# Clang's compilation database file
/compile_commands.json
+
+# Documentation toolchain
+sphinx_*/
Andrew Vasquez <andrew.vasquez@qlogic.com>
Andrey Ryabinin <ryabinin.a.a@gmail.com> <a.ryabinin@samsung.com>
Andy Adamson <andros@citi.umich.edu>
-Antoine Tenart <antoine.tenart@free-electrons.com>
+Antoine Tenart <atenart@kernel.org> <antoine.tenart@bootlin.com>
+Antoine Tenart <atenart@kernel.org> <antoine.tenart@free-electrons.com>
Antonio Ospite <ao2@ao2.it> <ao2@amarulasolutions.com>
Archit Taneja <archit@ti.com>
Ard Biesheuvel <ardb@kernel.org> <ard.biesheuvel@linaro.org>
Linas Vepstas <linas@austin.ibm.com>
Linus Lüssing <linus.luessing@c0d3.blue> <linus.luessing@ascom.ch>
Linus Lüssing <linus.luessing@c0d3.blue> <linus.luessing@web.de>
+<linux-hardening@vger.kernel.org> <kernel-hardening@lists.openwall.com>
Li Yang <leoyang.li@nxp.com> <leoli@freescale.com>
Li Yang <leoyang.li@nxp.com> <leo@zh-kernel.org>
Lukasz Luba <lukasz.luba@arm.com> <l.luba@partner.samsung.com>
Marcin Nowakowski <marcin.nowakowski@mips.com> <marcin.nowakowski@imgtec.com>
Marc Zyngier <maz@kernel.org> <marc.zyngier@arm.com>
Mark Brown <broonie@sirena.org.uk>
+Mark Starovoytov <mstarovo@pm.me> <mstarovoitov@marvell.com>
Mark Yao <markyao0591@gmail.com> <mark.yao@rock-chips.com>
Martin Kepplinger <martink@posteo.de> <martin.kepplinger@ginzinger.com>
Martin Kepplinger <martink@posteo.de> <martin.kepplinger@puri.sm>
--- /dev/null
+What: /sys/kernel/notes
+Date: July 2009
+Contact: <linux-kernel@vger.kernel.org>
+Description: The /sys/kernel/notes file contains the binary representation
+ of the running vmlinux's .notes section.
pciebus-howto
pci-iov-howto
msi-howto
+ sysfs-pci
acpi-info
pci-error-recovery
pcieaer-howto
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+============================================
+Accessing PCI device resources through sysfs
+============================================
+
+sysfs, usually mounted at /sys, provides access to PCI resources on platforms
+that support it. For example, a given bus might look like this::
+
+ /sys/devices/pci0000:17
+ |-- 0000:17:00.0
+ | |-- class
+ | |-- config
+ | |-- device
+ | |-- enable
+ | |-- irq
+ | |-- local_cpus
+ | |-- remove
+ | |-- resource
+ | |-- resource0
+ | |-- resource1
+ | |-- resource2
+ | |-- revision
+ | |-- rom
+ | |-- subsystem_device
+ | |-- subsystem_vendor
+ | `-- vendor
+ `-- ...
+
+The topmost element describes the PCI domain and bus number. In this case,
+the domain number is 0000 and the bus number is 17 (both values are in hex).
+This bus contains a single function device in slot 0. The domain and bus
+numbers are reproduced for convenience. Under the device directory are several
+files, each with their own function.
+
+ =================== =====================================================
+ file function
+ =================== =====================================================
+ class PCI class (ascii, ro)
+ config PCI config space (binary, rw)
+ device PCI device (ascii, ro)
+ enable Whether the device is enabled (ascii, rw)
+ irq IRQ number (ascii, ro)
+ local_cpus nearby CPU mask (cpumask, ro)
+ remove remove device from kernel's list (ascii, wo)
+ resource PCI resource host addresses (ascii, ro)
+ resource0..N PCI resource N, if present (binary, mmap, rw\ [1]_)
+ resource0_wc..N_wc PCI WC map resource N, if prefetchable (binary, mmap)
+ revision PCI revision (ascii, ro)
+ rom PCI ROM resource, if present (binary, ro)
+ subsystem_device PCI subsystem device (ascii, ro)
+ subsystem_vendor PCI subsystem vendor (ascii, ro)
+ vendor PCI vendor (ascii, ro)
+ =================== =====================================================
+
+::
+
+ ro - read only file
+ rw - file is readable and writable
+ wo - write only file
+ mmap - file is mmapable
+ ascii - file contains ascii text
+ binary - file contains binary data
+ cpumask - file contains a cpumask type
+
+.. [1] rw for IORESOURCE_IO (I/O port) regions only
+
+The read only files are informational, writes to them will be ignored, with
+the exception of the 'rom' file. Writable files can be used to perform
+actions on the device (e.g. changing config space, detaching a device).
+mmapable files are available via an mmap of the file at offset 0 and can be
+used to do actual device programming from userspace. Note that some platforms
+don't support mmapping of certain resources, so be sure to check the return
+value from any attempted mmap. The most notable of these are I/O port
+resources, which also provide read/write access.
+
+The 'enable' file provides a counter that indicates how many times the device
+has been enabled. If the 'enable' file currently returns '4', and a '1' is
+echoed into it, it will then return '5'. Echoing a '0' into it will decrease
+the count. Even when it returns to 0, though, some of the initialisation
+may not be reversed.
+
+The 'rom' file is special in that it provides read-only access to the device's
+ROM file, if available. It's disabled by default, however, so applications
+should write the string "1" to the file to enable it before attempting a read
+call, and disable it following the access by writing "0" to the file. Note
+that the device must be enabled for a rom read to return data successfully.
+In the event a driver is not bound to the device, it can be enabled using the
+'enable' file, documented above.
+
+The 'remove' file is used to remove the PCI device, by writing a non-zero
+integer to the file. This does not involve any kind of hot-plug functionality,
+e.g. powering off the device. The device is removed from the kernel's list of
+PCI devices, the sysfs directory for it is removed, and the device will be
+removed from any drivers attached to it. Removal of PCI root buses is
+disallowed.
+
+Accessing legacy resources through sysfs
+----------------------------------------
+
+Legacy I/O port and ISA memory resources are also provided in sysfs if the
+underlying platform supports them. They're located in the PCI class hierarchy,
+e.g.::
+
+ /sys/class/pci_bus/0000:17/
+ |-- bridge -> ../../../devices/pci0000:17
+ |-- cpuaffinity
+ |-- legacy_io
+ `-- legacy_mem
+
+The legacy_io file is a read/write file that can be used by applications to
+do legacy port I/O. The application should open the file, seek to the desired
+port (e.g. 0x3e8) and do a read or a write of 1, 2 or 4 bytes. The legacy_mem
+file should be mmapped with an offset corresponding to the memory offset
+desired, e.g. 0xa0000 for the VGA frame buffer. The application can then
+simply dereference the returned pointer (after checking for errors of course)
+to access legacy memory space.
+
+Supporting PCI access on new platforms
+--------------------------------------
+
+In order to support PCI resource mapping as described above, Linux platform
+code should ideally define ARCH_GENERIC_PCI_MMAP_RESOURCE and use the generic
+implementation of that functionality. To support the historical interface of
+mmap() through files in /proc/bus/pci, platforms may also set HAVE_PCI_MMAP.
+
+Alternatively, platforms which set HAVE_PCI_MMAP may provide their own
+implementation of pci_mmap_page_range() instead of defining
+ARCH_GENERIC_PCI_MMAP_RESOURCE.
+
+Platforms which support write-combining maps of PCI resources must define
+arch_can_pci_mmap_wc() which shall evaluate to non-zero at runtime when
+write-combining is permitted. Platforms which support maps of I/O resources
+define arch_can_pci_mmap_io() similarly.
+
+Legacy resources are protected by the HAVE_PCI_LEGACY define. Platforms
+wishing to support legacy functionality should define it and provide
+pci_legacy_read, pci_legacy_write and pci_mmap_legacy_page_range functions.
reboot, and enjoy!
If you ever need to change the default root device, video mode,
- ramdisk size, etc. in the kernel image, use the ``rdev`` program (or
- alternatively the LILO boot options when appropriate). No need to
- recompile the kernel to change these parameters.
+ etc. in the kernel image, use your bootloader's boot options
+ where appropriate. No need to recompile the kernel to change
+ these parameters.
- Reboot with the new kernel and enjoy.
Say you've got a big slow raid 6, and an ssd or three. Wouldn't it be
nice if you could use them as cache... Hence bcache.
-Wiki and git repositories are at:
+The bcache wiki can be found at:
+ https://bcache.evilpiepirate.org
- - https://bcache.evilpiepirate.org
- - http://evilpiepirate.org/git/linux-bcache.git
- - https://evilpiepirate.org/git/bcache-tools.git
+This is the git repository of bcache-tools:
+ https://git.kernel.org/pub/scm/linux/kernel/git/colyli/bcache-tools.git/
+
+The latest bcache kernel code can be found from mainline Linux kernel:
+ https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/
It's designed around the performance characteristics of SSDs - it only allocates
in erase block sized buckets, and it uses a hybrid btree/log to track cached
to be flushed.
Getting started:
-You'll need make-bcache from the bcache-tools repository. Both the cache device
+You'll need bcache util from the bcache-tools repository. Both the cache device
and backing device must be formatted before use::
- make-bcache -B /dev/sdb
- make-bcache -C /dev/sdc
+ bcache make -B /dev/sdb
+ bcache make -C /dev/sdc
-make-bcache has the ability to format multiple devices at the same time - if
+`bcache make` has the ability to format multiple devices at the same time - if
you format your backing devices and cache device at the same time, you won't
have to manually attach::
- make-bcache -B /dev/sda /dev/sdb -C /dev/sdc
+ bcache make -B /dev/sda /dev/sdb -C /dev/sdc
+
+If your bcache-tools is not updated to latest version and does not have the
+unified `bcache` utility, you may use the legacy `make-bcache` utility to format
+bcache device with same -B and -C parameters.
bcache-tools now ships udev rules, and bcache devices are known to the kernel
immediately. Without udev, you can manually register devices like this::
If bcache is not available in the kernel, a filesystem on the backing
device is still available at an 8KiB offset. So either via a loopdev
of the backing device created with --offset 8K, or any value defined by
---data-offset when you originally formatted bcache with `make-bcache`.
+--data-offset when you originally formatted bcache with `bcache make`.
For example::
After you boot back with bcache enabled, you recreate the cache and attach it::
- host:~# make-bcache -C /dev/sdh2
+ host:~# bcache make -C /dev/sdh2
UUID: 7be7e175-8f4c-4f99-94b2-9c904d227045
Set UUID: 5bc072a8-ab17-446d-9744-e247949913c1
version: 0
The default metadata size in bcache is 8k. If your backing device is
RAID based, then be sure to align this by a multiple of your stride
- width using `make-bcache --data-offset`. If you intend to expand your
+ width using `bcache make --data-offset`. If you intend to expand your
disk array in the future, then multiply a series of primes by your
raid stripe size to get the disk multiples that you would like.
1) Overview
2) Kernel Command Line Parameters
- 3) Using "rdev -r"
+ 3) Using "rdev"
4) An Example of Creating a Compressed RAM Disk
rd_size
See ramdisk_size.
-3) Using "rdev -r"
-------------------
+3) Using "rdev"
+---------------
-The usage of the word (two bytes) that "rdev -r" sets in the kernel image is
-as follows. The low 11 bits (0 -> 10) specify an offset (in 1 k blocks) of up
-to 2 MB (2^11) of where to find the RAM disk (this used to be the size). Bit
-14 indicates that a RAM disk is to be loaded, and bit 15 indicates whether a
-prompt/wait sequence is to be given before trying to read the RAM disk. Since
-the RAM disk dynamically grows as data is being written into it, a size field
-is not required. Bits 11 to 13 are not currently used and may as well be zero.
-These numbers are no magical secrets, as seen below::
+"rdev" is an obsolete, deprecated, antiquated utility that could be used
+to set the boot device in a Linux kernel image.
- ./arch/x86/kernel/setup.c:#define RAMDISK_IMAGE_START_MASK 0x07FF
- ./arch/x86/kernel/setup.c:#define RAMDISK_PROMPT_FLAG 0x8000
- ./arch/x86/kernel/setup.c:#define RAMDISK_LOAD_FLAG 0x4000
+Instead of using rdev, just place the boot device information on the
+kernel command line and pass it to the kernel from the bootloader.
-Consider a typical two floppy disk setup, where you will have the
-kernel on disk one, and have already put a RAM disk image onto disk #2.
+You can also pass arguments to the kernel by setting FDARGS in
+arch/x86/boot/Makefile and specify in initrd image by setting FDINITRD in
+arch/x86/boot/Makefile.
-Hence you want to set bits 0 to 13 as 0, meaning that your RAM disk
-starts at an offset of 0 kB from the beginning of the floppy.
-The command line equivalent is: "ramdisk_start=0"
+Some of the kernel command line boot options that may apply here are::
-You want bit 14 as one, indicating that a RAM disk is to be loaded.
-The command line equivalent is: "load_ramdisk=1"
-
-You want bit 15 as one, indicating that you want a prompt/keypress
-sequence so that you have a chance to switch floppy disks.
-The command line equivalent is: "prompt_ramdisk=1"
-
-Putting that together gives 2^15 + 2^14 + 0 = 49152 for an rdev word.
-So to create disk one of the set, you would do::
-
- /usr/src/linux# cat arch/x86/boot/zImage > /dev/fd0
- /usr/src/linux# rdev /dev/fd0 /dev/fd0
- /usr/src/linux# rdev -r /dev/fd0 49152
+ ramdisk_start=N
+ ramdisk_size=M
If you make a boot disk that has LILO, then for the above, you would use::
- append = "ramdisk_start=0 load_ramdisk=1 prompt_ramdisk=1"
-
-Since the default start = 0 and the default prompt = 1, you could use::
-
- append = "load_ramdisk=1"
-
+ append = "ramdisk_start=N ramdisk_size=M"
4) An Example of Creating a Compressed RAM Disk
-----------------------------------------------
dd if=/tmp/ram_image.gz of=/dev/fd0 bs=1k seek=400
-g) Use "rdev" to set the boot device, RAM disk offset, prompt flag, etc.
- For prompt_ramdisk=1, load_ramdisk=1, ramdisk_start=400, one would
- have 2^15 + 2^14 + 400 = 49552::
-
- rdev /dev/fd0 /dev/fd0
- rdev -r /dev/fd0 49552
+g) Make sure that you have already specified the boot information in
+ FDARGS and FDINITRD or that you use a bootloader to pass kernel
+ command line boot options to the kernel.
That is it. You now have your boot/root compressed RAM disk floppy. Some
users may wish to combine steps (d) and (f) by using a pipe.
Changelog:
----------
+SEPT-2020 :
+
+ Removed usage of "rdev"
+
10-22-04 :
Updated to reflect changes in command line options, remove
obsolete references, general cleanup.
James Nelson (james4765@gmail.com)
-
12-95 :
Original Document
+.. _cpusets:
+
=======
CPUSETS
=======
pgmajfault
Number of major page faults incurred
- workingset_refault
- Number of refaults of previously evicted pages
+ workingset_refault_anon
+ Number of refaults of previously evicted anonymous pages.
- workingset_activate
- Number of refaulted pages that were immediately activated
+ workingset_refault_file
+ Number of refaults of previously evicted file pages.
- workingset_restore
- Number of restored pages which have been detected as an active
- workingset before they got reclaimed.
+ workingset_activate_anon
+ Number of refaulted anonymous pages that were immediately
+ activated.
+
+ workingset_activate_file
+ Number of refaulted file pages that were immediately activated.
+
+ workingset_restore_anon
+ Number of restored anonymous pages which have been detected as
+ an active workingset before they got reclaimed.
+
+ workingset_restore_file
+ Number of restored file pages which have been detected as an
+ active workingset before they got reclaimed.
workingset_nodereclaim
Number of times a shadow node has been reclaimed
the value passed in <key_size>.
<key_type>
- Either 'logon' or 'user' kernel key type.
+ Either 'logon', 'user' or 'encrypted' kernel key type.
<key_description>
The kernel keyring key description crypt target should look for
thread because it benefits CFQ to have writes submitted using the
same context.
+no_read_workqueue
+ Bypass dm-crypt internal workqueue and process read requests synchronously.
+
+no_write_workqueue
+ Bypass dm-crypt internal workqueue and process write requests synchronously.
+ This option is automatically enabled for host-managed zoned block devices
+ (e.g. host-managed SMR hard-disks).
+
integrity:<bytes>:<type>
The device requires additional <bytes> metadata per-sector stored
in per-bio integrity structure. This metadata must by provided
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0-only
+
+GPIO Testing Driver
+===================
+
+The GPIO Testing Driver (gpio-mockup) provides a way to create simulated GPIO
+chips for testing purposes. The lines exposed by these chips can be accessed
+using the standard GPIO character device interface as well as manipulated
+using the dedicated debugfs directory structure.
+
+Creating simulated chips using module params
+--------------------------------------------
+
+When loading the gpio-mockup driver a number of parameters can be passed to the
+module.
+
+ gpio_mockup_ranges
+
+ This parameter takes an argument in the form of an array of integer
+ pairs. Each pair defines the base GPIO number (if any) and the number
+ of lines exposed by the chip. If the base GPIO is -1, the gpiolib
+ will assign it automatically.
+
+ Example: gpio_mockup_ranges=-1,8,-1,16,405,4
+
+ The line above creates three chips. The first one will expose 8 lines,
+ the second 16 and the third 4. The base GPIO for the third chip is set
+ to 405 while for two first chips it will be assigned automatically.
+
+ gpio_named_lines
+
+ This parameter doesn't take any arguments. It lets the driver know that
+ GPIO lines exposed by it should be named.
+
+ The name format is: gpio-mockup-X-Y where X is mockup chip's ID
+ and Y is the line offset.
+
+Manipulating simulated lines
+----------------------------
+
+Each mockup chip creates its own subdirectory in /sys/kernel/debug/gpio-mockup/.
+The directory is named after the chip's label. A symlink is also created, named
+after the chip's name, which points to the label directory.
+
+Inside each subdirectory, there's a separate attribute for each GPIO line. The
+name of the attribute represents the line's offset in the chip.
+
+Reading from a line attribute returns the current value. Writing to it (0 or 1)
+changes the configuration of the simulated pull-up/pull-down resistor
+(1 - pull-up, 0 - pull-down).
gpio-aggregator
sysfs
+ gpio-mockup
.. only:: subproject and html
dump kernel.
You can also use the Crash utility to analyze dump files in Kdump
-format. Crash is available on Dave Anderson's site at the following URL:
+format. Crash is available at the following URL:
- http://people.redhat.com/~anderson/
+ https://github.com/crash-utility/crash
+
+Crash document can be found at:
+ https://crash-utility.github.io/
Trigger Kdump on WARN()
=======================
loops can be debugged more effectively on production
systems.
- clearcpuid=BITNUM [X86]
+ clearcpuid=BITNUM[,BITNUM...] [X86]
Disable CPUID feature X for the kernel. See
arch/x86/include/asm/cpufeatures.h for the valid bit
numbers. Note the Linux specific bits are not necessarily
some critical bits.
cma=nn[MG]@[start[MG][-end[MG]]]
- [ARM,X86,KNL]
+ [KNL,CMA]
Sets the size of kernel global memory area for
contiguous memory allocations and optionally the
placement constraint by the physical address range of
Arch Perfmon v4 (Skylake and newer).
disable_ddw [PPC/PSERIES]
- Disable Dynamic DMA Window support. Use this if
+ Disable Dynamic DMA Window support. Use this
to workaround buggy firmware.
disable_ipv6= [IPV6]
what data is available or for reverse-engineering.
dyndbg[="val"] [KNL,DYNAMIC_DEBUG]
- module.dyndbg[="val"]
+ <module>.dyndbg[="val"]
Enable debug messages at boot time. See
Documentation/admin-guide/dynamic-debug-howto.rst
for details.
nopku [X86] Disable Memory Protection Keys CPU feature found
in some Intel CPUs.
- module.async_probe [KNL]
+ <module>.async_probe [KNL]
Enable asynchronous probe on this module.
early_ioremap_debug [KNL]
1 - Bypass the IOMMU for DMA.
unset - Use value of CONFIG_IOMMU_DEFAULT_PASSTHROUGH.
- io7= [HW] IO7 for Marvel based alpha systems
+ io7= [HW] IO7 for Marvel-based Alpha systems
See comment before marvel_specify_io7 in
arch/alpha/kernel/core_marvel.c.
kgdbwait [KGDB] Stop kernel execution and enter the
kernel debugger at the earliest opportunity.
- kmac= [MIPS] korina ethernet MAC address.
+ kmac= [MIPS] Korina ethernet MAC address.
Configure the RouterBoard 532 series on-chip
Ethernet adapter MAC address.
[KVM,ARM] Allow use of GICv4 for direct injection of
LPIs.
+ kvm_cma_resv_ratio=n [PPC]
+ Reserves given percentage from system memory area for
+ contiguous memory allocation for KVM hash pagetable
+ allocation.
+ By default it reserves 5% of total system memory.
+ Format: <integer>
+ Default: 5
+
kvm-intel.ept= [KVM,Intel] Disable extended page tables
(virtualized MMU) support on capable Intel chips.
Default is 1 (enabled)
lapic [X86-32,APIC] Enable the local APIC even if BIOS
disabled it.
- lapic= [X86,APIC] "notscdeadline" Do not use TSC deadline
+ lapic= [X86,APIC] Do not use TSC deadline
value for LAPIC timer one-shot implementation. Default
back to the programmable timer unit in the LAPIC.
+ Format: notscdeadline
lapic_timer_c2_ok [X86,APIC] trust the local apic timer
in C2 power state.
memblock=debug [KNL] Enable memblock debug messages.
- load_ramdisk= [RAM] List of ramdisks to load from floppy
- See Documentation/admin-guide/blockdev/ramdisk.rst.
+ load_ramdisk= [RAM] [Deprecated]
lockd.nlm_grace_period=P [NFS] Assign grace period.
Format: <integer>
(machvec) in a generic kernel.
Example: machvec=hpzx1
- machtype= [Loongson] Share the same kernel image file between different
- yeeloong laptop.
+ machtype= [Loongson] Share the same kernel image file between
+ different yeeloong laptops.
Example: machtype=lemote-yeeloong-2f-7inch
max_addr=nn[KMG] [KNL,BOOT,ia64] All physical memory greater
register save and restore. The kernel will only save
legacy floating-point registers on task switch.
- nohugeiomap [KNL,X86,PPC] Disable kernel huge I/O mappings.
+ nohugeiomap [KNL,X86,PPC,ARM64] Disable kernel huge I/O mappings.
nosmt [KNL,S390] Disable symmetric multithreading (SMT).
Equivalent to smt=1.
Param: <number> - step/bucket size as a power of 2 for
statistical time based profiling.
- prompt_ramdisk= [RAM] List of RAM disks to prompt for floppy disk
- before loading.
- See Documentation/admin-guide/blockdev/ramdisk.rst.
+ prompt_ramdisk= [RAM] [Deprecated]
prot_virt= [S390] enable hosting protected virtual machines
isolated from the hypervisor (if hardware supports
ramdisk_size= [RAM] Sizes of RAM disks in kilobytes
See Documentation/admin-guide/blockdev/ramdisk.rst.
+ ramdisk_start= [RAM] RAM disk image start address
+
random.trust_cpu={on,off}
[KNL] Enable or disable trusting the use of the
CPU's random number generator (if available) to
- 0572:0320
* - DVBSky T680CI
- 0572:680c
- * - MyGica Mini DVB-T2 USB Stick T230
+ * - MyGica Mini DVB-(T/T2/C) USB Stick T230
- 0572:c688
- * - MyGica Mini DVB-T2 USB Stick T230C
+ * - MyGica Mini DVB-(T/T2/C) USB Stick T230C
- 0572:c689
- * - MyGica Mini DVB-T2 USB Stick T230C Lite
+ * - MyGica Mini DVB-(T/T2/C) USB Stick T230C Lite
- 0572:c699
- * - MyGica Mini DVB-T2 USB Stick T230C v2
+ * - MyGica Mini DVB-(T/T2/C) USB Stick T230C v2
- 0572:c68a
* - TechnoTrend TT-connect CT2-4650 CI
- 0b48:3012
- 0b48:3011
* - TerraTec Cinergy S USB
- 0ccd:0064
+ * - Terratec Cinergy S2 PCIe Dual Port 1
+ - 153b:1181
+ * - Terratec Cinergy S2 PCIe Dual Port 2
+ - 153b:1182
* - Terratec Cinergy S2 USB BOX
- 0ccd:0x0105
* - Terratec Cinergy S2 USB HD
- PCTV DVB-S2 Stick (461e v2)
- em28178
- 2013:0461, 2013:0259
+ * - 105
+ - MyGica iGrabber
+ - em2860
+ - 1f4d:1abe
Using the media contorller APIs, the ov5670 sensor is configured to send
frames in packed raw Bayer format to IPU3 CSI2 receiver.
-# This example assumes /dev/media0 as the CIO2 media device
-
-export MDEV=/dev/media0
-
-# and that ov5670 sensor is connected to i2c bus 10 with address 0x36
-
-export SDEV=$(media-ctl -d $MDEV -e "ov5670 10-0036")
+.. code-block:: none
-# Establish the link for the media devices using media-ctl [#f3]_
-media-ctl -d $MDEV -l "ov5670:0 -> ipu3-csi2 0:0[1]"
+ # This example assumes /dev/media0 as the CIO2 media device
+ export MDEV=/dev/media0
-# Set the format for the media devices
-media-ctl -d $MDEV -V "ov5670:0 [fmt:SGRBG10/2592x1944]"
+ # and that ov5670 sensor is connected to i2c bus 10 with address 0x36
+ export SDEV=$(media-ctl -d $MDEV -e "ov5670 10-0036")
-media-ctl -d $MDEV -V "ipu3-csi2 0:0 [fmt:SGRBG10/2592x1944]"
+ # Establish the link for the media devices using media-ctl [#f3]_
+ media-ctl -d $MDEV -l "ov5670:0 -> ipu3-csi2 0:0[1]"
-media-ctl -d $MDEV -V "ipu3-csi2 0:1 [fmt:SGRBG10/2592x1944]"
+ # Set the format for the media devices
+ media-ctl -d $MDEV -V "ov5670:0 [fmt:SGRBG10/2592x1944]"
+ media-ctl -d $MDEV -V "ipu3-csi2 0:0 [fmt:SGRBG10/2592x1944]"
+ media-ctl -d $MDEV -V "ipu3-csi2 0:1 [fmt:SGRBG10/2592x1944]"
Once the media pipeline is configured, desired sensor specific settings
(such as exposure and gain settings) can be set, using the yavta tool.
e.g
-yavta -w 0x009e0903 444 $SDEV
-
-yavta -w 0x009e0913 1024 $SDEV
+.. code-block:: none
-yavta -w 0x009e0911 2046 $SDEV
+ yavta -w 0x009e0903 444 $SDEV
+ yavta -w 0x009e0913 1024 $SDEV
+ yavta -w 0x009e0911 2046 $SDEV
Once the desired sensor settings are set, frame captures can be done as below.
e.g
-yavta --data-prefix -u -c10 -n5 -I -s2592x1944 --file=/tmp/frame-#.bin \
- -f IPU3_SGRBG10 $(media-ctl -d $MDEV -e "ipu3-cio2 0")
+.. code-block:: none
+
+ yavta --data-prefix -u -c10 -n5 -I -s2592x1944 --file=/tmp/frame-#.bin \
+ -f IPU3_SGRBG10 $(media-ctl -d $MDEV -e "ipu3-cio2 0")
With the above command, 10 frames are captured at 2592x1944 resolution, with
sGRBG10 format and output as IPU3_SGRBG10 format.
Let us take "ipu3-imgu 0" subdev as an example.
-media-ctl -d $MDEV -r
-
-media-ctl -d $MDEV -l "ipu3-imgu 0 input":0 -> "ipu3-imgu 0":0[1]
-
-media-ctl -d $MDEV -l "ipu3-imgu 0":2 -> "ipu3-imgu 0 output":0[1]
-
-media-ctl -d $MDEV -l "ipu3-imgu 0":3 -> "ipu3-imgu 0 viewfinder":0[1]
+.. code-block:: none
-media-ctl -d $MDEV -l "ipu3-imgu 0":4 -> "ipu3-imgu 0 3a stat":0[1]
+ media-ctl -d $MDEV -r
+ media-ctl -d $MDEV -l "ipu3-imgu 0 input":0 -> "ipu3-imgu 0":0[1]
+ media-ctl -d $MDEV -l "ipu3-imgu 0":2 -> "ipu3-imgu 0 output":0[1]
+ media-ctl -d $MDEV -l "ipu3-imgu 0":3 -> "ipu3-imgu 0 viewfinder":0[1]
+ media-ctl -d $MDEV -l "ipu3-imgu 0":4 -> "ipu3-imgu 0 3a stat":0[1]
Also the pipe mode of the corresponding V4L2 subdev should be set as desired
(e.g 0 for video mode or 1 for still mode) through the control id 0x009819a1 as
below.
-yavta -w "0x009819A1 1" /dev/v4l-subdev7
+.. code-block:: none
+
+ yavta -w "0x009819A1 1" /dev/v4l-subdev7
Certain hardware blocks in ImgU pipeline can change the frame resolution by
cropping or scaling, these hardware blocks include Input Feeder(IF), Bayer Down
the desired results for the main output image and the viewfinder output, in NV12
format.
-v4l2n --pipe=4 --load=/tmp/frame-#.bin --open=/dev/video4
---fmt=type:VIDEO_OUTPUT_MPLANE,width=2592,height=1944,pixelformat=0X47337069
---reqbufs=type:VIDEO_OUTPUT_MPLANE,count:1 --pipe=1 --output=/tmp/frames.out
---open=/dev/video5
---fmt=type:VIDEO_CAPTURE_MPLANE,width=2560,height=1920,pixelformat=NV12
---reqbufs=type:VIDEO_CAPTURE_MPLANE,count:1 --pipe=2 --output=/tmp/frames.vf
---open=/dev/video6
---fmt=type:VIDEO_CAPTURE_MPLANE,width=2560,height=1920,pixelformat=NV12
---reqbufs=type:VIDEO_CAPTURE_MPLANE,count:1 --pipe=3 --open=/dev/video7
---output=/tmp/frames.3A --fmt=type:META_CAPTURE,?
---reqbufs=count:1,type:META_CAPTURE --pipe=1,2,3,4 --stream=5
+.. code-block:: none
+
+ v4l2n --pipe=4 --load=/tmp/frame-#.bin --open=/dev/video4
+ --fmt=type:VIDEO_OUTPUT_MPLANE,width=2592,height=1944,pixelformat=0X47337069 \
+ --reqbufs=type:VIDEO_OUTPUT_MPLANE,count:1 --pipe=1 \
+ --output=/tmp/frames.out --open=/dev/video5 \
+ --fmt=type:VIDEO_CAPTURE_MPLANE,width=2560,height=1920,pixelformat=NV12 \
+ --reqbufs=type:VIDEO_CAPTURE_MPLANE,count:1 --pipe=2 \
+ --output=/tmp/frames.vf --open=/dev/video6 \
+ --fmt=type:VIDEO_CAPTURE_MPLANE,width=2560,height=1920,pixelformat=NV12 \
+ --reqbufs=type:VIDEO_CAPTURE_MPLANE,count:1 --pipe=3 --open=/dev/video7 \
+ --output=/tmp/frames.3A --fmt=type:META_CAPTURE,? \
+ --reqbufs=count:1,type:META_CAPTURE --pipe=1,2,3,4 --stream=5
You can also use yavta [#f2]_ command to do same thing as above:
.. code-block:: none
- yavta --data-prefix -Bcapture-mplane -c10 -n5 -I -s2592x1944 \
- --file=frame-#.out-f NV12 /dev/video5 & \
- yavta --data-prefix -Bcapture-mplane -c10 -n5 -I -s2592x1944 \
- --file=frame-#.vf -f NV12 /dev/video6 & \
- yavta --data-prefix -Bmeta-capture -c10 -n5 -I \
- --file=frame-#.3a /dev/video7 & \
- yavta --data-prefix -Boutput-mplane -c10 -n5 -I -s2592x1944 \
- --file=/tmp/frame-in.cio2 -f IPU3_SGRBG10 /dev/video4
+ yavta --data-prefix -Bcapture-mplane -c10 -n5 -I -s2592x1944 \
+ --file=frame-#.out-f NV12 /dev/video5 & \
+ yavta --data-prefix -Bcapture-mplane -c10 -n5 -I -s2592x1944 \
+ --file=frame-#.vf -f NV12 /dev/video6 & \
+ yavta --data-prefix -Bmeta-capture -c10 -n5 -I \
+ --file=frame-#.3a /dev/video7 & \
+ yavta --data-prefix -Boutput-mplane -c10 -n5 -I -s2592x1944 \
+ --file=/tmp/frame-in.cio2 -f IPU3_SGRBG10 /dev/video4
where /dev/video4, /dev/video5, /dev/video6 and /dev/video7 devices point to
input, output, viewfinder and 3A statistics video nodes respectively.
Main output frames
~~~~~~~~~~~~~~~~~~
-raw2pnm -x2560 -y1920 -fNV12 /tmp/frames.out /tmp/frames.out.ppm
+.. code-block:: none
+
+ raw2pnm -x2560 -y1920 -fNV12 /tmp/frames.out /tmp/frames.out.ppm
where 2560x1920 is output resolution, NV12 is the video format, followed
by input frame and output PNM file.
Viewfinder output frames
~~~~~~~~~~~~~~~~~~~~~~~~
-raw2pnm -x2560 -y1920 -fNV12 /tmp/frames.vf /tmp/frames.vf.ppm
+.. code-block:: none
+
+ raw2pnm -x2560 -y1920 -fNV12 /tmp/frames.vf /tmp/frames.vf.ppm
where 2560x1920 is output resolution, NV12 is the video format, followed
by input frame and output PNM file.
Optical Black Correction Optical Black Correction block subtracts a pre-defined
value from the respective pixel values to obtain better
image quality.
- Defined in :c:type:`ipu3_uapi_obgrid_param`.
+ Defined in struct ipu3_uapi_obgrid_param.
Linearization This algo block uses linearization parameters to
address non-linearity sensor effects. The Lookup table
table is defined in
- :c:type:`ipu3_uapi_isp_lin_vmem_params`.
+ struct ipu3_uapi_isp_lin_vmem_params.
SHD Lens shading correction is used to correct spatial
non-uniformity of the pixel response due to optical
lens shading. This is done by applying a different gain
for each pixel. The gain, black level etc are
- configured in :c:type:`ipu3_uapi_shd_config_static`.
+ configured in struct ipu3_uapi_shd_config_static.
BNR Bayer noise reduction block removes image noise by
applying a bilateral filter.
- See :c:type:`ipu3_uapi_bnr_static_config` for details.
+ See struct ipu3_uapi_bnr_static_config for details.
ANR Advanced Noise Reduction is a block based algorithm
that performs noise reduction in the Bayer domain. The
convolution matrix etc can be found in
- :c:type:`ipu3_uapi_anr_config`.
+ struct ipu3_uapi_anr_config.
DM Demosaicing converts raw sensor data in Bayer format
into RGB (Red, Green, Blue) presentation. Then add
outputs of estimation of Y channel for following stream
processing by Firmware. The struct is defined as
- :c:type:`ipu3_uapi_dm_config`.
+ struct ipu3_uapi_dm_config.
Color Correction Color Correction algo transforms sensor specific color
space to the standard "sRGB" color space. This is done
by applying 3x3 matrix defined in
- :c:type:`ipu3_uapi_ccm_mat_config`.
-Gamma correction Gamma correction :c:type:`ipu3_uapi_gamma_config` is a
+ struct ipu3_uapi_ccm_mat_config.
+Gamma correction Gamma correction struct ipu3_uapi_gamma_config is a
basic non-linear tone mapping correction that is
applied per pixel for each pixel component.
CSC Color space conversion transforms each pixel from the
RGB primary presentation to YUV (Y: brightness,
UV: Luminance) presentation. This is done by applying
a 3x3 matrix defined in
- :c:type:`ipu3_uapi_csc_mat_config`
+ struct ipu3_uapi_csc_mat_config
CDS Chroma down sampling
After the CSC is performed, the Chroma Down Sampling
is applied for a UV plane down sampling by a factor
of 2 in each direction for YUV 4:2:0 using a 4x2
- configurable filter :c:type:`ipu3_uapi_cds_params`.
+ configurable filter struct ipu3_uapi_cds_params.
CHNR Chroma noise reduction
This block processes only the chrominance pixels and
performs noise reduction by cleaning the high
frequency noise.
- See struct :c:type:`ipu3_uapi_yuvp1_chnr_config`.
+ See struct struct ipu3_uapi_yuvp1_chnr_config.
TCC Total color correction as defined in struct
- :c:type:`ipu3_uapi_yuvp2_tcc_static_config`.
+ struct ipu3_uapi_yuvp2_tcc_static_config.
XNR3 eXtreme Noise Reduction V3 is the third revision of
noise reduction algorithm used to improve image
quality. This removes the low frequency noise in the
captured image. Two related structs are being defined,
- :c:type:`ipu3_uapi_isp_xnr3_params` for ISP data memory
- and :c:type:`ipu3_uapi_isp_xnr3_vmem_params` for vector
+ struct ipu3_uapi_isp_xnr3_params for ISP data memory
+ and struct ipu3_uapi_isp_xnr3_vmem_params for vector
memory.
TNR Temporal Noise Reduction block compares successive
frames in time to remove anomalies / noise in pixel
- values. :c:type:`ipu3_uapi_isp_tnr3_vmem_params` and
- :c:type:`ipu3_uapi_isp_tnr3_params` are defined for ISP
+ values. struct ipu3_uapi_isp_tnr3_vmem_params and
+ struct ipu3_uapi_isp_tnr3_params are defined for ISP
vector and data memory respectively.
======================== =======================================================
called accelerator cluster (ACC) to crunch pixel data and produce statistics.
ACC parameters of individual algorithms, as defined by
-:c:type:`ipu3_uapi_acc_param`, can be chosen to be applied by the user
-space through struct :c:type:`ipu3_uapi_flags` embedded in
-:c:type:`ipu3_uapi_params` structure. For parameters that are configured as
+struct ipu3_uapi_acc_param, can be chosen to be applied by the user
+space through struct struct ipu3_uapi_flags embedded in
+struct ipu3_uapi_params structure. For parameters that are configured as
not enabled by the user space, the corresponding structs are ignored by the
driver, in which case the existing configuration of the algorithm will be
preserved.
tw5864 Techwell TW5864 video/audio grabber and encoder
tw686x Intersil/Techwell TW686x
tw68 Techwell tw68x Video For Linux
+zoran Zoran-36057/36067 JPEG codec
================ ========================================================
Some of those drivers support multiple devices, as shown at the card
ivtv-cardlist
saa7134-cardlist
saa7164-cardlist
+ zoran-cardlist
--- /dev/null
+digraph board {
+ rankdir=TB
+ n00000001 [label="{{<port0> 0 | <port1> 1} | rkisp1_isp\n/dev/v4l-subdev0 | {<port2> 2 | <port3> 3}}", shape=Mrecord, style=filled, fillcolor=green]
+ n00000001:port2 -> n00000006:port0
+ n00000001:port2 -> n00000009:port0
+ n00000001:port3 -> n00000014 [style=bold]
+ n00000006 [label="{{<port0> 0} | rkisp1_resizer_mainpath\n/dev/v4l-subdev1 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green]
+ n00000006:port1 -> n0000000c [style=bold]
+ n00000009 [label="{{<port0> 0} | rkisp1_resizer_selfpath\n/dev/v4l-subdev2 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green]
+ n00000009:port1 -> n00000010 [style=bold]
+ n0000000c [label="rkisp1_mainpath\n/dev/video0", shape=box, style=filled, fillcolor=yellow]
+ n00000010 [label="rkisp1_selfpath\n/dev/video1", shape=box, style=filled, fillcolor=yellow]
+ n00000014 [label="rkisp1_stats\n/dev/video2", shape=box, style=filled, fillcolor=yellow]
+ n00000018 [label="rkisp1_params\n/dev/video3", shape=box, style=filled, fillcolor=yellow]
+ n00000018 -> n00000001:port1 [style=bold]
+ n0000001c [label="{{} | imx219 4-0010\n/dev/v4l-subdev3 | {<port0> 0}}", shape=Mrecord, style=filled, fillcolor=green]
+ n0000001c:port0 -> n00000001:port0
+}
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: <isonum.txt>
+
+=========================================
+Rockchip Image Signal Processor (rkisp1)
+=========================================
+
+Introduction
+============
+
+This file documents the driver for the Rockchip ISP1 that is part of RK3288
+and RK3399 SoCs. The driver is located under drivers/staging/media/rkisp1
+and uses the Media-Controller API.
+
+Topology
+========
+.. _rkisp1_topology_graph:
+
+.. kernel-figure:: rkisp1.dot
+ :alt: Diagram of the default media pipeline topology
+ :align: center
+
+
+The driver has 4 video devices:
+
+- rkisp1_mainpath: capture device for retrieving images, usually in higher
+ resolution.
+- rkisp1_selfpath: capture device for retrieving images.
+- rkisp1_stats: a metadata capture device that sends statistics.
+- rkisp1_params: a metadata output device that receives parameters
+ configurations from userspace.
+
+The driver has 3 subdevices:
+
+- rkisp1_resizer_mainpath: used to resize and downsample frames for the
+ mainpath capture device.
+- rkisp1_resizer_selfpath: used to resize and downsample frames for the
+ selfpath capture device.
+- rkisp1_isp: is connected to the sensor and is responsible for all the isp
+ operations.
+
+
+rkisp1_mainpath, rkisp1_selfpath - Frames Capture Video Nodes
+-------------------------------------------------------------
+Those are the `mainpath` and `selfpath` capture devices to capture frames.
+Those entities are the DMA engines that write the frames to memory.
+The selfpath video device can capture YUV/RGB formats. Its input is YUV encoded
+stream and it is able to convert it to RGB. The selfpath is not able to
+capture bayer formats.
+The mainpath can capture both bayer and YUV formats but it is not able to
+capture RGB formats.
+Both capture videos support
+the ``V4L2_CAP_IO_MC`` :ref:`capability <device-capabilities>`.
+
+
+rkisp1_resizer_mainpath, rkisp1_resizer_selfpath - Resizers Subdevices Nodes
+----------------------------------------------------------------------------
+Those are resizer entities for the mainpath and the selfpath. Those entities
+can scale the frames up and down and also change the YUV sampling (for example
+YUV4:2:2 -> YUV4:2:0). They also have cropping capability on the sink pad.
+The resizers entities can only operate on YUV:4:2:2 format
+(MEDIA_BUS_FMT_YUYV8_2X8).
+The mainpath capture device supports capturing video in bayer formats. In that
+case the resizer of the mainpath is set to 'bypass' mode - it just forward the
+frame without operating on it.
+
+rkisp1_isp - Image Signal Processing Subdevice Node
+---------------------------------------------------
+This is the isp entity. It is connected to the sensor on sink pad 0 and
+receives the frames using the CSI-2 protocol. It is responsible of configuring
+the CSI-2 protocol. It has a cropping capability on sink pad 0 that is
+connected to the sensor and on source pad 2 connected to the resizer entities.
+Cropping on sink pad 0 defines the image region from the sensor.
+Cropping on source pad 2 defines the region for the Image Stabilizer (IS).
+
+.. _rkisp1_stats:
+
+rkisp1_stats - Statistics Video Node
+------------------------------------
+The statistics video node outputs the 3A (auto focus, auto exposure and auto
+white balance) statistics, and also histogram statistics for the frames that
+are being processed by the rkisp1 to userspace applications.
+Using these data, applications can implement algorithms and re-parameterize
+the driver through the rkisp_params node to improve image quality during a
+video stream.
+The buffer format is defined by struct :c:type:`rkisp1_stat_buffer`, and
+userspace should set
+:ref:`V4L2_META_FMT_RK_ISP1_STAT_3A <v4l2-meta-fmt-stat-rkisp1>` as the
+dataformat.
+
+.. _rkisp1_params:
+
+rkisp1_params - Parameters Video Node
+-------------------------------------
+The rkisp1_params video node receives a set of parameters from userspace
+to be applied to the hardware during a video stream, allowing userspace
+to dynamically modify values such as black level, cross talk corrections
+and others.
+
+The buffer format is defined by struct :c:type:`rkisp1_params_cfg`, and
+userspace should set
+:ref:`V4L2_META_FMT_RK_ISP1_PARAMS <v4l2-meta-fmt-params-rkisp1>` as the
+dataformat.
+
+
+Capturing Video Frames Example
+==============================
+
+In the following example, the sensor connected to pad 0 of 'rkisp1_isp' is
+imx219.
+
+The following commands can be used to capture video from the selfpath video
+node with dimension 900x800 planar format YUV 4:2:2. It uses all cropping
+capabilities possible, (see explanation right below)
+
+.. code-block:: bash
+
+ # set the links
+ "media-ctl" "-d" "platform:rkisp1" "-r"
+ "media-ctl" "-d" "platform:rkisp1" "-l" "'imx219 4-0010':0 -> 'rkisp1_isp':0 [1]"
+ "media-ctl" "-d" "platform:rkisp1" "-l" "'rkisp1_isp':2 -> 'rkisp1_resizer_selfpath':0 [1]"
+ "media-ctl" "-d" "platform:rkisp1" "-l" "'rkisp1_isp':2 -> 'rkisp1_resizer_mainpath':0 [0]"
+
+ # set format for imx219 4-0010:0
+ "media-ctl" "-d" "platform:rkisp1" "--set-v4l2" '"imx219 4-0010":0 [fmt:SRGGB10_1X10/1640x1232]'
+
+ # set format for rkisp1_isp pads:
+ "media-ctl" "-d" "platform:rkisp1" "--set-v4l2" '"rkisp1_isp":0 [fmt:SRGGB10_1X10/1640x1232 crop: (0,0)/1600x1200]'
+ "media-ctl" "-d" "platform:rkisp1" "--set-v4l2" '"rkisp1_isp":2 [fmt:YUYV8_2X8/1600x1200 crop: (0,0)/1500x1100]'
+
+ # set format for rkisp1_resizer_selfpath pads:
+ "media-ctl" "-d" "platform:rkisp1" "--set-v4l2" '"rkisp1_resizer_selfpath":0 [fmt:YUYV8_2X8/1500x1100 crop: (300,400)/1400x1000]'
+ "media-ctl" "-d" "platform:rkisp1" "--set-v4l2" '"rkisp1_resizer_selfpath":1 [fmt:YUYV8_2X8/900x800]'
+
+ # set format for rkisp1_selfpath:
+ "v4l2-ctl" "-z" "platform:rkisp1" "-d" "rkisp1_selfpath" "-v" "width=900,height=800,"
+ "v4l2-ctl" "-z" "platform:rkisp1" "-d" "rkisp1_selfpath" "-v" "pixelformat=422P"
+
+ # start streaming:
+ v4l2-ctl "-z" "platform:rkisp1" "-d" "rkisp1_selfpath" "--stream-mmap" "--stream-count" "10"
+
+
+In the above example the sensor is configured to bayer format:
+`SRGGB10_1X10/1640x1232`. The rkisp1_isp:0 pad should be configured to the
+same mbus format and dimensions as the sensor, otherwise streaming will fail
+with 'EPIPE' error. So it is also configured to `SRGGB10_1X10/1640x1232`.
+In addition, the rkisp1_isp:0 pad is configured to cropping `(0,0)/1600x1200`.
+
+The cropping dimensions are automatically propagated to be the format of the
+isp source pad `rkisp1_isp:2`. Another cropping operation is configured on
+the isp source pad: `(0,0)/1500x1100`.
+
+The resizer's sink pad `rkisp1_resizer_selfpath` should be configured to format
+`YUYV8_2X8/1500x1100` in order to match the format on the other side of the
+link. In addition a cropping `(300,400)/1400x1000` is configured on it.
+
+The source pad of the resizer, `rkisp1_resizer_selfpath:1` is configured to
+format `YUYV8_2X8/900x800`. That means that the resizer first crop a window
+of `(300,400)/1400x100` from the received frame and then scales this window
+to dimension `900x800`.
+
+Note that the above example does not uses the stats-params control loop.
+Therefore the capture frames will not go through the 3A algorithms and
+probably won't have a good quality, and can even look dark and greenish.
+
+Configuring Quantization
+========================
+
+The driver supports limited and full range quantization on YUV formats,
+where limited is the default.
+To switch between one or the other, userspace should use the Colorspace
+Conversion API (CSC) for subdevices on source pad 2 of the
+isp (`rkisp1_isp:2`). The quantization configured on this pad is the
+quantization of the captured video frames on the mainpath and selfpath
+video nodes.
+Note that the resizer and capture entities will always report
+``V4L2_QUANTIZATION_DEFAULT`` even if the quantization is configured to full
+range on `rkisp1_isp:2`. So in order to get the configured quantization,
+application should get it from pad `rkisp1_isp:2`.
+
- 2040:1801
* - Hauppauge WinTV MiniCard
- 2040:2000, 2040:200a, 2040:2010, 2040:2011, 2040:2019
- * - Hauppauge WinTV MiniCard
+ * - Hauppauge WinTV MiniCard Rev 2
- 2040:2009
* - Hauppauge WinTV MiniStick
- 2040:5500, 2040:5510, 2040:5520, 2040:5530, 2040:5580, 2040:5590, 2040:b900, 2040:b910, 2040:b980, 2040:b990, 2040:c000, 2040:c010, 2040:c080, 2040:c090, 2040:c0a0, 2040:f5a0
em28xx-cardlist
tm6000-cardlist
siano-cardlist
- usbvision-cardlist
gspca-cardlist
+++ /dev/null
-.. SPDX-License-Identifier: GPL-2.0
-
-USBvision cards list
-====================
-
-.. tabularcolumns:: |p{1.4cm}|p{11.1cm}|p{4.2cm}|
-
-.. flat-table::
- :header-rows: 1
- :widths: 2 19 18
- :stub-columns: 0
-
- * - Card number
- - Card name
- - USB IDs
-
- * - 0
- - Xanboo
- - 0a6f:0400
-
- * - 1
- - Belkin USB VideoBus II Adapter
- - 050d:0106
-
- * - 2
- - Belkin Components USB VideoBus
- - 050d:0207
-
- * - 3
- - Belkin USB VideoBus II
- - 050d:0208
-
- * - 4
- - echoFX InterView Lite
- - 0571:0002
-
- * - 5
- - USBGear USBG-V1 resp. HAMA USB
- - 0573:0003
-
- * - 6
- - D-Link V100
- - 0573:0400
-
- * - 7
- - X10 USB Camera
- - 0573:2000
-
- * - 8
- - Hauppauge WinTV USB Live (PAL B/G)
- - 0573:2d00
-
- * - 9
- - Hauppauge WinTV USB Live Pro (NTSC M/N)
- - 0573:2d01
-
- * - 10
- - Zoran Co. PMD (Nogatech) AV-grabber Manhattan
- - 0573:2101
-
- * - 11
- - Nogatech USB-TV (NTSC) FM
- - 0573:4100
-
- * - 12
- - PNY USB-TV (NTSC) FM
- - 0573:4110
-
- * - 13
- - PixelView PlayTv-USB PRO (PAL) FM
- - 0573:4450
-
- * - 14
- - ZTV ZT-721 2.4GHz USB A/V Receiver
- - 0573:4550
-
- * - 15
- - Hauppauge WinTV USB (NTSC M/N)
- - 0573:4d00
-
- * - 16
- - Hauppauge WinTV USB (PAL B/G)
- - 0573:4d01
-
- * - 17
- - Hauppauge WinTV USB (PAL I)
- - 0573:4d02
-
- * - 18
- - Hauppauge WinTV USB (PAL/SECAM L)
- - 0573:4d03
-
- * - 19
- - Hauppauge WinTV USB (PAL D/K)
- - 0573:4d04
-
- * - 20
- - Hauppauge WinTV USB (NTSC FM)
- - 0573:4d10
-
- * - 21
- - Hauppauge WinTV USB (PAL B/G FM)
- - 0573:4d11
-
- * - 22
- - Hauppauge WinTV USB (PAL I FM)
- - 0573:4d12
-
- * - 23
- - Hauppauge WinTV USB (PAL D/K FM)
- - 0573:4d14
-
- * - 24
- - Hauppauge WinTV USB Pro (NTSC M/N)
- - 0573:4d2a
-
- * - 25
- - Hauppauge WinTV USB Pro (NTSC M/N) V2
- - 0573:4d2b
-
- * - 26
- - Hauppauge WinTV USB Pro (PAL/SECAM B/G/I/D/K/L)
- - 0573:4d2c
-
- * - 27
- - Hauppauge WinTV USB Pro (NTSC M/N) V3
- - 0573:4d20
-
- * - 28
- - Hauppauge WinTV USB Pro (PAL B/G)
- - 0573:4d21
-
- * - 29
- - Hauppauge WinTV USB Pro (PAL I)
- - 0573:4d22
-
- * - 30
- - Hauppauge WinTV USB Pro (PAL/SECAM L)
- - 0573:4d23
-
- * - 31
- - Hauppauge WinTV USB Pro (PAL D/K)
- - 0573:4d24
-
- * - 32
- - Hauppauge WinTV USB Pro (PAL/SECAM BGDK/I/L)
- - 0573:4d25
-
- * - 33
- - Hauppauge WinTV USB Pro (PAL/SECAM BGDK/I/L) V2
- - 0573:4d26
-
- * - 34
- - Hauppauge WinTV USB Pro (PAL B/G) V2
- - 0573:4d27
-
- * - 35
- - Hauppauge WinTV USB Pro (PAL B/G,D/K)
- - 0573:4d28
-
- * - 36
- - Hauppauge WinTV USB Pro (PAL I,D/K)
- - 0573:4d29
-
- * - 37
- - Hauppauge WinTV USB Pro (NTSC M/N FM)
- - 0573:4d30
-
- * - 38
- - Hauppauge WinTV USB Pro (PAL B/G FM)
- - 0573:4d31
-
- * - 39
- - Hauppauge WinTV USB Pro (PAL I FM)
- - 0573:4d32
-
- * - 40
- - Hauppauge WinTV USB Pro (PAL D/K FM)
- - 0573:4d34
-
- * - 41
- - Hauppauge WinTV USB Pro (Temic PAL/SECAM B/G/I/D/K/L FM)
- - 0573:4d35
-
- * - 42
- - Hauppauge WinTV USB Pro (Temic PAL B/G FM)
- - 0573:4d36
-
- * - 43
- - Hauppauge WinTV USB Pro (PAL/SECAM B/G/I/D/K/L FM)
- - 0573:4d37
-
- * - 44
- - Hauppauge WinTV USB Pro (NTSC M/N FM) V2
- - 0573:4d38
-
- * - 45
- - Camtel Technology USB TV Genie Pro FM Model TVB330
- - 0768:0006
-
- * - 46
- - Digital Video Creator I
- - 07d0:0001
-
- * - 47
- - Global Village GV-007 (NTSC)
- - 07d0:0002
-
- * - 48
- - Dazzle Fusion Model DVC-50 Rev 1 (NTSC)
- - 07d0:0003
-
- * - 49
- - Dazzle Fusion Model DVC-80 Rev 1 (PAL)
- - 07d0:0004
-
- * - 50
- - Dazzle Fusion Model DVC-90 Rev 1 (SECAM)
- - 07d0:0005
-
- * - 51
- - Eskape Labs MyTV2Go
- - 07f8:9104
-
- * - 52
- - Pinnacle Studio PCTV USB (PAL)
- - 2304:010d
-
- * - 53
- - Pinnacle Studio PCTV USB (SECAM)
- - 2304:0109
-
- * - 54
- - Pinnacle Studio PCTV USB (PAL) FM
- - 2304:0110
-
- * - 55
- - Miro PCTV USB
- - 2304:0111
-
- * - 56
- - Pinnacle Studio PCTV USB (NTSC) FM
- - 2304:0112
-
- * - 57
- - Pinnacle Studio PCTV USB (PAL) FM V2
- - 2304:0210
-
- * - 58
- - Pinnacle Studio PCTV USB (NTSC) FM V2
- - 2304:0212
-
- * - 59
- - Pinnacle Studio PCTV USB (PAL) FM V3
- - 2304:0214
-
- * - 60
- - Pinnacle Studio Linx Video input cable (NTSC)
- - 2304:0300
-
- * - 61
- - Pinnacle Studio Linx Video input cable (PAL)
- - 2304:0301
-
- * - 62
- - Pinnacle PCTV Bungee USB (PAL) FM
- - 2304:0419
-
- * - 63
- - Hauppauge WinTv-USB
- - 2400:4200
-
- * - 64
- - Pinnacle Studio PCTV USB (NTSC) FM V3
- - 2304:0113
-
- * - 65
- - Nogatech USB MicroCam NTSC (NV3000N)
- - 0573:3000
-
- * - 66
- - Nogatech USB MicroCam PAL (NV3001P)
- - 0573:3001
philips
qcom_camss
rcar-fdp1
+ rkisp1
saa7134
si470x
si4713
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+Zoran cards list
+================
+
+.. tabularcolumns:: |p{1.4cm}|p{11.1cm}|p{4.2cm}|
+
+.. flat-table::
+ :header-rows: 1
+ :widths: 2 19 18
+ :stub-columns: 0
+
+ * - Card number
+ - Card name
+ - PCI subsystem IDs
+
+ * - 0
+ - DC10(old)
+ - <any>
+
+ * - 1
+ - DC10(new)
+ - <any>
+
+ * - 2
+ - DC10_PLUS
+ - 1031:7efe
+
+ * - 3
+ - DC30
+ - <any>
+
+ * - 4
+ - DC30_PLUS
+ - 1031:d801
+
+ * - 5
+ - LML33
+ - <any>
+
+ * - 6
+ - LML33R10
+ - 12f8:8a02
+
+ * - 7
+ - Buz
+ - 13ca:4231
+
+ * - 8
+ - 6-Eyes
+ - <any>
--- /dev/null
+=============================
+Arm Coherent Mesh Network PMU
+=============================
+
+CMN-600 is a configurable mesh interconnect consisting of a rectangular
+grid of crosspoints (XPs), with each crosspoint supporting up to two
+device ports to which various AMBA CHI agents are attached.
+
+CMN implements a distributed PMU design as part of its debug and trace
+functionality. This consists of a local monitor (DTM) at every XP, which
+counts up to 4 event signals from the connected device nodes and/or the
+XP itself. Overflow from these local counters is accumulated in up to 8
+global counters implemented by the main controller (DTC), which provides
+overall PMU control and interrupts for global counter overflow.
+
+PMU events
+----------
+
+The PMU driver registers a single PMU device for the whole interconnect,
+see /sys/bus/event_source/devices/arm_cmn. Multi-chip systems may link
+more than one CMN together via external CCIX links - in this situation,
+each mesh counts its own events entirely independently, and additional
+PMU devices will be named arm_cmn_{1..n}.
+
+Most events are specified in a format based directly on the TRM
+definitions - "type" selects the respective node type, and "eventid" the
+event number. Some events require an additional occupancy ID, which is
+specified by "occupid".
+
+* Since RN-D nodes do not have any distinct events from RN-I nodes, they
+ are treated as the same type (0xa), and the common event templates are
+ named "rnid_*".
+
+* The cycle counter is treated as a synthetic event belonging to the DTC
+ node ("type" == 0x3, "eventid" is ignored).
+
+* XP events also encode the port and channel in the "eventid" field, to
+ match the underlying pmu_event0_id encoding for the pmu_event_sel
+ register. The event templates are named with prefixes to cover all
+ permutations.
+
+By default each event provides an aggregate count over all nodes of the
+given type. To target a specific node, "bynodeid" must be set to 1 and
+"nodeid" to the appropriate value derived from the CMN configuration
+(as defined in the "Node ID Mapping" section of the TRM).
+
+Watchpoints
+-----------
+
+The PMU can also count watchpoint events to monitor specific flit
+traffic. Watchpoints are treated as a synthetic event type, and like PMU
+events can be global or targeted with a particular XP's "nodeid" value.
+Since the watchpoint direction is otherwise implicit in the underlying
+register selection, separate events are provided for flit uploads and
+downloads.
+
+The flit match value and mask are passed in config1 and config2 ("val"
+and "mask" respectively). "wp_dev_sel", "wp_chn_sel", "wp_grp" and
+"wp_exclusive" are specified per the TRM definitions for dtm_wp_config0.
+Where a watchpoint needs to match fields from both match groups on the
+REQ or SNP channel, it can be specified as two events - one for each
+group - with the same nonzero "combine" value. The count for such a
+pair of combined events will be attributed to the primary match.
+Watchpoint events with a "combine" value of 0 are considered independent
+and will count individually.
qcom_l2_pmu
qcom_l3_pmu
arm-ccn
+ arm-cmn
xgene-pmu
arm_dsu_pmu
thunderx2-pmu
instruction of the CPUs (which, as a rule, suspends the execution of the program
and causes the hardware to attempt to enter the shallowest available idle state)
for this purpose, and if ``idle=poll`` is used, idle CPUs will execute a
-more or less ``lightweight'' sequence of instructions in a tight loop. [Note
+more or less "lightweight" sequence of instructions in a tight loop. [Note
that using ``idle=poll`` is somewhat drastic in many cases, as preventing idle
CPUs from saving almost any energy at all may not be the only effect of it.
For example, on Intel hardware it effectively prevents CPUs from using
This small document describes the "Video Mode Selection" feature which
allows the use of various special video modes supported by the video BIOS. Due
to usage of the BIOS, the selection is limited to boot time (before the
-kernel decompression starts) and works only on 80X86 machines.
+kernel decompression starts) and works only on 80X86 machines that are
+booted through BIOS firmware (as opposed to through UEFI, kexec, etc.).
.. note::
The video mode to be used is selected by a kernel parameter which can be
specified in the kernel Makefile (the SVGA_MODE=... line) or by the "vga=..."
-option of LILO (or some other boot loader you use) or by the "vidmode" utility
+option of LILO (or some other boot loader you use) or by the "xrandr" utility
(present in standard Linux utility packages). You can use the following values
of this parameter::
better to use absolute mode numbers instead.
0x.... - Hexadecimal video mode ID (also displayed on the menu, see below
- for exact meaning of the ID). Warning: rdev and LILO don't support
+ for exact meaning of the ID). Warning: LILO doesn't support
hexadecimal numbers -- you have to convert it to decimal manually.
Menu
+.. SPDX-License-Identifier: GPL-2.0+
+
================================
Documentation for /proc/sys/abi/
================================
-kernel version 2.6.0.test2
+.. See scripts/check-sysctl-docs to keep this up to date:
+.. scripts/check-sysctl-docs -vtable="abi" \
+.. Documentation/admin-guide/sysctl/abi.rst \
+.. $(git grep -l register_sysctl_)
-Copyright (c) 2003, Fabian Frederick <ffrederick@users.sourceforge.net>
+Copyright (c) 2020, Stephen Kitt
-For general info: index.rst.
+For general info, see :doc:`index`.
------------------------------------------------------------------------------
-This path is binary emulation relevant aka personality types aka abi.
-When a process is executed, it's linked to an exec_domain whose
-personality is defined using values available from /proc/sys/abi.
-You can find further details about abi in include/linux/personality.h.
-
-Here are the files featuring in 2.6 kernel:
-
-- defhandler_coff
-- defhandler_elf
-- defhandler_lcall7
-- defhandler_libcso
-- fake_utsname
-- trace
-
-defhandler_coff
----------------
-
-defined value:
- PER_SCOSVR3::
-
- 0x0003 | STICKY_TIMEOUTS | WHOLE_SECONDS | SHORT_INODE
-
-defhandler_elf
---------------
-
-defined value:
- PER_LINUX::
-
- 0
-
-defhandler_lcall7
------------------
-
-defined value :
- PER_SVR4::
-
- 0x0001 | STICKY_TIMEOUTS | MMAP_PAGE_ZERO,
-
-defhandler_libsco
------------------
-
-defined value:
- PER_SVR4::
+The files in ``/proc/sys/abi`` can be used to see and modify
+ABI-related settings.
- 0x0001 | STICKY_TIMEOUTS | MMAP_PAGE_ZERO,
+Currently, these files might (depending on your configuration)
+show up in ``/proc/sys/kernel``:
-fake_utsname
-------------
+.. contents:: :local:
-Unused
+vsyscall32 (x86)
+================
-trace
------
+Determines whether the kernels maps a vDSO page into 32-bit processes;
+can be set to 1 to enable, or 0 to disable. Defaults to enabled if
+``CONFIG_COMPAT_VDSO`` is set, disabled otherwide.
-Unused
+This controls the same setting as the ``vdso32`` kernel boot
+parameter.
5) ``B`` If a page-release function has found a bad page reference or some
unexpected page flags. This indicates a hardware problem or a kernel bug;
there should be other information in the log indicating why this tainting
- occured.
+ occurred.
6) ``U`` if a user or user application specifically requested that the
Tainted flag be set, ``' '`` otherwise.
* Datasheet
- http://dl.linux-sunxi.org/H3/Allwinner_H3_Datasheet_V1.0.pdf
+ https://linux-sunxi.org/images/4/4b/Allwinner_H3_Datasheet_V1.2.pdf
- Allwinner R40 (sun8i)
For actually enabling [U]EFI support, enable:
- CONFIG_EFI=y
-- CONFIG_EFI_VARS=y or m
+- CONFIG_EFIVAR_FS=y or m
The implementation depends on receiving information about the UEFI environment
in a Flattened Device Tree (FDT) - so is only available with CONFIG_OF.
+.. _amu_index:
+
=======================================================
Activity Monitors Unit (AMU) extension in AArch64 Linux
=======================================================
+------------------------------+---------+---------+
| Name | bits | visible |
+------------------------------+---------+---------+
+ | MTE | [11-8] | y |
+ +------------------------------+---------+---------+
| SSBS | [7-4] | y |
+------------------------------+---------+---------+
| BT | [3-0] | y |
Functionality implied by ID_AA64PFR0_EL1.BT == 0b0001.
+HWCAP2_MTE
+
+ Functionality implied by ID_AA64PFR1_EL1.MTE == 0b0010, as described
+ by Documentation/arm64/memory-tagging-extension.rst.
4. Unused AT_HWCAP bits
-----------------------
+.. _arm64_index:
+
==================
ARM64 Architecture
==================
hugetlbpage
legacy_instructions
memory
+ memory-tagging-extension
perf
pointer-authentication
silicon-errata
--- /dev/null
+===============================================
+Memory Tagging Extension (MTE) in AArch64 Linux
+===============================================
+
+Authors: Vincenzo Frascino <vincenzo.frascino@arm.com>
+ Catalin Marinas <catalin.marinas@arm.com>
+
+Date: 2020-02-25
+
+This document describes the provision of the Memory Tagging Extension
+functionality in AArch64 Linux.
+
+Introduction
+============
+
+ARMv8.5 based processors introduce the Memory Tagging Extension (MTE)
+feature. MTE is built on top of the ARMv8.0 virtual address tagging TBI
+(Top Byte Ignore) feature and allows software to access a 4-bit
+allocation tag for each 16-byte granule in the physical address space.
+Such memory range must be mapped with the Normal-Tagged memory
+attribute. A logical tag is derived from bits 59-56 of the virtual
+address used for the memory access. A CPU with MTE enabled will compare
+the logical tag against the allocation tag and potentially raise an
+exception on mismatch, subject to system registers configuration.
+
+Userspace Support
+=================
+
+When ``CONFIG_ARM64_MTE`` is selected and Memory Tagging Extension is
+supported by the hardware, the kernel advertises the feature to
+userspace via ``HWCAP2_MTE``.
+
+PROT_MTE
+--------
+
+To access the allocation tags, a user process must enable the Tagged
+memory attribute on an address range using a new ``prot`` flag for
+``mmap()`` and ``mprotect()``:
+
+``PROT_MTE`` - Pages allow access to the MTE allocation tags.
+
+The allocation tag is set to 0 when such pages are first mapped in the
+user address space and preserved on copy-on-write. ``MAP_SHARED`` is
+supported and the allocation tags can be shared between processes.
+
+**Note**: ``PROT_MTE`` is only supported on ``MAP_ANONYMOUS`` and
+RAM-based file mappings (``tmpfs``, ``memfd``). Passing it to other
+types of mapping will result in ``-EINVAL`` returned by these system
+calls.
+
+**Note**: The ``PROT_MTE`` flag (and corresponding memory type) cannot
+be cleared by ``mprotect()``.
+
+**Note**: ``madvise()`` memory ranges with ``MADV_DONTNEED`` and
+``MADV_FREE`` may have the allocation tags cleared (set to 0) at any
+point after the system call.
+
+Tag Check Faults
+----------------
+
+When ``PROT_MTE`` is enabled on an address range and a mismatch between
+the logical and allocation tags occurs on access, there are three
+configurable behaviours:
+
+- *Ignore* - This is the default mode. The CPU (and kernel) ignores the
+ tag check fault.
+
+- *Synchronous* - The kernel raises a ``SIGSEGV`` synchronously, with
+ ``.si_code = SEGV_MTESERR`` and ``.si_addr = <fault-address>``. The
+ memory access is not performed. If ``SIGSEGV`` is ignored or blocked
+ by the offending thread, the containing process is terminated with a
+ ``coredump``.
+
+- *Asynchronous* - The kernel raises a ``SIGSEGV``, in the offending
+ thread, asynchronously following one or multiple tag check faults,
+ with ``.si_code = SEGV_MTEAERR`` and ``.si_addr = 0`` (the faulting
+ address is unknown).
+
+The user can select the above modes, per thread, using the
+``prctl(PR_SET_TAGGED_ADDR_CTRL, flags, 0, 0, 0)`` system call where
+``flags`` contain one of the following values in the ``PR_MTE_TCF_MASK``
+bit-field:
+
+- ``PR_MTE_TCF_NONE`` - *Ignore* tag check faults
+- ``PR_MTE_TCF_SYNC`` - *Synchronous* tag check fault mode
+- ``PR_MTE_TCF_ASYNC`` - *Asynchronous* tag check fault mode
+
+The current tag check fault mode can be read using the
+``prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)`` system call.
+
+Tag checking can also be disabled for a user thread by setting the
+``PSTATE.TCO`` bit with ``MSR TCO, #1``.
+
+**Note**: Signal handlers are always invoked with ``PSTATE.TCO = 0``,
+irrespective of the interrupted context. ``PSTATE.TCO`` is restored on
+``sigreturn()``.
+
+**Note**: There are no *match-all* logical tags available for user
+applications.
+
+**Note**: Kernel accesses to the user address space (e.g. ``read()``
+system call) are not checked if the user thread tag checking mode is
+``PR_MTE_TCF_NONE`` or ``PR_MTE_TCF_ASYNC``. If the tag checking mode is
+``PR_MTE_TCF_SYNC``, the kernel makes a best effort to check its user
+address accesses, however it cannot always guarantee it.
+
+Excluding Tags in the ``IRG``, ``ADDG`` and ``SUBG`` instructions
+-----------------------------------------------------------------
+
+The architecture allows excluding certain tags to be randomly generated
+via the ``GCR_EL1.Exclude`` register bit-field. By default, Linux
+excludes all tags other than 0. A user thread can enable specific tags
+in the randomly generated set using the ``prctl(PR_SET_TAGGED_ADDR_CTRL,
+flags, 0, 0, 0)`` system call where ``flags`` contains the tags bitmap
+in the ``PR_MTE_TAG_MASK`` bit-field.
+
+**Note**: The hardware uses an exclude mask but the ``prctl()``
+interface provides an include mask. An include mask of ``0`` (exclusion
+mask ``0xffff``) results in the CPU always generating tag ``0``.
+
+Initial process state
+---------------------
+
+On ``execve()``, the new process has the following configuration:
+
+- ``PR_TAGGED_ADDR_ENABLE`` set to 0 (disabled)
+- Tag checking mode set to ``PR_MTE_TCF_NONE``
+- ``PR_MTE_TAG_MASK`` set to 0 (all tags excluded)
+- ``PSTATE.TCO`` set to 0
+- ``PROT_MTE`` not set on any of the initial memory maps
+
+On ``fork()``, the new process inherits the parent's configuration and
+memory map attributes with the exception of the ``madvise()`` ranges
+with ``MADV_WIPEONFORK`` which will have the data and tags cleared (set
+to 0).
+
+The ``ptrace()`` interface
+--------------------------
+
+``PTRACE_PEEKMTETAGS`` and ``PTRACE_POKEMTETAGS`` allow a tracer to read
+the tags from or set the tags to a tracee's address space. The
+``ptrace()`` system call is invoked as ``ptrace(request, pid, addr,
+data)`` where:
+
+- ``request`` - one of ``PTRACE_PEEKMTETAGS`` or ``PTRACE_POKEMTETAGS``.
+- ``pid`` - the tracee's PID.
+- ``addr`` - address in the tracee's address space.
+- ``data`` - pointer to a ``struct iovec`` where ``iov_base`` points to
+ a buffer of ``iov_len`` length in the tracer's address space.
+
+The tags in the tracer's ``iov_base`` buffer are represented as one
+4-bit tag per byte and correspond to a 16-byte MTE tag granule in the
+tracee's address space.
+
+**Note**: If ``addr`` is not aligned to a 16-byte granule, the kernel
+will use the corresponding aligned address.
+
+``ptrace()`` return value:
+
+- 0 - tags were copied, the tracer's ``iov_len`` was updated to the
+ number of tags transferred. This may be smaller than the requested
+ ``iov_len`` if the requested address range in the tracee's or the
+ tracer's space cannot be accessed or does not have valid tags.
+- ``-EPERM`` - the specified process cannot be traced.
+- ``-EIO`` - the tracee's address range cannot be accessed (e.g. invalid
+ address) and no tags copied. ``iov_len`` not updated.
+- ``-EFAULT`` - fault on accessing the tracer's memory (``struct iovec``
+ or ``iov_base`` buffer) and no tags copied. ``iov_len`` not updated.
+- ``-EOPNOTSUPP`` - the tracee's address does not have valid tags (never
+ mapped with the ``PROT_MTE`` flag). ``iov_len`` not updated.
+
+**Note**: There are no transient errors for the requests above, so user
+programs should not retry in case of a non-zero system call return.
+
+``PTRACE_GETREGSET`` and ``PTRACE_SETREGSET`` with ``addr ==
+``NT_ARM_TAGGED_ADDR_CTRL`` allow ``ptrace()`` access to the tagged
+address ABI control and MTE configuration of a process as per the
+``prctl()`` options described in
+Documentation/arm64/tagged-address-abi.rst and above. The corresponding
+``regset`` is 1 element of 8 bytes (``sizeof(long))``).
+
+Example of correct usage
+========================
+
+*MTE Example code*
+
+.. code-block:: c
+
+ /*
+ * To be compiled with -march=armv8.5-a+memtag
+ */
+ #include <errno.h>
+ #include <stdint.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <unistd.h>
+ #include <sys/auxv.h>
+ #include <sys/mman.h>
+ #include <sys/prctl.h>
+
+ /*
+ * From arch/arm64/include/uapi/asm/hwcap.h
+ */
+ #define HWCAP2_MTE (1 << 18)
+
+ /*
+ * From arch/arm64/include/uapi/asm/mman.h
+ */
+ #define PROT_MTE 0x20
+
+ /*
+ * From include/uapi/linux/prctl.h
+ */
+ #define PR_SET_TAGGED_ADDR_CTRL 55
+ #define PR_GET_TAGGED_ADDR_CTRL 56
+ # define PR_TAGGED_ADDR_ENABLE (1UL << 0)
+ # define PR_MTE_TCF_SHIFT 1
+ # define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
+ # define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
+ # define PR_MTE_TCF_ASYNC (2UL << PR_MTE_TCF_SHIFT)
+ # define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
+ # define PR_MTE_TAG_SHIFT 3
+ # define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT)
+
+ /*
+ * Insert a random logical tag into the given pointer.
+ */
+ #define insert_random_tag(ptr) ({ \
+ uint64_t __val; \
+ asm("irg %0, %1" : "=r" (__val) : "r" (ptr)); \
+ __val; \
+ })
+
+ /*
+ * Set the allocation tag on the destination address.
+ */
+ #define set_tag(tagged_addr) do { \
+ asm volatile("stg %0, [%0]" : : "r" (tagged_addr) : "memory"); \
+ } while (0)
+
+ int main()
+ {
+ unsigned char *a;
+ unsigned long page_sz = sysconf(_SC_PAGESIZE);
+ unsigned long hwcap2 = getauxval(AT_HWCAP2);
+
+ /* check if MTE is present */
+ if (!(hwcap2 & HWCAP2_MTE))
+ return EXIT_FAILURE;
+
+ /*
+ * Enable the tagged address ABI, synchronous MTE tag check faults and
+ * allow all non-zero tags in the randomly generated set.
+ */
+ if (prctl(PR_SET_TAGGED_ADDR_CTRL,
+ PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT),
+ 0, 0, 0)) {
+ perror("prctl() failed");
+ return EXIT_FAILURE;
+ }
+
+ a = mmap(0, page_sz, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (a == MAP_FAILED) {
+ perror("mmap() failed");
+ return EXIT_FAILURE;
+ }
+
+ /*
+ * Enable MTE on the above anonymous mmap. The flag could be passed
+ * directly to mmap() and skip this step.
+ */
+ if (mprotect(a, page_sz, PROT_READ | PROT_WRITE | PROT_MTE)) {
+ perror("mprotect() failed");
+ return EXIT_FAILURE;
+ }
+
+ /* access with the default tag (0) */
+ a[0] = 1;
+ a[1] = 2;
+
+ printf("a[0] = %hhu a[1] = %hhu\n", a[0], a[1]);
+
+ /* set the logical and allocation tags */
+ a = (unsigned char *)insert_random_tag(a);
+ set_tag(a);
+
+ printf("%p\n", a);
+
+ /* non-zero tag access */
+ a[0] = 3;
+ printf("a[0] = %hhu a[1] = %hhu\n", a[0], a[1]);
+
+ /*
+ * If MTE is enabled correctly the next instruction will generate an
+ * exception.
+ */
+ printf("Expecting SIGSEGV...\n");
+ a[16] = 0xdd;
+
+ /* this should not be printed in the PR_MTE_TCF_SYNC mode */
+ printf("...haven't got one\n");
+
+ return EXIT_FAILURE;
+ }
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
-extensions = ['kerneldoc', 'rstFlatTable', 'kernel_include', 'cdomain',
+extensions = ['kerneldoc', 'rstFlatTable', 'kernel_include',
'kfigure', 'sphinx.ext.ifconfig', 'automarkup',
'maintainers_include', 'sphinx.ext.autosectionlabel' ]
+#
+# cdomain is badly broken in Sphinx 3+. Leaving it out generates *most*
+# of the docs correctly, but not all. Scream bloody murder but allow
+# the process to proceed; hopefully somebody will fix this properly soon.
+#
+if major >= 3:
+ sys.stderr.write('''WARNING: The kernel documentation build process
+ does not work correctly with Sphinx v3.0 and above. Expect errors
+ in the generated output.
+ ''')
+else:
+ extensions.append('cdomain')
+
# Ensure that autosectionlabel will produce unique names
autosectionlabel_prefix_document = True
autosectionlabel_maxdepth = 2
Command Line Switches
=====================
``maxcpus=n``
- Restrict boot time CPUs to *n*. Say if you have fourV CPUs, using
+ Restrict boot time CPUs to *n*. Say if you have four CPUs, using
``maxcpus=2`` will only boot two. You can choose to bring the
other CPUs later online.
struct sockaddr_alg sa = {
.salg_family = AF_ALG,
- .salg_type = "rng", /* this selects the symmetric cipher */
- .salg_name = "drbg_nopr_sha256" /* this is the cipher name */
+ .salg_type = "rng", /* this selects the random number generator */
+ .salg_name = "drbg_nopr_sha256" /* this is the RNG name */
};
Depending on the RNG type, the RNG must be seeded. The seed is provided
using the setsockopt interface to set the key. For example, the
ansi_cprng requires a seed. The DRBGs do not require a seed, but may be
-seeded.
+seeded. The seed is also known as a *Personalization String* in NIST SP 800-90A
+standard.
Using the read()/recvmsg() system calls, random numbers can be obtained.
The kernel generates at most 128 bytes in one call. If user space
system call multiple times. In this case, the returned file descriptors
have the same state.
+Following CAVP testing interfaces are enabled when kernel is built with
+CRYPTO_USER_API_RNG_CAVP option:
+
+- the concatenation of *Entropy* and *Nonce* can be provided to the RNG via
+ ALG_SET_DRBG_ENTROPY setsockopt interface. Setting the entropy requires
+ CAP_SYS_ADMIN permission.
+
+- *Additional Data* can be provided using the send()/sendmsg() system calls,
+ but only after the entropy has been set.
+
Zero-Copy Interface
-------------------
provided ciphertext is assumed to contain an authentication tag of
the given size (see section about AEAD memory layout below).
+- ALG_SET_DRBG_ENTROPY -- Setting the entropy of the random number generator.
+ This option is applicable to RNG cipher type only.
+
User space API example
----------------------
compatible:
items:
- const: raspberrypi,bcm2835-firmware
- - const: simple-bus
+ - const: simple-mfd
mboxes:
$ref: '/schemas/types.yaml#/definitions/phandle'
examples:
- |
firmware {
- compatible = "raspberrypi,bcm2835-firmware", "simple-bus";
+ compatible = "raspberrypi,bcm2835-firmware", "simple-mfd";
mboxes = <&mailbox>;
firmware_clocks: clocks {
};
mmc@5b010000 {
- compatible = "fsl,imx8qxp-usdhc";
+ compatible = "fsl,imx8qxp-usdhc", "fsl,imx7d-usdhc";
interrupts = <GIC_SPI 232 IRQ_TYPE_LEVEL_HIGH>;
reg = <0x5b010000 0x10000>;
clocks = <&conn_lpcg IMX_CONN_LPCG_SDHC0_IPG_CLK>,
main_crypto: crypto@4e00000 {
compatible = "ti,j721-sa2ul";
- reg = <0x0 0x4e00000 0x0 0x1200>;
+ reg = <0x4e00000 0x1200>;
power-domains = <&k3_pds 264 TI_SCI_PD_EXCLUSIVE>;
dmas = <&main_udmap 0xc000>, <&main_udmap 0x4000>,
<&main_udmap 0x4001>;
- vi
- Tegra210:
- power-domains: Must include venc powergate node as vi is in VE partition.
- - Tegra210 has CSI part of VI sharing same host interface and register space.
- So, VI device node should have CSI child node.
+
+ ports (optional node)
+ vi can have optional ports node and max 6 ports are supported. Each port
+ should have single 'endpoint' child node. All port nodes are grouped under
+ ports node. Please refer to the bindings defined in
+ Documentation/devicetree/bindings/media/video-interfaces.txt
+
+ csi (required node)
+ Tegra210 has CSI part of VI sharing same host interface and register space.
+ So, VI device node should have CSI child node.
- csi: mipi csi interface to vi
- power-domains: Must include sor powergate node as csicil is in
SOR partition.
+ channel (optional nodes)
+ Maximum 6 channels are supported with each csi brick as either x4 or x2
+ based on hw connectivity to sensor.
+
+ Required properties:
+ - reg: csi port number. Valid port numbers are 0 through 5.
+ - nvidia,mipi-calibrate: Should contain a phandle and a specifier
+ specifying which pads are used by this CSI port and need to be
+ calibrated. See also ../display/tegra/nvidia,tegra114-mipi.txt.
+
+ Each channel node must contain 2 port nodes which can be grouped
+ under 'ports' node and each port should have a single child 'endpoint'
+ node.
+
+ ports node
+ Please refer to the bindings defined in
+ Documentation/devicetree/bindings/media/video-interfaces.txt
+
+ ports node must contain below 2 port nodes.
+ port@0 with single child 'endpoint' node always a sink.
+ port@1 with single child 'endpoint' node always a source.
+
+ port@0 (required node)
+ Required properties:
+ - reg: 0
+
+ endpoint (required node)
+ Required properties:
+ - data-lanes: an array of data lane from 1 to 4. Valid array
+ lengths are 1/2/4.
+ - remote-endpoint: phandle to sensor 'endpoint' node.
+
+ port@1 (required node)
+ Required properties:
+ - reg: 1
+
+ endpoint (required node)
+ Required properties:
+ - remote-endpoint: phandle to vi port 'endpoint' node.
+
- epp: encoder pre-processor
Required properties:
ranges = <0x0 0x0 0x54080000 0x2000>;
+ ports {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ port@0 {
+ reg = <0>;
+ imx219_vi_in0: endpoint {
+ remote-endpoint = <&imx219_csi_out0>;
+ };
+ };
+ };
+
csi@838 {
compatible = "nvidia,tegra210-csi";
reg = <0x838 0x1300>;
<&tegra_car TEGRA210_CLK_CSI_TPG>;
clock-names = "csi", "cilab", "cilcd", "cile", "csi_tpg";
power-domains = <&pd_sor>;
+
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ channel@0 {
+ reg = <0>;
+ nvidia,mipi-calibrate = <&mipi 0x001>;
+
+ ports {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ port@0 {
+ reg = <0>;
+ imx219_csi_in0: endpoint {
+ data-lanes = <1 2>;
+ remote-endpoint = <&imx219_out0>;
+ };
+ };
+
+ port@1 {
+ reg = <1>;
+ imx219_csi_out0: endpoint {
+ remote-endpoint = <&imx219_vi_in0>;
+ };
+ };
+ };
+ };
};
};
display@fd4a0000 {
compatible = "xlnx,zynqmp-dpsub-1.7";
- reg = <0x0 0xfd4a0000 0x0 0x1000>,
- <0x0 0xfd4aa000 0x0 0x1000>,
- <0x0 0xfd4ab000 0x0 0x1000>,
- <0x0 0xfd4ac000 0x0 0x1000>;
+ reg = <0xfd4a0000 0x1000>,
+ <0xfd4aa000 0x1000>,
+ <0xfd4ab000 0x1000>,
+ <0xfd4ac000 0x1000>;
reg-names = "dp", "blend", "av_buf", "aud";
interrupts = <0 119 4>;
interrupt-parent = <&gic>;
dma: dma-controller@fd4c0000 {
compatible = "xlnx,zynqmp-dpdma";
- reg = <0x0 0xfd4c0000 0x0 0x1000>;
+ reg = <0xfd4c0000 0x1000>;
interrupts = <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>;
interrupt-parent = <&gic>;
clocks = <&dpdma_clk>;
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/edac/amazon,al-mc-edac.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Amazon's Annapurna Labs Memory Controller EDAC
+
+maintainers:
+ - Talel Shenhar <talel@amazon.com>
+ - Talel Shenhar <talelshenhar@gmail.com>
+
+description: |
+ EDAC node is defined to describe on-chip error detection and correction for
+ Amazon's Annapurna Labs Memory Controller.
+
+properties:
+
+ compatible:
+ const: amazon,al-mc-edac
+
+ reg:
+ maxItems: 1
+
+ "#address-cells":
+ const: 2
+
+ "#size-cells":
+ const: 2
+
+ interrupts:
+ minItems: 1
+ maxItems: 2
+ items:
+ - description: uncorrectable error interrupt
+ - description: correctable error interrupt
+
+ interrupt-names:
+ minItems: 1
+ maxItems: 2
+ items:
+ - const: ue
+ - const: ce
+
+required:
+ - compatible
+ - reg
+ - "#address-cells"
+ - "#size-cells"
+
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/irq.h>
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ edac@f0080000 {
+ #address-cells = <2>;
+ #size-cells = <2>;
+ compatible = "amazon,al-mc-edac";
+ reg = <0x0 0xf0080000 0x0 0x00010000>;
+ interrupt-parent = <&amazon_al_system_fabric>;
+ interrupt-names = "ue";
+ interrupts = <20 IRQ_TYPE_LEVEL_HIGH>;
+ };
+ };
compatible:
enum:
- ibm,fsi2spi
+ - ibm,fsi2spi-restricted
reg:
items:
properties:
compatible:
- enum:
- - fsl,imx1-gpio
- - fsl,imx21-gpio
- - fsl,imx31-gpio
- - fsl,imx35-gpio
- - fsl,imx7d-gpio
+ oneOf:
+ - enum:
+ - fsl,imx1-gpio
+ - fsl,imx21-gpio
+ - fsl,imx31-gpio
+ - fsl,imx35-gpio
+ - fsl,imx7d-gpio
+ - items:
+ - const: fsl,imx35-gpio
+ - const: fsl,imx31-gpio
+ - items:
+ - enum:
+ - fsl,imx50-gpio
+ - fsl,imx51-gpio
+ - fsl,imx53-gpio
+ - fsl,imx6q-gpio
+ - fsl,imx6sl-gpio
+ - fsl,imx6sll-gpio
+ - fsl,imx6sx-gpio
+ - fsl,imx6ul-gpio
+ - fsl,imx7d-gpio
+ - fsl,imx8mm-gpio
+ - fsl,imx8mn-gpio
+ - fsl,imx8mp-gpio
+ - fsl,imx8mq-gpio
+ - fsl,imx8qxp-gpio
+ - const: fsl,imx35-gpio
reg:
maxItems: 1
const: 2
gpio-controller: true
+ gpio-line-names: true
+ gpio-ranges: true
+
+ power-domains:
+ maxItems: 1
+
+patternProperties:
+ "^(hog-[0-9]+|.+-hog(-[0-9]+)?)$":
+ type: object
+ properties:
+ gpio-hog: true
+ gpios: true
+ input: true
+ output-high: true
+ output-low: true
+ line-name: true
+
+ required:
+ - gpio-hog
+ - gpios
+
+ additionalProperties: false
required:
- compatible
+++ /dev/null
-* MAX732x-compatible I/O expanders
-
-Required properties:
- - compatible: Should be one of the following:
- - "maxim,max7319": For the Maxim MAX7319
- - "maxim,max7320": For the Maxim MAX7320
- - "maxim,max7321": For the Maxim MAX7321
- - "maxim,max7322": For the Maxim MAX7322
- - "maxim,max7323": For the Maxim MAX7323
- - "maxim,max7324": For the Maxim MAX7324
- - "maxim,max7325": For the Maxim MAX7325
- - "maxim,max7326": For the Maxim MAX7326
- - "maxim,max7327": For the Maxim MAX7327
- - reg: I2C slave address for this device.
- - gpio-controller: Marks the device node as a GPIO controller.
- - #gpio-cells: Should be 2.
- - first cell is the GPIO number
- - second cell specifies GPIO flags, as defined in <dt-bindings/gpio/gpio.h>.
- Only the GPIO_ACTIVE_HIGH and GPIO_ACTIVE_LOW flags are supported.
-
-Optional properties:
-
- The I/O expander can detect input state changes, and thus optionally act as
- an interrupt controller. When the expander interrupt line is connected all the
- following properties must be set. For more information please see the
- interrupt controller device tree bindings documentation available at
- Documentation/devicetree/bindings/interrupt-controller/interrupts.txt.
-
- - interrupt-controller: Identifies the node as an interrupt controller.
- - #interrupt-cells: Number of cells to encode an interrupt source, shall be 2.
- - first cell is the pin number
- - second cell is used to specify flags
- - interrupts: Interrupt specifier for the controllers interrupt.
-
-Please refer to gpio.txt in this directory for details of the common GPIO
-bindings used by client devices.
-
-Example 1. MAX7325 with interrupt support enabled (CONFIG_GPIO_MAX732X_IRQ=y):
-
- expander: max7325@6d {
- compatible = "maxim,max7325";
- reg = <0x6d>;
- gpio-controller;
- #gpio-cells = <2>;
- interrupt-controller;
- #interrupt-cells = <2>;
- interrupt-parent = <&gpio4>;
- interrupts = <29 IRQ_TYPE_EDGE_FALLING>;
- };
-
-Example 2. MAX7325 with interrupt support disabled (CONFIG_GPIO_MAX732X_IRQ=n):
-
- expander: max7325@6d {
- compatible = "maxim,max7325";
- reg = <0x6d>;
- gpio-controller;
- #gpio-cells = <2>;
- };
+++ /dev/null
-* NXP PCA953x I2C GPIO multiplexer
-
-Required properties:
- - compatible: Has to contain one of the following:
- nxp,pca6416
- nxp,pca9505
- nxp,pca9534
- nxp,pca9535
- nxp,pca9536
- nxp,pca9537
- nxp,pca9538
- nxp,pca9539
- nxp,pca9554
- nxp,pca9555
- nxp,pca9556
- nxp,pca9557
- nxp,pca9574
- nxp,pca9575
- nxp,pca9698
- nxp,pcal6416
- nxp,pcal6524
- nxp,pcal9535
- nxp,pcal9555a
- maxim,max7310
- maxim,max7312
- maxim,max7313
- maxim,max7315
- ti,pca6107
- ti,pca9536
- ti,tca6408
- ti,tca6416
- ti,tca6424
- ti,tca9539
- ti,tca9554
- onnn,cat9554
- onnn,pca9654
- exar,xra1202
- - gpio-controller: if used as gpio expander.
- - #gpio-cells: if used as gpio expander.
- - interrupt-controller: if to be used as interrupt expander.
- - #interrupt-cells: if to be used as interrupt expander.
-
-Optional properties:
- - interrupts: interrupt specifier for the device's interrupt output.
- - reset-gpios: GPIO specification for the RESET input. This is an
- active low signal to the PCA953x.
- - vcc-supply: power supply regulator.
-
-Example:
-
-
- gpio@20 {
- compatible = "nxp,pca9505";
- reg = <0x20>;
- pinctrl-names = "default";
- pinctrl-0 = <&pinctrl_pca9505>;
- gpio-controller;
- #gpio-cells = <2>;
- interrupt-parent = <&gpio3>;
- interrupts = <23 IRQ_TYPE_LEVEL_LOW>;
- };
-
-
-Example with Interrupts:
-
-
- gpio99: gpio@22 {
- compatible = "nxp,pcal6524";
- reg = <0x22>;
- interrupt-parent = <&gpio6>;
- interrupts = <1 IRQ_TYPE_EDGE_FALLING>; /* gpio6_161 */
- interrupt-controller;
- #interrupt-cells = <2>;
- vcc-supply = <&vdds_1v8_main>;
- gpio-controller;
- #gpio-cells = <2>;
- gpio-line-names =
- "hdmi-ct-hpd", "hdmi.ls-oe", "p02", "p03", "vibra", "fault2", "p06", "p07",
- "en-usb", "en-host1", "en-host2", "chg-int", "p14", "p15", "mic-int", "en-modem",
- "shdn-hs-amp", "chg-status+red", "green", "blue", "en-esata", "fault1", "p26", "p27";
- };
-
- ts3a227@3b {
- compatible = "ti,ts3a227e";
- reg = <0x3b>;
- interrupt-parent = <&gpio99>;
- interrupts = <14 IRQ_TYPE_EDGE_RISING>;
- ti,micbias = <0>; /* 2.1V */
- };
-
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/gpio/gpio-pca95xx.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: NXP PCA95xx I2C GPIO multiplexer
+
+maintainers:
+ - Krzysztof Kozlowski <krzk@kernel.org>
+
+description: |+
+ Bindings for the family of I2C GPIO multiplexers/expanders: NXP PCA95xx,
+ Maxim MAX73xx
+
+properties:
+ compatible:
+ enum:
+ - exar,xra1202
+ - maxim,max7310
+ - maxim,max7312
+ - maxim,max7313
+ - maxim,max7315
+ - maxim,max7319
+ - maxim,max7320
+ - maxim,max7321
+ - maxim,max7322
+ - maxim,max7323
+ - maxim,max7324
+ - maxim,max7325
+ - maxim,max7326
+ - maxim,max7327
+ - nxp,pca6416
+ - nxp,pca9505
+ - nxp,pca9534
+ - nxp,pca9535
+ - nxp,pca9536
+ - nxp,pca9537
+ - nxp,pca9538
+ - nxp,pca9539
+ - nxp,pca9554
+ - nxp,pca9555
+ - nxp,pca9556
+ - nxp,pca9557
+ - nxp,pca9574
+ - nxp,pca9575
+ - nxp,pca9698
+ - nxp,pcal6416
+ - nxp,pcal6524
+ - nxp,pcal9535
+ - nxp,pcal9555a
+ - onnn,cat9554
+ - onnn,pca9654
+ - ti,pca6107
+ - ti,pca9536
+ - ti,tca6408
+ - ti,tca6416
+ - ti,tca6424
+ - ti,tca9539
+ - ti,tca9554
+
+ reg:
+ maxItems: 1
+
+ gpio-controller: true
+
+ '#gpio-cells':
+ const: 2
+
+ gpio-line-names:
+ minItems: 1
+ maxItems: 32
+
+ interrupts:
+ maxItems: 1
+
+ interrupt-controller: true
+
+ '#interrupt-cells':
+ const: 2
+
+ reset-gpios:
+ description:
+ GPIO specification for the RESET input. This is an active low signal to
+ the PCA953x. Not valid for Maxim MAX732x devices.
+
+ vcc-supply:
+ description:
+ Optional power supply. Not valid for Maxim MAX732x devices.
+
+ wakeup-source:
+ $ref: /schemas/types.yaml#/definitions/flag
+
+patternProperties:
+ "^(hog-[0-9]+|.+-hog(-[0-9]+)?)$":
+ type: object
+ properties:
+ gpio-hog: true
+ gpios: true
+ input: true
+ output-high: true
+ output-low: true
+ line-name: true
+
+ required:
+ - gpio-hog
+ - gpios
+
+ additionalProperties: false
+
+required:
+ - compatible
+ - reg
+ - gpio-controller
+ - "#gpio-cells"
+
+additionalProperties: false
+
+allOf:
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - maxim,max7320
+ - maxim,max7321
+ - maxim,max7322
+ - maxim,max7323
+ - maxim,max7324
+ - maxim,max7325
+ - maxim,max7326
+ - maxim,max7327
+ then:
+ properties:
+ reset-gpios: false
+ vcc-supply: false
+
+examples:
+ - |
+ #include <dt-bindings/gpio/gpio.h>
+ #include <dt-bindings/interrupt-controller/irq.h>
+
+ i2c0 {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ gpio@20 {
+ compatible = "nxp,pca9505";
+ reg = <0x20>;
+ pinctrl-names = "default";
+ pinctrl-0 = <&pinctrl_pca9505>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ interrupt-parent = <&gpio3>;
+ interrupts = <23 IRQ_TYPE_LEVEL_LOW>;
+
+ usb3-sata-sel-hog {
+ gpio-hog;
+ gpios = <4 GPIO_ACTIVE_HIGH>;
+ output-low;
+ line-name = "usb3_sata_sel";
+ };
+ };
+ };
+
+ - |
+ #include <dt-bindings/interrupt-controller/irq.h>
+
+ i2c1 {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ gpio99: gpio@22 {
+ compatible = "nxp,pcal6524";
+ reg = <0x22>;
+ interrupt-parent = <&gpio6>;
+ interrupts = <1 IRQ_TYPE_EDGE_FALLING>; /* gpio6_161 */
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ vcc-supply = <&vdds_1v8_main>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ gpio-line-names = "hdmi-ct-hpd", "hdmi.ls-oe", "p02", "p03",
+ "vibra", "fault2", "p06", "p07", "en-usb",
+ "en-host1", "en-host2", "chg-int", "p14", "p15",
+ "mic-int", "en-modem", "shdn-hs-amp",
+ "chg-status+red", "green", "blue", "en-esata",
+ "fault1", "p26", "p27";
+ };
+
+ ts3a227@3b {
+ compatible = "ti,ts3a227e";
+ reg = <0x3b>;
+ interrupt-parent = <&gpio99>;
+ interrupts = <14 IRQ_TYPE_EDGE_RISING>;
+ ti,micbias = <0>; /* 2.1V */
+ };
+ };
+
+ - |
+ #include <dt-bindings/interrupt-controller/irq.h>
+
+ i2c2 {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ /* MAX7325 with interrupt support enabled */
+ gpio@6d {
+ compatible = "maxim,max7325";
+ reg = <0x6d>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ interrupt-parent = <&gpio4>;
+ interrupts = <29 IRQ_TYPE_EDGE_FALLING>;
+ };
+ };
+
+ - |
+ i2c3 {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ /* MAX7325 with interrupt support disabled */
+ gpio@6e {
+ compatible = "maxim,max7325";
+ reg = <0x6e>;
+ gpio-controller;
+ #gpio-cells = <2>;
+ };
+ };
gpio-controller: true
+ gpio-line-names: true
+
gpio-ranges:
+ minItems: 1
maxItems: 8
required:
- renesas,gpio-r8a774a1 # RZ/G2M
- renesas,gpio-r8a774b1 # RZ/G2N
- renesas,gpio-r8a774c0 # RZ/G2E
+ - renesas,gpio-r8a774e1 # RZ/G2H
- renesas,gpio-r8a7795 # R-Car H3
- renesas,gpio-r8a7796 # R-Car M3-W
- renesas,gpio-r8a77961 # R-Car M3-W+
- gpio-controller : Marks the device node as a GPIO controller
- interrupts : Interrupt specifier, see interrupt-controller/interrupts.txt
- interrupt-controller : Mark the GPIO controller as an interrupt-controller
-- ngpios : number of GPIO lines, see gpio.txt
- (should be multiple of 8, up to 80 pins)
+- ngpios : number of *hardware* GPIO lines, see gpio.txt. This will expose
+ 2 software GPIOs per hardware GPIO: one for hardware input, one for hardware
+ output. Up to 80 pins, must be a multiple of 8.
- clocks : A phandle to the APB clock for SGPM clock division
- bus-frequency : SGPM CLK frequency
'#gpio-cells':
const: 2
+ ngpios:
+ default: 32
+ minimum: 1
+ maximum: 32
+
snps,nr-gpios:
description: The number of GPIO pins exported by the port.
+ deprecated: true
$ref: /schemas/types.yaml#/definitions/uint32
default: 32
minimum: 1
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/adi,adm1266.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices ADM1266 Cascadable Super Sequencer with Margin
+ Control and Fault Recording
+
+maintainers:
+ - Alexandru Tachici <alexandru.tachici@analog.com>
+
+description: |
+ Analog Devices ADM1266 Cascadable Super Sequencer with Margin
+ Control and Fault Recording.
+ https://www.analog.com/media/en/technical-documentation/data-sheets/ADM1266.pdf
+
+properties:
+ compatible:
+ enum:
+ - adi,adm1266
+
+ reg:
+ description: |
+ I2C address of slave device.
+ items:
+ minimum: 0x40
+ maximum: 0x4F
+
+ avcc-supply:
+ description: |
+ Phandle to the Avcc power supply.
+
+required:
+ - compatible
+ - reg
+
+additionalProperties: false
+
+examples:
+ - |
+ i2c0 {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ adm1266@40 {
+ compatible = "adi,adm1266";
+ reg = <0x40>;
+ };
+ };
+...
+++ /dev/null
-*LM75 hwmon sensor.
-
-Required properties:
-- compatible: manufacturer and chip name, one of
- "adi,adt75",
- "dallas,ds1775",
- "dallas,ds75",
- "dallas,ds7505",
- "gmt,g751",
- "national,lm75",
- "national,lm75a",
- "national,lm75b",
- "maxim,max6625",
- "maxim,max6626",
- "maxim,max31725",
- "maxim,max31726",
- "maxim,mcp980x",
- "nxp,pct2075",
- "st,stds75",
- "st,stlm75",
- "microchip,tcn75",
- "ti,tmp100",
- "ti,tmp101",
- "ti,tmp105",
- "ti,tmp112",
- "ti,tmp175",
- "ti,tmp275",
- "ti,tmp75",
- "ti,tmp75b",
- "ti,tmp75c",
-
-- reg: I2C bus address of the device
-
-Example:
-
-sensor@48 {
- compatible = "st,stlm75";
- reg = <0x48>;
-};
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only or BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/lm75.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: LM75 hwmon sensor
+
+maintainers:
+ - Jean Delvare <jdelvare@suse.com>
+ - Guenter Roeck <linux@roeck-us.net>
+
+properties:
+ compatible:
+ enum:
+ - adi,adt75
+ - dallas,ds1775
+ - dallas,ds75
+ - dallas,ds7505
+ - gmt,g751
+ - national,lm75
+ - national,lm75a
+ - national,lm75b
+ - maxim,max6625
+ - maxim,max6626
+ - maxim,max31725
+ - maxim,max31726
+ - maxim,mcp980x
+ - nxp,pct2075
+ - st,stds75
+ - st,stlm75
+ - microchip,tcn75
+ - ti,tmp100
+ - ti,tmp101
+ - ti,tmp105
+ - ti,tmp112
+ - ti,tmp175
+ - ti,tmp275
+ - ti,tmp75
+ - ti,tmp75b
+ - ti,tmp75c
+
+ reg:
+ maxItems: 1
+
+ vs-supply:
+ description: phandle to the regulator that provides the +VS supply
+
+required:
+ - compatible
+ - reg
+
+additionalProperties: false
+
+examples:
+ - |
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ sensor@48 {
+ compatible = "st,stlm75";
+ reg = <0x48>;
+ vs-supply = <&vs>;
+ };
+ };
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+
+$id: http://devicetree.org/schemas/hwmon/maxim,max20730.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Maxim max20730
+
+maintainers:
+ - Jean Delvare <jdelvare@suse.com>
+ - Guenter Roeck <linux@roeck-us.net>
+
+description: |
+ The MAX20730 is a fully integrated, highly efficient switching regulator
+ with PMBus for applications operating from 4.5V to 16V and requiring
+ up to 25A (max) load. This single-chip regulator provides extremely
+ compact, high efficiency power-delivery solutions with high-precision
+ output voltages and excellent transient response.
+
+ Datasheets:
+ https://datasheets.maximintegrated.com/en/ds/MAX20730.pdf
+ https://datasheets.maximintegrated.com/en/ds/MAX20734.pdf
+ https://datasheets.maximintegrated.com/en/ds/MAX20743.pdf
+
+properties:
+ compatible:
+ enum:
+ - maxim,max20730
+ - maxim,max20734
+ - maxim,max20743
+
+ reg:
+ maxItems: 1
+
+ vout-voltage-divider:
+ description: |
+ If voltage divider present at vout, the voltage at voltage sensor pin
+ will be scaled. The properties will convert the raw reading to a more
+ meaningful number if voltage divider present. It has two numbers,
+ the first number is the output resistor, the second number is the total
+ resistance. Therefore, the adjusted vout is equal to
+ Vout = Vout * output_resistance / total resistance.
+ $ref: /schemas/types.yaml#/definitions/uint32-array
+ minItems: 2
+ maxItems: 2
+
+required:
+ - compatible
+ - reg
+
+additionalProperties: false
+
+examples:
+ - |
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ max20730@10 {
+ compatible = "maxim,max20730";
+ reg = <0x10>;
+ vout-voltage-divider = <1000 2000>; // vout would be scaled to 0.5
+ };
+ };
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/moortec,mr75203.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Moortec Semiconductor MR75203 PVT Controller bindings
+
+maintainers:
+ - Rahul Tanwar <rtanwar@maxlinear.com>
+
+properties:
+ compatible:
+ const: moortec,mr75203
+
+ reg:
+ items:
+ - description: PVT common registers
+ - description: PVT temprature sensor registers
+ - description: PVT process detector registers
+ - description: PVT voltage monitor registers
+
+ reg-names:
+ items:
+ - const: common
+ - const: ts
+ - const: pd
+ - const: vm
+
+ intel,vm-map:
+ description:
+ PVT controller has 5 VM (voltage monitor) sensors.
+ vm-map defines CPU core to VM instance mapping. A
+ value of 0xff means that VM sensor is unused.
+ $ref: /schemas/types.yaml#definitions/uint8-array
+ maxItems: 5
+
+ clocks:
+ maxItems: 1
+
+ resets:
+ maxItems: 1
+
+ "#thermal-sensor-cells":
+ const: 1
+
+required:
+ - compatible
+ - reg
+ - reg-names
+ - intel,vm-map
+ - clocks
+ - resets
+ - "#thermal-sensor-cells"
+
+additionalProperties: false
+
+examples:
+ - |
+ pvt: pvt@e0680000 {
+ compatible = "moortec,mr75203";
+ reg = <0xe0680000 0x80>,
+ <0xe0680080 0x180>,
+ <0xe0680200 0x200>,
+ <0xe0680400 0xc00>;
+ reg-names = "common", "ts", "pd", "vm";
+ intel,vm-map = [03 01 04 ff ff];
+ clocks = <&osc0>;
+ resets = <&rcu0 0x40 7>;
+ #thermal-sensor-cells = <1>;
+ };
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/sensirion,shtc1.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Sensirion SHTC1 Humidity and Temperature Sensor IC
+
+maintainers:
+ - Christopher Ruehl chris.ruehl@gtsys.com.hk
+
+description: |
+ The SHTC1, SHTW1 and SHTC3 are digital humidity and temperature sensor
+ designed especially for battery-driven high-volume consumer electronics
+ applications.
+ For further information refere to Documentation/hwmon/shtc1.rst
+
+ This binding document describes the binding for the hardware monitor
+ portion of the driver.
+
+properties:
+ compatible:
+ enum:
+ - sensirion,shtc1
+ - sensirion,shtw1
+ - sensirion,shtc3
+
+ reg:
+ const: 0x70
+
+ sensirion,blocking-io:
+ $ref: /schemas/types.yaml#definitions/flag
+ description:
+ If set, the driver hold the i2c bus until measurement is finished.
+
+ sensirion,low-precision:
+ $ref: /schemas/types.yaml#definitions/flag
+ description:
+ If set, the sensor aquire data with low precision (not recommended).
+ The driver aquire data with high precision by default.
+
+required:
+ - compatible
+ - reg
+
+additionalProperties: false
+
+examples:
+ - |
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+ clock-frequency = <400000>;
+
+ shtc3@70 {
+ compatible = "sensirion,shtc3";
+ reg = <0x70>;
+ sensirion,blocking-io;
+ };
+ };
+...
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interrupt-controller/actions,owl-sirq.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Actions Semi Owl SoCs SIRQ interrupt controller
+
+maintainers:
+ - Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+ - Cristian Ciocaltea <cristian.ciocaltea@gmail.com>
+
+description: |
+ This interrupt controller is found in the Actions Semi Owl SoCs (S500, S700
+ and S900) and provides support for handling up to 3 external interrupt lines.
+
+properties:
+ compatible:
+ enum:
+ - actions,s500-sirq
+ - actions,s700-sirq
+ - actions,s900-sirq
+
+ reg:
+ maxItems: 1
+
+ interrupt-controller: true
+
+ '#interrupt-cells':
+ const: 2
+ description:
+ The first cell is the input IRQ number, between 0 and 2, while the second
+ cell is the trigger type as defined in interrupt.txt in this directory.
+
+ 'interrupts':
+ description: |
+ Contains the GIC SPI IRQs mapped to the external interrupt lines.
+ They shall be specified sequentially from output 0 to 2.
+ minItems: 3
+ maxItems: 3
+
+required:
+ - compatible
+ - reg
+ - interrupt-controller
+ - '#interrupt-cells'
+ - 'interrupts'
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+ sirq: interrupt-controller@b01b0200 {
+ compatible = "actions,s500-sirq";
+ reg = <0xb01b0200 0x4>;
+ interrupt-controller;
+ #interrupt-cells = <2>;
+ interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>, /* SIRQ0 */
+ <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>, /* SIRQ1 */
+ <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>; /* SIRQ2 */
+ };
+
+...
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interrupt-controller/mstar,mst-intc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MStar Interrupt Controller
+
+maintainers:
+ - Mark-PK Tsai <mark-pk.tsai@mediatek.com>
+
+description: |+
+ MStar, SigmaStar and Mediatek TV SoCs contain multiple legacy
+ interrupt controllers that routes interrupts to the GIC.
+
+ The HW block exposes a number of interrupt controllers, each
+ can support up to 64 interrupts.
+
+properties:
+ compatible:
+ const: mstar,mst-intc
+
+ interrupt-controller: true
+
+ "#interrupt-cells":
+ const: 3
+ description: |
+ Use the same format as specified by GIC in arm,gic.yaml.
+
+ reg:
+ maxItems: 1
+
+ mstar,irqs-map-range:
+ description: |
+ The range <start, end> of parent interrupt controller's interrupt
+ lines that are hardwired to mstar interrupt controller.
+ $ref: /schemas/types.yaml#/definitions/uint32-matrix
+ items:
+ minItems: 2
+ maxItems: 2
+
+ mstar,intc-no-eoi:
+ description:
+ Mark this controller has no End Of Interrupt(EOI) implementation.
+ type: boolean
+
+required:
+ - compatible
+ - reg
+ - mstar,irqs-map-range
+
+additionalProperties: false
+
+examples:
+ - |
+ mst_intc0: interrupt-controller@1f2032d0 {
+ compatible = "mstar,mst-intc";
+ interrupt-controller;
+ #interrupt-cells = <3>;
+ interrupt-parent = <&gic>;
+ reg = <0x1f2032d0 0x30>;
+ mstar,irqs-map-range = <0 63>;
+ };
+...
Synopsys DesignWare provides interrupt controller IP for APB known as
dw_apb_ictl. The IP is used as secondary interrupt controller in some SoCs with
-APB bus, e.g. Marvell Armada 1500.
+APB bus, e.g. Marvell Armada 1500. It can also be used as primary interrupt
+controller in some SoCs, e.g. Hisilicon SD5203.
Required properties:
- compatible: shall be "snps,dw-apb-ictl"
region starting with ENABLE_LOW register
- interrupt-controller: identifies the node as an interrupt controller
- #interrupt-cells: number of cells to encode an interrupt-specifier, shall be 1
+
+Additional required property when it's used as secondary interrupt controller:
- interrupts: interrupt reference to primary interrupt controller
The interrupt sources map to the corresponding bits in the interrupt
- (optional) fast interrupts start at 64.
Example:
+ /* dw_apb_ictl is used as secondary interrupt controller */
aic: interrupt-controller@3000 {
compatible = "snps,dw-apb-ictl";
reg = <0x3000 0xc00>;
interrupt-parent = <&gic>;
interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>;
};
+
+ /* dw_apb_ictl is used as primary interrupt controller */
+ vic: interrupt-controller@10130000 {
+ compatible = "snps,dw-apb-ictl";
+ reg = <0x10130000 0x1000>;
+ interrupt-controller;
+ #interrupt-cells = <1>;
+ };
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only or BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interrupt-controller/ti,pruss-intc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI PRU-ICSS Local Interrupt Controller
+
+maintainers:
+ - Suman Anna <s-anna@ti.com>
+
+description: |
+ Each PRU-ICSS has a single interrupt controller instance that is common
+ to all the PRU cores. Most interrupt controllers can route 64 input events
+ which are then mapped to 10 possible output interrupts through two levels
+ of mapping. The input events can be triggered by either the PRUs and/or
+ various other PRUSS internal and external peripherals. The first 2 output
+ interrupts (0, 1) are fed exclusively to the internal PRU cores, with the
+ remaining 8 (2 through 9) connected to external interrupt controllers
+ including the MPU and/or other PRUSS instances, DSPs or devices.
+
+ The property "ti,irqs-reserved" is used for denoting the connection
+ differences on the output interrupts 2 through 9. If this property is not
+ defined, it implies that all the PRUSS INTC output interrupts 2 through 9
+ (host_intr0 through host_intr7) are connected exclusively to the Arm interrupt
+ controller.
+
+ The K3 family of SoCs can handle 160 input events that can be mapped to 20
+ different possible output interrupts. The additional output interrupts (10
+ through 19) are connected to new sub-modules within the ICSSG instances.
+
+ This interrupt-controller node should be defined as a child node of the
+ corresponding PRUSS node. The node should be named "interrupt-controller".
+
+properties:
+ compatible:
+ enum:
+ - ti,pruss-intc
+ - ti,icssg-intc
+ description: |
+ Use "ti,pruss-intc" for OMAP-L13x/AM18x/DA850 SoCs,
+ AM335x family of SoCs,
+ AM437x family of SoCs,
+ AM57xx family of SoCs
+ 66AK2G family of SoCs
+ Use "ti,icssg-intc" for K3 AM65x & J721E family of SoCs
+
+ reg:
+ maxItems: 1
+
+ interrupts:
+ minItems: 1
+ maxItems: 8
+ description: |
+ All the interrupts generated towards the main host processor in the SoC.
+ A shared interrupt can be skipped if the desired destination and usage is
+ by a different processor/device.
+
+ interrupt-names:
+ minItems: 1
+ maxItems: 8
+ items:
+ pattern: host_intr[0-7]
+ description: |
+ Should use one of the above names for each valid host event interrupt
+ connected to Arm interrupt controller, the name should match the
+ corresponding host event interrupt number.
+
+ interrupt-controller: true
+
+ "#interrupt-cells":
+ const: 3
+ description: |
+ Client users shall use the PRU System event number (the interrupt source
+ that the client is interested in) [cell 1], PRU channel [cell 2] and PRU
+ host_event (target) [cell 3] as the value of the interrupts property in
+ their node. The system events can be mapped to some output host
+ interrupts through 2 levels of many-to-one mapping i.e. events to channel
+ mapping and channels to host interrupts so through this property entire
+ mapping is provided.
+
+ ti,irqs-reserved:
+ $ref: /schemas/types.yaml#definitions/uint8
+ description: |
+ Bitmask of host interrupts between 0 and 7 (corresponding to PRUSS INTC
+ output interrupts 2 through 9) that are not connected to the Arm interrupt
+ controller or are shared and used by other devices or processors in the
+ SoC. Define this property when any of 8 interrupts should not be handled
+ by Arm interrupt controller.
+ Eg: - AM437x and 66AK2G SoCs do not have "host_intr5" interrupt
+ connected to MPU
+ - AM65x and J721E SoCs have "host_intr5", "host_intr6" and
+ "host_intr7" interrupts connected to MPU, and other ICSSG
+ instances.
+
+required:
+ - compatible
+ - reg
+ - interrupts
+ - interrupt-names
+ - interrupt-controller
+ - "#interrupt-cells"
+
+additionalProperties: false
+
+examples:
+ - |
+ /* AM33xx PRU-ICSS */
+ pruss: pruss@0 {
+ compatible = "ti,am3356-pruss";
+ reg = <0x0 0x80000>;
+ #address-cells = <1>;
+ #size-cells = <1>;
+ ranges;
+
+ pruss_intc: interrupt-controller@20000 {
+ compatible = "ti,pruss-intc";
+ reg = <0x20000 0x2000>;
+ interrupts = <20 21 22 23 24 25 26 27>;
+ interrupt-names = "host_intr0", "host_intr1",
+ "host_intr2", "host_intr3",
+ "host_intr4", "host_intr5",
+ "host_intr6", "host_intr7";
+ interrupt-controller;
+ #interrupt-cells = <3>;
+ };
+ };
+
+ - |
+
+ /* AM4376 PRU-ICSS */
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ pruss@0 {
+ compatible = "ti,am4376-pruss";
+ reg = <0x0 0x40000>;
+ #address-cells = <1>;
+ #size-cells = <1>;
+ ranges;
+
+ interrupt-controller@20000 {
+ compatible = "ti,pruss-intc";
+ reg = <0x20000 0x2000>;
+ interrupt-controller;
+ #interrupt-cells = <3>;
+ interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-names = "host_intr0", "host_intr1",
+ "host_intr2", "host_intr3",
+ "host_intr4",
+ "host_intr6", "host_intr7";
+ ti,irqs-reserved = /bits/ 8 <0x20>; /* BIT(5) */
+ };
+ };
const: 0
patternProperties:
- "^multi-led[0-9a-f]$":
+ "^multi-led@[0-9a-b]$":
type: object
allOf:
- $ref: leds-class-multicolor.yaml#
Optional properties:
- linux,rc-map-name: see rc.txt file in the same
directory.
+ - linux,autosuspend-period: autosuspend delay time,
+ the unit is milisecond.
Example node:
compatible = "gpio-ir-receiver";
gpios = <&gpio0 19 1>;
linux,rc-map-name = "rc-rc6-mce";
+ linux,autosuspend-period = <125>;
};
+++ /dev/null
-* Sony 1/2.5-Inch 8.51Mp CMOS Digital Image Sensor
-
-The Sony imx274 is a 1/2.5-inch CMOS active pixel digital image sensor with
-an active array size of 3864H x 2202V. It is programmable through I2C
-interface. The I2C address is fixed to 0x1a as per sensor data sheet.
-Image data is sent through MIPI CSI-2, which is configured as 4 lanes
-at 1440 Mbps.
-
-
-Required Properties:
-- compatible: value should be "sony,imx274" for imx274 sensor
-- reg: I2C bus address of the device
-
-Optional Properties:
-- reset-gpios: Sensor reset GPIO
-- clocks: Reference to the input clock.
-- clock-names: Should be "inck".
-- VANA-supply: Sensor 2.8v analog supply.
-- VDIG-supply: Sensor 1.8v digital core supply.
-- VDDL-supply: Sensor digital IO 1.2v supply.
-
-The imx274 device node should contain one 'port' child node with
-an 'endpoint' subnode. For further reading on port node refer to
-Documentation/devicetree/bindings/media/video-interfaces.txt.
-
-Example:
- sensor@1a {
- compatible = "sony,imx274";
- reg = <0x1a>;
- #address-cells = <1>;
- #size-cells = <0>;
- reset-gpios = <&gpio_sensor 0 0>;
- port {
- sensor_out: endpoint {
- remote-endpoint = <&csiss_in>;
- };
- };
- };
+++ /dev/null
-Omnivision OV5647 raw image sensor
----------------------------------
-
-OV5647 is a raw image sensor with MIPI CSI-2 and CCP2 image data interfaces
-and CCI (I2C compatible) control bus.
-
-Required properties:
-
-- compatible : "ovti,ov5647".
-- reg : I2C slave address of the sensor.
-- clocks : Reference to the xclk clock.
-
-The common video interfaces bindings (see video-interfaces.txt) should be
-used to specify link to the image data receiver. The OV5647 device
-node should contain one 'port' child node with an 'endpoint' subnode.
-
-Endpoint node mandatory properties:
-
-- remote-endpoint: A phandle to the bus receiver's endpoint node.
-
-Example:
-
- i2c@2000 {
- ...
- ov: camera@36 {
- compatible = "ovti,ov5647";
- reg = <0x36>;
- clocks = <&camera_clk>;
- port {
- camera_1: endpoint {
- remote-endpoint = <&csi1_ep1>;
- };
- };
- };
- };
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/i2c/ov5647.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Omnivision OV5647 raw image sensor
+
+maintainers:
+ - Dave Stevenson <dave.stevenson@raspberrypi.com>
+ - Jacopo Mondi <jacopo@jmondi.org>
+
+description: |-
+ The OV5647 is a raw image sensor with MIPI CSI-2 and CCP2 image data
+ interfaces and CCI (I2C compatible) control bus.
+
+properties:
+ compatible:
+ const: ovti,ov5647
+
+ reg:
+ description: I2C device address.
+ maxItems: 1
+
+ clocks:
+ description: Reference to the xclk clock.
+ maxItems: 1
+
+ pwdn-gpios:
+ description: Reference to the GPIO connected to the pwdn pin. Active high.
+ maxItems: 1
+
+ port:
+ type: object
+ description: |-
+ Should contain one endpoint sub-node used to model connection to the
+ video receiver according to the specification defined in
+ Documentation/devicetree/bindings/media/video-interfaces.txt.
+
+ properties:
+ endpoint:
+ type: object
+
+ properties:
+ remote-endpoint:
+ description: |-
+ phandle to the video receiver input port.
+
+ clock-noncontinuous:
+ type: boolean
+ description: |-
+ Set to true to allow MIPI CSI-2 non-continuous clock operations.
+
+ additionalProperties: false
+
+ additionalProperties: false
+
+required:
+ - compatible
+ - reg
+ - clocks
+ - port
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/gpio/gpio.h>
+
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ ov5647: camera@36 {
+ compatible = "ovti,ov5647";
+ reg = <0x36>;
+ clocks = <&camera_clk>;
+ pwdn-gpios = <&pioE 29 GPIO_ACTIVE_HIGH>;
+
+ port {
+ camera_out: endpoint {
+ remote-endpoint = <&csi1_ep1>;
+ };
+ };
+ };
+ };
+
+...
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/i2c/sony,imx274.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Sony 1/2.5-Inch 8.51MP CMOS Digital Image Sensor
+
+maintainers:
+ - Leon Luo <leonl@leopardimaging.com>
+
+description: |
+ The Sony IMX274 is a 1/2.5-inch CMOS active pixel digital image sensor with an
+ active array size of 3864H x 2202V. It is programmable through I2C interface.
+ Image data is sent through MIPI CSI-2, which is configured as 4 lanes at 1440
+ Mbps.
+
+properties:
+ compatible:
+ const: sony,imx274
+
+ reg:
+ const: 0x1a
+
+ reset-gpios:
+ maxItems: 1
+
+ clocks:
+ maxItems: 1
+
+ clock-names:
+ const: inck
+
+ vana-supply:
+ description: Sensor 2.8 V analog supply.
+ maxItems: 1
+
+ vdig-supply:
+ description: Sensor 1.8 V digital core supply.
+ maxItems: 1
+
+ vddl-supply:
+ description: Sensor digital IO 1.2 V supply.
+ maxItems: 1
+
+ port:
+ type: object
+ description: Output video port. See ../video-interfaces.txt.
+
+required:
+ - compatible
+ - reg
+ - port
+
+additionalProperties: false
+
+examples:
+ - |
+ i2c0 {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ imx274: camera-sensor@1a {
+ compatible = "sony,imx274";
+ reg = <0x1a>;
+ reset-gpios = <&gpio_sensor 0 0>;
+
+ port {
+ sensor_out: endpoint {
+ remote-endpoint = <&csiss_in>;
+ };
+ };
+ };
+ };
+
+...
--- /dev/null
+* MediaTek JPEG Encoder
+
+MediaTek JPEG Encoder is the JPEG encode hardware present in MediaTek SoCs
+
+Required properties:
+- compatible : "mediatek,mt2701-jpgenc"
+ followed by "mediatek,mtk-jpgenc"
+- reg : physical base address of the JPEG encoder registers and length of
+ memory mapped region.
+- interrupts : interrupt number to the interrupt controller.
+- clocks: device clocks, see
+ Documentation/devicetree/bindings/clock/clock-bindings.txt for details.
+- clock-names: must contain "jpgenc". It is the clock of JPEG encoder.
+- power-domains: a phandle to the power domain, see
+ Documentation/devicetree/bindings/power/power_domain.txt for details.
+- mediatek,larb: must contain the local arbiters in the current SoCs, see
+ Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt
+ for details.
+- iommus: should point to the respective IOMMU block with master port as
+ argument, see Documentation/devicetree/bindings/iommu/mediatek,iommu.txt
+ for details.
+
+Example:
+ jpegenc: jpegenc@1500a000 {
+ compatible = "mediatek,mt2701-jpgenc",
+ "mediatek,mtk-jpgenc";
+ reg = <0 0x1500a000 0 0x1000>;
+ interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_LOW>;
+ clocks = <&imgsys CLK_IMG_VENC>;
+ clock-names = "jpgenc";
+ power-domains = <&scpsys MT2701_POWER_DOMAIN_ISP>;
+ mediatek,larb = <&larb2>;
+ iommus = <&iommu MT2701_M4U_PORT_JPGENC_RDMA>,
+ <&iommu MT2701_M4U_PORT_JPGENC_BSDMA>;
+ };
supports high resolution encoding and decoding functionalities.
Required properties:
-- compatible : "mediatek,mt8173-vcodec-enc" for encoder
- "mediatek,mt8173-vcodec-dec" for decoder.
+- compatible : "mediatek,mt8173-vcodec-enc" for MT8173 encoder
+ "mediatek,mt8183-vcodec-enc" for MT8183 encoder.
+ "mediatek,mt8173-vcodec-dec" for MT8173 decoder.
- reg : Physical base address of the video codec registers and length of
memory mapped region.
- interrupts : interrupt number to the cpu.
- iommus : should point to the respective IOMMU block with master port as
argument, see Documentation/devicetree/bindings/iommu/mediatek,iommu.txt
for details.
-- mediatek,vpu : the node of video processor unit
+One of the two following nodes:
+- mediatek,vpu : the node of the video processor unit, if using VPU.
+- mediatek,scp : the node of the SCP unit, if using SCP.
Example:
maxItems: 1
power-domains:
- maxItems: 2
+ minItems: 2
+ maxItems: 3
power-domain-names:
+ minItems: 2
+ maxItems: 3
items:
- const: venus
- const: vcodec0
+ - const: cx
clocks:
maxItems: 5
maxItems: 1
power-domains:
- maxItems: 3
+ minItems: 3
+ maxItems: 4
power-domain-names:
+ minItems: 3
+ maxItems: 4
items:
- const: venus
- const: vcodec0
- const: vcodec1
+ - const: cx
clocks:
maxItems: 7
- renesas,r8a774a1-csi2 # RZ/G2M
- renesas,r8a774b1-csi2 # RZ/G2N
- renesas,r8a774c0-csi2 # RZ/G2E
+ - renesas,r8a774e1-csi2 # RZ/G2H
- renesas,r8a7795-csi2 # R-Car H3
- renesas,r8a7796-csi2 # R-Car M3-W
- renesas,r8a77965-csi2 # R-Car M3-N
oneOf:
- items:
- enum:
+ - renesas,vin-r8a7742 # RZ/G1H
- renesas,vin-r8a7743 # RZ/G1M
- renesas,vin-r8a7744 # RZ/G1N
- renesas,vin-r8a7745 # RZ/G1E
- renesas,vin-r8a774a1 # RZ/G2M
- renesas,vin-r8a774b1 # RZ/G2N
- renesas,vin-r8a774c0 # RZ/G2E
+ - renesas,vin-r8a774e1 # RZ/G2H
- renesas,vin-r8a7778 # R-Car M1
- renesas,vin-r8a7779 # R-Car H1
- renesas,vin-r8a7795 # R-Car H3
This node should contain child 'port' nodes specifying active parallel video
input ports. It includes camera A and camera B inputs. 'reg' property in the
-port nodes specifies data input - 0, 1 indicates input A, B respectively.
+port nodes specifies data input - 1, 2 indicates input A, B respectively.
Optional properties
/* parallel camera ports */
parallel-ports {
/* camera A input */
- port@0 {
- reg = <0>;
+ port@1 {
+ reg = <1>;
fimc0_ep: endpoint {
remote-endpoint = <&s5k6aa_ep>;
bus-width = <8>;
properties:
compatible:
- enum:
- - fsl,imx25-esdhc
- - fsl,imx35-esdhc
- - fsl,imx51-esdhc
- - fsl,imx53-esdhc
- - fsl,imx6q-usdhc
- - fsl,imx6sl-usdhc
- - fsl,imx6sx-usdhc
- - fsl,imx6ull-usdhc
- - fsl,imx7d-usdhc
- - fsl,imx7ulp-usdhc
- - fsl,imx8mq-usdhc
- - fsl,imx8mm-usdhc
- - fsl,imx8mn-usdhc
- - fsl,imx8mp-usdhc
- - fsl,imx8qm-usdhc
- - fsl,imx8qxp-usdhc
+ oneOf:
+ - enum:
+ - fsl,imx25-esdhc
+ - fsl,imx35-esdhc
+ - fsl,imx51-esdhc
+ - fsl,imx53-esdhc
+ - fsl,imx6q-usdhc
+ - fsl,imx6sl-usdhc
+ - fsl,imx6sx-usdhc
+ - fsl,imx6ull-usdhc
+ - fsl,imx7d-usdhc
+ - fsl,imx7ulp-usdhc
+ - items:
+ - enum:
+ - fsl,imx8mm-usdhc
+ - fsl,imx8mn-usdhc
+ - fsl,imx8mp-usdhc
+ - fsl,imx8mq-usdhc
+ - fsl,imx8qxp-usdhc
+ - const: fsl,imx7d-usdhc
reg:
maxItems: 1
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mmc/microchip,dw-sparx5-sdhci.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Microchip Sparx5 Mobile Storage Host Controller Binding
+
+allOf:
+ - $ref: "mmc-controller.yaml"
+
+maintainers:
+ - Lars Povlsen <lars.povlsen@microchip.com>
+
+# Everything else is described in the common file
+properties:
+ compatible:
+ const: microchip,dw-sparx5-sdhci
+
+ reg:
+ maxItems: 1
+
+ interrupts:
+ maxItems: 1
+
+ clocks:
+ maxItems: 1
+ description:
+ Handle to "core" clock for the sdhci controller.
+
+ clock-names:
+ items:
+ - const: core
+
+ microchip,clock-delay:
+ description: Delay clock to card to meet setup time requirements.
+ Each step increase by 1.25ns.
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 1
+ maximum: 15
+
+required:
+ - compatible
+ - reg
+ - interrupts
+ - clocks
+ - clock-names
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ #include <dt-bindings/clock/microchip,sparx5.h>
+ sdhci0: mmc@600800000 {
+ compatible = "microchip,dw-sparx5-sdhci";
+ reg = <0x00800000 0x1000>;
+ pinctrl-0 = <&emmc_pins>;
+ pinctrl-names = "default";
+ clocks = <&clks CLK_ID_AUX1>;
+ clock-names = "core";
+ assigned-clocks = <&clks CLK_ID_AUX1>;
+ assigned-clock-rates = <800000000>;
+ interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
+ bus-width = <8>;
+ microchip,clock-delay = <10>;
+ };
that requires the respective functionality should implement them using
these definitions.
+ It is possible to assign a fixed index mmcN to an MMC host controller
+ (and the corresponding mmcblkN devices) by defining an alias in the
+ /aliases device tree node.
+
properties:
$nodename:
pattern: "^mmc(@.*)?$"
reset-gpios:
minItems: 1
+ # Put some limit to avoid false warnings
+ maxItems: 32
description:
contains a list of GPIO specifiers. The reset GPIOs are asserted
at initialization and prior we start the power up procedure of the card.
properties:
compatible:
- const: actions,owl-mmc
+ oneOf:
+ - const: actions,owl-mmc
+ - items:
+ - const: actions,s700-mmc
+ - const: actions,owl-mmc
reg:
maxItems: 1
- renesas,sdhi-r8a774a1 # RZ/G2M
- renesas,sdhi-r8a774b1 # RZ/G2N
- renesas,sdhi-r8a774c0 # RZ/G2E
+ - renesas,sdhi-r8a774e1 # RZ/G2H
- renesas,sdhi-r8a7795 # R-Car H3
- renesas,sdhi-r8a7796 # R-Car M3-W
- renesas,sdhi-r8a77961 # R-Car M3-W+
+++ /dev/null
-Device Tree Bindings for the SDHCI Controllers present on TI's AM654 SOCs
-
-The bindings follow the mmc[1], clock[2] and interrupt[3] bindings.
-Only deviations are documented here.
-
- [1] Documentation/devicetree/bindings/mmc/mmc.txt
- [2] Documentation/devicetree/bindings/clock/clock-bindings.txt
- [3] Documentation/devicetree/bindings/interrupt-controller/interrupts.txt
-
-Required Properties:
- - compatible: should be one of:
- "ti,am654-sdhci-5.1": SDHCI on AM654 device.
- "ti,j721e-sdhci-8bit": 8 bit SDHCI on J721E device.
- "ti,j721e-sdhci-4bit": 4 bit SDHCI on J721E device.
- - reg: Must be two entries.
- - The first should be the sdhci register space
- - The second should the subsystem/phy register space
- - clocks: Handles to the clock inputs.
- - clock-names: Tuple including "clk_xin" and "clk_ahb"
- - interrupts: Interrupt specifiers
- Output tap delay for each speed mode:
- - ti,otap-del-sel-legacy
- - ti,otap-del-sel-mmc-hs
- - ti,otap-del-sel-sd-hs
- - ti,otap-del-sel-sdr12
- - ti,otap-del-sel-sdr25
- - ti,otap-del-sel-sdr50
- - ti,otap-del-sel-sdr104
- - ti,otap-del-sel-ddr50
- - ti,otap-del-sel-ddr52
- - ti,otap-del-sel-hs200
- - ti,otap-del-sel-hs400
- These bindings must be provided otherwise the driver will disable the
- corresponding speed mode (i.e. all nodes must provide at least -legacy)
-
-Optional Properties (Required for ti,am654-sdhci-5.1 and ti,j721e-sdhci-8bit):
- - ti,trm-icp: DLL trim select
- - ti,driver-strength-ohm: driver strength in ohms.
- Valid values are 33, 40, 50, 66 and 100 ohms.
-Optional Properties:
- - ti,strobe-sel: strobe select delay for HS400 speed mode. Default value: 0x0.
- - ti,clkbuf-sel: Clock Delay Buffer Select
-
-Example:
-
- sdhci0: sdhci@4f80000 {
- compatible = "ti,am654-sdhci-5.1";
- reg = <0x0 0x4f80000 0x0 0x260>, <0x0 0x4f90000 0x0 0x134>;
- power-domains = <&k3_pds 47>;
- clocks = <&k3_clks 47 0>, <&k3_clks 47 1>;
- clock-names = "clk_ahb", "clk_xin";
- interrupts = <GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH>;
- sdhci-caps-mask = <0x80000007 0x0>;
- mmc-ddr-1_8v;
- ti,otap-del-sel-legacy = <0x0>;
- ti,otap-del-sel-mmc-hs = <0x0>;
- ti,otap-del-sel-ddr52 = <0x5>;
- ti,otap-del-sel-hs200 = <0x5>;
- ti,otap-del-sel-hs400 = <0x0>;
- ti,trm-icp = <0x8>;
- };
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+# Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/
+%YAML 1.2
+---
+$id: "http://devicetree.org/schemas/mmc/sdhci-am654.yaml#"
+$schema : "http://devicetree.org/meta-schemas/core.yaml#"
+
+title: TI AM654 MMC Controller
+
+maintainers:
+ - Ulf Hansson <ulf.hansson@linaro.org>
+
+allOf:
+ - $ref: mmc-controller.yaml#
+
+properties:
+ compatible:
+ enum:
+ - ti,am654-sdhci-5.1
+ - ti,j721e-sdhci-8bit
+ - ti,j721e-sdhci-4bit
+ - ti,j7200-sdhci-8bit
+ - ti,j721e-sdhci-4bit
+
+ reg:
+ maxItems: 2
+
+ interrupts:
+ maxItems: 1
+
+ power-domains:
+ maxItems: 1
+
+ clocks:
+ minItems: 1
+ maxItems: 2
+ description: Handles to input clocks
+
+ clock-names:
+ minItems: 1
+ maxItems: 2
+ items:
+ - const: clk_ahb
+ - const: clk_xin
+
+ # PHY output tap delays:
+ # Used to delay the data valid window and align it to the sampling clock.
+ # Binding needs to be provided for each supported speed mode otherwise the
+ # corresponding mode will be disabled.
+
+ ti,otap-del-sel-legacy:
+ description: Output tap delay for SD/MMC legacy timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ ti,otap-del-sel-mmc-hs:
+ description: Output tap delay for MMC high speed timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ ti,otap-del-sel-sd-hs:
+ description: Output tap delay for SD high speed timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ ti,otap-del-sel-sdr12:
+ description: Output tap delay for SD UHS SDR12 timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ ti,otap-del-sel-sdr25:
+ description: Output tap delay for SD UHS SDR25 timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ ti,otap-del-sel-sdr50:
+ description: Output tap delay for SD UHS SDR50 timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ ti,otap-del-sel-sdr104:
+ description: Output tap delay for SD UHS SDR104 timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ ti,otap-del-sel-ddr50:
+ description: Output tap delay for SD UHS DDR50 timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ ti,otap-del-sel-ddr52:
+ description: Output tap delay for eMMC DDR52 timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ ti,otap-del-sel-hs200:
+ description: Output tap delay for eMMC HS200 timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ ti,otap-del-sel-hs400:
+ description: Output tap delay for eMMC HS400 timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ # PHY input tap delays:
+ # Used to delay the data valid window and align it to the sampling clock for
+ # modes that don't support tuning
+
+ ti,itap-del-sel-legacy:
+ description: Input tap delay for SD/MMC legacy timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0x1f
+
+ ti,itap-del-sel-mmc-hs:
+ description: Input tap delay for MMC high speed timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0x1f
+
+ ti,itap-del-sel-sd-hs:
+ description: Input tap delay for SD high speed timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0x1f
+
+ ti,itap-del-sel-sdr12:
+ description: Input tap delay for SD UHS SDR12 timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0x1f
+
+ ti,itap-del-sel-sdr25:
+ description: Input tap delay for SD UHS SDR25 timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0x1f
+
+ ti,itap-del-sel-ddr52:
+ description: Input tap delay for MMC DDR52 timing
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0x1f
+
+ ti,trm-icp:
+ description: DLL trim select
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ minimum: 0
+ maximum: 0xf
+
+ ti,driver-strength-ohm:
+ description: DLL drive strength in ohms
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+ oneOf:
+ - enum:
+ - 33
+ - 40
+ - 50
+ - 66
+ - 100
+
+ ti,strobe-sel:
+ description: strobe select delay for HS400 speed mode.
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+
+ ti,clkbuf-sel:
+ description: Clock Delay Buffer Select
+ $ref: "/schemas/types.yaml#/definitions/uint32"
+
+required:
+ - compatible
+ - reg
+ - interrupts
+ - clocks
+ - clock-names
+ - ti,otap-del-sel-legacy
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/irq.h>
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+ bus {
+ #address-cells = <2>;
+ #size-cells = <2>;
+
+ mmc0: mmc@4f80000 {
+ compatible = "ti,am654-sdhci-5.1";
+ reg = <0x0 0x4f80000 0x0 0x260>, <0x0 0x4f90000 0x0 0x134>;
+ power-domains = <&k3_pds 47>;
+ clocks = <&k3_clks 47 0>, <&k3_clks 47 1>;
+ clock-names = "clk_ahb", "clk_xin";
+ interrupts = <GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH>;
+ sdhci-caps-mask = <0x80000007 0x0>;
+ mmc-ddr-1_8v;
+ ti,otap-del-sel-legacy = <0x0>;
+ ti,otap-del-sel-mmc-hs = <0x0>;
+ ti,otap-del-sel-ddr52 = <0x5>;
+ ti,otap-del-sel-hs200 = <0x5>;
+ ti,otap-del-sel-hs400 = <0x0>;
+ ti,itap-del-sel-legacy = <0x10>;
+ ti,itap-del-sel-mmc-hs = <0xa>;
+ ti,itap-del-sel-ddr52 = <0x3>;
+ ti,trm-icp = <0x8>;
+ };
+ };
- "renesas,etheravb-r8a774a1" for the R8A774A1 SoC.
- "renesas,etheravb-r8a774b1" for the R8A774B1 SoC.
- "renesas,etheravb-r8a774c0" for the R8A774C0 SoC.
+ - "renesas,etheravb-r8a774e1" for the R8A774E1 SoC.
- "renesas,etheravb-r8a7795" for the R8A7795 SoC.
- "renesas,etheravb-r8a7796" for the R8A77960 SoC.
- "renesas,etheravb-r8a77961" for the R8A77961 SoC.
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+# Copyright 2020 Arm Ltd.
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/perf/arm,cmn.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Arm CMN (Coherent Mesh Network) Performance Monitors
+
+maintainers:
+ - Robin Murphy <robin.murphy@arm.com>
+
+properties:
+ compatible:
+ const: arm,cmn-600
+
+ reg:
+ items:
+ - description: Physical address of the base (PERIPHBASE) and
+ size (up to 64MB) of the configuration address space.
+
+ interrupts:
+ minItems: 1
+ maxItems: 4
+ items:
+ - description: Overflow interrupt for DTC0
+ - description: Overflow interrupt for DTC1
+ - description: Overflow interrupt for DTC2
+ - description: Overflow interrupt for DTC3
+ description: One interrupt for each DTC domain implemented must
+ be specified, in order. DTC0 is always present.
+
+ arm,root-node:
+ $ref: /schemas/types.yaml#/definitions/uint32
+ description: Offset from PERIPHBASE of the configuration
+ discovery node (see TRM definition of ROOTNODEBASE).
+
+required:
+ - compatible
+ - reg
+ - interrupts
+ - arm,root-node
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/arm-gic.h>
+ #include <dt-bindings/interrupt-controller/irq.h>
+ pmu@50000000 {
+ compatible = "arm,cmn-600";
+ reg = <0x50000000 0x4000000>;
+ /* 4x2 mesh with one DTC, and CFG node at 0,1,1,0 */
+ interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
+ arm,root-node = <0x104000>;
+ };
+...
+++ /dev/null
-Monolithic Power Systems MP8867/MP8869 voltage regulator
-
-Required properties:
-- compatible: Must be one of the following.
- "mps,mp8867"
- "mps,mp8869"
-- reg: I2C slave address.
-- enable-gpios: enable gpios.
-- mps,fb-voltage-divider: An array of two integers containing the resistor
- values R1 and R2 of the feedback voltage divider in kilo ohms.
-
-Any property defined as part of the core regulator binding, defined in
-./regulator.txt, can also be used.
-
-Example:
-
- vcpu: regulator@62 {
- compatible = "mps,mp8869";
- regulator-name = "vcpu";
- regulator-min-microvolt = <700000>;
- regulator-max-microvolt = <850000>;
- regulator-always-on;
- regulator-boot-on;
- enable-gpios = <&porta 1 GPIO_ACTIVE_LOW>;
- mps,fb-voltage-divider = <80 240>;
- reg = <0x62>;
- };
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/mps,mp886x.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Monolithic Power Systems MP8867/MP8869 voltage regulator
+
+maintainers:
+ - Jisheng Zhang <jszhang@kernel.org>
+
+allOf:
+ - $ref: regulator.yaml#
+
+properties:
+ compatible:
+ enum:
+ - mps,mp8867
+ - mps,mp8869
+
+ reg:
+ maxItems: 1
+
+ enable-gpios:
+ description: GPIO to enable/disable the regulator.
+ maxItems: 1
+
+ mps,fb-voltage-divider:
+ description: An array of two integers containing the resistor
+ values R1 and R2 of the feedback voltage divider in kilo ohms.
+ $ref: "/schemas/types.yaml#/definitions/uint32-array"
+ maxItems: 2
+
+ mps,switch-frequency-hz:
+ description: The valid switch frequency in Hertz.
+ enum: [500000, 750000, 1000000, 1250000, 1500000]
+
+required:
+ - compatible
+ - reg
+ - enable-gpios
+ - mps,fb-voltage-divider
+
+examples:
+ - |
+ #include <dt-bindings/gpio/gpio.h>
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+ regulator@62 {
+ compatible = "mps,mp8869";
+ regulator-name = "vcpu";
+ regulator-min-microvolt = <800000>;
+ regulator-max-microvolt = <1150000>;
+ enable-gpios = <&porta 1 GPIO_ACTIVE_LOW>;
+ mps,fb-voltage-divider = <80 240>;
+ reg = <0x62>;
+ };
+ };
+
+...
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/mt6360-regulator.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MT6360 Regulator from MediaTek Integrated
+
+maintainers:
+ - Gene Chen <gene_chen@richtek.com>
+
+description: |
+ list of regulators provided by this controller, must be named
+ after their hardware counterparts buck1/2 or ldo1/2/3/5/6/7
+
+properties:
+ compatible:
+ const: mediatek,mt6360-regulator
+
+ LDO_VIN1-supply:
+ description: Input supply phandle(s) for LDO1/2/3
+ LDO_VIN2-supply:
+ description: Input supply phandle(s) for LDO5
+ LDO_VIN3-supply:
+ description: Input supply phandle(s) for LDO6/7
+
+patternProperties:
+ "^buck[12]$":
+ $ref: "regulator.yaml#"
+
+ "^ldo[123567]$":
+ $ref: "regulator.yaml#"
+
+required:
+ - compatible
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/irq.h>
+ #include <dt-bindings/regulator/mediatek,mt6360-regulator.h>
+ regulator {
+ compatible = "mediatek,mt6360-regulator";
+ LDO_VIN3-supply = <&BUCK2>;
+ buck1 {
+ regulator-compatible = "BUCK1";
+ regulator-name = "mt6360,buck1";
+ regulator-min-microvolt = <300000>;
+ regulator-max-microvolt = <1300000>;
+ regulator-allowed-modes = <MT6360_OPMODE_NORMAL
+ MT6360_OPMODE_LP
+ MT6360_OPMODE_ULP>;
+ };
+ BUCK2: buck2 {
+ regulator-compatible = "BUCK2";
+ regulator-name = "mt6360,buck2";
+ regulator-min-microvolt = <300000>;
+ regulator-max-microvolt = <1300000>;
+ regulator-allowed-modes = <MT6360_OPMODE_NORMAL
+ MT6360_OPMODE_LP
+ MT6360_OPMODE_ULP>;
+ };
+ ldo6 {
+ regulator-compatible = "LDO6";
+ regulator-name = "mt6360,ldo6";
+ regulator-min-microvolt = <500000>;
+ regulator-max-microvolt = <2100000>;
+ regulator-allowed-modes = <MT6360_OPMODE_NORMAL
+ MT6360_OPMODE_LP>;
+ };
+ ldo7 {
+ regulator-compatible = "LDO7";
+ regulator-name = "mt6360,ldo7";
+ regulator-min-microvolt = <500000>;
+ regulator-max-microvolt = <2100000>;
+ regulator-allowed-modes = <MT6360_OPMODE_NORMAL
+ MT6360_OPMODE_LP>;
+ };
+ ldo1 {
+ regulator-compatible = "LDO1";
+ regulator-name = "mt6360,ldo1";
+ regulator-min-microvolt = <1200000>;
+ regulator-max-microvolt = <3600000>;
+ regulator-allowed-modes = <MT6360_OPMODE_NORMAL
+ MT6360_OPMODE_LP>;
+ };
+ ldo2 {
+ regulator-compatible = "LDO2";
+ regulator-name = "mt6360,ldo2";
+ regulator-min-microvolt = <1200000>;
+ regulator-max-microvolt = <3600000>;
+ regulator-allowed-modes = <MT6360_OPMODE_NORMAL
+ MT6360_OPMODE_LP>;
+ };
+ ldo3 {
+ regulator-compatible = "LDO3";
+ regulator-name = "mt6360,ldo3";
+ regulator-min-microvolt = <1200000>;
+ regulator-max-microvolt = <3600000>;
+ regulator-allowed-modes = <MT6360_OPMODE_NORMAL
+ MT6360_OPMODE_LP>;
+ };
+ ldo5 {
+ regulator-compatible = "LDO5";
+ regulator-name = "mt6360,ldo5";
+ regulator-min-microvolt = <2700000>;
+ regulator-max-microvolt = <3600000>;
+ regulator-allowed-modes = <MT6360_OPMODE_NORMAL
+ MT6360_OPMODE_LP>;
+ };
+ };
+...
+++ /dev/null
-PFUZE100 family of regulators
-
-Required properties:
-- compatible: "fsl,pfuze100", "fsl,pfuze200", "fsl,pfuze3000", "fsl,pfuze3001"
-- reg: I2C slave address
-
-Optional properties:
-- fsl,pfuze-support-disable-sw: Boolean, if present disable all unused switch
- regulators to save power consumption. Attention, ensure that all important
- regulators (e.g. DDR ref, DDR supply) has set the "regulator-always-on"
- property. If not present, the switched regulators are always on and can't be
- disabled. This binding is a workaround to keep backward compatibility with
- old dtb's which rely on the fact that the switched regulators are always on
- and don't mark them explicit as "regulator-always-on".
-- fsl,pmic-stby-poweroff: if present, configure the PMIC to shutdown all
- power rails when PMIC_STBY_REQ line is asserted during the power off sequence.
- Use this option if the SoC should be powered off by external power
- management IC (PMIC) on PMIC_STBY_REQ signal.
- As opposite to PMIC_STBY_REQ boards can implement PMIC_ON_REQ signal.
-
-Required child node:
-- regulators: This is the list of child nodes that specify the regulator
- initialization data for defined regulators. Please refer to below doc
- Documentation/devicetree/bindings/regulator/regulator.txt.
-
- The valid names for regulators are:
- --PFUZE100
- sw1ab,sw1c,sw2,sw3a,sw3b,sw4,swbst,vsnvs,vrefddr,vgen1~vgen6
- --PFUZE200
- sw1ab,sw2,sw3a,sw3b,swbst,vsnvs,vrefddr,vgen1~vgen6,coin
- --PFUZE3000
- sw1a,sw1b,sw2,sw3,swbst,vsnvs,vrefddr,vldo1,vldo2,vccsd,v33,vldo3,vldo4
- --PFUZE3001
- sw1,sw2,sw3,vsnvs,vldo1,vldo2,vccsd,v33,vldo3,vldo4
-
-Each regulator is defined using the standard binding for regulators.
-
-Example 1: PFUZE100
-
- pfuze100: pmic@8 {
- compatible = "fsl,pfuze100";
- reg = <0x08>;
-
- regulators {
- sw1a_reg: sw1ab {
- regulator-min-microvolt = <300000>;
- regulator-max-microvolt = <1875000>;
- regulator-boot-on;
- regulator-always-on;
- regulator-ramp-delay = <6250>;
- };
-
- sw1c_reg: sw1c {
- regulator-min-microvolt = <300000>;
- regulator-max-microvolt = <1875000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- sw2_reg: sw2 {
- regulator-min-microvolt = <800000>;
- regulator-max-microvolt = <3300000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- sw3a_reg: sw3a {
- regulator-min-microvolt = <400000>;
- regulator-max-microvolt = <1975000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- sw3b_reg: sw3b {
- regulator-min-microvolt = <400000>;
- regulator-max-microvolt = <1975000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- sw4_reg: sw4 {
- regulator-min-microvolt = <800000>;
- regulator-max-microvolt = <3300000>;
- };
-
- swbst_reg: swbst {
- regulator-min-microvolt = <5000000>;
- regulator-max-microvolt = <5150000>;
- };
-
- snvs_reg: vsnvs {
- regulator-min-microvolt = <1000000>;
- regulator-max-microvolt = <3000000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- vref_reg: vrefddr {
- regulator-boot-on;
- regulator-always-on;
- };
-
- vgen1_reg: vgen1 {
- regulator-min-microvolt = <800000>;
- regulator-max-microvolt = <1550000>;
- };
-
- vgen2_reg: vgen2 {
- regulator-min-microvolt = <800000>;
- regulator-max-microvolt = <1550000>;
- };
-
- vgen3_reg: vgen3 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- };
-
- vgen4_reg: vgen4 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- vgen5_reg: vgen5 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- vgen6_reg: vgen6 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
- };
- };
-
-
-Example 2: PFUZE200
-
- pfuze200: pmic@8 {
- compatible = "fsl,pfuze200";
- reg = <0x08>;
-
- regulators {
- sw1a_reg: sw1ab {
- regulator-min-microvolt = <300000>;
- regulator-max-microvolt = <1875000>;
- regulator-boot-on;
- regulator-always-on;
- regulator-ramp-delay = <6250>;
- };
-
- sw2_reg: sw2 {
- regulator-min-microvolt = <800000>;
- regulator-max-microvolt = <3300000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- sw3a_reg: sw3a {
- regulator-min-microvolt = <400000>;
- regulator-max-microvolt = <1975000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- sw3b_reg: sw3b {
- regulator-min-microvolt = <400000>;
- regulator-max-microvolt = <1975000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- swbst_reg: swbst {
- regulator-min-microvolt = <5000000>;
- regulator-max-microvolt = <5150000>;
- };
-
- snvs_reg: vsnvs {
- regulator-min-microvolt = <1000000>;
- regulator-max-microvolt = <3000000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- vref_reg: vrefddr {
- regulator-boot-on;
- regulator-always-on;
- };
-
- vgen1_reg: vgen1 {
- regulator-min-microvolt = <800000>;
- regulator-max-microvolt = <1550000>;
- };
-
- vgen2_reg: vgen2 {
- regulator-min-microvolt = <800000>;
- regulator-max-microvolt = <1550000>;
- };
-
- vgen3_reg: vgen3 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- };
-
- vgen4_reg: vgen4 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- vgen5_reg: vgen5 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- vgen6_reg: vgen6 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- coin_reg: coin {
- regulator-min-microvolt = <2500000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
- };
- };
-
-Example 3: PFUZE3000
-
- pfuze3000: pmic@8 {
- compatible = "fsl,pfuze3000";
- reg = <0x08>;
-
- regulators {
- sw1a_reg: sw1a {
- regulator-min-microvolt = <700000>;
- regulator-max-microvolt = <1475000>;
- regulator-boot-on;
- regulator-always-on;
- regulator-ramp-delay = <6250>;
- };
- /* use sw1c_reg to align with pfuze100/pfuze200 */
- sw1c_reg: sw1b {
- regulator-min-microvolt = <700000>;
- regulator-max-microvolt = <1475000>;
- regulator-boot-on;
- regulator-always-on;
- regulator-ramp-delay = <6250>;
- };
-
- sw2_reg: sw2 {
- regulator-min-microvolt = <2500000>;
- regulator-max-microvolt = <3300000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- sw3a_reg: sw3 {
- regulator-min-microvolt = <900000>;
- regulator-max-microvolt = <1650000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- swbst_reg: swbst {
- regulator-min-microvolt = <5000000>;
- regulator-max-microvolt = <5150000>;
- };
-
- snvs_reg: vsnvs {
- regulator-min-microvolt = <1000000>;
- regulator-max-microvolt = <3000000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- vref_reg: vrefddr {
- regulator-boot-on;
- regulator-always-on;
- };
-
- vgen1_reg: vldo1 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- vgen2_reg: vldo2 {
- regulator-min-microvolt = <800000>;
- regulator-max-microvolt = <1550000>;
- };
-
- vgen3_reg: vccsd {
- regulator-min-microvolt = <2850000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- vgen4_reg: v33 {
- regulator-min-microvolt = <2850000>;
- regulator-max-microvolt = <3300000>;
- };
-
- vgen5_reg: vldo3 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- vgen6_reg: vldo4 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
- };
- };
-
-Example 4: PFUZE 3001
-
- pfuze3001: pmic@8 {
- compatible = "fsl,pfuze3001";
- reg = <0x08>;
-
- regulators {
- sw1_reg: sw1 {
- regulator-min-microvolt = <700000>;
- regulator-max-microvolt = <3300000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- sw2_reg: sw2 {
- regulator-min-microvolt = <1500000>;
- regulator-max-microvolt = <3300000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- sw3_reg: sw3 {
- regulator-min-microvolt = <900000>;
- regulator-max-microvolt = <1650000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- snvs_reg: vsnvs {
- regulator-min-microvolt = <1000000>;
- regulator-max-microvolt = <3000000>;
- regulator-boot-on;
- regulator-always-on;
- };
-
- vgen1_reg: vldo1 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- vgen2_reg: vldo2 {
- regulator-min-microvolt = <800000>;
- regulator-max-microvolt = <1550000>;
- regulator-always-on;
- };
-
- vgen3_reg: vccsd {
- regulator-min-microvolt = <2850000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- vgen4_reg: v33 {
- regulator-min-microvolt = <2850000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- vgen5_reg: vldo3 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
-
- vgen6_reg: vldo4 {
- regulator-min-microvolt = <1800000>;
- regulator-max-microvolt = <3300000>;
- regulator-always-on;
- };
- };
- };
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/pfuze100.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: PFUZE100 family of regulators
+
+maintainers:
+ - Robin Gong <yibin.gong@nxp.com>
+
+description: |
+ The valid names for regulators are:
+ --PFUZE100
+ sw1ab,sw1c,sw2,sw3a,sw3b,sw4,swbst,vsnvs,vrefddr,vgen1~vgen6
+ --PFUZE200
+ sw1ab,sw2,sw3a,sw3b,swbst,vsnvs,vrefddr,vgen1~vgen6,coin
+ --PFUZE3000
+ sw1a,sw1b,sw2,sw3,swbst,vsnvs,vrefddr,vldo1,vldo2,vccsd,v33,vldo3,vldo4
+ --PFUZE3001
+ sw1,sw2,sw3,vsnvs,vldo1,vldo2,vccsd,v33,vldo3,vldo4
+
+ Each regulator is defined using the standard binding for regulators.
+
+properties:
+ $nodename:
+ pattern: "^pmic@[0-9]$"
+
+ compatible:
+ enum:
+ - fsl,pfuze100
+ - fsl,pfuze200
+ - fsl,pfuze3000
+ - fsl,pfuze3001
+
+ reg:
+ maxItems: 1
+
+ fsl,pfuze-support-disable-sw:
+ $ref: /schemas/types.yaml#/definitions/flag
+ description: |
+ Boolean, if present disable all unused switch regulators to save power
+ consumption. Attention, ensure that all important regulators
+ (e.g. DDR ref, DDR supply) has set the "regulator-always-on" property.
+ If not present, the switched regulators are always on and can't be
+ disabled. This binding is a workaround to keep backward compatibility
+ with old dtb's which rely on the fact that the switched regulators are
+ always on and don't mark them explicit as "regulator-always-on".
+
+ fsl,pmic-stby-poweroff:
+ $ref: /schemas/types.yaml#/definitions/flag
+ description: |
+ if present, configure the PMIC to shutdown all
+ power rails when PMIC_STBY_REQ line is asserted during the power off sequence.
+ Use this option if the SoC should be powered off by external power management
+ IC (PMIC) on PMIC_STBY_REQ signal.
+ As opposite to PMIC_STBY_REQ boards can implement PMIC_ON_REQ signal.
+
+ regulators:
+ type: object
+ description: |
+ list of regulators provided by this controller.
+
+ patternProperties:
+ "^sw([1-4]|[1-4][a-c]|[1-4][a-c][a-c])$":
+ $ref: "regulator.yaml#"
+ type: object
+
+ "^vgen[1-6]$":
+ $ref: "regulator.yaml#"
+ type: object
+
+ "^(vsnvs|vref|vrefddr|swbst|coin)$":
+ $ref: "regulator.yaml#"
+ type: object
+
+ additionalProperties: false
+
+required:
+ - compatible
+ - reg
+
+examples:
+ - |
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ pmic@8 {
+ compatible = "fsl,pfuze100";
+ reg = <0x08>;
+
+ regulators {
+ sw1a_reg: sw1ab {
+ regulator-min-microvolt = <300000>;
+ regulator-max-microvolt = <1875000>;
+ regulator-boot-on;
+ regulator-always-on;
+ regulator-ramp-delay = <6250>;
+ };
+
+ sw1c_reg: sw1c {
+ regulator-min-microvolt = <300000>;
+ regulator-max-microvolt = <1875000>;
+ regulator-boot-on;
+ regulator-always-on;
+ };
+
+ sw2_reg: sw2 {
+ regulator-min-microvolt = <800000>;
+ regulator-max-microvolt = <3300000>;
+ regulator-boot-on;
+ regulator-always-on;
+ };
+
+ sw3a_reg: sw3a {
+ regulator-min-microvolt = <400000>;
+ regulator-max-microvolt = <1975000>;
+ regulator-boot-on;
+ regulator-always-on;
+ };
+
+ sw3b_reg: sw3b {
+ regulator-min-microvolt = <400000>;
+ regulator-max-microvolt = <1975000>;
+ regulator-boot-on;
+ regulator-always-on;
+ };
+
+ sw4_reg: sw4 {
+ regulator-min-microvolt = <800000>;
+ regulator-max-microvolt = <3300000>;
+ };
+
+ swbst_reg: swbst {
+ regulator-min-microvolt = <5000000>;
+ regulator-max-microvolt = <5150000>;
+ };
+
+ snvs_reg: vsnvs {
+ regulator-min-microvolt = <1000000>;
+ regulator-max-microvolt = <3000000>;
+ regulator-boot-on;
+ regulator-always-on;
+ };
+
+ vref_reg: vrefddr {
+ regulator-boot-on;
+ regulator-always-on;
+ };
+
+ vgen1_reg: vgen1 {
+ regulator-min-microvolt = <800000>;
+ regulator-max-microvolt = <1550000>;
+ };
+
+ vgen2_reg: vgen2 {
+ regulator-min-microvolt = <800000>;
+ regulator-max-microvolt = <1550000>;
+ };
+
+ vgen3_reg: vgen3 {
+ regulator-min-microvolt = <1800000>;
+ regulator-max-microvolt = <3300000>;
+ };
+
+ vgen4_reg: vgen4 {
+ regulator-min-microvolt = <1800000>;
+ regulator-max-microvolt = <3300000>;
+ regulator-always-on;
+ };
+
+ vgen5_reg: vgen5 {
+ regulator-min-microvolt = <1800000>;
+ regulator-max-microvolt = <3300000>;
+ regulator-always-on;
+ };
+
+ vgen6_reg: vgen6 {
+ regulator-min-microvolt = <1800000>;
+ regulator-max-microvolt = <3300000>;
+ regulator-always-on;
+ };
+ };
+ };
+ };
l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22, l23, l24, lvs1, lvs2,
lvs3, 5vs1, 5vs2
+ For pm8950 and pm8953, s1, s2, s3, s4, s5, s6, s7, l1, l2, l3, l4, l5, l6,
+ l7, l8, l9, l10, l11, l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22,
+ l23
+
For pm8994, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, l1, l2, l3,
l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16, l17, l18, l19,
l20, l21, l22, l23, l24, l25, l26, l27, l28, l29, l30, l31, l32, lvs1, lvs2
l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16, l17, l18, l19,
l20, l21, l22, l23, l24, l25, l26, l27, l28, lvs1, lvs2
+ For pm660, s1, s2, s3, s4, s5, s6, l1, l2, l3, l5, l6, l7, l8, l9, l10, l22,
+ l12, l13, l14, l15, l16, l17, l18, l19
+
+ For pm660l s1, s2, s3, s5, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, bob
+
For pma8084, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, l1, l2, l3,
l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16, l17, l18, l19,
l20, l21, l22, l23, l24, l25, l26, l27, lvs1, lvs2, lvs3, lvs4, 5vs1
- qcom,rpm-pm8916-regulators
- qcom,rpm-pm8941-regulators
- qcom,rpm-pm8950-regulators
+ - qcom,rpm-pm8953-regulators
- qcom,rpm-pm8994-regulators
- qcom,rpm-pm8998-regulators
+ - qcom,rpm-pm660-regulators
+ - qcom,rpm-pm660l-regulators
- qcom,rpm-pma8084-regulators
- qcom,rpm-pmi8994-regulators
- qcom,rpm-pmi8998-regulators
"qcom,pm8950-regulators"
"qcom,pm8994-regulators"
"qcom,pmi8994-regulators"
+ "qcom,pm660-regulators"
+ "qcom,pm660l-regulators"
"qcom,pms405-regulators"
- interrupts:
Definition: Reference to regulator supplying the input pin, as
described in the data sheet.
+- vdd_l1_l6_l7-supply:
+- vdd_l2_l3-supply:
+- vdd_l5-supply:
+- vdd_l8_l9_l10_l11_l12_l13_l14-supply:
+- vdd_l15_l16_l17_l18_l19-supply:
+- vdd_s1-supply:
+- vdd_s2-supply:
+- vdd_s3-supply:
+- vdd_s5-supply:
+- vdd_s6-supply:
+ Usage: optional (pm660 only)
+ Value type: <phandle>
+ Definition: Reference to regulator supplying the input pin, as
+ described in the data sheet.
+
+- vdd_l1_l9_l10-supply:
+- vdd_l2-supply:
+- vdd_l3_l5_l7_l8-supply:
+- vdd_l4_l6-supply:
+- vdd_s1-supply:
+- vdd_s2-supply:
+- vdd_s3-supply:
+- vdd_s4-supply:
+- vdd_s5-supply:
+ Usage: optional (pm660l only)
+ Value type: <phandle>
+ Definition: Reference to regulator supplying the input pin, as
+ described in the data sheet.
+
- vdd_l1_l2-supply:
- vdd_l3_l8-supply:
- vdd_l4-supply:
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/raspberrypi,7inch-touchscreen-panel-regulator.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: RaspberryPi 7" display ATTINY88-based regulator/backlight controller
+
+maintainers:
+ - Marek Vasut <marex@denx.de>
+
+description: |
+ The RaspberryPi 7" display has an ATTINY88-based regulator/backlight
+ controller on the PCB, which is used to turn the display unit on/off
+ and control the backlight.
+
+allOf:
+ - $ref: "regulator.yaml#"
+
+properties:
+ compatible:
+ const: raspberrypi,7inch-touchscreen-panel-regulator
+
+ reg:
+ maxItems: 1
+
+additionalProperties: false
+
+required:
+ - compatible
+ - reg
+
+examples:
+ - |
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+ regulator@45 {
+ compatible = "raspberrypi,7inch-touchscreen-panel-regulator";
+ reg = <0x45>;
+ };
+ };
+
+...
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/richtek,rt4801-regulator.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Richtek RT4801 Display Bias regulators
+
+maintainers:
+ - ChiYuan Huang <cy_huang@richtek.com>
+
+description: |
+ Regulator nodes should be named to DSVP and DSVN. The
+ definition for each of these nodes is defined using the standard
+ binding for regulators at
+ Documentation/devicetree/bindings/regulator/regulator.txt.
+ Datasheet is available at
+ https://www.richtek.com/assets/product_file/RT4801H/DS4801H-00.pdf
+
+#The valid names for RT4801 regulator nodes are:
+#DSVP, DSVN
+
+properties:
+ compatible:
+ enum:
+ - richtek,rt4801
+
+ reg:
+ maxItems: 1
+
+ enable-gpios:
+ description: GPIOs to use to enable DSVP/DSVN regulator.
+ The first one is ENP to enable DSVP, and second one is ENM to enable DSVN.
+ Number of GPIO in the array list could be 1 or 2.
+ If only one gpio is specified, only one gpio used to control ENP/ENM.
+ Else both are spefied, DSVP/DSVN could be controlled individually.
+ Othersie, this property not specified. treat both as always-on regulator.
+ minItems: 1
+ maxItems: 2
+
+patternProperties:
+ "^DSV(P|N)$":
+ type: object
+ $ref: regulator.yaml#
+ description:
+ Properties for single display bias regulator.
+
+required:
+ - compatible
+ - reg
+
+additionalProperties: false
+
+examples:
+ - |
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ rt4801@73 {
+ compatible = "richtek,rt4801";
+ reg = <0x73>;
+ enable-gpios = <&gpio26 2 0>, <&gpio26 3 0>;
+
+ dsvp: DSVP {
+ regulator-name = "rt4801,dsvp";
+ regulator-min-microvolt = <4000000>;
+ regulator-max-microvolt = <6000000>;
+ regulator-boot-on;
+ };
+ dsvn: DSVN {
+ regulator-name = "rt4801,dsvn";
+ regulator-min-microvolt = <4000000>;
+ regulator-max-microvolt = <6000000>;
+ regulator-boot-on;
+ };
+
+ };
+ };
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/richtek,rtmv20-regulator.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Richtek RTMV20 laser diode regulator
+
+maintainers:
+ - ChiYuan Huang <cy_huang@richtek.com>
+
+description: |
+ Richtek RTMV20 is a load switch current regulator that can supply up to 6A.
+ It is used to drive laser diode. There're two signals for chip controls
+ (Enable/Fail), Enable pin to turn chip on, and Fail pin as fault indication.
+ There're still four pins for camera control, two inputs (strobe and vsync),
+ the others for outputs (fsin1 and fsin2). Strobe input to start the current
+ supply, vsync input from IR camera, and fsin1/fsin2 output for the optional.
+
+properties:
+ compatible:
+ const: richtek,rtmv20
+
+ reg:
+ maxItems: 1
+
+ wakeup-source: true
+
+ interrupts:
+ maxItems: 1
+
+ enable-gpios:
+ description: A connection of the 'enable' gpio line.
+ maxItems: 1
+
+ richtek,ld-pulse-delay-us:
+ description: |
+ load current pulse delay in microsecond after strobe pin pulse high.
+ minimum: 0
+ maximum: 100000
+ default: 0
+
+ richtek,ld-pulse-width-us:
+ description: |
+ Load current pulse width in microsecond after strobe pin pulse high.
+ minimum: 0
+ maximum: 10000
+ default: 1200
+
+ richtek,fsin1-delay-us:
+ description: |
+ Fsin1 pulse high delay in microsecond after vsync signal pulse high.
+ minimum: 0
+ maximum: 100000
+ default: 23000
+
+ richtek,fsin1-width-us:
+ description: |
+ Fsin1 pulse high width in microsecond after vsync signal pulse high.
+ minimum: 40
+ maximum: 10000
+ default: 160
+
+ richtek,fsin2-delay-us:
+ description: |
+ Fsin2 pulse high delay in microsecond after vsync signal pulse high.
+ minimum: 0
+ maximum: 100000
+ default: 23000
+
+ richtek,fsin2-width-us:
+ description: |
+ Fsin2 pulse high width in microsecond after vsync signal pulse high.
+ minimum: 40
+ maximum: 10000
+ default: 160
+
+ richtek,es-pulse-width-us:
+ description: Eye safety function pulse width limit in microsecond.
+ minimum: 0
+ maximum: 10000
+ default: 1200
+
+ richtek,es-ld-current-microamp:
+ description: Eye safety function load current limit in microamp.
+ minimum: 0
+ maximum: 6000000
+ default: 3000000
+
+ richtek,lbp-level-microvolt:
+ description: Low battery protection level in microvolt.
+ minimum: 2400000
+ maximum: 3700000
+ default: 2700000
+
+ richtek,lbp-enable:
+ description: Low battery protection function enable control.
+ type: boolean
+
+ richtek,strobe-polarity-high:
+ description: Strobe pin active polarity control.
+ type: boolean
+
+ richtek,vsync-polarity-high:
+ description: Vsync pin active polarity control.
+ type: boolean
+
+ richtek,fsin-enable:
+ description: Fsin function enable control.
+ type: boolean
+
+ richtek,fsin-output:
+ description: Fsin function output control.
+ type: boolean
+
+ richtek,es-enable:
+ description: Eye safety function enable control.
+ type: boolean
+
+ lsw:
+ description: load switch current regulator description.
+ type: object
+ $ref: "regulator.yaml#"
+
+required:
+ - compatible
+ - reg
+ - wakeup-source
+ - interrupts
+ - enable-gpios
+ - lsw
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/interrupt-controller/irq.h>
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ rtmv20@34 {
+ compatible = "richtek,rtmv20";
+ reg = <0x34>;
+ wakeup-source;
+ interrupts-extended = <&gpio26 2 IRQ_TYPE_LEVEL_LOW>;
+ enable-gpios = <&gpio26 3 0>;
+
+ richtek,strobe-polarity-high;
+ richtek,vsync-polarity-high;
+
+ lsw {
+ regulator-name = "rtmv20,lsw";
+ regulator-min-microamp = <0>;
+ regulator-max-microamp = <6000000>;
+ };
+ };
+ };
+...
# ----------------------------------------------------------------
# rest | not supported | not supported | not supported
+ # BD71837 power outputs can either be controlled by the PMIC internal
+ # hardware state machine or by software. If you need regulators to be
+ # turned ON/OFF for example based on PMIC_STBY_REQ line (which toggles
+ # PMIC HW state machine) - then you should set this property.
+ # Tradeoff is that then SW can't control the ON/OFF state for this
+ # regulator (other than invoking a PMIC state change).
+ rohm,no-regulator-enable-control:
+ description: |
+ Enable/Disable control of this regulator must be left to the
+ PMIC hardware state machine.
+ type: boolean
required:
- regulator-name
# ----------------------------------------------------------------
# rest | not supported | not supported | not supported
+ # BD718(47/50) power outputs can either be controlled by the PMIC internal
+ # hardware state machine or by software. If you need regulators to be
+ # turned ON/OFF for example based on PMIC_STBY_REQ line (which toggles
+ # PMIC HW state machine) - then you should set this property.
+ # Tradeoff is that then SW can't control the ON/OFF state for this
+ # regulator (other than invoking a PMIC state change).
+ rohm,no-regulator-enable-control:
+ description: |
+ Enable/Disable control of this regulator must be left to the
+ PMIC hardware state machine.
+ type: boolean
required:
- regulator-name
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/rohm,bd9576-regulator.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ROHM BD9576 and BD9573 Power Management Integrated Circuit regulators
+
+maintainers:
+ - Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>
+
+description: |
+ This module is part of the ROHM BD9576 MFD device. For more details
+ see Documentation/devicetree/bindings/mfd/rohm,bd9576-pmic.yaml.
+
+ The regulator controller is represented as a sub-node of the PMIC node
+ on the device tree.
+
+ The valid names for BD9576 regulator nodes are
+ regulator-vd50, regulator-vd18, regulator-vdddr, regulator-vd10,
+ regulator-voutl1, regulator-vouts1
+
+patternProperties:
+ "regulator-.+":
+ type: object
+ description:
+ Properties for single regulator.
+ $ref: "regulator.yaml#"
+
+ required:
+ - regulator-name
+
+ unevaluatedProperties: false
+additionalProperties: false
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/silergy,sy8824x.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: silergy sy8824c,sy8824e,sy20276 and sy20278 PMIC
+
+maintainers:
+ - Jisheng Zhang <jszhang@kernel.org>
+
+allOf:
+ - $ref: regulator.yaml#
+
+properties:
+ compatible:
+ enum:
+ - silergy,sy8824c
+ - silergy,sy8824e
+ - silergy,sy20276
+ - silergy,sy20278
+
+ reg:
+ maxItems: 1
+
+required:
+ - compatible
+ - reg
+
+unevaluatedProperties: false
+
+examples:
+ - |
+ i2c {
+ #address-cells = <1>;
+ #size-cells = <0>;
+ regulator@60 {
+ compatible = "silergy,sy8824c";
+ regulator-min-microvolt = <800000>;
+ regulator-max-microvolt = <1150000>;
+ reg = <0x60>;
+ };
+ };
+
+...
+++ /dev/null
-SY8824C/SY8824E/SY20276 Voltage regulator
-
-Required properties:
-- compatible: Must be one of the following.
- "silergy,sy8824c"
- "silergy,sy8824e"
- "silergy,sy20276"
- "silergy,sy20278"
-- reg: I2C slave address
-
-Any property defined as part of the core regulator binding, defined in
-./regulator.txt, can also be used.
-
-Example:
-
- vcore: regulator@00 {
- compatible = "silergy,sy8824c";
- reg = <0x66>;
- regulator-name = "vcore";
- regulator-min-microvolt = <800000>;
- regulator-max-microvolt = <1150000>;
- regulator-boot-on;
- regulator-always-on;
- };
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/rng/ingenic,trng.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Bindings for DTRNG in Ingenic SoCs
+
+maintainers:
+ - 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
+
+description:
+ The True Random Number Generator in Ingenic SoCs.
+
+properties:
+ compatible:
+ enum:
+ - ingenic,x1830-dtrng
+
+ reg:
+ maxItems: 1
+
+ clocks:
+ maxItems: 1
+
+required:
+ - compatible
+ - reg
+ - clocks
+
+additionalProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/clock/x1830-cgu.h>
+
+ dtrng: trng@10072000 {
+ compatible = "ingenic,x1830-dtrng";
+ reg = <0x10072000 0xc>;
+
+ clocks = <&cgu X1830_CLK_DTRNG>;
+ };
+...
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/rng/xiphera,xip8001b-trng.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Xiphera XIP8001B-trng bindings
+
+maintainers:
+ - Atte Tommiska <atte.tommiska@xiphera.com>
+
+description: |
+ Xiphera FPGA-based true random number generator intellectual property core.
+
+properties:
+ compatible:
+ const: xiphera,xip8001b-trng
+
+ reg:
+ maxItems: 1
+
+required:
+ - compatible
+ - reg
+
+additionalProperties: false
+
+examples:
+ - |
+ rng@43c00000 {
+ compatible = "xiphera,xip8001b-trng";
+ reg = <0x43c00000 0x10000>;
+ };
BRCMSTB SoCs
"brcm,spi-bcm7435-qspi", "brcm,spi-bcm-qspi", "brcm,spi-brcmstb-mspi" : Second Instance of MSPI
BRCMSTB SoCs
+ "brcm,spi-bcm7445-qspi", "brcm,spi-bcm-qspi", "brcm,spi-brcmstb-mspi" : Second Instance of MSPI
+ BRCMSTB SoCs
"brcm,spi-bcm7216-qspi", "brcm,spi-bcm-qspi", "brcm,spi-brcmstb-mspi" : Second Instance of MSPI
BRCMSTB SoCs
"brcm,spi-bcm7278-qspi", "brcm,spi-bcm-qspi", "brcm,spi-brcmstb-mspi" : Second Instance of MSPI
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/spi/mediatek,spi-mtk-nor.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Serial NOR flash controller for MediaTek ARM SoCs
+
+maintainers:
+ - Bayi Cheng <bayi.cheng@mediatek.com>
+ - Chuanhong Guo <gch981213@gmail.com>
+
+description: |
+ This spi controller support single, dual, or quad mode transfer for
+ SPI NOR flash. There should be only one spi slave device following
+ generic spi bindings. It's not recommended to use this controller
+ for devices other than SPI NOR flash due to limited transfer
+ capability of this controller.
+
+allOf:
+ - $ref: /spi/spi-controller.yaml#
+
+properties:
+ compatible:
+ oneOf:
+ - items:
+ - enum:
+ - mediatek,mt2701-nor
+ - mediatek,mt2712-nor
+ - mediatek,mt7622-nor
+ - mediatek,mt7623-nor
+ - mediatek,mt7629-nor
+ - mediatek,mt8192-nor
+ - enum:
+ - mediatek,mt8173-nor
+ - items:
+ - const: mediatek,mt8173-nor
+ reg:
+ maxItems: 1
+
+ interrupts:
+ maxItems: 1
+
+ clocks:
+ items:
+ - description: clock used for spi bus
+ - description: clock used for controller
+
+ clock-names:
+ items:
+ - const: spi
+ - const: sf
+
+required:
+ - compatible
+ - reg
+ - interrupts
+ - clocks
+ - clock-names
+
+unevaluatedProperties: false
+
+examples:
+ - |
+ #include <dt-bindings/clock/mt8173-clk.h>
+
+ soc {
+ #address-cells = <2>;
+ #size-cells = <2>;
+
+ nor_flash: spi@1100d000 {
+ compatible = "mediatek,mt8173-nor";
+ reg = <0 0x1100d000 0 0xe0>;
+ interrupts = <&spi_flash_irq>;
+ clocks = <&pericfg CLK_PERI_SPI>, <&topckgen CLK_TOP_SPINFI_IFR_SEL>;
+ clock-names = "spi", "sf";
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ flash@0 {
+ compatible = "jedec,spi-nor";
+ reg = <0>;
+ };
+ };
+ };
+
- items:
- enum:
+ - renesas,qspi-r8a7742 # RZ/G1H
- renesas,qspi-r8a7743 # RZ/G1M
- renesas,qspi-r8a7744 # RZ/G1N
- renesas,qspi-r8a7745 # RZ/G1E
- renesas,msiof-r8a774e1 # RZ/G2H
- renesas,msiof-r8a7795 # R-Car H3
- renesas,msiof-r8a7796 # R-Car M3-W
+ - renesas,msiof-r8a77961 # R-Car M3-W+
- renesas,msiof-r8a77965 # R-Car M3-N
- renesas,msiof-r8a77970 # R-Car V3M
- renesas,msiof-r8a77980 # R-Car V3H
properties:
reg:
minItems: 2
+ - if:
+ properties:
+ compatible:
+ contains:
+ enum:
+ - baikal,bt1-sys-ssi
+ then:
+ properties:
+ mux-controls:
+ maxItems: 1
+ required:
+ - mux-controls
+ else:
+ required:
+ - interrupts
properties:
compatible:
- mscc,ocelot-spi
- mscc,jaguar2-spi
- const: snps,dw-apb-ssi
+ - description: Microchip Sparx5 SoC SPI Controller
+ const: microchip,sparx5-spi
- description: Amazon Alpine SPI Controller
const: amazon,alpine-dw-apb-ssi
- description: Renesas RZ/N1 SPI Controller
- const: snps,dw-apb-ssi
- description: Intel Keem Bay SPI Controller
const: intel,keembay-ssi
+ - description: Baikal-T1 SPI Controller
+ const: baikal,bt1-ssi
+ - description: Baikal-T1 System Boot SPI Controller
+ const: baikal,bt1-sys-ssi
reg:
minItems: 1
items:
- description: DW APB SSI controller memory mapped registers
- - description: SPI MST region map
+ - description: SPI MST region map or directly mapped SPI ROM
interrupts:
maxItems: 1
- const: tx
- const: rx
+ rx-sample-delay-ns:
+ default: 0
+ description: Default value of the rx-sample-delay-ns property.
+ This value will be used if the property is not explicitly defined
+ for a SPI slave device. See below.
+
patternProperties:
"^.*@[0-9a-f]+$":
type: object
spi-tx-bus-width:
const: 1
+ rx-sample-delay-ns:
+ description: SPI Rx sample delay offset, unit is nanoseconds.
+ The delay from the default sample time before the actual
+ sample of the rxd input signal occurs. The "rx_sample_delay"
+ is an optional feature of the designware controller, and the
+ upper limit is also subject to controller configuration.
+
unevaluatedProperties: false
required:
- reg
- "#address-cells"
- "#size-cells"
- - interrupts
- clocks
examples:
num-cs = <2>;
cs-gpios = <&gpio0 13 0>,
<&gpio0 14 0>;
+ rx-sample-delay-ns = <3>;
+ spi-flash@1 {
+ compatible = "spi-nand";
+ reg = <1>;
+ rx-sample-delay-ns = <7>;
+ };
+ };
+ - |
+ spi@1f040100 {
+ compatible = "baikal,bt1-sys-ssi";
+ reg = <0x1f040100 0x900>,
+ <0x1c000000 0x1000000>;
+ #address-cells = <1>;
+ #size-cells = <0>;
+ mux-controls = <&boot_mux>;
+ clocks = <&ccu_sys>;
+ clock-names = "ssi_clk";
};
...
+++ /dev/null
-* Serial NOR flash controller for MediaTek ARM SoCs
-
-Required properties:
-- compatible: For mt8173, compatible should be "mediatek,mt8173-nor",
- and it's the fallback compatible for other Soc.
- For every other SoC, should contain both the SoC-specific compatible
- string and "mediatek,mt8173-nor".
- The possible values are:
- "mediatek,mt2701-nor", "mediatek,mt8173-nor"
- "mediatek,mt2712-nor", "mediatek,mt8173-nor"
- "mediatek,mt7622-nor", "mediatek,mt8173-nor"
- "mediatek,mt7623-nor", "mediatek,mt8173-nor"
- "mediatek,mt7629-nor", "mediatek,mt8173-nor"
- "mediatek,mt8173-nor"
-- reg: physical base address and length of the controller's register
-- interrupts: Interrupt number used by the controller.
-- clocks: the phandle of the clocks needed by the nor controller
-- clock-names: the names of the clocks
- the clocks should be named "spi" and "sf". "spi" is used for spi bus,
- and "sf" is used for controller, these are the clocks witch
- hardware needs to enabling nor flash and nor flash controller.
- See Documentation/devicetree/bindings/clock/clock-bindings.txt for details.
-- #address-cells: should be <1>
-- #size-cells: should be <0>
-
-There should be only one spi slave device following generic spi bindings.
-It's not recommended to use this controller for devices other than SPI NOR
-flash due to limited transfer capability of this controller.
-
-Example:
-
-nor_flash: spi@1100d000 {
- compatible = "mediatek,mt8173-nor";
- reg = <0 0x1100d000 0 0xe0>;
- interrupts = <&spi_flash_irq>;
- clocks = <&pericfg CLK_PERI_SPI>,
- <&topckgen CLK_TOP_SPINFI_IFR_SEL>;
- clock-names = "spi", "sf";
- #address-cells = <1>;
- #size-cells = <0>;
-
- flash@0 {
- compatible = "jedec,spi-nor";
- reg = <0>;
- };
-};
-
- items:
- enum:
- renesas,r8a73a4-cmt0 # 32-bit CMT0 on R-Mobile APE6
+ - renesas,r8a7742-cmt0 # 32-bit CMT0 on RZ/G1H
- renesas,r8a7743-cmt0 # 32-bit CMT0 on RZ/G1M
- renesas,r8a7744-cmt0 # 32-bit CMT0 on RZ/G1N
- renesas,r8a7745-cmt0 # 32-bit CMT0 on RZ/G1E
- items:
- enum:
- renesas,r8a73a4-cmt1 # 48-bit CMT1 on R-Mobile APE6
+ - renesas,r8a7742-cmt1 # 48-bit CMT1 on RZ/G1H
- renesas,r8a7743-cmt1 # 48-bit CMT1 on RZ/G1M
- renesas,r8a7744-cmt1 # 48-bit CMT1 on RZ/G1N
- renesas,r8a7745-cmt1 # 48-bit CMT1 on RZ/G1E
- renesas,r8a774a1-cmt0 # 32-bit CMT0 on RZ/G2M
- renesas,r8a774b1-cmt0 # 32-bit CMT0 on RZ/G2N
- renesas,r8a774c0-cmt0 # 32-bit CMT0 on RZ/G2E
+ - renesas,r8a774e1-cmt0 # 32-bit CMT0 on RZ/G2H
- renesas,r8a7795-cmt0 # 32-bit CMT0 on R-Car H3
- renesas,r8a7796-cmt0 # 32-bit CMT0 on R-Car M3-W
- renesas,r8a77965-cmt0 # 32-bit CMT0 on R-Car M3-N
- renesas,r8a774a1-cmt1 # 48-bit CMT on RZ/G2M
- renesas,r8a774b1-cmt1 # 48-bit CMT on RZ/G2N
- renesas,r8a774c0-cmt1 # 48-bit CMT on RZ/G2E
+ - renesas,r8a774e1-cmt1 # 48-bit CMT on RZ/G2H
- renesas,r8a7795-cmt1 # 48-bit CMT on R-Car H3
- renesas,r8a7796-cmt1 # 48-bit CMT on R-Car M3-W
- renesas,r8a77965-cmt1 # 48-bit CMT on R-Car M3-N
- fsl,mpl3115
# MPR121: Proximity Capacitive Touch Sensor Controller
- fsl,mpr121
+ # Monolithic Power Systems Inc. multi-phase controller mp2975
+ - mps,mp2975
# G751: Digital Temperature Sensor and Thermal Watchdog with Two-Wire Interface
- gmt,g751
# Infineon IR38064 Voltage Regulator
- nuvoton,npct601
# Nuvoton Temperature Sensor
- nuvoton,w83773g
- # Octal SMBus and I2C registered interface
- - nxp,pca9556
- # 8-bit I2C-bus and SMBus I/O port with reset
- - nxp,pca9557
# OKI ML86V7667 video decoder
- oki,ml86v7667
# OV5642: Color CMOS QSXGA (5-megapixel) Image Sensor with OmniBSI and Embedded TrueFocus
- silabs,si7020
# Skyworks SKY81452: Six-Channel White LED Driver with Touch Panel Bias Supply
- skyworks,sky81452
+ # Socionext SynQuacer TPM MMIO module
+ - socionext,synquacer-tpm-mmio
# i2c serial eeprom (24cxx)
- st,24c256
# Ambient Light Sensor with SMBUS/Two Wire Serial Interface
description: Shenzhen Xingbangda Display Technology Co., Ltd
"^xinpeng,.*":
description: Shenzhen Xinpeng Technology Co., Ltd
+ "^xiphera,.*":
+ description: Xiphera Ltd.
"^xlnx,.*":
description: Xilinx
"^xnano,.*":
Cross-referencing from reStructuredText
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-To cross-reference the functions and types defined in the kernel-doc comments
-from reStructuredText documents, please use the `Sphinx C Domain`_
-references. For example::
-
- See function :c:func:`foo` and struct/union/enum/typedef :c:type:`bar`.
-
-While the type reference works with just the type name, without the
-struct/union/enum/typedef part in front, you may want to use::
-
- See :c:type:`struct foo <foo>`.
- See :c:type:`union bar <bar>`.
- See :c:type:`enum baz <baz>`.
- See :c:type:`typedef meh <meh>`.
-
-This will produce prettier links, and is in line with how kernel-doc does the
-cross-references.
+No additional syntax is needed to cross-reference the functions and types
+defined in the kernel-doc comments from reStructuredText documents.
+Just end function names with ``()`` and write ``struct``, ``union``, ``enum``
+or ``typedef`` before types.
+For example::
+
+ See foo().
+ See struct foo.
+ See union bar.
+ See enum baz.
+ See typedef meh.
+
+However, if you want custom text in the cross-reference link, that can be done
+through the following syntax::
+
+ See :c:func:`my custom link text for function foo <foo>`.
+ See :c:type:`my custom link text for struct bar <bar>`.
For further details, please refer to the `Sphinx C Domain`_ documentation.
- column 3
+Cross-referencing
+-----------------
+
+Cross-referencing from one documentation page to another can be done by passing
+the path to the file starting from the Documentation folder.
+For example, to cross-reference to this page (the .rst extension is optional)::
+
+ See Documentation/doc-guide/sphinx.rst.
+
+If you want to use a relative path, you need to use Sphinx's ``doc`` directive.
+For example, referencing this page from the same directory would be done as::
+
+ See :doc:`sphinx`.
+
+For information on cross-referencing to kernel-doc functions or types, see
+Documentation/doc-guide/kernel-doc.rst.
+
.. _sphinx_kfigure:
Figures & Images
- Memory mapping the contents of the DMA buffer is also supported. See the
discussion below on `CPU Access to DMA Buffer Objects`_ for the full details.
-- The DMA buffer FD is also pollable, see `Fence Poll Support`_ below for
+- The DMA buffer FD is also pollable, see `Implicit Fence Poll Support`_ below for
details.
Basic Operation and Device DMA Access
forced to a thread. The "fake?" raw lock can be used to work around this
problem::
- raw_spinlock_t wa_lock;
- static irqreturn_t omap_gpio_irq_handler(int irq, void *gpiobank)
- unsigned long wa_lock_flags;
- raw_spin_lock_irqsave(&bank->wa_lock, wa_lock_flags);
- generic_handle_irq(irq_find_mapping(bank->chip.irq.domain, bit));
- raw_spin_unlock_irqrestore(&bank->wa_lock, wa_lock_flags);
+ raw_spinlock_t wa_lock;
+ static irqreturn_t omap_gpio_irq_handler(int irq, void *gpiobank)
+ unsigned long wa_lock_flags;
+ raw_spin_lock_irqsave(&bank->wa_lock, wa_lock_flags);
+ generic_handle_irq(irq_find_mapping(bank->chip.irq.domain, bit));
+ raw_spin_unlock_irqrestore(&bank->wa_lock, wa_lock_flags);
- GENERIC CHAINED GPIO IRQCHIPS: these are the same as "CHAINED GPIO irqchips",
but chained IRQ handlers are not used. Instead GPIO IRQs dispatching is
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+Writing camera sensor drivers
+=============================
+
+CSI-2
+-----
+
+Please see what is written on :ref:`MIPI_CSI_2`.
+
+Handling clocks
+---------------
+
+Camera sensors have an internal clock tree including a PLL and a number of
+divisors. The clock tree is generally configured by the driver based on a few
+input parameters that are specific to the hardware:: the external clock frequency
+and the link frequency. The two parameters generally are obtained from system
+firmware. No other frequencies should be used in any circumstances.
+
+The reason why the clock frequencies are so important is that the clock signals
+come out of the SoC, and in many cases a specific frequency is designed to be
+used in the system. Using another frequency may cause harmful effects
+elsewhere. Therefore only the pre-determined frequencies are configurable by the
+user.
+
+Frame size
+----------
+
+There are two distinct ways to configure the frame size produced by camera
+sensors.
+
+Freely configurable camera sensor drivers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Freely configurable camera sensor drivers expose the device's internal
+processing pipeline as one or more sub-devices with different cropping and
+scaling configurations. The output size of the device is the result of a series
+of cropping and scaling operations from the device's pixel array's size.
+
+An example of such a driver is the smiapp driver (see drivers/media/i2c/smiapp).
+
+Register list based drivers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Register list based drivers generally, instead of able to configure the device
+they control based on user requests, are limited to a number of preset
+configurations that combine a number of different parameters that on hardware
+level are independent. How a driver picks such configuration is based on the
+format set on a source pad at the end of the device's internal pipeline.
+
+Most sensor drivers are implemented this way, see e.g.
+drivers/media/i2c/imx319.c for an example.
+
+Frame interval configuration
+----------------------------
+
+There are two different methods for obtaining possibilities for different frame
+intervals as well as configuring the frame interval. Which one to implement
+depends on the type of the device.
+
+Raw camera sensors
+~~~~~~~~~~~~~~~~~~
+
+Instead of a high level parameter such as frame interval, the frame interval is
+a result of the configuration of a number of camera sensor implementation
+specific parameters. Luckily, these parameters tend to be the same for more or
+less all modern raw camera sensors.
+
+The frame interval is calculated using the following equation::
+
+ frame interval = (analogue crop width + horizontal blanking) *
+ (analogue crop height + vertical blanking) / pixel rate
+
+The formula is bus independent and is applicable for raw timing parameters on
+large variety of devices beyond camera sensors. Devices that have no analogue
+crop, use the full source image size, i.e. pixel array size.
+
+Horizontal and vertical blanking are specified by ``V4L2_CID_HBLANK`` and
+``V4L2_CID_VBLANK``, respectively. The unit of these controls are lines. The
+pixel rate is specified by ``V4L2_CID_PIXEL_RATE`` in the same sub-device. The
+unit of that control is Hz.
+
+Register list based drivers need to implement read-only sub-device nodes for the
+purpose. Devices that are not register list based need these to configure the
+device's internal processing pipeline.
+
+The first entity in the linear pipeline is the pixel array. The pixel array may
+be followed by other entities that are there to allow configuring binning,
+skipping, scaling or digital crop :ref:`v4l2-subdev-selections`.
+
+USB cameras etc. devices
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+USB video class hardware, as well as many cameras offering a similar higher
+level interface natively, generally use the concept of frame interval (or frame
+rate) on device level in firmware or hardware. This means lower level controls
+implemented by raw cameras may not be used on uAPI (or even kAPI) to control the
+frame interval on these devices.
+
+Power management
+----------------
+
+Always use runtime PM to manage the power states of your device. Camera sensor
+drivers are in no way special in this respect: they are responsible for
+controlling the power state of the device they otherwise control as well. In
+general, the device must be powered on at least when its registers are being
+accessed and when it is streaming.
+
+Existing camera sensor drivers may rely on the old
+:c:type:`v4l2_subdev_core_ops`->s_power() callback for bridge or ISP drivers to
+manage their power state. This is however **deprecated**. If you feel you need
+to begin calling an s_power from an ISP or a bridge driver, instead please add
+runtime PM support to the sensor driver you are using. Likewise, new drivers
+should not use s_power.
+
+Please see examples in e.g. ``drivers/media/i2c/ov8856.c`` and
+``drivers/media/i2c/smiapp/smiapp-core.c``. The two drivers work in both ACPI
+and DT based systems.
+
+Control framework
+~~~~~~~~~~~~~~~~~
+
+``v4l2_ctrl_handler_setup()`` function may not be used in the device's runtime
+PM ``runtime_resume`` callback, as it has no way to figure out the power state
+of the device. This is because the power state of the device is only changed
+after the power state transition has taken place. The ``s_ctrl`` callback can be
+used to obtain device's power state after the power state transition:
+
+.. c:function::
+ int pm_runtime_get_if_in_use(struct device *dev);
+
+The function returns a non-zero value if it succeeded getting the power count or
+runtime PM was disabled, in either of which cases the driver may proceed to
+access the device.
calling cec_allocate_adapter() and deleted by calling cec_delete_adapter():
.. c:function::
- struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops, void *priv,
- const char *name, u32 caps, u8 available_las);
+ struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops, \
+ void *priv, const char *name, \
+ u32 caps, u8 available_las);
.. c:function::
void cec_delete_adapter(struct cec_adapter *adap);
CEC_CAP_RC is set) you call:
.. c:function::
- int cec_register_adapter(struct cec_adapter *adap, struct device *parent);
+ int cec_register_adapter(struct cec_adapter *adap, \
+ struct device *parent);
where parent is the parent device.
hardware:
-To enable/disable the hardware:
+To enable/disable the hardware::
-.. c:function::
int (*adap_enable)(struct cec_adapter *adap, bool enable);
This callback enables or disables the CEC hardware. Enabling the CEC hardware
Note that adap_enable must return 0 if enable is false.
-To enable/disable the 'monitor all' mode:
+To enable/disable the 'monitor all' mode::
-.. c:function::
int (*adap_monitor_all_enable)(struct cec_adapter *adap, bool enable);
If enabled, then the adapter should be put in a mode to also monitor messages
Note that adap_monitor_all_enable must return 0 if enable is false.
-To enable/disable the 'monitor pin' mode:
+To enable/disable the 'monitor pin' mode::
-.. c:function::
int (*adap_monitor_pin_enable)(struct cec_adapter *adap, bool enable);
If enabled, then the adapter should be put in a mode to also monitor CEC pin
Note that adap_monitor_pin_enable must return 0 if enable is false.
-To program a new logical address:
+To program a new logical address::
-.. c:function::
int (*adap_log_addr)(struct cec_adapter *adap, u8 logical_addr);
If logical_addr == CEC_LOG_ADDR_INVALID then all programmed logical addresses
Note that adap_log_addr must return 0 if logical_addr is CEC_LOG_ADDR_INVALID.
-To transmit a new message:
+To transmit a new message::
-.. c:function::
int (*adap_transmit)(struct cec_adapter *adap, u8 attempts,
u32 signal_free_time, struct cec_msg *msg);
microseconds (one data bit period is 2.4 ms).
-To log the current CEC hardware status:
+To log the current CEC hardware status::
-.. c:function::
void (*adap_status)(struct cec_adapter *adap, struct seq_file *file);
This optional callback can be used to show the status of the CEC hardware.
The status is available through debugfs: cat /sys/kernel/debug/cec/cecX/status
-To free any resources when the adapter is deleted:
+To free any resources when the adapter is deleted::
-.. c:function::
void (*adap_free)(struct cec_adapter *adap);
This optional callback can be used to free any resources that might have been
Your adapter driver will also have to react to events (typically interrupt
driven) by calling into the framework in the following situations:
-When a transmit finished (successfully or otherwise):
+When a transmit finished (successfully or otherwise)::
-.. c:function::
- void cec_transmit_done(struct cec_adapter *adap, u8 status, u8 arb_lost_cnt,
- u8 nack_cnt, u8 low_drive_cnt, u8 error_cnt);
+ void cec_transmit_done(struct cec_adapter *adap, u8 status,
+ u8 arb_lost_cnt, u8 nack_cnt, u8 low_drive_cnt,
+ u8 error_cnt);
-or:
+or::
-.. c:function::
void cec_transmit_attempt_done(struct cec_adapter *adap, u8 status);
The status can be one of:
$ cat einj.txt >error-inj
The first callback is called when this file is read and it should show the
-the current error injection state:
+the current error injection state::
-.. c:function::
int (*error_inj_show)(struct cec_adapter *adap, struct seq_file *sf);
It is recommended that it starts with a comment block with basic usage
information. It returns 0 for success and an error otherwise.
-The second callback will parse commands written to the ``error-inj`` file:
+The second callback will parse commands written to the ``error-inj`` file::
-.. c:function::
bool (*error_inj_parse_line)(struct cec_adapter *adap, char *line);
The ``line`` argument points to the start of the command. Any leading
};
The received() callback allows the driver to optionally handle a newly
-received CEC message
+received CEC message::
-.. c:function::
int (*received)(struct cec_adapter *adap, struct cec_msg *msg);
If the driver wants to process a CEC message, then it can implement this
CEC Adapter drivers can call the following CEC framework functions:
.. c:function::
- int cec_transmit_msg(struct cec_adapter *adap, struct cec_msg *msg,
- bool block);
+ int cec_transmit_msg(struct cec_adapter *adap, struct cec_msg *msg, \
+ bool block);
Transmit a CEC message. If block is true, then wait until the message has been
transmitted, otherwise just queue it and return.
.. c:function::
- void cec_s_phys_addr(struct cec_adapter *adap, u16 phys_addr,
- bool block);
+ void cec_s_phys_addr(struct cec_adapter *adap, u16 phys_addr, bool block);
Change the physical address. This function will set adap->phys_addr and
send an event if it has changed. If cec_s_log_addrs() has been called and
address to CEC_PHYS_ADDR_INVALID before enabling the new physical address.
.. c:function::
- void cec_s_phys_addr_from_edid(struct cec_adapter *adap,
- const struct edid *edid);
+ void cec_s_phys_addr_from_edid(struct cec_adapter *adap, \
+ const struct edid *edid);
A helper function that extracts the physical address from the edid struct
and calls cec_s_phys_addr() with that address, or CEC_PHYS_ADDR_INVALID
if the EDID did not contain a physical address or edid was a NULL pointer.
.. c:function::
- int cec_s_log_addrs(struct cec_adapter *adap,
+ int cec_s_log_addrs(struct cec_adapter *adap, \
struct cec_log_addrs *log_addrs, bool block);
Claim the CEC logical addresses. Should never be called if CEC_CAP_LOG_ADDRS
.. SPDX-License-Identifier: GPL-2.0
+.. _MIPI_CSI_2:
+
MIPI CSI-2
==========
sh_mobile_ceu_camera
tuners
vimc-devel
+ zoran
Digital TV drivers
dvb-usb
frontends
+ vidtv
contributors
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+================================
+vidtv: Virtual Digital TV driver
+================================
+
+Author: Daniel W. S. Almeida <dwlsalmeida@gmail.com>, June 2020.
+
+Background
+----------
+
+Vidtv is a virtual DVB driver that aims to serve as a reference for driver
+writers by serving as a template. It also validates the existing media DVB
+APIs, thus helping userspace application writers.
+
+Currently, it consists of:
+
+- A fake tuner driver, which will report a bad signal quality if the chosen
+ frequency is too far away from a table of valid frequencies for a
+ particular delivery system.
+
+- A fake demod driver, which will constantly poll the fake signal quality
+ returned by the tuner, simulating a device that can lose/reacquire a lock
+ on the signal depending on the CNR levels.
+
+- A fake bridge driver, which is the module responsible for modprobing the
+ fake tuner and demod modules and implementing the demux logic. This module
+ takes parameters at initialization that will dictate how the simulation
+ behaves.
+
+- Code reponsible for encoding a valid MPEG Transport Stream, which is then
+ passed to the bridge driver. This fake stream contains some hardcoded content.
+ For now, we have a single, audio-only channel containing a single MPEG
+ Elementary Stream, which in turn contains a SMPTE 302m encoded sine-wave.
+ Note that this particular encoder was chosen because it is the easiest
+ way to encode PCM audio data in a MPEG Transport Stream.
+
+Building vidtv
+--------------
+vidtv is a test driver and thus is **not** enabled by default when
+compiling the kernel.
+
+In order to enable compilation of vidtv:
+
+- Enable **DVB_TEST_DRIVERS**, then
+- Enable **DVB_VIDTV**
+
+When compiled as a module, expect the following .ko files:
+
+- dvb_vidtv_tuner.ko
+
+- dvb_vidtv_demod.ko
+
+- dvb_vidtv_bridge.ko
+
+Running vidtv
+-------------
+When compiled as a module, run::
+
+ modprobe vidtv
+
+That's it! The bridge driver will initialize the tuner and demod drivers as
+part of its own initialization.
+
+By default, it will accept the following frequencies:
+
+ - 474 MHz for DVB-T/T2/C;
+ - 11,362 GHz for DVB-S/S2.
+
+For satellite systems, the driver simulates an universal extended
+LNBf, with frequencies at Ku-Band, ranging from 10.7 GHz to 12.75 GHz.
+
+You can optionally define some command-line arguments to vidtv.
+
+Command-line arguments to vidtv
+-------------------------------
+Below is a list of all arguments that can be supplied to vidtv:
+
+drop_tslock_prob_on_low_snr
+ Probability of losing the TS lock if the signal quality is bad.
+ This probability be used by the fake demodulator driver to
+ eventually return a status of 0 when the signal quality is not
+ good.
+
+recover_tslock_prob_on_good_snr:
+ Probability recovering the TS lock when the signal improves. This
+ probability be used by the fake demodulator driver to eventually
+ return a status of 0x1f when/if the signal quality improves.
+
+mock_power_up_delay_msec
+ Simulate a power up delay. Default: 0.
+
+mock_tune_delay_msec
+ Simulate a tune delay. Default 0.
+
+vidtv_valid_dvb_t_freqs
+ Valid DVB-T frequencies to simulate, in Hz.
+
+vidtv_valid_dvb_c_freqs
+ Valid DVB-C frequencies to simulate, in Hz.
+
+vidtv_valid_dvb_s_freqs
+ Valid DVB-S/S2 frequencies to simulate at Ku-Band, in kHz.
+
+max_frequency_shift_hz,
+ Maximum shift in HZ allowed when tuning in a channel.
+
+si_period_msec
+ How often to send SI packets. Default: 40ms.
+
+pcr_period_msec
+ How often to send PCR packets. Default: 40ms.
+
+mux_rate_kbytes_sec
+ Attempt to maintain this bit rate by inserting TS null packets, if
+ necessary. Default: 4096.
+
+pcr_pid,
+ PCR PID for all channels. Default: 0x200.
+
+mux_buf_sz_pkts,
+ Size for the mux buffer in multiples of 188 bytes.
+
+vidtv internal structure
+------------------------
+The kernel modules are split in the following way:
+
+vidtv_tuner.[ch]
+ Implements a fake tuner DVB driver.
+
+vidtv_demod.[ch]
+ Implements a fake demodulator DVB driver.
+
+vidtv_bridge.[ch]
+ Implements a bridge driver.
+
+The MPEG related code is split in the following way:
+
+vidtv_ts.[ch]
+ Code to work with MPEG TS packets, such as TS headers, adaptation
+ fields, PCR packets and NULL packets.
+
+vidtv_psi.[ch]
+ This is the PSI generator. PSI packets contain general information
+ about a MPEG Transport Stream. A PSI generator is needed so
+ userspace apps can retrieve information about the Transport Stream
+ and eventually tune into a (dummy) channel.
+
+ Because the generator is implemented in a separate file, it can be
+ reused elsewhere in the media subsystem.
+
+ Currently vidtv supports working with 3 PSI tables: PAT, PMT and
+ SDT.
+
+ The specification for PAT and PMT can be found in *ISO 13818-1:
+ Systems*, while the specification for the SDT can be found in *ETSI
+ EN 300 468: Specification for Service Information (SI) in DVB
+ systems*.
+
+ It isn't strictly necessary, but using a real TS file helps when
+ debugging PSI tables. Vidtv currently tries to replicate the PSI
+ structure found in this file: `TS1Globo.ts
+ <https://tsduck.io/streams/brazil-isdb-tb/TS1globo.ts>`_.
+
+ A good way to visualize the structure of streams is by using
+ `DVBInspector <https://sourceforge.net/projects/dvbinspector/>`_.
+
+vidtv_pes.[ch]
+ Implements the PES logic to convert encoder data into MPEG TS
+ packets. These can then be fed into a TS multiplexer and eventually
+ into userspace.
+
+vidtv_encoder.h
+ An interface for vidtv encoders. New encoders can be added to this
+ driver by implementing the calls in this file.
+
+vidtv_s302m.[ch]
+ Implements a S302M encoder to make it possible to insert PCM audio
+ data in the generated MPEG Transport Stream. The relevant
+ specification is available online as *SMPTE 302M-2007: Television -
+ Mapping of AES3 Data into MPEG-2 Transport Stream*.
+
+
+ The resulting MPEG Elementary Stream is conveyed in a private
+ stream with a S302M registration descriptor attached.
+
+ This shall enable passing an audio signal into userspace so it can
+ be decoded and played by media software. The corresponding decoder
+ in ffmpeg is located in 'libavcodec/s302m.c' and is experimental.
+
+vidtv_channel.[ch]
+ Implements a 'channel' abstraction.
+
+ When vidtv boots, it will create some hardcoded channels:
+
+ #. Their services will be concatenated to populate the SDT.
+
+ #. Their programs will be concatenated to populate the PAT
+
+ #. For each program in the PAT, a PMT section will be created
+
+ #. The PMT section for a channel will be assigned its streams.
+
+ #. Every stream will have its corresponding encoder polled in a
+ loop to produce TS packets.
+ These packets may be interleaved by the muxer and then delivered
+ to the bridge.
+
+vidtv_mux.[ch]
+ Implements a MPEG TS mux, loosely based on the ffmpeg
+ implementation in "libavcodec/mpegtsenc.c"
+
+ The muxer runs a loop which is responsible for:
+
+ #. Keeping track of the amount of time elapsed since the last
+ iteration.
+
+ #. Polling encoders in order to fetch 'elapsed_time' worth of data.
+
+ #. Inserting PSI and/or PCR packets, if needed.
+
+ #. Padding the resulting stream with NULL packets if
+ necessary in order to maintain the chosen bit rate.
+
+ #. Delivering the resulting TS packets to the bridge
+ driver so it can pass them to the demux.
+
+Testing vidtv with v4l-utils
+----------------------------
+
+Using the tools in v4l-utils is a great way to test and inspect the output of
+vidtv. It is hosted here: `v4l-utils Documentation
+<https://linuxtv.org/wiki/index.php/V4l-utils>`_.
+
+From its webpage::
+
+ The v4l-utils are a series of packages for handling media devices.
+
+ It is hosted at http://git.linuxtv.org/v4l-utils.git, and packaged
+ on most distributions.
+
+ It provides a series of libraries and utilities to be used to
+ control several aspect of the media boards.
+
+
+Start by installing v4l-utils and then modprobing vidtv::
+
+ modprobe dvb_vidtv_bridge
+
+If the driver is OK, it should load and its probing code will run. This will
+pull in the tuner and demod drivers.
+
+Using dvb-fe-tool
+~~~~~~~~~~~~~~~~~
+
+The first step to check whether the demod loaded successfully is to run::
+
+ $ dvb-fe-tool
+
+This should return what is currently set up at the demod struct, i.e.::
+
+ static const struct dvb_frontend_ops vidtv_demod_ops = {
+ .delsys = {
+ SYS_DVBT,
+ SYS_DVBT2,
+ SYS_DVBC_ANNEX_A,
+ SYS_DVBS,
+ SYS_DVBS2,
+ },
+
+ .info = {
+ .name = "Dummy demod for DVB-T/T2/C/S/S2",
+ .frequency_min_hz = 51 * MHz,
+ .frequency_max_hz = 2150 * MHz,
+ .frequency_stepsize_hz = 62500,
+ .frequency_tolerance_hz = 29500 * kHz,
+ .symbol_rate_min = 1000000,
+ .symbol_rate_max = 45000000,
+
+ .caps = FE_CAN_FEC_1_2 |
+ FE_CAN_FEC_2_3 |
+ FE_CAN_FEC_3_4 |
+ FE_CAN_FEC_4_5 |
+ FE_CAN_FEC_5_6 |
+ FE_CAN_FEC_6_7 |
+ FE_CAN_FEC_7_8 |
+ FE_CAN_FEC_8_9 |
+ FE_CAN_QAM_16 |
+ FE_CAN_QAM_64 |
+ FE_CAN_QAM_32 |
+ FE_CAN_QAM_128 |
+ FE_CAN_QAM_256 |
+ FE_CAN_QAM_AUTO |
+ FE_CAN_QPSK |
+ FE_CAN_FEC_AUTO |
+ FE_CAN_INVERSION_AUTO |
+ FE_CAN_TRANSMISSION_MODE_AUTO |
+ FE_CAN_GUARD_INTERVAL_AUTO |
+ FE_CAN_HIERARCHY_AUTO,
+ }
+
+ ....
+
+For more information on dvb-fe-tools check its online documentation here:
+`dvb-fe-tool Documentation
+<https://www.linuxtv.org/wiki/index.php/Dvb-fe-tool>`_.
+
+Using dvb-scan
+~~~~~~~~~~~~~~
+
+In order to tune into a channel and read the PSI tables, we can use dvb-scan.
+
+For this, one should provide a configuration file known as a 'scan file',
+here's an example::
+
+ [Channel]
+ FREQUENCY = 330000000
+ MODULATION = QAM/AUTO
+ SYMBOL_RATE = 6940000
+ INNER_FEC = AUTO
+ DELIVERY_SYSTEM = DVBC/ANNEX_A
+
+.. note::
+ The parameters depend on the video standard you're testing.
+
+.. note::
+ Vidtv is a fake driver and does not validate much of the information
+ in the scan file. Just specifying 'FREQUENCY' and 'DELIVERY_SYSTEM'
+ should be enough for DVB-T/DVB-T2. For DVB-S/DVB-C however, you
+ should also provide 'SYMBOL_RATE'.
+
+You can browse scan tables online here: `dvb-scan-tables
+<https://git.linuxtv.org/dtv-scan-tables.git>`_.
+
+Assuming this channel is named 'channel.conf', you can then run::
+
+ $ dvbv5-scan channel.conf
+
+For more information on dvb-scan, check its documentation online here:
+`dvb-scan Documentation <https://www.linuxtv.org/wiki/index.php/Dvbscan>`_.
+
+Using dvb-zap
+~~~~~~~~~~~~~
+
+dvbv5-zap is a command line tool that can be used to record MPEG-TS to disk. The
+typical use is to tune into a channel and put it into record mode. The example
+below - which is taken from the documentation - illustrates that::
+
+ $ dvbv5-zap -c dvb_channel.conf "trilhas sonoras" -r
+ using demux '/dev/dvb/adapter0/demux0'
+ reading channels from file 'dvb_channel.conf'
+ service has pid type 05: 204
+ tuning to 573000000 Hz
+ audio pid 104
+ dvb_set_pesfilter 104
+ Lock (0x1f) Quality= Good Signal= 100.00% C/N= -13.80dB UCB= 70 postBER= 3.14x10^-3 PER= 0
+ DVR interface '/dev/dvb/adapter0/dvr0' can now be opened
+
+The channel can be watched by playing the contents of the DVR interface, with
+some player that recognizes the MPEG-TS format, such as *mplayer* or *vlc*.
+
+By playing the contents of the stream one can visually inspect the workings of
+vidtv, e.g.::
+
+ $ mplayer /dev/dvb/adapter0/dvr0
+
+For more information on dvb-zap check its online documentation here:
+`dvb-zap Documentation
+<https://www.linuxtv.org/wiki/index.php/Dvbv5-zap>`_.
+See also: `zap <https://www.linuxtv.org/wiki/index.php/Zap>`_.
+
+
+What can still be improved in vidtv
+-----------------------------------
+
+Add *debugfs* integration
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Although frontend drivers provide DVBv5 statistics via the .read_status
+call, a nice addition would be to make additional statistics available to
+userspace via debugfs, which is a simple-to-use, RAM-based filesystem
+specifically designed for debug purposes.
+
+The logic for this would be implemented on a separate file so as not to
+pollute the frontend driver. These statistics are driver-specific and can
+be useful during tests.
+
+The Siano driver is one example of a driver using
+debugfs to convey driver-specific statistics to userspace and it can be
+used as a reference.
+
+This should be further enabled and disabled via a Kconfig
+option for convenience.
+
+Add a way to test video
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently, vidtv can only encode PCM audio. It would be great to implement
+a barebones version of MPEG-2 video encoding so we can also test video. The
+first place to look into is *ISO 13818-2: Information technology — Generic
+coding of moving pictures and associated audio information — Part 2: Video*,
+which covers the encoding of compressed video in MPEG Transport Streams.
+
+This might optionally use the Video4Linux2 Test Pattern Generator, v4l2-tpg,
+which resides at::
+
+ drivers/media/common/v4l2-tpg/
+
+
+Add white noise simulation
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The vidtv tuner already has code to identify whether the chosen frequency
+is too far away from a table of valid frequencies. For now, this means that
+the demodulator can eventually lose the lock on the signal, since the tuner will
+report a bad signal quality.
+
+A nice addition is to simulate some noise when the signal quality is bad by:
+
+- Randomly dropping some TS packets. This will trigger a continuity error if the
+ continuity counter is updated but the packet is not passed on to the demux.
+
+- Updating the error statistics accordingly (e.g. BER, etc).
+
+- Simulating some noise in the encoded data.
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+The Zoran driver
+================
+
+unified zoran driver (zr360x7, zoran, buz, dc10(+), dc30(+), lml33)
+
+website: http://mjpeg.sourceforge.net/driver-zoran/
+
+
+Frequently Asked Questions
+--------------------------
+
+What cards are supported
+------------------------
+
+Iomega Buz, Linux Media Labs LML33/LML33R10, Pinnacle/Miro
+DC10/DC10+/DC30/DC30+ and related boards (available under various names).
+
+Iomega Buz
+~~~~~~~~~~
+
+* Zoran zr36067 PCI controller
+* Zoran zr36060 MJPEG codec
+* Philips saa7111 TV decoder
+* Philips saa7185 TV encoder
+
+Drivers to use: videodev, i2c-core, i2c-algo-bit,
+videocodec, saa7111, saa7185, zr36060, zr36067
+
+Inputs/outputs: Composite and S-video
+
+Norms: PAL, SECAM (720x576 @ 25 fps), NTSC (720x480 @ 29.97 fps)
+
+Card number: 7
+
+AverMedia 6 Eyes AVS6EYES
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Zoran zr36067 PCI controller
+* Zoran zr36060 MJPEG codec
+* Samsung ks0127 TV decoder
+* Conexant bt866 TV encoder
+
+Drivers to use: videodev, i2c-core, i2c-algo-bit,
+videocodec, ks0127, bt866, zr36060, zr36067
+
+Inputs/outputs:
+ Six physical inputs. 1-6 are composite,
+ 1-2, 3-4, 5-6 doubles as S-video,
+ 1-3 triples as component.
+ One composite output.
+
+Norms: PAL, SECAM (720x576 @ 25 fps), NTSC (720x480 @ 29.97 fps)
+
+Card number: 8
+
+.. note::
+
+ Not autodetected, card=8 is necessary.
+
+Linux Media Labs LML33
+~~~~~~~~~~~~~~~~~~~~~~
+
+* Zoran zr36067 PCI controller
+* Zoran zr36060 MJPEG codec
+* Brooktree bt819 TV decoder
+* Brooktree bt856 TV encoder
+
+Drivers to use: videodev, i2c-core, i2c-algo-bit,
+videocodec, bt819, bt856, zr36060, zr36067
+
+Inputs/outputs: Composite and S-video
+
+Norms: PAL (720x576 @ 25 fps), NTSC (720x480 @ 29.97 fps)
+
+Card number: 5
+
+Linux Media Labs LML33R10
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Zoran zr36067 PCI controller
+* Zoran zr36060 MJPEG codec
+* Philips saa7114 TV decoder
+* Analog Devices adv7170 TV encoder
+
+Drivers to use: videodev, i2c-core, i2c-algo-bit,
+videocodec, saa7114, adv7170, zr36060, zr36067
+
+Inputs/outputs: Composite and S-video
+
+Norms: PAL (720x576 @ 25 fps), NTSC (720x480 @ 29.97 fps)
+
+Card number: 6
+
+Pinnacle/Miro DC10(new)
+~~~~~~~~~~~~~~~~~~~~~~~
+
+* Zoran zr36057 PCI controller
+* Zoran zr36060 MJPEG codec
+* Philips saa7110a TV decoder
+* Analog Devices adv7176 TV encoder
+
+Drivers to use: videodev, i2c-core, i2c-algo-bit,
+videocodec, saa7110, adv7175, zr36060, zr36067
+
+Inputs/outputs: Composite, S-video and Internal
+
+Norms: PAL, SECAM (768x576 @ 25 fps), NTSC (640x480 @ 29.97 fps)
+
+Card number: 1
+
+Pinnacle/Miro DC10+
+~~~~~~~~~~~~~~~~~~~
+
+* Zoran zr36067 PCI controller
+* Zoran zr36060 MJPEG codec
+* Philips saa7110a TV decoder
+* Analog Devices adv7176 TV encoder
+
+Drivers to use: videodev, i2c-core, i2c-algo-bit,
+videocodec, saa7110, adv7175, zr36060, zr36067
+
+Inputs/outputs: Composite, S-video and Internal
+
+Norms: PAL, SECAM (768x576 @ 25 fps), NTSC (640x480 @ 29.97 fps)
+
+Card number: 2
+
+Pinnacle/Miro DC10(old)
+~~~~~~~~~~~~~~~~~~~~~~~
+
+* Zoran zr36057 PCI controller
+* Zoran zr36050 MJPEG codec
+* Zoran zr36016 Video Front End or Fuji md0211 Video Front End (clone?)
+* Micronas vpx3220a TV decoder
+* mse3000 TV encoder or Analog Devices adv7176 TV encoder
+
+Drivers to use: videodev, i2c-core, i2c-algo-bit,
+videocodec, vpx3220, mse3000/adv7175, zr36050, zr36016, zr36067
+
+Inputs/outputs: Composite, S-video and Internal
+
+Norms: PAL, SECAM (768x576 @ 25 fps), NTSC (640x480 @ 29.97 fps)
+
+Card number: 0
+
+Pinnacle/Miro DC30
+~~~~~~~~~~~~~~~~~~
+
+* Zoran zr36057 PCI controller
+* Zoran zr36050 MJPEG codec
+* Zoran zr36016 Video Front End
+* Micronas vpx3225d/vpx3220a/vpx3216b TV decoder
+* Analog Devices adv7176 TV encoder
+
+Drivers to use: videodev, i2c-core, i2c-algo-bit,
+videocodec, vpx3220/vpx3224, adv7175, zr36050, zr36016, zr36067
+
+Inputs/outputs: Composite, S-video and Internal
+
+Norms: PAL, SECAM (768x576 @ 25 fps), NTSC (640x480 @ 29.97 fps)
+
+Card number: 3
+
+Pinnacle/Miro DC30+
+~~~~~~~~~~~~~~~~~~~
+
+* Zoran zr36067 PCI controller
+* Zoran zr36050 MJPEG codec
+* Zoran zr36016 Video Front End
+* Micronas vpx3225d/vpx3220a/vpx3216b TV decoder
+* Analog Devices adv7176 TV encoder
+
+Drivers to use: videodev, i2c-core, i2c-algo-bit,
+videocodec, vpx3220/vpx3224, adv7175, zr36050, zr36015, zr36067
+
+Inputs/outputs: Composite, S-video and Internal
+
+Norms: PAL, SECAM (768x576 @ 25 fps), NTSC (640x480 @ 29.97 fps)
+
+Card number: 4
+
+.. note::
+
+ #) No module for the mse3000 is available yet
+ #) No module for the vpx3224 is available yet
+
+1.1 What the TV decoder can do an what not
+------------------------------------------
+
+The best know TV standards are NTSC/PAL/SECAM. but for decoding a frame that
+information is not enough. There are several formats of the TV standards.
+And not every TV decoder is able to handle every format. Also the every
+combination is supported by the driver. There are currently 11 different
+tv broadcast formats all aver the world.
+
+The CCIR defines parameters needed for broadcasting the signal.
+The CCIR has defined different standards: A,B,D,E,F,G,D,H,I,K,K1,L,M,N,...
+The CCIR says not much about the colorsystem used !!!
+And talking about a colorsystem says not to much about how it is broadcast.
+
+The CCIR standards A,E,F are not used any more.
+
+When you speak about NTSC, you usually mean the standard: CCIR - M using
+the NTSC colorsystem which is used in the USA, Japan, Mexico, Canada
+and a few others.
+
+When you talk about PAL, you usually mean: CCIR - B/G using the PAL
+colorsystem which is used in many Countries.
+
+When you talk about SECAM, you mean: CCIR - L using the SECAM Colorsystem
+which is used in France, and a few others.
+
+There the other version of SECAM, CCIR - D/K is used in Bulgaria, China,
+Slovakai, Hungary, Korea (Rep.), Poland, Rumania and a others.
+
+The CCIR - H uses the PAL colorsystem (sometimes SECAM) and is used in
+Egypt, Libya, Sri Lanka, Syrain Arab. Rep.
+
+The CCIR - I uses the PAL colorsystem, and is used in Great Britain, Hong Kong,
+Ireland, Nigeria, South Africa.
+
+The CCIR - N uses the PAL colorsystem and PAL frame size but the NTSC framerate,
+and is used in Argentinia, Uruguay, an a few others
+
+We do not talk about how the audio is broadcast !
+
+A rather good sites about the TV standards are:
+http://www.sony.jp/support/
+http://info.electronicwerkstatt.de/bereiche/fernsehtechnik/frequenzen_und_normen/Fernsehnormen/
+and http://www.cabl.com/restaurant/channel.html
+
+Other weird things around: NTSC 4.43 is a modificated NTSC, which is mainly
+used in PAL VCR's that are able to play back NTSC. PAL 60 seems to be the same
+as NTSC 4.43 . The Datasheets also talk about NTSC 44, It seems as if it would
+be the same as NTSC 4.43.
+NTSC Combs seems to be a decoder mode where the decoder uses a comb filter
+to split coma and luma instead of a Delay line.
+
+But I did not defiantly find out what NTSC Comb is.
+
+Philips saa7111 TV decoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- was introduced in 1997, is used in the BUZ and
+- can handle: PAL B/G/H/I, PAL N, PAL M, NTSC M, NTSC N, NTSC 4.43 and SECAM
+
+Philips saa7110a TV decoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- was introduced in 1995, is used in the Pinnacle/Miro DC10(new), DC10+ and
+- can handle: PAL B/G, NTSC M and SECAM
+
+Philips saa7114 TV decoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- was introduced in 2000, is used in the LML33R10 and
+- can handle: PAL B/G/D/H/I/N, PAL N, PAL M, NTSC M, NTSC 4.43 and SECAM
+
+Brooktree bt819 TV decoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- was introduced in 1996, and is used in the LML33 and
+- can handle: PAL B/D/G/H/I, NTSC M
+
+Micronas vpx3220a TV decoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- was introduced in 1996, is used in the DC30 and DC30+ and
+- can handle: PAL B/G/H/I, PAL N, PAL M, NTSC M, NTSC 44, PAL 60, SECAM,NTSC Comb
+
+Samsung ks0127 TV decoder
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- is used in the AVS6EYES card and
+- can handle: NTSC-M/N/44, PAL-M/N/B/G/H/I/D/K/L and SECAM
+
+
+What the TV encoder can do an what not
+--------------------------------------
+
+The TV encoder is doing the "same" as the decoder, but in the other direction.
+You feed them digital data and the generate a Composite or SVHS signal.
+For information about the colorsystems and TV norm take a look in the
+TV decoder section.
+
+Philips saa7185 TV Encoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- was introduced in 1996, is used in the BUZ
+- can generate: PAL B/G, NTSC M
+
+Brooktree bt856 TV Encoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- was introduced in 1994, is used in the LML33
+- can generate: PAL B/D/G/H/I/N, PAL M, NTSC M, PAL-N (Argentina)
+
+Analog Devices adv7170 TV Encoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- was introduced in 2000, is used in the LML300R10
+- can generate: PAL B/D/G/H/I/N, PAL M, NTSC M, PAL 60
+
+Analog Devices adv7175 TV Encoder
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- was introduced in 1996, is used in the DC10, DC10+, DC10 old, DC30, DC30+
+- can generate: PAL B/D/G/H/I/N, PAL M, NTSC M
+
+ITT mse3000 TV encoder
+~~~~~~~~~~~~~~~~~~~~~~
+
+- was introduced in 1991, is used in the DC10 old
+- can generate: PAL , NTSC , SECAM
+
+Conexant bt866 TV encoder
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- is used in AVS6EYES, and
+- can generate: NTSC/PAL, PALM, PALN
+
+The adv717x, should be able to produce PAL N. But you find nothing PAL N
+specific in the registers. Seem that you have to reuse a other standard
+to generate PAL N, maybe it would work if you use the PAL M settings.
+
+How do I get this damn thing to work
+------------------------------------
+
+Load zr36067.o. If it can't autodetect your card, use the card=X insmod
+option with X being the card number as given in the previous section.
+To have more than one card, use card=X1[,X2[,X3,[X4[..]]]]
+
+To automate this, add the following to your /etc/modprobe.d/zoran.conf:
+
+options zr36067 card=X1[,X2[,X3[,X4[..]]]]
+alias char-major-81-0 zr36067
+
+One thing to keep in mind is that this doesn't load zr36067.o itself yet. It
+just automates loading. If you start using xawtv, the device won't load on
+some systems, since you're trying to load modules as a user, which is not
+allowed ("permission denied"). A quick workaround is to add 'Load "v4l"' to
+XF86Config-4 when you use X by default, or to run 'v4l-conf -c <device>' in
+one of your startup scripts (normally rc.local) if you don't use X. Both
+make sure that the modules are loaded on startup, under the root account.
+
+What mainboard should I use (or why doesn't my card work)
+---------------------------------------------------------
+
+
+<insert lousy disclaimer here>. In short: good=SiS/Intel, bad=VIA.
+
+Experience tells us that people with a Buz, on average, have more problems
+than users with a DC10+/LML33. Also, it tells us that people owning a VIA-
+based mainboard (ktXXX, MVP3) have more problems than users with a mainboard
+based on a different chipset. Here's some notes from Andrew Stevens:
+
+Here's my experience of using LML33 and Buz on various motherboards:
+
+- VIA MVP3
+ - Forget it. Pointless. Doesn't work.
+- Intel 430FX (Pentium 200)
+ - LML33 perfect, Buz tolerable (3 or 4 frames dropped per movie)
+- Intel 440BX (early stepping)
+ - LML33 tolerable. Buz starting to get annoying (6-10 frames/hour)
+- Intel 440BX (late stepping)
+ - Buz tolerable, LML3 almost perfect (occasional single frame drops)
+- SiS735
+ - LML33 perfect, Buz tolerable.
+- VIA KT133(*)
+ - LML33 starting to get annoying, Buz poor enough that I have up.
+
+- Both 440BX boards were dual CPU versions.
+
+Bernhard Praschinger later added:
+
+- AMD 751
+ - Buz perfect-tolerable
+- AMD 760
+ - Buz perfect-tolerable
+
+In general, people on the user mailinglist won't give you much of a chance
+if you have a VIA-based motherboard. They may be cheap, but sometimes, you'd
+rather want to spend some more money on better boards. In general, VIA
+mainboard's IDE/PCI performance will also suck badly compared to others.
+You'll noticed the DC10+/DC30+ aren't mentioned anywhere in the overview.
+Basically, you can assume that if the Buz works, the LML33 will work too. If
+the LML33 works, the DC10+/DC30+ will work too. They're most tolerant to
+different mainboard chipsets from all of the supported cards.
+
+If you experience timeouts during capture, buy a better mainboard or lower
+the quality/buffersize during capture (see 'Concerning buffer sizes, quality,
+output size etc.'). If it hangs, there's little we can do as of now. Check
+your IRQs and make sure the card has its own interrupts.
+
+Programming interface
+---------------------
+
+This driver conforms to video4linux2. Support for V4L1 and for the custom
+zoran ioctls has been removed in kernel 2.6.38.
+
+For programming example, please, look at lavrec.c and lavplay.c code in
+the MJPEG-tools (http://mjpeg.sf.net/).
+
+Additional notes for software developers:
+
+ The driver returns maxwidth and maxheight parameters according to
+ the current TV standard (norm). Therefore, the software which
+ communicates with the driver and "asks" for these parameters should
+ first set the correct norm. Well, it seems logically correct: TV
+ standard is "more constant" for current country than geometry
+ settings of a variety of TV capture cards which may work in ITU or
+ square pixel format.
+
+Applications
+------------
+
+Applications known to work with this driver:
+
+TV viewing:
+
+* xawtv
+* kwintv
+* probably any TV application that supports video4linux or video4linux2.
+
+MJPEG capture/playback:
+
+* mjpegtools/lavtools (or Linux Video Studio)
+* gstreamer
+* mplayer
+
+General raw capture:
+
+* xawtv
+* gstreamer
+* probably any application that supports video4linux or video4linux2
+
+Video editing:
+
+* Cinelerra
+* MainActor
+* mjpegtools (or Linux Video Studio)
+
+
+Concerning buffer sizes, quality, output size etc.
+--------------------------------------------------
+
+
+The zr36060 can do 1:2 JPEG compression. This is really the theoretical
+maximum that the chipset can reach. The driver can, however, limit compression
+to a maximum (size) of 1:4. The reason for this is that some cards (e.g. Buz)
+can't handle 1:2 compression without stopping capture after only a few minutes.
+With 1:4, it'll mostly work. If you have a Buz, use 'low_bitrate=1' to go into
+1:4 max. compression mode.
+
+100% JPEG quality is thus 1:2 compression in practice. So for a full PAL frame
+(size 720x576). The JPEG fields are stored in YUY2 format, so the size of the
+fields are 720x288x16/2 bits/field (2 fields/frame) = 207360 bytes/field x 2 =
+414720 bytes/frame (add some more bytes for headers and DHT (huffman)/DQT
+(quantization) tables, and you'll get to something like 512kB per frame for
+1:2 compression. For 1:4 compression, you'd have frames of half this size.
+
+Some additional explanation by Martin Samuelsson, which also explains the
+importance of buffer sizes:
+--
+> Hmm, I do not think it is really that way. With the current (downloaded
+> at 18:00 Monday) driver I get that output sizes for 10 sec:
+> -q 50 -b 128 : 24.283.332 Bytes
+> -q 50 -b 256 : 48.442.368
+> -q 25 -b 128 : 24.655.992
+> -q 25 -b 256 : 25.859.820
+
+I woke up, and can't go to sleep again. I'll kill some time explaining why
+this doesn't look strange to me.
+
+Let's do some math using a width of 704 pixels. I'm not sure whether the Buz
+actually use that number or not, but that's not too important right now.
+
+704x288 pixels, one field, is 202752 pixels. Divided by 64 pixels per block;
+3168 blocks per field. Each pixel consist of two bytes; 128 bytes per block;
+1024 bits per block. 100% in the new driver mean 1:2 compression; the maximum
+output becomes 512 bits per block. Actually 510, but 512 is simpler to use
+for calculations.
+
+Let's say that we specify d1q50. We thus want 256 bits per block; times 3168
+becomes 811008 bits; 101376 bytes per field. We're talking raw bits and bytes
+here, so we don't need to do any fancy corrections for bits-per-pixel or such
+things. 101376 bytes per field.
+
+d1 video contains two fields per frame. Those sum up to 202752 bytes per
+frame, and one of those frames goes into each buffer.
+
+But wait a second! -b128 gives 128kB buffers! It's not possible to cram
+202752 bytes of JPEG data into 128kB!
+
+This is what the driver notice and automatically compensate for in your
+examples. Let's do some math using this information:
+
+128kB is 131072 bytes. In this buffer, we want to store two fields, which
+leaves 65536 bytes for each field. Using 3168 blocks per field, we get
+20.68686868... available bytes per block; 165 bits. We can't allow the
+request for 256 bits per block when there's only 165 bits available! The -q50
+option is silently overridden, and the -b128 option takes precedence, leaving
+us with the equivalence of -q32.
+
+This gives us a data rate of 165 bits per block, which, times 3168, sums up
+to 65340 bytes per field, out of the allowed 65536. The current driver has
+another level of rate limiting; it won't accept -q values that fill more than
+6/8 of the specified buffers. (I'm not sure why. "Playing it safe" seem to be
+a safe bet. Personally, I think I would have lowered requested-bits-per-block
+by one, or something like that.) We can't use 165 bits per block, but have to
+lower it again, to 6/8 of the available buffer space: We end up with 124 bits
+per block, the equivalence of -q24. With 128kB buffers, you can't use greater
+than -q24 at -d1. (And PAL, and 704 pixels width...)
+
+The third example is limited to -q24 through the same process. The second
+example, using very similar calculations, is limited to -q48. The only
+example that actually grab at the specified -q value is the last one, which
+is clearly visible, looking at the file size.
+--
+
+Conclusion: the quality of the resulting movie depends on buffer size, quality,
+whether or not you use 'low_bitrate=1' as insmod option for the zr36060.c
+module to do 1:4 instead of 1:2 compression, etc.
+
+If you experience timeouts, lowering the quality/buffersize or using
+'low_bitrate=1 as insmod option for zr36060.o might actually help, as is
+proven by the Buz.
+
+It hangs/crashes/fails/whatevers! Help!
+---------------------------------------
+
+Make sure that the card has its own interrupts (see /proc/interrupts), check
+the output of dmesg at high verbosity (load zr36067.o with debug=2,
+load all other modules with debug=1). Check that your mainboard is favorable
+(see question 2) and if not, test the card in another computer. Also see the
+notes given in question 3 and try lowering quality/buffersize/capturesize
+if recording fails after a period of time.
+
+If all this doesn't help, give a clear description of the problem including
+detailed hardware information (memory+brand, mainboard+chipset+brand, which
+MJPEG card, processor, other PCI cards that might be of interest), give the
+system PnP information (/proc/interrupts, /proc/dma, /proc/devices), and give
+the kernel version, driver version, glibc version, gcc version and any other
+information that might possibly be of interest. Also provide the dmesg output
+at high verbosity. See 'Contacting' on how to contact the developers.
+
+Maintainers/Contacting
+----------------------
+
+Previous maintainers/developers of this driver are
+- Laurent Pinchart <laurent.pinchart@skynet.be>
+- Ronald Bultje rbultje@ronald.bitfreak.net
+- Serguei Miridonov <mirsev@cicese.mx>
+- Wolfgang Scherr <scherr@net4you.net>
+- Dave Perks <dperks@ibm.net>
+- Rainer Johanni <Rainer@Johanni.de>
+
+Driver's License
+----------------
+
+ This driver is distributed under the terms of the General Public License.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+See http://www.gnu.org/ for more information.
mc-core
cec-core
csi2
+ camera-sensor
drivers/index
From the bridge driver perspective, you load the sub-device module and somehow
obtain the :c:type:`v4l2_subdev` pointer. For i2c devices this is easy: you call
``i2c_get_clientdata()``. For other buses something similar needs to be done.
-Helper functions exists for sub-devices on an I2C bus that do most of this
+Helper functions exist for sub-devices on an I2C bus that do most of this
tricky work for you.
Each :c:type:`v4l2_subdev` contains function pointers that sub-device drivers
and sink of the link. Subdev drivers are also free to use this function to
perform the checks mentioned above in addition to their own checks.
+Subdev registration
+~~~~~~~~~~~~~~~~~~~
+
There are currently two ways to register subdevices with the V4L2 core. The
first (traditional) possibility is to have subdevices registered by bridge
drivers. This can be done when the bridge driver has the complete information
Using one or the other registration method only affects the probing process, the
run-time bridge-subdevice interaction is in both cases the same.
-In the synchronous case a device (bridge) driver needs to register the
+In the **synchronous** case a device (bridge) driver needs to register the
:c:type:`v4l2_subdev` with the v4l2_device:
:c:func:`v4l2_device_register_subdev <v4l2_device_register_subdev>`
Afterwards the subdev module can be unloaded and
:c:type:`sd <v4l2_subdev>`->dev == ``NULL``.
-You can call an ops function either directly:
+In the **asynchronous** case subdevice probing can be invoked independently of
+the bridge driver availability. The subdevice driver then has to verify whether
+all the requirements for a successful probing are satisfied. This can include a
+check for a master clock availability. If any of the conditions aren't satisfied
+the driver might decide to return ``-EPROBE_DEFER`` to request further reprobing
+attempts. Once all conditions are met the subdevice shall be registered using
+the :c:func:`v4l2_async_register_subdev` function. Unregistration is
+performed using the :c:func:`v4l2_async_unregister_subdev` call. Subdevices
+registered this way are stored in a global list of subdevices, ready to be
+picked up by bridge drivers.
+
+Bridge drivers in turn have to register a notifier object. This is
+performed using the :c:func:`v4l2_async_notifier_register` call. To
+unregister the notifier the driver has to call
+:c:func:`v4l2_async_notifier_unregister`. The former of the two functions
+takes two arguments: a pointer to struct :c:type:`v4l2_device` and a
+pointer to struct :c:type:`v4l2_async_notifier`.
+
+Before registering the notifier, bridge drivers must do two things:
+first, the notifier must be initialized using the
+:c:func:`v4l2_async_notifier_init`. Second, bridge drivers can then
+begin to form a list of subdevice descriptors that the bridge device
+needs for its operation. Subdevice descriptors are added to the notifier
+using the :c:func:`v4l2_async_notifier_add_subdev` call. This function
+takes two arguments: a pointer to struct :c:type:`v4l2_async_notifier`,
+and a pointer to the subdevice descripter, which is of type struct
+:c:type:`v4l2_async_subdev`.
+
+The V4L2 core will then use these descriptors to match asynchronously
+registered subdevices to them. If a match is detected the ``.bound()``
+notifier callback is called. After all subdevices have been located the
+.complete() callback is called. When a subdevice is removed from the
+system the .unbind() method is called. All three callbacks are optional.
+
+Calling subdev operations
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The advantage of using :c:type:`v4l2_subdev` is that it is a generic struct and
+does not contain any knowledge about the underlying hardware. So a driver might
+contain several subdevs that use an I2C bus, but also a subdev that is
+controlled through GPIO pins. This distinction is only relevant when setting
+up the device, but once the subdev is registered it is completely transparent.
+
+Once te subdev has been registered you can call an ops function either
+directly:
.. code-block:: c
err = v4l2_subdev_call(sd, core, g_std, &norm);
-The macro will to the right ``NULL`` pointer checks and returns ``-ENODEV``
+The macro will do the right ``NULL`` pointer checks and returns ``-ENODEV``
if :c:type:`sd <v4l2_subdev>` is ``NULL``, ``-ENOIOCTLCMD`` if either
:c:type:`sd <v4l2_subdev>`->core or :c:type:`sd <v4l2_subdev>`->core->g_std is ``NULL``, or the actual result of the
:c:type:`sd <v4l2_subdev>`->ops->core->g_std ops.
whether there is a ``notify()`` callback defined and returns ``-ENODEV`` if not.
Otherwise the result of the ``notify()`` call is returned.
-The advantage of using :c:type:`v4l2_subdev` is that it is a generic struct and
-does not contain any knowledge about the underlying hardware. So a driver might
-contain several subdevs that use an I2C bus, but also a subdev that is
-controlled through GPIO pins. This distinction is only relevant when setting
-up the device, but once the subdev is registered it is completely transparent.
-
-In the asynchronous case subdevice probing can be invoked independently of the
-bridge driver availability. The subdevice driver then has to verify whether all
-the requirements for a successful probing are satisfied. This can include a
-check for a master clock availability. If any of the conditions aren't satisfied
-the driver might decide to return ``-EPROBE_DEFER`` to request further reprobing
-attempts. Once all conditions are met the subdevice shall be registered using
-the :c:func:`v4l2_async_register_subdev` function. Unregistration is
-performed using the :c:func:`v4l2_async_unregister_subdev` call. Subdevices
-registered this way are stored in a global list of subdevices, ready to be
-picked up by bridge drivers.
-
-Bridge drivers in turn have to register a notifier object. This is
-performed using the :c:func:`v4l2_async_notifier_register` call. To
-unregister the notifier the driver has to call
-:c:func:`v4l2_async_notifier_unregister`. The former of the two functions
-takes two arguments: a pointer to struct :c:type:`v4l2_device` and a
-pointer to struct :c:type:`v4l2_async_notifier`.
-
-Before registering the notifier, bridge drivers must do two things:
-first, the notifier must be initialized using the
-:c:func:`v4l2_async_notifier_init`. Second, bridge drivers can then
-begin to form a list of subdevice descriptors that the bridge device
-needs for its operation. Subdevice descriptors are added to the notifier
-using the :c:func:`v4l2_async_notifier_add_subdev` call. This function
-takes two arguments: a pointer to struct :c:type:`v4l2_async_notifier`,
-and a pointer to the subdevice descripter, which is of type struct
-:c:type:`v4l2_async_subdev`.
-
-The V4L2 core will then use these descriptors to match asynchronously
-registered subdevices to them. If a match is detected the ``.bound()``
-notifier callback is called. After all subdevices have been located the
-.complete() callback is called. When a subdevice is removed from the
-system the .unbind() method is called. All three callbacks are optional.
-
V4L2 sub-device userspace API
-----------------------------
---------------------------------------------
.. kernel-doc:: include/media/v4l2-subdev.h
-
-.. kernel-doc:: include/media/v4l2-async.h
nvdimm
btt
security
+ firmware-activate
the stream pointer for all DAIS connected to a stream and releases the
memory allocated for the stream.
- Not Supported
+Not Supported
=============
1. A single port with multiple channels supported cannot be used between two
-streams or across stream. For example a port with 4 channels cannot be used
-to handle 2 independent stereo streams even though it's possible in theory
-in SoundWire.
+ streams or across stream. For example a port with 4 channels cannot be used
+ to handle 2 independent stereo streams even though it's possible in theory
+ in SoundWire.
Note, not all drivers can handle font with widths not divisible by 8,
such as vga16fb.
-2. fbcon=scrollback:<value>[k]
- The scrollback buffer is memory that is used to preserve display
- contents that has already scrolled past your view. This is accessed
- by using the Shift-PageUp key combination. The value 'value' is any
- integer. It defaults to 32KB. The 'k' suffix is optional, and will
- multiply the 'value' by 1024.
-
-3. fbcon=map:<0123>
+2. fbcon=map:<0123>
This is an interesting option. It tells which driver gets mapped to
which console. The value '0123' is a sequence that gets repeated until
Later on, when you want to map the console the to the framebuffer
device, you can use the con2fbmap utility.
-4. fbcon=vc:<n1>-<n2>
+3. fbcon=vc:<n1>-<n2>
This option tells fbcon to take over only a range of consoles as
specified by the values 'n1' and 'n2'. The rest of the consoles
is typically located on the same video card. Thus, the consoles that
are controlled by the VGA console will be garbled.
-5. fbcon=rotate:<n>
+4. fbcon=rotate:<n>
This option changes the orientation angle of the console display. The
value 'n' accepts the following:
Actually, the underlying fb driver is totally ignorant of console
rotation.
-6. fbcon=margin:<color>
+5. fbcon=margin:<color>
This option specifies the color of the margins. The margins are the
leftover area at the right and the bottom of the screen that are not
used by text. By default, this area will be black. The 'color' value
is an integer number that depends on the framebuffer driver being used.
-7. fbcon=nodefer
+6. fbcon=nodefer
If the kernel is compiled with deferred fbcon takeover support, normally
the framebuffer contents, left in place by the firmware/bootloader, will
be preserved until there actually is some text is output to the console.
This option causes fbcon to bind immediately to the fbdev device.
-8. fbcon=logo-pos:<location>
+7. fbcon=logo-pos:<location>
The only possible 'location' is 'center' (without quotes), and when
given, the bootup logo is moved from the default top-left corner
displayed due to multiple CPUs, the collected line of logos is moved
as a whole.
-9. fbcon=logo-count:<n>
+8. fbcon=logo-count:<n>
The value 'n' overrides the number of bootup logos. 0 disables the
logo, and -1 gives the default which is the number of online CPUs.
- interlaced text mode is not supported; it looks like hardware limitation,
but I'm not sure.
- Gxx0 SGRAM/SDRAM is not autodetected.
- - If you are using more than one framebuffer device, you must boot kernel
- with 'video=scrollback:0'.
- maybe more...
And following misfeatures:
contact me.
- The 24/32 is not likely to work anytime soon, knowing that the
hardware does ... unusual things in 24/32 bpp.
-- When used with another video board, current limitations of the linux
- console subsystem can cause some troubles, specifically, you should
- disable software scrollback, as it can oops badly ...
Todo
====
* scrolling (fullscreen) is fast, because there is
no need to copy around data.
- * You'll get scrollback (the Shift-PgUp thing),
- the video memory can be used as scrollback buffer
kontra:
quota
seq_file
sharedsubtree
- sysfs-pci
- sysfs-tagging
automount-support
.. SPDX-License-Identifier: GPL-2.0
====================
-fILESYSTEM Mount API
+Filesystem Mount API
====================
.. CONTENTS
int vfs_parse_fs_param(struct fs_context *fc,
struct fs_parameter *param);
- Supply a single mount parameter to the filesystem context. This include
+ Supply a single mount parameter to the filesystem context. This includes
the specification of the source/device which is specified as the "source"
parameter (which may be specified multiple times if the filesystem
supports that).
one.
-=====================
-PARAMETER DESCRIPTION
+Parameter Description
=====================
Parameters are described using structures defined in linux/fs_parser.h.
called SEQ_START_TOKEN; it can be used if you wish to instruct your
show() function (described below) to print a header at the top of the
output. SEQ_START_TOKEN should only be used if the offset is zero,
-however.
+however. SEQ_START_TOKEN has no special meaning to the core seq_file
+code. It is provided as a convenience for a start() funciton to
+communicate with the next() and show() functions.
The next function to implement is called, amazingly, next(); its job is to
move the iterator forward to the next position in the sequence. The
return spos;
}
+The next() function should set ``*pos`` to a value that start() can use
+to find the new location in the sequence. When the iterator is being
+stored in the private data area, rather than being reinitialized on each
+start(), it might seem sufficient to simply set ``*pos`` to any non-zero
+value (zero always tells start() to restart the sequence). This is not
+sufficient due to historical problems.
+
+Historically, many next() functions have *not* updated ``*pos`` at
+end-of-file. If the value is then used by start() to initialise the
+iterator, this can result in corner cases where the last entry in the
+sequence is reported twice in the file. In order to discourage this bug
+from being resurrected, the core seq_file code now produces a warning if
+a next() function does not change the value of ``*pos``. Consequently a
+next() function *must* change the value of ``*pos``, and of course must
+set it to a non-zero value.
+
The stop() function closes a session; its job, of course, is to clean
up. If dynamic memory is allocated for the iterator, stop() is the
place to free it; if a lock was taken by start(), stop() must release
+++ /dev/null
-.. SPDX-License-Identifier: GPL-2.0
-
-============================================
-Accessing PCI device resources through sysfs
-============================================
-
-sysfs, usually mounted at /sys, provides access to PCI resources on platforms
-that support it. For example, a given bus might look like this::
-
- /sys/devices/pci0000:17
- |-- 0000:17:00.0
- | |-- class
- | |-- config
- | |-- device
- | |-- enable
- | |-- irq
- | |-- local_cpus
- | |-- remove
- | |-- resource
- | |-- resource0
- | |-- resource1
- | |-- resource2
- | |-- revision
- | |-- rom
- | |-- subsystem_device
- | |-- subsystem_vendor
- | `-- vendor
- `-- ...
-
-The topmost element describes the PCI domain and bus number. In this case,
-the domain number is 0000 and the bus number is 17 (both values are in hex).
-This bus contains a single function device in slot 0. The domain and bus
-numbers are reproduced for convenience. Under the device directory are several
-files, each with their own function.
-
- =================== =====================================================
- file function
- =================== =====================================================
- class PCI class (ascii, ro)
- config PCI config space (binary, rw)
- device PCI device (ascii, ro)
- enable Whether the device is enabled (ascii, rw)
- irq IRQ number (ascii, ro)
- local_cpus nearby CPU mask (cpumask, ro)
- remove remove device from kernel's list (ascii, wo)
- resource PCI resource host addresses (ascii, ro)
- resource0..N PCI resource N, if present (binary, mmap, rw\ [1]_)
- resource0_wc..N_wc PCI WC map resource N, if prefetchable (binary, mmap)
- revision PCI revision (ascii, ro)
- rom PCI ROM resource, if present (binary, ro)
- subsystem_device PCI subsystem device (ascii, ro)
- subsystem_vendor PCI subsystem vendor (ascii, ro)
- vendor PCI vendor (ascii, ro)
- =================== =====================================================
-
-::
-
- ro - read only file
- rw - file is readable and writable
- wo - write only file
- mmap - file is mmapable
- ascii - file contains ascii text
- binary - file contains binary data
- cpumask - file contains a cpumask type
-
-.. [1] rw for IORESOURCE_IO (I/O port) regions only
-
-The read only files are informational, writes to them will be ignored, with
-the exception of the 'rom' file. Writable files can be used to perform
-actions on the device (e.g. changing config space, detaching a device).
-mmapable files are available via an mmap of the file at offset 0 and can be
-used to do actual device programming from userspace. Note that some platforms
-don't support mmapping of certain resources, so be sure to check the return
-value from any attempted mmap. The most notable of these are I/O port
-resources, which also provide read/write access.
-
-The 'enable' file provides a counter that indicates how many times the device
-has been enabled. If the 'enable' file currently returns '4', and a '1' is
-echoed into it, it will then return '5'. Echoing a '0' into it will decrease
-the count. Even when it returns to 0, though, some of the initialisation
-may not be reversed.
-
-The 'rom' file is special in that it provides read-only access to the device's
-ROM file, if available. It's disabled by default, however, so applications
-should write the string "1" to the file to enable it before attempting a read
-call, and disable it following the access by writing "0" to the file. Note
-that the device must be enabled for a rom read to return data successfully.
-In the event a driver is not bound to the device, it can be enabled using the
-'enable' file, documented above.
-
-The 'remove' file is used to remove the PCI device, by writing a non-zero
-integer to the file. This does not involve any kind of hot-plug functionality,
-e.g. powering off the device. The device is removed from the kernel's list of
-PCI devices, the sysfs directory for it is removed, and the device will be
-removed from any drivers attached to it. Removal of PCI root buses is
-disallowed.
-
-Accessing legacy resources through sysfs
-----------------------------------------
-
-Legacy I/O port and ISA memory resources are also provided in sysfs if the
-underlying platform supports them. They're located in the PCI class hierarchy,
-e.g.::
-
- /sys/class/pci_bus/0000:17/
- |-- bridge -> ../../../devices/pci0000:17
- |-- cpuaffinity
- |-- legacy_io
- `-- legacy_mem
-
-The legacy_io file is a read/write file that can be used by applications to
-do legacy port I/O. The application should open the file, seek to the desired
-port (e.g. 0x3e8) and do a read or a write of 1, 2 or 4 bytes. The legacy_mem
-file should be mmapped with an offset corresponding to the memory offset
-desired, e.g. 0xa0000 for the VGA frame buffer. The application can then
-simply dereference the returned pointer (after checking for errors of course)
-to access legacy memory space.
-
-Supporting PCI access on new platforms
---------------------------------------
-
-In order to support PCI resource mapping as described above, Linux platform
-code should ideally define ARCH_GENERIC_PCI_MMAP_RESOURCE and use the generic
-implementation of that functionality. To support the historical interface of
-mmap() through files in /proc/bus/pci, platforms may also set HAVE_PCI_MMAP.
-
-Alternatively, platforms which set HAVE_PCI_MMAP may provide their own
-implementation of pci_mmap_page_range() instead of defining
-ARCH_GENERIC_PCI_MMAP_RESOURCE.
-
-Platforms which support write-combining maps of PCI resources must define
-arch_can_pci_mmap_wc() which shall evaluate to non-zero at runtime when
-write-combining is permitted. Platforms which support maps of I/O resources
-define arch_can_pci_mmap_io() similarly.
-
-Legacy resources are protected by the HAVE_PCI_LEGACY define. Platforms
-wishing to support legacy functionality should define it and provide
-pci_legacy_read, pci_legacy_write and pci_mmap_legacy_page_range functions.
+++ /dev/null
-.. SPDX-License-Identifier: GPL-2.0
-
-=============
-Sysfs tagging
-=============
-
-(Taken almost verbatim from Eric Biederman's netns tagging patch
-commit msg)
-
-The problem. Network devices show up in sysfs and with the network
-namespace active multiple devices with the same name can show up in
-the same directory, ouch!
-
-To avoid that problem and allow existing applications in network
-namespaces to see the same interface that is currently presented in
-sysfs, sysfs now has tagging directory support.
-
-By using the network namespace pointers as tags to separate out
-the sysfs directory entries we ensure that we don't have conflicts
-in the directories and applications only see a limited set of
-the network devices.
-
-Each sysfs directory entry may be tagged with a namespace via the
-``void *ns member`` of its ``kernfs_node``. If a directory entry is tagged,
-then ``kernfs_node->flags`` will have a flag between KOBJ_NS_TYPE_NONE
-and KOBJ_NS_TYPES, and ns will point to the namespace to which it
-belongs.
-
-Each sysfs superblock's kernfs_super_info contains an array
-``void *ns[KOBJ_NS_TYPES]``. When a task in a tagging namespace
-kobj_nstype first mounts sysfs, a new superblock is created. It
-will be differentiated from other sysfs mounts by having its
-``s_fs_info->ns[kobj_nstype]`` set to the new namespace. Note that
-through bind mounting and mounts propagation, a task can easily view
-the contents of other namespaces' sysfs mounts. Therefore, when a
-namespace exits, it will call kobj_ns_exit() to invalidate any
-kernfs_node->ns pointers pointing to it.
-
-Users of this interface:
-
-- define a type in the ``kobj_ns_type`` enumeration.
-- call kobj_ns_type_register() with its ``kobj_ns_type_operations`` which has
-
- - current_ns() which returns current's namespace
- - netlink_ns() which returns a socket's namespace
- - initial_ns() which returns the initial namesapce
-
-- call kobj_ns_exit() when an individual tag is no longer valid
To illustrate::
- #define to_dev(obj) container_of(obj, struct device, kobj)
#define to_dev_attr(_attr) container_of(_attr, struct device_attribute, attr)
static ssize_t dev_attr_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct device_attribute *dev_attr = to_dev_attr(attr);
- struct device *dev = to_dev(kobj);
+ struct device *dev = kobj_to_dev(kobj);
ssize_t ret = -EIO;
if (dev_attr->show)
.. SPDX-License-Identifier: GPL-2.0
-:orphan:
-
.. UBIFS Authentication
.. sigma star gmbh
.. 2018
+============================
+UBIFS Authentication Support
+============================
+
Introduction
============
lpit
video_extension
extcon-intel-int3496
+ intel-pmc-mux
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver adm1266
+=====================
+
+Supported chips:
+ * Analog Devices ADM1266
+ Prefix: 'adm1266'
+ Datasheet: https://www.analog.com/media/en/technical-documentation/data-sheets/ADM1266.pdf
+
+Author: Alexandru Tachici <alexandru.tachici@analog.com>
+
+
+Description
+-----------
+
+This driver supports hardware monitoring for Analog Devices ADM1266 sequencer.
+
+ADM1266 is a sequencer that features voltage readback from 17 channels via an
+integrated 12 bit SAR ADC, accessed using a PMBus interface.
+
+The driver is a client driver to the core PMBus driver. Please see
+Documentation/hwmon/pmbus for details on PMBus client drivers.
+
+
+Sysfs entries
+-------------
+
+The following attributes are supported. Limits are read-write, history reset
+attributes are write-only, all other attributes are read-only.
+
+inX_label "voutx"
+inX_input Measured voltage.
+inX_min Minimum Voltage.
+inX_max Maximum voltage.
+inX_min_alarm Voltage low alarm.
+inX_max_alarm Voltage high alarm.
running during probe, wakes up every 100secs and stops running
when driver is removed.
+Frequency of the accumulator thread is set during the probe
+based on the chosen energy unit resolution. For example
+A. fine grain (1.625 micro J)
+B. course grain (0.125 milli J)
+
A socket and core energy read would return the current register
value added to the respective energy accumulator.
the drive temperature.
+Usage Note
+----------
+
+Reading the drive temperature may reset the spin down timer on some drives.
+This has been observed with WD120EFAX drives, but may be seen with other
+drives as well. The same behavior is observed if the 'hdtemp' or 'smartd'
+tools are used to access the drive.
+With the WD120EFAX drive, reading the drive temperature using the drivetemp
+driver is still possible _after_ it transitioned to standby mode, and
+reading the drive temperature in this mode will not cause the drive to
+change its mode (meaning the drive will not spin up). It is unknown if other
+drives experience similar behavior.
+
+A known workaround for WD120EFAX drives is to read the drive temperature at
+intervals larger than twice the spin-down time. Otherwise affected drives
+will never spin down.
+
+
Sysfs entries
-------------
adm1026
adm1031
adm1177
+ adm1266
adm1275
adm9240
ads7828
ina209
ina2xx
ina3221
+ intel-m10-bmc-hwmon
ir35221
ir38064
isl68137
smsc47b397
smsc47m192
smsc47m1
+ sparx5-temp
tc654
tc74
thmc50
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver intel-m10-bmc-hwmon
+=================================
+
+Supported chips:
+
+ * Intel MAX 10 BMC for Intel PAC N3000
+
+ Prefix: 'n3000bmc-hwmon'
+
+Author: Xu Yilun <yilun.xu@intel.com>
+
+
+Description
+-----------
+
+This driver adds the temperature, voltage, current and power reading
+support for the Intel MAX 10 Board Management Controller (BMC) chip.
+The BMC chip is integrated in some Intel Programmable Acceleration
+Cards (PAC). It connects to a set of sensor chips to monitor the
+sensor data of different components on the board. The BMC firmware is
+responsible for sensor data sampling and recording in shared
+registers. The host driver reads the sensor data from these shared
+registers and exposes them to users as hwmon interfaces.
+
+The BMC chip is implemented using the Intel MAX 10 CPLD. It could be
+reprogramed to some variants in order to support different Intel
+PACs. The driver is designed to be able to distinguish between the
+variants, but now it only supports the BMC for Intel PAC N3000.
+
+
+Sysfs attributes
+----------------
+
+The following attributes are supported:
+
+- Intel MAX 10 BMC for Intel PAC N3000:
+
+======================= =======================================================
+tempX_input Temperature of the component (specified by tempX_label)
+tempX_max Temperature maximum setpoint of the component
+tempX_crit Temperature critical setpoint of the component
+tempX_max_hyst Hysteresis for temperature maximum of the component
+tempX_crit_hyst Hysteresis for temperature critical of the component
+temp1_label "Board Temperature"
+temp2_label "FPGA Die Temperature"
+temp3_label "QSFP0 Temperature"
+temp4_label "QSFP1 Temperature"
+temp5_label "Retimer A Temperature"
+temp6_label "Retimer A SerDes Temperature"
+temp7_label "Retimer B Temperature"
+temp8_label "Retimer B SerDes Temperature"
+
+inX_input Measured voltage of the component (specified by
+ inX_label)
+in0_label "QSFP0 Supply Voltage"
+in1_label "QSFP1 Supply Voltage"
+in2_label "FPGA Core Voltage"
+in3_label "12V Backplane Voltage"
+in4_label "1.2V Voltage"
+in5_label "12V AUX Voltage"
+in6_label "1.8V Voltage"
+in7_label "3.3V Voltage"
+
+currX_input Measured current of the component (specified by
+ currX_label)
+curr1_label "FPGA Core Current"
+curr2_label "12V Backplane Current"
+curr3_label "12V AUX Current"
+
+powerX_input Measured power of the component (specified by
+ powerX_label)
+power1_label "Board Power"
+
+======================= =======================================================
+
+All the attributes are read-only.
Datasheet:
- http://cds.linear.com/docs/en/datasheet/2945fa.pdf
+ https://www.analog.com/media/en/technical-documentation/data-sheets/2945fb.pdf
Author: Guenter Roeck <linux@roeck-us.net>
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver mp2975
+====================
+
+Supported chips:
+
+ * MPS MP12254
+
+ Prefix: 'mp2975'
+
+Author:
+
+ Vadim Pasternak <vadimp@nvidia.com>
+
+Description
+-----------
+
+This driver implements support for Monolithic Power Systems, Inc. (MPS)
+vendor dual-loop, digital, multi-phase controller MP2975.
+
+This device:
+- Supports up to two power rail.
+- Provides 8 pulse-width modulations (PWMs), and can be configured up
+ to 8-phase operation for rail 1 and up to 4-phase operation for rail
+ 2.
+- Supports two pages 0 and 1 for telemetry and also pages 2 and 3 for
+ configuration.
+- Can configured VOUT readout in direct or VID format and allows
+ setting of different formats on rails 1 and 2. For VID the following
+ protocols are available: VR13 mode with 5-mV DAC; VR13 mode with
+ 10-mV DAC, IMVP9 mode with 5-mV DAC.
+
+Device supports:
+- SVID interface.
+- AVSBus interface.
+
+Device complaint with:
+- PMBus rev 1.3 interface.
+
+Device supports direct format for reading output current, output voltage,
+input and output power and temperature.
+Device supports linear format for reading input voltage and input power.
+Device supports VID and direct formats for reading output voltage.
+The below VID modes are supported: VR12, VR13, IMVP9.
+
+The driver provides the next attributes for the current:
+- for current in: input, maximum alarm;
+- for current out input, maximum alarm and highest values;
+- for phase current: input and label.
+attributes.
+The driver exports the following attributes via the 'sysfs' files, where
+- 'n' is number of telemetry pages (from 1 to 2);
+- 'k' is number of configured phases (from 1 to 8);
+- indexes 1, 1*n for "iin";
+- indexes n+1, n+2 for "iout";
+- indexes 2*n+1 ... 2*n + k for phases.
+
+**curr[1-{2n}]_alarm**
+
+**curr[{n+1}-{n+2}]_highest**
+
+**curr[1-{2n+k}]_input**
+
+**curr[1-{2n+k}]_label**
+
+The driver provides the next attributes for the voltage:
+- for voltage in: input, high critical threshold, high critical alarm, all only
+ from page 0;
+- for voltage out: input, low and high critical thresholds, low and high
+ critical alarms, from pages 0 and 1;
+The driver exports the following attributes via the 'sysfs' files, where
+- 'n' is number of telemetry pages (from 1 to 2);
+- indexes 1 for "iin";
+- indexes n+1, n+2 for "vout";
+
+**in[1-{2n+1}]_crit**
+
+**in[1-{2n+1}]_crit_alarm**
+
+**in[1-{2n+1}]_input**
+
+**in[1-{2n+1}]_label**
+
+**in[2-{n+1}]_lcrit**
+
+**in[2-{n+1}1_lcrit_alarm**
+
+The driver provides the next attributes for the power:
+- for power in alarm and input.
+- for power out: highest and input.
+The driver exports the following attributes via the 'sysfs' files, where
+- 'n' is number of telemetry pages (from 1 to 2);
+- indexes 1 for "pin";
+- indexes n+1, n+2 for "pout";
+
+**power1_alarm**
+
+**power[2-{n+1}]_highest**
+
+**power[1-{2n+1}]_input**
+
+**power[1-{2n+1}]_label**
+
+The driver provides the next attributes for the temperature (only from page 0):
+
+
+**temp1_crit**
+
+**temp1_crit_alarm**
+
+**temp1_input**
+
+**temp1_max**
+
+**temp1_max_alarm**
::
- int pmbus_do_probe(struct i2c_client *client, const struct i2c_device_id *id,
- struct pmbus_driver_info *info);
+ int pmbus_do_probe(struct i2c_client *client, struct pmbus_driver_info *info);
Execute probe function. Similar to standard probe function for other drivers,
with the pointer to struct pmbus_driver_info as additional argument. Calls
| PMBUS_HAVE_FAN12 | PMBUS_HAVE_STATUS_FAN12,
};
- static int ds1200_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+ static int ds1200_probe(struct i2c_client *client)
{
- return pmbus_do_probe(client, id, &ds1200_info);
+ return pmbus_do_probe(client, &ds1200_info);
}
static int ds1200_remove(struct i2c_client *client)
.driver = {
.name = "ds1200",
},
- .probe = ds1200_probe,
+ .probe_new = ds1200_probe,
.remove = ds1200_remove,
.id_table = ds1200_id,
};
inX_crit_alarm Voltage critical high alarm.
From VOLTAGE_OV_FAULT status.
inX_label "vin", "vcap", or "voutY"
+inX_rated_min Minimum rated voltage.
+ From MFR_VIN_MIN or MFR_VOUT_MIN register.
+inX_rated_max Maximum rated voltage.
+ From MFR_VIN_MAX or MFR_VOUT_MAX register.
currX_input Measured current. From READ_IIN or READ_IOUT register.
currX_max Maximum current.
currX_label "iin", "iinY", "iinY.Z", "ioutY", or "ioutY.Z",
where Y reflects the page number and Z reflects the
phase.
+currX_rated_max Maximum rated current.
+ From MFR_IIN_MAX or MFR_IOUT_MAX register.
powerX_input Measured power. From READ_PIN or READ_POUT register.
powerX_cap Output power cap. From POUT_MAX register.
powerX_label "pin", "pinY", "pinY.Z", "poutY", or "poutY.Z",
where Y reflects the page number and Z reflects the
phase.
+powerX_rated_max Maximum rated power.
+ From MFR_PIN_MAX or MFR_POUT_MAX register.
tempX_input Measured temperature.
From READ_TEMPERATURE_X register.
-tempX_min Mimimum temperature. From UT_WARN_LIMIT register.
+tempX_min Minimum temperature. From UT_WARN_LIMIT register.
tempX_max Maximum temperature. From OT_WARN_LIMIT register.
tempX_lcrit Critical low temperature.
From UT_FAULT_LIMIT register.
tempX_crit_alarm Chip temperature critical high alarm. Set by comparing
READ_TEMPERATURE_X with OT_FAULT_LIMIT if
TEMP_OT_FAULT status is set.
+tempX_rated_min Minimum rated temperature.
+ From MFR_TAMBIENT_MIN register.
+tempX_rated_max Maximum rated temperature.
+ From MFR_TAMBIENT_MAX, MFR_MAX_TEMP_1, MFR_MAX_TEMP_2 or
+ MFR_MAX_TEMP_3 register.
======================= ========================================================
Affects the way the driver calculates the CPU core reference
voltage from the vid pins.
+`in[0-*]_rated_min`
+ Minimum rated voltage.
+
+ Unit: millivolt
+
+ RO
+
+`in[0-*]_rated_max`
+ Maximum rated voltage.
+
+ Unit: millivolt
+
+ RO
+
Also see the Alarms section for status flags associated with voltages.
RW
+`temp[1-*]_rated_min`
+ Minimum rated temperature.
+
+ Unit: millidegree Celsius
+
+ RO
+
+`temp[1-*]_rated_max`
+ Maximum rated temperature.
+
+ Unit: millidegree Celsius
+
+ RO
+
Some chips measure temperature using external thermistors and an ADC, and
report the temperature measurement as a voltage. Converting this voltage
back to a temperature (or the other way around for limits) requires
RW
+`curr[1-*]_rated_min`
+ Minimum rated current.
+
+ Unit: milliampere
+
+ RO
+
+`curr[1-*]_rated_max`
+ Maximum rated current.
+
+ Unit: milliampere
+
+ RO
+
Also see the Alarms section for status flags associated with currents.
*****
RW
+`power[1-*]_rated_min`
+ Minimum rated power.
+
+ Unit: microWatt
+
+ RO
+
+`power[1-*]_rated_max`
+ Maximum rated power.
+
+ Unit: microWatt
+
+ RO
+
Also see the Alarms section for status flags associated with power readings.
******
RW
+`humidity[1-*]_rated_min`
+ Minimum rated humidity.
+
+ Unit: milli-percent (per cent mille, pcm)
+
+ RO
+
+`humidity[1-*]_rated_max`
+ Maximum rated humidity.
+
+ Unit: milli-percent (per cent mille, pcm)
+
+ RO
+
******
Alarms
******
irq-redir
mca
serial
- xen
+++ /dev/null
-********************************************************
-Recipe for getting/building/running Xen/ia64 with pv_ops
-********************************************************
-This recipe describes how to get xen-ia64 source and build it,
-and run domU with pv_ops.
-
-Requirements
-============
-
- - python
- - mercurial
- it (aka "hg") is an open-source source code
- management software. See the below.
- http://www.selenic.com/mercurial/wiki/
- - git
- - bridge-utils
-
-Getting and Building Xen and Dom0
-=================================
-
- My environment is:
-
- - Machine : Tiger4
- - Domain0 OS : RHEL5
- - DomainU OS : RHEL5
-
- 1. Download source::
-
- # hg clone http://xenbits.xensource.com/ext/ia64/xen-unstable.hg
- # cd xen-unstable.hg
- # hg clone http://xenbits.xensource.com/ext/ia64/linux-2.6.18-xen.hg
-
- 2. # make world
-
- 3. # make install-tools
-
- 4. copy kernels and xen::
-
- # cp xen/xen.gz /boot/efi/efi/redhat/
- # cp build-linux-2.6.18-xen_ia64/vmlinux.gz \
- /boot/efi/efi/redhat/vmlinuz-2.6.18.8-xen
-
- 5. make initrd for Dom0/DomU::
-
- # make -C linux-2.6.18-xen.hg ARCH=ia64 modules_install \
- O=$(pwd)/build-linux-2.6.18-xen_ia64
- # mkinitrd -f /boot/efi/efi/redhat/initrd-2.6.18.8-xen.img \
- 2.6.18.8-xen --builtin mptspi --builtin mptbase \
- --builtin mptscsih --builtin uhci-hcd --builtin ohci-hcd \
- --builtin ehci-hcd
-
-Making a disk image for guest OS
-================================
-
- 1. make file::
-
- # dd if=/dev/zero of=/root/rhel5.img bs=1M seek=4096 count=0
- # mke2fs -F -j /root/rhel5.img
- # mount -o loop /root/rhel5.img /mnt
- # cp -ax /{dev,var,etc,usr,bin,sbin,lib} /mnt
- # mkdir /mnt/{root,proc,sys,home,tmp}
-
- Note: You may miss some device files. If so, please create them
- with mknod. Or you can use tar instead of cp.
-
- 2. modify DomU's fstab::
-
- # vi /mnt/etc/fstab
- /dev/xvda1 / ext3 defaults 1 1
- none /dev/pts devpts gid=5,mode=620 0 0
- none /dev/shm tmpfs defaults 0 0
- none /proc proc defaults 0 0
- none /sys sysfs defaults 0 0
-
- 3. modify inittab
-
- set runlevel to 3 to avoid X trying to start::
-
- # vi /mnt/etc/inittab
- id:3:initdefault:
-
- Start a getty on the hvc0 console::
-
- X0:2345:respawn:/sbin/mingetty hvc0
-
- tty1-6 mingetty can be commented out
-
- 4. add hvc0 into /etc/securetty::
-
- # vi /mnt/etc/securetty (add hvc0)
-
- 5. umount::
-
- # umount /mnt
-
-FYI, virt-manager can also make a disk image for guest OS.
-It's GUI tools and easy to make it.
-
-Boot Xen & Domain0
-==================
-
- 1. replace elilo
- elilo of RHEL5 can boot Xen and Dom0.
- If you use old elilo (e.g RHEL4), please download from the below
- http://elilo.sourceforge.net/cgi-bin/blosxom
- and copy into /boot/efi/efi/redhat/::
-
- # cp elilo-3.6-ia64.efi /boot/efi/efi/redhat/elilo.efi
-
- 2. modify elilo.conf (like the below)::
-
- # vi /boot/efi/efi/redhat/elilo.conf
- prompt
- timeout=20
- default=xen
- relocatable
-
- image=vmlinuz-2.6.18.8-xen
- label=xen
- vmm=xen.gz
- initrd=initrd-2.6.18.8-xen.img
- read-only
- append=" -- rhgb root=/dev/sda2"
-
-The append options before "--" are for xen hypervisor,
-the options after "--" are for dom0.
-
-FYI, your machine may need console options like
-"com1=19200,8n1 console=vga,com1". For example,
-append="com1=19200,8n1 console=vga,com1 -- rhgb console=tty0 \
-console=ttyS0 root=/dev/sda2"
-
-Getting and Building domU with pv_ops
-=====================================
-
- 1. get pv_ops tree::
-
- # git clone http://people.valinux.co.jp/~yamahata/xen-ia64/linux-2.6-xen-ia64.git/
-
- 2. git branch (if necessary)::
-
- # cd linux-2.6-xen-ia64/
- # git checkout -b your_branch origin/xen-ia64-domu-minimal-2008may19
-
- Note:
- The current branch is xen-ia64-domu-minimal-2008may19.
- But you would find the new branch. You can see with
- "git branch -r" to get the branch lists.
-
- http://people.valinux.co.jp/~yamahata/xen-ia64/for_eagl/linux-2.6-ia64-pv-ops.git/
-
- is also available.
-
- The tree is based on
-
- git://git.kernel.org/pub/scm/linux/kernel/git/aegl/linux-2.6 test)
-
- 3. copy .config for pv_ops of domU::
-
- # cp arch/ia64/configs/xen_domu_wip_defconfig .config
-
- 4. make kernel with pv_ops::
-
- # make oldconfig
- # make
-
- 5. install the kernel and initrd::
-
- # cp vmlinux.gz /boot/efi/efi/redhat/vmlinuz-2.6-pv_ops-xenU
- # make modules_install
- # mkinitrd -f /boot/efi/efi/redhat/initrd-2.6-pv_ops-xenU.img \
- 2.6.26-rc3xen-ia64-08941-g1b12161 --builtin mptspi \
- --builtin mptbase --builtin mptscsih --builtin uhci-hcd \
- --builtin ohci-hcd --builtin ehci-hcd
-
-Boot DomainU with pv_ops
-========================
-
- 1. make config of DomU::
-
- # vi /etc/xen/rhel5
- kernel = "/boot/efi/efi/redhat/vmlinuz-2.6-pv_ops-xenU"
- ramdisk = "/boot/efi/efi/redhat/initrd-2.6-pv_ops-xenU.img"
- vcpus = 1
- memory = 512
- name = "rhel5"
- disk = [ 'file:/root/rhel5.img,xvda1,w' ]
- root = "/dev/xvda1 ro"
- extra= "rhgb console=hvc0"
-
- 2. After boot xen and dom0, start xend::
-
- # /etc/init.d/xend start
-
- ( In the debugging case, `# XEND_DEBUG=1 xend trace_start` )
-
- 3. start domU::
-
- # xm create -c rhel5
-
-Reference
-=========
-- Wiki of Xen/IA64 upstream merge
- http://wiki.xensource.com/xenwiki/XenIA64/UpstreamMerge
-
-Written by Akio Takebe <takebe_akio@jp.fujitsu.com> on 28 May 2008
*/
}
- static int iio_trig_hrtimer_remove(struct iio_sw_trigger *swt)
+ static int iio_trig_sample_remove(struct iio_sw_trigger *swt)
{
/*
* This undoes the actions in iio_trig_sample_probe
+.. _kbuild_llvm:
+
==============================
Building Linux with Clang/LLVM
==============================
ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- make CC=clang
``CROSS_COMPILE`` is not used to prefix the Clang compiler binary, instead
-``CROSS_COMPILE`` is used to set a command line flag: ``--target <triple>``. For
+``CROSS_COMPILE`` is used to set a command line flag: ``--target=<triple>``. For
example: ::
- clang --target aarch64-linux-gnu foo.c
+ clang --target=aarch64-linux-gnu foo.c
LLVM Utilities
--------------
- `Wiki <https://github.com/ClangBuiltLinux/linux/wiki>`_
- `Beginner Bugs <https://github.com/ClangBuiltLinux/linux/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22>`_
+.. _getting_llvm:
+
Getting LLVM
-------------
a later run of this command to identify the leakers. This same output
can also help you find situations where runtime lock initialization has
been omitted.
+
+Recursive read locks:
+---------------------
+The whole of the rest document tries to prove a certain type of cycle is equivalent
+to deadlock possibility.
+
+There are three types of lockers: writers (i.e. exclusive lockers, like
+spin_lock() or write_lock()), non-recursive readers (i.e. shared lockers, like
+down_read()) and recursive readers (recursive shared lockers, like rcu_read_lock()).
+And we use the following notations of those lockers in the rest of the document:
+
+ W or E: stands for writers (exclusive lockers).
+ r: stands for non-recursive readers.
+ R: stands for recursive readers.
+ S: stands for all readers (non-recursive + recursive), as both are shared lockers.
+ N: stands for writers and non-recursive readers, as both are not recursive.
+
+Obviously, N is "r or W" and S is "r or R".
+
+Recursive readers, as their name indicates, are the lockers allowed to acquire
+even inside the critical section of another reader of the same lock instance,
+in other words, allowing nested read-side critical sections of one lock instance.
+
+While non-recursive readers will cause a self deadlock if trying to acquire inside
+the critical section of another reader of the same lock instance.
+
+The difference between recursive readers and non-recursive readers is because:
+recursive readers get blocked only by a write lock *holder*, while non-recursive
+readers could get blocked by a write lock *waiter*. Considering the follow example:
+
+ TASK A: TASK B:
+
+ read_lock(X);
+ write_lock(X);
+ read_lock_2(X);
+
+Task A gets the reader (no matter whether recursive or non-recursive) on X via
+read_lock() first. And when task B tries to acquire writer on X, it will block
+and become a waiter for writer on X. Now if read_lock_2() is recursive readers,
+task A will make progress, because writer waiters don't block recursive readers,
+and there is no deadlock. However, if read_lock_2() is non-recursive readers,
+it will get blocked by writer waiter B, and cause a self deadlock.
+
+Block conditions on readers/writers of the same lock instance:
+--------------------------------------------------------------
+There are simply four block conditions:
+
+1. Writers block other writers.
+2. Readers block writers.
+3. Writers block both recursive readers and non-recursive readers.
+4. And readers (recursive or not) don't block other recursive readers but
+ may block non-recursive readers (because of the potential co-existing
+ writer waiters)
+
+Block condition matrix, Y means the row blocks the column, and N means otherwise.
+
+ | E | r | R |
+ +---+---+---+---+
+ E | Y | Y | Y |
+ +---+---+---+---+
+ r | Y | Y | N |
+ +---+---+---+---+
+ R | Y | Y | N |
+
+ (W: writers, r: non-recursive readers, R: recursive readers)
+
+
+acquired recursively. Unlike non-recursive read locks, recursive read locks
+only get blocked by current write lock *holders* other than write lock
+*waiters*, for example:
+
+ TASK A: TASK B:
+
+ read_lock(X);
+
+ write_lock(X);
+
+ read_lock(X);
+
+is not a deadlock for recursive read locks, as while the task B is waiting for
+the lock X, the second read_lock() doesn't need to wait because it's a recursive
+read lock. However if the read_lock() is non-recursive read lock, then the above
+case is a deadlock, because even if the write_lock() in TASK B cannot get the
+lock, but it can block the second read_lock() in TASK A.
+
+Note that a lock can be a write lock (exclusive lock), a non-recursive read
+lock (non-recursive shared lock) or a recursive read lock (recursive shared
+lock), depending on the lock operations used to acquire it (more specifically,
+the value of the 'read' parameter for lock_acquire()). In other words, a single
+lock instance has three types of acquisition depending on the acquisition
+functions: exclusive, non-recursive read, and recursive read.
+
+To be concise, we call that write locks and non-recursive read locks as
+"non-recursive" locks and recursive read locks as "recursive" locks.
+
+Recursive locks don't block each other, while non-recursive locks do (this is
+even true for two non-recursive read locks). A non-recursive lock can block the
+corresponding recursive lock, and vice versa.
+
+A deadlock case with recursive locks involved is as follow:
+
+ TASK A: TASK B:
+
+ read_lock(X);
+ read_lock(Y);
+ write_lock(Y);
+ write_lock(X);
+
+Task A is waiting for task B to read_unlock() Y and task B is waiting for task
+A to read_unlock() X.
+
+Dependency types and strong dependency paths:
+---------------------------------------------
+Lock dependencies record the orders of the acquisitions of a pair of locks, and
+because there are 3 types for lockers, there are, in theory, 9 types of lock
+dependencies, but we can show that 4 types of lock dependencies are enough for
+deadlock detection.
+
+For each lock dependency:
+
+ L1 -> L2
+
+, which means lockdep has seen L1 held before L2 held in the same context at runtime.
+And in deadlock detection, we care whether we could get blocked on L2 with L1 held,
+IOW, whether there is a locker L3 that L1 blocks L3 and L2 gets blocked by L3. So
+we only care about 1) what L1 blocks and 2) what blocks L2. As a result, we can combine
+recursive readers and non-recursive readers for L1 (as they block the same types) and
+we can combine writers and non-recursive readers for L2 (as they get blocked by the
+same types).
+
+With the above combination for simplification, there are 4 types of dependency edges
+in the lockdep graph:
+
+1) -(ER)->: exclusive writer to recursive reader dependency, "X -(ER)-> Y" means
+ X -> Y and X is a writer and Y is a recursive reader.
+
+2) -(EN)->: exclusive writer to non-recursive locker dependency, "X -(EN)-> Y" means
+ X -> Y and X is a writer and Y is either a writer or non-recursive reader.
+
+3) -(SR)->: shared reader to recursive reader dependency, "X -(SR)-> Y" means
+ X -> Y and X is a reader (recursive or not) and Y is a recursive reader.
+
+4) -(SN)->: shared reader to non-recursive locker dependency, "X -(SN)-> Y" means
+ X -> Y and X is a reader (recursive or not) and Y is either a writer or
+ non-recursive reader.
+
+Note that given two locks, they may have multiple dependencies between them, for example:
+
+ TASK A:
+
+ read_lock(X);
+ write_lock(Y);
+ ...
+
+ TASK B:
+
+ write_lock(X);
+ write_lock(Y);
+
+, we have both X -(SN)-> Y and X -(EN)-> Y in the dependency graph.
+
+We use -(xN)-> to represent edges that are either -(EN)-> or -(SN)->, the
+similar for -(Ex)->, -(xR)-> and -(Sx)->
+
+A "path" is a series of conjunct dependency edges in the graph. And we define a
+"strong" path, which indicates the strong dependency throughout each dependency
+in the path, as the path that doesn't have two conjunct edges (dependencies) as
+-(xR)-> and -(Sx)->. In other words, a "strong" path is a path from a lock
+walking to another through the lock dependencies, and if X -> Y -> Z is in the
+path (where X, Y, Z are locks), and the walk from X to Y is through a -(SR)-> or
+-(ER)-> dependency, the walk from Y to Z must not be through a -(SN)-> or
+-(SR)-> dependency.
+
+We will see why the path is called "strong" in next section.
+
+Recursive Read Deadlock Detection:
+----------------------------------
+
+We now prove two things:
+
+Lemma 1:
+
+If there is a closed strong path (i.e. a strong circle), then there is a
+combination of locking sequences that causes deadlock. I.e. a strong circle is
+sufficient for deadlock detection.
+
+Lemma 2:
+
+If there is no closed strong path (i.e. strong circle), then there is no
+combination of locking sequences that could cause deadlock. I.e. strong
+circles are necessary for deadlock detection.
+
+With these two Lemmas, we can easily say a closed strong path is both sufficient
+and necessary for deadlocks, therefore a closed strong path is equivalent to
+deadlock possibility. As a closed strong path stands for a dependency chain that
+could cause deadlocks, so we call it "strong", considering there are dependency
+circles that won't cause deadlocks.
+
+Proof for sufficiency (Lemma 1):
+
+Let's say we have a strong circle:
+
+ L1 -> L2 ... -> Ln -> L1
+
+, which means we have dependencies:
+
+ L1 -> L2
+ L2 -> L3
+ ...
+ Ln-1 -> Ln
+ Ln -> L1
+
+We now can construct a combination of locking sequences that cause deadlock:
+
+Firstly let's make one CPU/task get the L1 in L1 -> L2, and then another get
+the L2 in L2 -> L3, and so on. After this, all of the Lx in Lx -> Lx+1 are
+held by different CPU/tasks.
+
+And then because we have L1 -> L2, so the holder of L1 is going to acquire L2
+in L1 -> L2, however since L2 is already held by another CPU/task, plus L1 ->
+L2 and L2 -> L3 are not -(xR)-> and -(Sx)-> (the definition of strong), which
+means either L2 in L1 -> L2 is a non-recursive locker (blocked by anyone) or
+the L2 in L2 -> L3, is writer (blocking anyone), therefore the holder of L1
+cannot get L2, it has to wait L2's holder to release.
+
+Moreover, we can have a similar conclusion for L2's holder: it has to wait L3's
+holder to release, and so on. We now can prove that Lx's holder has to wait for
+Lx+1's holder to release, and note that Ln+1 is L1, so we have a circular
+waiting scenario and nobody can get progress, therefore a deadlock.
+
+Proof for necessary (Lemma 2):
+
+Lemma 2 is equivalent to: If there is a deadlock scenario, then there must be a
+strong circle in the dependency graph.
+
+According to Wikipedia[1], if there is a deadlock, then there must be a circular
+waiting scenario, means there are N CPU/tasks, where CPU/task P1 is waiting for
+a lock held by P2, and P2 is waiting for a lock held by P3, ... and Pn is waiting
+for a lock held by P1. Let's name the lock Px is waiting as Lx, so since P1 is waiting
+for L1 and holding Ln, so we will have Ln -> L1 in the dependency graph. Similarly,
+we have L1 -> L2, L2 -> L3, ..., Ln-1 -> Ln in the dependency graph, which means we
+have a circle:
+
+ Ln -> L1 -> L2 -> ... -> Ln
+
+, and now let's prove the circle is strong:
+
+For a lock Lx, Px contributes the dependency Lx-1 -> Lx and Px+1 contributes
+the dependency Lx -> Lx+1, and since Px is waiting for Px+1 to release Lx,
+so it's impossible that Lx on Px+1 is a reader and Lx on Px is a recursive
+reader, because readers (no matter recursive or not) don't block recursive
+readers, therefore Lx-1 -> Lx and Lx -> Lx+1 cannot be a -(xR)-> -(Sx)-> pair,
+and this is true for any lock in the circle, therefore, the circle is strong.
+
+References:
+-----------
+[1]: https://en.wikipedia.org/wiki/Deadlock
+[2]: Shibu, K. (2009). Intro To Embedded Systems (1st ed.). Tata McGraw-Hill
Read path: same as in :ref:`seqcount_t`.
+
+.. _seqcount_latch_t:
+
+Latch sequence counters (``seqcount_latch_t``)
+----------------------------------------------
+
+Latch sequence counters are a multiversion concurrency control mechanism
+where the embedded seqcount_t counter even/odd value is used to switch
+between two copies of protected data. This allows the sequence counter
+read path to safely interrupt its own write side critical section.
+
+Use seqcount_latch_t when the write side sections cannot be protected
+from interruption by readers. This is typically the case when the read
+side can be invoked from NMI handlers.
+
+Check `raw_write_seqcount_latch()` for more information.
+
+
.. _seqlock_t:
Sequential locks (``seqlock_t``)
rebasing-and-merging
pull-requests
maintainer-entry-profile
+ modifying-patches
--- /dev/null
+.. _modifyingpatches:
+
+Modifying Patches
+=================
+
+If you are a subsystem or branch maintainer, sometimes you need to slightly
+modify patches you receive in order to merge them, because the code is not
+exactly the same in your tree and the submitters'. If you stick strictly to
+rule (c) of the developers certificate of origin, you should ask the submitter
+to rediff, but this is a totally counter-productive waste of time and energy.
+Rule (b) allows you to adjust the code, but then it is very impolite to change
+one submitters code and make him endorse your bugs. To solve this problem, it
+is recommended that you add a line between the last Signed-off-by header and
+yours, indicating the nature of your changes. While there is nothing mandatory
+about this, it seems like prepending the description with your mail and/or
+name, all enclosed in square brackets, is noticeable enough to make it obvious
+that you are responsible for last-minute changes. Example::
+
+ Signed-off-by: Random J Developer <random@developer.example.org>
+ [lucky@maintainer.example.org: struct foo moved from foo.c to foo.h]
+ Signed-off-by: Lucky K Maintainer <lucky@maintainer.example.org>
+
+This practice is particularly helpful if you maintain a stable branch and
+want at the same time to credit the author, track changes, merge the fix,
+and protect the submitter from complaints. Note that under no circumstances
+can you change the author's identity (the From header), as it is the one
+which appears in the changelog.
+
+Special note to back-porters: It seems to be a common and useful practice
+to insert an indication of the origin of a patch at the top of the commit
+message (just after the subject line) to facilitate tracking. For instance,
+here's what we see in a 3.x-stable release::
+
+ Date: Tue Oct 7 07:26:38 2014 -0400
+
+ libata: Un-break ATA blacklist
+
+ commit 1c40279960bcd7d52dbdf1d466b20d24b99176c8 upstream.
+
+And here's what might appear in an older kernel once a patch is backported::
+
+ Date: Tue May 13 22:12:27 2008 +0200
+
+ wireless, airo: waitbusy() won't delay
+
+ [backport of 2.6 commit b7acbdfbd1f277c1eb23f344f899cfa4cd0bf36a]
+
+Whatever the format, this information provides a valuable help to people
+tracking your trees, and to people trying to troubleshoot bugs in your
+tree.
[*] For information on bus mastering DMA and coherency please read:
Documentation/driver-api/pci/pci.rst
- Documentation/DMA-API-HOWTO.txt
- Documentation/DMA-API.txt
+ Documentation/core-api/dma-api-howto.rst
+ Documentation/core-api/dma-api.rst
DATA DEPENDENCY BARRIERS (HISTORICAL)
here.
See the subsection "Kernel I/O barrier effects" for more information on
- relaxed I/O accessors and the Documentation/DMA-API.txt file for more
- information on consistent memory.
+ relaxed I/O accessors and the Documentation/core-api/dma-api.rst file for
+ more information on consistent memory.
(*) pmem_wmb();
seg6-sysctl
strparser
switchdev
+ sysfs-tagging
tc-actions-env-rules
tcp-thin
team
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+=============
+Sysfs tagging
+=============
+
+(Taken almost verbatim from Eric Biederman's netns tagging patch
+commit msg)
+
+The problem. Network devices show up in sysfs and with the network
+namespace active multiple devices with the same name can show up in
+the same directory, ouch!
+
+To avoid that problem and allow existing applications in network
+namespaces to see the same interface that is currently presented in
+sysfs, sysfs now has tagging directory support.
+
+By using the network namespace pointers as tags to separate out
+the sysfs directory entries we ensure that we don't have conflicts
+in the directories and applications only see a limited set of
+the network devices.
+
+Each sysfs directory entry may be tagged with a namespace via the
+``void *ns member`` of its ``kernfs_node``. If a directory entry is tagged,
+then ``kernfs_node->flags`` will have a flag between KOBJ_NS_TYPE_NONE
+and KOBJ_NS_TYPES, and ns will point to the namespace to which it
+belongs.
+
+Each sysfs superblock's kernfs_super_info contains an array
+``void *ns[KOBJ_NS_TYPES]``. When a task in a tagging namespace
+kobj_nstype first mounts sysfs, a new superblock is created. It
+will be differentiated from other sysfs mounts by having its
+``s_fs_info->ns[kobj_nstype]`` set to the new namespace. Note that
+through bind mounting and mounts propagation, a task can easily view
+the contents of other namespaces' sysfs mounts. Therefore, when a
+namespace exits, it will call kobj_ns_exit() to invalidate any
+kernfs_node->ns pointers pointing to it.
+
+Users of this interface:
+
+- define a type in the ``kobj_ns_type`` enumeration.
+- call kobj_ns_type_register() with its ``kobj_ns_type_operations`` which has
+
+ - current_ns() which returns current's namespace
+ - netlink_ns() which returns a socket's namespace
+ - initial_ns() which returns the initial namesapce
+
+- call kobj_ns_exit() when an individual tag is no longer valid
http://vger.kernel.org/vger-lists.html
There are lists hosted elsewhere, though; a number of them are at
-lists.redhat.com.
+redhat.com/mailman/listinfo.
The core mailing list for kernel development is, of course, linux-kernel.
This list is an intimidating place to be; volume can reach 500 messages per
Program Minimal version Command to check the version
====================== =============== ========================================
GNU C 4.9 gcc --version
+Clang/LLVM (optional) 10.0.1 clang --version
GNU make 3.81 make --version
binutils 2.23 ld -v
flex 2.5.35 flex --version
The gcc version requirements may vary depending on the type of CPU in your
computer.
+Clang/LLVM (optional)
+---------------------
+
+The latest formal release of clang and LLVM utils (according to
+`releases.llvm.org <https://releases.llvm.org>`_) are supported for building
+kernels. Older releases aren't guaranteed to work, and we may drop workarounds
+from the kernel that were used to support older versions. Please see additional
+docs on :ref:`Building Linux with Clang/LLVM <kbuild_llvm>`.
+
Make
----
- <ftp://ftp.gnu.org/gnu/gcc/>
+Clang/LLVM
+----------
+
+- :ref:`Getting LLVM <getting_llvm>`.
+
Make
----
"unreachable" conditions. (For example, see commits like `this one
<https://git.kernel.org/linus/d4689846881d160a4d12a514e991a740bcb5d65a>`_.)
-uninitialized_var()
--------------------
-For any compiler warnings about uninitialized variables, just add
-an initializer. Using the uninitialized_var() macro (or similar
-warning-silencing tricks) is dangerous as it papers over `real bugs
-<https://lore.kernel.org/lkml/20200603174714.192027-1-glider@google.com/>`_
-(or can in the future), and suppresses unrelated compiler warnings
-(e.g. "unused variable"). If the compiler thinks it is uninitialized,
-either simply initialize the variable or make compiler changes. Keep in
-mind that in most cases, if an initialization is obviously redundant,
-the compiler's dead-store elimination pass will make sure there are no
-needless variable writes.
-
-As Linus has said, this macro
-`must <https://lore.kernel.org/lkml/CA+55aFw+Vbj0i=1TGqCR5vQkCzWJ0QxK6CernOU6eedsudAixw@mail.gmail.com/>`_
-`be <https://lore.kernel.org/lkml/CA+55aFwgbgqhbp1fkxvRKEpzyR5J8n1vKT1VZdz9knmPuXhOeg@mail.gmail.com/>`_
-`removed <https://lore.kernel.org/lkml/CA+55aFz2500WfbKXAx8s67wrm9=yVJu65TpLgN_ybYNv0VEOKA@mail.gmail.com/>`_.
-
open-coded arithmetic in allocator arguments
--------------------------------------------
Dynamic size calculations (especially multiplication) should not be
In the example above, we had to remember to calculate ``count - 1`` when using
the struct_size() helper, otherwise we would have --unintentionally-- allocated
memory for one too many ``items`` objects. The cleanest and least error-prone way
-to implement this is through the use of a `flexible array member`::
+to implement this is through the use of a `flexible array member`, together with
+struct_size() and flex_array_size() helpers::
struct something {
size_t count;
instance = kmalloc(struct_size(instance, items, count), GFP_KERNEL);
instance->count = count;
- size = sizeof(instance->items[0]) * instance->count;
- memcpy(instance->items, source, size);
+ memcpy(instance->items, source, flex_array_size(instance, items, instance->count));
it makes quoting portions of the patch more difficult in the patch
review process.
+It's also strongly recommended that you use plain text in your email body,
+for patches and other emails alike. https://useplaintext.email may be useful
+for information on how to configure your preferred email client, as well as
+listing recommended email clients should you not already have a preference.
+
Email clients that are used for Linux kernel patches should send the
patch text untouched. For example, they should not modify or delete tabs
or spaces, even at the beginning or end of lines.
The kernel is written in the C programming language [c-language]_.
More precisely, the kernel is typically compiled with ``gcc`` [gcc]_
under ``-std=gnu89`` [gcc-c-dialect-options]_: the GNU dialect of ISO C90
-(including some C99 features).
+(including some C99 features). ``clang`` [clang]_ is also supported, see
+docs on :ref:`Building Linux with Clang/LLVM <kbuild_llvm>`.
This dialect contains many extensions to the language [gnu-extensions]_,
and many of them are used within the kernel as a matter of course.
-There is some support for compiling the kernel with ``clang`` [clang]_
-and ``icc`` [icc]_ for several of the architectures, although at the time
-of writing it is not completed, requiring third-party patches.
+There is some support for compiling the kernel with ``icc`` [icc]_ for several
+of the architectures, although at the time of writing it is not completed,
+requiring third-party patches.
Attributes
----------
c) Builds successfully when using ``O=builddir``
+ d) Any Documentation/ changes build successfully without new warnings/errors.
+ Use ``make htmldocs`` or ``make pdfdocs`` to check the build and
+ fix any issues.
+
3) Builds on multiple CPU architectures by using local cross-compile tools
or some other build farm.
Licensing:
The code must be released to us under the
- GNU General Public License. We don't insist on any kind
- of exclusive GPL licensing, and if you wish the driver
- to be useful to other communities such as BSD you may well
- wish to release under multiple licenses.
+ GNU General Public License. If you wish the driver to be
+ useful to other communities such as BSD you may release
+ under multiple licenses. If you choose to release under
+ licenses other than the GPL, you should include your
+ rationale for your license choices in your cover letter.
See accepted licenses at include/linux/module.h
Copyright:
This document contains a large number of suggestions in a relatively terse
format. For detailed information on how the kernel development process
-works, see :ref:`Documentation/process <development_process_main>`.
-Also, read :ref:`Documentation/process/submit-checklist.rst <submitchecklist>`
-for a list of items to check before
-submitting code. If you are submitting a driver, also read
-:ref:`Documentation/process/submitting-drivers.rst <submittingdrivers>`;
-for device tree binding patches, read
-Documentation/devicetree/bindings/submitting-patches.rst.
-
-Many of these steps describe the default behavior of the ``git`` version
-control system; if you use ``git`` to prepare your patches, you'll find much
-of the mechanical work done for you, though you'll still need to prepare
-and document a sensible set of patches. In general, use of ``git`` will make
-your life as a kernel developer easier.
-
-0) Obtain a current source tree
--------------------------------
+works, see :doc:`development-process`. Also, read :doc:`submit-checklist`
+for a list of items to check before submitting code. If you are submitting
+a driver, also read :doc:`submitting-drivers`; for device tree binding patches,
+read :doc:`submitting-patches`.
+
+This documentation assumes that you're using ``git`` to prepare your patches.
+If you're unfamiliar with ``git``, you would be well-advised to learn how to
+use it, it will make your life as a kernel developer and in general much
+easier.
+
+Obtain a current source tree
+----------------------------
If you do not have a repository with the current kernel source handy, use
``git`` to obtain one. You'll want to start with the mainline repository,
in the MAINTAINERS file to find that tree, or simply ask the maintainer if
the tree is not listed there.
-It is still possible to download kernel releases via tarballs (as described
-in the next section), but that is the hard way to do kernel development.
-
-1) ``diff -up``
----------------
-
-If you must generate your patches by hand, use ``diff -up`` or ``diff -uprN``
-to create patches. Git generates patches in this form by default; if
-you're using ``git``, you can skip this section entirely.
-
-All changes to the Linux kernel occur in the form of patches, as
-generated by :manpage:`diff(1)`. When creating your patch, make sure to
-create it in "unified diff" format, as supplied by the ``-u`` argument
-to :manpage:`diff(1)`.
-Also, please use the ``-p`` argument which shows which C function each
-change is in - that makes the resultant ``diff`` a lot easier to read.
-Patches should be based in the root kernel source directory,
-not in any lower subdirectory.
-
-To create a patch for a single file, it is often sufficient to do::
-
- SRCTREE=linux
- MYFILE=drivers/net/mydriver.c
-
- cd $SRCTREE
- cp $MYFILE $MYFILE.orig
- vi $MYFILE # make your change
- cd ..
- diff -up $SRCTREE/$MYFILE{.orig,} > /tmp/patch
-
-To create a patch for multiple files, you should unpack a "vanilla",
-or unmodified kernel source tree, and generate a ``diff`` against your
-own source tree. For example::
-
- MYSRC=/devel/linux
-
- tar xvfz linux-3.19.tar.gz
- mv linux-3.19 linux-3.19-vanilla
- diff -uprN -X linux-3.19-vanilla/Documentation/dontdiff \
- linux-3.19-vanilla $MYSRC > /tmp/patch
-
-``dontdiff`` is a list of files which are generated by the kernel during
-the build process, and should be ignored in any :manpage:`diff(1)`-generated
-patch.
-
-Make sure your patch does not include any extra files which do not
-belong in a patch submission. Make sure to review your patch -after-
-generating it with :manpage:`diff(1)`, to ensure accuracy.
-
-If your changes produce a lot of deltas, you need to split them into
-individual patches which modify things in logical stages; see
-:ref:`split_changes`. This will facilitate review by other kernel developers,
-very important if you want your patch accepted.
-
-If you're using ``git``, ``git rebase -i`` can help you with this process. If
-you're not using ``git``, ``quilt`` <https://savannah.nongnu.org/projects/quilt>
-is another popular alternative.
-
.. _describe_changes:
-2) Describe your changes
-------------------------
+Describe your changes
+---------------------
Describe your problem. Whether your patch is a one-line bug fix or
5000 lines of a new feature, there must be an underlying problem that
.. _split_changes:
-3) Separate your changes
-------------------------
+Separate your changes
+---------------------
Separate each **logical change** into a separate patch.
-4) Style-check your changes
----------------------------
+Style-check your changes
+------------------------
Check your patch for basic style violations, details of which can be
found in
patch.
-5) Select the recipients for your patch
----------------------------------------
+Select the recipients for your patch
+------------------------------------
You should always copy the appropriate subsystem maintainer(s) on any patch
to code that they maintain; look through the MAINTAINERS file and the
If you have a patch that fixes an exploitable security bug, send that patch
to security@kernel.org. For severe bugs, a short embargo may be considered
to allow distributors to get the patch out to users; in such cases,
-obviously, the patch should not be sent to any public lists.
+obviously, the patch should not be sent to any public lists. See also
+:doc:`/admin-guide/security-bugs`.
Patches that fix a severe bug in a released kernel should be directed
toward the stable maintainers by putting a line like this::
-6) No MIME, no links, no compression, no attachments. Just plain text
-----------------------------------------------------------------------
+No MIME, no links, no compression, no attachments. Just plain text
+-------------------------------------------------------------------
Linus and other kernel developers need to be able to read and comment
on the changes you are submitting. It is important for a kernel
developer to be able to "quote" your changes, using standard e-mail
tools, so that they may comment on specific portions of your code.
-For this reason, all patches should be submitted by e-mail "inline".
+For this reason, all patches should be submitted by e-mail "inline". The
+easiest way to do this is with ``git send-email``, which is strongly
+recommended. An interactive tutorial for ``git send-email`` is available at
+https://git-send-email.io.
+
+If you choose not to use ``git send-email``:
.. warning::
Exception: If your mailer is mangling patches then someone may ask
you to re-send them using MIME.
-See :ref:`Documentation/process/email-clients.rst <email_clients>`
-for hints about configuring your e-mail client so that it sends your patches
-untouched.
-
-7) E-mail size
---------------
+See :doc:`/process/email-clients` for hints about configuring your e-mail
+client so that it sends your patches untouched.
-Large changes are not appropriate for mailing lists, and some
-maintainers. If your patch, uncompressed, exceeds 300 kB in size,
-it is preferred that you store your patch on an Internet-accessible
-server, and provide instead a URL (link) pointing to your patch. But note
-that if your patch exceeds 300 kB, it almost certainly needs to be broken up
-anyway.
-
-8) Respond to review comments
------------------------------
+Respond to review comments
+--------------------------
Your patch will almost certainly get comments from reviewers on ways in
-which the patch can be improved. You must respond to those comments;
-ignoring reviewers is a good way to get ignored in return. Review comments
-or questions that do not lead to a code change should almost certainly
+which the patch can be improved, in the form of a reply to your email. You must
+respond to those comments; ignoring reviewers is a good way to get ignored in
+return. You can simply reply to their emails to answer their comments. Review
+comments or questions that do not lead to a code change should almost certainly
bring about a comment or changelog entry so that the next reviewer better
understands what is going on.
reviewers sometimes get grumpy. Even in that case, though, respond
politely and address the problems they have pointed out.
+See :doc:`email-clients` for recommendations on email
+clients and mailing list etiquette.
-9) Don't get discouraged - or impatient
----------------------------------------
+
+Don't get discouraged - or impatient
+------------------------------------
After you have submitted your change, be patient and wait. Reviewers are
busy people and may not get to your patch right away.
busy times like merge windows.
-10) Include PATCH in the subject
---------------------------------
+Include PATCH in the subject
+-----------------------------
Due to high e-mail traffic to Linus, and to linux-kernel, it is common
convention to prefix your subject line with [PATCH]. This lets Linus
and other kernel developers more easily distinguish patches from other
e-mail discussions.
+``git send-email`` will do this for you automatically.
-11) Sign your work - the Developer's Certificate of Origin
-----------------------------------------------------------
+Sign your work - the Developer's Certificate of Origin
+------------------------------------------------------
To improve tracking of who did what, especially with patches that can
percolate to their final resting place in the kernel through several
Signed-off-by: Random J Developer <random@developer.example.org>
using your real name (sorry, no pseudonyms or anonymous contributions.)
+This will be done for you automatically if you use ``git commit -s``.
Some people also put extra tags at the end. They'll just be ignored for
now, but you can do this to mark internal company procedures or just
point out some special detail about the sign-off.
-If you are a subsystem or branch maintainer, sometimes you need to slightly
-modify patches you receive in order to merge them, because the code is not
-exactly the same in your tree and the submitters'. If you stick strictly to
-rule (c), you should ask the submitter to rediff, but this is a totally
-counter-productive waste of time and energy. Rule (b) allows you to adjust
-the code, but then it is very impolite to change one submitter's code and
-make him endorse your bugs. To solve this problem, it is recommended that
-you add a line between the last Signed-off-by header and yours, indicating
-the nature of your changes. While there is nothing mandatory about this, it
-seems like prepending the description with your mail and/or name, all
-enclosed in square brackets, is noticeable enough to make it obvious that
-you are responsible for last-minute changes. Example::
- Signed-off-by: Random J Developer <random@developer.example.org>
- [lucky@maintainer.example.org: struct foo moved from foo.c to foo.h]
- Signed-off-by: Lucky K Maintainer <lucky@maintainer.example.org>
-
-This practice is particularly helpful if you maintain a stable branch and
-want at the same time to credit the author, track changes, merge the fix,
-and protect the submitter from complaints. Note that under no circumstances
-can you change the author's identity (the From header), as it is the one
-which appears in the changelog.
-
-Special note to back-porters: It seems to be a common and useful practice
-to insert an indication of the origin of a patch at the top of the commit
-message (just after the subject line) to facilitate tracking. For instance,
-here's what we see in a 3.x-stable release::
-
- Date: Tue Oct 7 07:26:38 2014 -0400
-
- libata: Un-break ATA blacklist
-
- commit 1c40279960bcd7d52dbdf1d466b20d24b99176c8 upstream.
-
-And here's what might appear in an older kernel once a patch is backported::
-
- Date: Tue May 13 22:12:27 2008 +0200
-
- wireless, airo: waitbusy() won't delay
-
- [backport of 2.6 commit b7acbdfbd1f277c1eb23f344f899cfa4cd0bf36a]
-
-Whatever the format, this information provides a valuable help to people
-tracking your trees, and to people trying to troubleshoot bugs in your
-tree.
-
-
-12) When to use Acked-by:, Cc:, and Co-developed-by:
--------------------------------------------------------
+When to use Acked-by:, Cc:, and Co-developed-by:
+------------------------------------------------
The Signed-off-by: tag indicates that the signer was involved in the
development of the patch, or that he/she was in the patch's delivery path.
Signed-off-by: Submitting Co-Author <sub@coauthor.example.org>
-13) Using Reported-by:, Tested-by:, Reviewed-by:, Suggested-by: and Fixes:
---------------------------------------------------------------------------
+Using Reported-by:, Tested-by:, Reviewed-by:, Suggested-by: and Fixes:
+----------------------------------------------------------------------
The Reported-by tag gives credit to people who find bugs and report them and it
hopefully inspires them to help us again in the future. Please note that if
.. _the_canonical_patch_format:
-14) The canonical patch format
-------------------------------
+The canonical patch format
+--------------------------
This section describes how the patch itself should be formatted. Note
that, if you have your patches stored in a ``git`` repository, proper patch
.. _explicit_in_reply_to:
-15) Explicit In-Reply-To headers
---------------------------------
+Explicit In-Reply-To headers
+----------------------------
It can be helpful to manually add In-Reply-To: headers to a patch
(e.g., when using ``git send-email``) to associate the patch with
the cover email text) to link to an earlier version of the patch series.
-16) Providing base tree information
------------------------------------
+Providing base tree information
+-------------------------------
When other developers receive your patches and start the review process,
it is often useful for them to know where in the tree history they
content, right before your email signature.
-17) Sending ``git pull`` requests
----------------------------------
-
-If you have a series of patches, it may be most convenient to have the
-maintainer pull them directly into the subsystem repository with a
-``git pull`` operation. Note, however, that pulling patches from a developer
-requires a higher degree of trust than taking patches from a mailing list.
-As a result, many subsystem maintainers are reluctant to take pull
-requests, especially from new, unknown developers. If in doubt you can use
-the pull request as the cover letter for a normal posting of the patch
-series, giving the maintainer the option of using either.
-
-A pull request should have [GIT PULL] in the subject line. The
-request itself should include the repository name and the branch of
-interest on a single line; it should look something like::
-
- Please pull from
-
- git://jdelvare.pck.nerim.net/jdelvare-2.6 i2c-for-linus
-
- to get these changes:
-
-A pull request should also include an overall message saying what will be
-included in the request, a ``git shortlog`` listing of the patches
-themselves, and a ``diffstat`` showing the overall effect of the patch series.
-The easiest way to get all this information together is, of course, to let
-``git`` do it for you with the ``git request-pull`` command.
-
-Some maintainers (including Linus) want to see pull requests from signed
-commits; that increases their confidence that the request actually came
-from you. Linus, in particular, will not pull from public hosting sites
-like GitHub in the absence of a signed tag.
-
-The first step toward creating such tags is to make a GNUPG key and get it
-signed by one or more core kernel developers. This step can be hard for
-new developers, but there is no way around it. Attending conferences can
-be a good way to find developers who can sign your key.
-
-Once you have prepared a patch series in ``git`` that you wish to have somebody
-pull, create a signed tag with ``git tag -s``. This will create a new tag
-identifying the last commit in the series and containing a signature
-created with your private key. You will also have the opportunity to add a
-changelog-style message to the tag; this is an ideal place to describe the
-effects of the pull request as a whole.
-
-If the tree the maintainer will be pulling from is not the repository you
-are working from, don't forget to push the signed tag explicitly to the
-public tree.
-
-When generating your pull request, use the signed tag as the target. A
-command like this will do the trick::
-
- git request-pull master git://my.public.tree/linux.git my-signed-tag
-
-
References
----------
.. note::
Wakeup CPU selection in CFS can be eclipsed by Energy Aware Scheduling
- (EAS), which is described in Documentation/scheduling/sched-energy.rst.
+ (EAS), which is described in Documentation/scheduler/sched-energy.rst.
5.1.3 Load balancing
~~~~~~~~~~~~~~~~~~~~
looking for the presence of the SD_ASYM_CPUCAPACITY flag when the scheduling
domains are built.
-See Documentation/sched/sched-capacity.rst for requirements to be met for this
+See Documentation/scheduler/sched-capacity.rst for requirements to be met for this
flag to be set in the sched_domain hierarchy.
Please note that EAS is not fundamentally incompatible with SMP, but no
uid_t current_fsuid(void) Current's file access UID
gid_t current_fsgid(void) Current's file access GID
kernel_cap_t current_cap(void) Current's effective capabilities
- void *current_security(void) Current's LSM security pointer
struct user_struct *current_user(void) Current's user account
There are also convenience wrappers for retrieving specific associated pairs of
Or with the Intel TSS 2 stack::
- #> tpm2_createprimary --hierarchy o -G rsa2048 -o key.ctxt
+ #> tpm2_createprimary --hierarchy o -G rsa2048 -c key.ctxt
[...]
- handle: 0x800000FF
- #> tpm2_evictcontrol -c key.ctxt -p 0x81000001
+ #> tpm2_evictcontrol -c key.ctxt 0x81000001
persistentHandle: 0x81000001
Usage::
else:
from sphinx.errors import NoUri
import re
+from itertools import chain
#
# Regex nastiness. Of course.
# :c:func: block (i.e. ":c:func:`mmap()`s" flakes out), so the last
# bit tries to restrict matches to things that won't create trouble.
#
-RE_function = re.compile(r'([\w_][\w\d_]+\(\))')
+RE_function = re.compile(r'(([\w_][\w\d_]+)\(\))')
+RE_type = re.compile(r'(struct|union|enum|typedef)\s+([\w_][\w\d_]+)')
+#
+# Detects a reference to a documentation page of the form Documentation/... with
+# an optional extension
+#
+RE_doc = re.compile(r'Documentation(/[\w\-_/]+)(\.\w+)*')
#
# Many places in the docs refer to common system calls. It is
'select', 'poll', 'fork', 'execve', 'clone', 'ioctl',
'socket' ]
-#
-# Find all occurrences of function() and try to replace them with
-# appropriate cross references.
-#
-def markup_funcs(docname, app, node):
- cdom = app.env.domains['c']
+def markup_refs(docname, app, node):
t = node.astext()
done = 0
repl = [ ]
- for m in RE_function.finditer(t):
+ #
+ # Associate each regex with the function that will markup its matches
+ #
+ markup_func = {RE_type: markup_c_ref,
+ RE_function: markup_c_ref,
+ RE_doc: markup_doc_ref}
+ match_iterators = [regex.finditer(t) for regex in markup_func]
+ #
+ # Sort all references by the starting position in text
+ #
+ sorted_matches = sorted(chain(*match_iterators), key=lambda m: m.start())
+ for m in sorted_matches:
#
- # Include any text prior to function() as a normal text node.
+ # Include any text prior to match as a normal text node.
#
if m.start() > done:
repl.append(nodes.Text(t[done:m.start()]))
+
#
- # Go through the dance of getting an xref out of the C domain
- #
- target = m.group(1)[:-2]
- target_text = nodes.Text(target + '()')
- xref = None
- if target not in Skipfuncs:
- lit_text = nodes.literal(classes=['xref', 'c', 'c-func'])
- lit_text += target_text
- pxref = addnodes.pending_xref('', refdomain = 'c',
- reftype = 'function',
- reftarget = target, modname = None,
- classname = None)
- #
- # XXX The Latex builder will throw NoUri exceptions here,
- # work around that by ignoring them.
- #
- try:
- xref = cdom.resolve_xref(app.env, docname, app.builder,
- 'function', target, pxref, lit_text)
- except NoUri:
- xref = None
- #
- # Toss the xref into the list if we got it; otherwise just put
- # the function text.
+ # Call the function associated with the regex that matched this text and
+ # append its return to the text
#
- if xref:
- repl.append(xref)
- else:
- repl.append(target_text)
+ repl.append(markup_func[m.re](docname, app, m))
+
done = m.end()
if done < len(t):
repl.append(nodes.Text(t[done:]))
return repl
+#
+# Try to replace a C reference (function() or struct/union/enum/typedef
+# type_name) with an appropriate cross reference.
+#
+def markup_c_ref(docname, app, match):
+ class_str = {RE_function: 'c-func', RE_type: 'c-type'}
+ reftype_str = {RE_function: 'function', RE_type: 'type'}
+
+ cdom = app.env.domains['c']
+ #
+ # Go through the dance of getting an xref out of the C domain
+ #
+ target = match.group(2)
+ target_text = nodes.Text(match.group(0))
+ xref = None
+ if not (match.re == RE_function and target in Skipfuncs):
+ lit_text = nodes.literal(classes=['xref', 'c', class_str[match.re]])
+ lit_text += target_text
+ pxref = addnodes.pending_xref('', refdomain = 'c',
+ reftype = reftype_str[match.re],
+ reftarget = target, modname = None,
+ classname = None)
+ #
+ # XXX The Latex builder will throw NoUri exceptions here,
+ # work around that by ignoring them.
+ #
+ try:
+ xref = cdom.resolve_xref(app.env, docname, app.builder,
+ reftype_str[match.re], target, pxref,
+ lit_text)
+ except NoUri:
+ xref = None
+ #
+ # Return the xref if we got it; otherwise just return the plain text.
+ #
+ if xref:
+ return xref
+ else:
+ return target_text
+
+#
+# Try to replace a documentation reference of the form Documentation/... with a
+# cross reference to that page
+#
+def markup_doc_ref(docname, app, match):
+ stddom = app.env.domains['std']
+ #
+ # Go through the dance of getting an xref out of the std domain
+ #
+ target = match.group(1)
+ xref = None
+ pxref = addnodes.pending_xref('', refdomain = 'std', reftype = 'doc',
+ reftarget = target, modname = None,
+ classname = None, refexplicit = False)
+ #
+ # XXX The Latex builder will throw NoUri exceptions here,
+ # work around that by ignoring them.
+ #
+ try:
+ xref = stddom.resolve_xref(app.env, docname, app.builder, 'doc',
+ target, pxref, None)
+ except NoUri:
+ xref = None
+ #
+ # Return the xref if we got it; otherwise just return the plain text.
+ #
+ if xref:
+ return xref
+ else:
+ return nodes.Text(match.group(0))
+
def auto_markup(app, doctree, name):
#
# This loop could eventually be improved on. Someday maybe we
for para in doctree.traverse(nodes.paragraph):
for node in para.traverse(nodes.Text):
if not isinstance(node.parent, nodes.literal):
- node.parent.replace(node, markup_funcs(name, app, node))
+ node.parent.replace(node, markup_refs(name, app, node))
def setup(app):
app.connect('doctree-resolved', auto_markup)
MEMADDR : Address where the probe is inserted.
MAXACTIVE : Maximum number of instances of the specified function that
can be probed simultaneously, or 0 for the default value
- as defined in Documentation/staging/kprobes.rst section 1.3.1.
+ as defined in Documentation/trace/kprobes.rst section 1.3.1.
FETCHARGS : Arguments. Each probe can have up to 128 args.
%REG : Fetch register REG
-.. This file is dual-licensed: you can use it either under the terms
-.. of the GPL 2.0 or the GFDL 1.2 license, at your option. Note that this
-.. dual licensing only applies to this file, and not this project as a
-.. whole.
-..
-.. a) This file is free software; you can redistribute it and/or
-.. modify it under the terms of the GNU General Public License as
-.. published by the Free Software Foundation version 2 of
-.. the License.
-..
-.. This file is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License for more details.
-..
-.. Or, alternatively,
-..
-.. b) Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.2 version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GPL-2.0 OR GFDL-1.2 WITH no-invariant-sections
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.2-no-invariants-only
===========================
Lockless Ring Buffer Design
여러 메이저 넘버를 갖는 다양한 안정된 커널 트리들
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-3 자리 숫자로 이루어진 버젼의 커널들은 -stable 커널들이다. 그것들은 해당 메이저
-메인라인 릴리즈에서 발견된 큰 회귀들이나 보안 문제들 중 비교적 작고 중요한
-수정들을 포함하며, 앞의 두 버전 넘버는 같은 기반 버전을 의미한다.
+세개의 버젼 넘버로 이루어진 버젼의 커널들은 -stable 커널들이다. 그것들은 해당
+메이저 메인라인 릴리즈에서 발견된 큰 회귀들이나 보안 문제들 중 비교적 작고
+중요한 수정들을 포함한다. 주요 stable 시리즈 릴리즈는 세번째 버젼 넘버를
+증가시키며 앞의 두 버젼 넘버는 그대로 유지한다.
이것은 가장 최근의 안정적인 커널을 원하는 사용자에게 추천되는 브랜치이며,
개발/실험적 버젼을 테스트하는 것을 돕고자 하는 사용자들과는 별로 관련이 없다.
제안된 패치는 서브시스템 트리에 커밋되기 전에 메일링 리스트를 통해
리뷰된다(아래의 관련 섹션을 참고하기 바란다). 일부 커널 서브시스템의 경우, 이
리뷰 프로세스는 patchwork라는 도구를 통해 추적된다. patchwork은 등록된 패치와
-í\8c¨ì¹\98ì\97\90 ë\8c\80í\95\9c ì½\94ë©\98í\8a¸, í\8c¨ì¹\98ì\9d\98 ë²\84ì \84을 볼 수 있는 웹 인터페이스를 제공하고,
+í\8c¨ì¹\98ì\97\90 ë\8c\80í\95\9c ì½\94ë©\98í\8a¸, í\8c¨ì¹\98ì\9d\98 ë²\84ì ¼을 볼 수 있는 웹 인터페이스를 제공하고,
메인테이너는 패치를 리뷰 중, 리뷰 통과, 또는 반려됨으로 표시할 수 있다.
대부분의 이러한 patchwork 사이트는 https://patchwork.kernel.org/ 에 나열되어
있다.
- 컴파일러 배리어.
- CPU 메모리 배리어.
- - MMIO 쓰기 배리어.
(*) 암묵적 커널 메모리 배리어.
(*) CPU 간 ACQUIRING 배리어의 효과.
- Acquire vs 메모리 액세스.
- - Acquire vs I/O 액세스.
(*) 메모리 배리어가 필요한 곳
완료되기 전에 행해진 것처럼 보일 수 있습니다.
ACQUIRE 와 RELEASE 오퍼레이션의 사용은 일반적으로 다른 메모리 배리어의
- 필요성을 없앱니다 (하지만 "MMIO 쓰기 배리어" 서브섹션에서 설명되는 예외를
- 알아두세요). 또한, RELEASE+ACQUIRE 조합은 범용 메모리 배리어처럼 동작할
- 것을 보장하지 -않습니다-. 하지만, 어떤 변수에 대한 RELEASE 오퍼레이션을
- 앞서는 메모리 액세스들의 수행 결과는 이 RELEASE 오퍼레이션을 뒤이어 같은
- 변수에 대해 수행된 ACQUIRE 오퍼레이션을 뒤따르는 메모리 액세스에는 보여질
- 것이 보장됩니다. 다르게 말하자면, 주어진 변수의 크리티컬 섹션에서는, 해당
- 변수에 대한 앞의 크리티컬 섹션에서의 모든 액세스들이 완료되었을 것을
- 보장합니다.
+ 필요성을 없앱니다. 또한, RELEASE+ACQUIRE 조합은 범용 메모리 배리어처럼
+ 동작할 것을 보장하지 -않습니다-. 하지만, 어떤 변수에 대한 RELEASE
+ 오퍼레이션을 앞서는 메모리 액세스들의 수행 결과는 이 RELEASE 오퍼레이션을
+ 뒤이어 같은 변수에 대해 수행된 ACQUIRE 오퍼레이션을 뒤따르는 메모리
+ 액세스에는 보여질 것이 보장됩니다. 다르게 말하자면, 주어진 변수의
+ 크리티컬 섹션에서는, 해당 변수에 대한 앞의 크리티컬 섹션에서의 모든
+ 액세스들이 완료되었을 것을 보장합니다.
즉, ACQUIRE 는 최소한의 "취득" 동작처럼, 그리고 RELEASE 는 최소한의 "공개"
처럼 동작한다는 의미입니다.
(*) CPU 메모리 배리어.
- (*) MMIO 쓰기 배리어.
-
컴파일러 배리어
---------------
"커널 I/O 배리어의 효과" 섹션을, consistent memory 에 대한 자세한 내용을
위해선 Documentation/core-api/dma-api.rst 문서를 참고하세요.
+ (*) pmem_wmb();
+
+ 이것은 persistent memory 를 위한 것으로, persistent 저장소에 가해진 변경
+ 사항이 플랫폼 연속성 도메인에 도달했을 것을 보장하기 위한 것입니다.
+
+ 예를 들어, 임시적이지 않은 pmem 영역으로의 쓰기 후, 우리는 쓰기가 플랫폼
+ 연속성 도메인에 도달했을 것을 보장하기 위해 pmem_wmb() 를 사용합니다.
+ 이는 쓰기가 뒤따르는 instruction 들이 유발하는 어떠한 데이터 액세스나
+ 데이터 전송의 시작 전에 persistent 저장소를 업데이트 했을 것을 보장합니다.
+ 이는 wmb() 에 의해 이뤄지는 순서 규칙을 포함합니다.
+
+ Persistent memory 에서의 로드를 위해선 현재의 읽기 메모리 배리어로도 읽기
+ 순서를 보장하는데 충분합니다.
=========================
암묵적 커널 메모리 배리어
--- /dev/null
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/arm64/amu.rst <amu_index>`
+
+Translator: Bailu Lin <bailu.lin@vivo.com>
+
+=================================
+AArch64 Linux 中扩展的活动监控单元
+=================================
+
+作者: Ionela Voinescu <ionela.voinescu@arm.com>
+
+日期: 2019-09-10
+
+本文档简要描述了 AArch64 Linux 支持的活动监控单元的规范。
+
+
+架构总述
+--------
+
+活动监控是 ARMv8.4 CPU 架构引入的一个可选扩展特性。
+
+活动监控单元(在每个 CPU 中实现)为系统管理提供了性能计数器。既可以通
+过系统寄存器的方式访问计数器，同时也支持外部内存映射的方式访问计数器。
+
+AMUv1 架构实现了一个由4个固定的64位事件计数器组成的计数器组。
+
+ - CPU 周期计数器：同 CPU 的频率增长
+ - 常量计数器：同固定的系统时钟频率增长
+ - 淘汰指令计数器: 同每次架构指令执行增长
+ - 内存停顿周期计数器：计算由在时钟域内的最后一级缓存中未命中而引起
+ 的指令调度停顿周期数
+
+当处于 WFI 或者 WFE 状态时，计数器不会增长。
+
+AMU 架构提供了一个高达16位的事件计数器空间，未来新的 AMU 版本中可能
+用它来实现新增的事件计数器。
+
+另外，AMUv1 实现了一个多达16个64位辅助事件计数器的计数器组。
+
+冷复位时所有的计数器会清零。
+
+
+基本支持
+--------
+
+内核可以安全地运行在支持 AMU 和不支持 AMU 的 CPU 组合中。
+因此，当配置 CONFIG_ARM64_AMU_EXTN 后我们无条件使能后续
+(secondary or hotplugged) CPU 检测和使用这个特性。
+
+当在 CPU 上检测到该特性时，我们会标记为特性可用但是不能保证计数器的功能，
+仅表明有扩展属性。
+
+固件(代码运行在高异常级别，例如 arm-tf )需支持以下功能：
+
+ - 提供低异常级别(EL2 和 EL1)访问 AMU 寄存器的能力。
+ - 使能计数器。如果未使能，它的值应为 0。
+ - 在从电源关闭状态启动 CPU 前或后保存或者恢复计数器。
+
+当使用使能了该特性的内核启动但固件损坏时，访问计数器寄存器可能会遭遇
+panic 或者死锁。即使未发现这些症状，计数器寄存器返回的数据结果并不一
+定能反映真实情况。通常，计数器会返回 0，表明他们未被使能。
+
+如果固件没有提供适当的支持最好关闭 CONFIG_ARM64_AMU_EXTN。
+值得注意的是，出于安全原因，不要绕过 AMUSERRENR_EL0 设置而捕获从
+EL0(用户空间) 访问 EL1(内核空间)。 因此，固件应该确保访问 AMU寄存器
+不会困在 EL2或EL3。
+
+AMUv1 的固定计数器可以通过如下系统寄存器访问：
+
+ - SYS_AMEVCNTR0_CORE_EL0
+ - SYS_AMEVCNTR0_CONST_EL0
+ - SYS_AMEVCNTR0_INST_RET_EL0
+ - SYS_AMEVCNTR0_MEM_STALL_EL0
+
+特定辅助计数器可以通过 SYS_AMEVCNTR1_EL0(n) 访问，其中n介于0到15。
+
+详细信息定义在目录：arch/arm64/include/asm/sysreg.h。
+
+
+用户空间访问
+------------
+
+由于以下原因，当前禁止从用户空间访问 AMU 的寄存器：
+
+ - 安全因数：可能会暴露处于安全模式执行的代码信息。
+ - 意愿：AMU 是用于系统管理的。
+
+同样，该功能对用户空间不可见。
+
+
+虚拟化
+------
+
+由于以下原因，当前禁止从 KVM 客户端的用户空间(EL0)和内核空间(EL1)
+访问 AMU 的寄存器：
+
+ - 安全因数：可能会暴露给其他客户端或主机端执行的代码信息。
+
+任何试图访问 AMU 寄存器的行为都会触发一个注册在客户端的未定义异常。
--- /dev/null
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/arm64/index.rst <arm64_index>`
+:Translator: Bailu Lin <bailu.lin@vivo.com>
+
+.. _cn_arm64_index:
+
+
+==========
+ARM64 架构
+==========
+
+.. toctree::
+ :maxdepth: 2
+
+ amu
示例:
-#define to_dev(obj) container_of(obj, struct device, kobj)
#define to_dev_attr(_attr) container_of(_attr, struct device_attribute, attr)
static ssize_t dev_attr_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct device_attribute *dev_attr = to_dev_attr(attr);
- struct device *dev = to_dev(kobj);
+ struct device *dev = kobj_to_dev(kobj);
ssize_t ret = -EIO;
if (dev_attr->show)
admin-guide/index
process/index
filesystems/index
+ arm64/index
目录和表格
----------
error return:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- EACCES Access denied: requires CAP_SYS_ADMIN
- EINVAL value out of range [0 1]
- EBUSY Controller busy
error return:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- EACCES Access denied: requires CAP_SYS_ADMIN
- EINVAL value out of range supported by disk.
- EBUSY Controller busy or blockmode already set.
the ATA specification.
error returns:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- ENOMSG IDENTIFY DEVICE information not available
notes:
error return:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- EACCES Access denied: requires CAP_SYS_ADMIN
- EINVAL value out of range [0 1]
- EBUSY Controller busy
error return:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- EACCES Access denied: requires CAP_SYS_ADMIN
- EINVAL value out of range [0 3]
- EBUSY Controller busy
error return:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- EACCES Access denied: requires CAP_SYS_ADMIN
- EINVAL value out of range [0 1]
- EBUSY Controller busy
error return:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- EACCES Access denied: requires CAP_SYS_ADMIN
- EINVAL value out of range [0 1]
- EBUSY Controller busy
error return:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- EACCES Access denied: requires CAP_SYS_ADMIN
- EINVAL value out of range [0 255]
- EBUSY Controller busy
error return:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- EACCES Access denied: requires CAP_SYS_ADMIN
- EINVAL value out of range [0 1]
- EBUSY Controller busy
error return:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- EACCES Access denied: requires CAP_SYS_ADMIN
- EINVAL value out of range [0 254]
- EBUSY Controller busy
error return:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- EACCES Access denied: requires CAP_SYS_ADMIN
- EINVAL value out of range [0 2]
- EBUSY Controller busy
error return:
- - EINVAL (bdev != bdev->bd_contains) (not sure what this means)
+ - EINVAL Called on a partition instead of the whole disk device
- EACCES Access denied: requires CAP_SYS_ADMIN
- EINVAL value out of range [0 1]
- EBUSY Controller busy
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
-
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. include:: <isonum.txt>
.. _cec:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _cec-func-close:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _cec-func-ioctl:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _cec-func-open:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _cec-func-poll:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _cec-user-func:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _cec_header:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _cec-intro:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CEC_ADAP_G_CAPS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CEC_ADAP_LOG_ADDRS:
.. _CEC_ADAP_G_LOG_ADDRS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CEC_ADAP_PHYS_ADDR:
.. _CEC_ADAP_G_PHYS_ADDR:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CEC_DQEVENT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CEC_MODE:
.. _CEC_G_MODE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CEC_TRANSMIT:
.. _CEC_RECEIVE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
CEC Pin Framework Error Injection
=================================
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_BILINGUAL_CHANNEL_SELECT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_CHANNEL_SELECT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_CLEAR_BUFFER:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_CONTINUE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _audio_fclose:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _audio_fopen:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _audio_fwrite:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_GET_CAPABILITIES:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_GET_STATUS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_PAUSE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_PLAY:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_SELECT_SOURCE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_SET_AV_SYNC:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_SET_BYPASS_MODE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_SET_ID:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_SET_MIXER:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_SET_MUTE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_SET_STREAMTYPE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _AUDIO_STOP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dvb_audio:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _audio_data_types:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _audio_function_calls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _ca_fclose:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _ca_fopen:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CA_GET_CAP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CA_GET_DESCR_INFO:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CA_GET_MSG:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CA_GET_SLOT_INFO:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CA_RESET:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CA_SEND_MSG:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _CA_SET_DESCR:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dvb_ca:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _ca_data_types:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _ca_function_calls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dvb_demux:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_ADD_PID:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_EXPBUF:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dmx_fclose:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dmx_fopen:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dmx_fread:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dmx_fwrite:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_GET_PES_PIDS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_GET_STC:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dmx-mmap:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dmx-munmap:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_QBUF:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_QUERYBUF:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_REMOVE_PID:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_REQBUFS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_SET_BUFFER_SIZE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_SET_FILTER:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_SET_PES_FILTER:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_START:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _DMX_STOP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dmx_fcalls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dmx_types:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dvb-fe-read-status:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. c:type:: dvb_frontend_event
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. c:type:: dvb_frontend_parameters
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
-
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. include:: <isonum.txt>
.. _dvbapi:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _frontend-properties:
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- This file is dual-licensed: you can use it either under the terms
- of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
- dual licensing only applies to this file, and not this project as a
- whole.
-
- a) This file is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation version 2 of
- the License.
-
- This file is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- Or, alternatively,
-
- b) Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later -->
<svg id="svg2" width="15.847cm" height="8.4187cm" fill-rule="evenodd" stroke-linejoin="round" stroke-width="28.222" preserveAspectRatio="xMidYMid" version="1.2" viewBox="0 0 23770.123 12628.122" xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><defs id="defs142"><marker id="Arrow1Lend" overflow="visible" orient="auto"><path id="path954" transform="matrix(-.8 0 0 -.8 -10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker><marker id="marker1243" overflow="visible" orient="auto"><path id="path1241" transform="matrix(-.8 0 0 -.8 -10 0)" d="m0 0 5-5-17.5 5 17.5 5z" fill-rule="evenodd" stroke="#000" stroke-width="1pt"/></marker></defs><metadata id="metadata519"><rdf:RDF><cc:Work
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/><dc:title/></cc:Work></rdf:RDF></metadata><rect id="rect197" class="BoundingBox" x="5355.1" y="13.122" width="18403" height="9603" fill="none"/><path id="path199" d="m14556 9614.1h-9200v-9600h18400v9600z" fill="#fff"/><path id="path201" d="m14556 9614.1h-9200v-9600h18400v9600z" fill="none" stroke="#000"/><rect id="rect206" class="BoundingBox" x="13.122" y="4013.1" width="4544" height="2403" fill="none"/><path id="path208" d="m2285.1 6414.1h-2271v-2400h4541v2400z" fill="#fff"/><path id="path210" d="m2285.1 6414.1h-2271v-2400h4541v2400z" fill="none" stroke="#000"/><text id="text212" class="TextShape" x="-2443.8779" y="-4585.8779"><tspan id="tspan214" class="TextParagraph" font-family="sans-serif" font-size="635px" font-weight="400"><tspan id="tspan216" class="TextPosition"
x="1281.1219" y="5435.1221"><tspan id="tspan218" fill="#000000">Antena</tspan></tspan></tspan></text>
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dvb_examples:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
******************
Frontend bandwidth
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_DISEQC_RECV_SLAVE_REPLY:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_DISEQC_RESET_OVERLOAD:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_DISEQC_SEND_BURST:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_DISEQC_SEND_MASTER_CMD:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_DISHNETWORK_SEND_LEGACY_CMD:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_ENABLE_HIGH_LNB_VOLTAGE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_GET_EVENT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_GET_FRONTEND:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_GET_INFO:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_GET_PROPERTY:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_READ_BER:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_READ_SIGNAL_STRENGTH:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_READ_SNR:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_READ_STATUS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_READ_UNCORRECTED_BLOCKS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_SET_FRONTEND_TUNE_MODE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_SET_FRONTEND:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_SET_TONE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _FE_SET_VOLTAGE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
*************
Frontend type
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _fe_property_parameters:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
Frontend uAPI data types
========================
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _frontend-property-cable-systems:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _frontend-property-satellite-systems:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _frontend-property-terrestrial-systems:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _frontend-stat-properties:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dvb_frontend:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _frontend_f_close:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _frontend_f_open:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _frontend_fcalls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _frontend_legacy_types:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _frontend_legacy_dvbv3_api:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
****************************
Digital TV uAPI header files
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dvb_introdution:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _legacy_dvb_apis:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _NET_ADD_IF:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _NET_GET_IF:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _NET_REMOVE_IF:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _net_types:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _net:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _query-dvb-frontend-info:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_CLEAR_BUFFER:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_COMMAND:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_CONTINUE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_FAST_FORWARD:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _video_fclose:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _video_fopen:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_FREEZE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _video_fwrite:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_GET_CAPABILITIES:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_GET_EVENT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_GET_FRAME_COUNT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_GET_PTS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_GET_SIZE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_GET_STATUS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_PLAY:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_SELECT_SOURCE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_SET_BLANK:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_SET_DISPLAY_FORMAT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_SET_FORMAT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_SET_STREAMTYPE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_SLOWMOTION:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_STILLPICTURE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_STOP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDEO_TRY_COMMAND:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dvb_video:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _video_function_calls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _video_types:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _fdl:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _gen_errors:
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later
+
+========
+Glossary
+========
+
+.. note::
+
+ The goal of this section is to standardize the terms used within the media
+ userspace API documentation. This is Work In Progress.
+
+.. Please keep the glossary entries in alphabetical order
+
+.. glossary::
+
+ Bridge Driver
+ A :term:`Device Driver` that implements the main logic to talk with
+ media hardware.
+
+ CEC API
+ **Consumer Electronics Control API**
+
+ An API designed to receive and transmit data via an HDMI
+ CEC interface.
+
+ See :ref:`cec`.
+
+ Device Driver
+ Part of the Linux Kernel that implements support for a hardware
+ component.
+
+ Device Node
+ A character device node in the file system used to control and
+ transfer data in and out of a Kernel driver.
+
+ Digital TV API
+ **Previously known as DVB API**
+
+ An API designed to control a subset of the :term:`Media Hardware`
+ that implements digital TV (e. g. DVB, ATSC, ISDB, etc).
+
+ See :ref:`dvbapi`.
+
+ DSP
+ **Digital Signal Processor**
+
+ A specialized :term:`Microprocessor`, with its architecture
+ optimized for the operational needs of digital signal processing.
+
+ FPGA
+ **Field-programmable Gate Array**
+
+ An :term:`IC` circuit designed to be configured by a customer or
+ a designer after manufacturing.
+
+ See https://en.wikipedia.org/wiki/Field-programmable_gate_array.
+
+ Hardware Component
+ A subset of the :term:`Media Hardware`. For example an :term:`I²C` or
+ :term:`SPI` device, or an :term:`IP Block` inside an
+ :term:`SoC` or :term:`FPGA`.
+
+ Hardware Peripheral
+ A group of :term:`hardware components <Hardware Component>` that
+ together make a larger user-facing functional peripheral. For
+ instance, the :term:`SoC` :term:`ISP` :term:`IP Block`
+ and the external camera sensors together make a camera hardware
+ peripheral.
+
+ Also known as :term:`Peripheral`.
+
+ I²C
+ **Inter-Integrated Circuit**
+
+ A multi-master, multi-slave, packet switched, single-ended,
+ serial computer bus used to control some hardware components
+ like sub-device hardware components.
+
+ See http://www.nxp.com/docs/en/user-guide/UM10204.pdf.
+
+ IC
+ **Integrated circuit**
+
+ A set of electronic circuits on one small flat piece of
+ semiconductor material, normally silicon.
+
+ Also known as chip.
+
+ IP Block
+ **Intellectual property core**
+
+ In electronic design a semiconductor intellectual property core,
+ is a reusable unit of logic, cell, or integrated circuit layout
+ design that is the intellectual property of one party.
+ IP Blocks may be licensed to another party or can be owned
+ and used by a single party alone.
+
+ See https://en.wikipedia.org/wiki/Semiconductor_intellectual_property_core).
+
+ ISP
+ **Image Signal Processor**
+
+ A specialized processor that implements a set of algorithms for
+ processing image data. ISPs may implement algorithms for lens
+ shading correction, demosaicing, scaling and pixel format conversion
+ as well as produce statistics for the use of the control
+ algorithms (e.g. automatic exposure, white balance and focus).
+
+ Media API
+ A set of userspace APIs used to control the media hardware. It is
+ composed by:
+
+ - :term:`CEC API`;
+ - :term:`Digital TV API`;
+ - :term:`MC API`;
+ - :term:`RC API`; and
+ - :term:`V4L2 API`.
+
+ See :doc:`index`.
+
+ MC API
+ **Media Controller API**
+
+ An API designed to expose and control the relationships between
+ multimedia devices and sub-devices.
+
+ See :ref:`media_controller`.
+
+ MC-centric
+ :term:`V4L2 Hardware` device driver that requires :term:`MC API`.
+
+ Such drivers have ``V4L2_CAP_IO_MC`` device_caps field set
+ (see :ref:`VIDIOC_QUERYCAP`).
+
+ See :ref:`v4l2_hardware_control` for more details.
+
+ Media Hardware
+ Subset of the hardware that is supported by the Linux Media API.
+
+ This includes audio and video capture and playback hardware,
+ digital and analog TV, camera sensors, ISPs, remote controllers,
+ codecs, HDMI Consumer Electronics Control, HDMI capture, etc.
+
+ Microprocessor
+ Electronic circuitry that carries out the instructions of a
+ computer program by performing the basic arithmetic, logical,
+ control and input/output (I/O) operations specified by the
+ instructions on a single integrated circuit.
+
+ Peripheral
+ The same as :term:`Hardware Peripheral`.
+
+ RC API
+ **Remote Controller API**
+
+ An API designed to receive and transmit data from remote
+ controllers.
+
+ See :ref:`remote_controllers`.
+
+ SMBus
+ A subset of I²C, which defines a stricter usage of the bus.
+
+ SPI
+ **Serial Peripheral Interface Bus**
+
+ Synchronous serial communication interface specification used for
+ short distance communication, primarily in embedded systems.
+
+ SoC
+ **System on a Chip**
+
+ An integrated circuit that integrates all components of a computer
+ or other electronic systems.
+
+ V4L2 API
+ **V4L2 userspace API**
+
+ The userspace API defined in :ref:`v4l2spec`, which is used to
+ control a V4L2 hardware.
+
+ V4L2 Device Node
+ A :term:`Device Node` that is associated to a V4L driver.
+
+ The V4L2 device node naming is specified at :ref:`v4l2_device_naming`.
+
+ V4L2 Hardware
+ Part of the media hardware which is supported by the :term:`V4L2 API`.
+
+ V4L2 Sub-device
+ V4L2 hardware components that aren't controlled by a
+ :term:`Bridge Driver`. See :ref:`subdev`.
+
+ Video-node-centric
+ V4L2 device driver that doesn't require a media controller to be used.
+
+ Such drivers have the ``V4L2_CAP_IO_MC`` device_caps field unset
+ (see :ref:`VIDIOC_QUERYCAP`).
+
+ V4L2 Sub-device API
+ Part of the :term:`V4L2 API` which control
+ :term:`V4L2 sub-devices <V4L2 Sub-device>`, like sensors,
+ HDMI receivers, scalers, deinterlacers.
+
+ See :ref:`v4l2_hardware_control` for more details.
mediactl/media-controller
cec/cec-api
gen-errors
+
+ glossary
+
fdl-appendix
drivers/index
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media-controller-intro:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media-controller-model:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
-
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. include:: <isonum.txt>
.. _media_controller:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media-func-close:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media-func-ioctl:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media-func-open:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media-user-func:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media_header:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media_ioc_device_info:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media_ioc_enum_entities:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media_ioc_enum_links:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media_ioc_g_topology:
-.. This file is dual-licensed: you can use it either under the terms
-.. of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
-.. dual licensing only applies to this file, and not this project as a
-.. whole.
-..
-.. a) This file is free software; you can redistribute it and/or
-.. modify it under the terms of the GNU General Public License as
-.. published by the Free Software Foundation version 2 of
-.. the License.
-..
-.. This file is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License for more details.
-..
-.. Or, alternatively,
-..
-.. b) Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later
.. _media_ioc_request_alloc:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media_ioc_setup_link:
-.. This file is dual-licensed: you can use it either under the terms
-.. of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
-.. dual licensing only applies to this file, and not this project as a
-.. whole.
-..
-.. a) This file is free software; you can redistribute it and/or
-.. modify it under the terms of the GNU General Public License as
-.. published by the Free Software Foundation version 2 of
-.. the License.
-..
-.. This file is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License for more details.
-..
-.. Or, alternatively,
-..
-.. b) Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later
.. _media_request_ioc_queue:
-.. This file is dual-licensed: you can use it either under the terms
-.. of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
-.. dual licensing only applies to this file, and not this project as a
-.. whole.
-..
-.. a) This file is free software; you can redistribute it and/or
-.. modify it under the terms of the GNU General Public License as
-.. published by the Free Software Foundation version 2 of
-.. the License.
-..
-.. This file is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License for more details.
-..
-.. Or, alternatively,
-..
-.. b) Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later
.. _media_request_ioc_reinit:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _media-controller-types:
-.. This file is dual-licensed: you can use it either under the terms
-.. of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
-.. dual licensing only applies to this file, and not this project as a
-.. whole.
-..
-.. a) This file is free software; you can redistribute it and/or
-.. modify it under the terms of the GNU General Public License as
-.. published by the Free Software Foundation version 2 of
-.. the License.
-..
-.. This file is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License for more details.
-..
-.. Or, alternatively,
-..
-.. b) Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later
.. _media-request-api:
-.. This file is dual-licensed: you can use it either under the terms
-.. of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
-.. dual licensing only applies to this file, and not this project as a
-.. whole.
-..
-.. a) This file is free software; you can redistribute it and/or
-.. modify it under the terms of the GNU General Public License as
-.. published by the Free Software Foundation version 2 of
-.. the License.
-..
-.. This file is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License for more details.
-..
-.. Or, alternatively,
-..
-.. b) Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later
.. _request-func-close:
-.. This file is dual-licensed: you can use it either under the terms
-.. of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
-.. dual licensing only applies to this file, and not this project as a
-.. whole.
-..
-.. a) This file is free software; you can redistribute it and/or
-.. modify it under the terms of the GNU General Public License as
-.. published by the Free Software Foundation version 2 of
-.. the License.
-..
-.. This file is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License for more details.
-..
-.. Or, alternatively,
-..
-.. b) Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later
.. _request-func-ioctl:
-.. This file is dual-licensed: you can use it either under the terms
-.. of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
-.. dual licensing only applies to this file, and not this project as a
-.. whole.
-..
-.. a) This file is free software; you can redistribute it and/or
-.. modify it under the terms of the GNU General Public License as
-.. published by the Free Software Foundation version 2 of
-.. the License.
-..
-.. This file is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License for more details.
-..
-.. Or, alternatively,
-..
-.. b) Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later
.. _request-func-poll:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
file: uapi/v4l/keytable.c
=========================
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_dev_intro:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_dev:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_func:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_get_features:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_get_rec_mode:
.. _lirc_set_rec_mode:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_get_rec_resolution:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_get_send_mode:
.. _lirc_set_send_mode:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_get_min_timeout:
.. _lirc_get_max_timeout:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_header:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc-read:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_set_measure_carrier_mode:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_set_rec_carrier_range:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_set_rec_carrier:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_set_rec_timeout_reports:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_set_rec_timeout:
.. _lirc_get_rec_timeout:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_set_send_carrier:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_set_send_duty_cycle:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_set_transmitter_mask:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc_set_wideband_receiver:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _lirc-write:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _Remote_controllers_Intro:
-.. SPDX-License-Identifier: GPL-2.0
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _Remote_controllers_Protocols:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _remote_controllers_sysfs_nodes:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _Remote_controllers_table_change:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _Remote_controllers_tables:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
-
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. include:: <isonum.txt>
.. _remote_controllers:
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later -->
<svg id="svg2" width="235mm" height="179mm" clip-path="url(#a)" fill-rule="evenodd" stroke-linejoin="round" stroke-width="28.222" preserveAspectRatio="xMidYMid" version="1.2" viewBox="0 0 22648.239 17899.829" xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><metadata id="metadata1533"><rdf:RDF><cc:Work rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/><dc:title/></cc:Work></rdf:RDF></metadata><defs id="defs4"><clipPath id="a"><rect id="rect7" width="28000" height="21000"/></clipPath></defs><path id="path11" d="m10146 2636c-518.06 0-1035.1 515-1035.1 1031v4124c0 516 517.06 1032 1035.1 1032h8572.2c518.06 0 1036.1-516 1036.1-1032v-4124c0-516-518.06-1031-1036.1-1031h-8572.2z"
fill="#fcf" style=""/><path id="path15" d="m1505.5 13443c-293 0-585 292-585 585v2340c0 293 292 586 585 586h3275c293 0 586-293 586-586v-2340c0-293-293-585-586-585h-3275z" fill="#ffc" style=""/><path id="path19" d="m517.15 22.013c-461 0-922 461-922 922v11169c0 461 461 923 922 923h3692c461 0 922-462 922-923v-11169c0-461-461-922-922-922h-3692z" fill="#e6e6e6" style=""/><path id="path23" d="m2371.5 6438h-2260v-1086h4520v1086h-2260z" fill="#ff8080" style=""/><path id="path25" d="m2371.5 6438h-2260v-1086h4520v1086h-2260z" fill="none" stroke="#3465af" style=""/><text id="text27" class="TextShape" x="-2089.4541" y="-2163.9871" font-family="Serif, serif" font-size="493.88px"><tspan id="tspan29" class="TextParagraph" font-family="Serif, serif" font-size="493.88px"><tspan id="tspan31" class="TextPosition" x="489.5459" y="6111.0132" font-family="Serif, serif" font-size="493.88px"><tspan id="tspan33"
fill="#000000" font-family="Serif, serif" font-size="493.88px">Audio decoder</tspan></tspan></tspan></text>
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _app-pri:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _async:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _audio:
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- This file is dual-licensed: you can use it either under the terms
- of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
- dual licensing only applies to this file, and not this project as a
- whole.
-
- a) This file is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation version 2 of
- the License.
-
- This file is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- Or, alternatively,
-
- b) Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later -->
<svg id="svg2" width="164.15mm" height="46.771mm" fill-rule="evenodd" stroke-linejoin="round" stroke-width="28.222" preserveAspectRatio="xMidYMid" version="1.2" viewBox="0 0 16415.333 4677.1107" xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><metadata id="metadata652"><rdf:RDF><cc:Work rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/><dc:title/></cc:Work></rdf:RDF></metadata><g id="g186" class="com.sun.star.drawing.CustomShape" transform="translate(-3285.9 -3185.9)"><g id="id6"><rect id="rect189" class="BoundingBox" x="3299" y="3199" width="1303" height="1203" fill="none"/><path id="path191" d="m3950 4400h-650v-1200h1300v1200h-650z" fill="#00f"/><path id="path193" d="m3950
4400h-650v-1200h1300v1200h-650z" fill="none" stroke="#3465a4"/><text id="text195" class="TextShape"><tspan id="tspan197" class="TextParagraph" font-family="sans-serif" font-size="635px" font-weight="400"><tspan id="tspan199" class="TextPosition" x="3739" y="4021"><tspan id="tspan201" fill="#ffffff">B</tspan></tspan></tspan></text>
</g></g><g id="g203" class="com.sun.star.drawing.CustomShape" transform="translate(-3285.9 -3185.9)"><g id="id7"><rect id="rect206" class="BoundingBox" x="4599" y="3199" width="1303" height="1203" fill="none"/><path id="path208" d="m5250 4400h-650v-1200h1300v1200h-650z" fill="#0c0"/><path id="path210" d="m5250 4400h-650v-1200h1300v1200h-650z" fill="none" stroke="#3465a4"/><text id="text212" class="TextShape"><tspan id="tspan214" class="TextParagraph" font-family="sans-serif" font-size="635px" font-weight="400"><tspan id="tspan216" class="TextPosition" x="5003" y="4021"><tspan id="tspan218" fill="#ffffff">G</tspan></tspan></tspan></text>
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
**********
References
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _buffer:
:stub-columns: 0
:widths: 3 1 4
- * .. _`V4L2-FLAG-MEMORY-NON-CONSISTENT`:
-
- - ``V4L2_FLAG_MEMORY_NON_CONSISTENT``
- - 0x00000001
- - A buffer is allocated either in consistent (it will be automatically
- coherent between the CPU and the bus) or non-consistent memory. The
- latter can provide performance gains, for instance the CPU cache
- sync/flush operations can be avoided if the buffer is accessed by the
- corresponding device only and the CPU does not read/write to/from that
- buffer. However, this requires extra care from the driver -- it must
- guarantee memory consistency by issuing a cache flush/sync when
- consistency is needed. If this flag is set V4L2 will attempt to
- allocate the buffer in non-consistent memory. The flag takes effect
- only if the buffer is used for :ref:`memory mapping <mmap>` I/O and the
- queue reports the :ref:`V4L2_BUF_CAP_SUPPORTS_MMAP_CACHE_HINTS
- <V4L2-BUF-CAP-SUPPORTS-MMAP-CACHE-HINTS>` capability.
-
.. c:type:: v4l2_memory
enum v4l2_memory
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _capture-example:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
file: media/v4l/capture.c
=========================
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
****************************
Defining Colorspaces in V4L2
:c:type:`v4l2_hsv_encoding` specifies which encoding is used.
.. note:: The default R'G'B' quantization is full range for all
- colorspaces except for BT.2020 which uses limited range R'G'B'
- quantization.
+ colorspaces. HSV formats are always full range.
.. tabularcolumns:: |p{6.7cm}|p{10.8cm}|
- Details
* - ``V4L2_QUANTIZATION_DEFAULT``
- Use the default quantization encoding as defined by the
- colorspace. This is always full range for R'G'B' (except for the
- BT.2020 colorspace) and HSV. It is usually limited range for Y'CbCr.
+ colorspace. This is always full range for R'G'B' and HSV.
+ It is usually limited range for Y'CbCr.
* - ``V4L2_QUANTIZATION_FULL_RANGE``
- Use the full range quantization encoding. I.e. the range [0…1] is
mapped to [0…255] (with possible clipping to [1…254] to avoid the
* - ``V4L2_QUANTIZATION_LIM_RANGE``
- Use the limited range quantization encoding. I.e. the range [0…1]
is mapped to [16…235]. Cb and Cr are mapped from [-0.5…0.5] to
- [16…240].
+ [16…240]. Limited Range cannot be used with HSV.
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
********************************
Detailed Colorspace Descriptions
The :ref:`itu2020` standard defines the colorspace used by Ultra-high
definition television (UHDTV). The default transfer function is
``V4L2_XFER_FUNC_709``. The default Y'CbCr encoding is
-``V4L2_YCBCR_ENC_BT2020``. The default R'G'B' quantization is limited
-range (!), and so is the default Y'CbCr quantization. The chromaticities
-of the primary colors and the white reference are:
+``V4L2_YCBCR_ENC_BT2020``. The default Y'CbCr quantization is limited range.
+The chromaticities of the primary colors and the white reference are:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _colorspaces:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _common-defs:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _common:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _compat:
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- This file is dual-licensed: you can use it either under the terms
- of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
- dual licensing only applies to this file, and not this project as a
- whole.
-
- a) This file is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation version 2 of
- the License.
-
- This file is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- Or, alternatively,
-
- b) Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later -->
<svg id="svg2" width="249.01mm" height="143.01mm" fill-rule="evenodd" stroke-linejoin="round" stroke-width="28.222" preserveAspectRatio="xMidYMid" version="1.2" viewBox="0 0 24900.998 14300.999" xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><metadata id="metadata325"><rdf:RDF><cc:Work rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/><dc:title/></cc:Work></rdf:RDF></metadata><defs id="defs4" class="ClipPathGroup"><marker id="marker6261" overflow="visible" orient="auto"><path id="path6263" transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5-5-5z" fill="#f00" fill-rule="evenodd" stroke="#f00" stroke-width="1pt"/></marker><marker id="marker6125" overflow="visible"
orient="auto"><path id="path6127" transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5-5-5z" fill="#f00" fill-rule="evenodd" stroke="#f00" stroke-width="1pt"/></marker><marker id="marker6001" overflow="visible" orient="auto"><path id="path6003" transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5-5-5z" fill="#f00" fill-rule="evenodd" stroke="#f00" stroke-width="1pt"/></marker><marker id="marker5693" overflow="visible" orient="auto"><path id="path5695" transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5-5-5z" fill="#f00" fill-rule="evenodd" stroke="#f00" stroke-width="1pt"/></marker><marker id="marker5575" overflow="visible" orient="auto"><path id="path5577" transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5-5-5z" fill="#000080" fill-rule="evenodd" stroke="#000080" stroke-width="1pt"/></marker><marker id="marker5469" overflow="visible"
orient="auto"><path id="path5471" transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5-5-5z" fill="#000080" fill-rule="evenodd" stroke="#000080" stroke-width="1pt"/></marker><marker id="marker5259" overflow="visible" orient="auto"><path id="path5261" transform="matrix(-.4 0 0 -.4 -4 0)" d="m0 0 5-5-17.5 5 17.5 5-5-5z" fill="#000080" fill-rule="evenodd" stroke="#000080" stroke-width="1pt"/></marker><marker id="Arrow2Mend" overflow="visible" orient="auto"><path id="path4241" transform="scale(-.6)" d="m8.7186 4.0337-10.926-4.0177 10.926-4.0177c-1.7455 2.3721-1.7354 5.6175-6e-7 8.0354z" fill="#000080" fill-rule="evenodd" stroke="#000080" stroke-linejoin="round" stroke-width=".625"/></marker></defs><g id="g204" class="com.sun.star.drawing.CustomShape" transform="translate(-1350,-3250)"><g id="id6"><rect id="rect207" class="BoundingBox" x="1350" y="3250" width="24901" height="14301"
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _control:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _crop:
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
- Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _depth-formats:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _capture:
-.. This file is dual-licensed: you can use it either under the terms
-.. of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
-.. dual licensing only applies to this file, and not this project as a
-.. whole.
-..
-.. a) This file is free software; you can redistribute it and/or
-.. modify it under the terms of the GNU General Public License as
-.. published by the Free Software Foundation version 2 of
-.. the License.
-..
-.. This file is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License for more details.
-..
-.. Or, alternatively,
-..
-.. b) Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later
.. _encoder:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _event:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _mem2mem:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _metadata:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _osd:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _output:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _overlay:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _radio:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _raw-vbi:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _rds:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _sdr:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _sliced:
:stub-columns: 0
:widths: 3 3 2 2 2
- * - __u32
+ * - __u16
- ``service_set``
- :cspan:`2`
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _subdev:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _touch:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _devices:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _diff-v4l:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dmabuf:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dv-timings:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _camera-controls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _mpeg-controls:
- Variable bitrate
* - ``V4L2_MPEG_VIDEO_BITRATE_MODE_CBR``
- Constant bitrate
+ * - ``V4L2_MPEG_VIDEO_BITRATE_MODE_CQ``
+ - Constant quality
the average video bitrate. It is ignored if the video bitrate mode
is set to constant bitrate.
+``V4L2_CID_MPEG_VIDEO_CONSTANT_QUALITY (integer)``
+ Constant quality level control. This control is applicable when
+ ``V4L2_CID_MPEG_VIDEO_BITRATE_MODE`` value is
+ ``V4L2_MPEG_VIDEO_BITRATE_MODE_CQ``. Valid range is 1 to 100
+ where 1 indicates lowest quality and 100 indicates highest quality.
+ Encoder will decide the appropriate quantization parameter and
+ bitrate to produce requested frame quality.
+
+
+``V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE (enum)``
+
+enum v4l2_mpeg_video_frame_skip_mode -
+ Indicates in what conditions the encoder should skip frames. If
+ encoding a frame would cause the encoded stream to be larger then a
+ chosen data limit then the frame will be skipped. Possible values
+ are:
+
+
+.. tabularcolumns:: |p{9.2cm}|p{8.3cm}|
+
+.. raw:: latex
+
+ \small
+
+.. flat-table::
+ :header-rows: 0
+ :stub-columns: 0
+
+ * - ``V4L2_MPEG_FRAME_SKIP_MODE_DISABLED``
+ - Frame skip mode is disabled.
+ * - ``V4L2_MPEG_FRAME_SKIP_MODE_LEVEL_LIMIT``
+ - Frame skip mode enabled and buffer limit is set by the chosen
+ level and is defined by the standard.
+ * - ``V4L2_MPEG_FRAME_SKIP_MODE_BUF_LIMIT``
+ - Frame skip mode enabled and buffer limit is set by the
+ :ref:`VBV (MPEG1/2/4) <v4l2-mpeg-video-vbv-size>` or
+ :ref:`CPB (H264) buffer size <v4l2-mpeg-video-h264-cpb-size>` control.
+
+.. raw:: latex
+
+ \normalsize
+
``V4L2_CID_MPEG_VIDEO_TEMPORAL_DECIMATION (integer)``
For every captured frame, skip this many subsequent frames (default
0).
Quantization parameter for an B frame for MPEG4. Valid range: from 1
to 31.
+.. _v4l2-mpeg-video-vbv-size:
+
``V4L2_CID_MPEG_VIDEO_VBV_SIZE (integer)``
The Video Buffer Verifier size in kilobytes, it is used as a
limitation of frame skip. The VBV is defined in the standard as a
Force a key frame for the next queued buffer. Applicable to
encoders. This is a general, codec-agnostic keyframe control.
+.. _v4l2-mpeg-video-h264-cpb-size:
+
``V4L2_CID_MPEG_VIDEO_H264_CPB_SIZE (integer)``
The Coded Picture Buffer size in kilobytes, it is used as a
limitation of frame skip. The CPB is defined in the H264 standard as
* - ``V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE``
- 0x00000040
-
- * - ``V4L2_H264_PPS_FLAG_PIC_SCALING_MATRIX_PRESENT``
+ * - ``V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT``
- 0x00000080
- -
+ - Indicates that ``V4L2_CID_MPEG_VIDEO_H264_SCALING_MATRIX``
+ must be used for this picture.
``V4L2_CID_MPEG_VIDEO_H264_SCALING_MATRIX (struct)``
Specifies the scaling matrix (as extracted from the bitstream) for
- ``scaling_list_4x4[6][16]``
- Scaling matrix after applying the inverse scanning process.
Expected list order is Intra Y, Intra Cb, Intra Cr, Inter Y,
- Inter Cb, Inter Cr.
+ Inter Cb, Inter Cr. The values on each scaling list are
+ expected in raster scan order.
* - __u8
- ``scaling_list_8x8[6][64]``
- Scaling matrix after applying the inverse scanning process.
Expected list order is Intra Y, Inter Y, Intra Cb, Inter Cb,
- Intra Cr, Inter Cr.
+ Intra Cr, Inter Cr. The values on each scaling list are
+ expected in raster scan order.
``V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS (struct)``
Specifies the slice parameters (as extracted from the bitstream)
This compound control is not yet part of the public kernel API
and it is expected to change.
- This structure is expected to be passed as an array, with one
- entry for each slice included in the bitstream buffer.
-
.. c:type:: v4l2_ctrl_h264_slice_params
.. cssclass:: longtable
:stub-columns: 0
:widths: 1 1 2
- * - __u32
- - ``size``
- -
- * - __u32
- - ``start_byte_offset``
- Offset (in bytes) from the beginning of the OUTPUT buffer to the start
- of the slice. If the slice starts with a start code, then this is the
- offset to such start code. When operating in slice-based decoding mode
- (see :c:type:`v4l2_mpeg_video_h264_decode_mode`), this field should
- be set to 0. When operating in frame-based decoding mode, this field
- should be 0 for the first slice.
* - __u32
- ``header_bit_size``
- -
- * - __u16
+ - Offset in bits to slice_data() from the beginning of this slice.
+ * - __u32
- ``first_mb_in_slice``
-
* - __u8
- ``slice_type``
-
- * - __u8
- - ``pic_parameter_set_id``
- -
* - __u8
- ``colour_plane_id``
-
* - __u8
- ``redundant_pic_cnt``
-
- * - __u16
- - ``frame_num``
- -
- * - __u16
- - ``idr_pic_id``
- -
- * - __u16
- - ``pic_order_cnt_lsb``
- -
- * - __s32
- - ``delta_pic_order_cnt_bottom``
- -
- * - __s32
- - ``delta_pic_order_cnt0``
- -
- * - __s32
- - ``delta_pic_order_cnt1``
- -
- * - struct :c:type:`v4l2_h264_pred_weight_table`
- - ``pred_weight_table``
- -
- * - __u32
- - ``dec_ref_pic_marking_bit_size``
- - Size in bits of the dec_ref_pic_marking() syntax element.
- * - __u32
- - ``pic_order_cnt_bit_size``
- -
* - __u8
- ``cabac_init_idc``
-
- ``num_ref_idx_l1_active_minus1``
- If num_ref_idx_active_override_flag is not set, this field must be
set to the value of num_ref_idx_l1_default_active_minus1.
- * - __u32
- - ``slice_group_change_cycle``
- -
* - __u8
+ - ``reserved``
+ - Applications and drivers must set this to zero.
+ * - struct :c:type:`v4l2_h264_reference`
- ``ref_pic_list0[32]``
- Reference picture list after applying the per-slice modifications
- * - __u8
+ * - struct :c:type:`v4l2_h264_reference`
- ``ref_pic_list1[32]``
- Reference picture list after applying the per-slice modifications
* - __u32
:stub-columns: 0
:widths: 1 1 2
- * - ``V4L2_H264_SLICE_FLAG_FIELD_PIC``
- - 0x00000001
- -
- * - ``V4L2_H264_SLICE_FLAG_BOTTOM_FIELD``
- - 0x00000002
- -
* - ``V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED``
- - 0x00000004
+ - 0x00000001
-
* - ``V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH``
- - 0x00000008
+ - 0x00000002
-
-``Prediction Weight Table``
+``V4L2_CID_MPEG_VIDEO_H264_PRED_WEIGHTS (struct)``
+ Prediction weight table defined according to :ref:`h264`,
+ section 7.4.3.2 "Prediction Weight Table Semantics".
+ The prediction weight table must be passed by applications
+ under the conditions explained in section 7.3.3 "Slice header
+ syntax".
- The bitstream parameters are defined according to :ref:`h264`,
- section 7.4.3.2 "Prediction Weight Table Semantics". For further
- documentation, refer to the above specification, unless there is
- an explicit comment stating otherwise.
+ .. note::
+
+ This compound control is not yet part of the public kernel API and
+ it is expected to change.
-.. c:type:: v4l2_h264_pred_weight_table
+.. c:type:: v4l2_ctrl_h264_pred_weights
.. cssclass:: longtable
-.. flat-table:: struct v4l2_h264_pred_weight_table
+.. flat-table:: struct v4l2_ctrl_h264_pred_weights
:header-rows: 0
:stub-columns: 0
:widths: 1 1 2
- ``chroma_offset[32][2]``
-
+``Picture Reference``
+
+.. c:type:: v4l2_h264_reference
+
+.. cssclass:: longtable
+
+.. flat-table:: struct v4l2_h264_reference
+ :header-rows: 0
+ :stub-columns: 0
+ :widths: 1 1 2
+
+ * - __u8
+ - ``fields``
+ - Specifies how the picture is referenced. See :ref:`Reference Fields <h264_ref_fields>`
+ * - __u8
+ - ``index``
+ - Index into the :c:type:`v4l2_ctrl_h264_decode_params`.dpb array.
+
+.. _h264_ref_fields:
+
+``Reference Fields``
+
+.. cssclass:: longtable
+
+.. flat-table::
+ :header-rows: 0
+ :stub-columns: 0
+ :widths: 1 1 2
+
+ * - ``V4L2_H264_TOP_FIELD_REF``
+ - 0x1
+ - The top field in field pair is used for short-term reference.
+ * - ``V4L2_H264_BOTTOM_FIELD_REF``
+ - 0x2
+ - The bottom field in field pair is used for short-term reference.
+ * - ``V4L2_H264_FRAME_REF``
+ - 0x3
+ - The frame (or the top/bottom fields, if it's a field pair)
+ is used for short-term reference.
+
``V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS (struct)``
Specifies the decode parameters (as extracted from the bitstream)
for the associated H264 slice data. This includes the necessary
* - struct :c:type:`v4l2_h264_dpb_entry`
- ``dpb[16]``
-
- * - __u16
- - ``num_slices``
- - Number of slices needed to decode the current frame/field. When
- operating in slice-based decoding mode (see
- :c:type:`v4l2_mpeg_video_h264_decode_mode`), this field
- should always be set to one.
* - __u16
- ``nal_ref_idc``
- NAL reference ID value coming from the NAL Unit header
+ * - __u16
+ - ``frame_num``
+ -
* - __s32
- ``top_field_order_cnt``
- Picture Order Count for the coded top field
* - __s32
- ``bottom_field_order_cnt``
- Picture Order Count for the coded bottom field
+ * - __u16
+ - ``idr_pic_id``
+ -
+ * - __u16
+ - ``pic_order_cnt_lsb``
+ -
+ * - __s32
+ - ``delta_pic_order_cnt_bottom``
+ -
+ * - __s32
+ - ``delta_pic_order_cnt0``
+ -
+ * - __s32
+ - ``delta_pic_order_cnt1``
+ -
+ * - __u32
+ - ``dec_ref_pic_marking_bit_size``
+ - Size in bits of the dec_ref_pic_marking() syntax element.
+ * - __u32
+ - ``pic_order_cnt_bit_size``
+ - Combined size in bits of the picture order count related syntax
+ elements: pic_order_cnt_lsb, delta_pic_order_cnt_bottom,
+ delta_pic_order_cnt0, and delta_pic_order_cnt1.
+ * - __u32
+ - ``slice_group_change_cycle``
+ -
+ * - __u32
+ - ``reserved``
+ - Applications and drivers must set this to zero.
* - __u32
- ``flags``
- See :ref:`Decode Parameters Flags <h264_decode_params_flags>`
* - ``V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC``
- 0x00000001
- That picture is an IDR picture
+ * - ``V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC``
+ - 0x00000002
+ -
+ * - ``V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD``
+ - 0x00000004
+ -
.. c:type:: v4l2_h264_dpb_entry
``timestamp`` field in struct :c:type:`v4l2_buffer`. Use the
:c:func:`v4l2_timeval_to_ns()` function to convert the struct
:c:type:`timeval` in struct :c:type:`v4l2_buffer` to a __u64.
- * - __u16
- - ``frame_num``
+ * - __u32
+ - ``pic_num``
-
* - __u16
- - ``pic_num``
+ - ``frame_num``
-
+ * - __u8
+ - ``fields``
+ - Specifies how the DPB entry is referenced. See :ref:`Reference Fields <h264_ref_fields>`
+ * - __u8
+ - ``reserved[5]``
+ - Applications and drivers must set this to zero.
* - __s32
- ``top_field_order_cnt``
-
* - ``V4L2_H264_DPB_ENTRY_FLAG_VALID``
- 0x00000001
- - The DPB entry is valid and should be considered
+ - The DPB entry is valid (non-empty) and should be considered.
* - ``V4L2_H264_DPB_ENTRY_FLAG_ACTIVE``
- 0x00000002
- - The DPB entry is currently being used as a reference frame
+ - The DPB entry is used for reference.
* - ``V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM``
- 0x00000004
- - The DPB entry is a long term reference frame
+ - The DPB entry is used for long-term reference.
* - ``V4L2_H264_DPB_ENTRY_FLAG_FIELD``
- 0x00000008
- - The DPB entry is a field reference, which means only one of the field
- will be used when decoding the new frame/field. When not set the DPB
- entry is a frame reference (both fields will be used). Note that this
- flag does not say anything about the number of fields contained in the
- reference frame, it just describes the one used to decode the new
- field/frame
- * - ``V4L2_H264_DPB_ENTRY_FLAG_BOTTOM_FIELD``
- - 0x00000010
- - The DPB entry is a bottom field reference (only the bottom field of the
- reference frame is needed to decode the new frame/field). Only valid if
- V4L2_H264_DPB_ENTRY_FLAG_FIELD is set. When
- V4L2_H264_DPB_ENTRY_FLAG_FIELD is set but
- V4L2_H264_DPB_ENTRY_FLAG_BOTTOM_FIELD is not, that means the
- DPB entry is a top field reference
+ - The DPB entry is a single field or a complementary field pair.
``V4L2_CID_MPEG_VIDEO_H264_DECODE_MODE (enum)``
Specifies the decoding mode to use. Currently exposes slice-based and
* - ``V4L2_MPEG_VIDEO_H264_DECODE_MODE_SLICE_BASED``
- 0
- Decoding is done at the slice granularity.
- In this mode, ``num_slices`` field in struct
- :c:type:`v4l2_ctrl_h264_decode_params` should be set to 1,
- and ``start_byte_offset`` in struct
- :c:type:`v4l2_ctrl_h264_slice_params` should be set to 0.
The OUTPUT buffer must contain a single slice.
+ When this mode is selected, the ``V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS``
+ control shall be set. When multiple slices compose a frame,
+ use of ``V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF`` flag
+ is required.
* - ``V4L2_MPEG_VIDEO_H264_DECODE_MODE_FRAME_BASED``
- 1
- - Decoding is done at the frame granularity.
- In this mode, ``num_slices`` field in struct
- :c:type:`v4l2_ctrl_h264_decode_params` should be set to the number
- of slices in the frame, and ``start_byte_offset`` in struct
- :c:type:`v4l2_ctrl_h264_slice_params` should be set accordingly
- for each slice. For the first slice, ``start_byte_offset`` should
- be zero.
+ - Decoding is done at the frame granularity,
The OUTPUT buffer must contain all slices needed to decode the
frame. The OUTPUT buffer must also contain both fields.
+ This mode will be supported by devices that
+ parse the slice(s) header(s) in hardware. When this mode is
+ selected, the ``V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS``
+ control shall not be set.
``V4L2_CID_MPEG_VIDEO_H264_START_CODE (enum)``
Specifies the H264 slice start code expected for each slice.
``V4L2_CID_MPEG_MFC51_VIDEO_FRAME_SKIP_MODE``
(enum)
+ .. note::
+
+ This control is deprecated. Use the standard
+ ``V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE`` control instead.
+
enum v4l2_mpeg_mfc51_video_frame_skip_mode -
Indicates in what conditions the encoder should skip frames. If
encoding a frame would cause the encoded stream to be larger then a
* - ``V4L2_MPEG_VIDEO_VP9_PROFILE_3``
- Profile 3
+.. _v4l2-mpeg-video-vp9-level:
+
+``V4L2_CID_MPEG_VIDEO_VP9_LEVEL (enum)``
+
+enum v4l2_mpeg_video_vp9_level -
+ This control allows selecting the level for VP9 encoder.
+ This is also used to enumerate supported levels by VP9 encoder or decoder.
+ More information can be found at
+ `webmproject <https://www.webmproject.org/vp9/levels/>`__. Possible values are:
+
+.. flat-table::
+ :header-rows: 0
+ :stub-columns: 0
+
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_1_0``
+ - Level 1
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_1_1``
+ - Level 1.1
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_2_0``
+ - Level 2
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_2_1``
+ - Level 2.1
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_3_0``
+ - Level 3
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_3_1``
+ - Level 3.1
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_4_0``
+ - Level 4
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_4_1``
+ - Level 4.1
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_5_0``
+ - Level 5
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_5_1``
+ - Level 5.1
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_5_2``
+ - Level 5.2
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_6_0``
+ - Level 6
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_6_1``
+ - Level 6.1
+ * - ``V4L2_MPEG_VIDEO_VP9_LEVEL_6_2``
+ - Level 6.2
+
High Efficiency Video Coding (HEVC/H.265) Control Reference
===========================================================
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _detect-controls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _dv-controls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _flash-controls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _fm-rx-controls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _fm-tx-controls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _image-process-controls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _image-source-controls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _jpeg-controls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _rf-tuner-controls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _extended-controls:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _field-order:
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
- Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
- Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _format:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _func-close:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _func-ioctl:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _func-mmap:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _func-munmap:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _func-open:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _func-poll:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _func-read:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _func-select:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _func-write:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _hist-v4l2:
Codec API was released.
1998-11-08: Many minor changes. Most symbols have been renamed. Some
-material changes to struct :c:type:`v4l2_capability`.
+material changes to struct v4l2_capability.
1998-11-12: The read/write directon of some ioctls was misdefined.
to simplify the API, while making it more extensible and following
common Linux driver API conventions.
-1. Some typos in ``V4L2_FMT_FLAG`` symbols were fixed. struct
- :c:type:`v4l2_clip` was changed for compatibility with
- v4l. (1999-08-30)
+1. Some typos in ``V4L2_FMT_FLAG`` symbols were fixed. struct v4l2_clip
+ was changed for compatibility with v4l. (1999-08-30)
2. ``V4L2_TUNER_SUB_LANG1`` was added. (1999-09-05)
This change obsoletes the following ioctls: ``VIDIOC_S_INFMT``,
``VIDIOC_G_INFMT``, ``VIDIOC_S_OUTFMT``, ``VIDIOC_G_OUTFMT``,
``VIDIOC_S_VBIFMT`` and ``VIDIOC_G_VBIFMT``. The image format
- structure struct :c:type:`v4l2_format` was renamed to struct
- :c:type:`v4l2_pix_format`, while struct
- :c:type:`v4l2_format` is now the envelopping structure
+ struct v4l2_format was renamed to struct v4l2_pix_format, while
+ struct v4l2_format is now the envelopping structure
for all format negotiations.
5. Similar to the changes above, the ``VIDIOC_G_PARM`` and
``VIDIOC_S_PARM`` ioctls were merged with ``VIDIOC_G_OUTPARM`` and
- ``VIDIOC_S_OUTPARM``. A ``type`` field in the new struct
- :c:type:`v4l2_streamparm` selects the respective
- union member.
+ ``VIDIOC_S_OUTPARM``. A ``type`` field in the new struct v4l2_streamparm
+ selects the respective union member.
This change obsoletes the ``VIDIOC_G_OUTPARM`` and
``VIDIOC_S_OUTPARM`` ioctls.
categories might have a greater separation, or may even appear in
separate windows.
-7. The struct :c:type:`v4l2_buffer` ``timestamp`` was
+7. The struct v4l2_buffer ``timestamp`` was
changed to a 64 bit integer, containing the sampling or output time
of the frame in nanoseconds. Additionally timestamps will be in
absolute system time, not starting from zero at the beginning of a
v4l2_masterclock_gettime() function (used only by drivers) to
return a 64-bit integer.
-8. A ``sequence`` field was added to struct
- :c:type:`v4l2_buffer`. The ``sequence`` field counts
- captured frames, it is ignored by output devices. When a capture
- driver drops a frame, the sequence number of that frame is skipped.
+8. A ``sequence`` field was added to struct v4l2_buffer. The ``sequence``
+ field counts captured frames, it is ignored by output devices. When a
+ capture driver drops a frame, the sequence number of that frame is skipped.
V4L2 Version 0.20 incremental changes
=====================================
-1999-12-23: In struct :c:type:`v4l2_vbi_format` the
+1999-12-23: In struct v4l2_vbi_format the
``reserved1`` field became ``offset``. Previously drivers were required
to clear the ``reserved1`` field.
compatibility* as the :ref:`VIDIOC_G_FMT <VIDIOC_G_FMT>` and
:ref:`VIDIOC_S_FMT <VIDIOC_G_FMT>` ioctls may fail now if the
struct ``v4l2_fmt`` ``type`` field does not contain
-``V4L2_BUF_TYPE_VBI``. In the documentation of the struct
-:c:type:`v4l2_vbi_format` ``offset`` field the
-ambiguous phrase "rising edge" was changed to "leading edge".
-
+``V4L2_BUF_TYPE_VBI``. In the documentation of the struct v4l2_vbi_format`,
+the ``offset`` field the ambiguous phrase "rising edge" was changed to
+"leading edge".
V4L2 Version 0.20 2000-11-23
============================
until the application attempts to initiate a data exchange, see
:ref:`open`.
-3. The struct :c:type:`v4l2_capability` changed
+3. The struct v4l2_capability changed
dramatically. Note that also the size of the structure changed,
which is encoded in the ioctl request code, thus older V4L2 devices
will respond with an ``EINVAL`` error code to the new
``V4L2_FLAG_MONOCHROME`` flag was removed, this information is
available as described in :ref:`format`.
-4. In struct :c:type:`v4l2_input` the ``assoc_audio``
+4. In struct v4l2_input the ``assoc_audio``
field and the ``capability`` field and its only flag
``V4L2_INPUT_CAP_AUDIO`` was replaced by the new ``audioset`` field.
Instead of linking one video input to one audio input this field
New fields are ``tuner`` (reversing the former link from tuners to
video inputs), ``std`` and ``status``.
- Accordingly struct :c:type:`v4l2_output` lost its
+ Accordingly struct v4l2_output lost its
``capability`` and ``assoc_audio`` fields. ``audioset``,
``modulator`` and ``std`` where added instead.
-5. The struct :c:type:`v4l2_audio` field ``audio`` was
+5. The struct v4l2_audio field ``audio`` was
renamed to ``index``, for consistency with other structures. A new
capability flag ``V4L2_AUDCAP_STEREO`` was added to indicated if the
audio input in question supports stereo sound.
where removed. This can be easily implemented using controls.
(However the same applies to AVL which is still there.)
- Again for consistency the struct
- :c:type:`v4l2_audioout` field ``audio`` was renamed
+ Again for consistency the struct v4l2_audioout field ``audio`` was renamed
to ``index``.
-6. The struct :c:type:`v4l2_tuner` ``input`` field was
+6. The struct v4l2_tuner ``input`` field was
replaced by an ``index`` field, permitting devices with multiple
tuners. The link between video inputs and tuners is now reversed,
inputs point to their tuner. The ``std`` substructure became a
- simple set (more about this below) and moved into struct
- :c:type:`v4l2_input`. A ``type`` field was added.
+ simple set (more about this below) and moved into struct v4l2_input.
+ A ``type`` field was added.
- Accordingly in struct :c:type:`v4l2_modulator` the
+ Accordingly in struct v4l2_modulator the
``output`` was replaced by an ``index`` field.
- In struct :c:type:`v4l2_frequency` the ``port``
+ In struct v4l2_frequency the ``port``
field was replaced by a ``tuner`` field containing the respective
tuner or modulator index number. A tuner ``type`` field was added
and the ``reserved`` field became larger for future extensions
:ref:`VIDIOC_S_STD <VIDIOC_G_STD>` now take a pointer to this
type as argument. :ref:`VIDIOC_QUERYSTD` was
added to autodetect the received standard, if the hardware has this
- capability. In struct :c:type:`v4l2_standard` an
+ capability. In struct v4l2_standard an
``index`` field was added for
:ref:`VIDIOC_ENUMSTD`. A
:ref:`v4l2_std_id <v4l2-std-id>` field named ``id`` was added as
Struct ``v4l2_enumstd`` ceased to be.
:ref:`VIDIOC_ENUMSTD` now takes a pointer to a
- struct :c:type:`v4l2_standard` directly. The
+ struct v4l2_standard directly. The
information which standards are supported by a particular video
- input or output moved into struct :c:type:`v4l2_input`
- and struct :c:type:`v4l2_output` fields named ``std``,
+ input or output moved into struct v4l2_input
+ and struct v4l2_output fields named ``std``,
respectively.
8. The struct :ref:`v4l2_queryctrl <v4l2-queryctrl>` fields
:ref:`VIDIOC_S_FMT <VIDIOC_G_FMT>`, but without the overhead of
programming the hardware and regardless of I/O in progress.
- In struct :c:type:`v4l2_format` the ``fmt`` union was
- extended to contain struct :c:type:`v4l2_window`. All
+ In struct v4l2_format the ``fmt`` union was
+ extended to contain struct v4l2_window. All
image format negotiations are now possible with ``VIDIOC_G_FMT``,
``VIDIOC_S_FMT`` and ``VIDIOC_TRY_FMT``; ioctl. The ``VIDIOC_G_WIN``
and ``VIDIOC_S_WIN`` ioctls to prepare for a video overlay were
- removed. The ``type`` field changed to type enum
- :c:type:`v4l2_buf_type` and the buffer type names
- changed as follows.
+ removed. The ``type`` field changed to type enum v4l2_buf_type and
+ the buffer type names changed as follows.
:stub-columns: 0
* - Old defines
- - enum :c:type:`v4l2_buf_type`
+ - enum v4l2_buf_type
* - ``V4L2_BUF_TYPE_CAPTURE``
- ``V4L2_BUF_TYPE_VIDEO_CAPTURE``
* - ``V4L2_BUF_TYPE_CODECIN``
* - ``V4L2_BUF_TYPE_PRIVATE_BASE``
- ``V4L2_BUF_TYPE_PRIVATE`` (but this is deprecated)
+10. In struct v4l2_fmtdesc a enum v4l2_buf_type field named ``type`` was
+ added as in struct v4l2_format. The ``VIDIOC_ENUM_FBUFFMT`` ioctl is no
+ longer needed and was removed. These calls can be replaced by
+ :ref:`VIDIOC_ENUM_FMT` with type ``V4L2_BUF_TYPE_VIDEO_OVERLAY``.
-10. In struct :c:type:`v4l2_fmtdesc` a enum
- :c:type:`v4l2_buf_type` field named ``type`` was
- added as in struct :c:type:`v4l2_format`. The
- ``VIDIOC_ENUM_FBUFFMT`` ioctl is no longer needed and was removed.
- These calls can be replaced by
- :ref:`VIDIOC_ENUM_FMT` with type
- ``V4L2_BUF_TYPE_VIDEO_OVERLAY``.
-
-11. In struct :c:type:`v4l2_pix_format` the ``depth``
+11. In struct v4l2_pix_format the ``depth``
field was removed, assuming applications which recognize the format
by its four-character-code already know the color depth, and others
do not care about it. The same rationale lead to the removal of the
Since the remaining flags were replaced as well, the ``flags`` field
itself was removed.
- The interlace flags were replaced by a enum
- :c:type:`v4l2_field` value in a newly added ``field``
- field.
-
-
+ The interlace flags were replaced by a enum v4l2_field value in a
+ newly added ``field`` field.
.. flat-table::
:header-rows: 1
:stub-columns: 0
* - Old flag
- - enum :c:type:`v4l2_field`
+ - enum v4l2_field
* - ``V4L2_FMT_FLAG_NOT_INTERLACED``
- ?
* - ``V4L2_FMT_FLAG_INTERLACED`` = ``V4L2_FMT_FLAG_COMBINED``
* - ``-``
- ``V4L2_FIELD_ALTERNATE``
-
- The color space flags were replaced by a enum
- :c:type:`v4l2_colorspace` value in a newly added
- ``colorspace`` field, where one of ``V4L2_COLORSPACE_SMPTE170M``,
- ``V4L2_COLORSPACE_BT878``, ``V4L2_COLORSPACE_470_SYSTEM_M`` or
+ The color space flags were replaced by a enum v4l2_colorspace value in
+ a newly added ``colorspace`` field, where one of
+ ``V4L2_COLORSPACE_SMPTE170M``, ``V4L2_COLORSPACE_BT878``,
+ ``V4L2_COLORSPACE_470_SYSTEM_M`` or
``V4L2_COLORSPACE_470_SYSTEM_BG`` replaces ``V4L2_FMT_CS_601YUV``.
-12. In struct :c:type:`v4l2_requestbuffers` the
- ``type`` field was properly defined as enum
- :c:type:`v4l2_buf_type`. Buffer types changed as
- mentioned above. A new ``memory`` field of type enum
- :c:type:`v4l2_memory` was added to distinguish between
+12. In struct v4l2_requestbuffers the
+ ``type`` field was properly defined as enum v4l2_buf_type. Buffer types
+ changed as mentioned above. A new ``memory`` field of type
+ enum v4l2_memory was added to distinguish between
I/O methods using buffers allocated by the driver or the
application. See :ref:`io` for details.
-13. In struct :c:type:`v4l2_buffer` the ``type`` field was
- properly defined as enum :c:type:`v4l2_buf_type`.
+13. In struct v4l2_buffer the ``type`` field was
+ properly defined as enum v4l2_buf_type.
Buffer types changed as mentioned above. A ``field`` field of type
- enum :c:type:`v4l2_field` was added to indicate if a
+ enum v4l2_field was added to indicate if a
buffer contains a top or bottom field. The old field flags were
removed. Since no unadjusted system time clock was added to the
kernel as planned, the ``timestamp`` field changed back from type
stamp_t, an unsigned 64 bit integer expressing the sample time in
- nanoseconds, to struct :c:type:`timeval`. With the addition
+ nanoseconds, to struct timeval. With the addition
of a second memory mapping method the ``offset`` field moved into
- union ``m``, and a new ``memory`` field of type enum
- :c:type:`v4l2_memory` was added to distinguish between
+ union ``m``, and a new ``memory`` field of type enum v4l2_memory
+ was added to distinguish between
I/O methods. See :ref:`io` for details.
The ``V4L2_BUF_REQ_CONTIG`` flag was used by the V4L compatibility
indeed allocated in device memory rather than DMA-able system
memory. It was barely useful and so was removed.
-14. In struct :c:type:`v4l2_framebuffer` the
+14. In struct v4l2_framebuffer the
``base[3]`` array anticipating double- and triple-buffering in
off-screen video memory, however without defining a synchronization
mechanism, was replaced by a single pointer. The
``V4L2_FBUF_CAP_LIST_CLIPPING`` and
``V4L2_FBUF_CAP_BITMAP_CLIPPING``.
-15. In struct :c:type:`v4l2_clip` the ``x``, ``y``,
+15. In struct v4l2_clip the ``x``, ``y``,
``width`` and ``height`` field moved into a ``c`` substructure of
- type struct :c:type:`v4l2_rect`. The ``x`` and ``y``
+ type struct v4l2_rect. The ``x`` and ``y``
fields were renamed to ``left`` and ``top``, i. e. offsets to a
context dependent origin.
-16. In struct :c:type:`v4l2_window` the ``x``, ``y``,
+16. In struct v4l2_window the ``x``, ``y``,
``width`` and ``height`` field moved into a ``w`` substructure as
- above. A ``field`` field of type :c:type:`v4l2_field` was added to
+ above. A ``field`` field of type enum v4l2_field was added to
distinguish between field and frame (interlaced) overlay.
-17. The digital zoom interface, including struct
- struct ``v4l2_zoomcap``, struct
+17. The digital zoom interface, including struct ``v4l2_zoomcap``,
struct ``v4l2_zoom``, ``V4L2_ZOOM_NONCAP`` and
``V4L2_ZOOM_WHILESTREAMING`` was replaced by a new cropping and
- scaling interface. The previously unused struct
- struct :c:type:`v4l2_cropcap` and struct :c:type:`v4l2_crop`
+ scaling interface. The previously unused
+ struct v4l2_cropcap and struct v4l2_crop
where redefined for this purpose. See :ref:`crop` for details.
-18. In struct :c:type:`v4l2_vbi_format` the
+18. In struct v4l2_vbi_format the
``SAMPLE_FORMAT`` field now contains a four-character-code as used
to identify video image formats and ``V4L2_PIX_FMT_GREY`` replaces
the ``V4L2_VBI_SF_UBYTE`` define. The ``reserved`` field was
extended.
-19. In struct :c:type:`v4l2_captureparm` the type of
- the ``timeperframe`` field changed from unsigned long to struct
- :c:type:`v4l2_fract`. This allows the accurate
+19. In struct v4l2_captureparm the type of
+ the ``timeperframe`` field changed from unsigned long to
+ struct v4l2_fract. This allows the accurate
expression of multiples of the NTSC-M frame rate 30000 / 1001. A new
field ``readbuffers`` was added to control the driver behaviour in
read I/O mode.
- Similar changes were made to struct
- :c:type:`v4l2_outputparm`.
+ Similar changes were made to struct v4l2_outputparm.
20. The struct ``v4l2_performance`` and
``VIDIOC_G_PERF`` ioctl were dropped. Except when using the
===================
1. A new field ``input`` (former ``reserved[0]``) was added to the
- struct :c:type:`v4l2_buffer` structure. Purpose of this
+ struct v4l2_buffer. Purpose of this
field is to alternate between video inputs (e. g. cameras) in step
with the video capturing process. This function must be enabled with
the new ``V4L2_BUF_FLAG_INPUT`` flag. The ``flags`` field is no
4. The documentation of the :ref:`VIDIOC_QBUF` and
:ref:`VIDIOC_DQBUF <VIDIOC_QBUF>` ioctls did not mention the
- struct :c:type:`v4l2_buffer` ``memory`` field. It was
+ struct v4l2_buffer ``memory`` field. It was
also missing from examples. Also on the ``VIDIOC_DQBUF`` page the ``EIO``
error code was not documented.
V4L2 spec erratum 2006-01-10
============================
-1. The ``V4L2_IN_ST_COLOR_KILL`` flag in struct
- :c:type:`v4l2_input` not only indicates if the color
- killer is enabled, but also if it is active. (The color killer
- disables color decoding when it detects no color in the video signal
- to improve the image quality.)
+1. The ``V4L2_IN_ST_COLOR_KILL`` flag in struct v4l2_input not only
+ indicates if the color killer is enabled, but also if it is active.
+ (The color killer disables color decoding when it detects no color
+ in the video signal to improve the image quality.)
2. :ref:`VIDIOC_S_PARM <VIDIOC_G_PARM>` is a write-read ioctl, not
write-only as stated on its reference page. The ioctl changed in 2003
V4L2 spec erratum 2006-02-03
============================
-1. In struct :c:type:`v4l2_captureparm` and struct
- :c:type:`v4l2_outputparm` the ``timeperframe``
+1. In struct v4l2_captureparm and struct v4l2_outputparm the ``timeperframe``
field gives the time in seconds, not microseconds.
V4L2 spec erratum 2006-02-04
============================
-1. The ``clips`` field in struct :c:type:`v4l2_window`
- must point to an array of struct :c:type:`v4l2_clip`, not
- a linked list, because drivers ignore the struct
- struct :c:type:`v4l2_clip`. ``next`` pointer.
-
+1. The ``clips`` field in struct v4l2_window
+ must point to an array of struct v4l2_clip, not
+ a linked list, because drivers ignore the
+ struct v4l2_clip. ``next`` pointer.
V4L2 in Linux 2.6.17
====================
``V4L2_BUF_TYPE_SLICED_VBI_OUTPUT`` of the sliced VBI interface were
not mentioned along with other buffer types.
-2. In :ref:`VIDIOC_G_AUDIO <VIDIOC_G_AUDIO>` it was clarified that the struct
- :c:type:`v4l2_audio` ``mode`` field is a flags field.
+2. In :ref:`VIDIOC_G_AUDIO <VIDIOC_G_AUDIO>` it was clarified that the
+ struct v4l2_audio ``mode`` field is a flags field.
3. :ref:`VIDIOC_QUERYCAP` did not mention the sliced VBI and radio
capability flags.
-4. In :ref:`VIDIOC_G_FREQUENCY <VIDIOC_G_FREQUENCY>` it was clarified that applications
- must initialize the tuner ``type`` field of struct
- :c:type:`v4l2_frequency` before calling
+4. In :ref:`VIDIOC_G_FREQUENCY <VIDIOC_G_FREQUENCY>` it was clarified that
+ applications must initialize the tuner ``type`` field of
+ struct v4l2_frequency before calling
:ref:`VIDIOC_S_FREQUENCY <VIDIOC_G_FREQUENCY>`.
-5. The ``reserved`` array in struct
- :c:type:`v4l2_requestbuffers` has 2 elements,
+5. The ``reserved`` array in struct v4l2_requestbuffers has 2 elements,
not 32.
6. In :ref:`output` and :ref:`raw-vbi` the device file names
flag to skip unsupported controls with
:ref:`VIDIOC_QUERYCTRL`, new control types
``V4L2_CTRL_TYPE_INTEGER64`` and ``V4L2_CTRL_TYPE_CTRL_CLASS``
- (:c:type:`v4l2_ctrl_type`), and new control flags
+ (enum v4l2_ctrl_type), and new control flags
``V4L2_CTRL_FLAG_READ_ONLY``, ``V4L2_CTRL_FLAG_UPDATE``,
``V4L2_CTRL_FLAG_INACTIVE`` and ``V4L2_CTRL_FLAG_SLIDER``
(:ref:`control-flags`). See :ref:`extended-controls` for details.
V4L2 in Linux 2.6.19
====================
-1. In struct :c:type:`v4l2_sliced_vbi_cap` a
+1. In struct v4l2_sliced_vbi_cap a
buffer type field was added replacing a reserved field. Note on
architectures where the size of enum types differs from int types the
size of the structure changed. The
====================
1. Two new field orders ``V4L2_FIELD_INTERLACED_TB`` and
- ``V4L2_FIELD_INTERLACED_BT`` were added. See :c:type:`v4l2_field` for
+ ``V4L2_FIELD_INTERLACED_BT`` were added. See enum v4l2_field for
details.
2. Three new clipping/blending methods with a global or straight or
See the description of the :ref:`VIDIOC_G_FBUF <VIDIOC_G_FBUF>`
and :ref:`VIDIOC_S_FBUF <VIDIOC_G_FBUF>` ioctls for details.
- A new ``global_alpha`` field was added to
- :c:type:`v4l2_window`, extending the structure. This
- may *break compatibility* with applications using a struct
- struct :c:type:`v4l2_window` directly. However the
+ A new ``global_alpha`` field was added to struct v4l2_window,
+ extending the structure. This may **break compatibility** with
+ applications using a struct v4l2_window directly. However the
:ref:`VIDIOC_G/S/TRY_FMT <VIDIOC_G_FMT>` ioctls, which take a
- pointer to a :c:type:`v4l2_format` parent structure
+ pointer to a struct v4l2_format parent structure
with padding bytes at the end, are not affected.
-3. The format of the ``chromakey`` field in struct
- :c:type:`v4l2_window` changed from "host order RGB32"
- to a pixel value in the same format as the framebuffer. This may
- *break compatibility* with existing applications. Drivers supporting
- the "host order RGB32" format are not known.
+3. The format of the ``chromakey`` field in struct v4l2_window changed from
+ "host order RGB32" to a pixel value in the same format as the framebuffer.
+ This may **break compatibility** with existing applications. Drivers
+ supporting the "host order RGB32" format are not known.
V4L2 in Linux 2.6.24
1. The ``VIDIOC_G_CHIP_IDENT`` ioctl was renamed to
``VIDIOC_G_CHIP_IDENT_OLD`` and ``VIDIOC_DBG_G_CHIP_IDENT`` was
- introduced in its place. The old struct
- struct ``v4l2_chip_ident`` was renamed to
+ introduced in its place. The old struct ``v4l2_chip_ident`` was renamed to
struct ``v4l2_chip_ident_old``.
2. The pixel formats ``V4L2_PIX_FMT_VYUY``, ``V4L2_PIX_FMT_NV16`` and
V4L2 in Linux 3.6
=================
-1. Replaced ``input`` in struct :c:type:`v4l2_buffer` by
+1. Replaced ``input`` in struct v4l2_buffer by
``reserved2`` and removed ``V4L2_BUF_FLAG_INPUT``.
2. Added V4L2_CAP_VIDEO_M2M and V4L2_CAP_VIDEO_M2M_MPLANE
=================
1. Added timestamp types to ``flags`` field in
- struct :c:type:`v4l2_buffer`. See :ref:`buffer-flags`.
+ struct v4l2_buffer. See :ref:`buffer-flags`.
2. Added ``V4L2_EVENT_CTRL_CH_RANGE`` control event changes flag. See
:ref:`ctrl-changes-flags`.
V4L2 in Linux 3.14
==================
-1. In struct :c:type:`v4l2_rect`, the type of ``width`` and
+1. In struct v4l2_rect, the type of ``width`` and
``height`` fields changed from _s32 to _u32.
V4L2 in Linux 3.17
==================
-1. Extended struct :c:type:`v4l2_pix_format`. Added
+1. Extended struct v4l2_pix_format. Added
format flags.
2. Added compound control types and
V4L2 in Linux 3.19
==================
-1. Rewrote Colorspace chapter, added new enum
- :c:type:`v4l2_ycbcr_encoding` and enum
- :c:type:`v4l2_quantization` fields to struct
- :c:type:`v4l2_pix_format`, struct
- :c:type:`v4l2_pix_format_mplane` and
- struct :c:type:`v4l2_mbus_framefmt`.
-
+1. Rewrote Colorspace chapter, added new enum v4l2_ycbcr_encoding
+ and enum v4l2_quantization fields to struct v4l2_pix_format,
+ struct v4l2_pix_format_mplane and struct v4l2_mbus_framefmt.
V4L2 in Linux 4.4
=================
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _hsv-formats:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _io:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _libv4l-introduction:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _libv4l:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _meta-formats:
pixfmt-meta-d4xx
pixfmt-meta-intel-ipu3
+ pixfmt-meta-rkisp1
pixfmt-meta-uvc
pixfmt-meta-vsp1-hgo
pixfmt-meta-vsp1-hgt
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _mmap:
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
- This file is dual-licensed: you can use it either under the terms
- of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
- dual licensing only applies to this file, and not this project as a
- whole.
-
- a) This file is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation version 2 of
- the License.
-
- This file is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- Or, alternatively,
-
- b) Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
- This file is dual-licensed: you can use it either under the terms
- of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
- dual licensing only applies to this file, and not this project as a
- whole.
-
- a) This file is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation version 2 of
- the License.
-
- This file is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- Or, alternatively,
-
- b) Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _open:
Opening and Closing Devices
***************************
+.. _v4l2_hardware_control:
-Device Naming
-=============
+Controlling a hardware peripheral via V4L2
+==========================================
+
+Hardware that is supported using the V4L2 uAPI often consists of multiple
+devices or peripherals, each of which have their own driver.
+
+The bridge driver exposes one or more V4L2 device nodes
+(see :ref:`v4l2_device_naming`).
+
+There are other drivers providing support for other components of
+the hardware, which may also expose device nodes, called V4L2 sub-devices.
+
+When such V4L2 sub-devices are exposed, they allow controlling those
+other hardware components - usually connected via a serial bus (like
+I²C, SMBus or SPI). Depending on the bridge driver, those sub-devices
+can be controlled indirectly via the bridge driver or explicitly via
+the :ref:`Media Controller <media_controller>` and via the
+:ref:`V4L2 sub-devices <subdev>`.
+
+The devices that require the use of the
+:ref:`Media Controller <media_controller>` are called **MC-centric**
+devices. The devices that are fully controlled via V4L2 device nodes
+are called **video-node-centric**.
+
+Userspace can check if a V4L2 hardware peripheral is MC-centric by
+calling :ref:`VIDIOC_QUERYCAP` and checking the
+:ref:`device_caps field <device-capabilities>`.
+
+If the device returns ``V4L2_CAP_IO_MC`` flag at ``device_caps``,
+then it is MC-centric, otherwise, it is video-node-centric.
+
+It is required for MC-centric drivers to identify the V4L2
+sub-devices and to configure the pipelines via the
+:ref:`media controller API <media_controller>` before using the peripheral.
+Also, the sub-devices' configuration shall be controlled via the
+:ref:`sub-device API <subdev>`.
+
+.. note::
+
+ A video-node-centric may still provide media-controller and
+ sub-device interfaces as well.
+
+ However, in that case the media-controller and the sub-device
+ interfaces are read-only and just provide information about the
+ device. The actual configuration is done via the video nodes.
+
+.. _v4l2_device_naming:
+
+V4L2 Device Node Naming
+=======================
V4L2 drivers are implemented as kernel modules, loaded manually by the
system administrator or automatically when a device is first discovered.
-The driver modules plug into the "videodev" kernel module. It provides
+The driver modules plug into the ``videodev`` kernel module. It provides
helper functions and a common application interface specified in this
document.
Each driver thus loaded registers one or more device nodes with major
-number 81 and a minor number between 0 and 255. Minor numbers are
-allocated dynamically unless the kernel is compiled with the kernel
-option CONFIG_VIDEO_FIXED_MINOR_RANGES. In that case minor numbers
-are allocated in ranges depending on the device node type (video, radio,
-etc.).
+number 81. Minor numbers are allocated dynamically unless the kernel
+is compiled with the kernel option CONFIG_VIDEO_FIXED_MINOR_RANGES.
+In that case minor numbers are allocated in ranges depending on the
+device node type.
+
+The device nodes supported by the Video4Linux subsystem are:
+
+======================== ====================================================
+Default device node name Usage
+======================== ====================================================
+``/dev/videoX`` Video and metadata for capture/output devices
+``/dev/vbiX`` Vertical blank data (i.e. closed captions, teletext)
+``/dev/radioX`` Radio tuners and modulators
+``/dev/swradioX`` Software Defined Radio tuners and modulators
+``/dev/v4l-touchX`` Touch sensors
+``/dev/v4l-subdevX`` Video sub-devices (used by sensors and other
+ components of the hardware peripheral)\ [#]_
+======================== ====================================================
+
+Where ``X`` is a non-negative integer.
+
+.. note::
+
+ 1. The actual device node name is system-dependent, as udev rules may apply.
+ 2. There is no guarantee that ``X`` will remain the same for the same
+ device, as the number depends on the device driver's probe order.
+ If you need an unique name, udev default rules produce
+ ``/dev/v4l/by-id/`` and ``/dev/v4l/by-path/`` directories containing
+ links that can be used uniquely to identify a V4L2 device node::
+
+ $ tree /dev/v4l
+ /dev/v4l
+ ├── by-id
+ │ └── usb-OmniVision._USB_Camera-B4.04.27.1-video-index0 -> ../../video0
+ └── by-path
+ └── pci-0000:00:14.0-usb-0:2:1.0-video-index0 -> ../../video0
+
+.. [#] **V4L2 sub-device nodes** (e. g. ``/dev/v4l-subdevX``) use a different
+ set of system calls, as covered at :ref:`subdev`.
Many drivers support "video_nr", "radio_nr" or "vbi_nr" module
options to select specific video/radio/vbi node numbers. This allows the
Devices can support several functions. For example video capturing, VBI
capturing and radio support.
-The V4L2 API creates different nodes for each of these functions.
+The V4L2 API creates different V4L2 device nodes for each of these functions.
The V4L2 API was designed with the idea that one device node could
support all functions. However, in practice this never worked: this
using the streaming I/O API, not with the
:ref:`read() <func-read>`/\ :ref:`write() <func-write>` API.
-Today each device node supports just one function.
+Today each V4L2 device node supports just one function.
Besides video input or output the hardware may also support audio
sampling or playback. If so, these functions are implemented as ALSA PCM
devices with optional ALSA audio mixer devices.
One problem with all these devices is that the V4L2 API makes no
-provisions to find these related devices. Some really complex devices
-use the Media Controller (see :ref:`media_controller`) which can be
-used for this purpose. But most drivers do not use it, and while some
-code exists that uses sysfs to discover related devices (see
+provisions to find these related V4L2 device nodes. Some really complex
+hardware use the Media Controller (see :ref:`media_controller`) which can
+be used for this purpose. But several drivers do not use it, and while some
+code exists that uses sysfs to discover related V4L2 device nodes (see
libmedia_dev in the
`v4l-utils <http://git.linuxtv.org/cgit.cgi/v4l-utils.git/>`__ git
repository), there is no library yet that can provide a single API
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _pixfmt-bayer:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
******************
Compressed Formats
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-GREY:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _pixfmt-indexed:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
**********************
Standard Image Formats
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-INZI:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-M420:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2-meta-fmt-d4xx:
-.. This file is dual-licensed: you can use it either under the terms
-.. of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
-.. dual licensing only applies to this file, and not this project as a
-.. whole.
-..
-.. a) This file is free software; you can redistribute it and/or
-.. modify it under the terms of the GNU General Public License version
-.. 2.0 as published by the Free Software Foundation.
-..
-.. This file is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License version 2.0 for more details.
-..
-.. Or, alternatively,
-..
-.. b) Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later
.. _v4l2-meta-fmt-params:
.. _v4l2-meta-fmt-stat-3a:
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _v4l2-meta-fmt-params-rkisp1:
+.. _v4l2-meta-fmt-stat-rkisp1:
+
+*****************************************************************************
+V4L2_META_FMT_RK_ISP1_PARAMS ('rk1p'), V4L2_META_FMT_RK_ISP1_STAT_3A ('rk1s')
+*****************************************************************************
+
+Configuration parameters
+========================
+
+The configuration parameters are passed to the
+:ref:`rkisp1_params <rkisp1_params>` metadata output video node, using
+the :c:type:`v4l2_meta_format` interface. The buffer contains
+a single instance of the C structure :c:type:`rkisp1_params_cfg` defined in
+``rkisp1-config.h``. So the structure can be obtained from the buffer by:
+
+.. code-block:: c
+
+ struct rkisp1_params_cfg *params = (struct rkisp1_params_cfg*) buffer;
+
+.. rkisp1_stat_buffer
+
+3A and histogram statistics
+===========================
+
+The ISP1 device collects different statistics over an input Bayer frame.
+Those statistics are obtained from the :ref:`rkisp1_stats <rkisp1_stats>`
+metadata capture video node,
+using the :c:type:`v4l2_meta_format` interface. The buffer contains a single
+instance of the C structure :c:type:`rkisp1_stat_buffer` defined in
+``rkisp1-config.h``. So the structure can be obtained from the buffer by:
+
+.. code-block:: c
+
+ struct rkisp1_stat_buffer *stats = (struct rkisp1_stat_buffer*) buffer;
+
+The statistics collected are Exposure, AWB (Auto-white balance), Histogram and
+AF (Auto-focus). See :c:type:`rkisp1_stat_buffer` for details of the statistics.
+
+The 3A statistics and configuration parameters described here are usually
+consumed and produced by dedicated user space libraries that comprise the
+important tuning tools using software control loop.
+
+rkisp1 uAPI data types
+======================
+
+.. kernel-doc:: drivers/staging/media/rkisp1/uapi/rkisp1-config.h
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2-meta-fmt-uvc:
-.. This file is dual-licensed: you can use it either under the terms
-.. of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
-.. dual licensing only applies to this file, and not this project as a
-.. whole.
-..
-.. a) This file is free software; you can redistribute it and/or
-.. modify it under the terms of the GNU General Public License as
-.. published by the Free Software Foundation version 2 of
-.. the License.
-..
-.. This file is distributed in the hope that it will be useful,
-.. but WITHOUT ANY WARRANTY; without even the implied warranty of
-.. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-.. GNU General Public License for more details.
-..
-.. Or, alternatively,
-..
-.. b) Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later
.. _v4l2-meta-fmt-vivid:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2-meta-fmt-vsp1-hgo:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2-meta-fmt-vsp1-hgt:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-NV12:
.. _V4L2-PIX-FMT-NV21:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-NV12M:
.. _v4l2-pix-fmt-nv12mt-16x16:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-NV12MT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-NV16:
.. _V4L2-PIX-FMT-NV61:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-NV16M:
.. _v4l2-pix-fmt-nv61m:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-NV24:
.. _V4L2-PIX-FMT-NV42:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _packed-hsv:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _packed-yuv:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _pixfmt-reserved:
of tiles, resulting in 32-aligned resolutions for the luminance plane
and 16-aligned resolutions for the chrominance plane (with 2x2
subsampling).
-
-.. tabularcolumns:: |p{6.6cm}|p{2.2cm}|p{8.7cm}|
-
-.. _format-flags:
-
-.. flat-table:: Format Flags
- :header-rows: 0
- :stub-columns: 0
- :widths: 3 1 4
-
- * - ``V4L2_PIX_FMT_FLAG_PREMUL_ALPHA``
- - 0x00000001
- - The color values are premultiplied by the alpha channel value. For
- example, if a light blue pixel with 50% transparency was described
- by RGBA values (128, 192, 255, 128), the same pixel described with
- premultiplied colors would be described by RGBA values (64, 96,
- 128, 128)
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _pixfmt-rgb:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2-sdr-fmt-cs8:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-SDR-FMT-CS14LE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2-sdr-fmt-cu8:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-SDR-FMT-CU16LE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-SDR-FMT-PCU16BE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-SDR-FMT-PCU18BE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-SDR-FMT-PCU20BE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-SDR-FMT-RU12LE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2-pix-fmt-ipu3-sbggr10:
.. _v4l2-pix-fmt-ipu3-sgbrg10:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-SRGGB10:
.. _v4l2-pix-fmt-sbggr10:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-SBGGR10ALAW8:
.. _v4l2-pix-fmt-sgbrg10alaw8:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-SBGGR10DPCM8:
.. _v4l2-pix-fmt-sgbrg10dpcm8:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-SRGGB10P:
.. _v4l2-pix-fmt-sbggr10p:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-SRGGB12:
.. _v4l2-pix-fmt-sbggr12:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-SRGGB12P:
.. _v4l2-pix-fmt-sbggr12p:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-SRGGB14:
.. _v4l2-pix-fmt-sbggr14:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-SRGGB14P:
.. _v4l2-pix-fmt-sbggr14p:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-SRGGB16:
.. _v4l2-pix-fmt-sbggr16:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-SRGGB8:
.. _v4l2-pix-fmt-sbggr8:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-TCH-FMT-DELTA-TD08:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-TCH-FMT-DELTA-TD16:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-TCH-FMT-TU08:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-TCH-FMT-TU16:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-UV8:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-UYVY:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
******************************
Multi-planar format structures
* - __u8
- ``ycbcr_enc``
- Y'CbCr encoding, from enum :c:type:`v4l2_ycbcr_encoding`.
- This information supplements the ``colorspace`` and must be set by
- the driver for capture streams and by the application for output
- streams, see :ref:`colorspaces`.
+ See struct :c:type:`v4l2_pix_format`.
* - __u8
- ``hsv_enc``
- HSV encoding, from enum :c:type:`v4l2_hsv_encoding`.
- This information supplements the ``colorspace`` and must be set by
- the driver for capture streams and by the application for output
- streams, see :ref:`colorspaces`.
+ See struct :c:type:`v4l2_pix_format`.
* - }
-
* - __u8
- ``quantization``
- Quantization range, from enum :c:type:`v4l2_quantization`.
- This information supplements the ``colorspace`` and must be set by
- the driver for capture streams and by the application for output
- streams, see :ref:`colorspaces`.
+ See struct :c:type:`v4l2_pix_format`.
* - __u8
- ``xfer_func``
- Transfer function, from enum :c:type:`v4l2_xfer_func`.
- This information supplements the ``colorspace`` and must be set by
- the driver for capture streams and by the application for output
- streams, see :ref:`colorspaces`.
+ See struct :c:type:`v4l2_pix_format`.
* - __u8
- ``reserved[7]``
- Reserved for future extensions. Should be zeroed by drivers and
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
******************************
Single-planar format structure
- Image colorspace, from enum :c:type:`v4l2_colorspace`.
This information supplements the ``pixelformat`` and must be set
by the driver for capture streams and by the application for
- output streams, see :ref:`colorspaces`.
+ output streams, see :ref:`colorspaces`. If the application sets the
+ flag ``V4L2_PIX_FMT_FLAG_SET_CSC`` then the application can set
+ this field for a capture stream to request a specific colorspace
+ for the captured image data. If the driver cannot handle requested
+ conversion, it will return another supported colorspace.
+ The driver indicates that colorspace conversion is supported by setting
+ the flag V4L2_FMT_FLAG_CSC_COLORSPACE in the corresponding struct
+ :c:type:`v4l2_fmtdesc` during enumeration. See :ref:`fmtdesc-flags`.
* - __u32
- ``priv``
- This field indicates whether the remaining fields of the
- Y'CbCr encoding, from enum :c:type:`v4l2_ycbcr_encoding`.
This information supplements the ``colorspace`` and must be set by
the driver for capture streams and by the application for output
- streams, see :ref:`colorspaces`.
+ streams, see :ref:`colorspaces`. If the application sets the
+ flag ``V4L2_PIX_FMT_FLAG_SET_CSC`` then the application can set
+ this field for a capture stream to request a specific Y'CbCr encoding
+ for the captured image data. If the driver cannot handle requested
+ conversion, it will return another supported encoding.
+ This field is ignored for HSV pixelformats. The driver indicates that
+ ycbcr_enc conversion is supported by setting the flag
+ V4L2_FMT_FLAG_CSC_YCBCR_ENC in the corresponding struct
+ :c:type:`v4l2_fmtdesc` during enumeration. See :ref:`fmtdesc-flags`.
* - __u32
- ``hsv_enc``
- HSV encoding, from enum :c:type:`v4l2_hsv_encoding`.
This information supplements the ``colorspace`` and must be set by
the driver for capture streams and by the application for output
- streams, see :ref:`colorspaces`.
+ streams, see :ref:`colorspaces`. If the application sets the flag
+ ``V4L2_PIX_FMT_FLAG_SET_CSC`` then the application can set this
+ field for a capture stream to request a specific HSV encoding for the
+ captured image data. If the driver cannot handle requested
+ conversion, it will return another supported encoding.
+ This field is ignored for non-HSV pixelformats. The driver indicates
+ that hsv_enc conversion is supported by setting the flag
+ V4L2_FMT_FLAG_CSC_HSV_ENC in the corresponding struct
+ :c:type:`v4l2_fmtdesc` during enumeration. See :ref:`fmtdesc-flags`.
* - }
-
* - __u32
- Quantization range, from enum :c:type:`v4l2_quantization`.
This information supplements the ``colorspace`` and must be set by
the driver for capture streams and by the application for output
- streams, see :ref:`colorspaces`.
+ streams, see :ref:`colorspaces`. If the application sets the flag
+ ``V4L2_PIX_FMT_FLAG_SET_CSC`` then the application can set
+ this field for a capture stream to request a specific quantization
+ range for the captured image data. If the driver cannot handle requested
+ conversion, it will return another supported quantization.
+ The driver indicates that quantization conversion is supported by setting
+ the flag V4L2_FMT_FLAG_CSC_QUANTIZATION in the corresponding struct
+ :c:type:`v4l2_fmtdesc` during enumeration. See :ref:`fmtdesc-flags`.
* - __u32
- ``xfer_func``
- Transfer function, from enum :c:type:`v4l2_xfer_func`.
This information supplements the ``colorspace`` and must be set by
the driver for capture streams and by the application for output
- streams, see :ref:`colorspaces`.
+ streams, see :ref:`colorspaces`. If the application sets the flag
+ ``V4L2_PIX_FMT_FLAG_SET_CSC`` then the application can set
+ this field for a capture stream to request a specific transfer function
+ for the captured image data. If the driver cannot handle requested
+ conversion, it will return another supported transfer function.
+ The driver indicates that xfer_func conversion is supported by setting
+ the flag V4L2_FMT_FLAG_CSC_XFER_FUNC in the corresponding struct
+ :c:type:`v4l2_fmtdesc` during enumeration. See :ref:`fmtdesc-flags`.
+
+.. tabularcolumns:: |p{6.6cm}|p{2.2cm}|p{8.7cm}|
+
+.. _format-flags:
+
+.. flat-table:: Format Flags
+ :header-rows: 0
+ :stub-columns: 0
+ :widths: 3 1 4
+
+ * - ``V4L2_PIX_FMT_FLAG_PREMUL_ALPHA``
+ - 0x00000001
+ - The color values are premultiplied by the alpha channel value. For
+ example, if a light blue pixel with 50% transparency was described
+ by RGBA values (128, 192, 255, 128), the same pixel described with
+ premultiplied colors would be described by RGBA values (64, 96,
+ 128, 128)
+ * .. _`v4l2-pix-fmt-flag-set-csc`:
+
+ - ``V4L2_PIX_FMT_FLAG_SET_CSC``
+ - 0x00000002
+ - Set by the application. It is only used for capture and is
+ ignored for output streams. If set, then request the device to do
+ colorspace conversion from the received colorspace to the requested
+ colorspace values. If the colorimetry field (``colorspace``, ``xfer_func``,
+ ``ycbcr_enc``, ``hsv_enc`` or ``quantization``) is set to ``*_DEFAULT``,
+ then that colorimetry setting will remain unchanged from what was received.
+ So in order to change the quantization, only the ``quantization`` field shall
+ be set to non default value (``V4L2_QUANTIZATION_FULL_RANGE`` or
+ ``V4L2_QUANTIZATION_LIM_RANGE``) and all other colorimetry fields shall
+ be set to ``*_DEFAULT``.
+
+ To check which conversions are supported by the hardware for the current
+ pixel format, see :ref:`fmtdesc-flags`.
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-VYUY:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-Y10:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-Y10BPACK:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-Y10P:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-Y12:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-Y12I:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-Y14:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-Y16-BE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-Y16:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-Y41P:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-Y8I:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-YVU410:
.. _v4l2-pix-fmt-yuv410:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-YUV411P:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-YVU420:
.. _V4L2-PIX-FMT-YUV420:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-YUV420M:
.. _v4l2-pix-fmt-yvu420m:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-YUV422M:
.. _v4l2-pix-fmt-yvu422m:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-YUV422P:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-YUV444M:
.. _v4l2-pix-fmt-yvu444m:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-YUYV:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-YVYU:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _V4L2-PIX-FMT-Z16:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _pixfmt:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _planar-apis:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _querycap:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _rw:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _sdr-formats:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
*************
Configuration
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
********
Examples
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
************
Introduction
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
*****************
Selection targets
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _selection-vs-crop:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _selection-api:
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- This file is dual-licensed: you can use it either under the terms
- of the GPL 2.0 or the GFDL 1.1+ license, at your option. Note that this
- dual licensing only applies to this file, and not this project as a
- whole.
-
- a) This file is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation version 2 of
- the License.
-
- This file is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- Or, alternatively,
-
- b) Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GPL-2.0 OR GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GPL-2.0 OR GFDL-1.1-no-invariants-or-later -->
<svg enable-background="new" version="1" viewBox="0 0 4226.3 1686.8" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<defs>
<pattern id="ig" xlink:href="#ka" patternTransform="matrix(5.4432 0 0 10.1 1722.4 161.06)"/>
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2-selections-common:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _standard:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _streaming-par:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2-mbus-format:
:ref:`field-order` for details.
* - __u32
- ``colorspace``
- - Image colorspace, from enum
- :c:type:`v4l2_colorspace`. See
- :ref:`colorspaces` for details.
+ - Image colorspace, from enum :c:type:`v4l2_colorspace`.
+ Must be set by the driver for subdevices. If the application sets the
+ flag ``V4L2_MBUS_FRAMEFMT_SET_CSC`` then the application can set this
+ field on the source pad to request a specific colorspace for the media
+ bus data. If the driver cannot handle the requested conversion, it will
+ return another supported colorspace. The driver indicates that colorspace
+ conversion is supported by setting the flag
+ V4L2_SUBDEV_MBUS_CODE_CSC_COLORSPACE in the corresponding struct
+ :c:type:`v4l2_subdev_mbus_code_enum` during enumeration.
+ See :ref:`v4l2-subdev-mbus-code-flags`.
+ * - union {
+ - (anonymous)
* - __u16
- ``ycbcr_enc``
- Y'CbCr encoding, from enum :c:type:`v4l2_ycbcr_encoding`.
This information supplements the ``colorspace`` and must be set by
- the driver for capture streams and by the application for output
- streams, see :ref:`colorspaces`.
+ the driver for subdevices, see :ref:`colorspaces`. If the application
+ sets the flag ``V4L2_MBUS_FRAMEFMT_SET_CSC`` then the application can set
+ this field on a source pad to request a specific Y'CbCr encoding
+ for the media bus data. If the driver cannot handle the requested
+ conversion, it will return another supported encoding.
+ This field is ignored for HSV media bus formats. The driver indicates
+ that ycbcr_enc conversion is supported by setting the flag
+ V4L2_SUBDEV_MBUS_CODE_CSC_YCBCR_ENC in the corresponding struct
+ :c:type:`v4l2_subdev_mbus_code_enum` during enumeration.
+ See :ref:`v4l2-subdev-mbus-code-flags`.
+ * - __u16
+ - ``hsv_enc``
+ - HSV encoding, from enum :c:type:`v4l2_hsv_encoding`.
+ This information supplements the ``colorspace`` and must be set by
+ the driver for subdevices, see :ref:`colorspaces`. If the application
+ sets the flag ``V4L2_MBUS_FRAMEFMT_SET_CSC`` then the application can set
+ this field on a source pad to request a specific HSV encoding
+ for the media bus data. If the driver cannot handle the requested
+ conversion, it will return another supported encoding.
+ This field is ignored for Y'CbCr media bus formats. The driver indicates
+ that hsv_enc conversion is supported by setting the flag
+ V4L2_SUBDEV_MBUS_CODE_CSC_HSV_ENC in the corresponding struct
+ :c:type:`v4l2_subdev_mbus_code_enum` during enumeration.
+ See :ref:`v4l2-subdev-mbus-code-flags`
+ * - }
+ -
* - __u16
- ``quantization``
- Quantization range, from enum :c:type:`v4l2_quantization`.
This information supplements the ``colorspace`` and must be set by
- the driver for capture streams and by the application for output
- streams, see :ref:`colorspaces`.
+ the driver for subdevices, see :ref:`colorspaces`. If the application
+ sets the flag ``V4L2_MBUS_FRAMEFMT_SET_CSC`` then the application can set
+ this field on a source pad to request a specific quantization
+ for the media bus data. If the driver cannot handle the requested
+ conversion, it will return another supported quantization.
+ The driver indicates that quantization conversion is supported by
+ setting the flag V4L2_SUBDEV_MBUS_CODE_CSC_QUANTIZATION in the
+ corresponding struct :c:type:`v4l2_subdev_mbus_code_enum`
+ during enumeration. See :ref:`v4l2-subdev-mbus-code-flags`.
+
* - __u16
- ``xfer_func``
- Transfer function, from enum :c:type:`v4l2_xfer_func`.
This information supplements the ``colorspace`` and must be set by
- the driver for capture streams and by the application for output
- streams, see :ref:`colorspaces`.
+ the driver for subdevices, see :ref:`colorspaces`. If the application
+ sets the flag ``V4L2_MBUS_FRAMEFMT_SET_CSC`` then the application can set
+ this field on a source pad to request a specific transfer
+ function for the media bus data. If the driver cannot handle the requested
+ conversion, it will return another supported transfer function.
+ The driver indicates that the transfer function conversion is supported by
+ setting the flag V4L2_SUBDEV_MBUS_CODE_CSC_XFER_FUNC in the
+ corresponding struct :c:type:`v4l2_subdev_mbus_code_enum`
+ during enumeration. See :ref:`v4l2-subdev-mbus-code-flags`.
* - __u16
- - ``reserved``\ [11]
+ - ``flags``
+ - flags See: :ref:v4l2-mbus-framefmt-flags
+ * - __u16
+ - ``reserved``\ [10]
- Reserved for future extensions. Applications and drivers must set
the array to zero.
+.. _v4l2-mbus-framefmt-flags:
+
+.. flat-table:: v4l2_mbus_framefmt Flags
+ :header-rows: 0
+ :stub-columns: 0
+ :widths: 3 1 4
+
+ * .. _`mbus-framefmt-set-csc`:
+
+ - ``V4L2_MBUS_FRAMEFMT_SET_CSC``
+ - 0x0001
+ - Set by the application. It is only used for source pads and is
+ ignored for sink pads. If set, then request the subdevice to do
+ colorspace conversion from the received colorspace to the requested
+ colorspace values. If the colorimetry field (``colorspace``, ``xfer_func``,
+ ``ycbcr_enc``, ``hsv_enc`` or ``quantization``) is set to ``*_DEFAULT``,
+ then that colorimetry setting will remain unchanged from what was received.
+ So in order to change the quantization, only the ``quantization`` field shall
+ be set to non default value (``V4L2_QUANTIZATION_FULL_RANGE`` or
+ ``V4L2_QUANTIZATION_LIM_RANGE``) and all other colorimetry fields shall
+ be set to ``*_DEFAULT``.
+
+ To check which conversions are supported by the hardware for the current
+ media bus frame format, see :ref:`v4l2-subdev-mbus-code-flags`.
.. _v4l2-mbus-pixelcode:
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
- Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
- Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
- Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _tch-formats:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _tuner:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _user-func:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _userp:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2-selection-flags:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2-selection-targets:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
-
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. include:: <isonum.txt>
+
.. _v4l2spec:
############################
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _v4l2grab-example:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
file: media/v4l/v4l2grab.c
==========================
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
- Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
- Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!--
- Permission is granted to copy, distribute and/or modify this
- document under the terms of the GNU Free Documentation License,
- Version 1.1 or any later version published by the Free Software
- Foundation, with no Invariant Sections, no Front-Cover Texts
- and no Back-Cover Texts. A copy of the license is included at
- Documentation/userspace-api/media/fdl-appendix.rst.
-
- TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
--->
+<!-- SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _video:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _videodev:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_CREATE_BUFS:
If you want to just query the capabilities without making any
other changes, then set ``count`` to 0, ``memory`` to
``V4L2_MEMORY_MMAP`` and ``format.type`` to the buffer type.
- * - __u32
- - ``flags``
- - Specifies additional buffer management attributes.
- See :ref:`memory-flags`.
* - __u32
- - ``reserved``\ [6]
+ - ``reserved``\ [7]
- A place holder for future extensions. Drivers and applications
must set the array to zero.
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_CROPCAP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_DBG_G_CHIP_INFO:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_DBG_G_REGISTER:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_DECODER_CMD:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_DQEVENT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_DV_TIMINGS_CAP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_ENCODER_CMD:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_ENUM_DV_TIMINGS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_ENUM_FMT:
This flag can only be used in combination with the
``V4L2_FMT_FLAG_COMPRESSED`` flag, since this applies to
compressed formats only. This flag is valid for stateful encoders only.
+ * - ``V4L2_FMT_FLAG_CSC_COLORSPACE``
+ - 0x0020
+ - The driver allows the application to try to change the default
+ colorspace. This flag is relevant only for capture devices.
+ The application can ask to configure the colorspace of the capture device
+ when calling the :ref:`VIDIOC_S_FMT <VIDIOC_G_FMT>` ioctl with
+ :ref:`V4L2_PIX_FMT_FLAG_SET_CSC <v4l2-pix-fmt-flag-set-csc>` set.
+ * - ``V4L2_FMT_FLAG_CSC_XFER_FUNC``
+ - 0x0040
+ - The driver allows the application to try to change the default
+ transfer function. This flag is relevant only for capture devices.
+ The application can ask to configure the transfer function of the capture
+ device when calling the :ref:`VIDIOC_S_FMT <VIDIOC_G_FMT>` ioctl with
+ :ref:`V4L2_PIX_FMT_FLAG_SET_CSC <v4l2-pix-fmt-flag-set-csc>` set.
+ * - ``V4L2_FMT_FLAG_CSC_YCBCR_ENC``
+ - 0x0080
+ - The driver allows the application to try to change the default
+ Y'CbCr encoding. This flag is relevant only for capture devices.
+ The application can ask to configure the Y'CbCr encoding of the capture device
+ when calling the :ref:`VIDIOC_S_FMT <VIDIOC_G_FMT>` ioctl with
+ :ref:`V4L2_PIX_FMT_FLAG_SET_CSC <v4l2-pix-fmt-flag-set-csc>` set.
+ * - ``V4L2_FMT_FLAG_CSC_HSV_ENC``
+ - 0x0080
+ - The driver allows the application to try to change the default
+ HSV encoding. This flag is relevant only for capture devices.
+ The application can ask to configure the HSV encoding of the capture device
+ when calling the :ref:`VIDIOC_S_FMT <VIDIOC_G_FMT>` ioctl with
+ :ref:`V4L2_PIX_FMT_FLAG_SET_CSC <v4l2-pix-fmt-flag-set-csc>` set.
+ * - ``V4L2_FMT_FLAG_CSC_QUANTIZATION``
+ - 0x0100
+ - The driver allows the application to try to change the default
+ quantization. This flag is relevant only for capture devices.
+ The application can ask to configure the quantization of the capture
+ device when calling the :ref:`VIDIOC_S_FMT <VIDIOC_G_FMT>` ioctl with
+ :ref:`V4L2_PIX_FMT_FLAG_SET_CSC <v4l2-pix-fmt-flag-set-csc>` set.
Return Value
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_ENUM_FRAMEINTERVALS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_ENUM_FRAMESIZES:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_ENUM_FREQ_BANDS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_ENUMAUDIO:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_ENUMAUDOUT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_ENUMINPUT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_ENUMOUTPUT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_ENUMSTD:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_EXPBUF:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_AUDIO:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_AUDOUT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_CROP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_CTRL:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_DV_TIMINGS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_EDID:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_ENC_INDEX:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_EXT_CTRLS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_FBUF:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_FMT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_FREQUENCY:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_INPUT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_JPEGCOMP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_MODULATOR:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_OUTPUT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_PARM:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_PRIORITY:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_SELECTION:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_SLICED_VBI_CAP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_STD:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_G_TUNER:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_LOG_STATUS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_OVERLAY:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_PREPARE_BUF:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_QBUF:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_QUERY_DV_TIMINGS:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_QUERYBUF:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_QUERYCAP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_QUERYCTRL:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_QUERYSTD:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_REQBUFS:
``V4L2_MEMORY_MMAP`` and ``type`` set to the buffer type. This will
free any previously allocated buffers, so this is typically something
that will be done at the start of the application.
- * - union {
- - (anonymous)
- * - __u32
- - ``flags``
- - Specifies additional buffer management attributes.
- See :ref:`memory-flags`.
* - __u32
- ``reserved``\ [1]
- - Kept for backwards compatibility. Use ``flags`` instead.
- * - }
- -
+ - A place holder for future extensions. Drivers and applications
+ must set the array to zero.
.. tabularcolumns:: |p{6.1cm}|p{2.2cm}|p{8.7cm}|
- This capability is set by the driver to indicate that the queue supports
cache and memory management hints. However, it's only valid when the
queue is used for :ref:`memory mapping <mmap>` streaming I/O. See
- :ref:`V4L2_FLAG_MEMORY_NON_CONSISTENT <V4L2-FLAG-MEMORY-NON-CONSISTENT>`,
:ref:`V4L2_BUF_FLAG_NO_CACHE_INVALIDATE <V4L2-BUF-FLAG-NO-CACHE-INVALIDATE>` and
:ref:`V4L2_BUF_FLAG_NO_CACHE_CLEAN <V4L2-BUF-FLAG-NO-CACHE-CLEAN>`.
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_S_HW_FREQ_SEEK:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_STREAMON:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_SUBDEV_ENUM_FRAME_INTERVAL:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_SUBDEV_ENUM_FRAME_SIZE:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_SUBDEV_ENUM_MBUS_CODE:
- Media bus format codes to be enumerated, from enum
:ref:`v4l2_subdev_format_whence <v4l2-subdev-format-whence>`.
* - __u32
- - ``reserved``\ [8]
+ - ``flags``
+ - See :ref:`v4l2-subdev-mbus-code-flags`
+ * - __u32
+ - ``reserved``\ [7]
- Reserved for future extensions. Applications and drivers must set
the array to zero.
+
+.. tabularcolumns:: |p{4.4cm}|p{4.4cm}|p{7.7cm}|
+
+.. _v4l2-subdev-mbus-code-flags:
+
+.. flat-table:: Subdev Media Bus Code Enumerate Flags
+ :header-rows: 0
+ :stub-columns: 0
+ :widths: 1 1 2
+
+ * - V4L2_SUBDEV_MBUS_CODE_CSC_COLORSPACE
+ - 0x00000001
+ - The driver allows the application to try to change the default colorspace
+ encoding. The application can ask to configure the colorspace of the
+ subdevice when calling the :ref:`VIDIOC_SUBDEV_S_FMT <VIDIOC_SUBDEV_G_FMT>`
+ ioctl with :ref:`V4L2_MBUS_FRAMEFMT_SET_CSC <mbus-framefmt-set-csc>` set.
+ See :ref:`v4l2-mbus-format` on how to do this.
+ * - V4L2_SUBDEV_MBUS_CODE_CSC_XFER_FUNC
+ - 0x00000002
+ - The driver allows the application to try to change the default transform function.
+ The application can ask to configure the transform function of
+ the subdevice when calling the :ref:`VIDIOC_SUBDEV_S_FMT <VIDIOC_SUBDEV_G_FMT>`
+ ioctl with :ref:`V4L2_MBUS_FRAMEFMT_SET_CSC <mbus-framefmt-set-csc>` set.
+ See :ref:`v4l2-mbus-format` on how to do this.
+ * - V4L2_SUBDEV_MBUS_CODE_CSC_YCBCR_ENC
+ - 0x00000004
+ - The driver allows the application to try to change the default Y'CbCr
+ encoding. The application can ask to configure the Y'CbCr encoding of the
+ subdevice when calling the :ref:`VIDIOC_SUBDEV_S_FMT <VIDIOC_SUBDEV_G_FMT>`
+ ioctl with :ref:`V4L2_MBUS_FRAMEFMT_SET_CSC <mbus-framefmt-set-csc>` set.
+ See :ref:`v4l2-mbus-format` on how to do this.
+ * - V4L2_SUBDEV_MBUS_CODE_CSC_HSV_ENC
+ - 0x00000004
+ - The driver allows the application to try to change the default HSV
+ encoding. The application can ask to configure the HSV encoding of the
+ subdevice when calling the :ref:`VIDIOC_SUBDEV_S_FMT <VIDIOC_SUBDEV_G_FMT>`
+ ioctl with :ref:`V4L2_MBUS_FRAMEFMT_SET_CSC <mbus-framefmt-set-csc>` set.
+ See :ref:`v4l2-mbus-format` on how to do this.
+ * - V4L2_SUBDEV_MBUS_CODE_CSC_QUANTIZATION
+ - 0x00000008
+ - The driver allows the application to try to change the default
+ quantization. The application can ask to configure the quantization of
+ the subdevice when calling the :ref:`VIDIOC_SUBDEV_S_FMT <VIDIOC_SUBDEV_G_FMT>`
+ ioctl with :ref:`V4L2_MBUS_FRAMEFMT_SET_CSC <mbus-framefmt-set-csc>` set.
+ See :ref:`v4l2-mbus-format` on how to do this.
+
Return Value
============
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_SUBDEV_G_CROP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_SUBDEV_G_FMT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_SUBDEV_G_FRAME_INTERVAL:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_SUBDEV_G_SELECTION:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_SUBDEV_QUERYCAP:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _VIDIOC_SUBSCRIBE_EVENT:
.. _VIDIOC_UNSUBSCRIBE_EVENT:
-.. Permission is granted to copy, distribute and/or modify this
-.. document under the terms of the GNU Free Documentation License,
-.. Version 1.1 or any later version published by the Free Software
-.. Foundation, with no Invariant Sections, no Front-Cover Texts
-.. and no Back-Cover Texts. A copy of the license is included at
-.. Documentation/userspace-api/media/fdl-appendix.rst.
-..
-.. TODO: replace it to GFDL-1.1-or-later WITH no-invariant-sections
+.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later
.. _yuv-formats:
# V4L2 pix flags
replace define V4L2_PIX_FMT_PRIV_MAGIC :c:type:`v4l2_pix_format`
-replace define V4L2_PIX_FMT_FLAG_PREMUL_ALPHA reserved-formats
+replace define V4L2_PIX_FMT_FLAG_PREMUL_ALPHA format-flags
# V4L2 format flags
replace define V4L2_FMT_FLAG_COMPRESSED fmtdesc-flags
replace define V4L2_FMT_FLAG_CONTINUOUS_BYTESTREAM fmtdesc-flags
replace define V4L2_FMT_FLAG_DYN_RESOLUTION fmtdesc-flags
replace define V4L2_FMT_FLAG_ENC_CAP_FRAME_INTERVAL fmtdesc-flags
+replace define V4L2_FMT_FLAG_CSC_COLORSPACE fmtdesc-flags
+replace define V4L2_FMT_FLAG_CSC_XFER_FUNC fmtdesc-flags
+replace define V4L2_FMT_FLAG_CSC_YCBCR_ENC fmtdesc-flags
+replace define V4L2_FMT_FLAG_CSC_HSV_ENC fmtdesc-flags
+replace define V4L2_FMT_FLAG_CSC_QUANTIZATION fmtdesc-flags
# V4L2 timecode types
replace define V4L2_TC_TYPE_24FPS timecode-type
:maxdepth: 2
kvm/index
- uml/user_mode_linux
+ uml/user_mode_linux_howto_v2
paravirt_ops
guest-halt-polling
encrypting bootstrap code, snapshot, migrating and debugging the guest. For more
information, see the SEV Key Management spec [api-spec]_
-The main ioctl to access SEV is KVM_MEM_ENCRYPT_OP. If the argument
-to KVM_MEM_ENCRYPT_OP is NULL, the ioctl returns 0 if SEV is enabled
+The main ioctl to access SEV is KVM_MEMORY_ENCRYPT_OP. If the argument
+to KVM_MEMORY_ENCRYPT_OP is NULL, the ioctl returns 0 if SEV is enabled
and ``ENOTTY` if it is disabled (on some older versions of Linux,
the ioctl runs normally even with a NULL argument, and therefore will
-likely return ``EFAULT``). If non-NULL, the argument to KVM_MEM_ENCRYPT_OP
+likely return ``EFAULT``). If non-NULL, the argument to KVM_MEMORY_ENCRYPT_OP
must be a struct kvm_sev_cmd::
struct kvm_sev_cmd {
:Capability: basic
:Architectures: x86
-:Type: system
+:Type: vm
:Parameters: an opaque platform specific structure (in/out)
:Returns: 0 on success; -1 on error
#define KVM_STATE_NESTED_VMX_SMM_GUEST_MODE 0x00000001
#define KVM_STATE_NESTED_VMX_SMM_VMXON 0x00000002
-#define KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE 0x00000001
+ #define KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE 0x00000001
struct kvm_vmx_nested_state_hdr {
__u64 vmxon_pa;
x3 = x1's value when entering the next payload (arm64)
x4 = x2's value when entering the next payload (arm64)
- Mask all exceptions, disable the MMU, move the arguments into place
- (arm64 only), and jump to the restart address while at HYP/EL2. This
- hypercall is not expected to return to its caller.
+ Mask all exceptions, disable the MMU, clear I+D bits, move the arguments
+ into place (arm64 only), and jump to the restart address while at HYP/EL2.
+ This hypercall is not expected to return to its caller.
Any other value of r0/x0 triggers a hypervisor-specific handling,
which is not documented here.
before enabling paravirtualized
sebd IPIs
-KVM_FEATURE_PV_POLL_CONTROL 12 host-side polling on HLT can
+KVM_FEATURE_POLL_CONTROL 12 host-side polling on HLT can
be disabled by writing
to msr 0x4b564d05.
+++ /dev/null
-.. SPDX-License-Identifier: GPL-2.0
-
-=====================
-User Mode Linux HOWTO
-=====================
-
-:Author: User Mode Linux Core Team
-:Last-updated: Sat Jan 25 16:07:55 CET 2020
-
-This document describes the use and abuse of Jeff Dike's User Mode
-Linux: a port of the Linux kernel as a normal Intel Linux process.
-
-
-.. Table of Contents
-
- 1. Introduction
-
- 1.1 How is User Mode Linux Different?
- 1.2 Why Would I Want User Mode Linux?
-
- 2. Compiling the kernel and modules
-
- 2.1 Compiling the kernel
- 2.2 Compiling and installing kernel modules
- 2.3 Compiling and installing uml_utilities
-
- 3. Running UML and logging in
-
- 3.1 Running UML
- 3.2 Logging in
- 3.3 Examples
-
- 4. UML on 2G/2G hosts
-
- 4.1 Introduction
- 4.2 The problem
- 4.3 The solution
-
- 5. Setting up serial lines and consoles
-
- 5.1 Specifying the device
- 5.2 Specifying the channel
- 5.3 Examples
-
- 6. Setting up the network
-
- 6.1 General setup
- 6.2 Userspace daemons
- 6.3 Specifying ethernet addresses
- 6.4 UML interface setup
- 6.5 Multicast
- 6.6 TUN/TAP with the uml_net helper
- 6.7 TUN/TAP with a preconfigured tap device
- 6.8 Ethertap
- 6.9 The switch daemon
- 6.10 Slip
- 6.11 Slirp
- 6.12 pcap
- 6.13 Setting up the host yourself
-
- 7. Sharing Filesystems between Virtual Machines
-
- 7.1 A warning
- 7.2 Using layered block devices
- 7.3 Note!
- 7.4 Another warning
- 7.5 uml_moo : Merging a COW file with its backing file
-
- 8. Creating filesystems
-
- 8.1 Create the filesystem file
- 8.2 Assign the file to a UML device
- 8.3 Creating and mounting the filesystem
-
- 9. Host file access
-
- 9.1 Using hostfs
- 9.2 hostfs as the root filesystem
- 9.3 Building hostfs
-
- 10. The Management Console
- 10.1 version
- 10.2 halt and reboot
- 10.3 config
- 10.4 remove
- 10.5 sysrq
- 10.6 help
- 10.7 cad
- 10.8 stop
- 10.9 go
-
- 11. Kernel debugging
-
- 11.1 Starting the kernel under gdb
- 11.2 Examining sleeping processes
- 11.3 Running ddd on UML
- 11.4 Debugging modules
- 11.5 Attaching gdb to the kernel
- 11.6 Using alternate debuggers
-
- 12. Kernel debugging examples
-
- 12.1 The case of the hung fsck
- 12.2 Episode 2: The case of the hung fsck
-
- 13. What to do when UML doesn't work
-
- 13.1 Strange compilation errors when you build from source
- 13.2 (obsolete)
- 13.3 A variety of panics and hangs with /tmp on a reiserfs filesystem
- 13.4 The compile fails with errors about conflicting types for 'open', 'dup', and 'waitpid'
- 13.5 UML doesn't work when /tmp is an NFS filesystem
- 13.6 UML hangs on boot when compiled with gprof support
- 13.7 syslogd dies with a SIGTERM on startup
- 13.8 TUN/TAP networking doesn't work on a 2.4 host
- 13.9 You can network to the host but not to other machines on the net
- 13.10 I have no root and I want to scream
- 13.11 UML build conflict between ptrace.h and ucontext.h
- 13.12 The UML BogoMips is exactly half the host's BogoMips
- 13.13 When you run UML, it immediately segfaults
- 13.14 xterms appear, then immediately disappear
- 13.15 Any other panic, hang, or strange behavior
-
- 14. Diagnosing Problems
-
- 14.1 Case 1 : Normal kernel panics
- 14.2 Case 2 : Tracing thread panics
- 14.3 Case 3 : Tracing thread panics caused by other threads
- 14.4 Case 4 : Hangs
-
- 15. Thanks
-
- 15.1 Code and Documentation
- 15.2 Flushing out bugs
- 15.3 Buglets and clean-ups
- 15.4 Case Studies
- 15.5 Other contributions
-
-
-1. Introduction
-================
-
- Welcome to User Mode Linux. It's going to be fun.
-
-
-
-1.1. How is User Mode Linux Different?
----------------------------------------
-
- Normally, the Linux Kernel talks straight to your hardware (video
- card, keyboard, hard drives, etc), and any programs which run ask the
- kernel to operate the hardware, like so::
-
-
-
- +-----------+-----------+----+
- | Process 1 | Process 2 | ...|
- +-----------+-----------+----+
- | Linux Kernel |
- +----------------------------+
- | Hardware |
- +----------------------------+
-
-
-
-
- The User Mode Linux Kernel is different; instead of talking to the
- hardware, it talks to a `real` Linux kernel (called the `host kernel`
- from now on), like any other program. Programs can then run inside
- User-Mode Linux as if they were running under a normal kernel, like
- so::
-
-
-
- +----------------+
- | Process 2 | ...|
- +-----------+----------------+
- | Process 1 | User-Mode Linux|
- +----------------------------+
- | Linux Kernel |
- +----------------------------+
- | Hardware |
- +----------------------------+
-
-
-
-
-
-1.2. Why Would I Want User Mode Linux?
----------------------------------------
-
-
- 1. If User Mode Linux crashes, your host kernel is still fine.
-
- 2. You can run a usermode kernel as a non-root user.
-
- 3. You can debug the User Mode Linux like any normal process.
-
- 4. You can run gprof (profiling) and gcov (coverage testing).
-
- 5. You can play with your kernel without breaking things.
-
- 6. You can use it as a sandbox for testing new apps.
-
- 7. You can try new development kernels safely.
-
- 8. You can run different distributions simultaneously.
-
- 9. It's extremely fun.
-
-
-
-.. _Compiling_the_kernel_and_modules:
-
-2. Compiling the kernel and modules
-====================================
-
-
-
-
-2.1. Compiling the kernel
---------------------------
-
-
- Compiling the user mode kernel is just like compiling any other
- kernel.
-
-
- 1. Download the latest kernel from your favourite kernel mirror,
- such as:
-
- https://mirrors.edge.kernel.org/pub/linux/kernel/v5.x/linux-5.4.14.tar.xz
-
- 2. Make a directory and unpack the kernel into it::
-
- host%
- mkdir ~/uml
-
- host%
- cd ~/uml
-
- host%
- tar xvf linux-5.4.14.tar.xz
-
-
- 3. Run your favorite config; ``make xconfig ARCH=um`` is the most
- convenient. ``make config ARCH=um`` and ``make menuconfig ARCH=um``
- will work as well. The defaults will give you a useful kernel. If
- you want to change something, go ahead, it probably won't hurt
- anything.
-
-
- Note: If the host is configured with a 2G/2G address space split
- rather than the usual 3G/1G split, then the packaged UML binaries
- will not run. They will immediately segfault. See
- :ref:`UML_on_2G/2G_hosts` for the scoop on running UML on your system.
-
-
-
- 4. Finish with ``make linux ARCH=um``: the result is a file called
- ``linux`` in the top directory of your source tree.
-
-
-2.2. Compiling and installing kernel modules
----------------------------------------------
-
- UML modules are built in the same way as the native kernel (with the
- exception of the 'ARCH=um' that you always need for UML)::
-
-
- host% make modules ARCH=um
-
-
-
-
- Any modules that you want to load into this kernel need to be built in
- the user-mode pool. Modules from the native kernel won't work.
-
- You can install them by using ftp or something to copy them into the
- virtual machine and dropping them into ``/lib/modules/$(uname -r)``.
-
- You can also get the kernel build process to install them as follows:
-
- 1. with the kernel not booted, mount the root filesystem in the top
- level of the kernel pool::
-
-
- host% mount root_fs mnt -o loop
-
-
-
-
-
-
- 2. run::
-
-
- host%
- make modules_install INSTALL_MOD_PATH=`pwd`/mnt ARCH=um
-
-
-
-
-
-
- 3. unmount the filesystem::
-
-
- host% umount mnt
-
-
-
-
-
-
- 4. boot the kernel on it
-
-
- When the system is booted, you can use insmod as usual to get the
- modules into the kernel. A number of things have been loaded into UML
- as modules, especially filesystems and network protocols and filters,
- so most symbols which need to be exported probably already are.
- However, if you do find symbols that need exporting, let us
- know at http://user-mode-linux.sourceforge.net/, and
- they'll be "taken care of".
-
-
-
-2.3. Compiling and installing uml_utilities
---------------------------------------------
-
- Many features of the UML kernel require a user-space helper program,
- so a uml_utilities package is distributed separately from the kernel
- patch which provides these helpers. Included within this is:
-
- - port-helper - Used by consoles which connect to xterms or ports
-
- - tunctl - Configuration tool to create and delete tap devices
-
- - uml_net - Setuid binary for automatic tap device configuration
-
- - uml_switch - User-space virtual switch required for daemon
- transport
-
- The uml_utilities tree is compiled with::
-
-
- host#
- make && make install
-
-
-
-
- Note that UML kernel patches may require a specific version of the
- uml_utilities distribution. If you don't keep up with the mailing
- lists, ensure that you have the latest release of uml_utilities if you
- are experiencing problems with your UML kernel, particularly when
- dealing with consoles or command-line switches to the helper programs
-
-
-
-
-
-
-
-
-3. Running UML and logging in
-==============================
-
-
-
-3.1. Running UML
------------------
-
- It runs on 2.2.15 or later, and all kernel versions since 2.4.
-
-
- Booting UML is straightforward. Simply run 'linux': it will try to
- mount the file ``root_fs`` in the current directory. You do not need to
- run it as root. If your root filesystem is not named ``root_fs``, then
- you need to put a ``ubd0=root_fs_whatever`` switch on the linux command
- line.
-
-
- You will need a filesystem to boot UML from. There are a number
- available for download from http://user-mode-linux.sourceforge.net.
- There are also several tools at
- http://user-mode-linux.sourceforge.net/ which can be
- used to generate UML-compatible filesystem images from media.
- The kernel will boot up and present you with a login prompt.
-
-
-Note:
- If the host is configured with a 2G/2G address space split
- rather than the usual 3G/1G split, then the packaged UML binaries will
- not run. They will immediately segfault. See :ref:`UML_on_2G/2G_hosts`
- for the scoop on running UML on your system.
-
-
-
-3.2. Logging in
-----------------
-
-
-
- The prepackaged filesystems have a root account with password 'root'
- and a user account with password 'user'. The login banner will
- generally tell you how to log in. So, you log in and you will find
- yourself inside a little virtual machine. Our filesystems have a
- variety of commands and utilities installed (and it is fairly easy to
- add more), so you will have a lot of tools with which to poke around
- the system.
-
- There are a couple of other ways to log in:
-
- - On a virtual console
-
-
-
- Each virtual console that is configured (i.e. the device exists in
- /dev and /etc/inittab runs a getty on it) will come up in its own
- xterm. If you get tired of the xterms, read
- :ref:`setting_up_serial_lines_and_consoles` to see how to attach
- the consoles to something else, like host ptys.
-
-
-
- - Over the serial line
-
-
- In the boot output, find a line that looks like::
-
-
-
- serial line 0 assigned pty /dev/ptyp1
-
-
-
-
- Attach your favorite terminal program to the corresponding tty. I.e.
- for minicom, the command would be::
-
-
- host% minicom -o -p /dev/ttyp1
-
-
-
-
-
-
- - Over the net
-
-
- If the network is running, then you can telnet to the virtual
- machine and log in to it. See :ref:`Setting_up_the_network` to learn
- about setting up a virtual network.
-
- When you're done using it, run halt, and the kernel will bring itself
- down and the process will exit.
-
-
-3.3. Examples
---------------
-
- Here are some examples of UML in action:
-
- - A login session http://user-mode-linux.sourceforge.net/old/login.html
-
- - A virtual network http://user-mode-linux.sourceforge.net/old/net.html
-
-
-
-
-
-.. _UML_on_2G/2G_hosts:
-
-4. UML on 2G/2G hosts
-======================
-
-
-
-
-4.1. Introduction
-------------------
-
-
- Most Linux machines are configured so that the kernel occupies the
- upper 1G (0xc0000000 - 0xffffffff) of the 4G address space and
- processes use the lower 3G (0x00000000 - 0xbfffffff). However, some
- machine are configured with a 2G/2G split, with the kernel occupying
- the upper 2G (0x80000000 - 0xffffffff) and processes using the lower
- 2G (0x00000000 - 0x7fffffff).
-
-
-
-
-4.2. The problem
------------------
-
-
- The prebuilt UML binaries on this site will not run on 2G/2G hosts
- because UML occupies the upper .5G of the 3G process address space
- (0xa0000000 - 0xbfffffff). Obviously, on 2G/2G hosts, this is right
- in the middle of the kernel address space, so UML won't even load - it
- will immediately segfault.
-
-
-
-
-4.3. The solution
-------------------
-
-
- The fix for this is to rebuild UML from source after enabling
- CONFIG_HOST_2G_2G (under 'General Setup'). This will cause UML to
- load itself in the top .5G of that smaller process address space,
- where it will run fine. See :ref:`Compiling_the_kernel_and_modules` if
- you need help building UML from source.
-
-
-
-
-
-
-
-.. _setting_up_serial_lines_and_consoles:
-
-
-5. Setting up serial lines and consoles
-========================================
-
-
- It is possible to attach UML serial lines and consoles to many types
- of host I/O channels by specifying them on the command line.
-
-
- You can attach them to host ptys, ttys, file descriptors, and ports.
- This allows you to do things like:
-
- - have a UML console appear on an unused host console,
-
- - hook two virtual machines together by having one attach to a pty
- and having the other attach to the corresponding tty
-
- - make a virtual machine accessible from the net by attaching a
- console to a port on the host.
-
-
- The general format of the command line option is ``device=channel``.
-
-
-
-5.1. Specifying the device
----------------------------
-
- Devices are specified with "con" or "ssl" (console or serial line,
- respectively), optionally with a device number if you are talking
- about a specific device.
-
-
- Using just "con" or "ssl" describes all of the consoles or serial
- lines. If you want to talk about console #3 or serial line #10, they
- would be "con3" and "ssl10", respectively.
-
-
- A specific device name will override a less general "con=" or "ssl=".
- So, for example, you can assign a pty to each of the serial lines
- except for the first two like this::
-
-
- ssl=pty ssl0=tty:/dev/tty0 ssl1=tty:/dev/tty1
-
-
-
-
- The specificity of the device name is all that matters; order on the
- command line is irrelevant.
-
-
-
-5.2. Specifying the channel
-----------------------------
-
- There are a number of different types of channels to attach a UML
- device to, each with a different way of specifying exactly what to
- attach to.
-
- - pseudo-terminals - device=pty pts terminals - device=pts
-
-
- This will cause UML to allocate a free host pseudo-terminal for the
- device. The terminal that it got will be announced in the boot
- log. You access it by attaching a terminal program to the
- corresponding tty:
-
- - screen /dev/pts/n
-
- - screen /dev/ttyxx
-
- - minicom -o -p /dev/ttyxx - minicom seems not able to handle pts
- devices
-
- - kermit - start it up, 'open' the device, then 'connect'
-
-
-
-
-
- - terminals - device=tty:tty device file
-
-
- This will make UML attach the device to the specified tty (i.e::
-
-
- con1=tty:/dev/tty3
-
-
-
-
- will attach UML's console 1 to the host's /dev/tty3). If the tty that
- you specify is the slave end of a tty/pty pair, something else must
- have already opened the corresponding pty in order for this to work.
-
-
-
-
-
- - xterms - device=xterm
-
-
- UML will run an xterm and the device will be attached to it.
-
-
-
-
-
- - Port - device=port:port number
-
-
- This will attach the UML devices to the specified host port.
- Attaching console 1 to the host's port 9000 would be done like
- this::
-
-
- con1=port:9000
-
-
-
-
- Attaching all the serial lines to that port would be done similarly::
-
-
- ssl=port:9000
-
-
-
-
- You access these devices by telnetting to that port. Each active
- telnet session gets a different device. If there are more telnets to a
- port than UML devices attached to it, then the extra telnet sessions
- will block until an existing telnet detaches, or until another device
- becomes active (i.e. by being activated in /etc/inittab).
-
- This channel has the advantage that you can both attach multiple UML
- devices to it and know how to access them without reading the UML boot
- log. It is also unique in allowing access to a UML from remote
- machines without requiring that the UML be networked. This could be
- useful in allowing public access to UMLs because they would be
- accessible from the net, but wouldn't need any kind of network
- filtering or access control because they would have no network access.
-
-
- If you attach the main console to a portal, then the UML boot will
- appear to hang. In reality, it's waiting for a telnet to connect, at
- which point the boot will proceed.
-
-
-
-
-
- - already-existing file descriptors - device=file descriptor
-
-
- If you set up a file descriptor on the UML command line, you can
- attach a UML device to it. This is most commonly used to put the
- main console back on stdin and stdout after assigning all the other
- consoles to something else::
-
-
- con0=fd:0,fd:1 con=pts
-
-
-
-
-
-
-
-
- - Nothing - device=null
-
-
- This allows the device to be opened, in contrast to 'none', but
- reads will block, and writes will succeed and the data will be
- thrown out.
-
-
-
-
-
- - None - device=none
-
-
- This causes the device to disappear.
-
-
-
- You can also specify different input and output channels for a device
- by putting a comma between them::
-
-
- ssl3=tty:/dev/tty2,xterm
-
-
-
-
- will cause serial line 3 to accept input on the host's /dev/tty2 and
- display output on an xterm. That's a silly example - the most common
- use of this syntax is to reattach the main console to stdin and stdout
- as shown above.
-
-
- If you decide to move the main console away from stdin/stdout, the
- initial boot output will appear in the terminal that you're running
- UML in. However, once the console driver has been officially
- initialized, then the boot output will start appearing wherever you
- specified that console 0 should be. That device will receive all
- subsequent output.
-
-
-
-5.3. Examples
---------------
-
- There are a number of interesting things you can do with this
- capability.
-
-
- First, this is how you get rid of those bleeding console xterms by
- attaching them to host ptys::
-
-
- con=pty con0=fd:0,fd:1
-
-
-
-
- This will make a UML console take over an unused host virtual console,
- so that when you switch to it, you will see the UML login prompt
- rather than the host login prompt::
-
-
- con1=tty:/dev/tty6
-
-
-
-
- You can attach two virtual machines together with what amounts to a
- serial line as follows:
-
- Run one UML with a serial line attached to a pty::
-
-
- ssl1=pty
-
-
-
-
- Look at the boot log to see what pty it got (this example will assume
- that it got /dev/ptyp1).
-
- Boot the other UML with a serial line attached to the corresponding
- tty::
-
-
- ssl1=tty:/dev/ttyp1
-
-
-
-
- Log in, make sure that it has no getty on that serial line, attach a
- terminal program like minicom to it, and you should see the login
- prompt of the other virtual machine.
-
-
-.. _setting_up_the_network:
-
-6. Setting up the network
-==========================
-
-
-
- This page describes how to set up the various transports and to
- provide a UML instance with network access to the host, other machines
- on the local net, and the rest of the net.
-
-
- As of 2.4.5, UML networking has been completely redone to make it much
- easier to set up, fix bugs, and add new features.
-
-
- There is a new helper, uml_net, which does the host setup that
- requires root privileges.
-
-
- There are currently five transport types available for a UML virtual
- machine to exchange packets with other hosts:
-
- - ethertap
-
- - TUN/TAP
-
- - Multicast
-
- - a switch daemon
-
- - slip
-
- - slirp
-
- - pcap
-
- The TUN/TAP, ethertap, slip, and slirp transports allow a UML
- instance to exchange packets with the host. They may be directed
- to the host or the host may just act as a router to provide access
- to other physical or virtual machines.
-
-
- The pcap transport is a synthetic read-only interface, using the
- libpcap binary to collect packets from interfaces on the host and
- filter them. This is useful for building preconfigured traffic
- monitors or sniffers.
-
-
- The daemon and multicast transports provide a completely virtual
- network to other virtual machines. This network is completely
- disconnected from the physical network unless one of the virtual
- machines on it is acting as a gateway.
-
-
- With so many host transports, which one should you use? Here's when
- you should use each one:
-
- - ethertap - if you want access to the host networking and it is
- running 2.2
-
- - TUN/TAP - if you want access to the host networking and it is
- running 2.4. Also, the TUN/TAP transport is able to use a
- preconfigured device, allowing it to avoid using the setuid uml_net
- helper, which is a security advantage.
-
- - Multicast - if you want a purely virtual network and you don't want
- to set up anything but the UML
-
- - a switch daemon - if you want a purely virtual network and you
- don't mind running the daemon in order to get somewhat better
- performance
-
- - slip - there is no particular reason to run the slip backend unless
- ethertap and TUN/TAP are just not available for some reason
-
- - slirp - if you don't have root access on the host to setup
- networking, or if you don't want to allocate an IP to your UML
-
- - pcap - not much use for actual network connectivity, but great for
- monitoring traffic on the host
-
- Ethertap is available on 2.4 and works fine. TUN/TAP is preferred
- to it because it has better performance and ethertap is officially
- considered obsolete in 2.4. Also, the root helper only needs to
- run occasionally for TUN/TAP, rather than handling every packet, as
- it does with ethertap. This is a slight security advantage since
- it provides fewer opportunities for a nasty UML user to somehow
- exploit the helper's root privileges.
-
-
-6.1. General setup
--------------------
-
- First, you must have the virtual network enabled in your UML. If are
- running a prebuilt kernel from this site, everything is already
- enabled. If you build the kernel yourself, under the "Network device
- support" menu, enable "Network device support", and then the three
- transports.
-
-
- The next step is to provide a network device to the virtual machine.
- This is done by describing it on the kernel command line.
-
- The general format is::
-
-
- eth <n> = <transport> , <transport args>
-
-
-
-
- For example, a virtual ethernet device may be attached to a host
- ethertap device as follows::
-
-
- eth0=ethertap,tap0,fe:fd:0:0:0:1,192.168.0.254
-
-
-
-
- This sets up eth0 inside the virtual machine to attach itself to the
- host /dev/tap0, assigns it an ethernet address, and assigns the host
- tap0 interface an IP address.
-
-
-
- Note that the IP address you assign to the host end of the tap device
- must be different than the IP you assign to the eth device inside UML.
- If you are short on IPs and don't want to consume two per UML, then
- you can reuse the host's eth IP address for the host ends of the tap
- devices. Internally, the UMLs must still get unique IPs for their eth
- devices. You can also give the UMLs non-routable IPs (192.168.x.x or
- 10.x.x.x) and have the host masquerade them. This will let outgoing
- connections work, but incoming connections won't without more work,
- such as port forwarding from the host.
- Also note that when you configure the host side of an interface, it is
- only acting as a gateway. It will respond to pings sent to it
- locally, but is not useful to do that since it's a host interface.
- You are not talking to the UML when you ping that interface and get a
- response.
-
-
- You can also add devices to a UML and remove them at runtime. See the
- :ref:`The_Management_Console` page for details.
-
-
- The sections below describe this in more detail.
-
-
- Once you've decided how you're going to set up the devices, you boot
- UML, log in, configure the UML side of the devices, and set up routes
- to the outside world. At that point, you will be able to talk to any
- other machines, physical or virtual, on the net.
-
-
- If ifconfig inside UML fails and the network refuses to come up, run
- tell you what went wrong.
-
-
-
-6.2. Userspace daemons
------------------------
-
- You will likely need the setuid helper, or the switch daemon, or both.
- They are both installed with the RPM and deb, so if you've installed
- either, you can skip the rest of this section.
-
-
- If not, then you need to check them out of CVS, build them, and
- install them. The helper is uml_net, in CVS /tools/uml_net, and the
- daemon is uml_switch, in CVS /tools/uml_router. They are both built
- with a plain 'make'. Both need to be installed in a directory that's
- in your path - /usr/bin is recommend. On top of that, uml_net needs
- to be setuid root.
-
-
-
-6.3. Specifying ethernet addresses
------------------------------------
-
- Below, you will see that the TUN/TAP, ethertap, and daemon interfaces
- allow you to specify hardware addresses for the virtual ethernet
- devices. This is generally not necessary. If you don't have a
- specific reason to do it, you probably shouldn't. If one is not
- specified on the command line, the driver will assign one based on the
- device IP address. It will provide the address fe:fd:nn:nn:nn:nn
- where nn.nn.nn.nn is the device IP address. This is nearly always
- sufficient to guarantee a unique hardware address for the device. A
- couple of exceptions are:
-
- - Another set of virtual ethernet devices are on the same network and
- they are assigned hardware addresses using a different scheme which
- may conflict with the UML IP address-based scheme
-
- - You aren't going to use the device for IP networking, so you don't
- assign the device an IP address
-
- If you let the driver provide the hardware address, you should make
- sure that the device IP address is known before the interface is
- brought up. So, inside UML, this will guarantee that::
-
-
-
- UML#
- ifconfig eth0 192.168.0.250 up
-
-
-
-
- If you decide to assign the hardware address yourself, make sure that
- the first byte of the address is even. Addresses with an odd first
- byte are broadcast addresses, which you don't want assigned to a
- device.
-
-
-
-6.4. UML interface setup
--------------------------
-
- Once the network devices have been described on the command line, you
- should boot UML and log in.
-
-
- The first thing to do is bring the interface up::
-
-
- UML# ifconfig ethn ip-address up
-
-
-
-
- You should be able to ping the host at this point.
-
-
- To reach the rest of the world, you should set a default route to the
- host::
-
-
- UML# route add default gw host ip
-
-
-
-
- Again, with host ip of 192.168.0.4::
-
-
- UML# route add default gw 192.168.0.4
-
-
-
-
- This page used to recommend setting a network route to your local net.
- This is wrong, because it will cause UML to try to figure out hardware
- addresses of the local machines by arping on the interface to the
- host. Since that interface is basically a single strand of ethernet
- with two nodes on it (UML and the host) and arp requests don't cross
- networks, they will fail to elicit any responses. So, what you want
- is for UML to just blindly throw all packets at the host and let it
- figure out what to do with them, which is what leaving out the network
- route and adding the default route does.
-
-
- Note: If you can't communicate with other hosts on your physical
- ethernet, it's probably because of a network route that's
- automatically set up. If you run 'route -n' and see a route that
- looks like this::
-
-
-
-
- Destination Gateway Genmask Flags Metric Ref Use Iface
- 192.168.0.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0
-
-
-
-
- with a mask that's not 255.255.255.255, then replace it with a route
- to your host::
-
-
- UML#
- route del -net 192.168.0.0 dev eth0 netmask 255.255.255.0
-
-
- UML#
- route add -host 192.168.0.4 dev eth0
-
-
-
-
- This, plus the default route to the host, will allow UML to exchange
- packets with any machine on your ethernet.
-
-
-
-6.5. Multicast
----------------
-
- The simplest way to set up a virtual network between multiple UMLs is
- to use the mcast transport. This was written by Harald Welte and is
- present in UML version 2.4.5-5um and later. Your system must have
- multicast enabled in the kernel and there must be a multicast-capable
- network device on the host. Normally, this is eth0, but if there is
- no ethernet card on the host, then you will likely get strange error
- messages when you bring the device up inside UML.
-
-
- To use it, run two UMLs with::
-
-
- eth0=mcast
-
-
-
-
- on their command lines. Log in, configure the ethernet device in each
- machine with different IP addresses::
-
-
- UML1# ifconfig eth0 192.168.0.254
-
-
- UML2# ifconfig eth0 192.168.0.253
-
-
-
-
- and they should be able to talk to each other.
-
- The full set of command line options for this transport are::
-
-
-
- ethn=mcast,ethernet address,multicast
- address,multicast port,ttl
-
-
-
- There is also a related point-to-point only "ucast" transport.
- This is useful when your network does not support multicast, and
- all network connections are simple point to point links.
-
- The full set of command line options for this transport are::
-
-
- ethn=ucast,ethernet address,remote address,listen port,remote port
-
-
-
-
-6.6. TUN/TAP with the uml_net helper
--------------------------------------
-
- TUN/TAP is the preferred mechanism on 2.4 to exchange packets with the
- host. The TUN/TAP backend has been in UML since 2.4.9-3um.
-
-
- The easiest way to get up and running is to let the setuid uml_net
- helper do the host setup for you. This involves insmod-ing the tun.o
- module if necessary, configuring the device, and setting up IP
- forwarding, routing, and proxy arp. If you are new to UML networking,
- do this first. If you're concerned about the security implications of
- the setuid helper, use it to get up and running, then read the next
- section to see how to have UML use a preconfigured tap device, which
- avoids the use of uml_net.
-
-
- If you specify an IP address for the host side of the device, the
- uml_net helper will do all necessary setup on the host - the only
- requirement is that TUN/TAP be available, either built in to the host
- kernel or as the tun.o module.
-
- The format of the command line switch to attach a device to a TUN/TAP
- device is::
-
-
- eth <n> =tuntap,,, <IP address>
-
-
-
-
- For example, this argument will attach the UML's eth0 to the next
- available tap device and assign an ethernet address to it based on its
- IP address::
-
-
- eth0=tuntap,,,192.168.0.254
-
-
-
-
-
-
- Note that the IP address that must be used for the eth device inside
- UML is fixed by the routing and proxy arp that is set up on the
- TUN/TAP device on the host. You can use a different one, but it won't
- work because reply packets won't reach the UML. This is a feature.
- It prevents a nasty UML user from doing things like setting the UML IP
- to the same as the network's nameserver or mail server.
-
-
- There are a couple potential problems with running the TUN/TAP
- transport on a 2.4 host kernel
-
- - TUN/TAP seems not to work on 2.4.3 and earlier. Upgrade the host
- kernel or use the ethertap transport.
-
- - With an upgraded kernel, TUN/TAP may fail with::
-
-
- File descriptor in bad state
-
-
-
-
- This is due to a header mismatch between the upgraded kernel and the
- kernel that was originally installed on the machine. The fix is to
- make sure that /usr/src/linux points to the headers for the running
- kernel.
-
- These were pointed out by Tim Robinson <timro at trkr dot net> in the past.
-
-
-
-6.7. TUN/TAP with a preconfigured tap device
----------------------------------------------
-
- If you prefer not to have UML use uml_net (which is somewhat
- insecure), with UML 2.4.17-11, you can set up a TUN/TAP device
- beforehand. The setup needs to be done as root, but once that's done,
- there is no need for root assistance. Setting up the device is done
- as follows:
-
- - Create the device with tunctl (available from the UML utilities
- tarball)::
-
-
-
-
- host# tunctl -u uid
-
-
-
-
- where uid is the user id or username that UML will be run as. This
- will tell you what device was created.
-
- - Configure the device IP (change IP addresses and device name to
- suit)::
-
-
-
-
- host# ifconfig tap0 192.168.0.254 up
-
-
-
-
-
- - Set up routing and arping if desired - this is my recipe, there are
- other ways of doing the same thing::
-
-
- host#
- bash -c 'echo 1 > /proc/sys/net/ipv4/ip_forward'
-
- host#
- route add -host 192.168.0.253 dev tap0
-
- host#
- bash -c 'echo 1 > /proc/sys/net/ipv4/conf/tap0/proxy_arp'
-
- host#
- arp -Ds 192.168.0.253 eth0 pub
-
-
-
-
- Note that this must be done every time the host boots - this configu-
- ration is not stored across host reboots. So, it's probably a good
- idea to stick it in an rc file. An even better idea would be a little
- utility which reads the information from a config file and sets up
- devices at boot time.
-
- - Rather than using up two IPs and ARPing for one of them, you can
- also provide direct access to your LAN by the UML by using a
- bridge::
-
-
- host#
- brctl addbr br0
-
-
- host#
- ifconfig eth0 0.0.0.0 promisc up
-
-
- host#
- ifconfig tap0 0.0.0.0 promisc up
-
-
- host#
- ifconfig br0 192.168.0.1 netmask 255.255.255.0 up
-
-
- host#
- brctl stp br0 off
-
-
- host#
- brctl setfd br0 1
-
-
- host#
- brctl sethello br0 1
-
-
- host#
- brctl addif br0 eth0
-
-
- host#
- brctl addif br0 tap0
-
-
-
-
- Note that 'br0' should be setup using ifconfig with the existing IP
- address of eth0, as eth0 no longer has its own IP.
-
- -
-
-
- Also, the /dev/net/tun device must be writable by the user running
- UML in order for the UML to use the device that's been configured
- for it. The simplest thing to do is::
-
-
- host# chmod 666 /dev/net/tun
-
-
-
-
- Making it world-writable looks bad, but it seems not to be
- exploitable as a security hole. However, it does allow anyone to cre-
- ate useless tap devices (useless because they can't configure them),
- which is a DOS attack. A somewhat more secure alternative would to be
- to create a group containing all the users who have preconfigured tap
- devices and chgrp /dev/net/tun to that group with mode 664 or 660.
-
-
- - Once the device is set up, run UML with 'eth0=tuntap,device name'
- (i.e. 'eth0=tuntap,tap0') on the command line (or do it with the
- mconsole config command).
-
- - Bring the eth device up in UML and you're in business.
-
- If you don't want that tap device any more, you can make it non-
- persistent with::
-
-
- host# tunctl -d tap device
-
-
-
-
- Finally, tunctl has a -b (for brief mode) switch which causes it to
- output only the name of the tap device it created. This makes it
- suitable for capture by a script::
-
-
- host# TAP=`tunctl -u 1000 -b`
-
-
-
-
-
-
-6.8. Ethertap
---------------
-
- Ethertap is the general mechanism on 2.2 for userspace processes to
- exchange packets with the kernel.
-
-
-
- To use this transport, you need to describe the virtual network device
- on the UML command line. The general format for this is::
-
-
- eth <n> =ethertap, <device> , <ethernet address> , <tap IP address>
-
-
-
-
- So, the previous example::
-
-
- eth0=ethertap,tap0,fe:fd:0:0:0:1,192.168.0.254
-
-
-
-
- attaches the UML eth0 device to the host /dev/tap0, assigns it the
- ethernet address fe:fd:0:0:0:1, and assigns the IP address
- 192.168.0.254 to the tap device.
-
-
-
- The tap device is mandatory, but the others are optional. If the
- ethernet address is omitted, one will be assigned to it.
-
-
- The presence of the tap IP address will cause the helper to run and do
- whatever host setup is needed to allow the virtual machine to
- communicate with the outside world. If you're not sure you know what
- you're doing, this is the way to go.
-
-
- If it is absent, then you must configure the tap device and whatever
- arping and routing you will need on the host. However, even in this
- case, the uml_net helper still needs to be in your path and it must be
- setuid root if you're not running UML as root. This is because the
- tap device doesn't support SIGIO, which UML needs in order to use
- something as a source of input. So, the helper is used as a
- convenient asynchronous IO thread.
-
- If you're using the uml_net helper, you can ignore the following host
- setup - uml_net will do it for you. You just need to make sure you
- have ethertap available, either built in to the host kernel or
- available as a module.
-
-
- If you want to set things up yourself, you need to make sure that the
- appropriate /dev entry exists. If it doesn't, become root and create
- it as follows::
-
-
- mknod /dev/tap <minor> c 36 <minor> + 16
-
-
-
-
- For example, this is how to create /dev/tap0::
-
-
- mknod /dev/tap0 c 36 0 + 16
-
-
-
-
- You also need to make sure that the host kernel has ethertap support.
- If ethertap is enabled as a module, you apparently need to insmod
- ethertap once for each ethertap device you want to enable. So,::
-
-
- host#
- insmod ethertap
-
-
-
-
- will give you the tap0 interface. To get the tap1 interface, you need
- to run::
-
-
- host#
- insmod ethertap unit=1 -o ethertap1
-
-
-
-
-
-
-
-6.9. The switch daemon
------------------------
-
- Note: This is the daemon formerly known as uml_router, but which was
- renamed so the network weenies of the world would stop growling at me.
-
-
- The switch daemon, uml_switch, provides a mechanism for creating a
- totally virtual network. By default, it provides no connection to the
- host network (but see -tap, below).
-
-
- The first thing you need to do is run the daemon. Running it with no
- arguments will make it listen on a default pair of unix domain
- sockets.
-
-
- If you want it to listen on a different pair of sockets, use::
-
-
- -unix control socket data socket
-
-
-
-
-
- If you want it to act as a hub rather than a switch, use::
-
-
- -hub
-
-
-
-
-
- If you want the switch to be connected to host networking (allowing
- the umls to get access to the outside world through the host), use::
-
-
- -tap tap0
-
-
-
-
-
- Note that the tap device must be preconfigured (see "TUN/TAP with a
- preconfigured tap device", above). If you're using a different tap
- device than tap0, specify that instead of tap0.
-
-
- uml_switch can be backgrounded as follows::
-
-
- host%
- uml_switch [ options ] < /dev/null > /dev/null
-
-
-
-
- The reason it doesn't background by default is that it listens to
- stdin for EOF. When it sees that, it exits.
-
-
- The general format of the kernel command line switch is::
-
-
-
- ethn=daemon,ethernet address,socket
- type,control socket,data socket
-
-
-
-
- You can leave off everything except the 'daemon'. You only need to
- specify the ethernet address if the one that will be assigned to it
- isn't acceptable for some reason. The rest of the arguments describe
- how to communicate with the daemon. You should only specify them if
- you told the daemon to use different sockets than the default. So, if
- you ran the daemon with no arguments, running the UML on the same
- machine with::
-
- eth0=daemon
-
-
-
-
- will cause the eth0 driver to attach itself to the daemon correctly.
-
-
-
-6.10. Slip
------------
-
- Slip is another, less general, mechanism for a process to communicate
- with the host networking. In contrast to the ethertap interface,
- which exchanges ethernet frames with the host and can be used to
- transport any higher-level protocol, it can only be used to transport
- IP.
-
-
- The general format of the command line switch is::
-
-
-
- ethn=slip,slip IP
-
-
-
-
- The slip IP argument is the IP address that will be assigned to the
- host end of the slip device. If it is specified, the helper will run
- and will set up the host so that the virtual machine can reach it and
- the rest of the network.
-
-
- There are some oddities with this interface that you should be aware
- of. You should only specify one slip device on a given virtual
- machine, and its name inside UML will be 'umn', not 'eth0' or whatever
- you specified on the command line. These problems will be fixed at
- some point.
-
-
-
-6.11. Slirp
-------------
-
- slirp uses an external program, usually /usr/bin/slirp, to provide IP
- only networking connectivity through the host. This is similar to IP
- masquerading with a firewall, although the translation is performed in
- user-space, rather than by the kernel. As slirp does not set up any
- interfaces on the host, or changes routing, slirp does not require
- root access or setuid binaries on the host.
-
-
- The general format of the command line switch for slirp is::
-
-
-
- ethn=slirp,ethernet address,slirp path
-
-
-
-
- The ethernet address is optional, as UML will set up the interface
- with an ethernet address based upon the initial IP address of the
- interface. The slirp path is generally /usr/bin/slirp, although it
- will depend on distribution.
-
-
- The slirp program can have a number of options passed to the command
- line and we can't add them to the UML command line, as they will be
- parsed incorrectly. Instead, a wrapper shell script can be written or
- the options inserted into the /.slirprc file. More information on
- all of the slirp options can be found in its man pages.
-
-
- The eth0 interface on UML should be set up with the IP 10.2.0.15,
- although you can use anything as long as it is not used by a network
- you will be connecting to. The default route on UML should be set to
- use::
-
-
- UML#
- route add default dev eth0
-
-
-
-
- slirp provides a number of useful IP addresses which can be used by
- UML, such as 10.0.2.3 which is an alias for the DNS server specified
- in /etc/resolv.conf on the host or the IP given in the 'dns' option
- for slirp.
-
-
- Even with a baudrate setting higher than 115200, the slirp connection
- is limited to 115200. If you need it to go faster, the slirp binary
- needs to be compiled with FULL_BOLT defined in config.h.
-
-
-
-6.12. pcap
------------
-
- The pcap transport is attached to a UML ethernet device on the command
- line or with uml_mconsole with the following syntax::
-
-
-
- ethn=pcap,host interface,filter
- expression,option1,option2
-
-
-
-
- The expression and options are optional.
-
-
- The interface is whatever network device on the host you want to
- sniff. The expression is a pcap filter expression, which is also what
- tcpdump uses, so if you know how to specify tcpdump filters, you will
- use the same expressions here. The options are up to two of
- 'promisc', control whether pcap puts the host interface into
- promiscuous mode. 'optimize' and 'nooptimize' control whether the pcap
- expression optimizer is used.
-
-
- Example::
-
-
-
- eth0=pcap,eth0,tcp
-
- eth1=pcap,eth0,!tcp
-
-
-
- will cause the UML eth0 to emit all tcp packets on the host eth0 and
- the UML eth1 to emit all non-tcp packets on the host eth0.
-
-
-
-6.13. Setting up the host yourself
------------------------------------
-
- If you don't specify an address for the host side of the ethertap or
- slip device, UML won't do any setup on the host. So this is what is
- needed to get things working (the examples use a host-side IP of
- 192.168.0.251 and a UML-side IP of 192.168.0.250 - adjust to suit your
- own network):
-
- - The device needs to be configured with its IP address. Tap devices
- are also configured with an mtu of 1484. Slip devices are
- configured with a point-to-point address pointing at the UML ip
- address::
-
-
- host# ifconfig tap0 arp mtu 1484 192.168.0.251 up
-
-
- host#
- ifconfig sl0 192.168.0.251 pointopoint 192.168.0.250 up
-
-
-
-
-
- - If a tap device is being set up, a route is set to the UML IP::
-
-
- UML# route add -host 192.168.0.250 gw 192.168.0.251
-
-
-
-
-
- - To allow other hosts on your network to see the virtual machine,
- proxy arp is set up for it::
-
-
- host# arp -Ds 192.168.0.250 eth0 pub
-
-
-
-
-
- - Finally, the host is set up to route packets::
-
-
- host# echo 1 > /proc/sys/net/ipv4/ip_forward
-
-
-
-
-
-
-
-
-
-
-7. Sharing Filesystems between Virtual Machines
-================================================
-
-
-
-
-7.1. A warning
----------------
-
- Don't attempt to share filesystems simply by booting two UMLs from the
- same file. That's the same thing as booting two physical machines
- from a shared disk. It will result in filesystem corruption.
-
-
-
-7.2. Using layered block devices
----------------------------------
-
- The way to share a filesystem between two virtual machines is to use
- the copy-on-write (COW) layering capability of the ubd block driver.
- As of 2.4.6-2um, the driver supports layering a read-write private
- device over a read-only shared device. A machine's writes are stored
- in the private device, while reads come from either device - the
- private one if the requested block is valid in it, the shared one if
- not. Using this scheme, the majority of data which is unchanged is
- shared between an arbitrary number of virtual machines, each of which
- has a much smaller file containing the changes that it has made. With
- a large number of UMLs booting from a large root filesystem, this
- leads to a huge disk space saving. It will also help performance,
- since the host will be able to cache the shared data using a much
- smaller amount of memory, so UML disk requests will be served from the
- host's memory rather than its disks.
-
-
-
-
- To add a copy-on-write layer to an existing block device file, simply
- add the name of the COW file to the appropriate ubd switch::
-
-
- ubd0=root_fs_cow,root_fs_debian_22
-
-
-
-
- where 'root_fs_cow' is the private COW file and 'root_fs_debian_22' is
- the existing shared filesystem. The COW file need not exist. If it
- doesn't, the driver will create and initialize it. Once the COW file
- has been initialized, it can be used on its own on the command line::
-
-
- ubd0=root_fs_cow
-
-
-
-
- The name of the backing file is stored in the COW file header, so it
- would be redundant to continue specifying it on the command line.
-
-
-
-7.3. Note!
------------
-
- When checking the size of the COW file in order to see the gobs of
- space that you're saving, make sure you use 'ls -ls' to see the actual
- disk consumption rather than the length of the file. The COW file is
- sparse, so the length will be very different from the disk usage.
- Here is a 'ls -l' of a COW file and backing file from one boot and
- shutdown::
-
- host% ls -l cow.debian debian2.2
- -rw-r--r-- 1 jdike jdike 492504064 Aug 6 21:16 cow.debian
- -rwxrw-rw- 1 jdike jdike 537919488 Aug 6 20:42 debian2.2
-
-
-
-
- Doesn't look like much saved space, does it? Well, here's 'ls -ls'::
-
-
- host% ls -ls cow.debian debian2.2
- 880 -rw-r--r-- 1 jdike jdike 492504064 Aug 6 21:16 cow.debian
- 525832 -rwxrw-rw- 1 jdike jdike 537919488 Aug 6 20:42 debian2.2
-
-
-
-
- Now, you can see that the COW file has less than a meg of disk, rather
- than 492 meg.
-
-
-
-7.4. Another warning
----------------------
-
- Once a filesystem is being used as a readonly backing file for a COW
- file, do not boot directly from it or modify it in any way. Doing so
- will invalidate any COW files that are using it. The mtime and size
- of the backing file are stored in the COW file header at its creation,
- and they must continue to match. If they don't, the driver will
- refuse to use the COW file.
-
-
-
-
- If you attempt to evade this restriction by changing either the
- backing file or the COW header by hand, you will get a corrupted
- filesystem.
-
-
-
-
- Among other things, this means that upgrading the distribution in a
- backing file and expecting that all of the COW files using it will see
- the upgrade will not work.
-
-
-
-
-7.5. uml_moo : Merging a COW file with its backing file
---------------------------------------------------------
-
- Depending on how you use UML and COW devices, it may be advisable to
- merge the changes in the COW file into the backing file every once in
- a while.
-
-
-
-
- The utility that does this is uml_moo. Its usage is::
-
-
- host% uml_moo COW file new backing file
-
-
-
-
- There's no need to specify the backing file since that information is
- already in the COW file header. If you're paranoid, boot the new
- merged file, and if you're happy with it, move it over the old backing
- file.
-
-
-
-
- uml_moo creates a new backing file by default as a safety measure. It
- also has a destructive merge option which will merge the COW file
- directly into its current backing file. This is really only usable
- when the backing file only has one COW file associated with it. If
- there are multiple COWs associated with a backing file, a -d merge of
- one of them will invalidate all of the others. However, it is
- convenient if you're short of disk space, and it should also be
- noticeably faster than a non-destructive merge.
-
-
-
-
- uml_moo is installed with the UML deb and RPM. If you didn't install
- UML from one of those packages, you can also get it from the UML
- utilities http://user-mode-linux.sourceforge.net/utilities tar file
- in tools/moo.
-
-
-
-
-
-
-
-
-8. Creating filesystems
-========================
-
-
- You may want to create and mount new UML filesystems, either because
- your root filesystem isn't large enough or because you want to use a
- filesystem other than ext2.
-
-
- This was written on the occasion of reiserfs being included in the
- 2.4.1 kernel pool, and therefore the 2.4.1 UML, so the examples will
- talk about reiserfs. This information is generic, and the examples
- should be easy to translate to the filesystem of your choice.
-
-
-8.1. Create the filesystem file
-================================
-
- dd is your friend. All you need to do is tell dd to create an empty
- file of the appropriate size. I usually make it sparse to save time
- and to avoid allocating disk space until it's actually used. For
- example, the following command will create a sparse 100 meg file full
- of zeroes::
-
-
- host%
- dd if=/dev/zero of=new_filesystem seek=100 count=1 bs=1M
-
-
-
-
-
-
- 8.2. Assign the file to a UML device
-
- Add an argument like the following to the UML command line::
-
- ubd4=new_filesystem
-
-
-
-
- making sure that you use an unassigned ubd device number.
-
-
-
- 8.3. Creating and mounting the filesystem
-
- Make sure that the filesystem is available, either by being built into
- the kernel, or available as a module, then boot up UML and log in. If
- the root filesystem doesn't have the filesystem utilities (mkfs, fsck,
- etc), then get them into UML by way of the net or hostfs.
-
-
- Make the new filesystem on the device assigned to the new file::
-
-
- host# mkreiserfs /dev/ubd/4
-
-
- <----------- MKREISERFSv2 ----------->
-
- ReiserFS version 3.6.25
- Block size 4096 bytes
- Block count 25856
- Used blocks 8212
- Journal - 8192 blocks (18-8209), journal header is in block 8210
- Bitmaps: 17
- Root block 8211
- Hash function "r5"
- ATTENTION: ALL DATA WILL BE LOST ON '/dev/ubd/4'! (y/n)y
- journal size 8192 (from 18)
- Initializing journal - 0%....20%....40%....60%....80%....100%
- Syncing..done.
-
-
-
-
- Now, mount it::
-
-
- UML#
- mount /dev/ubd/4 /mnt
-
-
-
-
- and you're in business.
-
-
-
-
-
-
-
-
-
-9. Host file access
-====================
-
-
- If you want to access files on the host machine from inside UML, you
- can treat it as a separate machine and either nfs mount directories
- from the host or copy files into the virtual machine with scp or rcp.
- However, since UML is running on the host, it can access those
- files just like any other process and make them available inside the
- virtual machine without needing to use the network.
-
-
- This is now possible with the hostfs virtual filesystem. With it, you
- can mount a host directory into the UML filesystem and access the
- files contained in it just as you would on the host.
-
-
-9.1. Using hostfs
-------------------
-
- To begin with, make sure that hostfs is available inside the virtual
- machine with::
-
-
- UML# cat /proc/filesystems
-
-
-
- . hostfs should be listed. If it's not, either rebuild the kernel
- with hostfs configured into it or make sure that hostfs is built as a
- module and available inside the virtual machine, and insmod it.
-
-
- Now all you need to do is run mount::
-
-
- UML# mount none /mnt/host -t hostfs
-
-
-
-
- will mount the host's / on the virtual machine's /mnt/host.
-
-
- If you don't want to mount the host root directory, then you can
- specify a subdirectory to mount with the -o switch to mount::
-
-
- UML# mount none /mnt/home -t hostfs -o /home
-
-
-
-
- will mount the hosts's /home on the virtual machine's /mnt/home.
-
-
-
-9.2. hostfs as the root filesystem
------------------------------------
-
- It's possible to boot from a directory hierarchy on the host using
- hostfs rather than using the standard filesystem in a file.
-
- To start, you need that hierarchy. The easiest way is to loop mount
- an existing root_fs file::
-
-
- host# mount root_fs uml_root_dir -o loop
-
-
-
-
- You need to change the filesystem type of / in etc/fstab to be
- 'hostfs', so that line looks like this::
-
- /dev/ubd/0 / hostfs defaults 1 1
-
-
-
-
- Then you need to chown to yourself all the files in that directory
- that are owned by root. This worked for me::
-
-
- host# find . -uid 0 -exec chown jdike {} \;
-
-
-
-
- Next, make sure that your UML kernel has hostfs compiled in, not as a
- module. Then run UML with the boot device pointing at that directory::
-
-
- ubd0=/path/to/uml/root/directory
-
-
-
-
- UML should then boot as it does normally.
-
-
-9.3. Building hostfs
----------------------
-
- If you need to build hostfs because it's not in your kernel, you have
- two choices:
-
-
-
- - Compiling hostfs into the kernel:
-
-
- Reconfigure the kernel and set the 'Host filesystem' option under
-
-
- - Compiling hostfs as a module:
-
-
- Reconfigure the kernel and set the 'Host filesystem' option under
- be in arch/um/fs/hostfs/hostfs.o. Install that in
- ``/lib/modules/$(uname -r)/fs`` in the virtual machine, boot it up, and::
-
-
- UML# insmod hostfs
-
-
-.. _The_Management_Console:
-
-10. The Management Console
-===========================
-
-
-
- The UML management console is a low-level interface to the kernel,
- somewhat like the i386 SysRq interface. Since there is a full-blown
- operating system under UML, there is much greater flexibility possible
- than with the SysRq mechanism.
-
-
- There are a number of things you can do with the mconsole interface:
-
- - get the kernel version
-
- - add and remove devices
-
- - halt or reboot the machine
-
- - Send SysRq commands
-
- - Pause and resume the UML
-
-
- You need the mconsole client (uml_mconsole) which is present in CVS
- (/tools/mconsole) in 2.4.5-9um and later, and will be in the RPM in
- 2.4.6.
-
-
- You also need CONFIG_MCONSOLE (under 'General Setup') enabled in UML.
- When you boot UML, you'll see a line like::
-
-
- mconsole initialized on /home/jdike/.uml/umlNJ32yL/mconsole
-
-
-
-
- If you specify a unique machine id one the UML command line, i.e.::
-
-
- umid=debian
-
-
-
-
- you'll see this::
-
-
- mconsole initialized on /home/jdike/.uml/debian/mconsole
-
-
-
-
- That file is the socket that uml_mconsole will use to communicate with
- UML. Run it with either the umid or the full path as its argument::
-
-
- host% uml_mconsole debian
-
-
-
-
- or::
-
-
- host% uml_mconsole /home/jdike/.uml/debian/mconsole
-
-
-
-
- You'll get a prompt, at which you can run one of these commands:
-
- - version
-
- - halt
-
- - reboot
-
- - config
-
- - remove
-
- - sysrq
-
- - help
-
- - cad
-
- - stop
-
- - go
-
-
-10.1. version
---------------
-
- This takes no arguments. It prints the UML version::
-
-
- (mconsole) version
- OK Linux usermode 2.4.5-9um #1 Wed Jun 20 22:47:08 EDT 2001 i686
-
-
-
-
- There are a couple actual uses for this. It's a simple no-op which
- can be used to check that a UML is running. It's also a way of
- sending an interrupt to the UML. This is sometimes useful on SMP
- hosts, where there's a bug which causes signals to UML to be lost,
- often causing it to appear to hang. Sending such a UML the mconsole
- version command is a good way to 'wake it up' before networking has
- been enabled, as it does not do anything to the function of the UML.
-
-
-
-10.2. halt and reboot
-----------------------
-
- These take no arguments. They shut the machine down immediately, with
- no syncing of disks and no clean shutdown of userspace. So, they are
- pretty close to crashing the machine::
-
-
- (mconsole) halt
- OK
-
-
-
-
-
-
-10.3. config
--------------
-
- "config" adds a new device to the virtual machine. Currently the ubd
- and network drivers support this. It takes one argument, which is the
- device to add, with the same syntax as the kernel command line::
-
-
-
-
- (mconsole)
- config ubd3=/home/jdike/incoming/roots/root_fs_debian22
-
- OK
- (mconsole) config eth1=mcast
- OK
-
-
-
-
-
-
-10.4. remove
--------------
-
- "remove" deletes a device from the system. Its argument is just the
- name of the device to be removed. The device must be idle in whatever
- sense the driver considers necessary. In the case of the ubd driver,
- the removed block device must not be mounted, swapped on, or otherwise
- open, and in the case of the network driver, the device must be down::
-
-
- (mconsole) remove ubd3
- OK
- (mconsole) remove eth1
- OK
-
-
-
-
-
-
-10.5. sysrq
-------------
-
- This takes one argument, which is a single letter. It calls the
- generic kernel's SysRq driver, which does whatever is called for by
- that argument. See the SysRq documentation in
- Documentation/admin-guide/sysrq.rst in your favorite kernel tree to
- see what letters are valid and what they do.
-
-
-
-10.6. help
------------
-
- "help" returns a string listing the valid commands and what each one
- does.
-
-
-
-10.7. cad
-----------
-
- This invokes the Ctl-Alt-Del action on init. What exactly this ends
- up doing is up to /etc/inittab. Normally, it reboots the machine.
- With UML, this is usually not desired, so if a halt would be better,
- then find the section of inittab that looks like this::
-
-
- # What to do when CTRL-ALT-DEL is pressed.
- ca:12345:ctrlaltdel:/sbin/shutdown -t1 -a -r now
-
-
-
-
- and change the command to halt.
-
-
-
-10.8. stop
------------
-
- This puts the UML in a loop reading mconsole requests until a 'go'
- mconsole command is received. This is very useful for making backups
- of UML filesystems, as the UML can be stopped, then synced via 'sysrq
- s', so that everything is written to the filesystem. You can then copy
- the filesystem and then send the UML 'go' via mconsole.
-
-
- Note that a UML running with more than one CPU will have problems
- after you send the 'stop' command, as only one CPU will be held in a
- mconsole loop and all others will continue as normal. This is a bug,
- and will be fixed.
-
-
-
-10.9. go
----------
-
- This resumes a UML after being paused by a 'stop' command. Note that
- when the UML has resumed, TCP connections may have timed out and if
- the UML is paused for a long period of time, crond might go a little
- crazy, running all the jobs it didn't do earlier.
-
-
-
-
-
-
-.. _Kernel_debugging:
-
-11. Kernel debugging
-=====================
-
-
- Note: The interface that makes debugging, as described here, possible
- is present in 2.4.0-test6 kernels and later.
-
-
- Since the user-mode kernel runs as a normal Linux process, it is
- possible to debug it with gdb almost like any other process. It is
- slightly different because the kernel's threads are already being
- ptraced for system call interception, so gdb can't ptrace them.
- However, a mechanism has been added to work around that problem.
-
-
- In order to debug the kernel, you need build it from source. See
- :ref:`Compiling_the_kernel_and_modules` for information on doing that.
- Make sure that you enable CONFIG_DEBUGSYM and CONFIG_PT_PROXY during
- the config. These will compile the kernel with ``-g``, and enable the
- ptrace proxy so that gdb works with UML, respectively.
-
-
-
-
-11.1. Starting the kernel under gdb
-------------------------------------
-
- You can have the kernel running under the control of gdb from the
- beginning by putting 'debug' on the command line. You will get an
- xterm with gdb running inside it. The kernel will send some commands
- to gdb which will leave it stopped at the beginning of start_kernel.
- At this point, you can get things going with 'next', 'step', or
- 'cont'.
-
-
- There is a transcript of a debugging session here <debug-
- session.html> , with breakpoints being set in the scheduler and in an
- interrupt handler.
-
-
-11.2. Examining sleeping processes
------------------------------------
-
-
- Not every bug is evident in the currently running process. Sometimes,
- processes hang in the kernel when they shouldn't because they've
- deadlocked on a semaphore or something similar. In this case, when
- you ^C gdb and get a backtrace, you will see the idle thread, which
- isn't very relevant.
-
-
- What you want is the stack of whatever process is sleeping when it
- shouldn't be. You need to figure out which process that is, which is
- generally fairly easy. Then you need to get its host process id,
- which you can do either by looking at ps on the host or at
- task.thread.extern_pid in gdb.
-
-
- Now what you do is this:
-
- - detach from the current thread::
-
-
- (UML gdb) det
-
-
-
-
-
- - attach to the thread you are interested in::
-
-
- (UML gdb) att <host pid>
-
-
-
-
-
- - look at its stack and anything else of interest::
-
-
- (UML gdb) bt
-
-
-
-
- Note that you can't do anything at this point that requires that a
- process execute, e.g. calling a function
-
- - when you're done looking at that process, reattach to the current
- thread and continue it::
-
-
- (UML gdb)
- att 1
-
-
- (UML gdb)
- c
-
-
-
-
- Here, specifying any pid which is not the process id of a UML thread
- will cause gdb to reattach to the current thread. I commonly use 1,
- but any other invalid pid would work.
-
-
-
-11.3. Running ddd on UML
--------------------------
-
- ddd works on UML, but requires a special kludge. The process goes
- like this:
-
- - Start ddd::
-
-
- host% ddd linux
-
-
-
-
-
- - With ps, get the pid of the gdb that ddd started. You can ask the
- gdb to tell you, but for some reason that confuses things and
- causes a hang.
-
- - run UML with 'debug=parent gdb-pid=<pid>' added to the command line
- - it will just sit there after you hit return
-
- - type 'att 1' to the ddd gdb and you will see something like::
-
-
- 0xa013dc51 in __kill ()
-
-
- (gdb)
-
-
-
-
-
- - At this point, type 'c', UML will boot up, and you can use ddd just
- as you do on any other process.
-
-
-
-11.4. Debugging modules
-------------------------
-
-
- gdb has support for debugging code which is dynamically loaded into
- the process. This support is what is needed to debug kernel modules
- under UML.
-
-
- Using that support is somewhat complicated. You have to tell gdb what
- object file you just loaded into UML and where in memory it is. Then,
- it can read the symbol table, and figure out where all the symbols are
- from the load address that you provided. It gets more interesting
- when you load the module again (i.e. after an rmmod). You have to
- tell gdb to forget about all its symbols, including the main UML ones
- for some reason, then load then all back in again.
-
-
- There's an easy way and a hard way to do this. The easy way is to use
- the umlgdb expect script written by Chandan Kudige. It basically
- automates the process for you.
-
-
- First, you must tell it where your modules are. There is a list in
- the script that looks like this::
-
- set MODULE_PATHS {
- "fat" "/usr/src/uml/linux-2.4.18/fs/fat/fat.o"
- "isofs" "/usr/src/uml/linux-2.4.18/fs/isofs/isofs.o"
- "minix" "/usr/src/uml/linux-2.4.18/fs/minix/minix.o"
- }
-
-
-
-
- You change that to list the names and paths of the modules that you
- are going to debug. Then you run it from the toplevel directory of
- your UML pool and it basically tells you what to do::
-
-
- ******** GDB pid is 21903 ********
- Start UML as: ./linux <kernel switches> debug gdb-pid=21903
-
-
-
- GNU gdb 5.0rh-5 Red Hat Linux 7.1
- Copyright 2001 Free Software Foundation, Inc.
- GDB is free software, covered by the GNU General Public License, and you are
- welcome to change it and/or distribute copies of it under certain conditions.
- Type "show copying" to see the conditions.
- There is absolutely no warranty for GDB. Type "show warranty" for details.
- This GDB was configured as "i386-redhat-linux"...
- (gdb) b sys_init_module
- Breakpoint 1 at 0xa0011923: file module.c, line 349.
- (gdb) att 1
-
-
-
-
- After you run UML and it sits there doing nothing, you hit return at
- the 'att 1' and continue it::
-
-
- Attaching to program: /home/jdike/linux/2.4/um/./linux, process 1
- 0xa00f4221 in __kill ()
- (UML gdb) c
- Continuing.
-
-
-
-
- At this point, you debug normally. When you insmod something, the
- expect magic will kick in and you'll see something like::
-
-
- *** Module hostfs loaded ***
- Breakpoint 1, sys_init_module (name_user=0x805abb0 "hostfs",
- mod_user=0x8070e00) at module.c:349
- 349 char *name, *n_name, *name_tmp = NULL;
- (UML gdb) finish
- Run till exit from #0 sys_init_module (name_user=0x805abb0 "hostfs",
- mod_user=0x8070e00) at module.c:349
- 0xa00e2e23 in execute_syscall (r=0xa8140284) at syscall_kern.c:411
- 411 else res = EXECUTE_SYSCALL(syscall, regs);
- Value returned is $1 = 0
- (UML gdb)
- p/x (int)module_list + module_list->size_of_struct
-
- $2 = 0xa9021054
- (UML gdb) symbol-file ./linux
- Load new symbol table from "./linux"? (y or n) y
- Reading symbols from ./linux...
- done.
- (UML gdb)
- add-symbol-file /home/jdike/linux/2.4/um/arch/um/fs/hostfs/hostfs.o 0xa9021054
-
- add symbol table from file "/home/jdike/linux/2.4/um/arch/um/fs/hostfs/hostfs.o" at
- .text_addr = 0xa9021054
- (y or n) y
-
- Reading symbols from /home/jdike/linux/2.4/um/arch/um/fs/hostfs/hostfs.o...
- done.
- (UML gdb) p *module_list
- $1 = {size_of_struct = 84, next = 0xa0178720, name = 0xa9022de0 "hostfs",
- size = 9016, uc = {usecount = {counter = 0}, pad = 0}, flags = 1,
- nsyms = 57, ndeps = 0, syms = 0xa9023170, deps = 0x0, refs = 0x0,
- init = 0xa90221f0 <init_hostfs>, cleanup = 0xa902222c <exit_hostfs>,
- ex_table_start = 0x0, ex_table_end = 0x0, persist_start = 0x0,
- persist_end = 0x0, can_unload = 0, runsize = 0, kallsyms_start = 0x0,
- kallsyms_end = 0x0,
- archdata_start = 0x1b855 <Address 0x1b855 out of bounds>,
- archdata_end = 0xe5890000 <Address 0xe5890000 out of bounds>,
- kernel_data = 0xf689c35d <Address 0xf689c35d out of bounds>}
- >> Finished loading symbols for hostfs ...
-
-
-
-
- That's the easy way. It's highly recommended. The hard way is
- described below in case you're interested in what's going on.
-
-
- Boot the kernel under the debugger and load the module with insmod or
- modprobe. With gdb, do::
-
-
- (UML gdb) p module_list
-
-
-
-
- This is a list of modules that have been loaded into the kernel, with
- the most recently loaded module first. Normally, the module you want
- is at module_list. If it's not, walk down the next links, looking at
- the name fields until find the module you want to debug. Take the
- address of that structure, and add module.size_of_struct (which in
- 2.4.10 kernels is 96 (0x60)) to it. Gdb can make this hard addition
- for you :-)::
-
-
-
- (UML gdb)
- printf "%#x\n", (int)module_list module_list->size_of_struct
-
-
-
-
- The offset from the module start occasionally changes (before 2.4.0,
- it was module.size_of_struct + 4), so it's a good idea to check the
- init and cleanup addresses once in a while, as describe below. Now
- do::
-
-
- (UML gdb)
- add-symbol-file /path/to/module/on/host that_address
-
-
-
-
- Tell gdb you really want to do it, and you're in business.
-
-
- If there's any doubt that you got the offset right, like breakpoints
- appear not to work, or they're appearing in the wrong place, you can
- check it by looking at the module structure. The init and cleanup
- fields should look like::
-
-
- init = 0x588066b0 <init_hostfs>, cleanup = 0x588066c0 <exit_hostfs>
-
-
-
-
- with no offsets on the symbol names. If the names are right, but they
- are offset, then the offset tells you how much you need to add to the
- address you gave to add-symbol-file.
-
-
- When you want to load in a new version of the module, you need to get
- gdb to forget about the old one. The only way I've found to do that
- is to tell gdb to forget about all symbols that it knows about::
-
-
- (UML gdb) symbol-file
-
-
-
-
- Then reload the symbols from the kernel binary::
-
-
- (UML gdb) symbol-file /path/to/kernel
-
-
-
-
- and repeat the process above. You'll also need to re-enable break-
- points. They were disabled when you dumped all the symbols because
- gdb couldn't figure out where they should go.
-
-
-
-11.5. Attaching gdb to the kernel
-----------------------------------
-
- If you don't have the kernel running under gdb, you can attach gdb to
- it later by sending the tracing thread a SIGUSR1. The first line of
- the console output identifies its pid::
-
- tracing thread pid = 20093
-
-
-
-
- When you send it the signal::
-
-
- host% kill -USR1 20093
-
-
-
-
- you will get an xterm with gdb running in it.
-
-
- If you have the mconsole compiled into UML, then the mconsole client
- can be used to start gdb::
-
-
- (mconsole) (mconsole) config gdb=xterm
-
-
-
-
- will fire up an xterm with gdb running in it.
-
-
-
-11.6. Using alternate debuggers
---------------------------------
-
- UML has support for attaching to an already running debugger rather
- than starting gdb itself. This is present in CVS as of 17 Apr 2001.
- I sent it to Alan for inclusion in the ac tree, and it will be in my
- 2.4.4 release.
-
-
- This is useful when gdb is a subprocess of some UI, such as emacs or
- ddd. It can also be used to run debuggers other than gdb on UML.
- Below is an example of using strace as an alternate debugger.
-
-
- To do this, you need to get the pid of the debugger and pass it in
- with the
-
-
- If you are using gdb under some UI, then tell it to 'att 1', and
- you'll find yourself attached to UML.
-
-
- If you are using something other than gdb as your debugger, then
- you'll need to get it to do the equivalent of 'att 1' if it doesn't do
- it automatically.
-
-
- An example of an alternate debugger is strace. You can strace the
- actual kernel as follows:
-
- - Run the following in a shell::
-
-
- host%
- sh -c 'echo pid=$$; echo -n hit return; read x; exec strace -p 1 -o strace.out'
-
-
-
- - Run UML with 'debug' and 'gdb-pid=<pid>' with the pid printed out
- by the previous command
-
- - Hit return in the shell, and UML will start running, and strace
- output will start accumulating in the output file.
-
- Note that this is different from running::
-
-
- host% strace ./linux
-
-
-
-
- That will strace only the main UML thread, the tracing thread, which
- doesn't do any of the actual kernel work. It just oversees the vir-
- tual machine. In contrast, using strace as described above will show
- you the low-level activity of the virtual machine.
-
-
-
-
-
-12. Kernel debugging examples
-==============================
-
-12.1. The case of the hung fsck
---------------------------------
-
- When booting up the kernel, fsck failed, and dropped me into a shell
- to fix things up. I ran fsck -y, which hung::
-
-
- Setting hostname uml [ OK ]
- Checking root filesystem
- /dev/fhd0 was not cleanly unmounted, check forced.
- Error reading block 86894 (Attempt to read block from filesystem resulted in short read) while reading indirect blocks of inode 19780.
-
- /dev/fhd0: UNEXPECTED INCONSISTENCY; RUN fsck MANUALLY.
- (i.e., without -a or -p options)
- [ FAILED ]
-
- *** An error occurred during the file system check.
- *** Dropping you to a shell; the system will reboot
- *** when you leave the shell.
- Give root password for maintenance
- (or type Control-D for normal startup):
-
- [root@uml /root]# fsck -y /dev/fhd0
- fsck -y /dev/fhd0
- Parallelizing fsck version 1.14 (9-Jan-1999)
- e2fsck 1.14, 9-Jan-1999 for EXT2 FS 0.5b, 95/08/09
- /dev/fhd0 contains a file system with errors, check forced.
- Pass 1: Checking inodes, blocks, and sizes
- Error reading block 86894 (Attempt to read block from filesystem resulted in short read) while reading indirect blocks of inode 19780. Ignore error? yes
-
- Inode 19780, i_blocks is 1548, should be 540. Fix? yes
-
- Pass 2: Checking directory structure
- Error reading block 49405 (Attempt to read block from filesystem resulted in short read). Ignore error? yes
-
- Directory inode 11858, block 0, offset 0: directory corrupted
- Salvage? yes
-
- Missing '.' in directory inode 11858.
- Fix? yes
-
- Missing '..' in directory inode 11858.
- Fix? yes
-
-
- The standard drill in this sort of situation is to fire up gdb on the
- signal thread, which, in this case, was pid 1935. In another window,
- I run gdb and attach pid 1935::
-
-
- ~/linux/2.3.26/um 1016: gdb linux
- GNU gdb 4.17.0.11 with Linux support
- Copyright 1998 Free Software Foundation, Inc.
- GDB is free software, covered by the GNU General Public License, and you are
- welcome to change it and/or distribute copies of it under certain conditions.
- Type "show copying" to see the conditions.
- There is absolutely no warranty for GDB. Type "show warranty" for details.
- This GDB was configured as "i386-redhat-linux"...
-
- (gdb) att 1935
- Attaching to program `/home/dike/linux/2.3.26/um/linux', Pid 1935
- 0x100756d9 in __wait4 ()
-
-
- Let's see what's currently running::
-
-
-
- (gdb) p current_task.pid
- $1 = 0
-
-
-
-
-
- It's the idle thread, which means that fsck went to sleep for some
- reason and never woke up.
-
-
- Let's guess that the last process in the process list is fsck::
-
-
-
- (gdb) p current_task.prev_task.comm
- $13 = "fsck.ext2\000\000\000\000\000\000"
-
-
-
-
-
- It is, so let's see what it thinks it's up to::
-
-
-
- (gdb) p current_task.prev_task.thread
- $14 = {extern_pid = 1980, tracing = 0, want_tracing = 0, forking = 0,
- kernel_stack_page = 0, signal_stack = 1342627840, syscall = {id = 4, args = {
- 3, 134973440, 1024, 0, 1024}, have_result = 0, result = 50590720},
- request = {op = 2, u = {exec = {ip = 1350467584, sp = 2952789424}, fork = {
- regs = {1350467584, 2952789424, 0 <repeats 15 times>}, sigstack = 0,
- pid = 0}, switch_to = 0x507e8000, thread = {proc = 0x507e8000,
- arg = 0xaffffdb0, flags = 0, new_pid = 0}, input_request = {
- op = 1350467584, fd = -1342177872, proc = 0, pid = 0}}}}
-
-
-
- The interesting things here are the fact that its .thread.syscall.id
- is __NR_write (see the big switch in arch/um/kernel/syscall_kern.c or
- the defines in include/asm-um/arch/unistd.h), and that it never
- returned. Also, its .request.op is OP_SWITCH (see
- arch/um/include/user_util.h). These mean that it went into a write,
- and, for some reason, called schedule().
-
-
- The fact that it never returned from write means that its stack should
- be fairly interesting. Its pid is 1980 (.thread.extern_pid). That
- process is being ptraced by the signal thread, so it must be detached
- before gdb can attach it::
-
-
-
- (gdb) call detach(1980)
-
- Program received signal SIGSEGV, Segmentation fault.
- <function called from gdb>
- The program being debugged stopped while in a function called from GDB.
- When the function (detach) is done executing, GDB will silently
- stop (instead of continuing to evaluate the expression containing
- the function call).
- (gdb) call detach(1980)
- $15 = 0
-
-
- The first detach segfaults for some reason, and the second one
- succeeds.
-
-
- Now I detach from the signal thread, attach to the fsck thread, and
- look at its stack::
-
-
- (gdb) det
- Detaching from program: /home/dike/linux/2.3.26/um/linux Pid 1935
- (gdb) att 1980
- Attaching to program `/home/dike/linux/2.3.26/um/linux', Pid 1980
- 0x10070451 in __kill ()
- (gdb) bt
- #0 0x10070451 in __kill ()
- #1 0x10068ccd in usr1_pid (pid=1980) at process.c:30
- #2 0x1006a03f in _switch_to (prev=0x50072000, next=0x507e8000)
- at process_kern.c:156
- #3 0x1006a052 in switch_to (prev=0x50072000, next=0x507e8000, last=0x50072000)
- at process_kern.c:161
- #4 0x10001d12 in schedule () at core.c:777
- #5 0x1006a744 in __down (sem=0x507d241c) at semaphore.c:71
- #6 0x1006aa10 in __down_failed () at semaphore.c:157
- #7 0x1006c5d8 in segv_handler (sc=0x5006e940) at trap_user.c:174
- #8 0x1006c5ec in kern_segv_handler (sig=11) at trap_user.c:182
- #9 <signal handler called>
- #10 0x10155404 in errno ()
- #11 0x1006c0aa in segv (address=1342179328, is_write=2) at trap_kern.c:50
- #12 0x1006c5d8 in segv_handler (sc=0x5006eaf8) at trap_user.c:174
- #13 0x1006c5ec in kern_segv_handler (sig=11) at trap_user.c:182
- #14 <signal handler called>
- #15 0xc0fd in ?? ()
- #16 0x10016647 in sys_write (fd=3,
- buf=0x80b8800 <Address 0x80b8800 out of bounds>, count=1024)
- at read_write.c:159
- #17 0x1006d5b3 in execute_syscall (syscall=4, args=0x5006ef08)
- at syscall_kern.c:254
- #18 0x1006af87 in really_do_syscall (sig=12) at syscall_user.c:35
- #19 <signal handler called>
- #20 0x400dc8b0 in ?? ()
-
-
-
-
-
- The interesting things here are:
-
- - There are two segfaults on this stack (frames 9 and 14)
-
- - The first faulting address (frame 11) is 0x50000800::
-
- (gdb) p (void *)1342179328
- $16 = (void *) 0x50000800
-
-
-
-
-
- The initial faulting address is interesting because it is on the idle
- thread's stack. I had been seeing the idle thread segfault for no
- apparent reason, and the cause looked like stack corruption. In hopes
- of catching the culprit in the act, I had turned off all protections
- to that stack while the idle thread wasn't running. This apparently
- tripped that trap.
-
-
- However, the more immediate problem is that second segfault and I'm
- going to concentrate on that. First, I want to see where the fault
- happened, so I have to go look at the sigcontent struct in frame 8::
-
-
-
- (gdb) up
- #1 0x10068ccd in usr1_pid (pid=1980) at process.c:30
- 30 kill(pid, SIGUSR1);
- (gdb)
- #2 0x1006a03f in _switch_to (prev=0x50072000, next=0x507e8000)
- at process_kern.c:156
- 156 usr1_pid(getpid());
- (gdb)
- #3 0x1006a052 in switch_to (prev=0x50072000, next=0x507e8000, last=0x50072000)
- at process_kern.c:161
- 161 _switch_to(prev, next);
- (gdb)
- #4 0x10001d12 in schedule () at core.c:777
- 777 switch_to(prev, next, prev);
- (gdb)
- #5 0x1006a744 in __down (sem=0x507d241c) at semaphore.c:71
- 71 schedule();
- (gdb)
- #6 0x1006aa10 in __down_failed () at semaphore.c:157
- 157 }
- (gdb)
- #7 0x1006c5d8 in segv_handler (sc=0x5006e940) at trap_user.c:174
- 174 segv(sc->cr2, sc->err & 2);
- (gdb)
- #8 0x1006c5ec in kern_segv_handler (sig=11) at trap_user.c:182
- 182 segv_handler(sc);
- (gdb) p *sc
- Cannot access memory at address 0x0.
-
-
-
-
- That's not very useful, so I'll try a more manual method::
-
-
- (gdb) p *((struct sigcontext *) (&sig + 1))
- $19 = {gs = 0, __gsh = 0, fs = 0, __fsh = 0, es = 43, __esh = 0, ds = 43,
- __dsh = 0, edi = 1342179328, esi = 1350378548, ebp = 1342630440,
- esp = 1342630420, ebx = 1348150624, edx = 1280, ecx = 0, eax = 0,
- trapno = 14, err = 4, eip = 268480945, cs = 35, __csh = 0, eflags = 66118,
- esp_at_signal = 1342630420, ss = 43, __ssh = 0, fpstate = 0x0, oldmask = 0,
- cr2 = 1280}
-
-
-
- The ip is in handle_mm_fault::
-
-
- (gdb) p (void *)268480945
- $20 = (void *) 0x1000b1b1
- (gdb) i sym $20
- handle_mm_fault + 57 in section .text
-
-
-
-
-
- Specifically, it's in pte_alloc::
-
-
- (gdb) i line *$20
- Line 124 of "/home/dike/linux/2.3.26/um/include/asm/pgalloc.h"
- starts at address 0x1000b1b1 <handle_mm_fault+57>
- and ends at 0x1000b1b7 <handle_mm_fault+63>.
-
-
-
-
-
- To find where in handle_mm_fault this is, I'll jump forward in the
- code until I see an address in that procedure::
-
-
-
- (gdb) i line *0x1000b1c0
- Line 126 of "/home/dike/linux/2.3.26/um/include/asm/pgalloc.h"
- starts at address 0x1000b1b7 <handle_mm_fault+63>
- and ends at 0x1000b1c3 <handle_mm_fault+75>.
- (gdb) i line *0x1000b1d0
- Line 131 of "/home/dike/linux/2.3.26/um/include/asm/pgalloc.h"
- starts at address 0x1000b1d0 <handle_mm_fault+88>
- and ends at 0x1000b1da <handle_mm_fault+98>.
- (gdb) i line *0x1000b1e0
- Line 61 of "/home/dike/linux/2.3.26/um/include/asm/pgalloc.h"
- starts at address 0x1000b1da <handle_mm_fault+98>
- and ends at 0x1000b1e1 <handle_mm_fault+105>.
- (gdb) i line *0x1000b1f0
- Line 134 of "/home/dike/linux/2.3.26/um/include/asm/pgalloc.h"
- starts at address 0x1000b1f0 <handle_mm_fault+120>
- and ends at 0x1000b200 <handle_mm_fault+136>.
- (gdb) i line *0x1000b200
- Line 135 of "/home/dike/linux/2.3.26/um/include/asm/pgalloc.h"
- starts at address 0x1000b200 <handle_mm_fault+136>
- and ends at 0x1000b208 <handle_mm_fault+144>.
- (gdb) i line *0x1000b210
- Line 139 of "/home/dike/linux/2.3.26/um/include/asm/pgalloc.h"
- starts at address 0x1000b210 <handle_mm_fault+152>
- and ends at 0x1000b219 <handle_mm_fault+161>.
- (gdb) i line *0x1000b220
- Line 1168 of "memory.c" starts at address 0x1000b21e <handle_mm_fault+166>
- and ends at 0x1000b222 <handle_mm_fault+170>.
-
-
-
-
-
- Something is apparently wrong with the page tables or vma_structs, so
- lets go back to frame 11 and have a look at them::
-
-
-
- #11 0x1006c0aa in segv (address=1342179328, is_write=2) at trap_kern.c:50
- 50 handle_mm_fault(current, vma, address, is_write);
- (gdb) call pgd_offset_proc(vma->vm_mm, address)
- $22 = (pgd_t *) 0x80a548c
-
-
-
-
-
- That's pretty bogus. Page tables aren't supposed to be in process
- text or data areas. Let's see what's in the vma::
-
-
- (gdb) p *vma
- $23 = {vm_mm = 0x507d2434, vm_start = 0, vm_end = 134512640,
- vm_next = 0x80a4f8c, vm_page_prot = {pgprot = 0}, vm_flags = 31200,
- vm_avl_height = 2058, vm_avl_left = 0x80a8c94, vm_avl_right = 0x80d1000,
- vm_next_share = 0xaffffdb0, vm_pprev_share = 0xaffffe63,
- vm_ops = 0xaffffe7a, vm_pgoff = 2952789626, vm_file = 0xafffffec,
- vm_private_data = 0x62}
- (gdb) p *vma.vm_mm
- $24 = {mmap = 0x507d2434, mmap_avl = 0x0, mmap_cache = 0x8048000,
- pgd = 0x80a4f8c, mm_users = {counter = 0}, mm_count = {counter = 134904288},
- map_count = 134909076, mmap_sem = {count = {counter = 135073792},
- sleepers = -1342177872, wait = {lock = <optimized out or zero length>,
- task_list = {next = 0xaffffe63, prev = 0xaffffe7a},
- __magic = -1342177670, __creator = -1342177300}, __magic = 98},
- page_table_lock = {}, context = 138, start_code = 0, end_code = 0,
- start_data = 0, end_data = 0, start_brk = 0, brk = 0, start_stack = 0,
- arg_start = 0, arg_end = 0, env_start = 0, env_end = 0, rss = 1350381536,
- total_vm = 0, locked_vm = 0, def_flags = 0, cpu_vm_mask = 0, swap_cnt = 0,
- swap_address = 0, segments = 0x0}
-
-
-
- This also pretty bogus. With all of the 0x80xxxxx and 0xaffffxxx
- addresses, this is looking like a stack was plonked down on top of
- these structures. Maybe it's a stack overflow from the next page::
-
-
- (gdb) p vma
- $25 = (struct vm_area_struct *) 0x507d2434
-
-
-
- That's towards the lower quarter of the page, so that would have to
- have been pretty heavy stack overflow::
-
-
- (gdb) x/100x $25
- 0x507d2434: 0x507d2434 0x00000000 0x08048000 0x080a4f8c
- 0x507d2444: 0x00000000 0x080a79e0 0x080a8c94 0x080d1000
- 0x507d2454: 0xaffffdb0 0xaffffe63 0xaffffe7a 0xaffffe7a
- 0x507d2464: 0xafffffec 0x00000062 0x0000008a 0x00000000
- 0x507d2474: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d2484: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d2494: 0x00000000 0x00000000 0x507d2fe0 0x00000000
- 0x507d24a4: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d24b4: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d24c4: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d24d4: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d24e4: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d24f4: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d2504: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d2514: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d2524: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d2534: 0x00000000 0x00000000 0x507d25dc 0x00000000
- 0x507d2544: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d2554: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d2564: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d2574: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d2584: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d2594: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d25a4: 0x00000000 0x00000000 0x00000000 0x00000000
- 0x507d25b4: 0x00000000 0x00000000 0x00000000 0x00000000
-
-
-
- It's not stack overflow. The only "stack-like" piece of this data is
- the vma_struct itself.
-
-
- At this point, I don't see any avenues to pursue, so I just have to
- admit that I have no idea what's going on. What I will do, though, is
- stick a trap on the segfault handler which will stop if it sees any
- writes to the idle thread's stack. That was the thing that happened
- first, and it may be that if I can catch it immediately, what's going
- on will be somewhat clearer.
-
-
-12.2. Episode 2: The case of the hung fsck
--------------------------------------------
-
- After setting a trap in the SEGV handler for accesses to the signal
- thread's stack, I reran the kernel.
-
-
- fsck hung again, this time by hitting the trap::
-
-
-
- Setting hostname uml [ OK ]
- Checking root filesystem
- /dev/fhd0 contains a file system with errors, check forced.
- Error reading block 86894 (Attempt to read block from filesystem resulted in short read) while reading indirect blocks of inode 19780.
-
- /dev/fhd0: UNEXPECTED INCONSISTENCY; RUN fsck MANUALLY.
- (i.e., without -a or -p options)
- [ FAILED ]
-
- *** An error occurred during the file system check.
- *** Dropping you to a shell; the system will reboot
- *** when you leave the shell.
- Give root password for maintenance
- (or type Control-D for normal startup):
-
- [root@uml /root]# fsck -y /dev/fhd0
- fsck -y /dev/fhd0
- Parallelizing fsck version 1.14 (9-Jan-1999)
- e2fsck 1.14, 9-Jan-1999 for EXT2 FS 0.5b, 95/08/09
- /dev/fhd0 contains a file system with errors, check forced.
- Pass 1: Checking inodes, blocks, and sizes
- Error reading block 86894 (Attempt to read block from filesystem resulted in short read) while reading indirect blocks of inode 19780. Ignore error? yes
-
- Pass 2: Checking directory structure
- Error reading block 49405 (Attempt to read block from filesystem resulted in short read). Ignore error? yes
-
- Directory inode 11858, block 0, offset 0: directory corrupted
- Salvage? yes
-
- Missing '.' in directory inode 11858.
- Fix? yes
-
- Missing '..' in directory inode 11858.
- Fix? yes
-
- Untested (4127) [100fe44c]: trap_kern.c line 31
-
-
-
-
-
- I need to get the signal thread to detach from pid 4127 so that I can
- attach to it with gdb. This is done by sending it a SIGUSR1, which is
- caught by the signal thread, which detaches the process::
-
-
- kill -USR1 4127
-
-
-
-
-
- Now I can run gdb on it::
-
-
- ~/linux/2.3.26/um 1034: gdb linux
- GNU gdb 4.17.0.11 with Linux support
- Copyright 1998 Free Software Foundation, Inc.
- GDB is free software, covered by the GNU General Public License, and you are
- welcome to change it and/or distribute copies of it under certain conditions.
- Type "show copying" to see the conditions.
- There is absolutely no warranty for GDB. Type "show warranty" for details.
- This GDB was configured as "i386-redhat-linux"...
- (gdb) att 4127
- Attaching to program `/home/dike/linux/2.3.26/um/linux', Pid 4127
- 0x10075891 in __libc_nanosleep ()
-
-
-
-
-
- The backtrace shows that it was in a write and that the fault address
- (address in frame 3) is 0x50000800, which is right in the middle of
- the signal thread's stack page::
-
-
- (gdb) bt
- #0 0x10075891 in __libc_nanosleep ()
- #1 0x1007584d in __sleep (seconds=1000000)
- at ../sysdeps/unix/sysv/linux/sleep.c:78
- #2 0x1006ce9a in stop () at user_util.c:191
- #3 0x1006bf88 in segv (address=1342179328, is_write=2) at trap_kern.c:31
- #4 0x1006c628 in segv_handler (sc=0x5006eaf8) at trap_user.c:174
- #5 0x1006c63c in kern_segv_handler (sig=11) at trap_user.c:182
- #6 <signal handler called>
- #7 0xc0fd in ?? ()
- #8 0x10016647 in sys_write (fd=3, buf=0x80b8800 "R.", count=1024)
- at read_write.c:159
- #9 0x1006d603 in execute_syscall (syscall=4, args=0x5006ef08)
- at syscall_kern.c:254
- #10 0x1006af87 in really_do_syscall (sig=12) at syscall_user.c:35
- #11 <signal handler called>
- #12 0x400dc8b0 in ?? ()
- #13 <signal handler called>
- #14 0x400dc8b0 in ?? ()
- #15 0x80545fd in ?? ()
- #16 0x804daae in ?? ()
- #17 0x8054334 in ?? ()
- #18 0x804d23e in ?? ()
- #19 0x8049632 in ?? ()
- #20 0x80491d2 in ?? ()
- #21 0x80596b5 in ?? ()
- (gdb) p (void *)1342179328
- $3 = (void *) 0x50000800
-
-
-
- Going up the stack to the segv_handler frame and looking at where in
- the code the access happened shows that it happened near line 110 of
- block_dev.c::
-
-
-
- (gdb) up
- #1 0x1007584d in __sleep (seconds=1000000)
- at ../sysdeps/unix/sysv/linux/sleep.c:78
- ../sysdeps/unix/sysv/linux/sleep.c:78: No such file or directory.
- (gdb)
- #2 0x1006ce9a in stop () at user_util.c:191
- 191 while(1) sleep(1000000);
- (gdb)
- #3 0x1006bf88 in segv (address=1342179328, is_write=2) at trap_kern.c:31
- 31 KERN_UNTESTED();
- (gdb)
- #4 0x1006c628 in segv_handler (sc=0x5006eaf8) at trap_user.c:174
- 174 segv(sc->cr2, sc->err & 2);
- (gdb) p *sc
- $1 = {gs = 0, __gsh = 0, fs = 0, __fsh = 0, es = 43, __esh = 0, ds = 43,
- __dsh = 0, edi = 1342179328, esi = 134973440, ebp = 1342631484,
- esp = 1342630864, ebx = 256, edx = 0, ecx = 256, eax = 1024, trapno = 14,
- err = 6, eip = 268550834, cs = 35, __csh = 0, eflags = 66070,
- esp_at_signal = 1342630864, ss = 43, __ssh = 0, fpstate = 0x0, oldmask = 0,
- cr2 = 1342179328}
- (gdb) p (void *)268550834
- $2 = (void *) 0x1001c2b2
- (gdb) i sym $2
- block_write + 1090 in section .text
- (gdb) i line *$2
- Line 209 of "/home/dike/linux/2.3.26/um/include/asm/arch/string.h"
- starts at address 0x1001c2a1 <block_write+1073>
- and ends at 0x1001c2bf <block_write+1103>.
- (gdb) i line *0x1001c2c0
- Line 110 of "block_dev.c" starts at address 0x1001c2bf <block_write+1103>
- and ends at 0x1001c2e3 <block_write+1139>.
-
-
-
- Looking at the source shows that the fault happened during a call to
- copy_from_user to copy the data into the kernel::
-
-
- 107 count -= chars;
- 108 copy_from_user(p,buf,chars);
- 109 p += chars;
- 110 buf += chars;
-
-
-
- p is the pointer which must contain 0x50000800, since buf contains
- 0x80b8800 (frame 8 above). It is defined as::
-
-
- p = offset + bh->b_data;
-
-
-
-
-
- I need to figure out what bh is, and it just so happens that bh is
- passed as an argument to mark_buffer_uptodate and mark_buffer_dirty a
- few lines later, so I do a little disassembly::
-
-
- (gdb) disas 0x1001c2bf 0x1001c2e0
- Dump of assembler code from 0x1001c2bf to 0x1001c2d0:
- 0x1001c2bf <block_write+1103>: addl %eax,0xc(%ebp)
- 0x1001c2c2 <block_write+1106>: movl 0xfffffdd4(%ebp),%edx
- 0x1001c2c8 <block_write+1112>: btsl $0x0,0x18(%edx)
- 0x1001c2cd <block_write+1117>: btsl $0x1,0x18(%edx)
- 0x1001c2d2 <block_write+1122>: sbbl %ecx,%ecx
- 0x1001c2d4 <block_write+1124>: testl %ecx,%ecx
- 0x1001c2d6 <block_write+1126>: jne 0x1001c2e3 <block_write+1139>
- 0x1001c2d8 <block_write+1128>: pushl $0x0
- 0x1001c2da <block_write+1130>: pushl %edx
- 0x1001c2db <block_write+1131>: call 0x1001819c <__mark_buffer_dirty>
- End of assembler dump.
-
-
-
-
-
- At that point, bh is in %edx (address 0x1001c2da), which is calculated
- at 0x1001c2c2 as %ebp + 0xfffffdd4, so I figure exactly what that is,
- taking %ebp from the sigcontext_struct above::
-
-
- (gdb) p (void *)1342631484
- $5 = (void *) 0x5006ee3c
- (gdb) p 0x5006ee3c+0xfffffdd4
- $6 = 1342630928
- (gdb) p (void *)$6
- $7 = (void *) 0x5006ec10
- (gdb) p *((void **)$7)
- $8 = (void *) 0x50100200
-
-
-
-
-
- Now, I look at the structure to see what's in it, and particularly,
- what its b_data field contains::
-
-
- (gdb) p *((struct buffer_head *)0x50100200)
- $13 = {b_next = 0x50289380, b_blocknr = 49405, b_size = 1024, b_list = 0,
- b_dev = 15872, b_count = {counter = 1}, b_rdev = 15872, b_state = 24,
- b_flushtime = 0, b_next_free = 0x501001a0, b_prev_free = 0x50100260,
- b_this_page = 0x501001a0, b_reqnext = 0x0, b_pprev = 0x507fcf58,
- b_data = 0x50000800 "", b_page = 0x50004000,
- b_end_io = 0x10017f60 <end_buffer_io_sync>, b_dev_id = 0x0,
- b_rsector = 98810, b_wait = {lock = <optimized out or zero length>,
- task_list = {next = 0x50100248, prev = 0x50100248}, __magic = 1343226448,
- __creator = 0}, b_kiobuf = 0x0}
-
-
-
-
-
- The b_data field is indeed 0x50000800, so the question becomes how
- that happened. The rest of the structure looks fine, so this probably
- is not a case of data corruption. It happened on purpose somehow.
-
-
- The b_page field is a pointer to the page_struct representing the
- 0x50000000 page. Looking at it shows the kernel's idea of the state
- of that page::
-
-
-
- (gdb) p *$13.b_page
- $17 = {list = {next = 0x50004a5c, prev = 0x100c5174}, mapping = 0x0,
- index = 0, next_hash = 0x0, count = {counter = 1}, flags = 132, lru = {
- next = 0x50008460, prev = 0x50019350}, wait = {
- lock = <optimized out or zero length>, task_list = {next = 0x50004024,
- prev = 0x50004024}, __magic = 1342193708, __creator = 0},
- pprev_hash = 0x0, buffers = 0x501002c0, virtual = 1342177280,
- zone = 0x100c5160}
-
-
-
-
-
- Some sanity-checking: the virtual field shows the "virtual" address of
- this page, which in this kernel is the same as its "physical" address,
- and the page_struct itself should be mem_map[0], since it represents
- the first page of memory::
-
-
-
- (gdb) p (void *)1342177280
- $18 = (void *) 0x50000000
- (gdb) p mem_map
- $19 = (mem_map_t *) 0x50004000
-
-
-
-
-
- These check out fine.
-
-
- Now to check out the page_struct itself. In particular, the flags
- field shows whether the page is considered free or not::
-
-
- (gdb) p (void *)132
- $21 = (void *) 0x84
-
-
-
-
-
- The "reserved" bit is the high bit, which is definitely not set, so
- the kernel considers the signal stack page to be free and available to
- be used.
-
-
- At this point, I jump to conclusions and start looking at my early
- boot code, because that's where that page is supposed to be reserved.
-
-
- In my setup_arch procedure, I have the following code which looks just
- fine::
-
-
-
- bootmap_size = init_bootmem(start_pfn, end_pfn - start_pfn);
- free_bootmem(__pa(low_physmem) + bootmap_size, high_physmem - low_physmem);
-
-
-
-
-
- Two stack pages have already been allocated, and low_physmem points to
- the third page, which is the beginning of free memory.
- The init_bootmem call declares the entire memory to the boot memory
- manager, which marks it all reserved. The free_bootmem call frees up
- all of it, except for the first two pages. This looks correct to me.
-
-
- So, I decide to see init_bootmem run and make sure that it is marking
- those first two pages as reserved. I never get that far.
-
-
- Stepping into init_bootmem, and looking at bootmem_map before looking
- at what it contains shows the following::
-
-
-
- (gdb) p bootmem_map
- $3 = (void *) 0x50000000
-
-
-
-
-
- Aha! The light dawns. That first page is doing double duty as a
- stack and as the boot memory map. The last thing that the boot memory
- manager does is to free the pages used by its memory map, so this page
- is getting freed even its marked as reserved.
-
-
- The fix was to initialize the boot memory manager before allocating
- those two stack pages, and then allocate them through the boot memory
- manager. After doing this, and fixing a couple of subsequent buglets,
- the stack corruption problem disappeared.
-
-
-
-
-
-13. What to do when UML doesn't work
-=====================================
-
-
-
-
-13.1. Strange compilation errors when you build from source
-------------------------------------------------------------
-
- As of test11, it is necessary to have "ARCH=um" in the environment or
- on the make command line for all steps in building UML, including
- clean, distclean, or mrproper, config, menuconfig, or xconfig, dep,
- and linux. If you forget for any of them, the i386 build seems to
- contaminate the UML build. If this happens, start from scratch with::
-
-
- host%
- make mrproper ARCH=um
-
-
-
-
- and repeat the build process with ARCH=um on all the steps.
-
-
- See :ref:`Compiling_the_kernel_and_modules` for more details.
-
-
- Another cause of strange compilation errors is building UML in
- /usr/src/linux. If you do this, the first thing you need to do is
- clean up the mess you made. The /usr/src/linux/asm link will now
- point to /usr/src/linux/asm-um. Make it point back to
- /usr/src/linux/asm-i386. Then, move your UML pool someplace else and
- build it there. Also see below, where a more specific set of symptoms
- is described.
-
-
-
-13.3. A variety of panics and hangs with /tmp on a reiserfs filesystem
------------------------------------------------------------------------
-
- I saw this on reiserfs 3.5.21 and it seems to be fixed in 3.5.27.
- Panics preceded by::
-
-
- Detaching pid nnnn
-
-
-
- are diagnostic of this problem. This is a reiserfs bug which causes a
- thread to occasionally read stale data from a mmapped page shared with
- another thread. The fix is to upgrade the filesystem or to have /tmp
- be an ext2 filesystem.
-
-
-
- 13.4. The compile fails with errors about conflicting types for
- 'open', 'dup', and 'waitpid'
-
- This happens when you build in /usr/src/linux. The UML build makes
- the include/asm link point to include/asm-um. /usr/include/asm points
- to /usr/src/linux/include/asm, so when that link gets moved, files
- which need to include the asm-i386 versions of headers get the
- incompatible asm-um versions. The fix is to move the include/asm link
- back to include/asm-i386 and to do UML builds someplace else.
-
-
-
-13.5. UML doesn't work when /tmp is an NFS filesystem
-------------------------------------------------------
-
- This seems to be a similar situation with the ReiserFS problem above.
- Some versions of NFS seems not to handle mmap correctly, which UML
- depends on. The workaround is have /tmp be a non-NFS directory.
-
-
-13.6. UML hangs on boot when compiled with gprof support
----------------------------------------------------------
-
- If you build UML with gprof support and, early in the boot, it does
- this::
-
-
- kernel BUG at page_alloc.c:100!
-
-
-
-
- you have a buggy gcc. You can work around the problem by removing
- UM_FASTCALL from CFLAGS in arch/um/Makefile-i386. This will open up
- another bug, but that one is fairly hard to reproduce.
-
-
-
-13.7. syslogd dies with a SIGTERM on startup
----------------------------------------------
-
- The exact boot error depends on the distribution that you're booting,
- but Debian produces this::
-
-
- /etc/rc2.d/S10sysklogd: line 49: 93 Terminated
- start-stop-daemon --start --quiet --exec /sbin/syslogd -- $SYSLOGD
-
-
-
-
- This is a syslogd bug. There's a race between a parent process
- installing a signal handler and its child sending the signal.
-
-
-
-13.8. TUN/TAP networking doesn't work on a 2.4 host
-----------------------------------------------------
-
- There are a couple of problems which were reported by
- Tim Robinson <timro at trkr dot net>
-
- - It doesn't work on hosts running 2.4.7 (or thereabouts) or earlier.
- The fix is to upgrade to something more recent and then read the
- next item.
-
- - If you see::
-
-
- File descriptor in bad state
-
-
-
- when you bring up the device inside UML, you have a header mismatch
- between the original kernel and the upgraded one. Make /usr/src/linux
- point at the new headers. This will only be a problem if you build
- uml_net yourself.
-
-
-
-13.9. You can network to the host but not to other machines on the net
-=======================================================================
-
- If you can connect to the host, and the host can connect to UML, but
- you cannot connect to any other machines, then you may need to enable
- IP Masquerading on the host. Usually this is only experienced when
- using private IP addresses (192.168.x.x or 10.x.x.x) for host/UML
- networking, rather than the public address space that your host is
- connected to. UML does not enable IP Masquerading, so you will need
- to create a static rule to enable it::
-
-
- host%
- iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
-
-
-
-
- Replace eth0 with the interface that you use to talk to the rest of
- the world.
-
-
- Documentation on IP Masquerading, and SNAT, can be found at
- http://www.netfilter.org.
-
-
- If you can reach the local net, but not the outside Internet, then
- that is usually a routing problem. The UML needs a default route::
-
-
- UML#
- route add default gw gateway IP
-
-
-
-
- The gateway IP can be any machine on the local net that knows how to
- reach the outside world. Usually, this is the host or the local net-
- work's gateway.
-
-
- Occasionally, we hear from someone who can reach some machines, but
- not others on the same net, or who can reach some ports on other
- machines, but not others. These are usually caused by strange
- firewalling somewhere between the UML and the other box. You track
- this down by running tcpdump on every interface the packets travel
- over and see where they disappear. When you find a machine that takes
- the packets in, but does not send them onward, that's the culprit.
-
-
-
-13.10. I have no root and I want to scream
-===========================================
-
- Thanks to Birgit Wahlich for telling me about this strange one. It
- turns out that there's a limit of six environment variables on the
- kernel command line. When that limit is reached or exceeded, argument
- processing stops, which means that the 'root=' argument that UML
- usually adds is not seen. So, the filesystem has no idea what the
- root device is, so it panics.
-
-
- The fix is to put less stuff on the command line. Glomming all your
- setup variables into one is probably the best way to go.
-
-
-
-13.11. UML build conflict between ptrace.h and ucontext.h
-==========================================================
-
- On some older systems, /usr/include/asm/ptrace.h and
- /usr/include/sys/ucontext.h define the same names. So, when they're
- included together, the defines from one completely mess up the parsing
- of the other, producing errors like::
-
- /usr/include/sys/ucontext.h:47: parse error before
- `10`
-
-
-
-
- plus a pile of warnings.
-
-
- This is a libc botch, which has since been fixed, and I don't see any
- way around it besides upgrading.
-
-
-
-13.12. The UML BogoMips is exactly half the host's BogoMips
-------------------------------------------------------------
-
- On i386 kernels, there are two ways of running the loop that is used
- to calculate the BogoMips rating, using the TSC if it's there or using
- a one-instruction loop. The TSC produces twice the BogoMips as the
- loop. UML uses the loop, since it has nothing resembling a TSC, and
- will get almost exactly the same BogoMips as a host using the loop.
- However, on a host with a TSC, its BogoMips will be double the loop
- BogoMips, and therefore double the UML BogoMips.
-
-
-
-13.13. When you run UML, it immediately segfaults
---------------------------------------------------
-
- If the host is configured with the 2G/2G address space split, that's
- why. See ref:`UML_on_2G/2G_hosts` for the details on getting UML to
- run on your host.
-
-
-
-13.14. xterms appear, then immediately disappear
--------------------------------------------------
-
- If you're running an up to date kernel with an old release of
- uml_utilities, the port-helper program will not work properly, so
- xterms will exit straight after they appear. The solution is to
- upgrade to the latest release of uml_utilities. Usually this problem
- occurs when you have installed a packaged release of UML then compiled
- your own development kernel without upgrading the uml_utilities from
- the source distribution.
-
-
-
-13.15. Any other panic, hang, or strange behavior
---------------------------------------------------
-
- If you're seeing truly strange behavior, such as hangs or panics that
- happen in random places, or you try running the debugger to see what's
- happening and it acts strangely, then it could be a problem in the
- host kernel. If you're not running a stock Linus or -ac kernel, then
- try that. An early version of the preemption patch and a 2.4.10 SuSE
- kernel have caused very strange problems in UML.
-
-
- Otherwise, let me know about it. Send a message to one of the UML
- mailing lists - either the developer list - user-mode-linux-devel at
- lists dot sourceforge dot net (subscription info) or the user list -
- user-mode-linux-user at lists dot sourceforge do net (subscription
- info), whichever you prefer. Don't assume that everyone knows about
- it and that a fix is imminent.
-
-
- If you want to be super-helpful, read :ref:`Diagnosing_Problems` and
- follow the instructions contained therein.
-
-.. _Diagnosing_Problems:
-
-14. Diagnosing Problems
-========================
-
-
- If you get UML to crash, hang, or otherwise misbehave, you should
- report this on one of the project mailing lists, either the developer
- list - user-mode-linux-devel at lists dot sourceforge dot net
- (subscription info) or the user list - user-mode-linux-user at lists
- dot sourceforge dot net (subscription info). When you do, it is
- likely that I will want more information. So, it would be helpful to
- read the stuff below, do whatever is applicable in your case, and
- report the results to the list.
-
-
- For any diagnosis, you're going to need to build a debugging kernel.
- The binaries from this site aren't debuggable. If you haven't done
- this before, read about :ref:`Compiling_the_kernel_and_modules` and
- :ref:`Kernel_debugging` UML first.
-
-
-14.1. Case 1 : Normal kernel panics
-------------------------------------
-
- The most common case is for a normal thread to panic. To debug this,
- you will need to run it under the debugger (add 'debug' to the command
- line). An xterm will start up with gdb running inside it. Continue
- it when it stops in start_kernel and make it crash. Now ``^C gdb`` and
-
-
- If the panic was a "Kernel mode fault", then there will be a segv
- frame on the stack and I'm going to want some more information. The
- stack might look something like this::
-
-
- (UML gdb) backtrace
- #0 0x1009bf76 in __sigprocmask (how=1, set=0x5f347940, oset=0x0)
- at ../sysdeps/unix/sysv/linux/sigprocmask.c:49
- #1 0x10091411 in change_sig (signal=10, on=1) at process.c:218
- #2 0x10094785 in timer_handler (sig=26) at time_kern.c:32
- #3 0x1009bf38 in __restore ()
- at ../sysdeps/unix/sysv/linux/i386/sigaction.c:125
- #4 0x1009534c in segv (address=8, ip=268849158, is_write=2, is_user=0)
- at trap_kern.c:66
- #5 0x10095c04 in segv_handler (sig=11) at trap_user.c:285
- #6 0x1009bf38 in __restore ()
-
-
-
-
- I'm going to want to see the symbol and line information for the value
- of ip in the segv frame. In this case, you would do the following::
-
-
- (UML gdb) i sym 268849158
-
-
-
-
- and::
-
-
- (UML gdb) i line *268849158
-
-
-
-
- The reason for this is the __restore frame right above the segv_han-
- dler frame is hiding the frame that actually segfaulted. So, I have
- to get that information from the faulting ip.
-
-
-14.2. Case 2 : Tracing thread panics
--------------------------------------
-
- The less common and more painful case is when the tracing thread
- panics. In this case, the kernel debugger will be useless because it
- needs a healthy tracing thread in order to work. The first thing to
- do is get a backtrace from the tracing thread. This is done by
- figuring out what its pid is, firing up gdb, and attaching it to that
- pid. You can figure out the tracing thread pid by looking at the
- first line of the console output, which will look like this::
-
-
- tracing thread pid = 15851
-
-
-
-
- or by running ps on the host and finding the line that looks like
- this::
-
-
- jdike 15851 4.5 0.4 132568 1104 pts/0 S 21:34 0:05 ./linux [(tracing thread)]
-
-
-
-
- If the panic was 'segfault in signals', then follow the instructions
- above for collecting information about the location of the seg fault.
-
-
- If the tracing thread flaked out all by itself, then send that
- backtrace in and wait for our crack debugging team to fix the problem.
-
-
- 14.3. Case 3 : Tracing thread panics caused by other threads
-
- However, there are cases where the misbehavior of another thread
- caused the problem. The most common panic of this type is::
-
-
- wait_for_stop failed to wait for <pid> to stop with <signal number>
-
-
-
-
- In this case, you'll need to get a backtrace from the process men-
- tioned in the panic, which is complicated by the fact that the kernel
- debugger is defunct and without some fancy footwork, another gdb can't
- attach to it. So, this is how the fancy footwork goes:
-
- In a shell::
-
-
- host% kill -STOP pid
-
-
-
-
- Run gdb on the tracing thread as described in case 2 and do::
-
-
- (host gdb) call detach(pid)
-
-
- If you get a segfault, do it again. It always works the second time.
-
- Detach from the tracing thread and attach to that other thread::
-
-
- (host gdb) detach
-
-
-
-
-
-
- (host gdb) attach pid
-
-
-
-
- If gdb hangs when attaching to that process, go back to a shell and
- do::
-
-
- host%
- kill -CONT pid
-
-
-
-
- And then get the backtrace::
-
-
- (host gdb) backtrace
-
-
-
-
-
-14.4. Case 4 : Hangs
----------------------
-
- Hangs seem to be fairly rare, but they sometimes happen. When a hang
- happens, we need a backtrace from the offending process. Run the
- kernel debugger as described in case 1 and get a backtrace. If the
- current process is not the idle thread, then send in the backtrace.
- You can tell that it's the idle thread if the stack looks like this::
-
-
- #0 0x100b1401 in __libc_nanosleep ()
- #1 0x100a2885 in idle_sleep (secs=10) at time.c:122
- #2 0x100a546f in do_idle () at process_kern.c:445
- #3 0x100a5508 in cpu_idle () at process_kern.c:471
- #4 0x100ec18f in start_kernel () at init/main.c:592
- #5 0x100a3e10 in start_kernel_proc (unused=0x0) at um_arch.c:71
- #6 0x100a383f in signal_tramp (arg=0x100a3dd8) at trap_user.c:50
-
-
-
-
- If this is the case, then some other process is at fault, and went to
- sleep when it shouldn't have. Run ps on the host and figure out which
- process should not have gone to sleep and stayed asleep. Then attach
- to it with gdb and get a backtrace as described in case 3.
-
-
-
-
-
-
-15. Thanks
-===========
-
-
- A number of people have helped this project in various ways, and this
- page gives recognition where recognition is due.
-
-
- If you're listed here and you would prefer a real link on your name,
- or no link at all, instead of the despammed email address pseudo-link,
- let me know.
-
-
- If you're not listed here and you think maybe you should be, please
- let me know that as well. I try to get everyone, but sometimes my
- bookkeeping lapses and I forget about contributions.
-
-
-15.1. Code and Documentation
------------------------------
-
- Rusty Russell <rusty at linuxcare.com.au> -
-
- - wrote the HOWTO
- http://user-mode-linux.sourceforge.net/old/UserModeLinux-HOWTO.html
-
- - prodded me into making this project official and putting it on
- SourceForge
-
- - came up with the way cool UML logo
- http://user-mode-linux.sourceforge.net/uml-small.png
-
- - redid the config process
-
-
- Peter Moulder <reiter at netspace.net.au> - Fixed my config and build
- processes, and added some useful code to the block driver
-
-
- Bill Stearns <wstearns at pobox.com> -
-
- - HOWTO updates
-
- - lots of bug reports
-
- - lots of testing
-
- - dedicated a box (uml.ists.dartmouth.edu) to support UML development
-
- - wrote the mkrootfs script, which allows bootable filesystems of
- RPM-based distributions to be cranked out
-
- - cranked out a large number of filesystems with said script
-
-
- Jim Leu <jleu at mindspring.com> - Wrote the virtual ethernet driver
- and associated usermode tools
-
- Lars Brinkhoff http://lars.nocrew.org/ - Contributed the ptrace
- proxy from his own project to allow easier kernel debugging
-
-
- Andrea Arcangeli <andrea at suse.de> - Redid some of the early boot
- code so that it would work on machines with Large File Support
-
-
- Chris Emerson - Did the first UML port to Linux/ppc
-
-
- Harald Welte <laforge at gnumonks.org> - Wrote the multicast
- transport for the network driver
-
-
- Jorgen Cederlof - Added special file support to hostfs
-
-
- Greg Lonnon <glonnon at ridgerun dot com> - Changed the ubd driver
- to allow it to layer a COW file on a shared read-only filesystem and
- wrote the iomem emulation support
-
-
- Henrik Nordstrom http://hem.passagen.se/hno/ - Provided a variety
- of patches, fixes, and clues
-
-
- Lennert Buytenhek - Contributed various patches, a rewrite of the
- network driver, the first implementation of the mconsole driver, and
- did the bulk of the work needed to get SMP working again.
-
-
- Yon Uriarte - Fixed the TUN/TAP network backend while I slept.
-
-
- Adam Heath - Made a bunch of nice cleanups to the initialization code,
- plus various other small patches.
-
-
- Matt Zimmerman - Matt volunteered to be the UML Debian maintainer and
- is doing a real nice job of it. He also noticed and fixed a number of
- actually and potentially exploitable security holes in uml_net. Plus
- the occasional patch. I like patches.
-
-
- James McMechan - James seems to have taken over maintenance of the ubd
- driver and is doing a nice job of it.
-
-
- Chandan Kudige - wrote the umlgdb script which automates the reloading
- of module symbols.
-
-
- Steve Schmidtke - wrote the UML slirp transport and hostaudio drivers,
- enabling UML processes to access audio devices on the host. He also
- submitted patches for the slip transport and lots of other things.
-
-
- David Coulson http://davidcoulson.net -
-
- - Set up the http://usermodelinux.org site,
- which is a great way of keeping the UML user community on top of
- UML goings-on.
-
- - Site documentation and updates
-
- - Nifty little UML management daemon UMLd
-
- - Lots of testing and bug reports
-
-
-
-
-15.2. Flushing out bugs
-------------------------
-
-
-
- - Yuri Pudgorodsky
-
- - Gerald Britton
-
- - Ian Wehrman
-
- - Gord Lamb
-
- - Eugene Koontz
-
- - John H. Hartman
-
- - Anders Karlsson
-
- - Daniel Phillips
-
- - John Fremlin
-
- - Rainer Burgstaller
-
- - James Stevenson
-
- - Matt Clay
-
- - Cliff Jefferies
-
- - Geoff Hoff
-
- - Lennert Buytenhek
-
- - Al Viro
-
- - Frank Klingenhoefer
-
- - Livio Baldini Soares
-
- - Jon Burgess
-
- - Petru Paler
-
- - Paul
-
- - Chris Reahard
-
- - Sverker Nilsson
-
- - Gong Su
-
- - johan verrept
-
- - Bjorn Eriksson
-
- - Lorenzo Allegrucci
-
- - Muli Ben-Yehuda
-
- - David Mansfield
-
- - Howard Goff
-
- - Mike Anderson
-
- - John Byrne
-
- - Sapan J. Batia
-
- - Iris Huang
-
- - Jan Hudec
-
- - Voluspa
-
-
-
-
-15.3. Buglets and clean-ups
-----------------------------
-
-
-
- - Dave Zarzycki
-
- - Adam Lazur
-
- - Boria Feigin
-
- - Brian J. Murrell
-
- - JS
-
- - Roman Zippel
-
- - Wil Cooley
-
- - Ayelet Shemesh
-
- - Will Dyson
-
- - Sverker Nilsson
-
- - dvorak
-
- - v.naga srinivas
-
- - Shlomi Fish
-
- - Roger Binns
-
- - johan verrept
-
- - MrChuoi
-
- - Peter Cleve
-
- - Vincent Guffens
-
- - Nathan Scott
-
- - Patrick Caulfield
-
- - jbearce
-
- - Catalin Marinas
-
- - Shane Spencer
-
- - Zou Min
-
-
- - Ryan Boder
-
- - Lorenzo Colitti
-
- - Gwendal Grignou
-
- - Andre' Breiler
-
- - Tsutomu Yasuda
-
-
-
-15.4. Case Studies
--------------------
-
-
- - Jon Wright
-
- - William McEwan
-
- - Michael Richardson
-
-
-
-15.5. Other contributions
---------------------------
-
-
- Bill Carr <Bill.Carr at compaq.com> made the Red Hat mkrootfs script
- work with RH 6.2.
-
- Michael Jennings <mikejen at hevanet.com> sent in some material which
- is now gracing the top of the index page
- http://user-mode-linux.sourceforge.net/ of this site.
-
- SGI (and more specifically Ralf Baechle <ralf at
- uni-koblenz.de> ) gave me an account on oss.sgi.com.
- The bandwidth there made it possible to
- produce most of the filesystems available on the project download
- page.
-
- Laurent Bonnaud <Laurent.Bonnaud at inpg.fr> took the old grotty
- Debian filesystem that I've been distributing and updated it to 2.2.
- It is now available by itself here.
-
- Rik van Riel gave me some ftp space on ftp.nl.linux.org so I can make
- releases even when Sourceforge is broken.
-
- Rodrigo de Castro looked at my broken pte code and told me what was
- wrong with it, letting me fix a long-standing (several weeks) and
- serious set of bugs.
-
- Chris Reahard built a specialized root filesystem for running a DNS
- server jailed inside UML. It's available from the download
- http://user-mode-linux.sourceforge.net/old/dl-sf.html page in the Jail
- Filesystems section.
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+#########
+UML HowTo
+#########
+
+.. contents:: :local:
+
+************
+Introduction
+************
+
+Welcome to User Mode Linux
+
+User Mode Linux is the first Open Source virtualization platform (first
+release date 1991) and second virtualization platform for an x86 PC.
+
+How is UML Different from a VM using Virtualization package X?
+==============================================================
+
+We have come to assume that virtualization also means some level of
+hardware emulation. In fact, it does not. As long as a virtualization
+package provides the OS with devices which the OS can recognize and
+has a driver for, the devices do not need to emulate real hardware.
+Most OSes today have built-in support for a number of "fake"
+devices used only under virtualization.
+User Mode Linux takes this concept to the ultimate extreme - there
+is not a single real device in sight. It is 100% artificial or if
+we use the correct term 100% paravirtual. All UML devices are abstract
+concepts which map onto something provided by the host - files, sockets,
+pipes, etc.
+
+The other major difference between UML and various virtualization
+packages is that there is a distinct difference between the way the UML
+kernel and the UML programs operate.
+The UML kernel is just a process running on Linux - same as any other
+program. It can be run by an unprivileged user and it does not require
+anything in terms of special CPU features.
+The UML userspace, however, is a bit different. The Linux kernel on the
+host machine assists UML in intercepting everything the program running
+on a UML instance is trying to do and making the UML kernel handle all
+of its requests.
+This is different from other virtualization packages which do not make any
+difference between the guest kernel and guest programs. This difference
+results in a number of advantages and disadvantages of UML over let's say
+QEMU which we will cover later in this document.
+
+
+Why Would I Want User Mode Linux?
+=================================
+
+
+* If User Mode Linux kernel crashes, your host kernel is still fine. It
+ is not accelerated in any way (vhost, kvm, etc) and it is not trying to
+ access any devices directly. It is, in fact, a process like any other.
+
+* You can run a usermode kernel as a non-root user (you may need to
+ arrange appropriate permissions for some devices).
+
+* You can run a very small VM with a minimal footprint for a specific
+ task (for example 32M or less).
+
+* You can get extremely high performance for anything which is a "kernel
+ specific task" such as forwarding, firewalling, etc while still being
+ isolated from the host kernel.
+
+* You can play with kernel concepts without breaking things.
+
+* You are not bound by "emulating" hardware, so you can try weird and
+ wonderful concepts which are very difficult to support when emulating
+ real hardware such as time travel and making your system clock
+ dependent on what UML does (very useful for things like tests).
+
+* It's fun.
+
+Why not to run UML
+==================
+
+* The syscall interception technique used by UML makes it inherently
+ slower for any userspace applications. While it can do kernel tasks
+ on par with most other virtualization packages, its userspace is
+ **slow**. The root cause is that UML has a very high cost of creating
+ new processes and threads (something most Unix/Linux applications
+ take for granted).
+
+* UML is strictly uniprocessor at present. If you want to run an
+ application which needs many CPUs to function, it is clearly the
+ wrong choice.
+
+***********************
+Building a UML instance
+***********************
+
+There is no UML installer in any distribution. While you can use off
+the shelf install media to install into a blank VM using a virtualization
+package, there is no UML equivalent. You have to use appropriate tools on
+your host to build a viable filesystem image.
+
+This is extremely easy on Debian - you can do it using debootstrap. It is
+also easy on OpenWRT - the build process can build UML images. All other
+distros - YMMV.
+
+Creating an image
+=================
+
+Create a sparse raw disk image::
+
+ # dd if=/dev/zero of=disk_image_name bs=1 count=1 seek=16G
+
+This will create a 16G disk image. The OS will initially allocate only one
+block and will allocate more as they are written by UML. As of kernel
+version 4.19 UML fully supports TRIM (as usually used by flash drives).
+Using TRIM inside the UML image by specifying discard as a mount option
+or by running ``tune2fs -o discard /dev/ubdXX`` will request UML to
+return any unused blocks to the OS.
+
+Create a filesystem on the disk image and mount it::
+
+ # mkfs.ext4 ./disk_image_name && mount ./disk_image_name /mnt
+
+This example uses ext4, any other filesystem such as ext3, btrfs, xfs,
+jfs, etc will work too.
+
+Create a minimal OS installation on the mounted filesystem::
+
+ # debootstrap buster /mnt http://deb.debian.org/debian
+
+debootstrap does not set up the root password, fstab, hostname or
+anything related to networking. It is up to the user to do that.
+
+Set the root password -t he easiest way to do that is to chroot into the
+mounted image::
+
+ # chroot /mnt
+ # passwd
+ # exit
+
+Edit key system files
+=====================
+
+UML block devices are called ubds. The fstab created by debootstrap
+will be empty and it needs an entry for the root file system::
+
+ /dev/ubd0 ext4 discard,errors=remount-ro 0 1
+
+The image hostname will be set to the same as the host on which you
+are creating it image. It is a good idea to change that to avoid
+"Oh, bummer, I rebooted the wrong machine".
+
+UML supports two classes of network devices - the older uml_net ones
+which are scheduled for obsoletion. These are called ethX. It also
+supports the newer vector IO devices which are significantly faster
+and have support for some standard virtual network encapsulations like
+Ethernet over GRE and Ethernet over L2TPv3. These are called vec0.
+
+Depending on which one is in use, ``/etc/network/interfaces`` will
+need entries like::
+
+ # legacy UML network devices
+ auto eth0
+ iface eth0 inet dhcp
+
+ # vector UML network devices
+ auto vec0
+ iface eth0 inet dhcp
+
+We now have a UML image which is nearly ready to run, all we need is a
+UML kernel and modules for it.
+
+Most distributions have a UML package. Even if you intend to use your own
+kernel, testing the image with a stock one is always a good start. These
+packages come with a set of modules which should be copied to the target
+filesystem. The location is distribution dependent. For Debian these
+reside under /usr/lib/uml/modules. Copy recursively the content of this
+directory to the mounted UML filesystem::
+
+ # cp -rax /usr/lib/uml/modules /mnt/lib/modules
+
+If you have compiled your own kernel, you need to use the usual "install
+modules to a location" procedure by running::
+
+ # make install MODULES_DIR=/mnt/lib/modules
+
+At this point the image is ready to be brought up.
+
+*************************
+Setting Up UML Networking
+*************************
+
+UML networking is designed to emulate an Ethernet connection. This
+connection may be either a point-to-point (similar to a connection
+between machines using a back-to-back cable) or a connection to a
+switch. UML supports a wide variety of means to build these
+connections to all of: local machine, remote machine(s), local and
+remote UML and other VM instances.
+
+
++-----------+--------+------------------------------------+------------+
+| Transport | Type | Capabilities | Throughput |
++===========+========+====================================+============+
+| tap | vector | checksum, tso | > 8Gbit |
++-----------+--------+------------------------------------+------------+
+| hybrid | vector | checksum, tso, multipacket rx | > 6GBit |
++-----------+--------+------------------------------------+------------+
+| raw | vector | checksum, tso, multipacket rx, tx" | > 6GBit |
++-----------+--------+------------------------------------+------------+
+| EoGRE | vector | multipacket rx, tx | > 3Gbit |
++-----------+--------+------------------------------------+------------+
+| Eol2tpv3 | vector | multipacket rx, tx | > 3Gbit |
++-----------+--------+------------------------------------+------------+
+| bess | vector | multipacket rx, tx | > 3Gbit |
++-----------+--------+------------------------------------+------------+
+| fd | vector | dependent on fd type | varies |
++-----------+--------+------------------------------------+------------+
+| tuntap | legacy | none | ~ 500Mbit |
++-----------+--------+------------------------------------+------------+
+| daemon | legacy | none | ~ 450Mbit |
++-----------+--------+------------------------------------+------------+
+| socket | legacy | none | ~ 450Mbit |
++-----------+--------+------------------------------------+------------+
+| pcap | legacy | rx only | ~ 450Mbit |
++-----------+--------+------------------------------------+------------+
+| ethertap | legacy | obsolete | ~ 500Mbit |
++-----------+--------+------------------------------------+------------+
+| vde | legacy | obsolete | ~ 500Mbit |
++-----------+--------+------------------------------------+------------+
+
+* All transports which have tso and checksum offloads can deliver speeds
+ approaching 10G on TCP streams.
+
+* All transports which have multi-packet rx and/or tx can deliver pps
+ rates of up to 1Mps or more.
+
+* All legacy transports are generally limited to ~600-700MBit and 0.05Mps
+
+* GRE and L2TPv3 allow connections to all of: local machine, remote
+ machines, remote network devices and remote UML instances.
+
+* Socket allows connections only between UML instances.
+
+* Daemon and bess require running a local switch. This switch may be
+ connected to the host as well.
+
+
+Network configuration privileges
+================================
+
+The majority of the supported networking modes need ``root`` privileges.
+For example, in the legacy tuntap networking mode, users were required
+to be part of the group associated with the tunnel device.
+
+For newer network drivers like the vector transports, ``root`` privilege
+is required to fire an ioctl to setup the tun interface and/or use
+raw sockets where needed.
+
+This can be achieved by granting the user a particular capability instead
+of running UML as root. In case of vector transport, a user can add the
+capability ``CAP_NET_ADMIN`` or ``CAP_NET_RAW``, to the uml binary.
+Thenceforth, UML can be run with normal user privilges, along with
+full networking.
+
+For example::
+
+ # sudo setcap cap_net_raw,cap_net_admin+ep linux
+
+Configuring vector transports
+===============================
+
+All vector transports support a similar syntax:
+
+If X is the interface number as in vec0, vec1, vec2, etc, the general
+syntax for options is::
+
+ vecX:transport="Transport Name",option=value,option=value,...,option=value
+
+Common options
+--------------
+
+These options are common for all transports:
+
+* ``depth=int`` - sets the queue depth for vector IO. This is the
+ amount of packets UML will attempt to read or write in a single
+ system call. The default number is 64 and is generally sufficient
+ for most applications that need throughput in the 2-4 Gbit range.
+ Higher speeds may require larger values.
+
+* ``mac=XX:XX:XX:XX:XX`` - sets the interface MAC address value.
+
+* ``gro=[0,1]`` - sets GRO on or off. Enables receive/transmit offloads.
+ The effect of this option depends on the host side support in the transport
+ which is being configured. In most cases it will enable TCP segmentation and
+ RX/TX checksumming offloads. The setting must be identical on the host side
+ and the UML side. The UML kernel will produce warnings if it is not.
+ For example, GRO is enabled by default on local machine interfaces
+ (e.g. veth pairs, bridge, etc), so it should be enabled in UML in the
+ corresponding UML transports (raw, tap, hybrid) in order for networking to
+ operate correctly.
+
+* ``mtu=int`` - sets the interface MTU
+
+* ``headroom=int`` - adjusts the default headroom (32 bytes) reserved
+ if a packet will need to be re-encapsulated into for instance VXLAN.
+
+* ``vec=0`` - disable multipacket io and fall back to packet at a
+ time mode
+
+Shared Options
+--------------
+
+* ``ifname=str`` Transports which bind to a local network interface
+ have a shared option - the name of the interface to bind to.
+
+* ``src, dst, src_port, dst_port`` - all transports which use sockets
+ which have the notion of source and destination and/or source port
+ and destination port use these to specify them.
+
+* ``v6=[0,1]`` to specify if a v6 connection is desired for all
+ transports which operate over IP. Additionally, for transports that
+ have some differences in the way they operate over v4 and v6 (for example
+ EoL2TPv3), sets the correct mode of operation. In the absense of this
+ option, the socket type is determined based on what do the src and dst
+ arguments resolve/parse to.
+
+tap transport
+-------------
+
+Example::
+
+ vecX:transport=tap,ifname=tap0,depth=128,gro=1
+
+This will connect vec0 to tap0 on the host. Tap0 must already exist (for example
+created using tunctl) and UP.
+
+tap0 can be configured as a point-to-point interface and given an ip
+address so that UML can talk to the host. Alternatively, it is possible
+to connect UML to a tap interface which is connected to a bridge.
+
+While tap relies on the vector infrastructure, it is not a true vector
+transport at this point, because Linux does not support multi-packet
+IO on tap file descriptors for normal userspace apps like UML. This
+is a privilege which is offered only to something which can hook up
+to it at kernel level via specialized interfaces like vhost-net. A
+vhost-net like helper for UML is planned at some point in the future.
+
+Privileges required: tap transport requires either:
+
+* tap interface to exist and be created persistent and owned by the
+ UML user using tunctl. Example ``tunctl -u uml-user -t tap0``
+
+* binary to have ``CAP_NET_ADMIN`` privilege
+
+hybrid transport
+----------------
+
+Example::
+
+ vecX:transport=hybrid,ifname=tap0,depth=128,gro=1
+
+This is an experimental/demo transport which couples tap for transmit
+and a raw socket for receive. The raw socket allows multi-packet
+receive resulting in significantly higher packet rates than normal tap
+
+Privileges required: hybrid requires ``CAP_NET_RAW`` capability by
+the UML user as well as the requirements for the tap transport.
+
+raw socket transport
+--------------------
+
+Example::
+
+ vecX:transport=raw,ifname=p-veth0,depth=128,gro=1
+
+
+This transport uses vector IO on raw sockets. While you can bind to any
+interface including a physical one, the most common use it to bind to
+the "peer" side of a veth pair with the other side configured on the
+host.
+
+Example host configuration for Debian:
+
+**/etc/network/interfaces**::
+
+ auto veth0
+ iface veth0 inet static
+ address 192.168.4.1
+ netmask 255.255.255.252
+ broadcast 192.168.4.3
+ pre-up ip link add veth0 type veth peer name p-veth0 && \
+ ifconfig p-veth0 up
+
+UML can now bind to p-veth0 like this::
+
+ vec0:transport=raw,ifname=p-veth0,depth=128,gro=1
+
+
+If the UML guest is configured with 192.168.4.2 and netmask 255.255.255.0
+it can talk to the host on 192.168.4.1
+
+The raw transport also provides some support for offloading some of the
+filtering to the host. The two options to control it are:
+
+* ``bpffile=str`` filename of raw bpf code to be loaded as a socket filter
+
+* ``bpfflash=int`` 0/1 allow loading of bpf from inside User Mode Linux.
+ This option allows the use of the ethtool load firmware command to
+ load bpf code.
+
+In either case the bpf code is loaded into the host kernel. While this is
+presently limited to legacy bpf syntax (not ebpf), it is still a security
+risk. It is not recommended to allow this unless the User Mode Linux
+instance is considered trusted.
+
+Privileges required: raw socket transport requires `CAP_NET_RAW`
+capability.
+
+GRE socket transport
+--------------------
+
+Example::
+
+ vecX:transport=gre,src=$src_host,dst=$dst_host
+
+
+This will configure an Ethernet over ``GRE`` (aka ``GRETAP`` or
+``GREIRB``) tunnel which will connect the UML instance to a ``GRE``
+endpoint at host dst_host. ``GRE`` supports the following additional
+options:
+
+* ``rx_key=int`` - GRE 32 bit integer key for rx packets, if set,
+ ``txkey`` must be set too
+
+* ``tx_key=int`` - GRE 32 bit integer key for tx packets, if set
+ ``rx_key`` must be set too
+
+* ``sequence=[0,1]`` - enable GRE sequence
+
+* ``pin_sequence=[0,1]`` - pretend that the sequence is always reset
+ on each packet (needed to interoperate with some really broken
+ implementations)
+
+* ``v6=[0,1]`` - force IPv4 or IPv6 sockets respectively
+
+* GRE checksum is not presently supported
+
+GRE has a number of caveats:
+
+* You can use only one GRE connection per ip address. There is no way to
+ multiplex connections as each GRE tunnel is terminated directly on
+ the UML instance.
+
+* The key is not really a security feature. While it was intended as such
+ it's "security" is laughable. It is, however, a useful feature to
+ ensure that the tunnel is not misconfigured.
+
+An example configuration for a Linux host with a local address of
+192.168.128.1 to connect to a UML instance at 192.168.129.1
+
+**/etc/network/interfaces**::
+
+ auto gt0
+ iface gt0 inet static
+ address 10.0.0.1
+ netmask 255.255.255.0
+ broadcast 10.0.0.255
+ mtu 1500
+ pre-up ip link add gt0 type gretap local 192.168.128.1 \
+ remote 192.168.129.1 || true
+ down ip link del gt0 || true
+
+Additionally, GRE has been tested versus a variety of network equipment.
+
+Privileges required: GRE requires ``CAP_NET_RAW``
+
+l2tpv3 socket transport
+-----------------------
+
+_Warning_. L2TPv3 has a "bug". It is the "bug" known as "has more
+options than GNU ls". While it has some advantages, there are usually
+easier (and less verbose) ways to connect a UML instance to something.
+For example, most devices which support L2TPv3 also support GRE.
+
+Example::
+
+ vec0:transport=l2tpv3,udp=1,src=$src_host,dst=$dst_host,srcport=$src_port,dstport=$dst_port,depth=128,rx_session=0xffffffff,tx_session=0xffff
+
+This will configure an Ethernet over L2TPv3 fixed tunnel which will
+connect the UML instance to a L2TPv3 endpoint at host $dst_host using
+the L2TPv3 UDP flavour and UDP destination port $dst_port.
+
+L2TPv3 always requires the following additional options:
+
+* ``rx_session=int`` - l2tpv3 32 bit integer session for rx packets
+
+* ``tx_session=int`` - l2tpv3 32 bit integer session for tx packets
+
+As the tunnel is fixed these are not negotiated and they are
+preconfigured on both ends.
+
+Additionally, L2TPv3 supports the following optional parameters
+
+* ``rx_cookie=int`` - l2tpv3 32 bit integer cookie for rx packets - same
+ functionality as GRE key, more to prevent misconfiguration than provide
+ actual security
+
+* ``tx_cookie=int`` - l2tpv3 32 bit integer cookie for tx packets
+
+* ``cookie64=[0,1]`` - use 64 bit cookies instead of 32 bit.
+
+* ``counter=[0,1]`` - enable l2tpv3 counter
+
+* ``pin_counter=[0,1]`` - pretend that the counter is always reset on
+ each packet (needed to interoperate with some really broken
+ implementations)
+
+* ``v6=[0,1]`` - force v6 sockets
+
+* ``udp=[0,1]`` - use raw sockets (0) or UDP (1) version of the protocol
+
+L2TPv3 has a number of caveats:
+
+* you can use only one connection per ip address in raw mode. There is
+ no way to multiplex connections as each L2TPv3 tunnel is terminated
+ directly on the UML instance. UDP mode can use different ports for
+ this purpose.
+
+Here is an example of how to configure a linux host to connect to UML
+via L2TPv3:
+
+**/etc/network/interfaces**::
+
+ auto l2tp1
+ iface l2tp1 inet static
+ address 192.168.126.1
+ netmask 255.255.255.0
+ broadcast 192.168.126.255
+ mtu 1500
+ pre-up ip l2tp add tunnel remote 127.0.0.1 \
+ local 127.0.0.1 encap udp tunnel_id 2 \
+ peer_tunnel_id 2 udp_sport 1706 udp_dport 1707 && \
+ ip l2tp add session name l2tp1 tunnel_id 2 \
+ session_id 0xffffffff peer_session_id 0xffffffff
+ down ip l2tp del session tunnel_id 2 session_id 0xffffffff && \
+ ip l2tp del tunnel tunnel_id 2
+
+
+Privileges required: L2TPv3 requires ``CAP_NET_RAW`` for raw IP mode and
+no special privileges for the UDP mode.
+
+BESS socket transport
+---------------------
+
+BESS is a high performance modular network switch.
+
+https://github.com/NetSys/bess
+
+It has support for a simple sequential packet socket mode which in the
+more recent versions is using vector IO for high performance.
+
+Example::
+
+ vecX:transport=bess,src=$unix_src,dst=$unix_dst
+
+This will configure a BESS transport using the unix_src Unix domain
+socket address as source and unix_dst socket address as destination.
+
+For BESS configuration and how to allocate a BESS Unix domain socket port
+please see the BESS documentation.
+
+https://github.com/NetSys/bess/wiki/Built-In-Modules-and-Ports
+
+BESS transport does not require any special privileges.
+
+Configuring Legacy transports
+=============================
+
+Legacy transports are now considered obsolete. Please use the vector
+versions.
+
+***********
+Running UML
+***********
+
+This section assumes that either the user-mode-linux package from the
+distribution or a custom built kernel has been installed on the host.
+
+These add an executable called linux to the system. This is the UML
+kernel. It can be run just like any other executable.
+It will take most normal linux kernel arguments as command line
+arguments. Additionally, it will need some UML specific arguments
+in order to do something useful.
+
+Arguments
+=========
+
+Mandatory Arguments:
+--------------------
+
+* ``mem=int[K,M,G]`` - amount of memory. By default bytes. It will
+ also accept K, M or G qualifiers.
+
+* ``ubdX[s,d,c,t]=`` virtual disk specification. This is not really
+ mandatory, but it is likely to be needed in nearly all cases so we can
+ specify a root file system.
+ The simplest possible image specification is the name of the image
+ file for the filesystem (created using one of the methods described
+ in `Creating an image`_)
+
+ * UBD devices support copy on write (COW). The changes are kept in
+ a separate file which can be discarded allowing a rollback to the
+ original pristine image. If COW is desired, the UBD image is
+ specified as: ``cow_file,master_image``.
+ Example:``ubd0=Filesystem.cow,Filesystem.img``
+
+ * UBD devices can be set to use synchronous IO. Any writes are
+ immediately flushed to disk. This is done by adding ``s`` after
+ the ``ubdX`` specification
+
+ * UBD performs some euristics on devices specified as a single
+ filename to make sure that a COW file has not been specified as
+ the image. To turn them off, use the ``d`` flag after ``ubdX``
+
+ * UBD supports TRIM - asking the Host OS to reclaim any unused
+ blocks in the image. To turn it off, specify the ``t`` flag after
+ ``ubdX``
+
+* ``root=`` root device - most likely ``/dev/ubd0`` (this is a Linux
+ filesystem image)
+
+Important Optional Arguments
+----------------------------
+
+If UML is run as "linux" with no extra arguments, it will try to start an
+xterm for every console configured inside the image (up to 6 in most
+linux distributions). Each console is started inside an
+xterm. This makes it nice and easy to use UML on a host with a GUI. It is,
+however, the wrong approach if UML is to be used as a testing harness or run
+in a text-only environment.
+
+In order to change this behaviour we need to specify an alternative console
+and wire it to one of the supported "line" channels. For this we need to map a
+console to use something different from the default xterm.
+
+Example which will divert console number 1 to stdin/stdout::
+
+ con1=fd:0,fd:1
+
+UML supports a wide variety of serial line channels which are specified using
+the following syntax
+
+ conX=channel_type:options[,channel_type:options]
+
+
+If the channel specification contains two parts separated by comma, the first
+one is input, the second one output.
+
+* The null channel - Discard all input or output. Example ``con=null`` will set
+ all consoles to null by default.
+
+* The fd channel - use file descriptor numbers for input/out. Example:
+ ``con1=fd:0,fd:1.``
+
+* The port channel - listen on tcp port number. Example: ``con1=port:4321``
+
+* The pty and pts channels - use system pty/pts.
+
+* The tty channel - bind to an existing system tty. Example: ``con1=/dev/tty8``
+ will make UML use the host 8th console (usually unused).
+
+* The xterm channel - this is the default - bring up an xterm on this channel
+ and direct IO to it. Note, that in order for xterm to work, the host must
+ have the UML distribution package installed. This usually contains the
+ port-helper and other utilities needed for UML to communicate with the xterm.
+ Alternatively, these need to be complied and installed from source. All
+ options applicable to consoles also apply to UML serial lines which are
+ presented as ttyS inside UML.
+
+Starting UML
+============
+
+We can now run UML.
+::
+ # linux mem=2048M umid=TEST \
+ ubd0=Filesystem.img \
+ vec0:transport=tap,ifname=tap0,depth=128,gro=1 \
+ root=/dev/ubda con=null con0=null,fd:2 con1=fd:0,fd:1
+
+This will run an instance with ``2048M RAM``, try to use the image file
+called ``Filesystem.img`` as root. It will connect to the host using tap0.
+All consoles except ``con1`` will be disabled and console 1 will
+use standard input/output making it appear in the same terminal it was started.
+
+Logging in
+============
+
+If you have not set up a password when generating the image, you will have to
+shut down the UML instance, mount the image, chroot into it and set it - as
+described in the Generating an Image section. If the password is already set,
+you can just log in.
+
+The UML Management Console
+============================
+
+In addition to managing the image from "the inside" using normal sysadmin tools,
+it is possible to perform a number of low level operations using the UML
+management console. The UML management console is a low-level interface to the
+kernel on a running UML instance, somewhat like the i386 SysRq interface. Since
+there is a full-blown operating system under UML, there is much greater
+flexibility possible than with the SysRq mechanism.
+
+There are a number of things you can do with the mconsole interface:
+
+* get the kernel version
+* add and remove devices
+* halt or reboot the machine
+* Send SysRq commands
+* Pause and resume the UML
+* Inspect processes running inside UML
+* Inspect UML internal /proc state
+
+You need the mconsole client (uml\_mconsole) which is a part of the UML
+tools package available in most Linux distritions.
+
+You also need ``CONFIG_MCONSOLE`` (under 'General Setup') enabled in the UML
+kernel. When you boot UML, you'll see a line like::
+
+ mconsole initialized on /home/jdike/.uml/umlNJ32yL/mconsole
+
+If you specify a unique machine id one the UML command line, i.e.
+``umid=debian``, you'll see this::
+
+ mconsole initialized on /home/jdike/.uml/debian/mconsole
+
+
+That file is the socket that uml_mconsole will use to communicate with
+UML. Run it with either the umid or the full path as its argument::
+
+ # uml_mconsole debian
+
+or
+
+ # uml_mconsole /home/jdike/.uml/debian/mconsole
+
+
+You'll get a prompt, at which you can run one of these commands:
+
+* version
+* help
+* halt
+* reboot
+* config
+* remove
+* sysrq
+* help
+* cad
+* stop
+* go
+* proc
+* stack
+
+version
+-------
+
+This command takes no arguments. It prints the UML version::
+
+ (mconsole) version
+ OK Linux OpenWrt 4.14.106 #0 Tue Mar 19 08:19:41 2019 x86_64
+
+
+There are a couple actual uses for this. It's a simple no-op which
+can be used to check that a UML is running. It's also a way of
+sending a device interrupt to the UML. UML mconsole is treated internally as
+a UML device.
+
+help
+----
+
+This command takes no arguments. It prints a short help screen with the
+supported mconsole commands.
+
+
+halt and reboot
+---------------
+
+These commands take no arguments. They shut the machine down immediately, with
+no syncing of disks and no clean shutdown of userspace. So, they are
+pretty close to crashing the machine::
+
+ (mconsole) halt
+ OK
+
+config
+------
+
+"config" adds a new device to the virtual machine. This is supported
+by most UML device drivers. It takes one argument, which is the
+device to add, with the same syntax as the kernel command line::
+
+ (mconsole) config ubd3=/home/jdike/incoming/roots/root_fs_debian22
+
+remove
+------
+
+"remove" deletes a device from the system. Its argument is just the
+name of the device to be removed. The device must be idle in whatever
+sense the driver considers necessary. In the case of the ubd driver,
+the removed block device must not be mounted, swapped on, or otherwise
+open, and in the case of the network driver, the device must be down::
+
+ (mconsole) remove ubd3
+
+sysrq
+-----
+
+This command takes one argument, which is a single letter. It calls the
+generic kernel's SysRq driver, which does whatever is called for by
+that argument. See the SysRq documentation in
+Documentation/admin-guide/sysrq.rst in your favorite kernel tree to
+see what letters are valid and what they do.
+
+cad
+---
+
+This invokes the ``Ctl-Alt-Del`` action in the running image. What exactly
+this ends up doing is up to init, systemd, etc. Normally, it reboots the
+machine.
+
+stop
+----
+
+This puts the UML in a loop reading mconsole requests until a 'go'
+mconsole command is received. This is very useful as a
+debugging/snapshotting tool.
+
+go
+--
+
+This resumes a UML after being paused by a 'stop' command. Note that
+when the UML has resumed, TCP connections may have timed out and if
+the UML is paused for a long period of time, crond might go a little
+crazy, running all the jobs it didn't do earlier.
+
+proc
+----
+
+This takes one argument - the name of a file in /proc which is printed
+to the mconsole standard output
+
+stack
+-----
+
+This takes one argument - the pid number of a process. Its stack is
+printed to a standard output.
+
+*******************
+Advanced UML Topics
+*******************
+
+Sharing Filesystems between Virtual Machines
+============================================
+
+Don't attempt to share filesystems simply by booting two UMLs from the
+same file. That's the same thing as booting two physical machines
+from a shared disk. It will result in filesystem corruption.
+
+Using layered block devices
+---------------------------
+
+The way to share a filesystem between two virtual machines is to use
+the copy-on-write (COW) layering capability of the ubd block driver.
+Any changed blocks are stored in the private COW file, while reads come
+from either device - the private one if the requested block is valid in
+it, the shared one if not. Using this scheme, the majority of data
+which is unchanged is shared between an arbitrary number of virtual
+machines, each of which has a much smaller file containing the changes
+that it has made. With a large number of UMLs booting from a large root
+filesystem, this leads to a huge disk space saving.
+
+Sharing file system data will also help performance, since the host will
+be able to cache the shared data using a much smaller amount of memory,
+so UML disk requests will be served from the host's memory rather than
+its disks. There is a major caveat in doing this on multisocket NUMA
+machines. On such hardware, running many UML instances with a shared
+master image and COW changes may caise issues like NMIs from excess of
+inter-socket traffic.
+
+If you are running UML on high end hardware like this, make sure to
+bind UML to a set of logical cpus residing on the same socket using the
+``taskset`` command or have a look at the "tuning" section.
+
+To add a copy-on-write layer to an existing block device file, simply
+add the name of the COW file to the appropriate ubd switch::
+
+ ubd0=root_fs_cow,root_fs_debian_22
+
+where ``root_fs_cow`` is the private COW file and ``root_fs_debian_22`` is
+the existing shared filesystem. The COW file need not exist. If it
+doesn't, the driver will create and initialize it.
+
+Disk Usage
+----------
+
+UML has TRIM support which will release any unused space in its disk
+image files to the underlying OS. It is important to use either ls -ls
+or du to verify the actual file size.
+
+COW validity.
+-------------
+
+Any changes to the master image will invalidate all COW files. If this
+happens, UML will *NOT* automatically delete any of the COW files and
+will refuse to boot. In this case the only solution is to either
+restore the old image (including its last modified timestamp) or remove
+all COW files which will result in their recreation. Any changes in
+the COW files will be lost.
+
+Cows can moo - uml_moo : Merging a COW file with its backing file
+-----------------------------------------------------------------
+
+Depending on how you use UML and COW devices, it may be advisable to
+merge the changes in the COW file into the backing file every once in
+a while.
+
+The utility that does this is uml_moo. Its usage is::
+
+ uml_moo COW_file new_backing_file
+
+
+There's no need to specify the backing file since that information is
+already in the COW file header. If you're paranoid, boot the new
+merged file, and if you're happy with it, move it over the old backing
+file.
+
+``uml_moo`` creates a new backing file by default as a safety measure.
+It also has a destructive merge option which will merge the COW file
+directly into its current backing file. This is really only usable
+when the backing file only has one COW file associated with it. If
+there are multiple COWs associated with a backing file, a -d merge of
+one of them will invalidate all of the others. However, it is
+convenient if you're short of disk space, and it should also be
+noticeably faster than a non-destructive merge.
+
+``uml_moo`` is installed with the UML distribution packages and is
+available as a part of UML utilities.
+
+Host file access
+==================
+
+If you want to access files on the host machine from inside UML, you
+can treat it as a separate machine and either nfs mount directories
+from the host or copy files into the virtual machine with scp.
+However, since UML is running on the host, it can access those
+files just like any other process and make them available inside the
+virtual machine without the need to use the network.
+This is possible with the hostfs virtual filesystem. With it, you
+can mount a host directory into the UML filesystem and access the
+files contained in it just as you would on the host.
+
+*SECURITY WARNING*
+
+Hostfs without any parameters to the UML Image will allow the image
+to mount any part of the host filesystem and write to it. Always
+confine hostfs to a specific "harmless" directory (for example ``/var/tmp``)
+if running UML. This is especially important if UML is being run as root.
+
+Using hostfs
+------------
+
+To begin with, make sure that hostfs is available inside the virtual
+machine with::
+
+ # cat /proc/filesystems
+
+``hostfs`` should be listed. If it's not, either rebuild the kernel
+with hostfs configured into it or make sure that hostfs is built as a
+module and available inside the virtual machine, and insmod it.
+
+
+Now all you need to do is run mount::
+
+ # mount none /mnt/host -t hostfs
+
+will mount the host's ``/`` on the virtual machine's ``/mnt/host``.
+If you don't want to mount the host root directory, then you can
+specify a subdirectory to mount with the -o switch to mount::
+
+ # mount none /mnt/home -t hostfs -o /home
+
+will mount the hosts's /home on the virtual machine's /mnt/home.
+
+hostfs as the root filesystem
+-----------------------------
+
+It's possible to boot from a directory hierarchy on the host using
+hostfs rather than using the standard filesystem in a file.
+To start, you need that hierarchy. The easiest way is to loop mount
+an existing root_fs file::
+
+ # mount root_fs uml_root_dir -o loop
+
+
+You need to change the filesystem type of ``/`` in ``etc/fstab`` to be
+'hostfs', so that line looks like this::
+
+ /dev/ubd/0 / hostfs defaults 1 1
+
+Then you need to chown to yourself all the files in that directory
+that are owned by root. This worked for me::
+
+ # find . -uid 0 -exec chown jdike {} \;
+
+Next, make sure that your UML kernel has hostfs compiled in, not as a
+module. Then run UML with the boot device pointing at that directory::
+
+ ubd0=/path/to/uml/root/directory
+
+UML should then boot as it does normally.
+
+Hostfs Caveats
+--------------
+
+Hostfs does not support keeping track of host filesystem changes on the
+host (outside UML). As a result, if a file is changed without UML's
+knowledge, UML will not know about it and its own in-memory cache of
+the file may be corrupt. While it is possible to fix this, it is not
+something which is being worked on at present.
+
+Tuning UML
+============
+
+UML at present is strictly uniprocessor. It will, however spin up a
+number of threads to handle various functions.
+
+The UBD driver, SIGIO and the MMU emulation do that. If the system is
+idle, these threads will be migrated to other processors on a SMP host.
+This, unfortunately, will usually result in LOWER performance because of
+all of the cache/memory synchronization traffic between cores. As a
+result, UML will usually benefit from being pinned on a single CPU
+especially on a large system. This can result in performance differences
+of 5 times or higher on some benchmarks.
+
+Similarly, on large multi-node NUMA systems UML will benefit if all of
+its memory is allocated from the same NUMA node it will run on. The
+OS will *NOT* do that by default. In order to do that, the sysadmin
+needs to create a suitable tmpfs ramdisk bound to a particular node
+and use that as the source for UML RAM allocation by specifying it
+in the TMP or TEMP environment variables. UML will look at the values
+of ``TMPDIR``, ``TMP`` or ``TEMP`` for that. If that fails, it will
+look for shmfs mounted under ``/dev/shm``. If everything else fails use
+``/tmp/`` regardless of the filesystem type used for it::
+
+ mount -t tmpfs -ompol=bind:X none /mnt/tmpfs-nodeX
+ TEMP=/mnt/tmpfs-nodeX taskset -cX linux options options options..
+
+*******************************************
+Contributing to UML and Developing with UML
+*******************************************
+
+UML is an excellent platform to develop new Linux kernel concepts -
+filesystems, devices, virtualization, etc. It provides unrivalled
+opportunities to create and test them without being constrained to
+emulating specific hardware.
+
+Example - want to try how linux will work with 4096 "proper" network
+devices?
+
+Not an issue with UML. At the same time, this is something which
+is difficult with other virtualization packages - they are
+constrained by the number of devices allowed on the hardware bus
+they are trying to emulate (for example 16 on a PCI bus in qemu).
+
+If you have something to contribute such as a patch, a bugfix, a
+new feature, please send it to ``linux-um@lists.infradead.org``
+
+Please follow all standard Linux patch guidelines such as cc-ing
+relevant maintainers and run ``./sripts/checkpatch.pl`` on your patch.
+For more details see ``Documentation/process/submitting-patches.rst``
+
+Note - the list does not accept HTML or attachments, all emails must
+be formatted as plain text.
+
+Developing always goes hand in hand with debugging. First of all,
+you can always run UML under gdb and there will be a whole section
+later on on how to do that. That, however, is not the only way to
+debug a linux kernel. Quite often adding tracing statements and/or
+using UML specific approaches such as ptracing the UML kernel process
+are significantly more informative.
+
+Tracing UML
+=============
+
+When running UML consists of a main kernel thread and a number of
+helper threads. The ones of interest for tracing are NOT the ones
+that are already ptraced by UML as a part of its MMU emulation.
+
+These are usually the first three threads visible in a ps display.
+The one with the lowest PID number and using most CPU is usually the
+kernel thread. The other threads are the disk
+(ubd) device helper thread and the sigio helper thread.
+Running ptrace on this thread usually results in the following picture::
+
+ host$ strace -p 16566
+ --- SIGIO {si_signo=SIGIO, si_code=POLL_IN, si_band=65} ---
+ epoll_wait(4, [{EPOLLIN, {u32=3721159424, u64=3721159424}}], 64, 0) = 1
+ epoll_wait(4, [], 64, 0) = 0
+ rt_sigreturn({mask=[PIPE]}) = 16967
+ ptrace(PTRACE_GETREGS, 16967, NULL, 0xd5f34f38) = 0
+ ptrace(PTRACE_GETREGSET, 16967, NT_X86_XSTATE, [{iov_base=0xd5f35010, iov_len=832}]) = 0
+ ptrace(PTRACE_GETSIGINFO, 16967, NULL, {si_signo=SIGTRAP, si_code=0x85, si_pid=16967, si_uid=0}) = 0
+ ptrace(PTRACE_SETREGS, 16967, NULL, 0xd5f34f38) = 0
+ ptrace(PTRACE_SETREGSET, 16967, NT_X86_XSTATE, [{iov_base=0xd5f35010, iov_len=2696}]) = 0
+ ptrace(PTRACE_SYSEMU, 16967, NULL, 0) = 0
+ --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_TRAPPED, si_pid=16967, si_uid=0, si_status=SIGTRAP, si_utime=65, si_stime=89} ---
+ wait4(16967, [{WIFSTOPPED(s) && WSTOPSIG(s) == SIGTRAP | 0x80}], WSTOPPED|__WALL, NULL) = 16967
+ ptrace(PTRACE_GETREGS, 16967, NULL, 0xd5f34f38) = 0
+ ptrace(PTRACE_GETREGSET, 16967, NT_X86_XSTATE, [{iov_base=0xd5f35010, iov_len=832}]) = 0
+ ptrace(PTRACE_GETSIGINFO, 16967, NULL, {si_signo=SIGTRAP, si_code=0x85, si_pid=16967, si_uid=0}) = 0
+ timer_settime(0, 0, {it_interval={tv_sec=0, tv_nsec=0}, it_value={tv_sec=0, tv_nsec=2830912}}, NULL) = 0
+ getpid() = 16566
+ clock_nanosleep(CLOCK_MONOTONIC, 0, {tv_sec=1, tv_nsec=0}, NULL) = ? ERESTART_RESTARTBLOCK (Interrupted by signal)
+ --- SIGALRM {si_signo=SIGALRM, si_code=SI_TIMER, si_timerid=0, si_overrun=0, si_value={int=1631716592, ptr=0x614204f0}} ---
+ rt_sigreturn({mask=[PIPE]}) = -1 EINTR (Interrupted system call)
+
+This is a typical picture from a mostly idle UML instance
+
+* UML interrupt controller uses epoll - this is UML waiting for IO
+ interrupts:
+
+ epoll_wait(4, [{EPOLLIN, {u32=3721159424, u64=3721159424}}], 64, 0) = 1
+
+* The sequence of ptrace calls is part of MMU emulation and runnin the
+ UML userspace
+* ``timer_settime`` is part of the UML high res timer subsystem mapping
+ timer requests from inside UML onto the host high resultion timers.
+* ``clock_nanosleep`` is UML going into idle (similar to the way a PC
+ will execute an ACPI idle).
+
+As you can see UML will generate quite a bit of output even in idle.The output
+can be very informative when observing IO. It shows the actual IO calls, their
+arguments and returns values.
+
+Kernel debugging
+================
+
+You can run UML under gdb now, though it will not necessarily agree to
+be started under it. If you are trying to track a runtime bug, it is
+much better to attach gdb to a running UML instance and let UML run.
+
+Assuming the same PID number as in the previous example, this would be::
+
+ # gdb -p 16566
+
+This will STOP the UML instance, so you must enter `cont` at the GDB
+command line to request it to continue. It may be a good idea to make
+this into a gdb script and pass it to gdb as an argument.
+
+Developing Device Drivers
+=========================
+
+Nearly all UML drivers are monolithic. While it is possible to build a
+UML driver as a kernel module, that limits the possible functionality
+to in-kernel only and non-UML specific. The reason for this is that
+in order to really leverage UML, one needs to write a piece of
+userspace code which maps driver concepts onto actual userspace host
+calls.
+
+This forms the so called "user" portion of the driver. While it can
+reuse a lot of kernel concepts, it is generally just another piece of
+userspace code. This portion needs some matching "kernel" code which
+resides inside the UML image and which implements the Linux kernel part.
+
+*Note: There are very few limitations in the way "kernel" and "user" interact*.
+
+UML does not have a strictly defined kernel to host API. It does not
+try to emulate a specific architecture or bus. UML's "kernel" and
+"user" can share memory, code and interact as needed to implement
+whatever design the software developer has in mind. The only
+limitations are purely technical. Due to a lot of functions and
+variables having the same names, the developer should be careful
+which includes and libraries they are trying to refer to.
+
+As a result a lot of userspace code consists of simple wrappers.
+F.e. ``os_close_file()`` is just a wrapper around ``close()``
+which ensures that the userspace function close does not clash
+with similarly named function(s) in the kernel part.
+
+Security Considerations
+-----------------------
+
+Drivers or any new functionality should default to not
+accepting arbitrary filename, bpf code or other parameters
+which can affect the host from inside the UML instance.
+For example, specifying the socket used for IPC communication
+between a driver and the host at the UML command line is OK
+security-wise. Allowing it as a loadable module parameter
+isn't.
+
+If such functionality is desireable for a particular application
+(e.g. loading BPF "firmware" for raw socket network transports),
+it should be off by default and should be explicitly turned on
+as a command line parameter at startup.
+
+Even with this in mind, the level of isolation between UML
+and the host is relatively weak. If the UML userspace is
+allowed to load arbitrary kernel drivers, an attacker can
+use this to break out of UML. Thus, if UML is used in
+a production application, it is recommended that all modules
+are loaded at boot and kernel module loading is disabled
+afterwards.
-.. hmm:
+.. _hmm:
=====================================
Heterogeneous Memory Management (HMM)
Migration to and from device memory
===================================
-Because the CPU cannot access device memory, migration must use the device DMA
-engine to perform copy from and to device memory. For this we need to use
-migrate_vma_setup(), migrate_vma_pages(), and migrate_vma_finalize() helpers.
-
+Because the CPU cannot access device memory directly, the device driver must
+use hardware DMA or device specific load/store instructions to migrate data.
+The migrate_vma_setup(), migrate_vma_pages(), and migrate_vma_finalize()
+functions are designed to make drivers easier to write and to centralize common
+code across drivers.
+
+Before migrating pages to device private memory, special device private
+``struct page`` need to be created. These will be used as special "swap"
+page table entries so that a CPU process will fault if it tries to access
+a page that has been migrated to device private memory.
+
+These can be allocated and freed with::
+
+ struct resource *res;
+ struct dev_pagemap pagemap;
+
+ res = request_free_mem_region(&iomem_resource, /* number of bytes */,
+ "name of driver resource");
+ pagemap.type = MEMORY_DEVICE_PRIVATE;
+ pagemap.range.start = res->start;
+ pagemap.range.end = res->end;
+ pagemap.nr_range = 1;
+ pagemap.ops = &device_devmem_ops;
+ memremap_pages(&pagemap, numa_node_id());
+
+ memunmap_pages(&pagemap);
+ release_mem_region(pagemap.range.start, range_len(&pagemap.range));
+
+There are also devm_request_free_mem_region(), devm_memremap_pages(),
+devm_memunmap_pages(), and devm_release_mem_region() when the resources can
+be tied to a ``struct device``.
+
+The overall migration steps are similar to migrating NUMA pages within system
+memory (see :ref:`Page migration <page_migration>`) but the steps are split
+between device driver specific code and shared common code:
+
+1. ``mmap_read_lock()``
+
+ The device driver has to pass a ``struct vm_area_struct`` to
+ migrate_vma_setup() so the mmap_read_lock() or mmap_write_lock() needs to
+ be held for the duration of the migration.
+
+2. ``migrate_vma_setup(struct migrate_vma *args)``
+
+ The device driver initializes the ``struct migrate_vma`` fields and passes
+ the pointer to migrate_vma_setup(). The ``args->flags`` field is used to
+ filter which source pages should be migrated. For example, setting
+ ``MIGRATE_VMA_SELECT_SYSTEM`` will only migrate system memory and
+ ``MIGRATE_VMA_SELECT_DEVICE_PRIVATE`` will only migrate pages residing in
+ device private memory. If the latter flag is set, the ``args->pgmap_owner``
+ field is used to identify device private pages owned by the driver. This
+ avoids trying to migrate device private pages residing in other devices.
+ Currently only anonymous private VMA ranges can be migrated to or from
+ system memory and device private memory.
+
+ One of the first steps migrate_vma_setup() does is to invalidate other
+ device's MMUs with the ``mmu_notifier_invalidate_range_start(()`` and
+ ``mmu_notifier_invalidate_range_end()`` calls around the page table
+ walks to fill in the ``args->src`` array with PFNs to be migrated.
+ The ``invalidate_range_start()`` callback is passed a
+ ``struct mmu_notifier_range`` with the ``event`` field set to
+ ``MMU_NOTIFY_MIGRATE`` and the ``migrate_pgmap_owner`` field set to
+ the ``args->pgmap_owner`` field passed to migrate_vma_setup(). This is
+ allows the device driver to skip the invalidation callback and only
+ invalidate device private MMU mappings that are actually migrating.
+ This is explained more in the next section.
+
+ While walking the page tables, a ``pte_none()`` or ``is_zero_pfn()``
+ entry results in a valid "zero" PFN stored in the ``args->src`` array.
+ This lets the driver allocate device private memory and clear it instead
+ of copying a page of zeros. Valid PTE entries to system memory or
+ device private struct pages will be locked with ``lock_page()``, isolated
+ from the LRU (if system memory since device private pages are not on
+ the LRU), unmapped from the process, and a special migration PTE is
+ inserted in place of the original PTE.
+ migrate_vma_setup() also clears the ``args->dst`` array.
+
+3. The device driver allocates destination pages and copies source pages to
+ destination pages.
+
+ The driver checks each ``src`` entry to see if the ``MIGRATE_PFN_MIGRATE``
+ bit is set and skips entries that are not migrating. The device driver
+ can also choose to skip migrating a page by not filling in the ``dst``
+ array for that page.
+
+ The driver then allocates either a device private struct page or a
+ system memory page, locks the page with ``lock_page()``, and fills in the
+ ``dst`` array entry with::
+
+ dst[i] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
+
+ Now that the driver knows that this page is being migrated, it can
+ invalidate device private MMU mappings and copy device private memory
+ to system memory or another device private page. The core Linux kernel
+ handles CPU page table invalidations so the device driver only has to
+ invalidate its own MMU mappings.
+
+ The driver can use ``migrate_pfn_to_page(src[i])`` to get the
+ ``struct page`` of the source and either copy the source page to the
+ destination or clear the destination device private memory if the pointer
+ is ``NULL`` meaning the source page was not populated in system memory.
+
+4. ``migrate_vma_pages()``
+
+ This step is where the migration is actually "committed".
+
+ If the source page was a ``pte_none()`` or ``is_zero_pfn()`` page, this
+ is where the newly allocated page is inserted into the CPU's page table.
+ This can fail if a CPU thread faults on the same page. However, the page
+ table is locked and only one of the new pages will be inserted.
+ The device driver will see that the ``MIGRATE_PFN_MIGRATE`` bit is cleared
+ if it loses the race.
+
+ If the source page was locked, isolated, etc. the source ``struct page``
+ information is now copied to destination ``struct page`` finalizing the
+ migration on the CPU side.
+
+5. Device driver updates device MMU page tables for pages still migrating,
+ rolling back pages not migrating.
+
+ If the ``src`` entry still has ``MIGRATE_PFN_MIGRATE`` bit set, the device
+ driver can update the device MMU and set the write enable bit if the
+ ``MIGRATE_PFN_WRITE`` bit is set.
+
+6. ``migrate_vma_finalize()``
+
+ This step replaces the special migration page table entry with the new
+ page's page table entry and releases the reference to the source and
+ destination ``struct page``.
+
+7. ``mmap_read_unlock()``
+
+ The lock can now be released.
Memory cgroup (memcg) and rss accounting
========================================
:maxdepth: 1
active_mm
+ arch_pgtable_helpers
balance
cleancache
free_page_reporting
Page migration
==============
-Page migration allows the moving of the physical location of pages between
-nodes in a numa system while the process is running. This means that the
+Page migration allows moving the physical location of pages between
+nodes in a NUMA system while the process is running. This means that the
virtual addresses that the process sees do not change. However, the
system rearranges the physical location of those pages.
-The main intend of page migration is to reduce the latency of memory access
+Also see :ref:`Heterogeneous Memory Management (HMM) <hmm>`
+for migrating pages to or from device private memory.
+
+The main intent of page migration is to reduce the latency of memory accesses
by moving pages near to the processor where the process accessing that memory
is running.
Page migration allows a process to manually relocate the node on which its
pages are located through the MF_MOVE and MF_MOVE_ALL options while setting
-a new memory policy via mbind(). The pages of process can also be relocated
+a new memory policy via mbind(). The pages of a process can also be relocated
from another process using the sys_migrate_pages() function call. The
-migrate_pages function call takes two sets of nodes and moves pages of a
+migrate_pages() function call takes two sets of nodes and moves pages of a
process that are located on the from nodes to the destination nodes.
Page migration functions are provided by the numactl package by Andi Kleen
(a version later than 0.9.3 is required. Get it from
-ftp://oss.sgi.com/www/projects/libnuma/download/). numactl provides libnuma
-which provides an interface similar to other numa functionality for page
+https://github.com/numactl/numactl.git). numactl provides libnuma
+which provides an interface similar to other NUMA functionality for page
migration. cat ``/proc/<pid>/numa_maps`` allows an easy review of where the
pages of a process are located. See also the numa_maps documentation in the
proc(5) man page.
Manual migration is useful if for example the scheduler has relocated
a process to a processor on a distant node. A batch scheduler or an
administrator may detect the situation and move the pages of the process
-nearer to the new processor. The kernel itself does only provide
+nearer to the new processor. The kernel itself only provides
manual page migration support. Automatic page migration may be implemented
through user space processes that move pages. A special function call
"move_pages" allows the moving of individual pages within a process.
-A NUMA profiler may f.e. obtain a log showing frequent off node
+For example, A NUMA profiler may obtain a log showing frequent off-node
accesses and may use the result to move pages to more advantageous
locations.
Larger installations usually partition the system using cpusets into
sections of nodes. Paul Jackson has equipped cpusets with the ability to
move pages when a task is moved to another cpuset (See
-Documentation/admin-guide/cgroup-v1/cpusets.rst).
-Cpusets allows the automation of process locality. If a task is moved to
+:ref:`CPUSETS <cpusets>`).
+Cpusets allow the automation of process locality. If a task is moved to
a new cpuset then also all its pages are moved with it so that the
performance of the process does not sink dramatically. Also the pages
of processes in a cpuset are moved if the allowed memory nodes of a
Lists of pages to be migrated are generated by scanning over
pages and moving them into lists. This is done by
calling isolate_lru_page().
- Calling isolate_lru_page increases the references to the page
+ Calling isolate_lru_page() increases the references to the page
so that it cannot vanish while the page migration occurs.
- It also prevents the swapper or other scans to encounter
+ It also prevents the swapper or other scans from encountering
the page.
2. We need to have a function of type new_page_t that can be
Steps:
-1. Lock the page to be migrated
+1. Lock the page to be migrated.
2. Ensure that writeback is complete.
3. Lock the new page that we want to move to. It is locked so that accesses to
- this (not yet uptodate) page immediately lock while the move is in progress.
+ this (not yet uptodate) page immediately block while the move is in progress.
4. All the page table references to the page are converted to migration
entries. This decreases the mapcount of a page. If the resulting
mapcount is not zero then we do not migrate the page. All user space
- processes that attempt to access the page will now wait on the page lock.
+ processes that attempt to access the page will now wait on the page lock
+ or wait for the migration page table entry to be removed.
5. The i_pages lock is taken. This will cause all processes trying
to access the page via the mapping to block on the spinlock.
-6. The refcount of the page is examined and we back out if references remain
- otherwise we know that we are the only one referencing this page.
+6. The refcount of the page is examined and we back out if references remain.
+ Otherwise, we know that we are the only one referencing this page.
7. The radix tree is checked and if it does not contain the pointer to this
page then we back out because someone else modified the radix tree.
15. Queued up writeback on the new page is triggered.
-16. If migration entries were page then replace them with real ptes. Doing
- so will enable access for user space processes not already waiting for
- the page lock.
+16. If migration entries were inserted into the page table, then replace them
+ with real ptes. Doing so will enable access for user space processes not
+ already waiting for the page lock.
-19. The page locks are dropped from the old and new page.
+17. The page locks are dropped from the old and new page.
Processes waiting on the page lock will redo their page faults
and will reach the new page.
-20. The new page is moved to the LRU and can be scanned by the swapper
- etc again.
+18. The new page is moved to the LRU and can be scanned by the swapper,
+ etc. again.
Non-LRU page migration
======================
-Although original migration aimed for reducing the latency of memory access
-for NUMA, compaction who want to create high-order page is also main customer.
+Although migration originally aimed for reducing the latency of memory accesses
+for NUMA, compaction also uses migration to create high-order pages.
Current problem of the implementation is that it is designed to migrate only
-*LRU* pages. However, there are potential non-lru pages which can be migrated
+*LRU* pages. However, there are potential non-LRU pages which can be migrated
in drivers, for example, zsmalloc, virtio-balloon pages.
For virtio-balloon pages, some parts of migration code path have been hooked
up and added virtio-balloon specific functions to intercept migration logics.
It's too specific to a driver so other drivers who want to make their pages
-movable would have to add own specific hooks in migration path.
+movable would have to add their own specific hooks in the migration path.
-To overclome the problem, VM supports non-LRU page migration which provides
+To overcome the problem, VM supports non-LRU page migration which provides
generic functions for non-LRU movable pages without driver specific hooks
-migration path.
+in the migration path.
-If a driver want to make own pages movable, it should define three functions
+If a driver wants to make its pages movable, it should define three functions
which are function pointers of struct address_space_operations.
1. ``bool (*isolate_page) (struct page *page, isolate_mode_t mode);``
- What VM expects on isolate_page function of driver is to return *true*
- if driver isolates page successfully. On returing true, VM marks the page
+ What VM expects from isolate_page() function of driver is to return *true*
+ if driver isolates the page successfully. On returning true, VM marks the page
as PG_isolated so concurrent isolation in several CPUs skip the page
for isolation. If a driver cannot isolate the page, it should return *false*.
Once page is successfully isolated, VM uses page.lru fields so driver
- shouldn't expect to preserve values in that fields.
+ shouldn't expect to preserve values in those fields.
2. ``int (*migratepage) (struct address_space *mapping,``
| ``struct page *newpage, struct page *oldpage, enum migrate_mode);``
- After isolation, VM calls migratepage of driver with isolated page.
- The function of migratepage is to move content of the old page to new page
+ After isolation, VM calls migratepage() of driver with the isolated page.
+ The function of migratepage() is to move the contents of the old page to the
+ new page
and set up fields of struct page newpage. Keep in mind that you should
indicate to the VM the oldpage is no longer movable via __ClearPageMovable()
- under page_lock if you migrated the oldpage successfully and returns
+ under page_lock if you migrated the oldpage successfully and returned
MIGRATEPAGE_SUCCESS. If driver cannot migrate the page at the moment, driver
can return -EAGAIN. On -EAGAIN, VM will retry page migration in a short time
- because VM interprets -EAGAIN as "temporal migration failure". On returning
- any error except -EAGAIN, VM will give up the page migration without retrying
- in this time.
+ because VM interprets -EAGAIN as "temporary migration failure". On returning
+ any error except -EAGAIN, VM will give up the page migration without
+ retrying.
- Driver shouldn't touch page.lru field VM using in the functions.
+ Driver shouldn't touch the page.lru field while in the migratepage() function.
3. ``void (*putback_page)(struct page *);``
- If migration fails on isolated page, VM should return the isolated page
- to the driver so VM calls driver's putback_page with migration failed page.
- In this function, driver should put the isolated page back to the own data
+ If migration fails on the isolated page, VM should return the isolated page
+ to the driver so VM calls the driver's putback_page() with the isolated page.
+ In this function, the driver should put the isolated page back into its own data
structure.
-4. non-lru movable page flags
+4. non-LRU movable page flags
- There are two page flags for supporting non-lru movable page.
+ There are two page flags for supporting non-LRU movable page.
* PG_movable
- Driver should use the below function to make page movable under page_lock::
+ Driver should use the function below to make page movable under page_lock::
void __SetPageMovable(struct page *page, struct address_space *mapping)
It needs argument of address_space for registering migration
family functions which will be called by VM. Exactly speaking,
- PG_movable is not a real flag of struct page. Rather than, VM
- reuses page->mapping's lower bits to represent it.
+ PG_movable is not a real flag of struct page. Rather, VM
+ reuses the page->mapping's lower bits to represent it::
-::
#define PAGE_MAPPING_MOVABLE 0x2
page->mapping = page->mapping | PAGE_MAPPING_MOVABLE;
so driver shouldn't access page->mapping directly. Instead, driver should
- use page_mapping which mask off the low two bits of page->mapping under
- page lock so it can get right struct address_space.
-
- For testing of non-lru movable page, VM supports __PageMovable function.
- However, it doesn't guarantee to identify non-lru movable page because
- page->mapping field is unified with other variables in struct page.
- As well, if driver releases the page after isolation by VM, page->mapping
- doesn't have stable value although it has PAGE_MAPPING_MOVABLE
- (Look at __ClearPageMovable). But __PageMovable is cheap to catch whether
- page is LRU or non-lru movable once the page has been isolated. Because
- LRU pages never can have PAGE_MAPPING_MOVABLE in page->mapping. It is also
- good for just peeking to test non-lru movable pages before more expensive
- checking with lock_page in pfn scanning to select victim.
-
- For guaranteeing non-lru movable page, VM provides PageMovable function.
- Unlike __PageMovable, PageMovable functions validates page->mapping and
- mapping->a_ops->isolate_page under lock_page. The lock_page prevents sudden
- destroying of page->mapping.
-
- Driver using __SetPageMovable should clear the flag via __ClearMovablePage
- under page_lock before the releasing the page.
+ use page_mapping() which masks off the low two bits of page->mapping under
+ page lock so it can get the right struct address_space.
+
+ For testing of non-LRU movable pages, VM supports __PageMovable() function.
+ However, it doesn't guarantee to identify non-LRU movable pages because
+ the page->mapping field is unified with other variables in struct page.
+ If the driver releases the page after isolation by VM, page->mapping
+ doesn't have a stable value although it has PAGE_MAPPING_MOVABLE set
+ (look at __ClearPageMovable). But __PageMovable() is cheap to call whether
+ page is LRU or non-LRU movable once the page has been isolated because LRU
+ pages can never have PAGE_MAPPING_MOVABLE set in page->mapping. It is also
+ good for just peeking to test non-LRU movable pages before more expensive
+ checking with lock_page() in pfn scanning to select a victim.
+
+ For guaranteeing non-LRU movable page, VM provides PageMovable() function.
+ Unlike __PageMovable(), PageMovable() validates page->mapping and
+ mapping->a_ops->isolate_page under lock_page(). The lock_page() prevents
+ sudden destroying of page->mapping.
+
+ Drivers using __SetPageMovable() should clear the flag via
+ __ClearMovablePage() under page_lock() before the releasing the page.
* PG_isolated
To prevent concurrent isolation among several CPUs, VM marks isolated page
- as PG_isolated under lock_page. So if a CPU encounters PG_isolated non-lru
- movable page, it can skip it. Driver doesn't need to manipulate the flag
- because VM will set/clear it automatically. Keep in mind that if driver
- sees PG_isolated page, it means the page have been isolated by VM so it
- shouldn't touch page.lru field.
- PG_isolated is alias with PG_reclaim flag so driver shouldn't use the flag
- for own purpose.
+ as PG_isolated under lock_page(). So if a CPU encounters PG_isolated
+ non-LRU movable page, it can skip it. Driver doesn't need to manipulate the
+ flag because VM will set/clear it automatically. Keep in mind that if the
+ driver sees a PG_isolated page, it means the page has been isolated by the
+ VM so it shouldn't touch the page.lru field.
+ The PG_isolated flag is aliased with the PG_reclaim flag so drivers
+ shouldn't use PG_isolated for its own purposes.
Monitoring Migration
=====================
512.
2. PGMIGRATE_FAIL: Normal page migration failure. Same counting rules as for
- _SUCCESS, above: this will be increased by the number of subpages, if it was
- a THP.
+ PGMIGRATE_SUCCESS, above: this will be increased by the number of subpages,
+ if it was a THP.
3. THP_MIGRATION_SUCCESS: A THP was migrated without being split.
To manage a watch list, the following functions are provided:
- * ``void init_watch_list(struct watch_list *wlist,
- void (*release_watch)(struct watch *wlist));``
+ * ::
+
+ void init_watch_list(struct watch_list *wlist,
+ void (*release_watch)(struct watch *wlist));
Initialise a watch list. If ``release_watch`` is not NULL, then this
indicates a function that should be called when the watch_list object is
driver-settable fields in the watch struct must have been set before this
is called.
- * ``int remove_watch_from_object(struct watch_list *wlist,
- struct watch_queue *wqueue,
- u64 id, false);``
+ * ::
+
+ int remove_watch_from_object(struct watch_list *wlist,
+ struct watch_queue *wqueue,
+ u64 id, false);
Remove a watch from a watch list, where the watch must match the specified
watch queue (``wqueue``) and object identifier (``id``). A notification
In addition to read/modify/write the setup header of the struct
boot_params as that of 16-bit boot protocol, the boot loader should
-also fill the additional fields of the struct boot_params as that
-described in zero-page.txt.
+also fill the additional fields of the struct boot_params as
+described in chapter :doc:`zero-page`.
After setting up the struct boot_params, the boot loader can load the
32/64-bit kernel in the same way as that of 16-bit boot protocol.
In addition to read/modify/write the setup header of the struct
boot_params as that of 16-bit boot protocol, the boot loader should
also fill the additional fields of the struct boot_params as described
-in zero-page.txt.
+in chapter :doc:`zero-page`.
After setting up the struct boot_params, the boot loader can load
64-bit kernel in the same way as that of 16-bit boot protocol, but
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+=================
+x86 Feature Flags
+=================
+
+Introduction
+============
+
+On x86, flags appearing in /proc/cpuinfo have an X86_FEATURE definition
+in arch/x86/include/asm/cpufeatures.h. If the kernel cares about a feature
+or KVM want to expose the feature to a KVM guest, it can and should have
+an X86_FEATURE_* defined. These flags represent hardware features as
+well as software features.
+
+If users want to know if a feature is available on a given system, they
+try to find the flag in /proc/cpuinfo. If a given flag is present, it
+means that the kernel supports it and is currently making it available.
+If such flag represents a hardware feature, it also means that the
+hardware supports it.
+
+If the expected flag does not appear in /proc/cpuinfo, things are murkier.
+Users need to find out the reason why the flag is missing and find the way
+how to enable it, which is not always easy. There are several factors that
+can explain missing flags: the expected feature failed to enable, the feature
+is missing in hardware, platform firmware did not enable it, the feature is
+disabled at build or run time, an old kernel is in use, or the kernel does
+not support the feature and thus has not enabled it. In general, /proc/cpuinfo
+shows features which the kernel supports. For a full list of CPUID flags
+which the CPU supports, use tools/arch/x86/kcpuid.
+
+How are feature flags created?
+==============================
+
+a: Feature flags can be derived from the contents of CPUID leaves.
+------------------------------------------------------------------
+These feature definitions are organized mirroring the layout of CPUID
+leaves and grouped in words with offsets as mapped in enum cpuid_leafs
+in cpufeatures.h (see arch/x86/include/asm/cpufeatures.h for details).
+If a feature is defined with a X86_FEATURE_<name> definition in
+cpufeatures.h, and if it is detected at run time, the flags will be
+displayed accordingly in /proc/cpuinfo. For example, the flag "avx2"
+comes from X86_FEATURE_AVX2 in cpufeatures.h.
+
+b: Flags can be from scattered CPUID-based features.
+----------------------------------------------------
+Hardware features enumerated in sparsely populated CPUID leaves get
+software-defined values. Still, CPUID needs to be queried to determine
+if a given feature is present. This is done in init_scattered_cpuid_features().
+For instance, X86_FEATURE_CQM_LLC is defined as 11*32 + 0 and its presence is
+checked at runtime in the respective CPUID leaf [EAX=f, ECX=0] bit EDX[1].
+
+The intent of scattering CPUID leaves is to not bloat struct
+cpuinfo_x86.x86_capability[] unnecessarily. For instance, the CPUID leaf
+[EAX=7, ECX=0] has 30 features and is dense, but the CPUID leaf [EAX=7, EAX=1]
+has only one feature and would waste 31 bits of space in the x86_capability[]
+array. Since there is a struct cpuinfo_x86 for each possible CPU, the wasted
+memory is not trivial.
+
+c: Flags can be created synthetically under certain conditions for hardware features.
+-------------------------------------------------------------------------------------
+Examples of conditions include whether certain features are present in
+MSR_IA32_CORE_CAPS or specific CPU models are identified. If the needed
+conditions are met, the features are enabled by the set_cpu_cap or
+setup_force_cpu_cap macros. For example, if bit 5 is set in MSR_IA32_CORE_CAPS,
+the feature X86_FEATURE_SPLIT_LOCK_DETECT will be enabled and
+"split_lock_detect" will be displayed. The flag "ring3mwait" will be
+displayed only when running on INTEL_FAM6_XEON_PHI_[KNL|KNM] processors.
+
+d: Flags can represent purely software features.
+------------------------------------------------
+These flags do not represent hardware features. Instead, they represent a
+software feature implemented in the kernel. For example, Kernel Page Table
+Isolation is purely software feature and its feature flag X86_FEATURE_PTI is
+also defined in cpufeatures.h.
+
+Naming of Flags
+===============
+
+The script arch/x86/kernel/cpu/mkcapflags.sh processes the
+#define X86_FEATURE_<name> from cpufeatures.h and generates the
+x86_cap/bug_flags[] arrays in kernel/cpu/capflags.c. The names in the
+resulting x86_cap/bug_flags[] are used to populate /proc/cpuinfo. The naming
+of flags in the x86_cap/bug_flags[] are as follows:
+
+a: The name of the flag is from the string in X86_FEATURE_<name> by default.
+----------------------------------------------------------------------------
+By default, the flag <name> in /proc/cpuinfo is extracted from the respective
+X86_FEATURE_<name> in cpufeatures.h. For example, the flag "avx2" is from
+X86_FEATURE_AVX2.
+
+b: The naming can be overridden.
+--------------------------------
+If the comment on the line for the #define X86_FEATURE_* starts with a
+double-quote character (""), the string inside the double-quote characters
+will be the name of the flags. For example, the flag "sse4_1" comes from
+the comment "sse4_1" following the X86_FEATURE_XMM4_1 definition.
+
+There are situations in which overriding the displayed name of the flag is
+needed. For instance, /proc/cpuinfo is a userspace interface and must remain
+constant. If, for some reason, the naming of X86_FEATURE_<name> changes, one
+shall override the new naming with the name already used in /proc/cpuinfo.
+
+c: The naming override can be "", which means it will not appear in /proc/cpuinfo.
+----------------------------------------------------------------------------------
+The feature shall be omitted from /proc/cpuinfo if it does not make sense for
+the feature to be exposed to userspace. For example, X86_FEATURE_ALWAYS is
+defined in cpufeatures.h but that flag is an internal kernel feature used
+in the alternative runtime patching functionality. So, its name is overridden
+with "". Its flag will not appear in /proc/cpuinfo.
+
+Flags are missing when one or more of these happen
+==================================================
+
+a: The hardware does not enumerate support for it.
+--------------------------------------------------
+For example, when a new kernel is running on old hardware or the feature is
+not enabled by boot firmware. Even if the hardware is new, there might be a
+problem enabling the feature at run time, the flag will not be displayed.
+
+b: The kernel does not know about the flag.
+-------------------------------------------
+For example, when an old kernel is running on new hardware.
+
+c: The kernel disabled support for it at compile-time.
+------------------------------------------------------
+For example, if 5-level-paging is not enabled when building (i.e.,
+CONFIG_X86_5LEVEL is not selected) the flag "la57" will not show up [#f1]_.
+Even though the feature will still be detected via CPUID, the kernel disables
+it by clearing via setup_clear_cpu_cap(X86_FEATURE_LA57).
+
+d: The feature is disabled at boot-time.
+----------------------------------------
+A feature can be disabled either using a command-line parameter or because
+it failed to be enabled. The command-line parameter clearcpuid= can be used
+to disable features using the feature number as defined in
+/arch/x86/include/asm/cpufeatures.h. For instance, User Mode Instruction
+Protection can be disabled using clearcpuid=514. The number 514 is calculated
+from #define X86_FEATURE_UMIP (16*32 + 2).
+
+In addition, there exists a variety of custom command-line parameters that
+disable specific features. The list of parameters includes, but is not limited
+to, nofsgsbase, nosmap, and nosmep. 5-level paging can also be disabled using
+"no5lvl". SMAP and SMEP are disabled with the aforementioned parameters,
+respectively.
+
+e: The feature was known to be non-functional.
+----------------------------------------------
+The feature was known to be non-functional because a dependency was
+missing at runtime. For example, AVX flags will not show up if XSAVE feature
+is disabled since they depend on XSAVE feature. Another example would be broken
+CPUs and them missing microcode patches. Due to that, the kernel decides not to
+enable a feature.
+
+.. [#f1] 5-level paging uses linear address of 57 bits.
:numbered:
boot
+ cpuinfo
topology
exception-tables
kernel-stacks
usb-legacy-support
i386/index
x86_64/index
+ sva
non-linear. This field is purely informational
only.
+"thread_throttle_mode":
+ Indicator on Intel systems of how tasks running on threads
+ of a physical core are throttled in cases where they
+ request different memory bandwidth percentages:
+
+ "max":
+ the smallest percentage is applied
+ to all threads
+ "per-thread":
+ bandwidth percentages are directly applied to
+ the threads running on the core
+
If RDT monitoring is available there will be an "L3_MON" directory
with the following files:
The bandwidth throttling is a core specific mechanism on some of Intel
SKUs. Using a high bandwidth and a low bandwidth setting on two threads
-sharing a core will result in both threads being throttled to use the
-low bandwidth. The fact that Memory bandwidth allocation(MBA) is a core
+sharing a core may result in both threads being throttled to use the
+low bandwidth (see "thread_throttle_mode").
+
+The fact that Memory bandwidth allocation(MBA) may be a core
specific mechanism where as memory bandwidth monitoring(MBM) is done at
the package level may lead to confusion when users try to apply control
via the MBA and then monitor the bandwidth to see if the controls are
--- /dev/null
+.. SPDX-License-Identifier: GPL-2.0
+
+===========================================
+Shared Virtual Addressing (SVA) with ENQCMD
+===========================================
+
+Background
+==========
+
+Shared Virtual Addressing (SVA) allows the processor and device to use the
+same virtual addresses avoiding the need for software to translate virtual
+addresses to physical addresses. SVA is what PCIe calls Shared Virtual
+Memory (SVM).
+
+In addition to the convenience of using application virtual addresses
+by the device, it also doesn't require pinning pages for DMA.
+PCIe Address Translation Services (ATS) along with Page Request Interface
+(PRI) allow devices to function much the same way as the CPU handling
+application page-faults. For more information please refer to the PCIe
+specification Chapter 10: ATS Specification.
+
+Use of SVA requires IOMMU support in the platform. IOMMU is also
+required to support the PCIe features ATS and PRI. ATS allows devices
+to cache translations for virtual addresses. The IOMMU driver uses the
+mmu_notifier() support to keep the device TLB cache and the CPU cache in
+sync. When an ATS lookup fails for a virtual address, the device should
+use the PRI in order to request the virtual address to be paged into the
+CPU page tables. The device must use ATS again in order the fetch the
+translation before use.
+
+Shared Hardware Workqueues
+==========================
+
+Unlike Single Root I/O Virtualization (SR-IOV), Scalable IOV (SIOV) permits
+the use of Shared Work Queues (SWQ) by both applications and Virtual
+Machines (VM's). This allows better hardware utilization vs. hard
+partitioning resources that could result in under utilization. In order to
+allow the hardware to distinguish the context for which work is being
+executed in the hardware by SWQ interface, SIOV uses Process Address Space
+ID (PASID), which is a 20-bit number defined by the PCIe SIG.
+
+PASID value is encoded in all transactions from the device. This allows the
+IOMMU to track I/O on a per-PASID granularity in addition to using the PCIe
+Resource Identifier (RID) which is the Bus/Device/Function.
+
+
+ENQCMD
+======
+
+ENQCMD is a new instruction on Intel platforms that atomically submits a
+work descriptor to a device. The descriptor includes the operation to be
+performed, virtual addresses of all parameters, virtual address of a completion
+record, and the PASID (process address space ID) of the current process.
+
+ENQCMD works with non-posted semantics and carries a status back if the
+command was accepted by hardware. This allows the submitter to know if the
+submission needs to be retried or other device specific mechanisms to
+implement fairness or ensure forward progress should be provided.
+
+ENQCMD is the glue that ensures applications can directly submit commands
+to the hardware and also permits hardware to be aware of application context
+to perform I/O operations via use of PASID.
+
+Process Address Space Tagging
+=============================
+
+A new thread-scoped MSR (IA32_PASID) provides the connection between
+user processes and the rest of the hardware. When an application first
+accesses an SVA-capable device, this MSR is initialized with a newly
+allocated PASID. The driver for the device calls an IOMMU-specific API
+that sets up the routing for DMA and page-requests.
+
+For example, the Intel Data Streaming Accelerator (DSA) uses
+iommu_sva_bind_device(), which will do the following:
+
+- Allocate the PASID, and program the process page-table (%cr3 register) in the
+ PASID context entries.
+- Register for mmu_notifier() to track any page-table invalidations to keep
+ the device TLB in sync. For example, when a page-table entry is invalidated,
+ the IOMMU propagates the invalidation to the device TLB. This will force any
+ future access by the device to this virtual address to participate in
+ ATS. If the IOMMU responds with proper response that a page is not
+ present, the device would request the page to be paged in via the PCIe PRI
+ protocol before performing I/O.
+
+This MSR is managed with the XSAVE feature set as "supervisor state" to
+ensure the MSR is updated during context switch.
+
+PASID Management
+================
+
+The kernel must allocate a PASID on behalf of each process which will use
+ENQCMD and program it into the new MSR to communicate the process identity to
+platform hardware. ENQCMD uses the PASID stored in this MSR to tag requests
+from this process. When a user submits a work descriptor to a device using the
+ENQCMD instruction, the PASID field in the descriptor is auto-filled with the
+value from MSR_IA32_PASID. Requests for DMA from the device are also tagged
+with the same PASID. The platform IOMMU uses the PASID in the transaction to
+perform address translation. The IOMMU APIs setup the corresponding PASID
+entry in IOMMU with the process address used by the CPU (e.g. %cr3 register in
+x86).
+
+The MSR must be configured on each logical CPU before any application
+thread can interact with a device. Threads that belong to the same
+process share the same page tables, thus the same MSR value.
+
+PASID is cleared when a process is created. The PASID allocation and MSR
+programming may occur long after a process and its threads have been created.
+One thread must call iommu_sva_bind_device() to allocate the PASID for the
+process. If a thread uses ENQCMD without the MSR first being populated, a #GP
+will be raised. The kernel will update the PASID MSR with the PASID for all
+threads in the process. A single process PASID can be used simultaneously
+with multiple devices since they all share the same address space.
+
+One thread can call iommu_sva_unbind_device() to free the allocated PASID.
+The kernel will clear the PASID MSR for all threads belonging to the process.
+
+New threads inherit the MSR value from the parent.
+
+Relationships
+=============
+
+ * Each process has many threads, but only one PASID.
+ * Devices have a limited number (~10's to 1000's) of hardware workqueues.
+ The device driver manages allocating hardware workqueues.
+ * A single mmap() maps a single hardware workqueue as a "portal" and
+ each portal maps down to a single workqueue.
+ * For each device with which a process interacts, there must be
+ one or more mmap()'d portals.
+ * Many threads within a process can share a single portal to access
+ a single device.
+ * Multiple processes can separately mmap() the same portal, in
+ which case they still share one device hardware workqueue.
+ * The single process-wide PASID is used by all threads to interact
+ with all devices. There is not, for instance, a PASID for each
+ thread or each thread<->device pair.
+
+FAQ
+===
+
+* What is SVA/SVM?
+
+Shared Virtual Addressing (SVA) permits I/O hardware and the processor to
+work in the same address space, i.e., to share it. Some call it Shared
+Virtual Memory (SVM), but Linux community wanted to avoid confusing it with
+POSIX Shared Memory and Secure Virtual Machines which were terms already in
+circulation.
+
+* What is a PASID?
+
+A Process Address Space ID (PASID) is a PCIe-defined Transaction Layer Packet
+(TLP) prefix. A PASID is a 20-bit number allocated and managed by the OS.
+PASID is included in all transactions between the platform and the device.
+
+* How are shared workqueues different?
+
+Traditionally, in order for userspace applications to interact with hardware,
+there is a separate hardware instance required per process. For example,
+consider doorbells as a mechanism of informing hardware about work to process.
+Each doorbell is required to be spaced 4k (or page-size) apart for process
+isolation. This requires hardware to provision that space and reserve it in
+MMIO. This doesn't scale as the number of threads becomes quite large. The
+hardware also manages the queue depth for Shared Work Queues (SWQ), and
+consumers don't need to track queue depth. If there is no space to accept
+a command, the device will return an error indicating retry.
+
+A user should check Deferrable Memory Write (DMWr) capability on the device
+and only submits ENQCMD when the device supports it. In the new DMWr PCIe
+terminology, devices need to support DMWr completer capability. In addition,
+it requires all switch ports to support DMWr routing and must be enabled by
+the PCIe subsystem, much like how PCIe atomic operations are managed for
+instance.
+
+SWQ allows hardware to provision just a single address in the device. When
+used with ENQCMD to submit work, the device can distinguish the process
+submitting the work since it will include the PASID assigned to that
+process. This helps the device scale to a large number of processes.
+
+* Is this the same as a user space device driver?
+
+Communicating with the device via the shared workqueue is much simpler
+than a full blown user space driver. The kernel driver does all the
+initialization of the hardware. User space only needs to worry about
+submitting work and processing completions.
+
+* Is this the same as SR-IOV?
+
+Single Root I/O Virtualization (SR-IOV) focuses on providing independent
+hardware interfaces for virtualizing hardware. Hence, it's required to be
+almost fully functional interface to software supporting the traditional
+BARs, space for interrupts via MSI-X, its own register layout.
+Virtual Functions (VFs) are assisted by the Physical Function (PF)
+driver.
+
+Scalable I/O Virtualization builds on the PASID concept to create device
+instances for virtualization. SIOV requires host software to assist in
+creating virtual devices; each virtual device is represented by a PASID
+along with the bus/device/function of the device. This allows device
+hardware to optimize device resource creation and can grow dynamically on
+demand. SR-IOV creation and management is very static in nature. Consult
+references below for more details.
+
+* Why not just create a virtual function for each app?
+
+Creating PCIe SR-IOV type Virtual Functions (VF) is expensive. VFs require
+duplicated hardware for PCI config space and interrupts such as MSI-X.
+Resources such as interrupts have to be hard partitioned between VFs at
+creation time, and cannot scale dynamically on demand. The VFs are not
+completely independent from the Physical Function (PF). Most VFs require
+some communication and assistance from the PF driver. SIOV, in contrast,
+creates a software-defined device where all the configuration and control
+aspects are mediated via the slow path. The work submission and completion
+happen without any mediation.
+
+* Does this support virtualization?
+
+ENQCMD can be used from within a guest VM. In these cases, the VMM helps
+with setting up a translation table to translate from Guest PASID to Host
+PASID. Please consult the ENQCMD instruction set reference for more
+details.
+
+* Does memory need to be pinned?
+
+When devices support SVA along with platform hardware such as IOMMU
+supporting such devices, there is no need to pin memory for DMA purposes.
+Devices that support SVA also support other PCIe features that remove the
+pinning requirement for memory.
+
+Device TLB support - Device requests the IOMMU to lookup an address before
+use via Address Translation Service (ATS) requests. If the mapping exists
+but there is no page allocated by the OS, IOMMU hardware returns that no
+mapping exists.
+
+Device requests the virtual address to be mapped via Page Request
+Interface (PRI). Once the OS has successfully completed the mapping, it
+returns the response back to the device. The device requests again for
+a translation and continues.
+
+IOMMU works with the OS in managing consistency of page-tables with the
+device. When removing pages, it interacts with the device to remove any
+device TLB entry that might have been cached before removing the mappings from
+the OS.
+
+References
+==========
+
+VT-D:
+https://01.org/blogs/ashokraj/2018/recent-enhancements-intel-virtualization-technology-directed-i/o-intel-vt-d
+
+SIOV:
+https://01.org/blogs/2019/assignable-interfaces-intel-scalable-i/o-virtualization-linux
+
+ENQCMD in ISE:
+https://software.intel.com/sites/default/files/managed/c5/15/architecture-instruction-set-extensions-programming-reference.pdf
+
+DSA spec:
+https://software.intel.com/sites/default/files/341204-intel-data-streaming-accelerator-spec.pdf
F: Documentation/devicetree/bindings/interrupt-controller/amazon,al-fic.txt
F: drivers/irqchip/irq-al-fic.c
+AMAZON ANNAPURNA LABS MEMORY CONTROLLER EDAC
+M: Talel Shenhar <talel@amazon.com>
+M: Talel Shenhar <talelshenhar@gmail.com>
+S: Maintained
+F: Documentation/devicetree/bindings/edac/amazon,al-mc-edac.yaml
+F: drivers/edac/al_mc_edac.c
+
AMAZON ANNAPURNA LABS THERMAL MMIO DRIVER
M: Talel Shenhar <talel@amazon.com>
S: Maintained
F: drivers/amba/
F: include/linux/amba/bus.h
+ARM PRIMECELL CLCD PL110 DRIVER
+M: Russell King <linux@armlinux.org.uk>
+S: Odd Fixes
+F: drivers/video/fbdev/amba-clcd.*
+
ARM PRIMECELL KMI PL050 DRIVER
M: Russell King <linux@armlinux.org.uk>
S: Odd Fixes
F: Documentation/devicetree/bindings/clock/actions,owl-cmu.txt
F: Documentation/devicetree/bindings/dma/owl-dma.txt
F: Documentation/devicetree/bindings/i2c/i2c-owl.txt
+F: Documentation/devicetree/bindings/interrupt-controller/actions,owl-sirq.yaml
F: Documentation/devicetree/bindings/mmc/owl-mmc.yaml
F: Documentation/devicetree/bindings/pinctrl/actions,s900-pinctrl.txt
F: Documentation/devicetree/bindings/power/actions,owl-sps.txt
F: drivers/clocksource/timer-owl*
F: drivers/dma/owl-dma.c
F: drivers/i2c/busses/i2c-owl.c
+F: drivers/irqchip/irq-owl-sirq.c
F: drivers/mmc/host/owl-mmc.c
F: drivers/pinctrl/actions/*
F: drivers/soc/actions/
ARM/Annapurna Labs ALPINE ARCHITECTURE
M: Tsahee Zidenberg <tsahee@annapurnalabs.com>
-M: Antoine Tenart <antoine.tenart@bootlin.com>
+M: Antoine Tenart <atenart@kernel.org>
L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
S: Maintained
F: arch/arm/boot/dts/alpine*
L: linux-media@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/media/s5p-cec.txt
-F: drivers/media/platform/s5p-cec/
+F: drivers/media/cec/platform/s5p/
ARM/SAMSUNG S5P SERIES JPEG CODEC SUPPORT
M: Andrzej Pietrasiewicz <andrzejtp2010@gmail.com>
F: drivers/clk/socfpga/
ARM/SOCFPGA EDAC SUPPORT
-M: Thor Thayer <thor.thayer@linux.intel.com>
+M: Dinh Nguyen <dinguyen@kernel.org>
S: Maintained
F: drivers/edac/altera_edac.
L: linux-media@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/media/tegra-cec.txt
-F: drivers/media/platform/tegra-cec/
+F: drivers/media/cec/platform/tegra/
ARM/TETON BGA MACHINE SUPPORT
M: "Mark F. Brown" <mark.brown314@gmail.com>
W: http://linuxtv.org
T: git git://linuxtv.org/media_tree.git
F: Documentation/devicetree/bindings/media/cec-gpio.txt
-F: drivers/media/platform/cec-gpio/
+F: drivers/media/cec/platform/cec-gpio/
CELL BROADBAND ENGINE ARCHITECTURE
M: Arnd Bergmann <arnd@arndb.de>
F: drivers/edac/bluefield_edac.c
EDAC-CALXEDA
-M: Robert Richter <rric@kernel.org>
+M: Andre Przywara <andre.przywara@arm.com>
L: linux-edac@vger.kernel.org
S: Maintained
F: drivers/edac/highbank*
GCC PLUGINS
M: Kees Cook <keescook@chromium.org>
R: Emese Revfy <re.emese@gmail.com>
-L: kernel-hardening@lists.openwall.com
+L: linux-hardening@vger.kernel.org
S: Maintained
F: Documentation/kbuild/gcc-plugins.rst
F: scripts/Makefile.gcc-plugins
K: \b(ABS|SYN)_MT_
INSIDE SECURE CRYPTO DRIVER
-M: Antoine Tenart <antoine.tenart@bootlin.com>
+M: Antoine Tenart <atenart@kernel.org>
L: linux-crypto@vger.kernel.org
S: Maintained
F: drivers/crypto/inside-secure/
F: include/uapi/drm/i915_drm.h
INTEL ETHERNET DRIVERS
-M: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
+M: Jesse Brandeburg <jesse.brandeburg@intel.com>
+M: Tony Nguyen <anthony.l.nguyen@intel.com>
L: intel-wired-lan@lists.osuosl.org (moderated for non-subscribers)
S: Supported
W: http://www.intel.com/support/feedback.htm
M: Yong Zhi <yong.zhi@intel.com>
M: Sakari Ailus <sakari.ailus@linux.intel.com>
M: Bingbu Cao <bingbu.cao@intel.com>
-R: Tian Shu Qiu <tian.shu.qiu@intel.com>
+R: Tianshu Qiu <tian.shu.qiu@intel.com>
L: linux-media@vger.kernel.org
S: Maintained
F: Documentation/userspace-api/media/v4l/pixfmt-srggb10-ipu3.rst
INTEL IPU3 CSI-2 IMGU DRIVER
M: Sakari Ailus <sakari.ailus@linux.intel.com>
R: Bingbu Cao <bingbu.cao@intel.com>
-R: Tian Shu Qiu <tian.shu.qiu@intel.com>
+R: Tianshu Qiu <tian.shu.qiu@intel.com>
L: linux-media@vger.kernel.org
S: Maintained
F: Documentation/admin-guide/media/ipu3.rst
LEAKING_ADDRESSES
M: Tobin C. Harding <me@tobin.cc>
M: Tycho Andersen <tycho@tycho.pizza>
-L: kernel-hardening@lists.openwall.com
+L: linux-hardening@vger.kernel.org
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/tobin/leaks.git
F: scripts/leaking_addresses.pl
F: drivers/ata/pata_arasan_cf.c
F: include/linux/pata_arasan_cf_data.h
-LIBATA PATA DRIVERS
-M: Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
-M: Jens Axboe <axboe@kernel.dk>
-L: linux-ide@vger.kernel.org
-S: Maintained
-T: git git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-block.git
-F: drivers/ata/ata_generic.c
-F: drivers/ata/pata_*.c
-
LIBATA PATA FARADAY FTIDE010 AND GEMINI SATA BRIDGE DRIVERS
M: Linus Walleij <linus.walleij@linaro.org>
L: linux-ide@vger.kernel.org
W: http://linux-meson.com/
T: git git://linuxtv.org/media_tree.git
F: Documentation/devicetree/bindings/media/amlogic,meson-gx-ao-cec.yaml
-F: drivers/media/platform/meson/ao-cec-g12a.c
-F: drivers/media/platform/meson/ao-cec.c
+F: drivers/media/cec/platform/meson/ao-cec-g12a.c
+F: drivers/media/cec/platform/meson/ao-cec.c
MESON NAND CONTROLLER DRIVER FOR AMLOGIC SOCS
M: Liang Yang <liang.yang@amlogic.com>
F: drivers/mtd/nand/raw/meson_*
MESON VIDEO DECODER DRIVER FOR AMLOGIC SOCS
-M: Maxime Jourdan <mjourdan@baylibre.com>
M: Neil Armstrong <narmstrong@baylibre.com>
L: linux-media@vger.kernel.org
L: linux-amlogic@lists.infradead.org
T: git git://linuxtv.org/anttip/media_tree.git
F: drivers/media/usb/msi2500/
+MSTAR INTERRUPT CONTROLLER DRIVER
+M: Mark-PK Tsai <mark-pk.tsai@mediatek.com>
+M: Daniel Palmer <daniel@thingy.jp>
+S: Maintained
+F: Documentation/devicetree/bindings/interrupt-controller/mstar,mst-intc.yaml
+F: drivers/irqchip/irq-mst-intc.c
+
MSYSTEMS DISKONCHIP G3 MTD DRIVER
M: Robert Jarzmik <robert.jarzmik@free.fr>
L: linux-mtd@lists.infradead.org
M: Andrew Lunn <andrew@lunn.ch>
M: Vivien Didelot <vivien.didelot@gmail.com>
M: Florian Fainelli <f.fainelli@gmail.com>
+M: Vladimir Oltean <olteanv@gmail.com>
S: Maintained
F: Documentation/devicetree/bindings/net/dsa/
F: drivers/net/dsa/
F: drivers/media/i2c/ov2685.c
OMNIVISION OV2740 SENSOR DRIVER
-M: Tianshu Qiu <tian.shu.qiua@intel.com>
+M: Tianshu Qiu <tian.shu.qiu@intel.com>
R: Shawn Tu <shawnx.tu@intel.com>
R: Bingbu Cao <bingbu.cao@intel.com>
L: linux-media@vger.kernel.org
F: drivers/media/i2c/ov5640.c
OMNIVISION OV5647 SENSOR DRIVER
-M: Luis Oliveira <lolivei@synopsys.com>
+M: Dave Stevenson <dave.stevenson@raspberrypi.com>
+M: Jacopo Mondi <jacopo@jmondi.org>
L: linux-media@vger.kernel.org
S: Maintained
T: git git://linuxtv.org/media_tree.git
+F: Documentation/devicetree/bindings/media/i2c/ov5647.yaml
F: drivers/media/i2c/ov5647.c
OMNIVISION OV5670 SENSOR DRIVER
PADATA PARALLEL EXECUTION MECHANISM
M: Steffen Klassert <steffen.klassert@secunet.com>
+M: Daniel Jordan <daniel.m.jordan@oracle.com>
L: linux-crypto@vger.kernel.org
+L: linux-kernel@vger.kernel.org
S: Maintained
F: Documentation/core-api/padata.rst
F: include/linux/padata.h
PCI DRIVER FOR AARDVARK (Marvell Armada 3700)
M: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
+M: Pali Rohár <pali@kernel.org>
L: linux-pci@vger.kernel.org
L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
S: Maintained
M: GR-Linux-NIC-Dev@marvell.com
L: netdev@vger.kernel.org
S: Supported
-F: Documentation/networking/device_drivers/ethernet/qlogic/LICENSE.qla3xxx
+F: Documentation/networking/device_drivers/qlogic/LICENSE.qla3xxx
F: drivers/net/ethernet/qlogic/qla3xxx.*
QLOGIC QLA4XXX iSCSI DRIVER
L: linux-media@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/media/i2c/imi,rdacm2x-gmsl.yaml
-F: drivers/media/i2c/rdacm20.c
F: drivers/media/i2c/max9271.c
F: drivers/media/i2c/max9271.h
+F: drivers/media/i2c/rdacm20.c
RDC R-321X SoC
M: Florian Fainelli <florian@openwrt.org>
ROCKCHIP ISP V1 DRIVER
M: Helen Koike <helen.koike@collabora.com>
+M: Dafna Hirschfeld <dafna.hirschfeld@collabora.com>
L: linux-media@vger.kernel.org
S: Maintained
+F: Documentation/admin-guide/media/rkisp1.rst
+F: Documentation/userspace-api/media/v4l/pixfmt-meta-rkisp1.rst
F: drivers/staging/media/rkisp1/
ROCKCHIP RASTER 2D GRAPHIC ACCELERATION UNIT DRIVER
R: Steven Rostedt <rostedt@goodmis.org> (SCHED_FIFO/SCHED_RR)
R: Ben Segall <bsegall@google.com> (CONFIG_CFS_BANDWIDTH)
R: Mel Gorman <mgorman@suse.de> (CONFIG_NUMA_BALANCING)
+R: Daniel Bristot de Oliveira <bristot@redhat.com> (SCHED_DEADLINE)
L: linux-kernel@vger.kernel.org
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git sched/core
SECO BOARDS CEC DRIVER
M: Ettore Chimenti <ek5.chimenti@gmail.com>
S: Maintained
-F: drivers/media/platform/seco-cec/seco-cec.c
-F: drivers/media/platform/seco-cec/seco-cec.h
+F: drivers/media/cec/platform/seco/seco-cec.c
+F: drivers/media/cec/platform/seco/seco-cec.h
SECURE COMPUTING
M: Kees Cook <keescook@chromium.org>
SOFTLOGIC 6x10 MPEG CODEC
M: Bluecherry Maintainers <maintainers@bluecherrydvr.com>
M: Anton Sviridenko <anton@corp.bluecherry.net>
-M: Andrey Utkin <andrey.utkin@corp.bluecherry.net>
M: Andrey Utkin <andrey_utkin@fastmail.com>
M: Ismael Luceno <ismael@iodev.co.uk>
L: linux-media@vger.kernel.org
L: linux-media@vger.kernel.org
S: Maintained
T: git git://linuxtv.org/media_tree.git
-F: Documentation/devicetree/bindings/media/i2c/imx274.txt
+F: Documentation/devicetree/bindings/media/i2c/sony,imx274.yaml
F: drivers/media/i2c/imx274.c
SONY IMX290 SENSOR DRIVER
M: Benjamin Gaignard <benjamin.gaignard@linaro.org>
S: Maintained
F: Documentation/devicetree/bindings/media/stih-cec.txt
-F: drivers/media/platform/sti/cec/
+F: drivers/media/cec/platform/sti/
STK1160 USB VIDEO CAPTURE DRIVER
M: Ezequiel Garcia <ezequiel@vanguardiasur.com.ar>
F: Documentation/devicetree/bindings/gpio/snps,dw-apb-gpio.yaml
F: drivers/gpio/gpio-dwapb.c
+SYNOPSYS DESIGNWARE APB SSI DRIVER
+M: Serge Semin <fancer.lancer@gmail.com>
+L: linux-spi@vger.kernel.org
+S: Supported
+F: Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml
+F: drivers/spi/spi-dw*
+
SYNOPSYS DESIGNWARE AXI DMAC DRIVER
M: Eugeniy Paltsev <Eugeniy.Paltsev@synopsys.com>
S: Maintained
S: Maintained
W: https://kernsec.org/wiki/index.php/Linux_Kernel_Integrity
Q: https://patchwork.kernel.org/project/linux-integrity/list/
-T: git git://git.infradead.org/users/jjs/linux-tpmdd.git
+T: git git://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git
F: drivers/char/tpm/
TRACING
W: http://www.linux-mtd.infradead.org/doc/ubifs.html
T: git git://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git next
T: git git://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git fixes
+F: Documentation/filesystems/ubifs-authentication.rst
F: Documentation/filesystems/ubifs.rst
F: fs/ubifs/
F: drivers/media/usb/uvc/
F: include/uapi/linux/uvcvideo.h
-USB VISION DRIVER
-M: Hans Verkuil <hverkuil@xs4all.nl>
-L: linux-media@vger.kernel.org
-S: Odd Fixes
-W: https://linuxtv.org
-T: git git://linuxtv.org/media_tree.git
-F: drivers/staging/media/usbvision/
-
USB WEBCAM GADGET
M: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
L: linux-usb@vger.kernel.org
F: include/linux/vga_switcheroo.h
VIA RHINE NETWORK DRIVER
-S: Orphan
+S: Maintained
+M: Kevin Brace <kevinbrace@bracecomputerlab.com>
F: drivers/net/ethernet/via/via-rhine.c
VIA SD/MMC CARD CONTROLLER DRIVER
F: drivers/media/platform/video-mux.c
VIDEOBUF2 FRAMEWORK
-M: Pawel Osciak <pawel@osciak.com>
+M: Tomasz Figa <tfiga@chromium.org>
M: Marek Szyprowski <m.szyprowski@samsung.com>
-M: Kyungmin Park <kyungmin.park@samsung.com>
-R: Tomasz Figa <tfiga@chromium.org>
L: linux-media@vger.kernel.org
S: Maintained
F: drivers/media/common/videobuf2/*
T: git git://linuxtv.org/media_tree.git
F: drivers/media/test-drivers/vivid/*
+VIDTV VIRTUAL DIGITAL TV DRIVER
+M: Daniel W. S. Almeida <dwlsalmeida@gmail.com>
+L: linux-media@vger.kernel.org
+S: Maintained
+W: https://linuxtv.org
+T: git git://linuxtv.org/media_tree.git
+F: drivers/media/test-drivers/vidtv/*
+
VLYNQ BUS
M: Florian Fainelli <f.fainelli@gmail.com>
L: openwrt-devel@lists.openwrt.org (subscribers-only)
F: arch/x86/mm/
X86 PLATFORM DRIVERS
-M: Darren Hart <dvhart@infradead.org>
-M: Andy Shevchenko <andy@infradead.org>
+M: Hans de Goede <hdegoede@redhat.com>
+M: Mark Gross <mgross@linux.intel.com>
L: platform-driver-x86@vger.kernel.org
-S: Odd Fixes
+S: Maintained
T: git git://git.infradead.org/linux-platform-drivers-x86.git
F: drivers/platform/olpc/
F: drivers/platform/x86/
F: Documentation/filesystems/zonefs.rst
F: fs/zonefs/
+ZR36067 VIDEO FOR LINUX DRIVER
+M: Corentin Labbe <clabbe@baylibre.com>
+L: mjpeg-users@lists.sourceforge.net
+L: linux-media@vger.kernel.org
+S: Maintained
+W: http://mjpeg.sourceforge.net/driver-zoran/
+Q: https://patchwork.linuxtv.org/project/linux-media/list/
+F: Documentation/driver-api/media/drivers/zoran.rst
+F: drivers/staging/media/zoran/
+
ZPOOL COMPRESSED PAGE STORAGE API
M: Dan Streetman <ddstreet@ieee.org>
L: linux-mm@kvack.org
VERSION = 5
PATCHLEVEL = 9
SUBLEVEL = 0
-EXTRAVERSION = -rc6
+EXTRAVERSION =
NAME = Kleptomaniac Octopus
# *DOCUMENTATION*
help
Boot time self-test of the branch patching code.
+config STATIC_CALL_SELFTEST
+ bool "Static call selftest"
+ depends on HAVE_STATIC_CALL
+ help
+ Boot time self-test of the call patching code.
+
config OPTPROBES
def_bool y
depends on KPROBES && HAVE_OPTPROBES
config ARCH_HAS_VDSO_DATA
bool
+config HAVE_STATIC_CALL
+ bool
+
+config HAVE_STATIC_CALL_INLINE
+ bool
+ depends on HAVE_STATIC_CALL
+
source "kernel/gcov/Kconfig"
source "scripts/gcc-plugins/Kconfig"
* better 64-bit) boundary
*/
#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER
-__wsum csum_and_copy_from_user(const void __user *src, void *dst, int len, __wsum sum, int *errp);
+#define _HAVE_ARCH_CSUM_AND_COPY
+__wsum csum_and_copy_from_user(const void __user *src, void *dst, int len);
-__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum);
+__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len);
/*
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
DISCARDS
}
#define insqh(x,y,z) \
__asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
-
-#define __get_user_u(x,ptr) \
+#define __get_word(insn,x,ptr) \
({ \
long __guu_err; \
__asm__ __volatile__( \
- "1: ldq_u %0,%2\n" \
+ "1: "#insn" %0,%2\n" \
"2:\n" \
EXC(1b,2b,%0,%1) \
: "=r"(x), "=r"(__guu_err) \
__guu_err; \
})
-#define __put_user_u(x,ptr) \
-({ \
- long __puu_err; \
- __asm__ __volatile__( \
- "1: stq_u %2,%1\n" \
- "2:\n" \
- EXC(1b,2b,$31,%0) \
- : "=r"(__puu_err) \
- : "m"(__m(addr)), "rJ"(x), "0"(0)); \
- __puu_err; \
-})
-
-
static inline unsigned short from64to16(unsigned long x)
{
/* Using extract instructions is a bit more efficient
*/
static inline unsigned long
csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
- long len, unsigned long checksum,
- int *errp)
+ long len)
{
+ unsigned long checksum = ~0U;
unsigned long carry = 0;
- int err = 0;
while (len >= 0) {
unsigned long word;
- err |= __get_user(word, src);
+ if (__get_word(ldq, word, src))
+ return 0;
checksum += carry;
src++;
checksum += word;
checksum += carry;
if (len) {
unsigned long word, tmp;
- err |= __get_user(word, src);
+ if (__get_word(ldq, word, src))
+ return 0;
tmp = *dst;
mskql(word, len, word);
checksum += word;
*dst = word | tmp;
checksum += carry;
}
- if (err && errp) *errp = err;
return checksum;
}
csum_partial_cfu_dest_aligned(const unsigned long __user *src,
unsigned long *dst,
unsigned long soff,
- long len, unsigned long checksum,
- int *errp)
+ long len)
{
unsigned long first;
unsigned long word, carry;
unsigned long lastsrc = 7+len+(unsigned long)src;
- int err = 0;
+ unsigned long checksum = ~0U;
- err |= __get_user_u(first,src);
+ if (__get_word(ldq_u, first,src))
+ return 0;
carry = 0;
while (len >= 0) {
unsigned long second;
- err |= __get_user_u(second, src+1);
+ if (__get_word(ldq_u, second, src+1))
+ return 0;
extql(first, soff, word);
len -= 8;
src++;
if (len) {
unsigned long tmp;
unsigned long second;
- err |= __get_user_u(second, lastsrc);
+ if (__get_word(ldq_u, second, lastsrc))
+ return 0;
tmp = *dst;
extql(first, soff, word);
extqh(second, soff, first);
*dst = word | tmp;
checksum += carry;
}
- if (err && errp) *errp = err;
return checksum;
}
csum_partial_cfu_src_aligned(const unsigned long __user *src,
unsigned long *dst,
unsigned long doff,
- long len, unsigned long checksum,
- unsigned long partial_dest,
- int *errp)
+ long len,
+ unsigned long partial_dest)
{
unsigned long carry = 0;
unsigned long word;
unsigned long second_dest;
- int err = 0;
+ unsigned long checksum = ~0U;
mskql(partial_dest, doff, partial_dest);
while (len >= 0) {
- err |= __get_user(word, src);
+ if (__get_word(ldq, word, src))
+ return 0;
len -= 8;
insql(word, doff, second_dest);
checksum += carry;
len += 8;
if (len) {
checksum += carry;
- err |= __get_user(word, src);
+ if (__get_word(ldq, word, src))
+ return 0;
mskql(word, len, word);
len -= 8;
checksum += word;
stq_u(partial_dest | second_dest, dst);
out:
checksum += carry;
- if (err && errp) *errp = err;
return checksum;
}
csum_partial_cfu_unaligned(const unsigned long __user * src,
unsigned long * dst,
unsigned long soff, unsigned long doff,
- long len, unsigned long checksum,
- unsigned long partial_dest,
- int *errp)
+ long len, unsigned long partial_dest)
{
unsigned long carry = 0;
unsigned long first;
unsigned long lastsrc;
- int err = 0;
+ unsigned long checksum = ~0U;
- err |= __get_user_u(first, src);
+ if (__get_word(ldq_u, first, src))
+ return 0;
lastsrc = 7+len+(unsigned long)src;
mskql(partial_dest, doff, partial_dest);
while (len >= 0) {
unsigned long second, word;
unsigned long second_dest;
- err |= __get_user_u(second, src+1);
+ if (__get_word(ldq_u, second, src+1))
+ return 0;
extql(first, soff, word);
checksum += carry;
len -= 8;
unsigned long second, word;
unsigned long second_dest;
- err |= __get_user_u(second, lastsrc);
+ if (__get_word(ldq_u, second, lastsrc))
+ return 0;
extql(first, soff, word);
extqh(second, soff, first);
word |= first;
unsigned long second, word;
unsigned long second_dest;
- err |= __get_user_u(second, lastsrc);
+ if (__get_word(ldq_u, second, lastsrc))
+ return 0;
extql(first, soff, word);
extqh(second, soff, first);
word |= first;
stq_u(partial_dest | word | second_dest, dst);
checksum += carry;
}
- if (err && errp) *errp = err;
return checksum;
}
-__wsum
-csum_and_copy_from_user(const void __user *src, void *dst, int len,
- __wsum sum, int *errp)
+static __wsum __csum_and_copy(const void __user *src, void *dst, int len)
{
- unsigned long checksum = (__force u32) sum;
unsigned long soff = 7 & (unsigned long) src;
unsigned long doff = 7 & (unsigned long) dst;
-
- if (len) {
- if (!access_ok(src, len)) {
- if (errp) *errp = -EFAULT;
- memset(dst, 0, len);
- return sum;
- }
- if (!doff) {
- if (!soff)
- checksum = csum_partial_cfu_aligned(
- (const unsigned long __user *) src,
- (unsigned long *) dst,
- len-8, checksum, errp);
- else
- checksum = csum_partial_cfu_dest_aligned(
- (const unsigned long __user *) src,
- (unsigned long *) dst,
- soff, len-8, checksum, errp);
- } else {
- unsigned long partial_dest;
- ldq_u(partial_dest, dst);
- if (!soff)
- checksum = csum_partial_cfu_src_aligned(
- (const unsigned long __user *) src,
- (unsigned long *) dst,
- doff, len-8, checksum,
- partial_dest, errp);
- else
- checksum = csum_partial_cfu_unaligned(
- (const unsigned long __user *) src,
- (unsigned long *) dst,
- soff, doff, len-8, checksum,
- partial_dest, errp);
- }
- checksum = from64to16 (checksum);
+ unsigned long checksum;
+
+ if (!doff) {
+ if (!soff)
+ checksum = csum_partial_cfu_aligned(
+ (const unsigned long __user *) src,
+ (unsigned long *) dst, len-8);
+ else
+ checksum = csum_partial_cfu_dest_aligned(
+ (const unsigned long __user *) src,
+ (unsigned long *) dst,
+ soff, len-8);
+ } else {
+ unsigned long partial_dest;
+ ldq_u(partial_dest, dst);
+ if (!soff)
+ checksum = csum_partial_cfu_src_aligned(
+ (const unsigned long __user *) src,
+ (unsigned long *) dst,
+ doff, len-8, partial_dest);
+ else
+ checksum = csum_partial_cfu_unaligned(
+ (const unsigned long __user *) src,
+ (unsigned long *) dst,
+ soff, doff, len-8, partial_dest);
}
- return (__force __wsum)checksum;
+ return (__force __wsum)from64to16 (checksum);
+}
+
+__wsum
+csum_and_copy_from_user(const void __user *src, void *dst, int len)
+{
+ if (!access_ok(src, len))
+ return 0;
+ return __csum_and_copy(src, dst, len);
}
EXPORT_SYMBOL(csum_and_copy_from_user);
__wsum
-csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
+csum_partial_copy_nocheck(const void *src, void *dst, int len)
{
- __wsum checksum;
- mm_segment_t oldfs = get_fs();
- set_fs(KERNEL_DS);
- checksum = csum_and_copy_from_user((__force const void __user *)src,
- dst, len, sum, NULL);
- set_fs(oldfs);
- return checksum;
+ return __csum_and_copy((__force const void __user *)src,
+ dst, len);
}
EXPORT_SYMBOL(csum_partial_copy_nocheck);
{
ri->ret_addr = (kprobe_opcode_t *) regs->blink;
+ ri->fp = NULL;
/* Replace the return addr with trampoline addr */
regs->blink = (unsigned long)&kretprobe_trampoline;
static int __kprobes trampoline_probe_handler(struct kprobe *p,
struct pt_regs *regs)
{
- struct kretprobe_instance *ri = NULL;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address = 0;
- unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
-
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
-
- /*
- * It is possible to have multiple instances associated with a given
- * task either because an multiple functions in the call path
- * have a return probe installed on them, and/or more than one return
- * return probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always inserted at the head of the list
- * - when multiple return probes are registered for the same
- * function, the first instance's ret_addr will point to the
- * real return address, and all the rest will point to
- * kretprobe_trampoline
- */
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- if (ri->rp && ri->rp->handler)
- ri->rp->handler(ri, regs);
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address) {
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
- }
-
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
- regs->ret = orig_ret_address;
-
- kretprobe_hash_unlock(current, &flags);
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
+ regs->ret = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL);
/* By returning a non zero value, we are telling the kprobe handler
* that we don't want the post_handler to run
_end = . ;
STABS_DEBUG
+ ELF_DETAILS
DISCARDS
.arcextmap 0 : {
select GENERIC_ARCH_TOPOLOGY if ARM_CPU_TOPOLOGY
select GENERIC_ATOMIC64 if CPU_V7M || CPU_V6 || !CPU_32v6K || !AEABI
select GENERIC_CLOCKEVENTS_BROADCAST if SMP
+ select GENERIC_IRQ_IPI if SMP
select GENERIC_CPU_AUTOPROBE
select GENERIC_EARLY_IOREMAP
select GENERIC_IDLE_POLL_SETUP
KBUILD_LDFLAGS_MODULE += --be8
endif
+# We never want expected sections to be placed heuristically by the
+# linker. All sections should be explicitly named in the linker script.
+LDFLAGS_vmlinux += $(call ld-option, --orphan-handling=warn)
+
ifeq ($(CONFIG_ARM_MODULE_PLTS),y)
KBUILD_LDS_MODULE += $(srctree)/arch/arm/kernel/module.lds
endif
LDFLAGS_vmlinux += --no-undefined
# Delete all temporary local symbols
LDFLAGS_vmlinux += -X
+# Report orphan sections
+LDFLAGS_vmlinux += $(call ld-option, --orphan-handling=warn)
# Next argument is a linker script
LDFLAGS_vmlinux += -T
/*
* Copyright (C) 2000 Russell King
*/
+#include <asm/vmlinux.lds.h>
#ifdef CONFIG_CPU_ENDIAN_BE8
#define ZIMAGE_MAGIC(x) ( (((x) >> 24) & 0x000000ff) | \
SECTIONS
{
/DISCARD/ : {
+ COMMON_DISCARDS
*(.ARM.exidx*)
*(.ARM.extab*)
+ *(.note.*)
+ *(.rel.*)
/*
* Discard any r/w data - this produces a link error if we have any,
* which is required for PIC decompression. Local data generates
*(.start)
*(.text)
*(.text.*)
- *(.gnu.warning)
- *(.glue_7t)
- *(.glue_7)
+ ARM_STUBS_TEXT
}
.table : ALIGN(4) {
_table_start = .;
PROVIDE(__pecoff_data_size = ALIGN(512) - ADDR(.data));
PROVIDE(__pecoff_end = ALIGN(512));
- .stab 0 : { *(.stab) }
- .stabstr 0 : { *(.stabstr) }
- .stab.excl 0 : { *(.stab.excl) }
- .stab.exclstr 0 : { *(.stab.exclstr) }
- .stab.index 0 : { *(.stab.index) }
- .stab.indexstr 0 : { *(.stab.indexstr) }
- .comment 0 : { *(.comment) }
+ STABS_DEBUG
+ DWARF_DEBUG
+ ARM_DETAILS
+
+ ARM_ASSERTS
}
ASSERT(_edata_real == _edata, "error: zImage file size is incorrect");
soc {
firmware: firmware {
- compatible = "raspberrypi,bcm2835-firmware", "simple-bus";
+ compatible = "raspberrypi,bcm2835-firmware", "simple-mfd";
#address-cells = <1>;
#size-cells = <1>;
vldr \out\()h, \sym + 8
.endm
- .macro __adr, reg, lbl
- adr \reg, \lbl
-THUMB( orr \reg, \reg, #1 )
- .endm
-
.macro in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
veor \b2, \b2, \b1
veor \b5, \b5, \b6
push {r4-r6, lr}
ldr r5, [sp, #16] // number of blocks
-99: __adr ip, 0f
+99: adr ip, 0f
and lr, r5, #7
cmp r5, #8
sub ip, ip, lr, lsl #2
- bxlt ip // computed goto if blocks < 8
+ movlt pc, ip // computed goto if blocks < 8
vld1.8 {q0}, [r1]!
vld1.8 {q1}, [r1]!
mov rounds, r3
bl \do8
- __adr ip, 1f
+ adr ip, 1f
and lr, r5, #7
cmp r5, #8
sub ip, ip, lr, lsl #2
- bxlt ip // computed goto if blocks < 8
+ movlt pc, ip // computed goto if blocks < 8
vst1.8 {\o0}, [r0]!
vst1.8 {\o1}, [r0]!
push {r4-r6, lr}
ldm ip, {r5-r6} // load args 4-5
-99: __adr ip, 0f
+99: adr ip, 0f
and lr, r5, #7
cmp r5, #8
sub ip, ip, lr, lsl #2
mov lr, r1
- bxlt ip // computed goto if blocks < 8
+ movlt pc, ip // computed goto if blocks < 8
vld1.8 {q0}, [lr]!
vld1.8 {q1}, [lr]!
vmov q14, q8
vmov q15, q8
- __adr ip, 1f
+ adr ip, 1f
and lr, r5, #7
cmp r5, #8
sub ip, ip, lr, lsl #2
- bxlt ip // computed goto if blocks < 8
+ movlt pc, ip // computed goto if blocks < 8
vld1.8 {q9}, [r1]!
vld1.8 {q10}, [r1]!
vld1.8 {q15}, [r1]!
W(nop)
-1: __adr ip, 2f
+1: adr ip, 2f
sub ip, ip, lr, lsl #3
- bxlt ip // computed goto if blocks < 8
+ movlt pc, ip // computed goto if blocks < 8
veor q0, q0, q8
vst1.8 {q0}, [r0]!
vmov q6, q0
vmov q7, q0
- __adr ip, 0f
+ adr ip, 0f
sub lr, r5, #1
and lr, lr, #7
cmp r5, #8
sub ip, ip, lr, lsl #5
sub ip, ip, lr, lsl #2
- bxlt ip // computed goto if blocks < 8
+ movlt pc, ip // computed goto if blocks < 8
next_ctr q1
next_ctr q2
mov rounds, r3
bl aesbs_encrypt8
- __adr ip, 1f
+ adr ip, 1f
and lr, r5, #7
cmp r5, #8
movgt r4, #0
ldrle r4, [sp, #40] // load final in the last round
sub ip, ip, lr, lsl #2
- bxlt ip // computed goto if blocks < 8
+ movlt pc, ip // computed goto if blocks < 8
vld1.8 {q8}, [r1]!
vld1.8 {q9}, [r1]!
1: bne 2f
vld1.8 {q15}, [r1]!
-2: __adr ip, 3f
+2: adr ip, 3f
cmp r5, #8
sub ip, ip, lr, lsl #3
- bxlt ip // computed goto if blocks < 8
+ movlt pc, ip // computed goto if blocks < 8
veor q0, q0, q8
vst1.8 {q0}, [r0]!
vshr.u64 d30, d31, #7
vmov q12, q14
- __adr ip, 0f
+ adr ip, 0f
and r4, r6, #7
cmp r6, #8
sub ip, ip, r4, lsl #5
mov r4, sp
- bxlt ip // computed goto if blocks < 8
+ movlt pc, ip // computed goto if blocks < 8
vld1.8 {q0}, [r1]!
next_tweak q12, q14, q15, q13
push {r4-r8, lr}
mov r5, sp // preserve sp
ldrd r6, r7, [sp, #24] // get blocks and iv args
- ldr r8, [sp, #32] // reorder final tweak?
- rsb r8, r8, #1
+ rsb r8, ip, #1
sub ip, sp, #128 // make room for 8x tweak
bic ip, ip, #0xf // align sp to 16 bytes
mov sp, ip
mov rounds, r3
bl \do8
- __adr ip, 0f
+ adr ip, 0f
and lr, r6, #7
cmp r6, #8
sub ip, ip, lr, lsl #2
mov r4, sp
- bxlt ip // computed goto if blocks < 8
+ movlt pc, ip // computed goto if blocks < 8
vld1.8 {q8}, [r4, :128]!
vld1.8 {q9}, [r4, :128]!
vld1.8 {q14}, [r4, :128]!
vld1.8 {q15}, [r4, :128]
-0: __adr ip, 1f
+0: adr ip, 1f
sub ip, ip, lr, lsl #3
- bxlt ip // computed goto if blocks < 8
+ movlt pc, ip // computed goto if blocks < 8
veor \o0, \o0, q8
vst1.8 {\o0}, [r0]!
.endm
ENTRY(aesbs_xts_encrypt)
+ mov ip, #0 // never reorder final tweak
__xts_crypt aesbs_encrypt8, q0, q1, q4, q6, q3, q7, q2, q5
ENDPROC(aesbs_xts_encrypt)
ENTRY(aesbs_xts_decrypt)
+ ldr ip, [sp, #8] // reorder final tweak?
__xts_crypt aesbs_decrypt8, q0, q1, q6, q4, q2, q7, q3, q5
ENDPROC(aesbs_xts_decrypt)
#include <asm/neon.h>
#include <asm/simd.h>
#include <crypto/aes.h>
-#include <crypto/cbc.h>
#include <crypto/ctr.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
struct aesbs_cbc_ctx {
struct aesbs_ctx key;
- struct crypto_cipher *enc_tfm;
+ struct crypto_skcipher *enc_tfm;
};
struct aesbs_xts_ctx {
kernel_neon_end();
memzero_explicit(&rk, sizeof(rk));
- return crypto_cipher_setkey(ctx->enc_tfm, in_key, key_len);
+ return crypto_skcipher_setkey(ctx->enc_tfm, in_key, key_len);
}
-static void cbc_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst)
+static int cbc_encrypt(struct skcipher_request *req)
{
+ struct skcipher_request *subreq = skcipher_request_ctx(req);
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
- crypto_cipher_encrypt_one(ctx->enc_tfm, dst, src);
-}
+ skcipher_request_set_tfm(subreq, ctx->enc_tfm);
+ skcipher_request_set_callback(subreq,
+ skcipher_request_flags(req),
+ NULL, NULL);
+ skcipher_request_set_crypt(subreq, req->src, req->dst,
+ req->cryptlen, req->iv);
-static int cbc_encrypt(struct skcipher_request *req)
-{
- return crypto_cbc_encrypt_walk(req, cbc_encrypt_one);
+ return crypto_skcipher_encrypt(subreq);
}
static int cbc_decrypt(struct skcipher_request *req)
return err;
}
-static int cbc_init(struct crypto_tfm *tfm)
+static int cbc_init(struct crypto_skcipher *tfm)
{
- struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+ unsigned int reqsize;
- ctx->enc_tfm = crypto_alloc_cipher("aes", 0, 0);
+ ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC);
+ if (IS_ERR(ctx->enc_tfm))
+ return PTR_ERR(ctx->enc_tfm);
- return PTR_ERR_OR_ZERO(ctx->enc_tfm);
+ reqsize = sizeof(struct skcipher_request);
+ reqsize += crypto_skcipher_reqsize(ctx->enc_tfm);
+ crypto_skcipher_set_reqsize(tfm, reqsize);
+
+ return 0;
}
-static void cbc_exit(struct crypto_tfm *tfm)
+static void cbc_exit(struct crypto_skcipher *tfm)
{
- struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
- crypto_free_cipher(ctx->enc_tfm);
+ crypto_free_skcipher(ctx->enc_tfm);
}
static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key,
return aesbs_setkey(tfm, in_key, key_len);
}
-static int xts_init(struct crypto_tfm *tfm)
+static int xts_init(struct crypto_skcipher *tfm)
{
- struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
ctx->cts_tfm = crypto_alloc_cipher("aes", 0, 0);
if (IS_ERR(ctx->cts_tfm))
return PTR_ERR_OR_ZERO(ctx->tweak_tfm);
}
-static void xts_exit(struct crypto_tfm *tfm)
+static void xts_exit(struct crypto_skcipher *tfm)
{
- struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
crypto_free_cipher(ctx->tweak_tfm);
crypto_free_cipher(ctx->cts_tfm);
.base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx),
.base.cra_module = THIS_MODULE,
.base.cra_flags = CRYPTO_ALG_INTERNAL,
- .base.cra_init = cbc_init,
- .base.cra_exit = cbc_exit,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = aesbs_cbc_setkey,
.encrypt = cbc_encrypt,
.decrypt = cbc_decrypt,
+ .init = cbc_init,
+ .exit = cbc_exit,
}, {
.base.cra_name = "__ctr(aes)",
.base.cra_driver_name = "__ctr-aes-neonbs",
.base.cra_ctxsize = sizeof(struct aesbs_xts_ctx),
.base.cra_module = THIS_MODULE,
.base.cra_flags = CRYPTO_ALG_INTERNAL,
- .base.cra_init = xts_init,
- .base.cra_exit = xts_exit,
.min_keysize = 2 * AES_MIN_KEY_SIZE,
.max_keysize = 2 * AES_MAX_KEY_SIZE,
.setkey = aesbs_xts_setkey,
.encrypt = xts_encrypt,
.decrypt = xts_decrypt,
+ .init = xts_init,
+ .exit = xts_exit,
} };
static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
#include <linux/module.h>
#include <linux/init.h>
#include <linux/jump_label.h>
+#include <linux/scatterlist.h>
#include <crypto/curve25519.h>
asmlinkage void curve25519_neon(u8 mypublic[CURVE25519_KEY_SIZE],
void poly1305_init_arm(void *state, const u8 *key);
void poly1305_blocks_arm(void *state, const u8 *src, u32 len, u32 hibit);
+void poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit);
void poly1305_emit_arm(void *state, u8 *digest, const u32 *nonce);
void __weak poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit)
#else
.syntax unified
# ifdef __thumb2__
-# define adrl adr
.thumb
# else
.code 32
stmdb sp!,{r4-r12,lr}
sub $H,sp,#16*4+16
- adrl $Ktbl,K256
+ adr $Ktbl,.Lsha256_block_data_order
+ sub $Ktbl,$Ktbl,#.Lsha256_block_data_order-K256
bic $H,$H,#15 @ align for 128-bit stores
mov $t2,sp
mov sp,$H @ alloca
#else
.syntax unified
# ifdef __thumb2__
-# define adrl adr
.thumb
# else
.code 32
stmdb sp!,{r4-r12,lr}
sub r11,sp,#16*4+16
- adrl r14,K256
+ adr r14,.Lsha256_block_data_order
+ sub r14,r14,#.Lsha256_block_data_order-K256
bic r11,r11,#15 @ align for 128-bit stores
mov r12,sp
mov sp,r11 @ alloca
#else
.syntax unified
# ifdef __thumb2__
-# define adrl adr
.thumb
# else
.code 32
dmb @ errata #451034 on early Cortex A8
add $len,$inp,$len,lsl#7 @ len to point at the end of inp
VFP_ABI_PUSH
- adrl $Ktbl,K512
+ adr $Ktbl,.Lsha512_block_data_order
+ sub $Ktbl,$Ktbl,.Lsha512_block_data_order-K512
vldmia $ctx,{$A-$H} @ load context
.Loop_neon:
___
#else
.syntax unified
# ifdef __thumb2__
-# define adrl adr
.thumb
# else
.code 32
dmb @ errata #451034 on early Cortex A8
add r2,r1,r2,lsl#7 @ len to point at the end of inp
VFP_ABI_PUSH
- adrl r3,K512
+ adr r3,.Lsha512_block_data_order
+ sub r3,r3,.Lsha512_block_data_order-K512
vldmia r0,{d16-d23} @ load context
.Loop_neon:
vshr.u64 d24,d20,#14 @ 0
*/
__wsum
-csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum);
+csum_partial_copy_nocheck(const void *src, void *dst, int len);
__wsum
-csum_partial_copy_from_user(const void __user *src, void *dst, int len, __wsum sum, int *err_ptr);
+csum_partial_copy_from_user(const void __user *src, void *dst, int len);
#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER
+#define _HAVE_ARCH_CSUM_AND_COPY
static inline
-__wsum csum_and_copy_from_user (const void __user *src, void *dst,
- int len, __wsum sum, int *err_ptr)
+__wsum csum_and_copy_from_user(const void __user *src, void *dst, int len)
{
- if (access_ok(src, len))
- return csum_partial_copy_from_user(src, dst, len, sum, err_ptr);
+ if (!access_ok(src, len))
+ return 0;
- if (len)
- *err_ptr = -EFAULT;
-
- return sum;
+ return csum_partial_copy_from_user(src, dst, len);
}
/*
#define MAX_UNCOMP_KERNEL_SIZE SZ_32M
/*
- * The kernel zImage should preferably be located between 32 MB and 128 MB
- * from the base of DRAM. The min address leaves space for a maximal size
- * uncompressed image, and the max address is due to how the zImage decompressor
- * picks a destination address.
+ * phys-to-virt patching requires that the physical to virtual offset fits
+ * into the immediate field of an add/sub instruction, which comes down to the
+ * 24 least significant bits being zero, and so the offset should be a multiple
+ * of 16 MB. Since PAGE_OFFSET itself is a multiple of 16 MB, the physical
+ * base should be aligned to 16 MB as well.
*/
-#define ZIMAGE_OFFSET_LIMIT SZ_128M
-#define MIN_ZIMAGE_OFFSET MAX_UNCOMP_KERNEL_SIZE
+#define EFI_PHYS_ALIGN SZ_16M
-/* on ARM, the FDT should be located in the first 128 MB of RAM */
-static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base)
+/* on ARM, the FDT should be located in a lowmem region */
+static inline unsigned long efi_get_max_fdt_addr(unsigned long image_addr)
{
- return dram_base + ZIMAGE_OFFSET_LIMIT;
+ return round_down(image_addr, EFI_PHYS_ALIGN) + SZ_512M;
}
/* on ARM, the initrd should be loaded in a lowmem region */
-static inline unsigned long efi_get_max_initrd_addr(unsigned long dram_base,
- unsigned long image_addr)
+static inline unsigned long efi_get_max_initrd_addr(unsigned long image_addr)
{
- return dram_base + SZ_512M;
+ return round_down(image_addr, EFI_PHYS_ALIGN) + SZ_512M;
}
struct efi_arm_entry_state {
#include <linux/threads.h>
#include <asm/irq.h>
-/* number of IPIS _not_ including IPI_CPU_BACKTRACE */
-#define NR_IPI 7
-
typedef struct {
unsigned int __softirq_pending;
-#ifdef CONFIG_SMP
- unsigned int ipi_irqs[NR_IPI];
-#endif
} ____cacheline_aligned irq_cpustat_t;
#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */
-#define __inc_irq_stat(cpu, member) __IRQ_STAT(cpu, member)++
-#define __get_irq_stat(cpu, member) __IRQ_STAT(cpu, member)
-
-#ifdef CONFIG_SMP
-u64 smp_irq_stat_cpu(unsigned int cpu);
-#else
-#define smp_irq_stat_cpu(cpu) 0
-#endif
-
-#define arch_irq_stat_cpu smp_irq_stat_cpu
-
#define __ARCH_IRQ_EXIT_IRQS_DISABLED 1
#endif /* __ASM_HARDIRQ_H */
*/
extern void smp_init_cpus(void);
-
/*
- * Provide a function to raise an IPI cross call on CPUs in callmap.
+ * Register IPI interrupts with the arch SMP code
*/
-extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
+extern void set_smp_ipi_range(int ipi_base, int nr_ipi);
/*
* Called from platform specific assembly code, this is the
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <asm-generic/vmlinux.lds.h>
+
+#ifdef CONFIG_HOTPLUG_CPU
+#define ARM_CPU_DISCARD(x)
+#define ARM_CPU_KEEP(x) x
+#else
+#define ARM_CPU_DISCARD(x) x
+#define ARM_CPU_KEEP(x)
+#endif
+
+#if (defined(CONFIG_SMP_ON_UP) && !defined(CONFIG_DEBUG_SPINLOCK)) || \
+ defined(CONFIG_GENERIC_BUG) || defined(CONFIG_JUMP_LABEL)
+#define ARM_EXIT_KEEP(x) x
+#define ARM_EXIT_DISCARD(x)
+#else
+#define ARM_EXIT_KEEP(x)
+#define ARM_EXIT_DISCARD(x) x
+#endif
+
+#ifdef CONFIG_MMU
+#define ARM_MMU_KEEP(x) x
+#define ARM_MMU_DISCARD(x)
+#else
+#define ARM_MMU_KEEP(x)
+#define ARM_MMU_DISCARD(x) x
+#endif
+
+#define PROC_INFO \
+ . = ALIGN(4); \
+ __proc_info_begin = .; \
+ *(.proc.info.init) \
+ __proc_info_end = .;
+
+#define IDMAP_TEXT \
+ ALIGN_FUNCTION(); \
+ __idmap_text_start = .; \
+ *(.idmap.text) \
+ __idmap_text_end = .; \
+
+#define ARM_DISCARD \
+ *(.ARM.exidx.exit.text) \
+ *(.ARM.extab.exit.text) \
+ *(.ARM.exidx.text.exit) \
+ *(.ARM.extab.text.exit) \
+ ARM_CPU_DISCARD(*(.ARM.exidx.cpuexit.text)) \
+ ARM_CPU_DISCARD(*(.ARM.extab.cpuexit.text)) \
+ ARM_EXIT_DISCARD(EXIT_TEXT) \
+ ARM_EXIT_DISCARD(EXIT_DATA) \
+ EXIT_CALL \
+ ARM_MMU_DISCARD(*(.text.fixup)) \
+ ARM_MMU_DISCARD(*(__ex_table)) \
+ COMMON_DISCARDS
+
+/*
+ * Sections that should stay zero sized, which is safer to explicitly
+ * check instead of blindly discarding.
+ */
+#define ARM_ASSERTS \
+ .plt : { \
+ *(.iplt) *(.rel.iplt) *(.iplt) *(.igot.plt) \
+ } \
+ ASSERT(SIZEOF(.plt) == 0, \
+ "Unexpected run-time procedure linkages detected!")
+
+#define ARM_DETAILS \
+ ELF_DETAILS \
+ .ARM.attributes 0 : { *(.ARM.attributes) }
+
+#define ARM_STUBS_TEXT \
+ *(.gnu.warning) \
+ *(.glue_7) \
+ *(.glue_7t) \
+ *(.vfp11_veneer) \
+ *(.v4_bx)
+
+#define ARM_TEXT \
+ IDMAP_TEXT \
+ __entry_text_start = .; \
+ *(.entry.text) \
+ __entry_text_end = .; \
+ IRQENTRY_TEXT \
+ SOFTIRQENTRY_TEXT \
+ TEXT_TEXT \
+ SCHED_TEXT \
+ CPUIDLE_TEXT \
+ LOCK_TEXT \
+ KPROBES_TEXT \
+ ARM_STUBS_TEXT \
+ . = ALIGN(4); \
+ *(.got) /* Global offset table */ \
+ ARM_CPU_KEEP(PROC_INFO)
+
+/* Stack unwinding tables */
+#define ARM_UNWIND_SECTIONS \
+ . = ALIGN(8); \
+ .ARM.unwind_idx : { \
+ __start_unwind_idx = .; \
+ *(.ARM.exidx*) \
+ __stop_unwind_idx = .; \
+ } \
+ .ARM.unwind_tab : { \
+ __start_unwind_tab = .; \
+ *(.ARM.extab*) \
+ __stop_unwind_tab = .; \
+ }
+
+/*
+ * The vectors and stubs are relocatable code, and the
+ * only thing that matters is their relative offsets
+ */
+#define ARM_VECTORS \
+ __vectors_start = .; \
+ .vectors 0xffff0000 : AT(__vectors_start) { \
+ *(.vectors) \
+ } \
+ . = __vectors_start + SIZEOF(.vectors); \
+ __vectors_end = .; \
+ \
+ __stubs_start = .; \
+ .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_start) { \
+ *(.stubs) \
+ } \
+ . = __stubs_start + SIZEOF(.stubs); \
+ __stubs_end = .; \
+ \
+ PROVIDE(vector_fiq_offset = vector_fiq - ADDR(.vectors));
+
+#define ARM_TCM \
+ __itcm_start = ALIGN(4); \
+ .text_itcm ITCM_OFFSET : AT(__itcm_start - LOAD_OFFSET) { \
+ __sitcm_text = .; \
+ *(.tcm.text) \
+ *(.tcm.rodata) \
+ . = ALIGN(4); \
+ __eitcm_text = .; \
+ } \
+ . = __itcm_start + SIZEOF(.text_itcm); \
+ \
+ __dtcm_start = .; \
+ .data_dtcm DTCM_OFFSET : AT(__dtcm_start - LOAD_OFFSET) { \
+ __sdtcm_data = .; \
+ *(.tcm.data) \
+ . = ALIGN(4); \
+ __edtcm_data = .; \
+ } \
+ . = __dtcm_start + SIZEOF(.data_dtcm);
* IRQ's are in fact implemented a bit like signal handlers for the kernel.
* Naturally it's not a 1:1 relation, but there are similarities.
*/
-#include <linux/kernel_stat.h>
#include <linux/signal.h>
#include <linux/ioport.h>
#include <linux/interrupt.h>
#include <linux/completion.h>
#include <linux/cpufreq.h>
#include <linux/irq_work.h>
+#include <linux/kernel_stat.h>
#include <linux/atomic.h>
#include <asm/bugs.h>
IPI_CPU_STOP,
IPI_IRQ_WORK,
IPI_COMPLETION,
+ NR_IPI,
/*
* CPU_BACKTRACE is special and not included in NR_IPI
* or tracable with trace_ipi_*
*/
- IPI_CPU_BACKTRACE,
+ IPI_CPU_BACKTRACE = NR_IPI,
/*
* SGI8-15 can be reserved by secure firmware, and thus may
* not be usable by the kernel. Please keep the above limited
* to at most 8 entries.
*/
+ MAX_IPI
};
+static int ipi_irq_base __read_mostly;
+static int nr_ipi __read_mostly = NR_IPI;
+static struct irq_desc *ipi_desc[MAX_IPI] __read_mostly;
+
+static void ipi_setup(int cpu);
+
static DECLARE_COMPLETION(cpu_running);
static struct smp_operations smp_ops __ro_after_init;
return cpu != 0;
}
+static void ipi_teardown(int cpu)
+{
+ int i;
+
+ if (WARN_ON_ONCE(!ipi_irq_base))
+ return;
+
+ for (i = 0; i < nr_ipi; i++)
+ disable_percpu_irq(ipi_irq_base + i);
+}
+
/*
* __cpu_disable runs on the processor to be shutdown.
*/
* and we must not schedule until we're ready to give up the cpu.
*/
set_cpu_online(cpu, false);
+ ipi_teardown(cpu);
/*
* OK - migrate IRQs away from this CPU
notify_cpu_starting(cpu);
+ ipi_setup(cpu);
+
calibrate_delay();
smp_store_cpu_info(cpu);
}
}
-static void (*__smp_cross_call)(const struct cpumask *, unsigned int);
-
-void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
-{
- if (!__smp_cross_call)
- __smp_cross_call = fn;
-}
-
static const char *ipi_types[NR_IPI] __tracepoint_string = {
#define S(x,s) [x] = s
S(IPI_WAKEUP, "CPU wakeup interrupts"),
S(IPI_COMPLETION, "completion interrupts"),
};
-static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
-{
- trace_ipi_raise_rcuidle(target, ipi_types[ipinr]);
- __smp_cross_call(target, ipinr);
-}
+static void smp_cross_call(const struct cpumask *target, unsigned int ipinr);
void show_ipi_list(struct seq_file *p, int prec)
{
unsigned int cpu, i;
for (i = 0; i < NR_IPI; i++) {
+ unsigned int irq;
+
+ if (!ipi_desc[i])
+ continue;
+
+ irq = irq_desc_get_irq(ipi_desc[i]);
seq_printf(p, "%*s%u: ", prec - 1, "IPI", i);
for_each_online_cpu(cpu)
- seq_printf(p, "%10u ",
- __get_irq_stat(cpu, ipi_irqs[i]));
+ seq_printf(p, "%10u ", kstat_irqs_cpu(irq, cpu));
seq_printf(p, " %s\n", ipi_types[i]);
}
}
-u64 smp_irq_stat_cpu(unsigned int cpu)
-{
- u64 sum = 0;
- int i;
-
- for (i = 0; i < NR_IPI; i++)
- sum += __get_irq_stat(cpu, ipi_irqs[i]);
-
- return sum;
-}
-
void arch_send_call_function_ipi_mask(const struct cpumask *mask)
{
smp_cross_call(mask, IPI_CALL_FUNC);
handle_IPI(ipinr, regs);
}
-void handle_IPI(int ipinr, struct pt_regs *regs)
+static void do_handle_IPI(int ipinr)
{
unsigned int cpu = smp_processor_id();
- struct pt_regs *old_regs = set_irq_regs(regs);
- if ((unsigned)ipinr < NR_IPI) {
+ if ((unsigned)ipinr < NR_IPI)
trace_ipi_entry_rcuidle(ipi_types[ipinr]);
- __inc_irq_stat(cpu, ipi_irqs[ipinr]);
- }
switch (ipinr) {
case IPI_WAKEUP:
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
case IPI_TIMER:
- irq_enter();
tick_receive_broadcast();
- irq_exit();
break;
#endif
break;
case IPI_CALL_FUNC:
- irq_enter();
generic_smp_call_function_interrupt();
- irq_exit();
break;
case IPI_CPU_STOP:
- irq_enter();
ipi_cpu_stop(cpu);
- irq_exit();
break;
#ifdef CONFIG_IRQ_WORK
case IPI_IRQ_WORK:
- irq_enter();
irq_work_run();
- irq_exit();
break;
#endif
case IPI_COMPLETION:
- irq_enter();
ipi_complete(cpu);
- irq_exit();
break;
case IPI_CPU_BACKTRACE:
printk_nmi_enter();
- irq_enter();
- nmi_cpu_backtrace(regs);
- irq_exit();
+ nmi_cpu_backtrace(get_irq_regs());
printk_nmi_exit();
break;
if ((unsigned)ipinr < NR_IPI)
trace_ipi_exit_rcuidle(ipi_types[ipinr]);
+}
+
+/* Legacy version, should go away once all irqchips have been converted */
+void handle_IPI(int ipinr, struct pt_regs *regs)
+{
+ struct pt_regs *old_regs = set_irq_regs(regs);
+
+ irq_enter();
+ do_handle_IPI(ipinr);
+ irq_exit();
+
set_irq_regs(old_regs);
}
+static irqreturn_t ipi_handler(int irq, void *data)
+{
+ do_handle_IPI(irq - ipi_irq_base);
+ return IRQ_HANDLED;
+}
+
+static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
+{
+ trace_ipi_raise_rcuidle(target, ipi_types[ipinr]);
+ __ipi_send_mask(ipi_desc[ipinr], target);
+}
+
+static void ipi_setup(int cpu)
+{
+ int i;
+
+ if (WARN_ON_ONCE(!ipi_irq_base))
+ return;
+
+ for (i = 0; i < nr_ipi; i++)
+ enable_percpu_irq(ipi_irq_base + i, 0);
+}
+
+void __init set_smp_ipi_range(int ipi_base, int n)
+{
+ int i;
+
+ WARN_ON(n < MAX_IPI);
+ nr_ipi = min(n, MAX_IPI);
+
+ for (i = 0; i < nr_ipi; i++) {
+ int err;
+
+ err = request_percpu_irq(ipi_base + i, ipi_handler,
+ "IPI", &irq_stat);
+ WARN_ON(err);
+
+ ipi_desc[i] = irq_to_desc(ipi_base + i);
+ irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);
+ }
+
+ ipi_irq_base = ipi_base;
+
+ /* Setup the boot CPU immediately */
+ ipi_setup(smp_processor_id());
+}
+
void smp_send_reschedule(int cpu)
{
smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
static void raise_nmi(cpumask_t *mask)
{
- __smp_cross_call(mask, IPI_CPU_BACKTRACE);
+ __ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask);
}
void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self)
static inline void update_cpu_capacity(unsigned int cpuid) {}
#endif
-/*
- * The current assumption is that we can power gate each core independently.
- * This will be superseded by DT binding once available.
- */
-const struct cpumask *cpu_corepower_mask(int cpu)
-{
- return &cpu_topology[cpu].thread_sibling;
-}
-
/*
* store_cpu_topology is called at boot when only one cpu is running
* and with the mutex cpu_hotplug.lock locked, when several cpus have booted,
update_siblings_masks(cpuid);
}
-static inline int cpu_corepower_flags(void)
-{
- return SD_SHARE_PKG_RESOURCES | SD_SHARE_POWERDOMAIN;
-}
-
-static struct sched_domain_topology_level arm_topology[] = {
-#ifdef CONFIG_SCHED_MC
- { cpu_corepower_mask, cpu_corepower_flags, SD_INIT_NAME(GMC) },
- { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
-#endif
- { cpu_cpu_mask, SD_INIT_NAME(DIE) },
- { NULL, },
-};
-
/*
* init_cpu_topology is called at boot when only one cpu is running
* which prevent simultaneous write access to cpu_topology array
smp_wmb();
parse_dt_topology();
-
- /* Set scheduler topology descriptor */
- set_sched_topology(arm_topology);
}
#include <linux/sizes.h>
-#include <asm-generic/vmlinux.lds.h>
+#include <asm/vmlinux.lds.h>
#include <asm/cache.h>
#include <asm/thread_info.h>
#include <asm/memory.h>
#include <asm/mpu.h>
#include <asm/page.h>
-#include "vmlinux.lds.h"
-
OUTPUT_ARCH(arm)
ENTRY(stext)
_end = .;
STABS_DEBUG
+ DWARF_DEBUG
+ ARM_DETAILS
+
+ ARM_ASSERTS
}
/*
#else
#include <linux/pgtable.h>
-#include <asm-generic/vmlinux.lds.h>
+#include <asm/vmlinux.lds.h>
#include <asm/cache.h>
#include <asm/thread_info.h>
#include <asm/memory.h>
#include <asm/mpu.h>
#include <asm/page.h>
-#include "vmlinux.lds.h"
-
OUTPUT_ARCH(arm)
ENTRY(stext)
_end = .;
STABS_DEBUG
+ DWARF_DEBUG
+ ARM_DETAILS
+
+ ARM_ASSERTS
}
#ifdef CONFIG_STRICT_KERNEL_RWX
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-
-#ifdef CONFIG_HOTPLUG_CPU
-#define ARM_CPU_DISCARD(x)
-#define ARM_CPU_KEEP(x) x
-#else
-#define ARM_CPU_DISCARD(x) x
-#define ARM_CPU_KEEP(x)
-#endif
-
-#if (defined(CONFIG_SMP_ON_UP) && !defined(CONFIG_DEBUG_SPINLOCK)) || \
- defined(CONFIG_GENERIC_BUG) || defined(CONFIG_JUMP_LABEL)
-#define ARM_EXIT_KEEP(x) x
-#define ARM_EXIT_DISCARD(x)
-#else
-#define ARM_EXIT_KEEP(x)
-#define ARM_EXIT_DISCARD(x) x
-#endif
-
-#ifdef CONFIG_MMU
-#define ARM_MMU_KEEP(x) x
-#define ARM_MMU_DISCARD(x)
-#else
-#define ARM_MMU_KEEP(x)
-#define ARM_MMU_DISCARD(x) x
-#endif
-
-#define PROC_INFO \
- . = ALIGN(4); \
- __proc_info_begin = .; \
- *(.proc.info.init) \
- __proc_info_end = .;
-
-#define IDMAP_TEXT \
- ALIGN_FUNCTION(); \
- __idmap_text_start = .; \
- *(.idmap.text) \
- __idmap_text_end = .; \
-
-#define ARM_DISCARD \
- *(.ARM.exidx.exit.text) \
- *(.ARM.extab.exit.text) \
- *(.ARM.exidx.text.exit) \
- *(.ARM.extab.text.exit) \
- ARM_CPU_DISCARD(*(.ARM.exidx.cpuexit.text)) \
- ARM_CPU_DISCARD(*(.ARM.extab.cpuexit.text)) \
- ARM_EXIT_DISCARD(EXIT_TEXT) \
- ARM_EXIT_DISCARD(EXIT_DATA) \
- EXIT_CALL \
- ARM_MMU_DISCARD(*(.text.fixup)) \
- ARM_MMU_DISCARD(*(__ex_table)) \
- *(.discard) \
- *(.discard.*)
-
-#define ARM_TEXT \
- IDMAP_TEXT \
- __entry_text_start = .; \
- *(.entry.text) \
- __entry_text_end = .; \
- IRQENTRY_TEXT \
- SOFTIRQENTRY_TEXT \
- TEXT_TEXT \
- SCHED_TEXT \
- CPUIDLE_TEXT \
- LOCK_TEXT \
- KPROBES_TEXT \
- *(.gnu.warning) \
- *(.glue_7) \
- *(.glue_7t) \
- . = ALIGN(4); \
- *(.got) /* Global offset table */ \
- ARM_CPU_KEEP(PROC_INFO)
-
-/* Stack unwinding tables */
-#define ARM_UNWIND_SECTIONS \
- . = ALIGN(8); \
- .ARM.unwind_idx : { \
- __start_unwind_idx = .; \
- *(.ARM.exidx*) \
- __stop_unwind_idx = .; \
- } \
- .ARM.unwind_tab : { \
- __start_unwind_tab = .; \
- *(.ARM.extab*) \
- __stop_unwind_tab = .; \
- }
-
-/*
- * The vectors and stubs are relocatable code, and the
- * only thing that matters is their relative offsets
- */
-#define ARM_VECTORS \
- __vectors_start = .; \
- .vectors 0xffff0000 : AT(__vectors_start) { \
- *(.vectors) \
- } \
- . = __vectors_start + SIZEOF(.vectors); \
- __vectors_end = .; \
- \
- __stubs_start = .; \
- .stubs ADDR(.vectors) + 0x1000 : AT(__stubs_start) { \
- *(.stubs) \
- } \
- . = __stubs_start + SIZEOF(.stubs); \
- __stubs_end = .; \
- \
- PROVIDE(vector_fiq_offset = vector_fiq - ADDR(.vectors));
-
-#define ARM_TCM \
- __itcm_start = ALIGN(4); \
- .text_itcm ITCM_OFFSET : AT(__itcm_start - LOAD_OFFSET) { \
- __sitcm_text = .; \
- *(.tcm.text) \
- *(.tcm.rodata) \
- . = ALIGN(4); \
- __eitcm_text = .; \
- } \
- . = __itcm_start + SIZEOF(.text_itcm); \
- \
- __dtcm_start = .; \
- .data_dtcm DTCM_OFFSET : AT(__dtcm_start - LOAD_OFFSET) { \
- __sdtcm_data = .; \
- *(.tcm.data) \
- . = ALIGN(4); \
- __edtcm_data = .; \
- } \
- . = __dtcm_start + SIZEOF(.data_dtcm);
.text
-/* Function: __u32 csum_partial_copy_nocheck(const char *src, char *dst, int len, __u32 sum)
- * Params : r0 = src, r1 = dst, r2 = len, r3 = checksum
+/* Function: __u32 csum_partial_copy_nocheck(const char *src, char *dst, int len)
+ * Params : r0 = src, r1 = dst, r2 = len
* Returns : r0 = new checksum
*/
FN_ENTRY
save_regs
+ mov sum, #-1
cmp len, #8 @ Ensure that we have at least
blo .Lless8 @ 8 bytes to copy.
/*
* unsigned int
- * csum_partial_copy_from_user(const char *src, char *dst, int len, int sum, int *err_ptr)
- * r0 = src, r1 = dst, r2 = len, r3 = sum, [sp] = *err_ptr
- * Returns : r0 = checksum, [[sp, #0], #0] = 0 or -EFAULT
+ * csum_partial_copy_from_user(const char *src, char *dst, int len)
+ * r0 = src, r1 = dst, r2 = len
+ * Returns : r0 = checksum or 0
*/
#define FN_ENTRY ENTRY(csum_partial_copy_from_user)
#include "csumpartialcopygeneric.S"
/*
- * FIXME: minor buglet here
- * We don't return the checksum for the data present in the buffer. To do
- * so properly, we would have to add in whatever registers were loaded before
- * the fault, which, with the current asm above is not predictable.
+ * We report fault by returning 0 csum - impossible in normal case, since
+ * we start with 0xffffffff for initial sum.
*/
.pushsection .text.fixup,"ax"
.align 4
-9001: mov r4, #-EFAULT
-#ifdef CONFIG_CPU_SW_DOMAIN_PAN
- ldr r5, [sp, #9*4] @ *err_ptr
-#else
- ldr r5, [sp, #8*4] @ *err_ptr
-#endif
- str r4, [r5]
- ldmia sp, {r1, r2} @ retrieve dst, len
- add r2, r2, r1
- mov r0, #0 @ zero the buffer
-9002: teq r2, r1
- strbne r0, [r1], #1
- bne 9002b
+9001: mov r0, #0
load_regs
.popsection
imx6_set_lpm(WAIT_UNCLOCKED);
raw_spin_unlock(&cpuidle_lock);
+ rcu_idle_enter();
cpu_do_idle();
+ rcu_idle_exit();
raw_spin_lock(&cpuidle_lock);
if (num_idle_cpus-- == num_online_cpus())
{
.exit_latency = 50,
.target_residency = 75,
- .flags = CPUIDLE_FLAG_TIMER_STOP,
+ .flags = CPUIDLE_FLAG_TIMER_STOP | CPUIDLE_FLAG_RCU_IDLE,
.enter = imx6q_enter_wait,
.name = "WAIT",
.desc = "Clock off",
/* Called from kretprobe_trampoline */
static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
{
- struct kretprobe_instance *ri = NULL;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address = 0;
- unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
- kprobe_opcode_t *correct_ret_addr = NULL;
-
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
-
- /*
- * It is possible to have multiple instances associated with a given
- * task either because multiple functions in the call path have
- * a return probe installed on them, and/or more than one return
- * probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always inserted at the head of the list
- * - when multiple return probes are registered for the same
- * function, the first instance's ret_addr will point to the
- * real return address, and all the rest will point to
- * kretprobe_trampoline
- */
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- orig_ret_address = (unsigned long)ri->ret_addr;
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
-
- correct_ret_addr = ri->ret_addr;
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- if (ri->rp && ri->rp->handler) {
- __this_cpu_write(current_kprobe, &ri->rp->kp);
- get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
- ri->ret_addr = correct_ret_addr;
- ri->rp->handler(ri, regs);
- __this_cpu_write(current_kprobe, NULL);
- }
-
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_hash_unlock(current, &flags);
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
-
- return (void *)orig_ret_address;
+ return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline,
+ (void *)regs->ARM_fp);
}
void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *)regs->ARM_lr;
+ ri->fp = (void *)regs->ARM_fp;
/* Replace the return addr with trampoline addr. */
regs->ARM_lr = (unsigned long)&kretprobe_trampoline;
pr_info("Xen: initializing cpu%d\n", cpu);
vcpup = per_cpu_ptr(xen_vcpu_info, cpu);
- info.mfn = virt_to_gfn(vcpup);
+ info.mfn = percpu_to_gfn(vcpup);
info.offset = xen_offset_in_page(vcpup);
err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, xen_vcpu_nr(cpu),
select ARCH_HAS_SETUP_DMA_OPS
select ARCH_HAS_SET_DIRECT_MAP
select ARCH_HAS_SET_MEMORY
+ select ARCH_STACKWALK
select ARCH_HAS_STRICT_KERNEL_RWX
select ARCH_HAS_STRICT_MODULE_RWX
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
select GENERIC_CPU_VULNERABILITIES
select GENERIC_EARLY_IOREMAP
select GENERIC_IDLE_POLL_SETUP
+ select GENERIC_IRQ_IPI
select GENERIC_IRQ_MULTI_HANDLER
select GENERIC_IRQ_PROBE
select GENERIC_IRQ_SHOW
default 14 if ARM64_16K_PAGES
default 12
-config ARM64_CONT_SHIFT
+config ARM64_CONT_PTE_SHIFT
int
default 5 if ARM64_64K_PAGES
default 7 if ARM64_16K_PAGES
default 4
+config ARM64_CONT_PMD_SHIFT
+ int
+ default 5 if ARM64_64K_PAGES
+ default 5 if ARM64_16K_PAGES
+ default 4
+
config ARCH_MMAP_RND_BITS_MIN
default 14 if ARM64_64K_PAGES
default 16 if ARM64_16K_PAGES
If unsure, say Y.
-config HARDEN_BRANCH_PREDICTOR
- bool "Harden the branch predictor against aliasing attacks" if EXPERT
- default y
- help
- Speculation attacks against some high-performance processors rely on
- being able to manipulate the branch predictor for a victim context by
- executing aliasing branches in the attacker context. Such attacks
- can be partially mitigated against by clearing internal branch
- predictor state and limiting the prediction logic in some situations.
-
- This config option will take CPU-specific actions to harden the
- branch predictor against aliasing attacks and may rely on specific
- instruction sequences or control bits being set by the system
- firmware.
-
- If unsure, say Y.
-
-config ARM64_SSBD
- bool "Speculative Store Bypass Disable" if EXPERT
- default y
- help
- This enables mitigation of the bypassing of previous stores
- by speculative loads.
-
- If unsure, say Y.
-
config RODATA_FULL_DEFAULT_ENABLED
bool "Apply r/o permissions of VM areas also to their linear aliases"
default y
provides a high bandwidth, cryptographically secure
hardware random number generator.
+config ARM64_AS_HAS_MTE
+ # Initial support for MTE went in binutils 2.32.0, checked with
+ # ".arch armv8.5-a+memtag" below. However, this was incomplete
+ # as a late addition to the final architecture spec (LDGM/STGM)
+ # is only supported in the newer 2.32.x and 2.33 binutils
+ # versions, hence the extra "stgm" instruction check below.
+ def_bool $(as-instr,.arch armv8.5-a+memtag\nstgm xzr$(comma)[x0])
+
+config ARM64_MTE
+ bool "Memory Tagging Extension support"
+ default y
+ depends on ARM64_AS_HAS_MTE && ARM64_TAGGED_ADDR_ABI
+ select ARCH_USES_HIGH_VMA_FLAGS
+ help
+ Memory Tagging (part of the ARMv8.5 Extensions) provides
+ architectural support for run-time, always-on detection of
+ various classes of memory error to aid with software debugging
+ to eliminate vulnerabilities arising from memory-unsafe
+ languages.
+
+ This option enables the support for the Memory Tagging
+ Extension at EL0 (i.e. for userspace).
+
+ Selecting this option allows the feature to be detected at
+ runtime. Any secondary CPU not implementing this feature will
+ not be allowed a late bring-up.
+
+ Userspace binaries that want to use this feature must
+ explicitly opt in. The mechanism for the userspace is
+ described in:
+
+ Documentation/arm64/memory-tagging-extension.rst.
+
endmenu
config ARM64_SVE
def_bool y
depends on HUGETLB_PAGE && MIGRATION
+config ARCH_ENABLE_THP_MIGRATION
+ def_bool y
+ depends on TRANSPARENT_HUGEPAGE
+
menu "Power management options"
source "kernel/power/Kconfig"
# Copyright (C) 1995-2001 by Russell King
LDFLAGS_vmlinux :=--no-undefined -X
-CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET)
ifeq ($(CONFIG_RELOCATABLE), y)
# Pass --no-apply-dynamic-relocs to restore pre-binutils-2.27 behaviour
endif
endif
+# We never want expected sections to be placed heuristically by the
+# linker. All sections should be explicitly named in the linker script.
+LDFLAGS_vmlinux += $(call ld-option, --orphan-handling=warn)
+
ifeq ($(CONFIG_ARM64_USE_LSE_ATOMICS), y)
ifneq ($(CONFIG_ARM64_LSE_ATOMICS), y)
$(warning LSE atomics not supported by binutils)
KBUILD_CFLAGS += -mgeneral-regs-only \
$(compat_vdso) $(cc_has_k_constraint)
-KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
KBUILD_CFLAGS += $(call cc-disable-warning, psabi)
KBUILD_AFLAGS += $(compat_vdso)
KBUILD_CFLAGS += $(call cc-option,-mabi=lp64)
KBUILD_AFLAGS += $(call cc-option,-mabi=lp64)
+# Avoid generating .eh_frame* sections.
+KBUILD_CFLAGS += -fno-asynchronous-unwind-tables -fno-unwind-tables
+KBUILD_AFLAGS += -fno-asynchronous-unwind-tables -fno-unwind-tables
+
ifeq ($(CONFIG_STACKPROTECTOR_PER_TASK),y)
prepare: stack_protector_prepare
stack_protector_prepare: prepare0
# Default value
head-y := arch/arm64/kernel/head.o
-# The byte offset of the kernel image in RAM from the start of RAM.
-TEXT_OFFSET := 0x0
-
ifeq ($(CONFIG_KASAN_SW_TAGS), y)
KASAN_SHADOW_SCALE_SHIFT := 4
else
KBUILD_CPPFLAGS += -DKASAN_SHADOW_SCALE_SHIFT=$(KASAN_SHADOW_SCALE_SHIFT)
KBUILD_AFLAGS += -DKASAN_SHADOW_SCALE_SHIFT=$(KASAN_SHADOW_SCALE_SHIFT)
-export TEXT_OFFSET
-
core-y += arch/arm64/
libs-y := arch/arm64/lib/ $(libs-y)
libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a
0: mov bskey, x21
mov rounds, x22
- br x7
+ br x16
SYM_FUNC_END(__xts_crypt8)
.macro __xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
uzp1 v30.4s, v30.4s, v25.4s
ld1 {v25.16b}, [x24]
-99: adr x7, \do8
+99: adr x16, \do8
bl __xts_crypt8
ldp q16, q17, [sp, #.Lframe_local_offset]
u8 buf[AES_BLOCK_SIZE];
u8 iv[AES_BLOCK_SIZE];
u64 dg[2] = {};
- u128 lengths;
+ be128 lengths;
u8 *tag;
int err;
u8 buf[AES_BLOCK_SIZE];
u8 iv[AES_BLOCK_SIZE];
u64 dg[2] = {};
- u128 lengths;
+ be128 lengths;
u8 *tag;
int err;
u32 finalize;
};
+extern const u32 sha1_ce_offsetof_count;
+extern const u32 sha1_ce_offsetof_finalize;
+
asmlinkage void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src,
int blocks);
u32 finalize;
};
+extern const u32 sha256_ce_offsetof_count;
+extern const u32 sha256_ce_offsetof_finalize;
+
asmlinkage void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src,
int blocks);
static inline void gic_pmr_mask_irqs(void)
{
- BUILD_BUG_ON(GICD_INT_DEF_PRI < (GIC_PRIO_IRQOFF |
+ BUILD_BUG_ON(GICD_INT_DEF_PRI < (__GIC_PRIO_IRQOFF |
GIC_PRIO_PSR_I_SET));
BUILD_BUG_ON(GICD_INT_DEF_PRI >= GIC_PRIO_IRQON);
/*
* are applied to IRQ priorities
*/
BUILD_BUG_ON((0x80 | (GICD_INT_DEF_PRI >> 1)) >= GIC_PRIO_IRQON);
+ /*
+ * Same situation as above, but now we make sure that we can mask
+ * regular interrupts.
+ */
+ BUILD_BUG_ON((0x80 | (GICD_INT_DEF_PRI >> 1)) < (__GIC_PRIO_IRQOFF_NS |
+ GIC_PRIO_PSR_I_SET));
gic_write_pmr(GIC_PRIO_IRQOFF);
}
}
#define arch_get_random_seed_long_early arch_get_random_seed_long_early
-#else
-
-static inline bool __arm64_rndr(unsigned long *v) { return false; }
-static inline bool __init __early_cpu_has_rndr(void) { return false; }
-
#endif /* CONFIG_ARCH_RANDOM */
#endif /* _ASM_ARCHRANDOM_H */
#define MAX_FDT_SIZE SZ_2M
/*
- * arm64 requires the kernel image to placed
- * TEXT_OFFSET bytes beyond a 2 MB aligned base
+ * arm64 requires the kernel image to placed at a 2 MB aligned base address
*/
#define MIN_KIMG_ALIGN SZ_2M
typedef u16 compat_ipc_pid_t;
typedef u32 compat_caddr_t;
typedef __kernel_fsid_t compat_fsid_t;
-typedef s64 compat_s64;
-typedef u64 compat_u64;
struct compat_stat {
#ifdef __AARCH64EB__
* mechanism for doing so, tests whether it is possible to boot
* the given CPU.
* @cpu_boot: Boots a cpu into the kernel.
- * @cpu_postboot: Optionally, perform any post-boot cleanup or necesary
+ * @cpu_postboot: Optionally, perform any post-boot cleanup or necessary
* synchronisation. Called from the cpu being booted.
* @cpu_can_disable: Determines whether a CPU can be disabled based on
* mechanism-specific information.
#define ARM64_HAS_DCPOP 21
#define ARM64_SVE 22
#define ARM64_UNMAP_KERNEL_AT_EL0 23
-#define ARM64_HARDEN_BRANCH_PREDICTOR 24
+#define ARM64_SPECTRE_V2 24
#define ARM64_HAS_RAS_EXTN 25
#define ARM64_WORKAROUND_843419 26
#define ARM64_HAS_CACHE_IDC 27
#define ARM64_HAS_CACHE_DIC 28
#define ARM64_HW_DBM 29
-#define ARM64_SSBD 30
+#define ARM64_SPECTRE_V4 30
#define ARM64_MISMATCHED_CACHE_TYPE 31
#define ARM64_HAS_STAGE2_FWB 32
#define ARM64_HAS_CRC32 33
#define ARM64_BTI 54
#define ARM64_HAS_ARMv8_4_TTL 55
#define ARM64_HAS_TLB_RANGE 56
+#define ARM64_MTE 57
-#define ARM64_NCAPS 57
+#define ARM64_NCAPS 58
#endif /* __ASM_CPUCAPS_H */
}
/*
- * Generic helper for handling capabilties with multiple (match,enable) pairs
+ * Generic helper for handling capabilities with multiple (match,enable) pairs
* of call backs, sharing the same capability bit.
* Iterate over each entry to see if at least one matches.
*/
cpus_have_const_cap(ARM64_HAS_IRQ_PRIO_MASKING);
}
+static inline bool system_supports_mte(void)
+{
+ return IS_ENABLED(CONFIG_ARM64_MTE) &&
+ cpus_have_const_cap(ARM64_MTE);
+}
+
static inline bool system_has_prio_mask_debugging(void)
{
return IS_ENABLED(CONFIG_ARM64_DEBUG_PRIORITY_MASKING) &&
cpus_have_const_cap(ARM64_HAS_TLB_RANGE);
}
-#define ARM64_BP_HARDEN_UNKNOWN -1
-#define ARM64_BP_HARDEN_WA_NEEDED 0
-#define ARM64_BP_HARDEN_NOT_REQUIRED 1
-
-int get_spectre_v2_workaround_state(void);
-
-#define ARM64_SSBD_UNKNOWN -1
-#define ARM64_SSBD_FORCE_DISABLE 0
-#define ARM64_SSBD_KERNEL 1
-#define ARM64_SSBD_FORCE_ENABLE 2
-#define ARM64_SSBD_MITIGATED 3
-
-static inline int arm64_get_ssbd_state(void)
-{
-#ifdef CONFIG_ARM64_SSBD
- extern int ssbd_state;
- return ssbd_state;
-#else
- return ARM64_SSBD_UNKNOWN;
-#endif
-}
-
-void arm64_set_ssbd_mitigation(bool state);
-
extern int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
static inline u32 id_aa64mmfr0_parange_to_phys_shift(int parange)
(SEGMENT_ALIGN > THREAD_ALIGN ? SEGMENT_ALIGN : THREAD_ALIGN)
/* on arm64, the FDT may be located anywhere in system RAM */
-static inline unsigned long efi_get_max_fdt_addr(unsigned long dram_base)
+static inline unsigned long efi_get_max_fdt_addr(unsigned long image_addr)
{
return ULONG_MAX;
}
* apply to other bootloaders, and are required for some kernel
* configurations.
*/
-static inline unsigned long efi_get_max_initrd_addr(unsigned long dram_base,
- unsigned long image_addr)
+static inline unsigned long efi_get_max_initrd_addr(unsigned long image_addr)
{
return (image_addr & ~(SZ_1G - 1UL)) + (1UL << (VA_BITS_MIN - 1));
}
#define ESR_ELx_EC_SYS64 (0x18)
#define ESR_ELx_EC_SVE (0x19)
#define ESR_ELx_EC_ERET (0x1a) /* EL2 only */
-/* Unallocated EC: 0x1b - 0x1E */
+/* Unallocated EC: 0x1B */
+#define ESR_ELx_EC_FPAC (0x1C) /* EL1 and above */
+/* Unallocated EC: 0x1D - 0x1E */
#define ESR_ELx_EC_IMP_DEF (0x1f) /* EL3 only */
#define ESR_ELx_EC_IABT_LOW (0x20)
#define ESR_ELx_EC_IABT_CUR (0x21)
void do_cp15instr(unsigned int esr, struct pt_regs *regs);
void do_el0_svc(struct pt_regs *regs);
void do_el0_svc_compat(struct pt_regs *regs);
+void do_ptrauth_fault(struct pt_regs *regs, unsigned int esr);
#endif /* __ASM_EXCEPTION_H */
#define ARCH_HAS_RELATIVE_EXTABLE
+static inline bool in_bpf_jit(struct pt_regs *regs)
+{
+ if (!IS_ENABLED(CONFIG_BPF_JIT))
+ return false;
+
+ return regs->pc >= BPF_JIT_REGION_START &&
+ regs->pc < BPF_JIT_REGION_END;
+}
+
#ifdef CONFIG_BPF_JIT
int arm64_bpf_fixup_exception(const struct exception_table_entry *ex,
struct pt_regs *regs);
extern void sve_save_state(void *state, u32 *pfpsr);
extern void sve_load_state(void const *state, u32 const *pfpsr,
unsigned long vq_minus_1);
+extern void sve_flush_live(void);
+extern void sve_load_from_fpsimd_state(struct user_fpsimd_state const *state,
+ unsigned long vq_minus_1);
extern unsigned int sve_get_vl(void);
struct arm64_cpu_capabilities;
| ((\np) << 5)
.endm
+/* PFALSE P\np.B */
+.macro _sve_pfalse np
+ _sve_check_preg \np
+ .inst 0x2518e400 \
+ | (\np)
+.endm
+
.macro __for from:req, to:req
.if (\from) == (\to)
- _for__body \from
+ _for__body %\from
.else
- __for \from, (\from) + ((\to) - (\from)) / 2
- __for (\from) + ((\to) - (\from)) / 2 + 1, \to
+ __for %\from, %((\from) + ((\to) - (\from)) / 2)
+ __for %((\from) + ((\to) - (\from)) / 2 + 1), %\to
.endif
.endm
.macro _for var:req, from:req, to:req, insn:vararg
.macro _for__body \var:req
+ .noaltmacro
\insn
+ .altmacro
.endm
+ .altmacro
__for \from, \to
+ .noaltmacro
.purgem _for__body
.endm
+/* Update ZCR_EL1.LEN with the new VQ */
+.macro sve_load_vq xvqminus1, xtmp, xtmp2
+ mrs_s \xtmp, SYS_ZCR_EL1
+ bic \xtmp2, \xtmp, ZCR_ELx_LEN_MASK
+ orr \xtmp2, \xtmp2, \xvqminus1
+ cmp \xtmp2, \xtmp
+ b.eq 921f
+ msr_s SYS_ZCR_EL1, \xtmp2 //self-synchronising
+921:
+.endm
+
+/* Preserve the first 128-bits of Znz and zero the rest. */
+.macro _sve_flush_z nz
+ _sve_check_zreg \nz
+ mov v\nz\().16b, v\nz\().16b
+.endm
+
+.macro sve_flush
+ _for n, 0, 31, _sve_flush_z \n
+ _for n, 0, 15, _sve_pfalse \n
+ _sve_wrffr 0
+.endm
+
.macro sve_save nxbase, xpfpsr, nxtmp
_for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34
_for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16
.endm
.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2
- mrs_s x\nxtmp, SYS_ZCR_EL1
- bic \xtmp2, x\nxtmp, ZCR_ELx_LEN_MASK
- orr \xtmp2, \xtmp2, \xvqminus1
- cmp \xtmp2, x\nxtmp
- b.eq 921f
- msr_s SYS_ZCR_EL1, \xtmp2 // self-synchronising
-921:
+ sve_load_vq \xvqminus1, x\nxtmp, \xtmp2
_for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34
_sve_ldr_p 0, \nxbase
_sve_wrffr 0
#include <asm/kvm_arm.h>
#include <asm/sysreg.h>
-#define NR_IPI 7
-
typedef struct {
unsigned int __softirq_pending;
- unsigned int ipi_irqs[NR_IPI];
} ____cacheline_aligned irq_cpustat_t;
#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */
-#define __inc_irq_stat(cpu, member) __IRQ_STAT(cpu, member)++
-#define __get_irq_stat(cpu, member) __IRQ_STAT(cpu, member)
-
-u64 smp_irq_stat_cpu(unsigned int cpu);
-#define arch_irq_stat_cpu smp_irq_stat_cpu
-
#define __ARCH_IRQ_EXIT_IRQS_DISABLED 1
struct nmi_ctx {
#include <uapi/asm/hwcap.h>
#include <asm/cpufeature.h>
+#define COMPAT_HWCAP_SWP (1 << 0)
#define COMPAT_HWCAP_HALF (1 << 1)
#define COMPAT_HWCAP_THUMB (1 << 2)
+#define COMPAT_HWCAP_26BIT (1 << 3)
#define COMPAT_HWCAP_FAST_MULT (1 << 4)
+#define COMPAT_HWCAP_FPA (1 << 5)
#define COMPAT_HWCAP_VFP (1 << 6)
#define COMPAT_HWCAP_EDSP (1 << 7)
+#define COMPAT_HWCAP_JAVA (1 << 8)
+#define COMPAT_HWCAP_IWMMXT (1 << 9)
+#define COMPAT_HWCAP_CRUNCH (1 << 10)
+#define COMPAT_HWCAP_THUMBEE (1 << 11)
#define COMPAT_HWCAP_NEON (1 << 12)
#define COMPAT_HWCAP_VFPv3 (1 << 13)
+#define COMPAT_HWCAP_VFPV3D16 (1 << 14)
#define COMPAT_HWCAP_TLS (1 << 15)
#define COMPAT_HWCAP_VFPv4 (1 << 16)
#define COMPAT_HWCAP_IDIVA (1 << 17)
#define COMPAT_HWCAP_IDIVT (1 << 18)
#define COMPAT_HWCAP_IDIV (COMPAT_HWCAP_IDIVA|COMPAT_HWCAP_IDIVT)
+#define COMPAT_HWCAP_VFPD32 (1 << 19)
#define COMPAT_HWCAP_LPAE (1 << 20)
#define COMPAT_HWCAP_EVTSTRM (1 << 21)
#define KERNEL_HWCAP_DGH __khwcap2_feature(DGH)
#define KERNEL_HWCAP_RNG __khwcap2_feature(RNG)
#define KERNEL_HWCAP_BTI __khwcap2_feature(BTI)
-/* reserved for KERNEL_HWCAP_MTE __khwcap2_feature(MTE) */
+#define KERNEL_HWCAP_MTE __khwcap2_feature(MTE)
/*
* This yields a mask that user programs can use to figure out what
__AARCH64_INSN_FUNCS(exception, 0xFF000000, 0xD4000000)
__AARCH64_INSN_FUNCS(hint, 0xFFFFF01F, 0xD503201F)
__AARCH64_INSN_FUNCS(br, 0xFFFFFC1F, 0xD61F0000)
+__AARCH64_INSN_FUNCS(br_auth, 0xFEFFF800, 0xD61F0800)
__AARCH64_INSN_FUNCS(blr, 0xFFFFFC1F, 0xD63F0000)
+__AARCH64_INSN_FUNCS(blr_auth, 0xFEFFF800, 0xD63F0800)
__AARCH64_INSN_FUNCS(ret, 0xFFFFFC1F, 0xD65F0000)
+__AARCH64_INSN_FUNCS(ret_auth, 0xFFFFFBFF, 0xD65F0BFF)
__AARCH64_INSN_FUNCS(eret, 0xFFFFFFFF, 0xD69F03E0)
+__AARCH64_INSN_FUNCS(eret_auth, 0xFFFFFBFF, 0xD69F0BFF)
__AARCH64_INSN_FUNCS(mrs, 0xFFF00000, 0xD5300000)
__AARCH64_INSN_FUNCS(msr_imm, 0xFFF8F01F, 0xD500401F)
__AARCH64_INSN_FUNCS(msr_reg, 0xFFF00000, 0xD5100000)
#ifndef __ASM_IRQ_WORK_H
#define __ASM_IRQ_WORK_H
-#include <asm/smp.h>
-
static inline bool arch_irq_work_has_interrupt(void)
{
- return !!__smp_cross_call;
+ return true;
}
#endif /* __ASM_IRQ_WORK_H */
+ EARLY_PGDS((vstart), (vend)) /* each PGDIR needs a next level page table */ \
+ EARLY_PUDS((vstart), (vend)) /* each PUD needs a next level page table */ \
+ EARLY_PMDS((vstart), (vend))) /* each PMD needs a next level page table */
-#define INIT_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR + TEXT_OFFSET, _end))
+#define INIT_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR, _end))
#define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE)
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
#include <asm/types.h>
/* Hyp Configuration Register (HCR) bits */
+#define HCR_ATA (UL(1) << 56)
#define HCR_FWB (UL(1) << 46)
#define HCR_API (UL(1) << 41)
#define HCR_APK (UL(1) << 40)
* TWI: Trap WFI
* TIDCP: Trap L2CTLR/L2ECTLR
* BSU_IS: Upgrade barriers to the inner shareable domain
- * FB: Force broadcast of all maintainance operations
+ * FB: Force broadcast of all maintenance operations
* AMO: Override CPSR.A and enable signaling with VA
* IMO: Override CPSR.I and enable signaling with VI
* FMO: Override CPSR.F and enable signaling with VF
HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \
HCR_FMO | HCR_IMO | HCR_PTW )
#define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF)
-#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK)
+#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK | HCR_ATA)
#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)
/* TCR_EL2 Registers bits */
#include <asm/virt.h>
-#define VCPU_WORKAROUND_2_FLAG_SHIFT 0
-#define VCPU_WORKAROUND_2_FLAG (_AC(1, UL) << VCPU_WORKAROUND_2_FLAG_SHIFT)
-
#define ARM_EXIT_WITH_SERROR_BIT 31
#define ARM_EXCEPTION_CODE(x) ((x) & ~(1U << ARM_EXIT_WITH_SERROR_BIT))
#define ARM_EXCEPTION_IS_TRAP(x) (ARM_EXCEPTION_CODE((x)) == ARM_EXCEPTION_TRAP)
#define __kvm_hyp_init CHOOSE_NVHE_SYM(__kvm_hyp_init)
#define __kvm_hyp_vector CHOOSE_HYP_SYM(__kvm_hyp_vector)
-#ifdef CONFIG_KVM_INDIRECT_VECTORS
extern atomic_t arm64_el2_vector_last_slot;
DECLARE_KVM_HYP_SYM(__bp_harden_hyp_vecs);
#define __bp_harden_hyp_vecs CHOOSE_HYP_SYM(__bp_harden_hyp_vecs)
-#endif
extern void __kvm_flush_vm_context(void);
extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa,
return vcpu_read_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
}
-static inline bool kvm_arm_get_vcpu_workaround_2_flag(struct kvm_vcpu *vcpu)
-{
- return vcpu->arch.workaround_flags & VCPU_WORKAROUND_2_FLAG;
-}
-
-static inline void kvm_arm_set_vcpu_workaround_2_flag(struct kvm_vcpu *vcpu,
- bool flag)
-{
- if (flag)
- vcpu->arch.workaround_flags |= VCPU_WORKAROUND_2_FLAG;
- else
- vcpu->arch.workaround_flags &= ~VCPU_WORKAROUND_2_FLAG;
-}
-
static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
{
if (vcpu_mode_is_32bit(vcpu)) {
static inline void kvm_clr_pmu_events(u32 clr) {}
#endif
-#define KVM_BP_HARDEN_UNKNOWN -1
-#define KVM_BP_HARDEN_WA_NEEDED 0
-#define KVM_BP_HARDEN_NOT_REQUIRED 1
-
-static inline int kvm_arm_harden_branch_predictor(void)
-{
- switch (get_spectre_v2_workaround_state()) {
- case ARM64_BP_HARDEN_WA_NEEDED:
- return KVM_BP_HARDEN_WA_NEEDED;
- case ARM64_BP_HARDEN_NOT_REQUIRED:
- return KVM_BP_HARDEN_NOT_REQUIRED;
- case ARM64_BP_HARDEN_UNKNOWN:
- default:
- return KVM_BP_HARDEN_UNKNOWN;
- }
-}
-
-#define KVM_SSBD_UNKNOWN -1
-#define KVM_SSBD_FORCE_DISABLE 0
-#define KVM_SSBD_KERNEL 1
-#define KVM_SSBD_FORCE_ENABLE 2
-#define KVM_SSBD_MITIGATED 3
-
-static inline int kvm_arm_have_ssbd(void)
-{
- switch (arm64_get_ssbd_state()) {
- case ARM64_SSBD_FORCE_DISABLE:
- return KVM_SSBD_FORCE_DISABLE;
- case ARM64_SSBD_KERNEL:
- return KVM_SSBD_KERNEL;
- case ARM64_SSBD_FORCE_ENABLE:
- return KVM_SSBD_FORCE_ENABLE;
- case ARM64_SSBD_MITIGATED:
- return KVM_SSBD_MITIGATED;
- case ARM64_SSBD_UNKNOWN:
- default:
- return KVM_SSBD_UNKNOWN;
- }
-}
-
void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu);
void kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu *vcpu);
#include <asm/page.h>
#include <asm/memory.h>
+#include <asm/mmu.h>
#include <asm/cpufeature.h>
/*
return ret;
}
-#ifdef CONFIG_KVM_INDIRECT_VECTORS
/*
* EL2 vectors can be mapped and rerouted in a number of ways,
* depending on the kernel configuration and CPU present:
*
- * - If the CPU has the ARM64_HARDEN_BRANCH_PREDICTOR cap, the
- * hardening sequence is placed in one of the vector slots, which is
- * executed before jumping to the real vectors.
+ * - If the CPU is affected by Spectre-v2, the hardening sequence is
+ * placed in one of the vector slots, which is executed before jumping
+ * to the real vectors.
*
- * - If the CPU has both the ARM64_HARDEN_EL2_VECTORS cap and the
- * ARM64_HARDEN_BRANCH_PREDICTOR cap, the slot containing the
- * hardening sequence is mapped next to the idmap page, and executed
- * before jumping to the real vectors.
+ * - If the CPU also has the ARM64_HARDEN_EL2_VECTORS cap, the slot
+ * containing the hardening sequence is mapped next to the idmap page,
+ * and executed before jumping to the real vectors.
*
* - If the CPU only has the ARM64_HARDEN_EL2_VECTORS cap, then an
* empty slot is selected, mapped next to the idmap page, and
* VHE, as we don't have hypervisor-specific mappings. If the system
* is VHE and yet selects this capability, it will be ignored.
*/
-#include <asm/mmu.h>
-
extern void *__kvm_bp_vect_base;
extern int __kvm_harden_el2_vector_slot;
-/* This is called on both VHE and !VHE systems */
static inline void *kvm_get_hyp_vector(void)
{
struct bp_hardening_data *data = arm64_get_bp_hardening_data();
void *vect = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
int slot = -1;
- if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR) && data->fn) {
+ if (cpus_have_const_cap(ARM64_SPECTRE_V2) && data->fn) {
vect = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
slot = data->hyp_vectors_slot;
}
return vect;
}
-/* This is only called on a !VHE system */
-static inline int kvm_map_vectors(void)
-{
- /*
- * HBP = ARM64_HARDEN_BRANCH_PREDICTOR
- * HEL2 = ARM64_HARDEN_EL2_VECTORS
- *
- * !HBP + !HEL2 -> use direct vectors
- * HBP + !HEL2 -> use hardened vectors in place
- * !HBP + HEL2 -> allocate one vector slot and use exec mapping
- * HBP + HEL2 -> use hardened vertors and use exec mapping
- */
- if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR)) {
- __kvm_bp_vect_base = kvm_ksym_ref(__bp_harden_hyp_vecs);
- __kvm_bp_vect_base = kern_hyp_va(__kvm_bp_vect_base);
- }
-
- if (cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS)) {
- phys_addr_t vect_pa = __pa_symbol(__bp_harden_hyp_vecs);
- unsigned long size = __BP_HARDEN_HYP_VECS_SZ;
-
- /*
- * Always allocate a spare vector slot, as we don't
- * know yet which CPUs have a BP hardening slot that
- * we can reuse.
- */
- __kvm_harden_el2_vector_slot = atomic_inc_return(&arm64_el2_vector_last_slot);
- BUG_ON(__kvm_harden_el2_vector_slot >= BP_HARDEN_EL2_SLOTS);
- return create_hyp_exec_mappings(vect_pa, size,
- &__kvm_bp_vect_base);
- }
-
- return 0;
-}
-#else
-static inline void *kvm_get_hyp_vector(void)
-{
- return kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
-}
-
-static inline int kvm_map_vectors(void)
-{
- return 0;
-}
-#endif
-
-#ifdef CONFIG_ARM64_SSBD
-DECLARE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
-
-static inline int hyp_map_aux_data(void)
-{
- int cpu, err;
-
- for_each_possible_cpu(cpu) {
- u64 *ptr;
-
- ptr = per_cpu_ptr(&arm64_ssbd_callback_required, cpu);
- err = create_hyp_mappings(ptr, ptr + 1, PAGE_HYP);
- if (err)
- return err;
- }
- return 0;
-}
-#else
-static inline int hyp_map_aux_data(void)
-{
- return 0;
-}
-#endif
-
#define kvm_phys_to_vttbr(addr) phys_to_ttbr(addr)
/*
/*
* Memory types available.
+ *
+ * IMPORTANT: MT_NORMAL must be index 0 since vm_get_page_prot() may 'or' in
+ * the MT_NORMAL_TAGGED memory type for PROT_MTE mappings. Note
+ * that protection_map[] only contains MT_NORMAL attributes.
*/
-#define MT_DEVICE_nGnRnE 0
-#define MT_DEVICE_nGnRE 1
-#define MT_DEVICE_GRE 2
-#define MT_NORMAL_NC 3
-#define MT_NORMAL 4
-#define MT_NORMAL_WT 5
+#define MT_NORMAL 0
+#define MT_NORMAL_TAGGED 1
+#define MT_NORMAL_NC 2
+#define MT_NORMAL_WT 3
+#define MT_DEVICE_nGnRnE 4
+#define MT_DEVICE_nGnRE 5
+#define MT_DEVICE_GRE 6
/*
* Memory types for Stage-2 translation
/* PHYS_OFFSET - the physical address of the start of memory. */
#define PHYS_OFFSET ({ VM_BUG_ON(memstart_addr & 1); memstart_addr; })
-/* the virtual base of the kernel image (minus TEXT_OFFSET) */
+/* the virtual base of the kernel image */
extern u64 kimage_vaddr;
/* the offset between the kernel virtual and physical mappings */
static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot,
unsigned long pkey __always_unused)
{
+ unsigned long ret = 0;
+
if (system_supports_bti() && (prot & PROT_BTI))
- return VM_ARM64_BTI;
+ ret |= VM_ARM64_BTI;
- return 0;
+ if (system_supports_mte() && (prot & PROT_MTE))
+ ret |= VM_MTE;
+
+ return ret;
}
#define arch_calc_vm_prot_bits(prot, pkey) arch_calc_vm_prot_bits(prot, pkey)
+static inline unsigned long arch_calc_vm_flag_bits(unsigned long flags)
+{
+ /*
+ * Only allow MTE on anonymous mappings as these are guaranteed to be
+ * backed by tags-capable memory. The vm_flags may be overridden by a
+ * filesystem supporting MTE (RAM-based).
+ */
+ if (system_supports_mte() && (flags & MAP_ANONYMOUS))
+ return VM_MTE_ALLOWED;
+
+ return 0;
+}
+#define arch_calc_vm_flag_bits(flags) arch_calc_vm_flag_bits(flags)
+
static inline pgprot_t arch_vm_get_page_prot(unsigned long vm_flags)
{
- return (vm_flags & VM_ARM64_BTI) ? __pgprot(PTE_GP) : __pgprot(0);
+ pteval_t prot = 0;
+
+ if (vm_flags & VM_ARM64_BTI)
+ prot |= PTE_GP;
+
+ /*
+ * There are two conditions required for returning a Normal Tagged
+ * memory type: (1) the user requested it via PROT_MTE passed to
+ * mmap() or mprotect() and (2) the corresponding vma supports MTE. We
+ * register (1) as VM_MTE in the vma->vm_flags and (2) as
+ * VM_MTE_ALLOWED. Note that the latter can only be set during the
+ * mmap() call since mprotect() does not accept MAP_* flags.
+ * Checking for VM_MTE only is sufficient since arch_validate_flags()
+ * does not permit (VM_MTE & !VM_MTE_ALLOWED).
+ */
+ if (vm_flags & VM_MTE)
+ prot |= PTE_ATTRINDX(MT_NORMAL_TAGGED);
+
+ return __pgprot(prot);
}
#define arch_vm_get_page_prot(vm_flags) arch_vm_get_page_prot(vm_flags)
if (system_supports_bti())
supported |= PROT_BTI;
+ if (system_supports_mte())
+ supported |= PROT_MTE;
+
return (prot & ~supported) == 0;
}
#define arch_validate_prot(prot, addr) arch_validate_prot(prot, addr)
+static inline bool arch_validate_flags(unsigned long vm_flags)
+{
+ if (!system_supports_mte())
+ return true;
+
+ /* only allow VM_MTE if VM_MTE_ALLOWED has been set previously */
+ return !(vm_flags & VM_MTE) || (vm_flags & VM_MTE_ALLOWED);
+}
+#define arch_validate_flags(vm_flags) arch_validate_flags(vm_flags)
+
#endif /* ! __ASM_MMAN_H__ */
#ifndef __ASSEMBLY__
+#include <linux/refcount.h>
+
typedef struct {
atomic64_t id;
#ifdef CONFIG_COMPAT
void *sigpage;
#endif
+ refcount_t pinned;
void *vdso;
unsigned long flags;
} mm_context_t;
bp_hardening_cb_t fn;
};
-#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
DECLARE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
static inline struct bp_hardening_data *arm64_get_bp_hardening_data(void)
{
struct bp_hardening_data *d;
- if (!cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR))
+ if (!cpus_have_const_cap(ARM64_SPECTRE_V2))
return;
d = arm64_get_bp_hardening_data();
if (d->fn)
d->fn();
}
-#else
-static inline struct bp_hardening_data *arm64_get_bp_hardening_data(void)
-{
- return NULL;
-}
-
-static inline void arm64_apply_bp_hardening(void) { }
-#endif /* CONFIG_HARDEN_BRANCH_PREDICTOR */
extern void arm64_memblock_init(void);
extern void paging_init(void);
#define destroy_context(mm) do { } while(0)
void check_and_switch_context(struct mm_struct *mm);
-#define init_new_context(tsk,mm) ({ atomic64_set(&(mm)->context.id, 0); 0; })
+static inline int
+init_new_context(struct task_struct *tsk, struct mm_struct *mm)
+{
+ atomic64_set(&mm->context.id, 0);
+ refcount_set(&mm->context.pinned, 0);
+ return 0;
+}
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
static inline void update_saved_ttbr0(struct task_struct *tsk,
void verify_cpu_asid_bits(void);
void post_ttbr_update_workaround(void);
+unsigned long arm64_mm_context_get(struct mm_struct *mm);
+void arm64_mm_context_put(struct mm_struct *mm);
+
#endif /* !__ASSEMBLY__ */
#endif /* !__ASM_MMU_CONTEXT_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 ARM Ltd.
+ */
+#ifndef __ASM_MTE_H
+#define __ASM_MTE_H
+
+#define MTE_GRANULE_SIZE UL(16)
+#define MTE_GRANULE_MASK (~(MTE_GRANULE_SIZE - 1))
+#define MTE_TAG_SHIFT 56
+#define MTE_TAG_SIZE 4
+
+#ifndef __ASSEMBLY__
+
+#include <linux/page-flags.h>
+
+#include <asm/pgtable-types.h>
+
+void mte_clear_page_tags(void *addr);
+unsigned long mte_copy_tags_from_user(void *to, const void __user *from,
+ unsigned long n);
+unsigned long mte_copy_tags_to_user(void __user *to, void *from,
+ unsigned long n);
+int mte_save_tags(struct page *page);
+void mte_save_page_tags(const void *page_addr, void *tag_storage);
+bool mte_restore_tags(swp_entry_t entry, struct page *page);
+void mte_restore_page_tags(void *page_addr, const void *tag_storage);
+void mte_invalidate_tags(int type, pgoff_t offset);
+void mte_invalidate_tags_area(int type);
+void *mte_allocate_tag_storage(void);
+void mte_free_tag_storage(char *storage);
+
+#ifdef CONFIG_ARM64_MTE
+
+/* track which pages have valid allocation tags */
+#define PG_mte_tagged PG_arch_2
+
+void mte_sync_tags(pte_t *ptep, pte_t pte);
+void mte_copy_page_tags(void *kto, const void *kfrom);
+void flush_mte_state(void);
+void mte_thread_switch(struct task_struct *next);
+void mte_suspend_exit(void);
+long set_mte_ctrl(struct task_struct *task, unsigned long arg);
+long get_mte_ctrl(struct task_struct *task);
+int mte_ptrace_copy_tags(struct task_struct *child, long request,
+ unsigned long addr, unsigned long data);
+
+#else
+
+/* unused if !CONFIG_ARM64_MTE, silence the compiler */
+#define PG_mte_tagged 0
+
+static inline void mte_sync_tags(pte_t *ptep, pte_t pte)
+{
+}
+static inline void mte_copy_page_tags(void *kto, const void *kfrom)
+{
+}
+static inline void flush_mte_state(void)
+{
+}
+static inline void mte_thread_switch(struct task_struct *next)
+{
+}
+static inline void mte_suspend_exit(void)
+{
+}
+static inline long set_mte_ctrl(struct task_struct *task, unsigned long arg)
+{
+ return 0;
+}
+static inline long get_mte_ctrl(struct task_struct *task)
+{
+ return 0;
+}
+static inline int mte_ptrace_copy_tags(struct task_struct *child,
+ long request, unsigned long addr,
+ unsigned long data)
+{
+ return -EIO;
+}
+
+#endif
+
+#endif /* __ASSEMBLY__ */
+#endif /* __ASM_MTE_H */
/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
static inline const struct cpumask *cpumask_of_node(int node)
{
+ if (node == NUMA_NO_NODE)
+ return cpu_all_mask;
+
return node_to_cpumask_map[node];
}
#endif
#include <linux/const.h>
/* PAGE_SHIFT determines the page size */
-/* CONT_SHIFT determines the number of pages which can be tracked together */
#define PAGE_SHIFT CONFIG_ARM64_PAGE_SHIFT
-#define CONT_SHIFT CONFIG_ARM64_CONT_SHIFT
#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT)
#define PAGE_MASK (~(PAGE_SIZE-1))
-#define CONT_SIZE (_AC(1, UL) << (CONT_SHIFT + PAGE_SHIFT))
-#define CONT_MASK (~(CONT_SIZE-1))
-
#endif /* __ASM_PAGE_DEF_H */
#include <linux/personality.h> /* for READ_IMPLIES_EXEC */
#include <asm/pgtable-types.h>
-extern void __cpu_clear_user_page(void *p, unsigned long user);
-extern void __cpu_copy_user_page(void *to, const void *from,
- unsigned long user);
+struct page;
+struct vm_area_struct;
+
extern void copy_page(void *to, const void *from);
extern void clear_page(void *to);
+void copy_user_highpage(struct page *to, struct page *from,
+ unsigned long vaddr, struct vm_area_struct *vma);
+#define __HAVE_ARCH_COPY_USER_HIGHPAGE
+
+void copy_highpage(struct page *to, struct page *from);
+#define __HAVE_ARCH_COPY_HIGHPAGE
+
#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \
alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr)
#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
-#define clear_user_page(addr,vaddr,pg) __cpu_clear_user_page(addr, vaddr)
-#define copy_user_page(to,from,vaddr,pg) __cpu_copy_user_page(to, from, vaddr)
+#define clear_user_page(page, vaddr, pg) clear_page(page)
+#define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
typedef struct page *pgtable_t;
#endif /* !__ASSEMBLY__ */
-#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_TSK_EXEC
+#define VM_DATA_DEFAULT_FLAGS (VM_DATA_FLAGS_TSK_EXEC | VM_MTE_ALLOWED)
#include <asm-generic/getorder.h>
#define pcibios_assign_all_busses() \
(pci_has_flag(PCI_REASSIGN_ALL_BUS))
+#define arch_can_pci_mmap_wc() 1
#define ARCH_GENERIC_PCI_MMAP_RESOURCE 1
extern int isa_dma_bridge_buggy;
#define ARMV8_PMU_USERENR_CR (1 << 2) /* Cycle counter can be read at EL0 */
#define ARMV8_PMU_USERENR_ER (1 << 3) /* Event counter can be read at EL0 */
+/* PMMIR_EL1.SLOTS mask */
+#define ARMV8_PMU_SLOTS_MASK 0xff
+
#ifdef CONFIG_PERF_EVENTS
struct pt_regs;
extern unsigned long perf_instruction_pointer(struct pt_regs *regs);
/*
* Contiguous page definitions.
*/
-#ifdef CONFIG_ARM64_64K_PAGES
-#define CONT_PTE_SHIFT (5 + PAGE_SHIFT)
-#define CONT_PMD_SHIFT (5 + PMD_SHIFT)
-#elif defined(CONFIG_ARM64_16K_PAGES)
-#define CONT_PTE_SHIFT (7 + PAGE_SHIFT)
-#define CONT_PMD_SHIFT (5 + PMD_SHIFT)
-#else
-#define CONT_PTE_SHIFT (4 + PAGE_SHIFT)
-#define CONT_PMD_SHIFT (4 + PMD_SHIFT)
-#endif
-
+#define CONT_PTE_SHIFT (CONFIG_ARM64_CONT_PTE_SHIFT + PAGE_SHIFT)
#define CONT_PTES (1 << (CONT_PTE_SHIFT - PAGE_SHIFT))
#define CONT_PTE_SIZE (CONT_PTES * PAGE_SIZE)
#define CONT_PTE_MASK (~(CONT_PTE_SIZE - 1))
+
+#define CONT_PMD_SHIFT (CONFIG_ARM64_CONT_PMD_SHIFT + PMD_SHIFT)
#define CONT_PMDS (1 << (CONT_PMD_SHIFT - PMD_SHIFT))
#define CONT_PMD_SIZE (CONT_PMDS * PMD_SIZE)
#define CONT_PMD_MASK (~(CONT_PMD_SIZE - 1))
-/* the numerical offset of the PTE within a range of CONT_PTES */
-#define CONT_RANGE_OFFSET(addr) (((addr)>>PAGE_SHIFT)&(CONT_PTES-1))
/*
* Hardware page table definitions.
#define PTE_DEVMAP (_AT(pteval_t, 1) << 57)
#define PTE_PROT_NONE (_AT(pteval_t, 1) << 58) /* only when !PTE_VALID */
+/*
+ * This bit indicates that the entry is present i.e. pmd_page()
+ * still points to a valid huge page in memory even if the pmd
+ * has been invalidated.
+ */
+#define PMD_PRESENT_INVALID (_AT(pteval_t, 1) << 59) /* only when !PMD_SECT_VALID */
+
#ifndef __ASSEMBLY__
#include <asm/cpufeature.h>
#define PROT_NORMAL_NC (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_NC))
#define PROT_NORMAL_WT (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_WT))
#define PROT_NORMAL (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL))
+#define PROT_NORMAL_TAGGED (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_TAGGED))
#define PROT_SECT_DEVICE_nGnRE (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_DEVICE_nGnRE))
#define PROT_SECT_NORMAL (PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
#define _HYP_PAGE_DEFAULT _PAGE_DEFAULT
#define PAGE_KERNEL __pgprot(PROT_NORMAL)
+#define PAGE_KERNEL_TAGGED __pgprot(PROT_NORMAL_TAGGED)
#define PAGE_KERNEL_RO __pgprot((PROT_NORMAL & ~PTE_WRITE) | PTE_RDONLY)
#define PAGE_KERNEL_ROX __pgprot((PROT_NORMAL & ~(PTE_WRITE | PTE_PXN)) | PTE_RDONLY)
#define PAGE_KERNEL_EXEC __pgprot(PROT_NORMAL & ~PTE_PXN)
#include <asm/proc-fns.h>
#include <asm/memory.h>
+#include <asm/mte.h>
#include <asm/pgtable-hwdef.h>
#include <asm/pgtable-prot.h>
#include <asm/tlbflush.h>
extern struct page *vmemmap;
-extern void __pte_error(const char *file, int line, unsigned long val);
-extern void __pmd_error(const char *file, int line, unsigned long val);
-extern void __pud_error(const char *file, int line, unsigned long val);
-extern void __pgd_error(const char *file, int line, unsigned long val);
-
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
__flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1)
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+/*
+ * Outside of a few very special situations (e.g. hibernation), we always
+ * use broadcast TLB invalidation instructions, therefore a spurious page
+ * fault on one CPU which has been handled concurrently by another CPU
+ * does not need to perform additional invalidation.
+ */
+#define flush_tlb_fix_spurious_fault(vma, address) do { } while (0)
+
/*
* ZERO_PAGE is a global shared page that is always zero: used
* for zero-mapped memory areas etc..
extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
#define ZERO_PAGE(vaddr) phys_to_page(__pa_symbol(empty_zero_page))
-#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte))
+#define pte_ERROR(e) \
+ pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e))
/*
* Macros to convert between a physical address and its placement in a
#define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN))
#define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT))
#define pte_devmap(pte) (!!(pte_val(pte) & PTE_DEVMAP))
+#define pte_tagged(pte) ((pte_val(pte) & PTE_ATTRINDX_MASK) == \
+ PTE_ATTRINDX(MT_NORMAL_TAGGED))
#define pte_cont_addr_end(addr, end) \
({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \
return pte;
}
+static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot)
+{
+ pmd_val(pmd) &= ~pgprot_val(prot);
+ return pmd;
+}
+
+static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot)
+{
+ pmd_val(pmd) |= pgprot_val(prot);
+ return pmd;
+}
+
static inline pte_t pte_wrprotect(pte_t pte)
{
pte = clear_pte_bit(pte, __pgprot(PTE_WRITE));
if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))
__sync_icache_dcache(pte);
+ if (system_supports_mte() &&
+ pte_present(pte) && pte_tagged(pte) && !pte_special(pte))
+ mte_sync_tags(ptep, pte);
+
__check_racy_pte_update(mm, ptep, pte);
set_pte(ptep, pte);
}
#endif
+#define pmd_present_invalid(pmd) (!!(pmd_val(pmd) & PMD_PRESENT_INVALID))
+
+static inline int pmd_present(pmd_t pmd)
+{
+ return pte_present(pmd_pte(pmd)) || pmd_present_invalid(pmd);
+}
+
/*
* THP definitions.
*/
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-#define pmd_trans_huge(pmd) (pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT))
+static inline int pmd_trans_huge(pmd_t pmd)
+{
+ return pmd_val(pmd) && pmd_present(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
+}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
-#define pmd_present(pmd) pte_present(pmd_pte(pmd))
#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
#define pmd_young(pmd) pte_young(pmd_pte(pmd))
#define pmd_valid(pmd) pte_valid(pmd_pte(pmd))
#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
-#define pmd_mkinvalid(pmd) (__pmd(pmd_val(pmd) & ~PMD_SECT_VALID))
+
+static inline pmd_t pmd_mkinvalid(pmd_t pmd)
+{
+ pmd = set_pmd_bit(pmd, __pgprot(PMD_PRESENT_INVALID));
+ pmd = clear_pmd_bit(pmd, __pgprot(PMD_SECT_VALID));
+
+ return pmd;
+}
#define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd))
#if CONFIG_PGTABLE_LEVELS > 2
-#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd_val(pmd))
+#define pmd_ERROR(e) \
+ pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e))
#define pud_none(pud) (!pud_val(pud))
#define pud_bad(pud) (!(pud_val(pud) & PUD_TABLE_BIT))
#if CONFIG_PGTABLE_LEVELS > 3
-#define pud_ERROR(pud) __pud_error(__FILE__, __LINE__, pud_val(pud))
+#define pud_ERROR(e) \
+ pr_err("%s:%d: bad pud %016llx.\n", __FILE__, __LINE__, pud_val(e))
#define p4d_none(p4d) (!p4d_val(p4d))
#define p4d_bad(p4d) (!(p4d_val(p4d) & 2))
#endif /* CONFIG_PGTABLE_LEVELS > 3 */
-#define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd))
+#define pgd_ERROR(e) \
+ pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e))
#define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
#define pgd_clear_fixmap() clear_fixmap(FIX_PGD)
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
+ /*
+ * Normal and Normal-Tagged are two different memory types and indices
+ * in MAIR_EL1. The mask below has to include PTE_ATTRINDX_MASK.
+ */
const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
- PTE_PROT_NONE | PTE_VALID | PTE_WRITE | PTE_GP;
+ PTE_PROT_NONE | PTE_VALID | PTE_WRITE | PTE_GP |
+ PTE_ATTRINDX_MASK;
/* preserve the hardware dirty information */
if (pte_hw_dirty(pte))
pte = pte_mkdirty(pte);
#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
#define __swp_entry_to_pte(swp) ((pte_t) { (swp).val })
+#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
+#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) })
+#define __swp_entry_to_pmd(swp) __pmd((swp).val)
+#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
+
/*
* Ensure that there are not more swap files than can be encoded in the kernel
* PTEs.
extern int kern_addr_valid(unsigned long addr);
+#ifdef CONFIG_ARM64_MTE
+
+#define __HAVE_ARCH_PREPARE_TO_SWAP
+static inline int arch_prepare_to_swap(struct page *page)
+{
+ if (system_supports_mte())
+ return mte_save_tags(page);
+ return 0;
+}
+
+#define __HAVE_ARCH_SWAP_INVALIDATE
+static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
+{
+ if (system_supports_mte())
+ mte_invalidate_tags(type, offset);
+}
+
+static inline void arch_swap_invalidate_area(int type)
+{
+ if (system_supports_mte())
+ mte_invalidate_tags_area(type);
+}
+
+#define __HAVE_ARCH_SWAP_RESTORE
+static inline void arch_swap_restore(swp_entry_t entry, struct page *page)
+{
+ if (system_supports_mte() && mte_restore_tags(entry, page))
+ set_bit(PG_mte_tagged, &page->flags);
+}
+
+#endif /* CONFIG_ARM64_MTE */
+
/*
* On AArch64, the cache coherency is handled via the set_pte_at() function.
*/
#include <asm/pgtable-hwdef.h>
#include <asm/pointer_auth.h>
#include <asm/ptrace.h>
+#include <asm/spectre.h>
#include <asm/types.h>
/*
struct ptrauth_keys_user keys_user;
struct ptrauth_keys_kernel keys_kernel;
#endif
+#ifdef CONFIG_ARM64_MTE
+ u64 sctlr_tcf0;
+ u64 gcr_user_incl;
+#endif
};
static inline void arch_thread_struct_whitelist(unsigned long *offset,
regs->pmr_save = GIC_PRIO_IRQON;
}
-static inline void set_ssbs_bit(struct pt_regs *regs)
-{
- regs->pstate |= PSR_SSBS_BIT;
-}
-
-static inline void set_compat_ssbs_bit(struct pt_regs *regs)
-{
- regs->pstate |= PSR_AA32_SSBS_BIT;
-}
-
static inline void start_thread(struct pt_regs *regs, unsigned long pc,
unsigned long sp)
{
start_thread_common(regs, pc);
regs->pstate = PSR_MODE_EL0t;
-
- if (arm64_get_ssbd_state() != ARM64_SSBD_FORCE_ENABLE)
- set_ssbs_bit(regs);
-
+ spectre_v4_enable_task_mitigation(current);
regs->sp = sp;
}
-static inline bool is_ttbr0_addr(unsigned long addr)
-{
- /* entry assembly clears tags for TTBR0 addrs */
- return addr < TASK_SIZE;
-}
-
-static inline bool is_ttbr1_addr(unsigned long addr)
-{
- /* TTBR1 addresses may have a tag if KASAN_SW_TAGS is in use */
- return arch_kasan_reset_tag(addr) >= PAGE_OFFSET;
-}
-
#ifdef CONFIG_COMPAT
static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc,
unsigned long sp)
regs->pstate |= PSR_AA32_E_BIT;
#endif
- if (arm64_get_ssbd_state() != ARM64_SSBD_FORCE_ENABLE)
- set_compat_ssbs_bit(regs);
-
+ spectre_v4_enable_task_mitigation(current);
regs->compat_sp = sp;
}
#endif
+static inline bool is_ttbr0_addr(unsigned long addr)
+{
+ /* entry assembly clears tags for TTBR0 addrs */
+ return addr < TASK_SIZE;
+}
+
+static inline bool is_ttbr1_addr(unsigned long addr)
+{
+ /* TTBR1 addresses may have a tag if KASAN_SW_TAGS is in use */
+ return arch_kasan_reset_tag(addr) >= PAGE_OFFSET;
+}
+
/* Forward declaration, a strange C thing */
struct task_struct;
#ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
/* PR_{SET,GET}_TAGGED_ADDR_CTRL prctl */
-long set_tagged_addr_ctrl(unsigned long arg);
-long get_tagged_addr_ctrl(void);
-#define SET_TAGGED_ADDR_CTRL(arg) set_tagged_addr_ctrl(arg)
-#define GET_TAGGED_ADDR_CTRL() get_tagged_addr_ctrl()
+long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg);
+long get_tagged_addr_ctrl(struct task_struct *task);
+#define SET_TAGGED_ADDR_CTRL(arg) set_tagged_addr_ctrl(current, arg)
+#define GET_TAGGED_ADDR_CTRL() get_tagged_addr_ctrl(current)
#endif
/*
* interrupt disabling temporarily does not rely on IRQ priorities.
*/
#define GIC_PRIO_IRQON 0xe0
-#define GIC_PRIO_IRQOFF (GIC_PRIO_IRQON & ~0x80)
+#define __GIC_PRIO_IRQOFF (GIC_PRIO_IRQON & ~0x80)
+#define __GIC_PRIO_IRQOFF_NS 0xa0
#define GIC_PRIO_PSR_I_SET (1 << 4)
+#define GIC_PRIO_IRQOFF \
+ ({ \
+ extern struct static_key_false gic_nonsecure_priorities;\
+ u8 __prio = __GIC_PRIO_IRQOFF; \
+ \
+ if (static_branch_unlikely(&gic_nonsecure_priorities)) \
+ __prio = __GIC_PRIO_IRQOFF_NS; \
+ \
+ __prio; \
+ })
+
/* Additional SPSR bits not exposed in the UABI */
#define PSR_MODE_THREAD_BIT (1 << 0)
#define PSR_IL_BIT (1 << 20)
struct seq_file;
-/*
- * generate IPI list text
- */
-extern void show_ipi_list(struct seq_file *p, int prec);
-
-/*
- * Called from C code, this handles an IPI.
- */
-extern void handle_IPI(int ipinr, struct pt_regs *regs);
-
/*
* Discover the set of possible CPUs and determine their
* SMP operations.
extern void smp_init_cpus(void);
/*
- * Provide a function to raise an IPI cross call on CPUs in callmap.
+ * Register IPI interrupts with the arch SMP code
*/
-extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
-
-extern void (*__smp_cross_call)(const struct cpumask *, unsigned int);
+extern void set_smp_ipi_range(int ipi_base, int nr_ipi);
/*
* Called from the secondary holding pen, this is the secondary CPU entry point.
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Interface for managing mitigations for Spectre vulnerabilities.
+ *
+ * Copyright (C) 2020 Google LLC
+ * Author: Will Deacon <will@kernel.org>
+ */
+
+#ifndef __ASM_SPECTRE_H
+#define __ASM_SPECTRE_H
+
+#include <asm/cpufeature.h>
+
+/* Watch out, ordering is important here. */
+enum mitigation_state {
+ SPECTRE_UNAFFECTED,
+ SPECTRE_MITIGATED,
+ SPECTRE_VULNERABLE,
+};
+
+struct task_struct;
+
+enum mitigation_state arm64_get_spectre_v2_state(void);
+bool has_spectre_v2(const struct arm64_cpu_capabilities *cap, int scope);
+void spectre_v2_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
+
+enum mitigation_state arm64_get_spectre_v4_state(void);
+bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope);
+void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused);
+void spectre_v4_enable_task_mitigation(struct task_struct *tsk);
+
+#endif /* __ASM_SPECTRE_H */
extern int unwind_frame(struct task_struct *tsk, struct stackframe *frame);
extern void walk_stackframe(struct task_struct *tsk, struct stackframe *frame,
- int (*fn)(struct stackframe *, void *), void *data);
+ bool (*fn)(void *, unsigned long), void *data);
extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
const char *loglvl);
#define PSTATE_PAN pstate_field(0, 4)
#define PSTATE_UAO pstate_field(0, 3)
#define PSTATE_SSBS pstate_field(3, 1)
+#define PSTATE_TCO pstate_field(3, 4)
#define SET_PSTATE_PAN(x) __emit_inst(0xd500401f | PSTATE_PAN | ((!!x) << PSTATE_Imm_shift))
#define SET_PSTATE_UAO(x) __emit_inst(0xd500401f | PSTATE_UAO | ((!!x) << PSTATE_Imm_shift))
#define SET_PSTATE_SSBS(x) __emit_inst(0xd500401f | PSTATE_SSBS | ((!!x) << PSTATE_Imm_shift))
+#define SET_PSTATE_TCO(x) __emit_inst(0xd500401f | PSTATE_TCO | ((!!x) << PSTATE_Imm_shift))
#define __SYS_BARRIER_INSN(CRm, op2, Rt) \
__emit_inst(0xd5000000 | sys_insn(0, 3, 3, (CRm), (op2)) | ((Rt) & 0x1f))
#define SYS_SCTLR_EL1 sys_reg(3, 0, 1, 0, 0)
#define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1)
#define SYS_CPACR_EL1 sys_reg(3, 0, 1, 0, 2)
+#define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5)
+#define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6)
#define SYS_ZCR_EL1 sys_reg(3, 0, 1, 2, 0)
#define SYS_ERXADDR_EL1 sys_reg(3, 0, 5, 4, 3)
#define SYS_ERXMISC0_EL1 sys_reg(3, 0, 5, 5, 0)
#define SYS_ERXMISC1_EL1 sys_reg(3, 0, 5, 5, 1)
+#define SYS_TFSR_EL1 sys_reg(3, 0, 5, 6, 0)
+#define SYS_TFSRE0_EL1 sys_reg(3, 0, 5, 6, 1)
#define SYS_FAR_EL1 sys_reg(3, 0, 6, 0, 0)
#define SYS_PAR_EL1 sys_reg(3, 0, 7, 4, 0)
#define SYS_PMINTENSET_EL1 sys_reg(3, 0, 9, 14, 1)
#define SYS_PMINTENCLR_EL1 sys_reg(3, 0, 9, 14, 2)
+#define SYS_PMMIR_EL1 sys_reg(3, 0, 9, 14, 6)
+
#define SYS_MAIR_EL1 sys_reg(3, 0, 10, 2, 0)
#define SYS_AMAIR_EL1 sys_reg(3, 0, 10, 3, 0)
#define SYS_CCSIDR_EL1 sys_reg(3, 1, 0, 0, 0)
#define SYS_CLIDR_EL1 sys_reg(3, 1, 0, 0, 1)
+#define SYS_GMID_EL1 sys_reg(3, 1, 0, 0, 4)
#define SYS_AIDR_EL1 sys_reg(3, 1, 0, 0, 7)
#define SYS_CSSELR_EL1 sys_reg(3, 2, 0, 0, 0)
#define SYS_ESR_EL2 sys_reg(3, 4, 5, 2, 0)
#define SYS_VSESR_EL2 sys_reg(3, 4, 5, 2, 3)
#define SYS_FPEXC32_EL2 sys_reg(3, 4, 5, 3, 0)
+#define SYS_TFSR_EL2 sys_reg(3, 4, 5, 6, 0)
#define SYS_FAR_EL2 sys_reg(3, 4, 6, 0, 0)
#define SYS_VDISR_EL2 sys_reg(3, 4, 12, 1, 1)
#define SYS_AFSR0_EL12 sys_reg(3, 5, 5, 1, 0)
#define SYS_AFSR1_EL12 sys_reg(3, 5, 5, 1, 1)
#define SYS_ESR_EL12 sys_reg(3, 5, 5, 2, 0)
+#define SYS_TFSR_EL12 sys_reg(3, 5, 5, 6, 0)
#define SYS_FAR_EL12 sys_reg(3, 5, 6, 0, 0)
#define SYS_MAIR_EL12 sys_reg(3, 5, 10, 2, 0)
#define SYS_AMAIR_EL12 sys_reg(3, 5, 10, 3, 0)
/* Common SCTLR_ELx flags. */
#define SCTLR_ELx_DSSBS (BIT(44))
+#define SCTLR_ELx_ATA (BIT(43))
+
+#define SCTLR_ELx_TCF_SHIFT 40
+#define SCTLR_ELx_TCF_NONE (UL(0x0) << SCTLR_ELx_TCF_SHIFT)
+#define SCTLR_ELx_TCF_SYNC (UL(0x1) << SCTLR_ELx_TCF_SHIFT)
+#define SCTLR_ELx_TCF_ASYNC (UL(0x2) << SCTLR_ELx_TCF_SHIFT)
+#define SCTLR_ELx_TCF_MASK (UL(0x3) << SCTLR_ELx_TCF_SHIFT)
+
+#define SCTLR_ELx_ITFSB (BIT(37))
#define SCTLR_ELx_ENIA (BIT(31))
#define SCTLR_ELx_ENIB (BIT(30))
#define SCTLR_ELx_ENDA (BIT(27))
#endif
/* SCTLR_EL1 specific flags. */
+#define SCTLR_EL1_ATA0 (BIT(42))
+
+#define SCTLR_EL1_TCF0_SHIFT 38
+#define SCTLR_EL1_TCF0_NONE (UL(0x0) << SCTLR_EL1_TCF0_SHIFT)
+#define SCTLR_EL1_TCF0_SYNC (UL(0x1) << SCTLR_EL1_TCF0_SHIFT)
+#define SCTLR_EL1_TCF0_ASYNC (UL(0x2) << SCTLR_EL1_TCF0_SHIFT)
+#define SCTLR_EL1_TCF0_MASK (UL(0x3) << SCTLR_EL1_TCF0_SHIFT)
+
#define SCTLR_EL1_BT1 (BIT(36))
#define SCTLR_EL1_BT0 (BIT(35))
#define SCTLR_EL1_UCI (BIT(26))
SCTLR_EL1_SA0 | SCTLR_EL1_SED | SCTLR_ELx_I |\
SCTLR_EL1_DZE | SCTLR_EL1_UCT |\
SCTLR_EL1_NTWE | SCTLR_ELx_IESB | SCTLR_EL1_SPAN |\
+ SCTLR_ELx_ITFSB| SCTLR_ELx_ATA | SCTLR_EL1_ATA0 |\
ENDIAN_SET_EL1 | SCTLR_EL1_UCI | SCTLR_EL1_RES1)
/* MAIR_ELx memory attributes (used by Linux) */
#define MAIR_ATTR_DEVICE_GRE UL(0x0c)
#define MAIR_ATTR_NORMAL_NC UL(0x44)
#define MAIR_ATTR_NORMAL_WT UL(0xbb)
+#define MAIR_ATTR_NORMAL_TAGGED UL(0xf0)
#define MAIR_ATTR_NORMAL UL(0xff)
#define MAIR_ATTR_MASK UL(0xff)
#define ID_AA64ISAR1_APA_SHIFT 4
#define ID_AA64ISAR1_DPB_SHIFT 0
-#define ID_AA64ISAR1_APA_NI 0x0
-#define ID_AA64ISAR1_APA_ARCHITECTED 0x1
-#define ID_AA64ISAR1_API_NI 0x0
-#define ID_AA64ISAR1_API_IMP_DEF 0x1
-#define ID_AA64ISAR1_GPA_NI 0x0
-#define ID_AA64ISAR1_GPA_ARCHITECTED 0x1
-#define ID_AA64ISAR1_GPI_NI 0x0
-#define ID_AA64ISAR1_GPI_IMP_DEF 0x1
+#define ID_AA64ISAR1_APA_NI 0x0
+#define ID_AA64ISAR1_APA_ARCHITECTED 0x1
+#define ID_AA64ISAR1_APA_ARCH_EPAC 0x2
+#define ID_AA64ISAR1_APA_ARCH_EPAC2 0x3
+#define ID_AA64ISAR1_APA_ARCH_EPAC2_FPAC 0x4
+#define ID_AA64ISAR1_APA_ARCH_EPAC2_FPAC_CMB 0x5
+#define ID_AA64ISAR1_API_NI 0x0
+#define ID_AA64ISAR1_API_IMP_DEF 0x1
+#define ID_AA64ISAR1_API_IMP_DEF_EPAC 0x2
+#define ID_AA64ISAR1_API_IMP_DEF_EPAC2 0x3
+#define ID_AA64ISAR1_API_IMP_DEF_EPAC2_FPAC 0x4
+#define ID_AA64ISAR1_API_IMP_DEF_EPAC2_FPAC_CMB 0x5
+#define ID_AA64ISAR1_GPA_NI 0x0
+#define ID_AA64ISAR1_GPA_ARCHITECTED 0x1
+#define ID_AA64ISAR1_GPI_NI 0x0
+#define ID_AA64ISAR1_GPI_IMP_DEF 0x1
/* id_aa64pfr0 */
#define ID_AA64PFR0_CSV3_SHIFT 60
#define ID_AA64PFR1_SSBS_PSTATE_INSNS 2
#define ID_AA64PFR1_BT_BTI 0x1
+#define ID_AA64PFR1_MTE_NI 0x0
+#define ID_AA64PFR1_MTE_EL0 0x1
+#define ID_AA64PFR1_MTE 0x2
+
/* id_aa64zfr0 */
#define ID_AA64ZFR0_F64MM_SHIFT 56
#define ID_AA64ZFR0_F32MM_SHIFT 52
#define CPACR_EL1_ZEN_EL0EN (BIT(17)) /* enable EL0 access, if EL1EN set */
#define CPACR_EL1_ZEN (CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN)
+/* TCR EL1 Bit Definitions */
+#define SYS_TCR_EL1_TCMA1 (BIT(58))
+#define SYS_TCR_EL1_TCMA0 (BIT(57))
+
+/* GCR_EL1 Definitions */
+#define SYS_GCR_EL1_RRND (BIT(16))
+#define SYS_GCR_EL1_EXCL_MASK 0xffffUL
+
+/* RGSR_EL1 Definitions */
+#define SYS_RGSR_EL1_TAG_MASK 0xfUL
+#define SYS_RGSR_EL1_SEED_SHIFT 8
+#define SYS_RGSR_EL1_SEED_MASK 0xffffUL
+
+/* GMID_EL1 field definitions */
+#define SYS_GMID_EL1_BS_SHIFT 0
+#define SYS_GMID_EL1_BS_SIZE 4
+
+/* TFSR{,E0}_EL1 bit definitions */
+#define SYS_TFSR_EL1_TF0_SHIFT 0
+#define SYS_TFSR_EL1_TF1_SHIFT 1
+#define SYS_TFSR_EL1_TF0 (UL(1) << SYS_TFSR_EL1_TF0_SHIFT)
+#define SYS_TFSR_EL1_TF1 (UK(2) << SYS_TFSR_EL1_TF1_SHIFT)
/* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */
#define SYS_MPIDR_SAFE_VAL (BIT(31))
write_sysreg(__scs_new, sysreg); \
} while (0)
+#define sysreg_clear_set_s(sysreg, clear, set) do { \
+ u64 __scs_val = read_sysreg_s(sysreg); \
+ u64 __scs_new = (__scs_val & ~(u64)(clear)) | (set); \
+ if (__scs_new != __scs_val) \
+ write_sysreg_s(__scs_new, sysreg); \
+} while (0)
+
#endif
#endif /* __ASM_SYSREG_H */
#define TIF_FOREIGN_FPSTATE 3 /* CPU's FP state is not current's */
#define TIF_UPROBE 4 /* uprobe breakpoint or singlestep */
#define TIF_FSCHECK 5 /* Check FS is USER_DS on return */
+#define TIF_MTE_ASYNC_FAULT 6 /* MTE Asynchronous Tag Check Fault */
#define TIF_SYSCALL_TRACE 8 /* syscall trace active */
#define TIF_SYSCALL_AUDIT 9 /* syscall auditing */
#define TIF_SYSCALL_TRACEPOINT 10 /* syscall tracepoint for ftrace */
#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)
#define _TIF_32BIT (1 << TIF_32BIT)
#define _TIF_SVE (1 << TIF_SVE)
+#define _TIF_MTE_ASYNC_FAULT (1 << TIF_MTE_ASYNC_FAULT)
#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
_TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
- _TIF_UPROBE | _TIF_FSCHECK)
+ _TIF_UPROBE | _TIF_FSCHECK | _TIF_MTE_ASYNC_FAULT)
#define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
_TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
void register_undef_hook(struct undef_hook *hook);
void unregister_undef_hook(struct undef_hook *hook);
-void force_signal_inject(int signal, int code, unsigned long address);
+void force_signal_inject(int signal, int code, unsigned long address, unsigned int err);
void arm64_notify_segfault(unsigned long addr);
void arm64_force_sig_fault(int signo, int code, void __user *addr, const char *str);
void arm64_force_sig_mceerr(int code, void __user *addr, short lsb, const char *str);
#define __NR_getpid 20
__SYSCALL(__NR_getpid, sys_getpid)
#define __NR_mount 21
-__SYSCALL(__NR_mount, compat_sys_mount)
+__SYSCALL(__NR_mount, sys_mount)
/* 22 was sys_umount */
__SYSCALL(22, sys_ni_syscall)
#define __NR_setuid 23
#define __NR_msync 144
__SYSCALL(__NR_msync, sys_msync)
#define __NR_readv 145
-__SYSCALL(__NR_readv, compat_sys_readv)
+__SYSCALL(__NR_readv, sys_readv)
#define __NR_writev 146
-__SYSCALL(__NR_writev, compat_sys_writev)
+__SYSCALL(__NR_writev, sys_writev)
#define __NR_getsid 147
__SYSCALL(__NR_getsid, sys_getsid)
#define __NR_fdatasync 148
#define __NR_tee 342
__SYSCALL(__NR_tee, sys_tee)
#define __NR_vmsplice 343
-__SYSCALL(__NR_vmsplice, compat_sys_vmsplice)
+__SYSCALL(__NR_vmsplice, sys_vmsplice)
#define __NR_move_pages 344
__SYSCALL(__NR_move_pages, compat_sys_move_pages)
#define __NR_getcpu 345
#define __NR_setns 375
__SYSCALL(__NR_setns, sys_setns)
#define __NR_process_vm_readv 376
-__SYSCALL(__NR_process_vm_readv, compat_sys_process_vm_readv)
+__SYSCALL(__NR_process_vm_readv, sys_process_vm_readv)
#define __NR_process_vm_writev 377
-__SYSCALL(__NR_process_vm_writev, compat_sys_process_vm_writev)
+__SYSCALL(__NR_process_vm_writev, sys_process_vm_writev)
#define __NR_kcmp 378
__SYSCALL(__NR_kcmp, sys_kcmp)
#define __NR_finit_module 379
#define HWCAP2_DGH (1 << 15)
#define HWCAP2_RNG (1 << 16)
#define HWCAP2_BTI (1 << 17)
-/* reserved for HWCAP2_MTE (1 << 18) */
+#define HWCAP2_MTE (1 << 18)
#endif /* _UAPI__ASM_HWCAP_H */
#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL 0
#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_AVAIL 1
#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_REQUIRED 2
+
+/*
+ * Only two states can be presented by the host kernel:
+ * - NOT_REQUIRED: the guest doesn't need to do anything
+ * - NOT_AVAIL: the guest isn't mitigated (it can still use SSBS if available)
+ *
+ * All the other values are deprecated. The host still accepts all
+ * values (they are ABI), but will narrow them to the above two.
+ */
#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2 KVM_REG_ARM_FW_REG(2)
#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL 0
#define KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_UNKNOWN 1
#include <asm-generic/mman.h>
#define PROT_BTI 0x10 /* BTI guarded page */
+#define PROT_MTE 0x20 /* Normal Tagged mapping */
#endif /* ! _UAPI__ASM_MMAN_H */
#define PSR_PAN_BIT 0x00400000
#define PSR_UAO_BIT 0x00800000
#define PSR_DIT_BIT 0x01000000
+#define PSR_TCO_BIT 0x02000000
#define PSR_V_BIT 0x10000000
#define PSR_C_BIT 0x20000000
#define PSR_Z_BIT 0x40000000
/* syscall emulation path in ptrace */
#define PTRACE_SYSEMU 31
#define PTRACE_SYSEMU_SINGLESTEP 32
+/* MTE allocation tag access */
+#define PTRACE_PEEKMTETAGS 33
+#define PTRACE_POKEMTETAGS 34
#ifndef __ASSEMBLY__
# Makefile for the linux kernel.
#
-CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET)
-AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)
CFLAGS_armv8_deprecated.o := -I$(src)
CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE)
return_address.o cpuinfo.o cpu_errata.o \
cpufeature.o alternative.o cacheinfo.o \
smp.o smp_spin_table.o topology.o smccc-call.o \
- syscall.o
+ syscall.o proton-pack.o
targets += efi-entry.o
obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
obj-$(CONFIG_CRASH_CORE) += crash_core.o
obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o
-obj-$(CONFIG_ARM64_SSBD) += ssbd.o
obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o
obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
+obj-$(CONFIG_ARM64_MTE) += mte.o
obj-y += vdso/ probes/
obj-$(CONFIG_COMPAT_VDSO) += vdso32/
case EFI_BOOT_SERVICES_DATA:
case EFI_CONVENTIONAL_MEMORY:
case EFI_PERSISTENT_MEMORY:
- pr_warn(FW_BUG "requested region covers kernel memory @ %pa\n", &phys);
- return NULL;
+ if (memblock_is_map_memory(phys) ||
+ !memblock_is_region_memory(phys, size)) {
+ pr_warn(FW_BUG "requested region covers kernel memory @ %pa\n", &phys);
+ return NULL;
+ }
+ /*
+ * Mapping kernel memory is permitted if the region in
+ * question is covered by a single memblock with the
+ * NOMAP attribute set: this enables the use of ACPI
+ * table overrides passed via initramfs, which are
+ * reserved in memory using arch_reserve_mem_area()
+ * below. As this particular use case only requires
+ * read access, fall through to the R/O mapping case.
+ */
+ fallthrough;
case EFI_RUNTIME_SERVICES_CODE:
/*
return err;
}
+
+void arch_reserve_mem_area(acpi_physical_address addr, size_t size)
+{
+ memblock_mark_nomap(addr, size);
+}
mov_q x13, SCTLR_ELx_FLAGS
bic x12, x12, x13
pre_disable_mmu_workaround
+ /*
+ * either disable EL1&0 translation regime or disable EL2&0 translation
+ * regime if HCR_EL2.E2H == 1
+ */
msr sctlr_el1, x12
isb
sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
}
-atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1);
-
-#include <asm/mmu_context.h>
-#include <asm/cacheflush.h>
-
-DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
-
-#ifdef CONFIG_KVM_INDIRECT_VECTORS
-static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
- const char *hyp_vecs_end)
-{
- void *dst = lm_alias(__bp_harden_hyp_vecs + slot * SZ_2K);
- int i;
-
- for (i = 0; i < SZ_2K; i += 0x80)
- memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
-
- __flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
-}
-
-static void install_bp_hardening_cb(bp_hardening_cb_t fn,
- const char *hyp_vecs_start,
- const char *hyp_vecs_end)
-{
- static DEFINE_RAW_SPINLOCK(bp_lock);
- int cpu, slot = -1;
-
- /*
- * detect_harden_bp_fw() passes NULL for the hyp_vecs start/end if
- * we're a guest. Skip the hyp-vectors work.
- */
- if (!hyp_vecs_start) {
- __this_cpu_write(bp_hardening_data.fn, fn);
- return;
- }
-
- raw_spin_lock(&bp_lock);
- for_each_possible_cpu(cpu) {
- if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
- slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
- break;
- }
- }
-
- if (slot == -1) {
- slot = atomic_inc_return(&arm64_el2_vector_last_slot);
- BUG_ON(slot >= BP_HARDEN_EL2_SLOTS);
- __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
- }
-
- __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
- __this_cpu_write(bp_hardening_data.fn, fn);
- raw_spin_unlock(&bp_lock);
-}
-#else
-static void install_bp_hardening_cb(bp_hardening_cb_t fn,
- const char *hyp_vecs_start,
- const char *hyp_vecs_end)
-{
- __this_cpu_write(bp_hardening_data.fn, fn);
-}
-#endif /* CONFIG_KVM_INDIRECT_VECTORS */
-
-#include <linux/arm-smccc.h>
-
-static void __maybe_unused call_smc_arch_workaround_1(void)
-{
- arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
-}
-
-static void call_hvc_arch_workaround_1(void)
-{
- arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
-}
-
-static void qcom_link_stack_sanitization(void)
-{
- u64 tmp;
-
- asm volatile("mov %0, x30 \n"
- ".rept 16 \n"
- "bl . + 4 \n"
- ".endr \n"
- "mov x30, %0 \n"
- : "=&r" (tmp));
-}
-
-static bool __nospectre_v2;
-static int __init parse_nospectre_v2(char *str)
-{
- __nospectre_v2 = true;
- return 0;
-}
-early_param("nospectre_v2", parse_nospectre_v2);
-
-/*
- * -1: No workaround
- * 0: No workaround required
- * 1: Workaround installed
- */
-static int detect_harden_bp_fw(void)
-{
- bp_hardening_cb_t cb;
- void *smccc_start, *smccc_end;
- struct arm_smccc_res res;
- u32 midr = read_cpuid_id();
-
- arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
- ARM_SMCCC_ARCH_WORKAROUND_1, &res);
-
- switch ((int)res.a0) {
- case 1:
- /* Firmware says we're just fine */
- return 0;
- case 0:
- break;
- default:
- return -1;
- }
-
- switch (arm_smccc_1_1_get_conduit()) {
- case SMCCC_CONDUIT_HVC:
- cb = call_hvc_arch_workaround_1;
- /* This is a guest, no need to patch KVM vectors */
- smccc_start = NULL;
- smccc_end = NULL;
- break;
-
-#if IS_ENABLED(CONFIG_KVM)
- case SMCCC_CONDUIT_SMC:
- cb = call_smc_arch_workaround_1;
- smccc_start = __smccc_workaround_1_smc;
- smccc_end = __smccc_workaround_1_smc +
- __SMCCC_WORKAROUND_1_SMC_SZ;
- break;
-#endif
-
- default:
- return -1;
- }
-
- if (((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR) ||
- ((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1))
- cb = qcom_link_stack_sanitization;
-
- if (IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR))
- install_bp_hardening_cb(cb, smccc_start, smccc_end);
-
- return 1;
-}
-
-DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
-
-int ssbd_state __read_mostly = ARM64_SSBD_KERNEL;
-static bool __ssb_safe = true;
-
-static const struct ssbd_options {
- const char *str;
- int state;
-} ssbd_options[] = {
- { "force-on", ARM64_SSBD_FORCE_ENABLE, },
- { "force-off", ARM64_SSBD_FORCE_DISABLE, },
- { "kernel", ARM64_SSBD_KERNEL, },
-};
-
-static int __init ssbd_cfg(char *buf)
-{
- int i;
-
- if (!buf || !buf[0])
- return -EINVAL;
-
- for (i = 0; i < ARRAY_SIZE(ssbd_options); i++) {
- int len = strlen(ssbd_options[i].str);
-
- if (strncmp(buf, ssbd_options[i].str, len))
- continue;
-
- ssbd_state = ssbd_options[i].state;
- return 0;
- }
-
- return -EINVAL;
-}
-early_param("ssbd", ssbd_cfg);
-
-void __init arm64_update_smccc_conduit(struct alt_instr *alt,
- __le32 *origptr, __le32 *updptr,
- int nr_inst)
-{
- u32 insn;
-
- BUG_ON(nr_inst != 1);
-
- switch (arm_smccc_1_1_get_conduit()) {
- case SMCCC_CONDUIT_HVC:
- insn = aarch64_insn_get_hvc_value();
- break;
- case SMCCC_CONDUIT_SMC:
- insn = aarch64_insn_get_smc_value();
- break;
- default:
- return;
- }
-
- *updptr = cpu_to_le32(insn);
-}
-
-void __init arm64_enable_wa2_handling(struct alt_instr *alt,
- __le32 *origptr, __le32 *updptr,
- int nr_inst)
-{
- BUG_ON(nr_inst != 1);
- /*
- * Only allow mitigation on EL1 entry/exit and guest
- * ARCH_WORKAROUND_2 handling if the SSBD state allows it to
- * be flipped.
- */
- if (arm64_get_ssbd_state() == ARM64_SSBD_KERNEL)
- *updptr = cpu_to_le32(aarch64_insn_gen_nop());
-}
-
-void arm64_set_ssbd_mitigation(bool state)
-{
- int conduit;
-
- if (!IS_ENABLED(CONFIG_ARM64_SSBD)) {
- pr_info_once("SSBD disabled by kernel configuration\n");
- return;
- }
-
- if (this_cpu_has_cap(ARM64_SSBS)) {
- if (state)
- asm volatile(SET_PSTATE_SSBS(0));
- else
- asm volatile(SET_PSTATE_SSBS(1));
- return;
- }
-
- conduit = arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, state,
- NULL);
-
- WARN_ON_ONCE(conduit == SMCCC_CONDUIT_NONE);
-}
-
-static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,
- int scope)
-{
- struct arm_smccc_res res;
- bool required = true;
- s32 val;
- bool this_cpu_safe = false;
- int conduit;
-
- WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
-
- if (cpu_mitigations_off())
- ssbd_state = ARM64_SSBD_FORCE_DISABLE;
-
- /* delay setting __ssb_safe until we get a firmware response */
- if (is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list))
- this_cpu_safe = true;
-
- if (this_cpu_has_cap(ARM64_SSBS)) {
- if (!this_cpu_safe)
- __ssb_safe = false;
- required = false;
- goto out_printmsg;
- }
-
- conduit = arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
- ARM_SMCCC_ARCH_WORKAROUND_2, &res);
-
- if (conduit == SMCCC_CONDUIT_NONE) {
- ssbd_state = ARM64_SSBD_UNKNOWN;
- if (!this_cpu_safe)
- __ssb_safe = false;
- return false;
- }
-
- val = (s32)res.a0;
-
- switch (val) {
- case SMCCC_RET_NOT_SUPPORTED:
- ssbd_state = ARM64_SSBD_UNKNOWN;
- if (!this_cpu_safe)
- __ssb_safe = false;
- return false;
-
- /* machines with mixed mitigation requirements must not return this */
- case SMCCC_RET_NOT_REQUIRED:
- pr_info_once("%s mitigation not required\n", entry->desc);
- ssbd_state = ARM64_SSBD_MITIGATED;
- return false;
-
- case SMCCC_RET_SUCCESS:
- __ssb_safe = false;
- required = true;
- break;
-
- case 1: /* Mitigation not required on this CPU */
- required = false;
- break;
-
- default:
- WARN_ON(1);
- if (!this_cpu_safe)
- __ssb_safe = false;
- return false;
- }
-
- switch (ssbd_state) {
- case ARM64_SSBD_FORCE_DISABLE:
- arm64_set_ssbd_mitigation(false);
- required = false;
- break;
-
- case ARM64_SSBD_KERNEL:
- if (required) {
- __this_cpu_write(arm64_ssbd_callback_required, 1);
- arm64_set_ssbd_mitigation(true);
- }
- break;
-
- case ARM64_SSBD_FORCE_ENABLE:
- arm64_set_ssbd_mitigation(true);
- required = true;
- break;
-
- default:
- WARN_ON(1);
- break;
- }
-
-out_printmsg:
- switch (ssbd_state) {
- case ARM64_SSBD_FORCE_DISABLE:
- pr_info_once("%s disabled from command-line\n", entry->desc);
- break;
-
- case ARM64_SSBD_FORCE_ENABLE:
- pr_info_once("%s forced from command-line\n", entry->desc);
- break;
- }
-
- return required;
-}
-
-/* known invulnerable cores */
-static const struct midr_range arm64_ssb_cpus[] = {
- MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
- MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
- MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
- MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
- MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
- MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
- {},
-};
-
#ifdef CONFIG_ARM64_ERRATUM_1463225
DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, \
CAP_MIDR_RANGE_LIST(midr_list)
-/* Track overall mitigation state. We are only mitigated if all cores are ok */
-static bool __hardenbp_enab = true;
-static bool __spectrev2_safe = true;
-
-int get_spectre_v2_workaround_state(void)
-{
- if (__spectrev2_safe)
- return ARM64_BP_HARDEN_NOT_REQUIRED;
-
- if (!__hardenbp_enab)
- return ARM64_BP_HARDEN_UNKNOWN;
-
- return ARM64_BP_HARDEN_WA_NEEDED;
-}
-
-/*
- * List of CPUs that do not need any Spectre-v2 mitigation at all.
- */
-static const struct midr_range spectre_v2_safe_list[] = {
- MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
- MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
- MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
- MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
- MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
- MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
- MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
- { /* sentinel */ }
-};
-
-/*
- * Track overall bp hardening for all heterogeneous cores in the machine.
- * We are only considered "safe" if all booted cores are known safe.
- */
-static bool __maybe_unused
-check_branch_predictor(const struct arm64_cpu_capabilities *entry, int scope)
-{
- int need_wa;
-
- WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
-
- /* If the CPU has CSV2 set, we're safe */
- if (cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64PFR0_EL1),
- ID_AA64PFR0_CSV2_SHIFT))
- return false;
-
- /* Alternatively, we have a list of unaffected CPUs */
- if (is_midr_in_range_list(read_cpuid_id(), spectre_v2_safe_list))
- return false;
-
- /* Fallback to firmware detection */
- need_wa = detect_harden_bp_fw();
- if (!need_wa)
- return false;
-
- __spectrev2_safe = false;
-
- if (!IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR)) {
- pr_warn_once("spectrev2 mitigation disabled by kernel configuration\n");
- __hardenbp_enab = false;
- return false;
- }
-
- /* forced off */
- if (__nospectre_v2 || cpu_mitigations_off()) {
- pr_info_once("spectrev2 mitigation disabled by command line option\n");
- __hardenbp_enab = false;
- return false;
- }
-
- if (need_wa < 0) {
- pr_warn_once("ARM_SMCCC_ARCH_WORKAROUND_1 missing from firmware\n");
- __hardenbp_enab = false;
- }
-
- return (need_wa > 0);
-}
-
static const __maybe_unused struct midr_range tx2_family_cpus[] = {
MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
},
#endif
{
- .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+ .desc = "Spectre-v2",
+ .capability = ARM64_SPECTRE_V2,
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
- .matches = check_branch_predictor,
+ .matches = has_spectre_v2,
+ .cpu_enable = spectre_v2_enable_mitigation,
},
#ifdef CONFIG_RANDOMIZE_BASE
{
},
#endif
{
- .desc = "Speculative Store Bypass Disable",
- .capability = ARM64_SSBD,
+ .desc = "Spectre-v4",
+ .capability = ARM64_SPECTRE_V4,
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
- .matches = has_ssbd_mitigation,
- .midr_range_list = arm64_ssb_cpus,
+ .matches = has_spectre_v4,
+ .cpu_enable = spectre_v4_enable_mitigation,
},
#ifdef CONFIG_ARM64_ERRATUM_1418040
{
{
}
};
-
-ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
- char *buf)
-{
- return sprintf(buf, "Mitigation: __user pointer sanitization\n");
-}
-
-ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
- char *buf)
-{
- switch (get_spectre_v2_workaround_state()) {
- case ARM64_BP_HARDEN_NOT_REQUIRED:
- return sprintf(buf, "Not affected\n");
- case ARM64_BP_HARDEN_WA_NEEDED:
- return sprintf(buf, "Mitigation: Branch predictor hardening\n");
- case ARM64_BP_HARDEN_UNKNOWN:
- default:
- return sprintf(buf, "Vulnerable\n");
- }
-}
-
-ssize_t cpu_show_spec_store_bypass(struct device *dev,
- struct device_attribute *attr, char *buf)
-{
- if (__ssb_safe)
- return sprintf(buf, "Not affected\n");
-
- switch (ssbd_state) {
- case ARM64_SSBD_KERNEL:
- case ARM64_SSBD_FORCE_ENABLE:
- if (IS_ENABLED(CONFIG_ARM64_SSBD))
- return sprintf(buf,
- "Mitigation: Speculative Store Bypass disabled via prctl\n");
- }
-
- return sprintf(buf, "Vulnerable\n");
-}
#include <asm/cpu_ops.h>
#include <asm/fpsimd.h>
#include <asm/mmu_context.h>
+#include <asm/mte.h>
#include <asm/processor.h>
#include <asm/sysreg.h>
#include <asm/traps.h>
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
- FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_API_SHIFT, 4, 0),
+ FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_API_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
- FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_APA_SHIFT, 4, 0),
+ FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_APA_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
ARM64_FTR_END,
};
static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_MPAMFRAC_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_RASFRAC_SHIFT, 4, 0),
- ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SSBS_SHIFT, 4, ID_AA64PFR1_SSBS_PSTATE_NI),
+ ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_MTE),
+ FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_MTE_SHIFT, 4, ID_AA64PFR1_MTE_NI),
+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SSBS_SHIFT, 4, ID_AA64PFR1_SSBS_PSTATE_NI),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_BTI),
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_BT_SHIFT, 4, 0),
ARM64_FTR_END,
};
static const struct arm64_ftr_bits ftr_id_pfr2[] = {
- ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR2_SSBS_SHIFT, 4, 0),
+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_SSBS_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_CSV3_SHIFT, 4, 0),
ARM64_FTR_END,
};
return 0;
return regp->sys_val;
}
+EXPORT_SYMBOL_GPL(read_sanitised_ftr_reg);
#define read_sysreg_case(r) \
case r: return read_sysreg_s(r)
write_sysreg(tcr, tcr_el1);
isb();
+ local_flush_tlb_all();
}
static bool cpu_has_broken_dbm(void)
WARN_ON(val & (7 << 27 | 7 << 21));
}
-#ifdef CONFIG_ARM64_SSBD
-static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
-{
- if (user_mode(regs))
- return 1;
-
- if (instr & BIT(PSTATE_Imm_shift))
- regs->pstate |= PSR_SSBS_BIT;
- else
- regs->pstate &= ~PSR_SSBS_BIT;
-
- arm64_skip_faulting_instruction(regs, 4);
- return 0;
-}
-
-static struct undef_hook ssbs_emulation_hook = {
- .instr_mask = ~(1U << PSTATE_Imm_shift),
- .instr_val = 0xd500401f | PSTATE_SSBS,
- .fn = ssbs_emulation_handler,
-};
-
-static void cpu_enable_ssbs(const struct arm64_cpu_capabilities *__unused)
-{
- static bool undef_hook_registered = false;
- static DEFINE_RAW_SPINLOCK(hook_lock);
-
- raw_spin_lock(&hook_lock);
- if (!undef_hook_registered) {
- register_undef_hook(&ssbs_emulation_hook);
- undef_hook_registered = true;
- }
- raw_spin_unlock(&hook_lock);
-
- if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) {
- sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
- arm64_set_ssbd_mitigation(false);
- } else {
- arm64_set_ssbd_mitigation(true);
- }
-}
-#endif /* CONFIG_ARM64_SSBD */
-
#ifdef CONFIG_ARM64_PAN
static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused)
{
#endif /* CONFIG_ARM64_RAS_EXTN */
#ifdef CONFIG_ARM64_PTR_AUTH
-static bool has_address_auth(const struct arm64_cpu_capabilities *entry,
- int __unused)
+static bool has_address_auth_cpucap(const struct arm64_cpu_capabilities *entry, int scope)
{
- return __system_matches_cap(ARM64_HAS_ADDRESS_AUTH_ARCH) ||
- __system_matches_cap(ARM64_HAS_ADDRESS_AUTH_IMP_DEF);
+ int boot_val, sec_val;
+
+ /* We don't expect to be called with SCOPE_SYSTEM */
+ WARN_ON(scope == SCOPE_SYSTEM);
+ /*
+ * The ptr-auth feature levels are not intercompatible with lower
+ * levels. Hence we must match ptr-auth feature level of the secondary
+ * CPUs with that of the boot CPU. The level of boot cpu is fetched
+ * from the sanitised register whereas direct register read is done for
+ * the secondary CPUs.
+ * The sanitised feature state is guaranteed to match that of the
+ * boot CPU as a mismatched secondary CPU is parked before it gets
+ * a chance to update the state, with the capability.
+ */
+ boot_val = cpuid_feature_extract_field(read_sanitised_ftr_reg(entry->sys_reg),
+ entry->field_pos, entry->sign);
+ if (scope & SCOPE_BOOT_CPU)
+ return boot_val >= entry->min_field_value;
+ /* Now check for the secondary CPUs with SCOPE_LOCAL_CPU scope */
+ sec_val = cpuid_feature_extract_field(__read_sysreg_by_encoding(entry->sys_reg),
+ entry->field_pos, entry->sign);
+ return sec_val == boot_val;
+}
+
+static bool has_address_auth_metacap(const struct arm64_cpu_capabilities *entry,
+ int scope)
+{
+ return has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH], scope) ||
+ has_address_auth_cpucap(cpu_hwcaps_ptrs[ARM64_HAS_ADDRESS_AUTH_IMP_DEF], scope);
}
static bool has_generic_auth(const struct arm64_cpu_capabilities *entry,
}
#endif /* CONFIG_ARM64_BTI */
+#ifdef CONFIG_ARM64_MTE
+static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap)
+{
+ static bool cleared_zero_page = false;
+
+ /*
+ * Clear the tags in the zero page. This needs to be done via the
+ * linear map which has the Tagged attribute.
+ */
+ if (!cleared_zero_page) {
+ cleared_zero_page = true;
+ mte_clear_page_tags(lm_alias(empty_zero_page));
+ }
+}
+#endif /* CONFIG_ARM64_MTE */
+
/* Internal helper functions to match cpu capability type */
static bool
cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
.field_pos = ID_AA64ISAR0_CRC32_SHIFT,
.min_field_value = 1,
},
-#ifdef CONFIG_ARM64_SSBD
{
.desc = "Speculative Store Bypassing Safe (SSBS)",
.capability = ARM64_SSBS,
- .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
+ .type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
.sys_reg = SYS_ID_AA64PFR1_EL1,
.field_pos = ID_AA64PFR1_SSBS_SHIFT,
.sign = FTR_UNSIGNED,
.min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY,
- .cpu_enable = cpu_enable_ssbs,
},
-#endif
#ifdef CONFIG_ARM64_CNP
{
.desc = "Common not Private translations",
.sign = FTR_UNSIGNED,
.field_pos = ID_AA64ISAR1_APA_SHIFT,
.min_field_value = ID_AA64ISAR1_APA_ARCHITECTED,
- .matches = has_cpuid_feature,
+ .matches = has_address_auth_cpucap,
},
{
.desc = "Address authentication (IMP DEF algorithm)",
.sign = FTR_UNSIGNED,
.field_pos = ID_AA64ISAR1_API_SHIFT,
.min_field_value = ID_AA64ISAR1_API_IMP_DEF,
- .matches = has_cpuid_feature,
+ .matches = has_address_auth_cpucap,
},
{
.capability = ARM64_HAS_ADDRESS_AUTH,
.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
- .matches = has_address_auth,
+ .matches = has_address_auth_metacap,
},
{
.desc = "Generic authentication (architected algorithm)",
.sign = FTR_UNSIGNED,
},
#endif
+#ifdef CONFIG_ARM64_MTE
+ {
+ .desc = "Memory Tagging Extension",
+ .capability = ARM64_MTE,
+ .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
+ .matches = has_cpuid_feature,
+ .sys_reg = SYS_ID_AA64PFR1_EL1,
+ .field_pos = ID_AA64PFR1_MTE_SHIFT,
+ .min_field_value = ID_AA64PFR1_MTE,
+ .sign = FTR_UNSIGNED,
+ .cpu_enable = cpu_enable_mte,
+ },
+#endif /* CONFIG_ARM64_MTE */
{},
};
HWCAP_MULTI_CAP(ptr_auth_hwcap_addr_matches, CAP_HWCAP, KERNEL_HWCAP_PACA),
HWCAP_MULTI_CAP(ptr_auth_hwcap_gen_matches, CAP_HWCAP, KERNEL_HWCAP_PACG),
#endif
+#ifdef CONFIG_ARM64_MTE
+ HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_MTE, CAP_HWCAP, KERNEL_HWCAP_MTE),
+#endif /* CONFIG_ARM64_MTE */
{},
};
unsigned long __icache_flags;
static const char *const hwcap_str[] = {
- "fp",
- "asimd",
- "evtstrm",
- "aes",
- "pmull",
- "sha1",
- "sha2",
- "crc32",
- "atomics",
- "fphp",
- "asimdhp",
- "cpuid",
- "asimdrdm",
- "jscvt",
- "fcma",
- "lrcpc",
- "dcpop",
- "sha3",
- "sm3",
- "sm4",
- "asimddp",
- "sha512",
- "sve",
- "asimdfhm",
- "dit",
- "uscat",
- "ilrcpc",
- "flagm",
- "ssbs",
- "sb",
- "paca",
- "pacg",
- "dcpodp",
- "sve2",
- "sveaes",
- "svepmull",
- "svebitperm",
- "svesha3",
- "svesm4",
- "flagm2",
- "frint",
- "svei8mm",
- "svef32mm",
- "svef64mm",
- "svebf16",
- "i8mm",
- "bf16",
- "dgh",
- "rng",
- "bti",
- /* reserved for "mte" */
- NULL
+ [KERNEL_HWCAP_FP] = "fp",
+ [KERNEL_HWCAP_ASIMD] = "asimd",
+ [KERNEL_HWCAP_EVTSTRM] = "evtstrm",
+ [KERNEL_HWCAP_AES] = "aes",
+ [KERNEL_HWCAP_PMULL] = "pmull",
+ [KERNEL_HWCAP_SHA1] = "sha1",
+ [KERNEL_HWCAP_SHA2] = "sha2",
+ [KERNEL_HWCAP_CRC32] = "crc32",
+ [KERNEL_HWCAP_ATOMICS] = "atomics",
+ [KERNEL_HWCAP_FPHP] = "fphp",
+ [KERNEL_HWCAP_ASIMDHP] = "asimdhp",
+ [KERNEL_HWCAP_CPUID] = "cpuid",
+ [KERNEL_HWCAP_ASIMDRDM] = "asimdrdm",
+ [KERNEL_HWCAP_JSCVT] = "jscvt",
+ [KERNEL_HWCAP_FCMA] = "fcma",
+ [KERNEL_HWCAP_LRCPC] = "lrcpc",
+ [KERNEL_HWCAP_DCPOP] = "dcpop",
+ [KERNEL_HWCAP_SHA3] = "sha3",
+ [KERNEL_HWCAP_SM3] = "sm3",
+ [KERNEL_HWCAP_SM4] = "sm4",
+ [KERNEL_HWCAP_ASIMDDP] = "asimddp",
+ [KERNEL_HWCAP_SHA512] = "sha512",
+ [KERNEL_HWCAP_SVE] = "sve",
+ [KERNEL_HWCAP_ASIMDFHM] = "asimdfhm",
+ [KERNEL_HWCAP_DIT] = "dit",
+ [KERNEL_HWCAP_USCAT] = "uscat",
+ [KERNEL_HWCAP_ILRCPC] = "ilrcpc",
+ [KERNEL_HWCAP_FLAGM] = "flagm",
+ [KERNEL_HWCAP_SSBS] = "ssbs",
+ [KERNEL_HWCAP_SB] = "sb",
+ [KERNEL_HWCAP_PACA] = "paca",
+ [KERNEL_HWCAP_PACG] = "pacg",
+ [KERNEL_HWCAP_DCPODP] = "dcpodp",
+ [KERNEL_HWCAP_SVE2] = "sve2",
+ [KERNEL_HWCAP_SVEAES] = "sveaes",
+ [KERNEL_HWCAP_SVEPMULL] = "svepmull",
+ [KERNEL_HWCAP_SVEBITPERM] = "svebitperm",
+ [KERNEL_HWCAP_SVESHA3] = "svesha3",
+ [KERNEL_HWCAP_SVESM4] = "svesm4",
+ [KERNEL_HWCAP_FLAGM2] = "flagm2",
+ [KERNEL_HWCAP_FRINT] = "frint",
+ [KERNEL_HWCAP_SVEI8MM] = "svei8mm",
+ [KERNEL_HWCAP_SVEF32MM] = "svef32mm",
+ [KERNEL_HWCAP_SVEF64MM] = "svef64mm",
+ [KERNEL_HWCAP_SVEBF16] = "svebf16",
+ [KERNEL_HWCAP_I8MM] = "i8mm",
+ [KERNEL_HWCAP_BF16] = "bf16",
+ [KERNEL_HWCAP_DGH] = "dgh",
+ [KERNEL_HWCAP_RNG] = "rng",
+ [KERNEL_HWCAP_BTI] = "bti",
+ [KERNEL_HWCAP_MTE] = "mte",
};
#ifdef CONFIG_COMPAT
+#define COMPAT_KERNEL_HWCAP(x) const_ilog2(COMPAT_HWCAP_ ## x)
static const char *const compat_hwcap_str[] = {
- "swp",
- "half",
- "thumb",
- "26bit",
- "fastmult",
- "fpa",
- "vfp",
- "edsp",
- "java",
- "iwmmxt",
- "crunch",
- "thumbee",
- "neon",
- "vfpv3",
- "vfpv3d16",
- "tls",
- "vfpv4",
- "idiva",
- "idivt",
- "vfpd32",
- "lpae",
- "evtstrm",
- NULL
+ [COMPAT_KERNEL_HWCAP(SWP)] = "swp",
+ [COMPAT_KERNEL_HWCAP(HALF)] = "half",
+ [COMPAT_KERNEL_HWCAP(THUMB)] = "thumb",
+ [COMPAT_KERNEL_HWCAP(26BIT)] = NULL, /* Not possible on arm64 */
+ [COMPAT_KERNEL_HWCAP(FAST_MULT)] = "fastmult",
+ [COMPAT_KERNEL_HWCAP(FPA)] = NULL, /* Not possible on arm64 */
+ [COMPAT_KERNEL_HWCAP(VFP)] = "vfp",
+ [COMPAT_KERNEL_HWCAP(EDSP)] = "edsp",
+ [COMPAT_KERNEL_HWCAP(JAVA)] = NULL, /* Not possible on arm64 */
+ [COMPAT_KERNEL_HWCAP(IWMMXT)] = NULL, /* Not possible on arm64 */
+ [COMPAT_KERNEL_HWCAP(CRUNCH)] = NULL, /* Not possible on arm64 */
+ [COMPAT_KERNEL_HWCAP(THUMBEE)] = NULL, /* Not possible on arm64 */
+ [COMPAT_KERNEL_HWCAP(NEON)] = "neon",
+ [COMPAT_KERNEL_HWCAP(VFPv3)] = "vfpv3",
+ [COMPAT_KERNEL_HWCAP(VFPV3D16)] = NULL, /* Not possible on arm64 */
+ [COMPAT_KERNEL_HWCAP(TLS)] = "tls",
+ [COMPAT_KERNEL_HWCAP(VFPv4)] = "vfpv4",
+ [COMPAT_KERNEL_HWCAP(IDIVA)] = "idiva",
+ [COMPAT_KERNEL_HWCAP(IDIVT)] = "idivt",
+ [COMPAT_KERNEL_HWCAP(VFPD32)] = NULL, /* Not possible on arm64 */
+ [COMPAT_KERNEL_HWCAP(LPAE)] = "lpae",
+ [COMPAT_KERNEL_HWCAP(EVTSTRM)] = "evtstrm",
};
+#define COMPAT_KERNEL_HWCAP2(x) const_ilog2(COMPAT_HWCAP2_ ## x)
static const char *const compat_hwcap2_str[] = {
- "aes",
- "pmull",
- "sha1",
- "sha2",
- "crc32",
- NULL
+ [COMPAT_KERNEL_HWCAP2(AES)] = "aes",
+ [COMPAT_KERNEL_HWCAP2(PMULL)] = "pmull",
+ [COMPAT_KERNEL_HWCAP2(SHA1)] = "sha1",
+ [COMPAT_KERNEL_HWCAP2(SHA2)] = "sha2",
+ [COMPAT_KERNEL_HWCAP2(CRC32)] = "crc32",
};
#endif /* CONFIG_COMPAT */
seq_puts(m, "Features\t:");
if (compat) {
#ifdef CONFIG_COMPAT
- for (j = 0; compat_hwcap_str[j]; j++)
- if (compat_elf_hwcap & (1 << j))
+ for (j = 0; j < ARRAY_SIZE(compat_hwcap_str); j++) {
+ if (compat_elf_hwcap & (1 << j)) {
+ /*
+ * Warn once if any feature should not
+ * have been present on arm64 platform.
+ */
+ if (WARN_ON_ONCE(!compat_hwcap_str[j]))
+ continue;
+
seq_printf(m, " %s", compat_hwcap_str[j]);
+ }
+ }
- for (j = 0; compat_hwcap2_str[j]; j++)
+ for (j = 0; j < ARRAY_SIZE(compat_hwcap2_str); j++)
if (compat_elf_hwcap2 & (1 << j))
seq_printf(m, " %s", compat_hwcap2_str[j]);
#endif /* CONFIG_COMPAT */
} else {
- for (j = 0; hwcap_str[j]; j++)
+ for (j = 0; j < ARRAY_SIZE(hwcap_str); j++)
if (cpu_have_feature(j))
seq_printf(m, " %s", hwcap_str[j]);
}
hook_debug_fault_code(DBG_ESR_EVT_HWSS, single_step_handler, SIGTRAP,
TRAP_TRACE, "single-step handler");
hook_debug_fault_code(DBG_ESR_EVT_BRK, brk_handler, SIGTRAP,
- TRAP_BRKPT, "ptrace BRK handler");
+ TRAP_BRKPT, "BRK handler");
}
/* Re-enable single step for syscall restarting. */
}
NOKPROBE_SYMBOL(el1_dbg);
+static void notrace el1_fpac(struct pt_regs *regs, unsigned long esr)
+{
+ local_daif_inherit(regs);
+ do_ptrauth_fault(regs, esr);
+}
+NOKPROBE_SYMBOL(el1_fpac);
+
asmlinkage void notrace el1_sync_handler(struct pt_regs *regs)
{
unsigned long esr = read_sysreg(esr_el1);
case ESR_ELx_EC_BRK64:
el1_dbg(regs, esr);
break;
+ case ESR_ELx_EC_FPAC:
+ el1_fpac(regs, esr);
+ break;
default:
el1_inv(regs, esr);
}
}
NOKPROBE_SYMBOL(el0_svc);
+static void notrace el0_fpac(struct pt_regs *regs, unsigned long esr)
+{
+ user_exit_irqoff();
+ local_daif_restore(DAIF_PROCCTX);
+ do_ptrauth_fault(regs, esr);
+}
+NOKPROBE_SYMBOL(el0_fpac);
+
asmlinkage void notrace el0_sync_handler(struct pt_regs *regs)
{
unsigned long esr = read_sysreg(esr_el1);
case ESR_ELx_EC_BRK64:
el0_dbg(regs, esr);
break;
+ case ESR_ELx_EC_FPAC:
+ el0_fpac(regs, esr);
+ break;
default:
el0_inv(regs, esr);
}
SYM_FUNC_END(fpsimd_load_state)
#ifdef CONFIG_ARM64_SVE
+
SYM_FUNC_START(sve_save_state)
sve_save 0, x1, 2
ret
_sve_rdvl 0, 1
ret
SYM_FUNC_END(sve_get_vl)
+
+/*
+ * Load SVE state from FPSIMD state.
+ *
+ * x0 = pointer to struct fpsimd_state
+ * x1 = VQ - 1
+ *
+ * Each SVE vector will be loaded with the first 128-bits taken from FPSIMD
+ * and the rest zeroed. All the other SVE registers will be zeroed.
+ */
+SYM_FUNC_START(sve_load_from_fpsimd_state)
+ sve_load_vq x1, x2, x3
+ fpsimd_restore x0, 8
+ _for n, 0, 15, _sve_pfalse \n
+ _sve_wrffr 0
+ ret
+SYM_FUNC_END(sve_load_from_fpsimd_state)
+
+/* Zero all SVE registers but the first 128-bits of each vector */
+SYM_FUNC_START(sve_flush_live)
+ sve_flush
+ ret
+SYM_FUNC_END(sve_flush_live)
+
#endif /* CONFIG_ARM64_SVE */
* them if required.
*/
.macro apply_ssbd, state, tmp1, tmp2
-#ifdef CONFIG_ARM64_SSBD
-alternative_cb arm64_enable_wa2_handling
- b .L__asm_ssbd_skip\@
+alternative_cb spectre_v4_patch_fw_mitigation_enable
+ b .L__asm_ssbd_skip\@ // Patched to NOP
alternative_cb_end
ldr_this_cpu \tmp2, arm64_ssbd_callback_required, \tmp1
cbz \tmp2, .L__asm_ssbd_skip\@
tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@
mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2
mov w1, #\state
-alternative_cb arm64_update_smccc_conduit
+alternative_cb spectre_v4_patch_fw_mitigation_conduit
nop // Patched to SMC/HVC #0
alternative_cb_end
.L__asm_ssbd_skip\@:
+ .endm
+
+ /* Check for MTE asynchronous tag check faults */
+ .macro check_mte_async_tcf, flgs, tmp
+#ifdef CONFIG_ARM64_MTE
+alternative_if_not ARM64_MTE
+ b 1f
+alternative_else_nop_endif
+ mrs_s \tmp, SYS_TFSRE0_EL1
+ tbz \tmp, #SYS_TFSR_EL1_TF0_SHIFT, 1f
+ /* Asynchronous TCF occurred for TTBR0 access, set the TI flag */
+ orr \flgs, \flgs, #_TIF_MTE_ASYNC_FAULT
+ str \flgs, [tsk, #TSK_TI_FLAGS]
+ msr_s SYS_TFSRE0_EL1, xzr
+1:
+#endif
+ .endm
+
+ /* Clear the MTE asynchronous tag check faults */
+ .macro clear_mte_async_tcf
+#ifdef CONFIG_ARM64_MTE
+alternative_if ARM64_MTE
+ dsb ish
+ msr_s SYS_TFSRE0_EL1, xzr
+alternative_else_nop_endif
#endif
.endm
ldr x19, [tsk, #TSK_TI_FLAGS]
disable_step_tsk x19, x20
+ /* Check for asynchronous tag check faults in user space */
+ check_mte_async_tcf x19, x22
apply_ssbd 1, x22, x23
ptrauth_keys_install_kernel tsk, x20, x22, x23
str x20, [sp, #S_PMR_SAVE]
alternative_else_nop_endif
+ /* Re-enable tag checking (TCO set on exception entry) */
+#ifdef CONFIG_ARM64_MTE
+alternative_if ARM64_MTE
+ SET_PSTATE_TCO(0)
+alternative_else_nop_endif
+#endif
+
/*
* Registers that may be useful after this macro is invoked:
*
bl trace_hardirqs_off
#endif
-#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
tbz x22, #55, 1f
bl do_el0_irq_bp_hardening
1:
-#endif
irq_handler
#ifdef CONFIG_TRACE_IRQFLAGS
and x2, x1, #_TIF_WORK_MASK
cbnz x2, work_pending
finish_ret_to_user:
+ /* Ignore asynchronous tag check faults in the uaccess routines */
+ clear_mte_async_tcf
enable_step_tsk x1, x2
#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
bl stackleak_erase
#include <linux/swab.h>
#include <asm/esr.h>
+#include <asm/exception.h>
#include <asm/fpsimd.h>
#include <asm/cpufeature.h>
#include <asm/cputype.h>
+#include <asm/neon.h>
#include <asm/processor.h>
#include <asm/simd.h>
#include <asm/sigcontext.h>
* re-enter user with corrupt state.
* There's no way to recover, so kill it:
*/
- force_signal_inject(SIGKILL, SI_KERNEL, 0);
+ force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
return;
}
* the SVE access trap will be disabled the next time this task
* reaches ret_to_user.
*
- * TIF_SVE should be clear on entry: otherwise, task_fpsimd_load()
+ * TIF_SVE should be clear on entry: otherwise, fpsimd_restore_current_state()
* would have disabled the SVE access trap for userspace during
* ret_to_user, making an SVE access trap impossible in that case.
*/
{
/* Even if we chose not to use SVE, the hardware could still trap: */
if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
- force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
+ force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
return;
}
#include "efi-header.S"
-#define __PHYS_OFFSET (KERNEL_START - TEXT_OFFSET)
+#define __PHYS_OFFSET KERNEL_START
-#if (TEXT_OFFSET & 0xfff) != 0
-#error TEXT_OFFSET must be at least 4KB aligned
-#elif (PAGE_OFFSET & 0x1fffff) != 0
+#if (PAGE_OFFSET & 0x1fffff) != 0
#error PAGE_OFFSET must be at least 2MB aligned
-#elif TEXT_OFFSET > 0x1fffff
-#error TEXT_OFFSET must be less than 2MB
#endif
/*
* x0 = physical address to the FDT blob.
*
* This code is mostly position independent so you call this at
- * __pa(PAGE_OFFSET + TEXT_OFFSET).
+ * __pa(PAGE_OFFSET).
*
* Note that the callee-saved registers are used for storing variables
* that are useful before the MMU is enabled. The allocations are described
b primary_entry // branch to kernel start, magic
.long 0 // reserved
#endif
- le64sym _kernel_offset_le // Image load offset from start of RAM, little-endian
+ .quad 0 // Image load offset from start of RAM, little-endian
le64sym _kernel_size_le // Effective size of kernel image, little-endian
le64sym _kernel_flags_le // Informative flags, little-endian
.quad 0 // reserved
* Map the kernel image (starting with PHYS_OFFSET).
*/
adrp x0, init_pg_dir
- mov_q x5, KIMAGE_VADDR + TEXT_OFFSET // compile time __va(_text)
+ mov_q x5, KIMAGE_VADDR // compile time __va(_text)
add x5, x5, x23 // add KASLR displacement
mov x4, PTRS_PER_PGD
adrp x6, _end // runtime __pa(_end)
.pushsection ".rodata", "a"
SYM_DATA_START(kimage_vaddr)
- .quad _text - TEXT_OFFSET
+ .quad _text
SYM_DATA_END(kimage_vaddr)
EXPORT_SYMBOL(kimage_vaddr)
.popsection
#include <linux/sched.h>
#include <linux/suspend.h>
#include <linux/utsname.h>
-#include <linux/version.h>
#include <asm/barrier.h>
#include <asm/cacheflush.h>
#include <asm/kexec.h>
#include <asm/memory.h>
#include <asm/mmu_context.h>
+#include <asm/mte.h>
#include <asm/pgalloc.h>
#include <asm/pgtable-hwdef.h>
#include <asm/sections.h>
#define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start))
+#ifdef CONFIG_ARM64_MTE
+
+static DEFINE_XARRAY(mte_pages);
+
+static int save_tags(struct page *page, unsigned long pfn)
+{
+ void *tag_storage, *ret;
+
+ tag_storage = mte_allocate_tag_storage();
+ if (!tag_storage)
+ return -ENOMEM;
+
+ mte_save_page_tags(page_address(page), tag_storage);
+
+ ret = xa_store(&mte_pages, pfn, tag_storage, GFP_KERNEL);
+ if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
+ mte_free_tag_storage(tag_storage);
+ return xa_err(ret);
+ } else if (WARN(ret, "swsusp: %s: Duplicate entry", __func__)) {
+ mte_free_tag_storage(ret);
+ }
+
+ return 0;
+}
+
+static void swsusp_mte_free_storage(void)
+{
+ XA_STATE(xa_state, &mte_pages, 0);
+ void *tags;
+
+ xa_lock(&mte_pages);
+ xas_for_each(&xa_state, tags, ULONG_MAX) {
+ mte_free_tag_storage(tags);
+ }
+ xa_unlock(&mte_pages);
+
+ xa_destroy(&mte_pages);
+}
+
+static int swsusp_mte_save_tags(void)
+{
+ struct zone *zone;
+ unsigned long pfn, max_zone_pfn;
+ int ret = 0;
+ int n = 0;
+
+ if (!system_supports_mte())
+ return 0;
+
+ for_each_populated_zone(zone) {
+ max_zone_pfn = zone_end_pfn(zone);
+ for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
+ struct page *page = pfn_to_online_page(pfn);
+
+ if (!page)
+ continue;
+
+ if (!test_bit(PG_mte_tagged, &page->flags))
+ continue;
+
+ ret = save_tags(page, pfn);
+ if (ret) {
+ swsusp_mte_free_storage();
+ goto out;
+ }
+
+ n++;
+ }
+ }
+ pr_info("Saved %d MTE pages\n", n);
+
+out:
+ return ret;
+}
+
+static void swsusp_mte_restore_tags(void)
+{
+ XA_STATE(xa_state, &mte_pages, 0);
+ int n = 0;
+ void *tags;
+
+ xa_lock(&mte_pages);
+ xas_for_each(&xa_state, tags, ULONG_MAX) {
+ unsigned long pfn = xa_state.xa_index;
+ struct page *page = pfn_to_online_page(pfn);
+
+ mte_restore_page_tags(page_address(page), tags);
+
+ mte_free_tag_storage(tags);
+ n++;
+ }
+ xa_unlock(&mte_pages);
+
+ pr_info("Restored %d MTE pages\n", n);
+
+ xa_destroy(&mte_pages);
+}
+
+#else /* CONFIG_ARM64_MTE */
+
+static int swsusp_mte_save_tags(void)
+{
+ return 0;
+}
+
+static void swsusp_mte_restore_tags(void)
+{
+}
+
+#endif /* CONFIG_ARM64_MTE */
+
int swsusp_arch_suspend(void)
{
int ret = 0;
/* make the crash dump kernel image visible/saveable */
crash_prepare_suspend();
+ ret = swsusp_mte_save_tags();
+ if (ret)
+ return ret;
+
sleep_cpu = smp_processor_id();
ret = swsusp_save();
} else {
dcache_clean_range(__hyp_text_start, __hyp_text_end);
}
+ swsusp_mte_restore_tags();
+
/* make the crash dump kernel image protected again */
crash_post_resume();
* mitigation off behind our back, let's set the state
* to what we expect it to be.
*/
- switch (arm64_get_ssbd_state()) {
- case ARM64_SSBD_FORCE_ENABLE:
- case ARM64_SSBD_KERNEL:
- arm64_set_ssbd_mitigation(true);
- }
+ spectre_v4_enable_mitigation(NULL);
}
local_daif_restore(flags);
#define KVM_NVHE_ALIAS(sym) __kvm_nvhe_##sym = sym;
/* Alternative callbacks for init-time patching of nVHE hyp code. */
-KVM_NVHE_ALIAS(arm64_enable_wa2_handling);
KVM_NVHE_ALIAS(kvm_patch_vector_branch);
KVM_NVHE_ALIAS(kvm_update_va_mask);
/* Global kernel state accessed by nVHE hyp code. */
-KVM_NVHE_ALIAS(arm64_ssbd_callback_required);
KVM_NVHE_ALIAS(kvm_host_data);
KVM_NVHE_ALIAS(kvm_vgic_global_state);
/* Static key checked in pmr_sync(). */
#ifdef CONFIG_ARM64_PSEUDO_NMI
KVM_NVHE_ALIAS(gic_pmr_sync);
+/* Static key checked in GIC_PRIO_IRQOFF. */
+KVM_NVHE_ALIAS(gic_nonsecure_priorities);
#endif
/* EL2 exception handling */
*/
#define HEAD_SYMBOLS \
DEFINE_IMAGE_LE64(_kernel_size_le, _end - _text); \
- DEFINE_IMAGE_LE64(_kernel_offset_le, TEXT_OFFSET); \
DEFINE_IMAGE_LE64(_kernel_flags_le, __HEAD_FLAGS);
#endif /* __ARM64_KERNEL_IMAGE_H */
case AARCH64_INSN_HINT_XPACLRI:
case AARCH64_INSN_HINT_PACIA_1716:
case AARCH64_INSN_HINT_PACIB_1716:
- case AARCH64_INSN_HINT_AUTIA_1716:
- case AARCH64_INSN_HINT_AUTIB_1716:
case AARCH64_INSN_HINT_PACIAZ:
case AARCH64_INSN_HINT_PACIASP:
case AARCH64_INSN_HINT_PACIBZ:
case AARCH64_INSN_HINT_PACIBSP:
- case AARCH64_INSN_HINT_AUTIAZ:
- case AARCH64_INSN_HINT_AUTIASP:
- case AARCH64_INSN_HINT_AUTIBZ:
- case AARCH64_INSN_HINT_AUTIBSP:
case AARCH64_INSN_HINT_BTI:
case AARCH64_INSN_HINT_BTIC:
case AARCH64_INSN_HINT_BTIJ:
bool __kprobes aarch64_insn_is_branch(u32 insn)
{
- /* b, bl, cb*, tb*, b.cond, br, blr */
+ /* b, bl, cb*, tb*, ret*, b.cond, br*, blr* */
return aarch64_insn_is_b(insn) ||
aarch64_insn_is_bl(insn) ||
aarch64_insn_is_tbz(insn) ||
aarch64_insn_is_tbnz(insn) ||
aarch64_insn_is_ret(insn) ||
+ aarch64_insn_is_ret_auth(insn) ||
aarch64_insn_is_br(insn) ||
+ aarch64_insn_is_br_auth(insn) ||
aarch64_insn_is_blr(insn) ||
+ aarch64_insn_is_blr_auth(insn) ||
aarch64_insn_is_bcond(insn);
}
* Copyright (C) 2012 ARM Ltd.
*/
-#include <linux/kernel_stat.h>
#include <linux/irq.h>
#include <linux/memory.h>
#include <linux/smp.h>
+#include <linux/hardirq.h>
#include <linux/init.h>
#include <linux/irqchip.h>
#include <linux/kprobes.h>
#include <asm/daifflags.h>
#include <asm/vmap_stack.h>
-unsigned long irq_err_count;
-
/* Only access this in an NMI enter/exit */
DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);
DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
-int arch_show_interrupts(struct seq_file *p, int prec)
-{
- show_ipi_list(p, prec);
- seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
- return 0;
-}
-
#ifdef CONFIG_VMAP_STACK
static void init_irq_stacks(void)
{
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 ARM Ltd.
+ */
+
+#include <linux/bitops.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/prctl.h>
+#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/string.h>
+#include <linux/swap.h>
+#include <linux/swapops.h>
+#include <linux/thread_info.h>
+#include <linux/uio.h>
+
+#include <asm/cpufeature.h>
+#include <asm/mte.h>
+#include <asm/ptrace.h>
+#include <asm/sysreg.h>
+
+static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap)
+{
+ pte_t old_pte = READ_ONCE(*ptep);
+
+ if (check_swap && is_swap_pte(old_pte)) {
+ swp_entry_t entry = pte_to_swp_entry(old_pte);
+
+ if (!non_swap_entry(entry) && mte_restore_tags(entry, page))
+ return;
+ }
+
+ mte_clear_page_tags(page_address(page));
+}
+
+void mte_sync_tags(pte_t *ptep, pte_t pte)
+{
+ struct page *page = pte_page(pte);
+ long i, nr_pages = compound_nr(page);
+ bool check_swap = nr_pages == 1;
+
+ /* if PG_mte_tagged is set, tags have already been initialised */
+ for (i = 0; i < nr_pages; i++, page++) {
+ if (!test_and_set_bit(PG_mte_tagged, &page->flags))
+ mte_sync_page_tags(page, ptep, check_swap);
+ }
+}
+
+int memcmp_pages(struct page *page1, struct page *page2)
+{
+ char *addr1, *addr2;
+ int ret;
+
+ addr1 = page_address(page1);
+ addr2 = page_address(page2);
+ ret = memcmp(addr1, addr2, PAGE_SIZE);
+
+ if (!system_supports_mte() || ret)
+ return ret;
+
+ /*
+ * If the page content is identical but at least one of the pages is
+ * tagged, return non-zero to avoid KSM merging. If only one of the
+ * pages is tagged, set_pte_at() may zero or change the tags of the
+ * other page via mte_sync_tags().
+ */
+ if (test_bit(PG_mte_tagged, &page1->flags) ||
+ test_bit(PG_mte_tagged, &page2->flags))
+ return addr1 != addr2;
+
+ return ret;
+}
+
+static void update_sctlr_el1_tcf0(u64 tcf0)
+{
+ /* ISB required for the kernel uaccess routines */
+ sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF0_MASK, tcf0);
+ isb();
+}
+
+static void set_sctlr_el1_tcf0(u64 tcf0)
+{
+ /*
+ * mte_thread_switch() checks current->thread.sctlr_tcf0 as an
+ * optimisation. Disable preemption so that it does not see
+ * the variable update before the SCTLR_EL1.TCF0 one.
+ */
+ preempt_disable();
+ current->thread.sctlr_tcf0 = tcf0;
+ update_sctlr_el1_tcf0(tcf0);
+ preempt_enable();
+}
+
+static void update_gcr_el1_excl(u64 incl)
+{
+ u64 excl = ~incl & SYS_GCR_EL1_EXCL_MASK;
+
+ /*
+ * Note that 'incl' is an include mask (controlled by the user via
+ * prctl()) while GCR_EL1 accepts an exclude mask.
+ * No need for ISB since this only affects EL0 currently, implicit
+ * with ERET.
+ */
+ sysreg_clear_set_s(SYS_GCR_EL1, SYS_GCR_EL1_EXCL_MASK, excl);
+}
+
+static void set_gcr_el1_excl(u64 incl)
+{
+ current->thread.gcr_user_incl = incl;
+ update_gcr_el1_excl(incl);
+}
+
+void flush_mte_state(void)
+{
+ if (!system_supports_mte())
+ return;
+
+ /* clear any pending asynchronous tag fault */
+ dsb(ish);
+ write_sysreg_s(0, SYS_TFSRE0_EL1);
+ clear_thread_flag(TIF_MTE_ASYNC_FAULT);
+ /* disable tag checking */
+ set_sctlr_el1_tcf0(SCTLR_EL1_TCF0_NONE);
+ /* reset tag generation mask */
+ set_gcr_el1_excl(0);
+}
+
+void mte_thread_switch(struct task_struct *next)
+{
+ if (!system_supports_mte())
+ return;
+
+ /* avoid expensive SCTLR_EL1 accesses if no change */
+ if (current->thread.sctlr_tcf0 != next->thread.sctlr_tcf0)
+ update_sctlr_el1_tcf0(next->thread.sctlr_tcf0);
+ update_gcr_el1_excl(next->thread.gcr_user_incl);
+}
+
+void mte_suspend_exit(void)
+{
+ if (!system_supports_mte())
+ return;
+
+ update_gcr_el1_excl(current->thread.gcr_user_incl);
+}
+
+long set_mte_ctrl(struct task_struct *task, unsigned long arg)
+{
+ u64 tcf0;
+ u64 gcr_incl = (arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT;
+
+ if (!system_supports_mte())
+ return 0;
+
+ switch (arg & PR_MTE_TCF_MASK) {
+ case PR_MTE_TCF_NONE:
+ tcf0 = SCTLR_EL1_TCF0_NONE;
+ break;
+ case PR_MTE_TCF_SYNC:
+ tcf0 = SCTLR_EL1_TCF0_SYNC;
+ break;
+ case PR_MTE_TCF_ASYNC:
+ tcf0 = SCTLR_EL1_TCF0_ASYNC;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (task != current) {
+ task->thread.sctlr_tcf0 = tcf0;
+ task->thread.gcr_user_incl = gcr_incl;
+ } else {
+ set_sctlr_el1_tcf0(tcf0);
+ set_gcr_el1_excl(gcr_incl);
+ }
+
+ return 0;
+}
+
+long get_mte_ctrl(struct task_struct *task)
+{
+ unsigned long ret;
+
+ if (!system_supports_mte())
+ return 0;
+
+ ret = task->thread.gcr_user_incl << PR_MTE_TAG_SHIFT;
+
+ switch (task->thread.sctlr_tcf0) {
+ case SCTLR_EL1_TCF0_NONE:
+ return PR_MTE_TCF_NONE;
+ case SCTLR_EL1_TCF0_SYNC:
+ ret |= PR_MTE_TCF_SYNC;
+ break;
+ case SCTLR_EL1_TCF0_ASYNC:
+ ret |= PR_MTE_TCF_ASYNC;
+ break;
+ }
+
+ return ret;
+}
+
+/*
+ * Access MTE tags in another process' address space as given in mm. Update
+ * the number of tags copied. Return 0 if any tags copied, error otherwise.
+ * Inspired by __access_remote_vm().
+ */
+static int __access_remote_tags(struct mm_struct *mm, unsigned long addr,
+ struct iovec *kiov, unsigned int gup_flags)
+{
+ struct vm_area_struct *vma;
+ void __user *buf = kiov->iov_base;
+ size_t len = kiov->iov_len;
+ int ret;
+ int write = gup_flags & FOLL_WRITE;
+
+ if (!access_ok(buf, len))
+ return -EFAULT;
+
+ if (mmap_read_lock_killable(mm))
+ return -EIO;
+
+ while (len) {
+ unsigned long tags, offset;
+ void *maddr;
+ struct page *page = NULL;
+
+ ret = get_user_pages_remote(mm, addr, 1, gup_flags, &page,
+ &vma, NULL);
+ if (ret <= 0)
+ break;
+
+ /*
+ * Only copy tags if the page has been mapped as PROT_MTE
+ * (PG_mte_tagged set). Otherwise the tags are not valid and
+ * not accessible to user. Moreover, an mprotect(PROT_MTE)
+ * would cause the existing tags to be cleared if the page
+ * was never mapped with PROT_MTE.
+ */
+ if (!test_bit(PG_mte_tagged, &page->flags)) {
+ ret = -EOPNOTSUPP;
+ put_page(page);
+ break;
+ }
+
+ /* limit access to the end of the page */
+ offset = offset_in_page(addr);
+ tags = min(len, (PAGE_SIZE - offset) / MTE_GRANULE_SIZE);
+
+ maddr = page_address(page);
+ if (write) {
+ tags = mte_copy_tags_from_user(maddr + offset, buf, tags);
+ set_page_dirty_lock(page);
+ } else {
+ tags = mte_copy_tags_to_user(buf, maddr + offset, tags);
+ }
+ put_page(page);
+
+ /* error accessing the tracer's buffer */
+ if (!tags)
+ break;
+
+ len -= tags;
+ buf += tags;
+ addr += tags * MTE_GRANULE_SIZE;
+ }
+ mmap_read_unlock(mm);
+
+ /* return an error if no tags copied */
+ kiov->iov_len = buf - kiov->iov_base;
+ if (!kiov->iov_len) {
+ /* check for error accessing the tracee's address space */
+ if (ret <= 0)
+ return -EIO;
+ else
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+/*
+ * Copy MTE tags in another process' address space at 'addr' to/from tracer's
+ * iovec buffer. Return 0 on success. Inspired by ptrace_access_vm().
+ */
+static int access_remote_tags(struct task_struct *tsk, unsigned long addr,
+ struct iovec *kiov, unsigned int gup_flags)
+{
+ struct mm_struct *mm;
+ int ret;
+
+ mm = get_task_mm(tsk);
+ if (!mm)
+ return -EPERM;
+
+ if (!tsk->ptrace || (current != tsk->parent) ||
+ ((get_dumpable(mm) != SUID_DUMP_USER) &&
+ !ptracer_capable(tsk, mm->user_ns))) {
+ mmput(mm);
+ return -EPERM;
+ }
+
+ ret = __access_remote_tags(mm, addr, kiov, gup_flags);
+ mmput(mm);
+
+ return ret;
+}
+
+int mte_ptrace_copy_tags(struct task_struct *child, long request,
+ unsigned long addr, unsigned long data)
+{
+ int ret;
+ struct iovec kiov;
+ struct iovec __user *uiov = (void __user *)data;
+ unsigned int gup_flags = FOLL_FORCE;
+
+ if (!system_supports_mte())
+ return -EIO;
+
+ if (get_user(kiov.iov_base, &uiov->iov_base) ||
+ get_user(kiov.iov_len, &uiov->iov_len))
+ return -EFAULT;
+
+ if (request == PTRACE_POKEMTETAGS)
+ gup_flags |= FOLL_WRITE;
+
+ /* align addr to the MTE tag granule */
+ addr &= MTE_GRANULE_MASK;
+
+ ret = access_remote_tags(child, addr, &kiov, gup_flags);
+ if (!ret)
+ ret = put_user(kiov.iov_len, &uiov->iov_len);
+
+ return ret;
+}
* whist unwinding the stackframe and is like a subroutine return so we use
* the PC.
*/
-static int callchain_trace(struct stackframe *frame, void *data)
+static bool callchain_trace(void *data, unsigned long pc)
{
struct perf_callchain_entry_ctx *entry = data;
- perf_callchain_store(entry, frame->pc);
- return 0;
+ perf_callchain_store(entry, pc);
+ return true;
}
void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
[C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB,
+ [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD,
+ [C(LL)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_LL_CACHE_RD,
+
[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
[C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
};
.attrs = armv8_pmuv3_format_attrs,
};
+static ssize_t slots_show(struct device *dev, struct device_attribute *attr,
+ char *page)
+{
+ struct pmu *pmu = dev_get_drvdata(dev);
+ struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
+ u32 slots = cpu_pmu->reg_pmmir & ARMV8_PMU_SLOTS_MASK;
+
+ return snprintf(page, PAGE_SIZE, "0x%08x\n", slots);
+}
+
+static DEVICE_ATTR_RO(slots);
+
+static struct attribute *armv8_pmuv3_caps_attrs[] = {
+ &dev_attr_slots.attr,
+ NULL,
+};
+
+static struct attribute_group armv8_pmuv3_caps_attr_group = {
+ .name = "caps",
+ .attrs = armv8_pmuv3_caps_attrs,
+};
+
/*
* Perf Events' indices
*/
#define ARMV8_IDX_CYCLE_COUNTER 0
#define ARMV8_IDX_COUNTER0 1
-#define ARMV8_IDX_COUNTER_LAST(cpu_pmu) \
- (ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
/*
#define ARMV8_IDX_TO_COUNTER(x) \
(((x) - ARMV8_IDX_COUNTER0) & ARMV8_PMU_COUNTER_MASK)
+/*
+ * This code is really good
+ */
+
+#define PMEVN_CASE(n, case_macro) \
+ case n: case_macro(n); break
+
+#define PMEVN_SWITCH(x, case_macro) \
+ do { \
+ switch (x) { \
+ PMEVN_CASE(0, case_macro); \
+ PMEVN_CASE(1, case_macro); \
+ PMEVN_CASE(2, case_macro); \
+ PMEVN_CASE(3, case_macro); \
+ PMEVN_CASE(4, case_macro); \
+ PMEVN_CASE(5, case_macro); \
+ PMEVN_CASE(6, case_macro); \
+ PMEVN_CASE(7, case_macro); \
+ PMEVN_CASE(8, case_macro); \
+ PMEVN_CASE(9, case_macro); \
+ PMEVN_CASE(10, case_macro); \
+ PMEVN_CASE(11, case_macro); \
+ PMEVN_CASE(12, case_macro); \
+ PMEVN_CASE(13, case_macro); \
+ PMEVN_CASE(14, case_macro); \
+ PMEVN_CASE(15, case_macro); \
+ PMEVN_CASE(16, case_macro); \
+ PMEVN_CASE(17, case_macro); \
+ PMEVN_CASE(18, case_macro); \
+ PMEVN_CASE(19, case_macro); \
+ PMEVN_CASE(20, case_macro); \
+ PMEVN_CASE(21, case_macro); \
+ PMEVN_CASE(22, case_macro); \
+ PMEVN_CASE(23, case_macro); \
+ PMEVN_CASE(24, case_macro); \
+ PMEVN_CASE(25, case_macro); \
+ PMEVN_CASE(26, case_macro); \
+ PMEVN_CASE(27, case_macro); \
+ PMEVN_CASE(28, case_macro); \
+ PMEVN_CASE(29, case_macro); \
+ PMEVN_CASE(30, case_macro); \
+ default: WARN(1, "Invalid PMEV* index\n"); \
+ } \
+ } while (0)
+
+#define RETURN_READ_PMEVCNTRN(n) \
+ return read_sysreg(pmevcntr##n##_el0)
+static unsigned long read_pmevcntrn(int n)
+{
+ PMEVN_SWITCH(n, RETURN_READ_PMEVCNTRN);
+ return 0;
+}
+
+#define WRITE_PMEVCNTRN(n) \
+ write_sysreg(val, pmevcntr##n##_el0)
+static void write_pmevcntrn(int n, unsigned long val)
+{
+ PMEVN_SWITCH(n, WRITE_PMEVCNTRN);
+}
+
+#define WRITE_PMEVTYPERN(n) \
+ write_sysreg(val, pmevtyper##n##_el0)
+static void write_pmevtypern(int n, unsigned long val)
+{
+ PMEVN_SWITCH(n, WRITE_PMEVTYPERN);
+}
+
static inline u32 armv8pmu_pmcr_read(void)
{
return read_sysreg(pmcr_el0);
return pmovsr & ARMV8_PMU_OVERFLOWED_MASK;
}
-static inline int armv8pmu_counter_valid(struct arm_pmu *cpu_pmu, int idx)
-{
- return idx >= ARMV8_IDX_CYCLE_COUNTER &&
- idx <= ARMV8_IDX_COUNTER_LAST(cpu_pmu);
-}
-
static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx)
{
return pmnc & BIT(ARMV8_IDX_TO_COUNTER(idx));
}
-static inline void armv8pmu_select_counter(int idx)
+static inline u32 armv8pmu_read_evcntr(int idx)
{
u32 counter = ARMV8_IDX_TO_COUNTER(idx);
- write_sysreg(counter, pmselr_el0);
- isb();
-}
-static inline u64 armv8pmu_read_evcntr(int idx)
-{
- armv8pmu_select_counter(idx);
- return read_sysreg(pmxevcntr_el0);
+ return read_pmevcntrn(counter);
}
static inline u64 armv8pmu_read_hw_counter(struct perf_event *event)
static u64 armv8pmu_read_counter(struct perf_event *event)
{
- struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
struct hw_perf_event *hwc = &event->hw;
int idx = hwc->idx;
u64 value = 0;
- if (!armv8pmu_counter_valid(cpu_pmu, idx))
- pr_err("CPU%u reading wrong counter %d\n",
- smp_processor_id(), idx);
- else if (idx == ARMV8_IDX_CYCLE_COUNTER)
+ if (idx == ARMV8_IDX_CYCLE_COUNTER)
value = read_sysreg(pmccntr_el0);
else
value = armv8pmu_read_hw_counter(event);
static inline void armv8pmu_write_evcntr(int idx, u64 value)
{
- armv8pmu_select_counter(idx);
- write_sysreg(value, pmxevcntr_el0);
+ u32 counter = ARMV8_IDX_TO_COUNTER(idx);
+
+ write_pmevcntrn(counter, value);
}
static inline void armv8pmu_write_hw_counter(struct perf_event *event,
static void armv8pmu_write_counter(struct perf_event *event, u64 value)
{
- struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
struct hw_perf_event *hwc = &event->hw;
int idx = hwc->idx;
value = armv8pmu_bias_long_counter(event, value);
- if (!armv8pmu_counter_valid(cpu_pmu, idx))
- pr_err("CPU%u writing wrong counter %d\n",
- smp_processor_id(), idx);
- else if (idx == ARMV8_IDX_CYCLE_COUNTER)
+ if (idx == ARMV8_IDX_CYCLE_COUNTER)
write_sysreg(value, pmccntr_el0);
else
armv8pmu_write_hw_counter(event, value);
static inline void armv8pmu_write_evtype(int idx, u32 val)
{
- armv8pmu_select_counter(idx);
+ u32 counter = ARMV8_IDX_TO_COUNTER(idx);
+
val &= ARMV8_PMU_EVTYPE_MASK;
- write_sysreg(val, pmxevtyper_el0);
+ write_pmevtypern(counter, val);
}
static inline void armv8pmu_write_event_type(struct perf_event *event)
armv8pmu_write_evtype(idx - 1, hwc->config_base);
armv8pmu_write_evtype(idx, chain_evt);
} else {
- armv8pmu_write_evtype(idx, hwc->config_base);
+ if (idx == ARMV8_IDX_CYCLE_COUNTER)
+ write_sysreg(hwc->config_base, pmccfiltr_el0);
+ else
+ armv8pmu_write_evtype(idx, hwc->config_base);
}
}
static inline void armv8pmu_enable_counter(u32 mask)
{
+ /*
+ * Make sure event configuration register writes are visible before we
+ * enable the counter.
+ * */
+ isb();
write_sysreg(mask, pmcntenset_el0);
}
static inline void armv8pmu_disable_counter(u32 mask)
{
write_sysreg(mask, pmcntenclr_el0);
+ /*
+ * Make sure the effects of disabling the counter are visible before we
+ * start configuring the event.
+ */
+ isb();
}
static inline void armv8pmu_disable_event_counter(struct perf_event *event)
static void armv8pmu_enable_event(struct perf_event *event)
{
- unsigned long flags;
- struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
- struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
-
/*
* Enable counter and interrupt, and set the counter to count
* the event that we're interested in.
*/
- raw_spin_lock_irqsave(&events->pmu_lock, flags);
/*
* Disable counter
armv8pmu_disable_event_counter(event);
/*
- * Set event (if destined for PMNx counters).
+ * Set event.
*/
armv8pmu_write_event_type(event);
* Enable counter
*/
armv8pmu_enable_event_counter(event);
-
- raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
}
static void armv8pmu_disable_event(struct perf_event *event)
{
- unsigned long flags;
- struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
- struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
-
- /*
- * Disable counter and interrupt
- */
- raw_spin_lock_irqsave(&events->pmu_lock, flags);
-
/*
* Disable counter
*/
* Disable interrupt for this counter
*/
armv8pmu_disable_event_irq(event);
-
- raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
}
static void armv8pmu_start(struct arm_pmu *cpu_pmu)
{
- unsigned long flags;
- struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
-
- raw_spin_lock_irqsave(&events->pmu_lock, flags);
/* Enable all counters */
armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E);
- raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
}
static void armv8pmu_stop(struct arm_pmu *cpu_pmu)
{
- unsigned long flags;
- struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
-
- raw_spin_lock_irqsave(&events->pmu_lock, flags);
/* Disable all counters */
armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E);
- raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
}
static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu)
if (!armpmu_event_set_period(event))
continue;
+ /*
+ * Perf event overflow will queue the processing of the event as
+ * an irq_work which will be taken care of in the handling of
+ * IPI_IRQ_WORK.
+ */
if (perf_event_overflow(event, &data, regs))
cpu_pmu->disable(event);
}
armv8pmu_start(cpu_pmu);
- /*
- * Handle the pending perf events.
- *
- * Note: this call *must* be run with interrupts disabled. For
- * platforms that can have the PMU interrupts raised as an NMI, this
- * will not work.
- */
- irq_work_run();
-
return IRQ_HANDLED;
}
bitmap_from_arr32(cpu_pmu->pmceid_ext_bitmap,
pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
+
+ /* store PMMIR_EL1 register for sysfs */
+ if (pmuver >= ID_AA64DFR0_PMUVER_8_4 && (pmceid_raw[1] & BIT(31)))
+ cpu_pmu->reg_pmmir = read_cpuid(PMMIR_EL1);
+ else
+ cpu_pmu->reg_pmmir = 0;
}
static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)
static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name,
int (*map_event)(struct perf_event *event),
const struct attribute_group *events,
- const struct attribute_group *format)
+ const struct attribute_group *format,
+ const struct attribute_group *caps)
{
int ret = armv8pmu_probe_pmu(cpu_pmu);
if (ret)
events : &armv8_pmuv3_events_attr_group;
cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = format ?
format : &armv8_pmuv3_format_attr_group;
+ cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_CAPS] = caps ?
+ caps : &armv8_pmuv3_caps_attr_group;
return 0;
}
+static int armv8_pmu_init_nogroups(struct arm_pmu *cpu_pmu, char *name,
+ int (*map_event)(struct perf_event *event))
+{
+ return armv8_pmu_init(cpu_pmu, name, map_event, NULL, NULL, NULL);
+}
+
static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_pmuv3",
- armv8_pmuv3_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_pmuv3",
+ armv8_pmuv3_map_event);
}
static int armv8_a34_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cortex_a34",
- armv8_pmuv3_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a34",
+ armv8_pmuv3_map_event);
}
static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cortex_a35",
- armv8_a53_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a35",
+ armv8_a53_map_event);
}
static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cortex_a53",
- armv8_a53_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a53",
+ armv8_a53_map_event);
}
static int armv8_a55_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cortex_a55",
- armv8_pmuv3_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a55",
+ armv8_pmuv3_map_event);
}
static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cortex_a57",
- armv8_a57_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a57",
+ armv8_a57_map_event);
}
static int armv8_a65_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cortex_a65",
- armv8_pmuv3_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a65",
+ armv8_pmuv3_map_event);
}
static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cortex_a72",
- armv8_a57_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a72",
+ armv8_a57_map_event);
}
static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cortex_a73",
- armv8_a73_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a73",
+ armv8_a73_map_event);
}
static int armv8_a75_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cortex_a75",
- armv8_pmuv3_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a75",
+ armv8_pmuv3_map_event);
}
static int armv8_a76_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cortex_a76",
- armv8_pmuv3_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a76",
+ armv8_pmuv3_map_event);
}
static int armv8_a77_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cortex_a77",
- armv8_pmuv3_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a77",
+ armv8_pmuv3_map_event);
}
static int armv8_e1_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_neoverse_e1",
- armv8_pmuv3_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_neoverse_e1",
+ armv8_pmuv3_map_event);
}
static int armv8_n1_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_neoverse_n1",
- armv8_pmuv3_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_neoverse_n1",
+ armv8_pmuv3_map_event);
}
static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_cavium_thunder",
- armv8_thunder_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cavium_thunder",
+ armv8_thunder_map_event);
}
static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu)
{
- return armv8_pmu_init(cpu_pmu, "armv8_brcm_vulcan",
- armv8_vulcan_map_event, NULL, NULL);
+ return armv8_pmu_init_nogroups(cpu_pmu, "armv8_brcm_vulcan",
+ armv8_vulcan_map_event);
}
static const struct of_device_id armv8_pmu_of_device_ids[] = {
/*
* Our handling of compat tasks (PERF_SAMPLE_REGS_ABI_32) is weird, but
- * we're stuck with it for ABI compatability reasons.
+ * we're stuck with it for ABI compatibility reasons.
*
* For a 32-bit consumer inspecting a 32-bit task, then it will look at
* the first 16 registers (see arch/arm/include/uapi/asm/perf_regs.h).
aarch64_insn_is_msr_imm(insn) ||
aarch64_insn_is_msr_reg(insn) ||
aarch64_insn_is_exception(insn) ||
- aarch64_insn_is_eret(insn))
+ aarch64_insn_is_eret(insn) ||
+ aarch64_insn_is_eret_auth(insn))
return false;
/*
!= AARCH64_INSN_SPCLREG_DAIF;
/*
- * The HINT instruction is is problematic when single-stepping,
- * except for the NOP case.
+ * The HINT instruction is steppable only if it is in whitelist
+ * and the rest of other such instructions are blocked for
+ * single stepping as they may cause exception or other
+ * unintended behaviour.
*/
if (aarch64_insn_is_hint(insn))
return aarch64_insn_is_steppable_hint(insn);
void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)
{
- struct kretprobe_instance *ri = NULL;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address = 0;
- unsigned long trampoline_address =
- (unsigned long)&kretprobe_trampoline;
- kprobe_opcode_t *correct_ret_addr = NULL;
-
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
-
- /*
- * It is possible to have multiple instances associated with a given
- * task either because multiple functions in the call path have
- * return probes installed on them, and/or more than one
- * return probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always pushed into the head of the list
- * - when multiple return probes are registered for the same
- * function, the (chronologically) first instance's ret_addr
- * will be the real return address, and all the rest will
- * point to kretprobe_trampoline.
- */
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- orig_ret_address = (unsigned long)ri->ret_addr;
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
-
- correct_ret_addr = ri->ret_addr;
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- if (ri->rp && ri->rp->handler) {
- __this_cpu_write(current_kprobe, &ri->rp->kp);
- get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
- ri->ret_addr = correct_ret_addr;
- ri->rp->handler(ri, regs);
- __this_cpu_write(current_kprobe, NULL);
- }
-
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_hash_unlock(current, &flags);
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
- return (void *)orig_ret_address;
+ return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline,
+ (void *)kernel_stack_pointer(regs));
}
void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *)regs->regs[30];
+ ri->fp = (void *)kernel_stack_pointer(regs);
/* replace return addr (x30) with trampoline */
regs->regs[30] = (long)&kretprobe_trampoline;
#include <linux/lockdep.h>
#include <linux/mman.h>
#include <linux/mm.h>
+#include <linux/nospec.h>
#include <linux/stddef.h>
#include <linux/sysctl.h>
#include <linux/unistd.h>
#include <asm/exec.h>
#include <asm/fpsimd.h>
#include <asm/mmu_context.h>
+#include <asm/mte.h>
#include <asm/processor.h>
#include <asm/pointer_auth.h>
#include <asm/stacktrace.h>
const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >>
PSR_BTYPE_SHIFT];
- printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO BTYPE=%s)\n",
+ printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO BTYPE=%s)\n",
pstate,
pstate & PSR_N_BIT ? 'N' : 'n',
pstate & PSR_Z_BIT ? 'Z' : 'z',
pstate & PSR_F_BIT ? 'F' : 'f',
pstate & PSR_PAN_BIT ? '+' : '-',
pstate & PSR_UAO_BIT ? '+' : '-',
+ pstate & PSR_TCO_BIT ? '+' : '-',
btype_str);
}
}
tls_thread_flush();
flush_ptrace_hw_breakpoint(current);
flush_tagged_addr_state();
+ flush_mte_state();
}
void release_thread(struct task_struct *dead_task)
dst->thread.sve_state = NULL;
clear_tsk_thread_flag(dst, TIF_SVE);
+ /* clear any pending asynchronous tag fault raised by the parent */
+ clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT);
+
return 0;
}
cpus_have_const_cap(ARM64_HAS_UAO))
childregs->pstate |= PSR_UAO_BIT;
- if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE)
- set_ssbs_bit(childregs);
+ spectre_v4_enable_task_mitigation(p);
if (system_uses_irq_prio_masking())
childregs->pmr_save = GIC_PRIO_IRQON;
*/
static void ssbs_thread_switch(struct task_struct *next)
{
- struct pt_regs *regs = task_pt_regs(next);
-
/*
* Nothing to do for kernel threads, but 'regs' may be junk
* (e.g. idle task) so check the flags and bail early.
* If all CPUs implement the SSBS extension, then we just need to
* context-switch the PSTATE field.
*/
- if (cpu_have_feature(cpu_feature(SSBS)))
+ if (cpus_have_const_cap(ARM64_SSBS))
return;
- /* If the mitigation is enabled, then we leave SSBS clear. */
- if ((arm64_get_ssbd_state() == ARM64_SSBD_FORCE_ENABLE) ||
- test_tsk_thread_flag(next, TIF_SSBD))
- return;
-
- if (compat_user_mode(regs))
- set_compat_ssbs_bit(regs);
- else if (user_mode(regs))
- set_ssbs_bit(regs);
+ spectre_v4_enable_task_mitigation(next);
}
/*
*/
dsb(ish);
+ /*
+ * MTE thread switching must happen after the DSB above to ensure that
+ * any asynchronous tag check faults have been logged in the TFSR*_EL1
+ * registers.
+ */
+ mte_thread_switch(next);
+
/* the actual thread switch */
last = cpu_switch_to(prev, next);
current->mm->context.flags = is_compat_task() ? MMCF_AARCH32 : 0;
ptrauth_thread_init_user(current);
+
+ if (task_spec_ssb_noexec(current)) {
+ arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS,
+ PR_SPEC_ENABLE);
+ }
}
#ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
*/
static unsigned int tagged_addr_disabled;
-long set_tagged_addr_ctrl(unsigned long arg)
+long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg)
{
- if (is_compat_task())
+ unsigned long valid_mask = PR_TAGGED_ADDR_ENABLE;
+ struct thread_info *ti = task_thread_info(task);
+
+ if (is_compat_thread(ti))
return -EINVAL;
- if (arg & ~PR_TAGGED_ADDR_ENABLE)
+
+ if (system_supports_mte())
+ valid_mask |= PR_MTE_TCF_MASK | PR_MTE_TAG_MASK;
+
+ if (arg & ~valid_mask)
return -EINVAL;
/*
if (arg & PR_TAGGED_ADDR_ENABLE && tagged_addr_disabled)
return -EINVAL;
- update_thread_flag(TIF_TAGGED_ADDR, arg & PR_TAGGED_ADDR_ENABLE);
+ if (set_mte_ctrl(task, arg) != 0)
+ return -EINVAL;
+
+ update_ti_thread_flag(ti, TIF_TAGGED_ADDR, arg & PR_TAGGED_ADDR_ENABLE);
return 0;
}
-long get_tagged_addr_ctrl(void)
+long get_tagged_addr_ctrl(struct task_struct *task)
{
- if (is_compat_task())
+ long ret = 0;
+ struct thread_info *ti = task_thread_info(task);
+
+ if (is_compat_thread(ti))
return -EINVAL;
- if (test_thread_flag(TIF_TAGGED_ADDR))
- return PR_TAGGED_ADDR_ENABLE;
+ if (test_ti_thread_flag(ti, TIF_TAGGED_ADDR))
+ ret = PR_TAGGED_ADDR_ENABLE;
- return 0;
+ ret |= get_mte_ctrl(task);
+
+ return ret;
}
/*
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Handle detection, reporting and mitigation of Spectre v1, v2 and v4, as
+ * detailed at:
+ *
+ * https://developer.arm.com/support/arm-security-updates/speculative-processor-vulnerability
+ *
+ * This code was originally written hastily under an awful lot of stress and so
+ * aspects of it are somewhat hacky. Unfortunately, changing anything in here
+ * instantly makes me feel ill. Thanks, Jann. Thann.
+ *
+ * Copyright (C) 2018 ARM Ltd, All Rights Reserved.
+ * Copyright (C) 2020 Google LLC
+ *
+ * "If there's something strange in your neighbourhood, who you gonna call?"
+ *
+ * Authors: Will Deacon <will@kernel.org> and Marc Zyngier <maz@kernel.org>
+ */
+
+#include <linux/arm-smccc.h>
+#include <linux/cpu.h>
+#include <linux/device.h>
+#include <linux/nospec.h>
+#include <linux/prctl.h>
+#include <linux/sched/task_stack.h>
+
+#include <asm/spectre.h>
+#include <asm/traps.h>
+
+/*
+ * We try to ensure that the mitigation state can never change as the result of
+ * onlining a late CPU.
+ */
+static void update_mitigation_state(enum mitigation_state *oldp,
+ enum mitigation_state new)
+{
+ enum mitigation_state state;
+
+ do {
+ state = READ_ONCE(*oldp);
+ if (new <= state)
+ break;
+
+ /* Userspace almost certainly can't deal with this. */
+ if (WARN_ON(system_capabilities_finalized()))
+ break;
+ } while (cmpxchg_relaxed(oldp, state, new) != state);
+}
+
+/*
+ * Spectre v1.
+ *
+ * The kernel can't protect userspace for this one: it's each person for
+ * themselves. Advertise what we're doing and be done with it.
+ */
+ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ return sprintf(buf, "Mitigation: __user pointer sanitization\n");
+}
+
+/*
+ * Spectre v2.
+ *
+ * This one sucks. A CPU is either:
+ *
+ * - Mitigated in hardware and advertised by ID_AA64PFR0_EL1.CSV2.
+ * - Mitigated in hardware and listed in our "safe list".
+ * - Mitigated in software by firmware.
+ * - Mitigated in software by a CPU-specific dance in the kernel.
+ * - Vulnerable.
+ *
+ * It's not unlikely for different CPUs in a big.LITTLE system to fall into
+ * different camps.
+ */
+static enum mitigation_state spectre_v2_state;
+
+static bool __read_mostly __nospectre_v2;
+static int __init parse_spectre_v2_param(char *str)
+{
+ __nospectre_v2 = true;
+ return 0;
+}
+early_param("nospectre_v2", parse_spectre_v2_param);
+
+static bool spectre_v2_mitigations_off(void)
+{
+ bool ret = __nospectre_v2 || cpu_mitigations_off();
+
+ if (ret)
+ pr_info_once("spectre-v2 mitigation disabled by command line option\n");
+
+ return ret;
+}
+
+ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ switch (spectre_v2_state) {
+ case SPECTRE_UNAFFECTED:
+ return sprintf(buf, "Not affected\n");
+ case SPECTRE_MITIGATED:
+ return sprintf(buf, "Mitigation: Branch predictor hardening\n");
+ case SPECTRE_VULNERABLE:
+ fallthrough;
+ default:
+ return sprintf(buf, "Vulnerable\n");
+ }
+}
+
+static enum mitigation_state spectre_v2_get_cpu_hw_mitigation_state(void)
+{
+ u64 pfr0;
+ static const struct midr_range spectre_v2_safe_list[] = {
+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
+ MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
+ MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
+ MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
+ MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
+ { /* sentinel */ }
+ };
+
+ /* If the CPU has CSV2 set, we're safe */
+ pfr0 = read_cpuid(ID_AA64PFR0_EL1);
+ if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT))
+ return SPECTRE_UNAFFECTED;
+
+ /* Alternatively, we have a list of unaffected CPUs */
+ if (is_midr_in_range_list(read_cpuid_id(), spectre_v2_safe_list))
+ return SPECTRE_UNAFFECTED;
+
+ return SPECTRE_VULNERABLE;
+}
+
+#define SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED (1)
+
+static enum mitigation_state spectre_v2_get_cpu_fw_mitigation_state(void)
+{
+ int ret;
+ struct arm_smccc_res res;
+
+ arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
+ ARM_SMCCC_ARCH_WORKAROUND_1, &res);
+
+ ret = res.a0;
+ switch (ret) {
+ case SMCCC_RET_SUCCESS:
+ return SPECTRE_MITIGATED;
+ case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
+ return SPECTRE_UNAFFECTED;
+ default:
+ fallthrough;
+ case SMCCC_RET_NOT_SUPPORTED:
+ return SPECTRE_VULNERABLE;
+ }
+}
+
+bool has_spectre_v2(const struct arm64_cpu_capabilities *entry, int scope)
+{
+ WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
+
+ if (spectre_v2_get_cpu_hw_mitigation_state() == SPECTRE_UNAFFECTED)
+ return false;
+
+ if (spectre_v2_get_cpu_fw_mitigation_state() == SPECTRE_UNAFFECTED)
+ return false;
+
+ return true;
+}
+
+DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
+
+enum mitigation_state arm64_get_spectre_v2_state(void)
+{
+ return spectre_v2_state;
+}
+
+#ifdef CONFIG_KVM
+#include <asm/cacheflush.h>
+#include <asm/kvm_asm.h>
+
+atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1);
+
+static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
+ const char *hyp_vecs_end)
+{
+ void *dst = lm_alias(__bp_harden_hyp_vecs + slot * SZ_2K);
+ int i;
+
+ for (i = 0; i < SZ_2K; i += 0x80)
+ memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
+
+ __flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
+}
+
+static void install_bp_hardening_cb(bp_hardening_cb_t fn)
+{
+ static DEFINE_RAW_SPINLOCK(bp_lock);
+ int cpu, slot = -1;
+ const char *hyp_vecs_start = __smccc_workaround_1_smc;
+ const char *hyp_vecs_end = __smccc_workaround_1_smc +
+ __SMCCC_WORKAROUND_1_SMC_SZ;
+
+ /*
+ * detect_harden_bp_fw() passes NULL for the hyp_vecs start/end if
+ * we're a guest. Skip the hyp-vectors work.
+ */
+ if (!is_hyp_mode_available()) {
+ __this_cpu_write(bp_hardening_data.fn, fn);
+ return;
+ }
+
+ raw_spin_lock(&bp_lock);
+ for_each_possible_cpu(cpu) {
+ if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
+ slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
+ break;
+ }
+ }
+
+ if (slot == -1) {
+ slot = atomic_inc_return(&arm64_el2_vector_last_slot);
+ BUG_ON(slot >= BP_HARDEN_EL2_SLOTS);
+ __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
+ }
+
+ __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
+ __this_cpu_write(bp_hardening_data.fn, fn);
+ raw_spin_unlock(&bp_lock);
+}
+#else
+static void install_bp_hardening_cb(bp_hardening_cb_t fn)
+{
+ __this_cpu_write(bp_hardening_data.fn, fn);
+}
+#endif /* CONFIG_KVM */
+
+static void call_smc_arch_workaround_1(void)
+{
+ arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
+}
+
+static void call_hvc_arch_workaround_1(void)
+{
+ arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
+}
+
+static void qcom_link_stack_sanitisation(void)
+{
+ u64 tmp;
+
+ asm volatile("mov %0, x30 \n"
+ ".rept 16 \n"
+ "bl . + 4 \n"
+ ".endr \n"
+ "mov x30, %0 \n"
+ : "=&r" (tmp));
+}
+
+static enum mitigation_state spectre_v2_enable_fw_mitigation(void)
+{
+ bp_hardening_cb_t cb;
+ enum mitigation_state state;
+
+ state = spectre_v2_get_cpu_fw_mitigation_state();
+ if (state != SPECTRE_MITIGATED)
+ return state;
+
+ if (spectre_v2_mitigations_off())
+ return SPECTRE_VULNERABLE;
+
+ switch (arm_smccc_1_1_get_conduit()) {
+ case SMCCC_CONDUIT_HVC:
+ cb = call_hvc_arch_workaround_1;
+ break;
+
+ case SMCCC_CONDUIT_SMC:
+ cb = call_smc_arch_workaround_1;
+ break;
+
+ default:
+ return SPECTRE_VULNERABLE;
+ }
+
+ install_bp_hardening_cb(cb);
+ return SPECTRE_MITIGATED;
+}
+
+static enum mitigation_state spectre_v2_enable_sw_mitigation(void)
+{
+ u32 midr;
+
+ if (spectre_v2_mitigations_off())
+ return SPECTRE_VULNERABLE;
+
+ midr = read_cpuid_id();
+ if (((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR) &&
+ ((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR_V1))
+ return SPECTRE_VULNERABLE;
+
+ install_bp_hardening_cb(qcom_link_stack_sanitisation);
+ return SPECTRE_MITIGATED;
+}
+
+void spectre_v2_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
+{
+ enum mitigation_state state;
+
+ WARN_ON(preemptible());
+
+ state = spectre_v2_get_cpu_hw_mitigation_state();
+ if (state == SPECTRE_VULNERABLE)
+ state = spectre_v2_enable_fw_mitigation();
+ if (state == SPECTRE_VULNERABLE)
+ state = spectre_v2_enable_sw_mitigation();
+
+ update_mitigation_state(&spectre_v2_state, state);
+}
+
+/*
+ * Spectre v4.
+ *
+ * If you thought Spectre v2 was nasty, wait until you see this mess. A CPU is
+ * either:
+ *
+ * - Mitigated in hardware and listed in our "safe list".
+ * - Mitigated in hardware via PSTATE.SSBS.
+ * - Mitigated in software by firmware (sometimes referred to as SSBD).
+ *
+ * Wait, that doesn't sound so bad, does it? Keep reading...
+ *
+ * A major source of headaches is that the software mitigation is enabled both
+ * on a per-task basis, but can also be forced on for the kernel, necessitating
+ * both context-switch *and* entry/exit hooks. To make it even worse, some CPUs
+ * allow EL0 to toggle SSBS directly, which can end up with the prctl() state
+ * being stale when re-entering the kernel. The usual big.LITTLE caveats apply,
+ * so you can have systems that have both firmware and SSBS mitigations. This
+ * means we actually have to reject late onlining of CPUs with mitigations if
+ * all of the currently onlined CPUs are safelisted, as the mitigation tends to
+ * be opt-in for userspace. Yes, really, the cure is worse than the disease.
+ *
+ * The only good part is that if the firmware mitigation is present, then it is
+ * present for all CPUs, meaning we don't have to worry about late onlining of a
+ * vulnerable CPU if one of the boot CPUs is using the firmware mitigation.
+ *
+ * Give me a VAX-11/780 any day of the week...
+ */
+static enum mitigation_state spectre_v4_state;
+
+/* This is the per-cpu state tracking whether we need to talk to firmware */
+DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
+
+enum spectre_v4_policy {
+ SPECTRE_V4_POLICY_MITIGATION_DYNAMIC,
+ SPECTRE_V4_POLICY_MITIGATION_ENABLED,
+ SPECTRE_V4_POLICY_MITIGATION_DISABLED,
+};
+
+static enum spectre_v4_policy __read_mostly __spectre_v4_policy;
+
+static const struct spectre_v4_param {
+ const char *str;
+ enum spectre_v4_policy policy;
+} spectre_v4_params[] = {
+ { "force-on", SPECTRE_V4_POLICY_MITIGATION_ENABLED, },
+ { "force-off", SPECTRE_V4_POLICY_MITIGATION_DISABLED, },
+ { "kernel", SPECTRE_V4_POLICY_MITIGATION_DYNAMIC, },
+};
+static int __init parse_spectre_v4_param(char *str)
+{
+ int i;
+
+ if (!str || !str[0])
+ return -EINVAL;
+
+ for (i = 0; i < ARRAY_SIZE(spectre_v4_params); i++) {
+ const struct spectre_v4_param *param = &spectre_v4_params[i];
+
+ if (strncmp(str, param->str, strlen(param->str)))
+ continue;
+
+ __spectre_v4_policy = param->policy;
+ return 0;
+ }
+
+ return -EINVAL;
+}
+early_param("ssbd", parse_spectre_v4_param);
+
+/*
+ * Because this was all written in a rush by people working in different silos,
+ * we've ended up with multiple command line options to control the same thing.
+ * Wrap these up in some helpers, which prefer disabling the mitigation if faced
+ * with contradictory parameters. The mitigation is always either "off",
+ * "dynamic" or "on".
+ */
+static bool spectre_v4_mitigations_off(void)
+{
+ bool ret = cpu_mitigations_off() ||
+ __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DISABLED;
+
+ if (ret)
+ pr_info_once("spectre-v4 mitigation disabled by command-line option\n");
+
+ return ret;
+}
+
+/* Do we need to toggle the mitigation state on entry to/exit from the kernel? */
+static bool spectre_v4_mitigations_dynamic(void)
+{
+ return !spectre_v4_mitigations_off() &&
+ __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DYNAMIC;
+}
+
+static bool spectre_v4_mitigations_on(void)
+{
+ return !spectre_v4_mitigations_off() &&
+ __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_ENABLED;
+}
+
+ssize_t cpu_show_spec_store_bypass(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ switch (spectre_v4_state) {
+ case SPECTRE_UNAFFECTED:
+ return sprintf(buf, "Not affected\n");
+ case SPECTRE_MITIGATED:
+ return sprintf(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n");
+ case SPECTRE_VULNERABLE:
+ fallthrough;
+ default:
+ return sprintf(buf, "Vulnerable\n");
+ }
+}
+
+enum mitigation_state arm64_get_spectre_v4_state(void)
+{
+ return spectre_v4_state;
+}
+
+static enum mitigation_state spectre_v4_get_cpu_hw_mitigation_state(void)
+{
+ static const struct midr_range spectre_v4_safe_list[] = {
+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
+ MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
+ MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
+ MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
+ MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
+ { /* sentinel */ },
+ };
+
+ if (is_midr_in_range_list(read_cpuid_id(), spectre_v4_safe_list))
+ return SPECTRE_UNAFFECTED;
+
+ /* CPU features are detected first */
+ if (this_cpu_has_cap(ARM64_SSBS))
+ return SPECTRE_MITIGATED;
+
+ return SPECTRE_VULNERABLE;
+}
+
+static enum mitigation_state spectre_v4_get_cpu_fw_mitigation_state(void)
+{
+ int ret;
+ struct arm_smccc_res res;
+
+ arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
+ ARM_SMCCC_ARCH_WORKAROUND_2, &res);
+
+ ret = res.a0;
+ switch (ret) {
+ case SMCCC_RET_SUCCESS:
+ return SPECTRE_MITIGATED;
+ case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
+ fallthrough;
+ case SMCCC_RET_NOT_REQUIRED:
+ return SPECTRE_UNAFFECTED;
+ default:
+ fallthrough;
+ case SMCCC_RET_NOT_SUPPORTED:
+ return SPECTRE_VULNERABLE;
+ }
+}
+
+bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope)
+{
+ enum mitigation_state state;
+
+ WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
+
+ state = spectre_v4_get_cpu_hw_mitigation_state();
+ if (state == SPECTRE_VULNERABLE)
+ state = spectre_v4_get_cpu_fw_mitigation_state();
+
+ return state != SPECTRE_UNAFFECTED;
+}
+
+static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
+{
+ if (user_mode(regs))
+ return 1;
+
+ if (instr & BIT(PSTATE_Imm_shift))
+ regs->pstate |= PSR_SSBS_BIT;
+ else
+ regs->pstate &= ~PSR_SSBS_BIT;
+
+ arm64_skip_faulting_instruction(regs, 4);
+ return 0;
+}
+
+static struct undef_hook ssbs_emulation_hook = {
+ .instr_mask = ~(1U << PSTATE_Imm_shift),
+ .instr_val = 0xd500401f | PSTATE_SSBS,
+ .fn = ssbs_emulation_handler,
+};
+
+static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
+{
+ static bool undef_hook_registered = false;
+ static DEFINE_RAW_SPINLOCK(hook_lock);
+ enum mitigation_state state;
+
+ /*
+ * If the system is mitigated but this CPU doesn't have SSBS, then
+ * we must be on the safelist and there's nothing more to do.
+ */
+ state = spectre_v4_get_cpu_hw_mitigation_state();
+ if (state != SPECTRE_MITIGATED || !this_cpu_has_cap(ARM64_SSBS))
+ return state;
+
+ raw_spin_lock(&hook_lock);
+ if (!undef_hook_registered) {
+ register_undef_hook(&ssbs_emulation_hook);
+ undef_hook_registered = true;
+ }
+ raw_spin_unlock(&hook_lock);
+
+ if (spectre_v4_mitigations_off()) {
+ sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
+ asm volatile(SET_PSTATE_SSBS(1));
+ return SPECTRE_VULNERABLE;
+ }
+
+ /* SCTLR_EL1.DSSBS was initialised to 0 during boot */
+ asm volatile(SET_PSTATE_SSBS(0));
+ return SPECTRE_MITIGATED;
+}
+
+/*
+ * Patch a branch over the Spectre-v4 mitigation code with a NOP so that
+ * we fallthrough and check whether firmware needs to be called on this CPU.
+ */
+void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt,
+ __le32 *origptr,
+ __le32 *updptr, int nr_inst)
+{
+ BUG_ON(nr_inst != 1); /* Branch -> NOP */
+
+ if (spectre_v4_mitigations_off())
+ return;
+
+ if (cpus_have_final_cap(ARM64_SSBS))
+ return;
+
+ if (spectre_v4_mitigations_dynamic())
+ *updptr = cpu_to_le32(aarch64_insn_gen_nop());
+}
+
+/*
+ * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction
+ * to call into firmware to adjust the mitigation state.
+ */
+void __init spectre_v4_patch_fw_mitigation_conduit(struct alt_instr *alt,
+ __le32 *origptr,
+ __le32 *updptr, int nr_inst)
+{
+ u32 insn;
+
+ BUG_ON(nr_inst != 1); /* NOP -> HVC/SMC */
+
+ switch (arm_smccc_1_1_get_conduit()) {
+ case SMCCC_CONDUIT_HVC:
+ insn = aarch64_insn_get_hvc_value();
+ break;
+ case SMCCC_CONDUIT_SMC:
+ insn = aarch64_insn_get_smc_value();
+ break;
+ default:
+ return;
+ }
+
+ *updptr = cpu_to_le32(insn);
+}
+
+static enum mitigation_state spectre_v4_enable_fw_mitigation(void)
+{
+ enum mitigation_state state;
+
+ state = spectre_v4_get_cpu_fw_mitigation_state();
+ if (state != SPECTRE_MITIGATED)
+ return state;
+
+ if (spectre_v4_mitigations_off()) {
+ arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, false, NULL);
+ return SPECTRE_VULNERABLE;
+ }
+
+ arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, true, NULL);
+
+ if (spectre_v4_mitigations_dynamic())
+ __this_cpu_write(arm64_ssbd_callback_required, 1);
+
+ return SPECTRE_MITIGATED;
+}
+
+void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
+{
+ enum mitigation_state state;
+
+ WARN_ON(preemptible());
+
+ state = spectre_v4_enable_hw_mitigation();
+ if (state == SPECTRE_VULNERABLE)
+ state = spectre_v4_enable_fw_mitigation();
+
+ update_mitigation_state(&spectre_v4_state, state);
+}
+
+static void __update_pstate_ssbs(struct pt_regs *regs, bool state)
+{
+ u64 bit = compat_user_mode(regs) ? PSR_AA32_SSBS_BIT : PSR_SSBS_BIT;
+
+ if (state)
+ regs->pstate |= bit;
+ else
+ regs->pstate &= ~bit;
+}
+
+void spectre_v4_enable_task_mitigation(struct task_struct *tsk)
+{
+ struct pt_regs *regs = task_pt_regs(tsk);
+ bool ssbs = false, kthread = tsk->flags & PF_KTHREAD;
+
+ if (spectre_v4_mitigations_off())
+ ssbs = true;
+ else if (spectre_v4_mitigations_dynamic() && !kthread)
+ ssbs = !test_tsk_thread_flag(tsk, TIF_SSBD);
+
+ __update_pstate_ssbs(regs, ssbs);
+}
+
+/*
+ * The Spectre-v4 mitigation can be controlled via a prctl() from userspace.
+ * This is interesting because the "speculation disabled" behaviour can be
+ * configured so that it is preserved across exec(), which means that the
+ * prctl() may be necessary even when PSTATE.SSBS can be toggled directly
+ * from userspace.
+ */
+static void ssbd_prctl_enable_mitigation(struct task_struct *task)
+{
+ task_clear_spec_ssb_noexec(task);
+ task_set_spec_ssb_disable(task);
+ set_tsk_thread_flag(task, TIF_SSBD);
+}
+
+static void ssbd_prctl_disable_mitigation(struct task_struct *task)
+{
+ task_clear_spec_ssb_noexec(task);
+ task_clear_spec_ssb_disable(task);
+ clear_tsk_thread_flag(task, TIF_SSBD);
+}
+
+static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl)
+{
+ switch (ctrl) {
+ case PR_SPEC_ENABLE:
+ /* Enable speculation: disable mitigation */
+ /*
+ * Force disabled speculation prevents it from being
+ * re-enabled.
+ */
+ if (task_spec_ssb_force_disable(task))
+ return -EPERM;
+
+ /*
+ * If the mitigation is forced on, then speculation is forced
+ * off and we again prevent it from being re-enabled.
+ */
+ if (spectre_v4_mitigations_on())
+ return -EPERM;
+
+ ssbd_prctl_disable_mitigation(task);
+ break;
+ case PR_SPEC_FORCE_DISABLE:
+ /* Force disable speculation: force enable mitigation */
+ /*
+ * If the mitigation is forced off, then speculation is forced
+ * on and we prevent it from being disabled.
+ */
+ if (spectre_v4_mitigations_off())
+ return -EPERM;
+
+ task_set_spec_ssb_force_disable(task);
+ fallthrough;
+ case PR_SPEC_DISABLE:
+ /* Disable speculation: enable mitigation */
+ /* Same as PR_SPEC_FORCE_DISABLE */
+ if (spectre_v4_mitigations_off())
+ return -EPERM;
+
+ ssbd_prctl_enable_mitigation(task);
+ break;
+ case PR_SPEC_DISABLE_NOEXEC:
+ /* Disable speculation until execve(): enable mitigation */
+ /*
+ * If the mitigation state is forced one way or the other, then
+ * we must fail now before we try to toggle it on execve().
+ */
+ if (task_spec_ssb_force_disable(task) ||
+ spectre_v4_mitigations_off() ||
+ spectre_v4_mitigations_on()) {
+ return -EPERM;
+ }
+
+ ssbd_prctl_enable_mitigation(task);
+ task_set_spec_ssb_noexec(task);
+ break;
+ default:
+ return -ERANGE;
+ }
+
+ spectre_v4_enable_task_mitigation(task);
+ return 0;
+}
+
+int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
+ unsigned long ctrl)
+{
+ switch (which) {
+ case PR_SPEC_STORE_BYPASS:
+ return ssbd_prctl_set(task, ctrl);
+ default:
+ return -ENODEV;
+ }
+}
+
+static int ssbd_prctl_get(struct task_struct *task)
+{
+ switch (spectre_v4_state) {
+ case SPECTRE_UNAFFECTED:
+ return PR_SPEC_NOT_AFFECTED;
+ case SPECTRE_MITIGATED:
+ if (spectre_v4_mitigations_on())
+ return PR_SPEC_NOT_AFFECTED;
+
+ if (spectre_v4_mitigations_dynamic())
+ break;
+
+ /* Mitigations are disabled, so we're vulnerable. */
+ fallthrough;
+ case SPECTRE_VULNERABLE:
+ fallthrough;
+ default:
+ return PR_SPEC_ENABLE;
+ }
+
+ /* Check the mitigation state for this task */
+ if (task_spec_ssb_force_disable(task))
+ return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
+
+ if (task_spec_ssb_noexec(task))
+ return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
+
+ if (task_spec_ssb_disable(task))
+ return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
+
+ return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
+}
+
+int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
+{
+ switch (which) {
+ case PR_SPEC_STORE_BYPASS:
+ return ssbd_prctl_get(task);
+ default:
+ return -ENODEV;
+ }
+}
#include <asm/cpufeature.h>
#include <asm/debug-monitors.h>
#include <asm/fpsimd.h>
+#include <asm/mte.h>
#include <asm/pointer_auth.h>
#include <asm/stacktrace.h>
#include <asm/syscall.h>
#endif /* CONFIG_CHECKPOINT_RESTORE */
#endif /* CONFIG_ARM64_PTR_AUTH */
+#ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
+static int tagged_addr_ctrl_get(struct task_struct *target,
+ const struct user_regset *regset,
+ struct membuf to)
+{
+ long ctrl = get_tagged_addr_ctrl(target);
+
+ if (IS_ERR_VALUE(ctrl))
+ return ctrl;
+
+ return membuf_write(&to, &ctrl, sizeof(ctrl));
+}
+
+static int tagged_addr_ctrl_set(struct task_struct *target, const struct
+ user_regset *regset, unsigned int pos,
+ unsigned int count, const void *kbuf, const
+ void __user *ubuf)
+{
+ int ret;
+ long ctrl;
+
+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, 0, -1);
+ if (ret)
+ return ret;
+
+ return set_tagged_addr_ctrl(target, ctrl);
+}
+#endif
+
enum aarch64_regset {
REGSET_GPR,
REGSET_FPR,
REGSET_PACG_KEYS,
#endif
#endif
+#ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
+ REGSET_TAGGED_ADDR_CTRL,
+#endif
};
static const struct user_regset aarch64_regsets[] = {
},
#endif
#endif
+#ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
+ [REGSET_TAGGED_ADDR_CTRL] = {
+ .core_note_type = NT_ARM_TAGGED_ADDR_CTRL,
+ .n = 1,
+ .size = sizeof(long),
+ .align = sizeof(long),
+ .regset_get = tagged_addr_ctrl_get,
+ .set = tagged_addr_ctrl_set,
+ },
+#endif
};
static const struct user_regset_view user_aarch64_view = {
long arch_ptrace(struct task_struct *child, long request,
unsigned long addr, unsigned long data)
{
+ switch (request) {
+ case PTRACE_PEEKMTETAGS:
+ case PTRACE_POKEMTETAGS:
+ return mte_ptrace_copy_tags(child, request, addr, data);
+ }
+
return ptrace_request(child, request, addr, data);
}
* We also reserve IL for the kernel; SS is handled dynamically.
*/
#define SPSR_EL1_AARCH64_RES0_BITS \
- (GENMASK_ULL(63, 32) | GENMASK_ULL(27, 25) | GENMASK_ULL(23, 22) | \
+ (GENMASK_ULL(63, 32) | GENMASK_ULL(27, 26) | GENMASK_ULL(23, 22) | \
GENMASK_ULL(20, 13) | GENMASK_ULL(5, 5))
#define SPSR_EL1_AARCH32_RES0_BITS \
(GENMASK_ULL(63, 32) | GENMASK_ULL(22, 22) | GENMASK_ULL(20, 20))
mov x14, xzr /* x14 = entry ptr */
mov x13, xzr /* x13 = copy dest */
- /* Clear the sctlr_el2 flags. */
- mrs x0, CurrentEL
- cmp x0, #CurrentEL_EL2
- b.ne 1f
- mrs x0, sctlr_el2
- mov_q x1, SCTLR_ELx_FLAGS
- bic x0, x0, x1
- pre_disable_mmu_workaround
- msr sctlr_el2, x0
- isb
-1:
-
/* Check if the new image needs relocation. */
tbnz x16, IND_DONE_BIT, .Ldone
void *addr;
};
-static int save_return_addr(struct stackframe *frame, void *d)
+static bool save_return_addr(void *d, unsigned long pc)
{
struct return_address_data *data = d;
if (!data->level) {
- data->addr = (void *)frame->pc;
- return 1;
+ data->addr = (void *)pc;
+ return false;
} else {
--data->level;
- return 0;
+ return true;
}
}
NOKPROBE_SYMBOL(save_return_addr);
if (vq) {
/*
* This assumes that the SVE state has already been saved to
- * the task struct by calling preserve_fpsimd_context().
+ * the task struct by calling the function
+ * fpsimd_signal_preserve_current_state().
*/
err |= __copy_to_user((char __user *)ctx + SVE_SIG_REGS_OFFSET,
current->thread.sve_state,
regs->pstate |= PSR_BTYPE_C;
}
+ /* TCO (Tag Check Override) always cleared for signal handlers */
+ regs->pstate &= ~PSR_TCO_BIT;
+
if (ka->sa.sa_flags & SA_RESTORER)
sigtramp = ka->sa.sa_restorer;
else
if (thread_flags & _TIF_UPROBE)
uprobe_notify_resume(regs);
+ if (thread_flags & _TIF_MTE_ASYNC_FAULT) {
+ clear_thread_flag(TIF_MTE_ASYNC_FAULT);
+ send_sig_fault(SIGSEGV, SEGV_MTEAERR,
+ (void __user *)NULL, current);
+ }
+
if (thread_flags & _TIF_SIGPENDING)
do_signal(regs);
#include <asm/assembler.h>
.macro SMCCC instr
- .cfi_startproc
\instr #0
ldr x4, [sp]
stp x0, x1, [x4, #ARM_SMCCC_RES_X0_OFFS]
b.ne 1f
str x6, [x4, ARM_SMCCC_QUIRK_STATE_OFFS]
1: ret
- .cfi_endproc
.endm
/*
#include <linux/completion.h>
#include <linux/of.h>
#include <linux/irq_work.h>
+#include <linux/kernel_stat.h>
#include <linux/kexec.h>
#include <linux/kvm_host.h>
IPI_CPU_CRASH_STOP,
IPI_TIMER,
IPI_IRQ_WORK,
- IPI_WAKEUP
+ IPI_WAKEUP,
+ NR_IPI
};
+static int ipi_irq_base __read_mostly;
+static int nr_ipi __read_mostly = NR_IPI;
+static struct irq_desc *ipi_desc[NR_IPI] __read_mostly;
+
+static void ipi_setup(int cpu);
+
#ifdef CONFIG_HOTPLUG_CPU
+static void ipi_teardown(int cpu);
static int op_cpu_kill(unsigned int cpu);
#else
static inline int op_cpu_kill(unsigned int cpu)
*/
notify_cpu_starting(cpu);
+ ipi_setup(cpu);
+
store_cpu_topology(cpu);
numa_add_cpu(cpu);
* and we must not schedule until we're ready to give up the cpu.
*/
set_cpu_online(cpu, false);
+ ipi_teardown(cpu);
/*
* OK - migrate IRQs away from this CPU
}
}
-void (*__smp_cross_call)(const struct cpumask *, unsigned int);
-
-void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
-{
- __smp_cross_call = fn;
-}
-
static const char *ipi_types[NR_IPI] __tracepoint_string = {
#define S(x,s) [x] = s
S(IPI_RESCHEDULE, "Rescheduling interrupts"),
S(IPI_WAKEUP, "CPU wake-up interrupts"),
};
-static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
-{
- trace_ipi_raise(target, ipi_types[ipinr]);
- __smp_cross_call(target, ipinr);
-}
+static void smp_cross_call(const struct cpumask *target, unsigned int ipinr);
-void show_ipi_list(struct seq_file *p, int prec)
+unsigned long irq_err_count;
+
+int arch_show_interrupts(struct seq_file *p, int prec)
{
unsigned int cpu, i;
for (i = 0; i < NR_IPI; i++) {
+ unsigned int irq = irq_desc_get_irq(ipi_desc[i]);
seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
prec >= 4 ? " " : "");
for_each_online_cpu(cpu)
- seq_printf(p, "%10u ",
- __get_irq_stat(cpu, ipi_irqs[i]));
+ seq_printf(p, "%10u ", kstat_irqs_cpu(irq, cpu));
seq_printf(p, " %s\n", ipi_types[i]);
}
-}
-
-u64 smp_irq_stat_cpu(unsigned int cpu)
-{
- u64 sum = 0;
- int i;
- for (i = 0; i < NR_IPI; i++)
- sum += __get_irq_stat(cpu, ipi_irqs[i]);
-
- return sum;
+ seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
+ return 0;
}
void arch_send_call_function_ipi_mask(const struct cpumask *mask)
#ifdef CONFIG_IRQ_WORK
void arch_irq_work_raise(void)
{
- if (__smp_cross_call)
- smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
+ smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
}
#endif
/*
* Main handler for inter-processor interrupts
*/
-void handle_IPI(int ipinr, struct pt_regs *regs)
+static void do_handle_IPI(int ipinr)
{
unsigned int cpu = smp_processor_id();
- struct pt_regs *old_regs = set_irq_regs(regs);
- if ((unsigned)ipinr < NR_IPI) {
+ if ((unsigned)ipinr < NR_IPI)
trace_ipi_entry_rcuidle(ipi_types[ipinr]);
- __inc_irq_stat(cpu, ipi_irqs[ipinr]);
- }
switch (ipinr) {
case IPI_RESCHEDULE:
break;
case IPI_CALL_FUNC:
- irq_enter();
generic_smp_call_function_interrupt();
- irq_exit();
break;
case IPI_CPU_STOP:
- irq_enter();
local_cpu_stop();
- irq_exit();
break;
case IPI_CPU_CRASH_STOP:
if (IS_ENABLED(CONFIG_KEXEC_CORE)) {
- irq_enter();
- ipi_cpu_crash_stop(cpu, regs);
+ ipi_cpu_crash_stop(cpu, get_irq_regs());
unreachable();
}
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
case IPI_TIMER:
- irq_enter();
tick_receive_broadcast();
- irq_exit();
break;
#endif
#ifdef CONFIG_IRQ_WORK
case IPI_IRQ_WORK:
- irq_enter();
irq_work_run();
- irq_exit();
break;
#endif
if ((unsigned)ipinr < NR_IPI)
trace_ipi_exit_rcuidle(ipi_types[ipinr]);
- set_irq_regs(old_regs);
+}
+
+static irqreturn_t ipi_handler(int irq, void *data)
+{
+ do_handle_IPI(irq - ipi_irq_base);
+ return IRQ_HANDLED;
+}
+
+static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
+{
+ trace_ipi_raise(target, ipi_types[ipinr]);
+ __ipi_send_mask(ipi_desc[ipinr], target);
+}
+
+static void ipi_setup(int cpu)
+{
+ int i;
+
+ if (WARN_ON_ONCE(!ipi_irq_base))
+ return;
+
+ for (i = 0; i < nr_ipi; i++)
+ enable_percpu_irq(ipi_irq_base + i, 0);
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static void ipi_teardown(int cpu)
+{
+ int i;
+
+ if (WARN_ON_ONCE(!ipi_irq_base))
+ return;
+
+ for (i = 0; i < nr_ipi; i++)
+ disable_percpu_irq(ipi_irq_base + i);
+}
+#endif
+
+void __init set_smp_ipi_range(int ipi_base, int n)
+{
+ int i;
+
+ WARN_ON(n < NR_IPI);
+ nr_ipi = min(n, NR_IPI);
+
+ for (i = 0; i < nr_ipi; i++) {
+ int err;
+
+ err = request_percpu_irq(ipi_base + i, ipi_handler,
+ "IPI", &cpu_number);
+ WARN_ON(err);
+
+ ipi_desc[i] = irq_to_desc(ipi_base + i);
+ irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);
+ }
+
+ ipi_irq_base = ipi_base;
+
+ /* Setup the boot CPU immediately */
+ ipi_setup(smp_processor_id());
}
void smp_send_reschedule(int cpu)
/*
* We write the release address as LE regardless of the native
- * endianess of the kernel. Therefore, any boot-loaders that
+ * endianness of the kernel. Therefore, any boot-loaders that
* read this address need to convert this address to the
- * boot-loader's endianess before jumping. This is mandated by
+ * boot-loader's endianness before jumping. This is mandated by
* the boot protocol.
*/
writeq_relaxed(__pa_symbol(secondary_holding_pen), release_addr);
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2018 ARM Ltd, All Rights Reserved.
- */
-
-#include <linux/compat.h>
-#include <linux/errno.h>
-#include <linux/prctl.h>
-#include <linux/sched.h>
-#include <linux/sched/task_stack.h>
-#include <linux/thread_info.h>
-
-#include <asm/cpufeature.h>
-
-static void ssbd_ssbs_enable(struct task_struct *task)
-{
- u64 val = is_compat_thread(task_thread_info(task)) ?
- PSR_AA32_SSBS_BIT : PSR_SSBS_BIT;
-
- task_pt_regs(task)->pstate |= val;
-}
-
-static void ssbd_ssbs_disable(struct task_struct *task)
-{
- u64 val = is_compat_thread(task_thread_info(task)) ?
- PSR_AA32_SSBS_BIT : PSR_SSBS_BIT;
-
- task_pt_regs(task)->pstate &= ~val;
-}
-
-/*
- * prctl interface for SSBD
- */
-static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl)
-{
- int state = arm64_get_ssbd_state();
-
- /* Unsupported */
- if (state == ARM64_SSBD_UNKNOWN)
- return -ENODEV;
-
- /* Treat the unaffected/mitigated state separately */
- if (state == ARM64_SSBD_MITIGATED) {
- switch (ctrl) {
- case PR_SPEC_ENABLE:
- return -EPERM;
- case PR_SPEC_DISABLE:
- case PR_SPEC_FORCE_DISABLE:
- return 0;
- }
- }
-
- /*
- * Things are a bit backward here: the arm64 internal API
- * *enables the mitigation* when the userspace API *disables
- * speculation*. So much fun.
- */
- switch (ctrl) {
- case PR_SPEC_ENABLE:
- /* If speculation is force disabled, enable is not allowed */
- if (state == ARM64_SSBD_FORCE_ENABLE ||
- task_spec_ssb_force_disable(task))
- return -EPERM;
- task_clear_spec_ssb_disable(task);
- clear_tsk_thread_flag(task, TIF_SSBD);
- ssbd_ssbs_enable(task);
- break;
- case PR_SPEC_DISABLE:
- if (state == ARM64_SSBD_FORCE_DISABLE)
- return -EPERM;
- task_set_spec_ssb_disable(task);
- set_tsk_thread_flag(task, TIF_SSBD);
- ssbd_ssbs_disable(task);
- break;
- case PR_SPEC_FORCE_DISABLE:
- if (state == ARM64_SSBD_FORCE_DISABLE)
- return -EPERM;
- task_set_spec_ssb_disable(task);
- task_set_spec_ssb_force_disable(task);
- set_tsk_thread_flag(task, TIF_SSBD);
- ssbd_ssbs_disable(task);
- break;
- default:
- return -ERANGE;
- }
-
- return 0;
-}
-
-int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
- unsigned long ctrl)
-{
- switch (which) {
- case PR_SPEC_STORE_BYPASS:
- return ssbd_prctl_set(task, ctrl);
- default:
- return -ENODEV;
- }
-}
-
-static int ssbd_prctl_get(struct task_struct *task)
-{
- switch (arm64_get_ssbd_state()) {
- case ARM64_SSBD_UNKNOWN:
- return -ENODEV;
- case ARM64_SSBD_FORCE_ENABLE:
- return PR_SPEC_DISABLE;
- case ARM64_SSBD_KERNEL:
- if (task_spec_ssb_force_disable(task))
- return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
- if (task_spec_ssb_disable(task))
- return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
- return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
- case ARM64_SSBD_FORCE_DISABLE:
- return PR_SPEC_ENABLE;
- default:
- return PR_SPEC_NOT_AFFECTED;
- }
-}
-
-int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
-{
- switch (which) {
- case PR_SPEC_STORE_BYPASS:
- return ssbd_prctl_get(task);
- default:
- return -ENODEV;
- }
-}
NOKPROBE_SYMBOL(unwind_frame);
void notrace walk_stackframe(struct task_struct *tsk, struct stackframe *frame,
- int (*fn)(struct stackframe *, void *), void *data)
+ bool (*fn)(void *, unsigned long), void *data)
{
while (1) {
int ret;
- if (fn(frame, data))
+ if (!fn(data, frame->pc))
break;
ret = unwind_frame(tsk, frame);
if (ret < 0)
}
NOKPROBE_SYMBOL(walk_stackframe);
-#ifdef CONFIG_STACKTRACE
-struct stack_trace_data {
- struct stack_trace *trace;
- unsigned int no_sched_functions;
- unsigned int skip;
-};
-
-static int save_trace(struct stackframe *frame, void *d)
+static void dump_backtrace_entry(unsigned long where, const char *loglvl)
{
- struct stack_trace_data *data = d;
- struct stack_trace *trace = data->trace;
- unsigned long addr = frame->pc;
-
- if (data->no_sched_functions && in_sched_functions(addr))
- return 0;
- if (data->skip) {
- data->skip--;
- return 0;
- }
-
- trace->entries[trace->nr_entries++] = addr;
-
- return trace->nr_entries >= trace->max_entries;
+ printk("%s %pS\n", loglvl, (void *)where);
}
-void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace)
+void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
+ const char *loglvl)
{
- struct stack_trace_data data;
struct stackframe frame;
+ int skip = 0;
- data.trace = trace;
- data.skip = trace->skip;
- data.no_sched_functions = 0;
+ pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
- start_backtrace(&frame, regs->regs[29], regs->pc);
- walk_stackframe(current, &frame, save_trace, &data);
-}
-EXPORT_SYMBOL_GPL(save_stack_trace_regs);
+ if (regs) {
+ if (user_mode(regs))
+ return;
+ skip = 1;
+ }
-static noinline void __save_stack_trace(struct task_struct *tsk,
- struct stack_trace *trace, unsigned int nosched)
-{
- struct stack_trace_data data;
- struct stackframe frame;
+ if (!tsk)
+ tsk = current;
if (!try_get_task_stack(tsk))
return;
- data.trace = trace;
- data.skip = trace->skip;
- data.no_sched_functions = nosched;
-
- if (tsk != current) {
- start_backtrace(&frame, thread_saved_fp(tsk),
- thread_saved_pc(tsk));
- } else {
- /* We don't want this function nor the caller */
- data.skip += 2;
+ if (tsk == current) {
start_backtrace(&frame,
(unsigned long)__builtin_frame_address(0),
- (unsigned long)__save_stack_trace);
+ (unsigned long)dump_backtrace);
+ } else {
+ /*
+ * task blocked in __switch_to
+ */
+ start_backtrace(&frame,
+ thread_saved_fp(tsk),
+ thread_saved_pc(tsk));
}
- walk_stackframe(tsk, &frame, save_trace, &data);
+ printk("%sCall trace:\n", loglvl);
+ do {
+ /* skip until specified stack frame */
+ if (!skip) {
+ dump_backtrace_entry(frame.pc, loglvl);
+ } else if (frame.fp == regs->regs[29]) {
+ skip = 0;
+ /*
+ * Mostly, this is the case where this function is
+ * called in panic/abort. As exception handler's
+ * stack frame does not contain the corresponding pc
+ * at which an exception has taken place, use regs->pc
+ * instead.
+ */
+ dump_backtrace_entry(regs->pc, loglvl);
+ }
+ } while (!unwind_frame(tsk, &frame));
put_task_stack(tsk);
}
-void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
+void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl)
{
- __save_stack_trace(tsk, trace, 1);
+ dump_backtrace(NULL, tsk, loglvl);
+ barrier();
}
-EXPORT_SYMBOL_GPL(save_stack_trace_tsk);
-void save_stack_trace(struct stack_trace *trace)
+#ifdef CONFIG_STACKTRACE
+
+void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie,
+ struct task_struct *task, struct pt_regs *regs)
{
- __save_stack_trace(current, trace, 0);
+ struct stackframe frame;
+
+ if (regs)
+ start_backtrace(&frame, regs->regs[29], regs->pc);
+ else if (task == current)
+ start_backtrace(&frame,
+ (unsigned long)__builtin_frame_address(0),
+ (unsigned long)arch_stack_walk);
+ else
+ start_backtrace(&frame, thread_saved_fp(task),
+ thread_saved_pc(task));
+
+ walk_stackframe(task, &frame, consume_entry, cookie);
}
-EXPORT_SYMBOL_GPL(save_stack_trace);
#endif
#include <asm/daifflags.h>
#include <asm/debug-monitors.h>
#include <asm/exec.h>
+#include <asm/mte.h>
#include <asm/memory.h>
#include <asm/mmu_context.h>
#include <asm/smp_plat.h>
* have turned the mitigation on. If the user has forcefully
* disabled it, make sure their wishes are obeyed.
*/
- if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE)
- arm64_set_ssbd_mitigation(false);
+ spectre_v4_enable_mitigation(NULL);
+
+ /* Restore additional MTE-specific configuration */
+ mte_suspend_exit();
}
/*
local_daif_restore(DAIF_PROCCTX);
user_exit();
+ if (system_supports_mte() && (flags & _TIF_MTE_ASYNC_FAULT)) {
+ /*
+ * Process the asynchronous tag check fault before the actual
+ * syscall. do_notify_resume() will send a signal to userspace
+ * before the syscall is restarted.
+ */
+ regs->regs[0] = -ERESTARTNOINTR;
+ return;
+ }
+
if (has_syscall_work(flags)) {
/*
* The de-facto standard way to skip a system call using ptrace
if (mpidr & MPIDR_UP_BITMASK)
return;
- /* Create cpu topology mapping based on MPIDR. */
- if (mpidr & MPIDR_MT_BITMASK) {
- /* Multiprocessor system : Multi-threads per core */
- cpuid_topo->thread_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
- cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
- cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 2) |
- MPIDR_AFFINITY_LEVEL(mpidr, 3) << 8;
- } else {
- /* Multiprocessor system : Single-thread per core */
- cpuid_topo->thread_id = -1;
- cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
- cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 1) |
- MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8 |
- MPIDR_AFFINITY_LEVEL(mpidr, 3) << 16;
- }
+ /*
+ * This would be the place to create cpu topology based on MPIDR.
+ *
+ * However, it cannot be trusted to depict the actual topology; some
+ * pieces of the architecture enforce an artificial cap on Aff0 values
+ * (e.g. GICv3's ICC_SGI1R_EL1 limits it to 15), leading to an
+ * artificial cycling of Aff1, Aff2 and Aff3 values. IOW, these end up
+ * having absolutely no relationship to the actual underlying system
+ * topology, and cannot be reasonably used as core / package ID.
+ *
+ * If the MT bit is set, Aff0 *could* be used to define a thread ID, but
+ * we still wouldn't be able to obtain a sane core ID. This means we
+ * need to entirely ignore MPIDR for any topology deduction.
+ */
+ cpuid_topo->thread_id = -1;
+ cpuid_topo->core_id = cpuid;
+ cpuid_topo->package_id = cpu_to_node(cpuid);
pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n",
cpuid, cpuid_topo->package_id, cpuid_topo->core_id,
#include <asm/daifflags.h>
#include <asm/debug-monitors.h>
#include <asm/esr.h>
+#include <asm/extable.h>
#include <asm/insn.h>
#include <asm/kprobes.h>
#include <asm/traps.h>
int show_unhandled_signals = 0;
-static void dump_backtrace_entry(unsigned long where, const char *loglvl)
-{
- printk("%s %pS\n", loglvl, (void *)where);
-}
-
static void dump_kernel_instr(const char *lvl, struct pt_regs *regs)
{
unsigned long addr = instruction_pointer(regs);
printk("%sCode: %s\n", lvl, str);
}
-void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
- const char *loglvl)
-{
- struct stackframe frame;
- int skip = 0;
-
- pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
-
- if (regs) {
- if (user_mode(regs))
- return;
- skip = 1;
- }
-
- if (!tsk)
- tsk = current;
-
- if (!try_get_task_stack(tsk))
- return;
-
- if (tsk == current) {
- start_backtrace(&frame,
- (unsigned long)__builtin_frame_address(0),
- (unsigned long)dump_backtrace);
- } else {
- /*
- * task blocked in __switch_to
- */
- start_backtrace(&frame,
- thread_saved_fp(tsk),
- thread_saved_pc(tsk));
- }
-
- printk("%sCall trace:\n", loglvl);
- do {
- /* skip until specified stack frame */
- if (!skip) {
- dump_backtrace_entry(frame.pc, loglvl);
- } else if (frame.fp == regs->regs[29]) {
- skip = 0;
- /*
- * Mostly, this is the case where this function is
- * called in panic/abort. As exception handler's
- * stack frame does not contain the corresponding pc
- * at which an exception has taken place, use regs->pc
- * instead.
- */
- dump_backtrace_entry(regs->pc, loglvl);
- }
- } while (!unwind_frame(tsk, &frame));
-
- put_task_stack(tsk);
-}
-
-void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl)
-{
- dump_backtrace(NULL, tsk, loglvl);
- barrier();
-}
-
#ifdef CONFIG_PREEMPT
#define S_PREEMPT " PREEMPT"
#elif defined(CONFIG_PREEMPT_RT)
oops_exit();
if (in_interrupt())
- panic("Fatal exception in interrupt");
+ panic("%s: Fatal exception in interrupt", str);
if (panic_on_oops)
- panic("Fatal exception");
+ panic("%s: Fatal exception", str);
raw_spin_unlock_irqrestore(&die_lock, flags);
return fn ? fn(regs, instr) : 1;
}
-void force_signal_inject(int signal, int code, unsigned long address)
+void force_signal_inject(int signal, int code, unsigned long address, unsigned int err)
{
const char *desc;
struct pt_regs *regs = current_pt_regs();
signal = SIGKILL;
}
- arm64_notify_die(desc, regs, signal, code, (void __user *)address, 0);
+ arm64_notify_die(desc, regs, signal, code, (void __user *)address, err);
}
/*
code = SEGV_ACCERR;
mmap_read_unlock(current->mm);
- force_signal_inject(SIGSEGV, code, addr);
+ force_signal_inject(SIGSEGV, code, addr, 0);
}
void do_undefinstr(struct pt_regs *regs)
return;
BUG_ON(!user_mode(regs));
- force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
+ force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
}
NOKPROBE_SYMBOL(do_undefinstr);
void do_bti(struct pt_regs *regs)
{
BUG_ON(!user_mode(regs));
- force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
+ force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
}
NOKPROBE_SYMBOL(do_bti);
+void do_ptrauth_fault(struct pt_regs *regs, unsigned int esr)
+{
+ /*
+ * Unexpected FPAC exception or pointer authentication failure in
+ * the kernel: kill the task before it does any more harm.
+ */
+ BUG_ON(!user_mode(regs));
+ force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr);
+}
+NOKPROBE_SYMBOL(do_ptrauth_fault);
+
#define __user_cache_maint(insn, address, res) \
if (address >= user_addr_max()) { \
res = -EFAULT; \
__user_cache_maint("ic ivau", address, ret);
break;
default:
- force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
+ force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
return;
}
sysreg = esr_sys64_to_sysreg(esr);
if (do_emulate_mrs(regs, sysreg, rt) != 0)
- force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
+ force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
}
static void wfi_handler(unsigned int esr, struct pt_regs *regs)
[ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
[ESR_ELx_EC_SVE] = "SVE",
[ESR_ELx_EC_ERET] = "ERET/ERETAA/ERETAB",
+ [ESR_ELx_EC_FPAC] = "FPAC",
[ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
[ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
[ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
}
NOKPROBE_SYMBOL(enter_from_user_mode);
-void __pte_error(const char *file, int line, unsigned long val)
-{
- pr_err("%s:%d: bad pte %016lx.\n", file, line, val);
-}
-
-void __pmd_error(const char *file, int line, unsigned long val)
-{
- pr_err("%s:%d: bad pmd %016lx.\n", file, line, val);
-}
-
-void __pud_error(const char *file, int line, unsigned long val)
-{
- pr_err("%s:%d: bad pud %016lx.\n", file, line, val);
-}
-
-void __pgd_error(const char *file, int line, unsigned long val)
-{
- pr_err("%s:%d: bad pgd %016lx.\n", file, line, val);
-}
-
/* GENERIC_BUG traps */
int is_valid_bugaddr(unsigned long addr)
.imm = BUG_BRK_IMM,
};
+static int reserved_fault_handler(struct pt_regs *regs, unsigned int esr)
+{
+ pr_err("%s generated an invalid instruction at %pS!\n",
+ in_bpf_jit(regs) ? "BPF JIT" : "Kernel text patching",
+ (void *)instruction_pointer(regs));
+
+ /* We cannot handle this */
+ return DBG_HOOK_ERROR;
+}
+
+static struct break_hook fault_break_hook = {
+ .fn = reserved_fault_handler,
+ .imm = FAULT_BRK_IMM,
+};
+
#ifdef CONFIG_KASAN_SW_TAGS
#define KASAN_ESR_RECOVER 0x20
void __init trap_init(void)
{
register_kernel_break_hook(&bug_break_hook);
+ register_kernel_break_hook(&fault_break_hook);
#ifdef CONFIG_KASAN_SW_TAGS
register_kernel_break_hook(&kasan_break_hook);
#endif
#include <asm/vdso.h>
extern char vdso_start[], vdso_end[];
-#ifdef CONFIG_COMPAT_VDSO
extern char vdso32_start[], vdso32_end[];
-#endif /* CONFIG_COMPAT_VDSO */
enum vdso_abi {
VDSO_ABI_AA64,
-#ifdef CONFIG_COMPAT_VDSO
VDSO_ABI_AA32,
-#endif /* CONFIG_COMPAT_VDSO */
};
enum vvar_pages {
/*
* Create and map the vectors page for AArch32 tasks.
*/
-#ifdef CONFIG_COMPAT_VDSO
static int aarch32_vdso_mremap(const struct vm_special_mapping *sm,
struct vm_area_struct *new_vma)
{
return __vdso_remap(VDSO_ABI_AA32, sm, new_vma);
}
-#endif /* CONFIG_COMPAT_VDSO */
enum aarch32_map {
AA32_MAP_VECTORS, /* kuser helpers */
-#ifdef CONFIG_COMPAT_VDSO
+ AA32_MAP_SIGPAGE,
AA32_MAP_VVAR,
AA32_MAP_VDSO,
-#endif
- AA32_MAP_SIGPAGE
};
static struct page *aarch32_vectors_page __ro_after_init;
.name = "[vectors]", /* ABI */
.pages = &aarch32_vectors_page,
},
-#ifdef CONFIG_COMPAT_VDSO
+ [AA32_MAP_SIGPAGE] = {
+ .name = "[sigpage]", /* ABI */
+ .pages = &aarch32_sig_page,
+ },
[AA32_MAP_VVAR] = {
.name = "[vvar]",
.fault = vvar_fault,
.name = "[vdso]",
.mremap = aarch32_vdso_mremap,
},
-#endif /* CONFIG_COMPAT_VDSO */
- [AA32_MAP_SIGPAGE] = {
- .name = "[sigpage]", /* ABI */
- .pages = &aarch32_sig_page,
- },
};
static int aarch32_alloc_kuser_vdso_page(void)
return 0;
}
-#ifdef CONFIG_COMPAT_VDSO
static int __aarch32_alloc_vdso_pages(void)
{
+
+ if (!IS_ENABLED(CONFIG_COMPAT_VDSO))
+ return 0;
+
vdso_info[VDSO_ABI_AA32].dm = &aarch32_vdso_maps[AA32_MAP_VVAR];
vdso_info[VDSO_ABI_AA32].cm = &aarch32_vdso_maps[AA32_MAP_VDSO];
return __vdso_init(VDSO_ABI_AA32);
}
-#endif /* CONFIG_COMPAT_VDSO */
static int __init aarch32_alloc_vdso_pages(void)
{
int ret;
-#ifdef CONFIG_COMPAT_VDSO
ret = __aarch32_alloc_vdso_pages();
if (ret)
return ret;
-#endif
ret = aarch32_alloc_sigpage();
if (ret)
if (ret)
goto out;
-#ifdef CONFIG_COMPAT_VDSO
- ret = __setup_additional_pages(VDSO_ABI_AA32,
- mm,
- bprm,
- uses_interp);
- if (ret)
- goto out;
-#endif /* CONFIG_COMPAT_VDSO */
+ if (IS_ENABLED(CONFIG_COMPAT_VDSO)) {
+ ret = __setup_additional_pages(VDSO_ABI_AA32, mm, bprm,
+ uses_interp);
+ if (ret)
+ goto out;
+ }
ret = aarch32_sigreturn_setup(mm);
out:
}
arch_initcall(vdso_init);
-int arch_setup_additional_pages(struct linux_binprm *bprm,
- int uses_interp)
+int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
{
struct mm_struct *mm = current->mm;
int ret;
if (mmap_write_lock_killable(mm))
return -EINTR;
- ret = __setup_additional_pages(VDSO_ABI_AA64,
- mm,
- bprm,
- uses_interp);
-
+ ret = __setup_additional_pages(VDSO_ABI_AA64, mm, bprm, uses_interp);
mmap_write_unlock(mm);
return ret;
*/
#define RO_EXCEPTION_TABLE_ALIGN 8
+#define RUNTIME_DISCARD_EXIT
#include <asm-generic/vmlinux.lds.h>
#include <asm/cache.h>
* matching the same input section name. There is no documented
* order of matching.
*/
+ DISCARDS
/DISCARD/ : {
- EXIT_CALL
- *(.discard)
- *(.discard.*)
*(.interp .dynamic)
*(.dynsym .dynstr .hash .gnu.hash)
- *(.eh_frame)
}
- . = KIMAGE_VADDR + TEXT_OFFSET;
+ . = KIMAGE_VADDR;
.head.text : {
_text = .;
*(.got) /* Global offset table */
}
+ /*
+ * Make sure that the .got.plt is either completely empty or it
+ * contains only the lazy dispatch entries.
+ */
+ .got.plt : { *(.got.plt) }
+ ASSERT(SIZEOF(.got.plt) == 0 || SIZEOF(.got.plt) == 0x18,
+ "Unexpected GOT/PLT entries detected!")
+
. = ALIGN(SEGMENT_ALIGN);
_etext = .; /* End of text section */
_end = .;
STABS_DEBUG
+ DWARF_DEBUG
+ ELF_DETAILS
HEAD_SYMBOLS
+
+ /*
+ * Sections that should stay zero sized, which is safer to
+ * explicitly check instead of blindly discarding.
+ */
+ .plt : {
+ *(.plt) *(.plt.*) *(.iplt) *(.igot)
+ }
+ ASSERT(SIZEOF(.plt) == 0, "Unexpected run-time procedure linkages detected!")
+
+ .data.rel.ro : { *(.data.rel.ro) }
+ ASSERT(SIZEOF(.data.rel.ro) == 0, "Unexpected RELRO detected!")
}
#include "image-vars.h"
/*
* If padding is applied before .head.text, virt<->phys conversions will fail.
*/
-ASSERT(_text == (KIMAGE_VADDR + TEXT_OFFSET), "HEAD is misaligned")
+ASSERT(_text == KIMAGE_VADDR, "HEAD is misaligned")
Adds support for a virtual Performance Monitoring Unit (PMU) in
virtual machines.
-config KVM_INDIRECT_VECTORS
- def_bool HARDEN_BRANCH_PREDICTOR || RANDOMIZE_BASE
-
endif # KVM
endif # VIRTUALIZATION
}
}
+static int kvm_map_vectors(void)
+{
+ /*
+ * SV2 = ARM64_SPECTRE_V2
+ * HEL2 = ARM64_HARDEN_EL2_VECTORS
+ *
+ * !SV2 + !HEL2 -> use direct vectors
+ * SV2 + !HEL2 -> use hardened vectors in place
+ * !SV2 + HEL2 -> allocate one vector slot and use exec mapping
+ * SV2 + HEL2 -> use hardened vectors and use exec mapping
+ */
+ if (cpus_have_const_cap(ARM64_SPECTRE_V2)) {
+ __kvm_bp_vect_base = kvm_ksym_ref(__bp_harden_hyp_vecs);
+ __kvm_bp_vect_base = kern_hyp_va(__kvm_bp_vect_base);
+ }
+
+ if (cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS)) {
+ phys_addr_t vect_pa = __pa_symbol(__bp_harden_hyp_vecs);
+ unsigned long size = __BP_HARDEN_HYP_VECS_SZ;
+
+ /*
+ * Always allocate a spare vector slot, as we don't
+ * know yet which CPUs have a BP hardening slot that
+ * we can reuse.
+ */
+ __kvm_harden_el2_vector_slot = atomic_inc_return(&arm64_el2_vector_last_slot);
+ BUG_ON(__kvm_harden_el2_vector_slot >= BP_HARDEN_EL2_SLOTS);
+ return create_hyp_exec_mappings(vect_pa, size,
+ &__kvm_bp_vect_base);
+ }
+
+ return 0;
+}
+
static void cpu_init_hyp_mode(void)
{
phys_addr_t pgd_ptr;
* at EL2.
*/
if (this_cpu_has_cap(ARM64_SSBS) &&
- arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) {
+ arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) {
kvm_call_hyp_nvhe(__kvm_enable_ssbs);
}
}
}
}
- err = hyp_map_aux_data();
- if (err)
- kvm_err("Cannot map host auxiliary data: %d\n", err);
-
return 0;
out_err:
-DDISABLE_BRANCH_PROFILING \
$(DISABLE_STACKLEAK_PLUGIN)
-obj-$(CONFIG_KVM) += vhe/ nvhe/
-obj-$(CONFIG_KVM_INDIRECT_VECTORS) += smccc_wa.o
+obj-$(CONFIG_KVM) += vhe/ nvhe/ smccc_wa.o
ARM_SMCCC_ARCH_WORKAROUND_2)
cbnz w1, el1_trap
-#ifdef CONFIG_ARM64_SSBD
-alternative_cb arm64_enable_wa2_handling
- b wa2_end
-alternative_cb_end
- get_vcpu_ptr x2, x0
- ldr x0, [x2, #VCPU_WORKAROUND_FLAGS]
-
- // Sanitize the argument and update the guest flags
- ldr x1, [sp, #8] // Guest's x1
- clz w1, w1 // Murphy's device:
- lsr w1, w1, #5 // w1 = !!w1 without using
- eor w1, w1, #1 // the flags...
- bfi x0, x1, #VCPU_WORKAROUND_2_FLAG_SHIFT, #1
- str x0, [x2, #VCPU_WORKAROUND_FLAGS]
-
- /* Check that we actually need to perform the call */
- hyp_ldr_this_cpu x0, arm64_ssbd_callback_required, x2
- cbz x0, wa2_end
-
- mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2
- smc #0
-
- /* Don't leak data from the SMC call */
- mov x3, xzr
-wa2_end:
- mov x2, xzr
- mov x1, xzr
-#endif
-
wa_epilogue:
mov x0, xzr
add sp, sp, #16
valid_vect el1_error // Error 32-bit EL1
SYM_CODE_END(__kvm_hyp_vector)
-#ifdef CONFIG_KVM_INDIRECT_VECTORS
.macro hyp_ventry
.align 7
1: esb
1: .org __bp_harden_hyp_vecs + __BP_HARDEN_HYP_VECS_SZ
.org 1b
SYM_CODE_END(__bp_harden_hyp_vecs)
-#endif
return false;
}
-static inline bool __needs_ssbd_off(struct kvm_vcpu *vcpu)
-{
- if (!cpus_have_final_cap(ARM64_SSBD))
- return false;
-
- return !(vcpu->arch.workaround_flags & VCPU_WORKAROUND_2_FLAG);
-}
-
-static inline void __set_guest_arch_workaround_state(struct kvm_vcpu *vcpu)
-{
-#ifdef CONFIG_ARM64_SSBD
- /*
- * The host runs with the workaround always present. If the
- * guest wants it disabled, so be it...
- */
- if (__needs_ssbd_off(vcpu) &&
- __hyp_this_cpu_read(arm64_ssbd_callback_required))
- arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, 0, NULL);
-#endif
-}
-
-static inline void __set_host_arch_workaround_state(struct kvm_vcpu *vcpu)
-{
-#ifdef CONFIG_ARM64_SSBD
- /*
- * If the guest has disabled the workaround, bring it back on.
- */
- if (__needs_ssbd_off(vcpu) &&
- __hyp_this_cpu_read(arm64_ssbd_callback_required))
- arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, 1, NULL);
-#endif
-}
-
static inline void __kvm_unexpected_el2_exception(void)
{
unsigned long addr, fixup;
__debug_switch_to_guest(vcpu);
- __set_guest_arch_workaround_state(vcpu);
-
do {
/* Jump in the fire! */
exit_code = __guest_enter(vcpu, host_ctxt);
/* And we're baaack! */
} while (fixup_guest_exit(vcpu, &exit_code));
- __set_host_arch_workaround_state(vcpu);
-
__sysreg_save_state_nvhe(guest_ctxt);
__sysreg32_save_state(vcpu);
__timer_disable_traps(vcpu);
isb();
}
+ /*
+ * __load_guest_stage2() includes an ISB only when the AT
+ * workaround is applied. Take care of the opposite condition,
+ * ensuring that we always have an ISB, but not two ISBs back
+ * to back.
+ */
__load_guest_stage2(mmu);
+ asm(ALTERNATIVE("isb", "nop", ARM64_WORKAROUND_SPECULATIVE_AT));
}
static void __tlb_switch_to_host(struct tlb_inv_context *cxt)
sysreg_restore_guest_state_vhe(guest_ctxt);
__debug_switch_to_guest(vcpu);
- __set_guest_arch_workaround_state(vcpu);
-
do {
/* Jump in the fire! */
exit_code = __guest_enter(vcpu, host_ctxt);
/* And we're baaack! */
} while (fixup_guest_exit(vcpu, &exit_code));
- __set_host_arch_workaround_state(vcpu);
-
sysreg_save_guest_state_vhe(guest_ctxt);
__deactivate_traps(vcpu);
feature = smccc_get_arg1(vcpu);
switch (feature) {
case ARM_SMCCC_ARCH_WORKAROUND_1:
- switch (kvm_arm_harden_branch_predictor()) {
- case KVM_BP_HARDEN_UNKNOWN:
+ switch (arm64_get_spectre_v2_state()) {
+ case SPECTRE_VULNERABLE:
break;
- case KVM_BP_HARDEN_WA_NEEDED:
+ case SPECTRE_MITIGATED:
val = SMCCC_RET_SUCCESS;
break;
- case KVM_BP_HARDEN_NOT_REQUIRED:
+ case SPECTRE_UNAFFECTED:
val = SMCCC_RET_NOT_REQUIRED;
break;
}
break;
case ARM_SMCCC_ARCH_WORKAROUND_2:
- switch (kvm_arm_have_ssbd()) {
- case KVM_SSBD_FORCE_DISABLE:
- case KVM_SSBD_UNKNOWN:
+ switch (arm64_get_spectre_v4_state()) {
+ case SPECTRE_VULNERABLE:
break;
- case KVM_SSBD_KERNEL:
- val = SMCCC_RET_SUCCESS;
- break;
- case KVM_SSBD_FORCE_ENABLE:
- case KVM_SSBD_MITIGATED:
+ case SPECTRE_MITIGATED:
+ /*
+ * SSBS everywhere: Indicate no firmware
+ * support, as the SSBS support will be
+ * indicated to the guest and the default is
+ * safe.
+ *
+ * Otherwise, expose a permanent mitigation
+ * to the guest, and hide SSBS so that the
+ * guest stays protected.
+ */
+ if (cpus_have_final_cap(ARM64_SSBS))
+ break;
+ fallthrough;
+ case SPECTRE_UNAFFECTED:
val = SMCCC_RET_NOT_REQUIRED;
break;
}
for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
kvm_pmu_release_perf_event(&pmu->pmc[i]);
+ irq_work_sync(&vcpu->arch.pmu.overflow_work);
}
u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)
kvm_pmu_update_state(vcpu);
}
+/**
+ * When perf interrupt is an NMI, we cannot safely notify the vcpu corresponding
+ * to the event.
+ * This is why we need a callback to do it once outside of the NMI context.
+ */
+static void kvm_pmu_perf_overflow_notify_vcpu(struct irq_work *work)
+{
+ struct kvm_vcpu *vcpu;
+ struct kvm_pmu *pmu;
+
+ pmu = container_of(work, struct kvm_pmu, overflow_work);
+ vcpu = kvm_pmc_to_vcpu(pmu->pmc);
+
+ kvm_vcpu_kick(vcpu);
+}
+
/**
* When the perf event overflows, set the overflow status and inform the vcpu.
*/
if (kvm_pmu_overflow_status(vcpu)) {
kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
- kvm_vcpu_kick(vcpu);
+
+ if (!in_nmi())
+ kvm_vcpu_kick(vcpu);
+ else
+ irq_work_queue(&vcpu->arch.pmu.overflow_work);
}
cpu_pmu->pmu.start(perf_event, PERF_EF_RELOAD);
return ret;
}
+ init_irq_work(&vcpu->arch.pmu.overflow_work,
+ kvm_pmu_perf_overflow_notify_vcpu);
+
vcpu->arch.pmu.created = true;
return 0;
}
{
switch (regid) {
case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
- switch (kvm_arm_harden_branch_predictor()) {
- case KVM_BP_HARDEN_UNKNOWN:
+ switch (arm64_get_spectre_v2_state()) {
+ case SPECTRE_VULNERABLE:
return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL;
- case KVM_BP_HARDEN_WA_NEEDED:
+ case SPECTRE_MITIGATED:
return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_AVAIL;
- case KVM_BP_HARDEN_NOT_REQUIRED:
+ case SPECTRE_UNAFFECTED:
return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_REQUIRED;
}
return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL;
case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2:
- switch (kvm_arm_have_ssbd()) {
- case KVM_SSBD_FORCE_DISABLE:
- return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL;
- case KVM_SSBD_KERNEL:
- return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL;
- case KVM_SSBD_FORCE_ENABLE:
- case KVM_SSBD_MITIGATED:
+ switch (arm64_get_spectre_v4_state()) {
+ case SPECTRE_MITIGATED:
+ /*
+ * As for the hypercall discovery, we pretend we
+ * don't have any FW mitigation if SSBS is there at
+ * all times.
+ */
+ if (cpus_have_final_cap(ARM64_SSBS))
+ return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL;
+ fallthrough;
+ case SPECTRE_UNAFFECTED:
return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED;
- case KVM_SSBD_UNKNOWN:
- default:
- return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_UNKNOWN;
+ case SPECTRE_VULNERABLE:
+ return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL;
}
}
val = kvm_psci_version(vcpu, vcpu->kvm);
break;
case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
- val = get_kernel_wa_level(reg->id) & KVM_REG_FEATURE_LEVEL_MASK;
- break;
case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2:
val = get_kernel_wa_level(reg->id) & KVM_REG_FEATURE_LEVEL_MASK;
-
- if (val == KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL &&
- kvm_arm_get_vcpu_workaround_2_flag(vcpu))
- val |= KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED;
break;
default:
return -ENOENT;
KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED))
return -EINVAL;
- wa_level = val & KVM_REG_FEATURE_LEVEL_MASK;
-
- if (get_kernel_wa_level(reg->id) < wa_level)
- return -EINVAL;
-
/* The enabled bit must not be set unless the level is AVAIL. */
- if (wa_level != KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL &&
- wa_level != val)
+ if ((val & KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED) &&
+ (val & KVM_REG_FEATURE_LEVEL_MASK) != KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL)
return -EINVAL;
- /* Are we finished or do we need to check the enable bit ? */
- if (kvm_arm_have_ssbd() != KVM_SSBD_KERNEL)
- return 0;
-
/*
- * If this kernel supports the workaround to be switched on
- * or off, make sure it matches the requested setting.
+ * Map all the possible incoming states to the only two we
+ * really want to deal with.
*/
- switch (wa_level) {
- case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL:
- kvm_arm_set_vcpu_workaround_2_flag(vcpu,
- val & KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED);
+ switch (val & KVM_REG_FEATURE_LEVEL_MASK) {
+ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL:
+ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_UNKNOWN:
+ wa_level = KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL;
break;
+ case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL:
case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED:
- kvm_arm_set_vcpu_workaround_2_flag(vcpu, true);
+ wa_level = KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED;
break;
+ default:
+ return -EINVAL;
}
+ /*
+ * We can deal with NOT_AVAIL on NOT_REQUIRED, but not the
+ * other way around.
+ */
+ if (get_kernel_wa_level(reg->id) < wa_level)
+ return -EINVAL;
+
return 0;
default:
return -ENOENT;
vcpu->arch.reset_state.reset = false;
}
- /* Default workaround setup is enabled (if supported) */
- if (kvm_arm_have_ssbd() == KVM_SSBD_KERNEL)
- vcpu->arch.workaround_flags |= VCPU_WORKAROUND_2_FLAG;
-
/* Reset timer */
ret = kvm_timer_vcpu_reset(vcpu);
out:
if (!vcpu_has_sve(vcpu))
val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
val &= ~(0xfUL << ID_AA64PFR0_AMU_SHIFT);
+ if (!(val & (0xfUL << ID_AA64PFR0_CSV2_SHIFT)) &&
+ arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED)
+ val |= (1UL << ID_AA64PFR0_CSV2_SHIFT);
+ } else if (id == SYS_ID_AA64PFR1_EL1) {
+ val &= ~(0xfUL << ID_AA64PFR1_MTE_SHIFT);
} else if (id == SYS_ID_AA64ISAR1_EL1 && !vcpu_has_ptrauth(vcpu)) {
val &= ~((0xfUL << ID_AA64ISAR1_APA_SHIFT) |
(0xfUL << ID_AA64ISAR1_API_SHIFT) |
return true;
}
+static bool access_mte_regs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
+ const struct sys_reg_desc *r)
+{
+ kvm_inject_undefined(vcpu);
+ return false;
+}
+
/* sys_reg_desc initialiser for known cpufeature ID registers */
#define ID_SANITISED(name) { \
SYS_DESC(SYS_##name), \
{ SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
{ SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
{ SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
+
+ { SYS_DESC(SYS_RGSR_EL1), access_mte_regs },
+ { SYS_DESC(SYS_GCR_EL1), access_mte_regs },
+
{ SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility },
{ SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
{ SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
{ SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
+ { SYS_DESC(SYS_TFSR_EL1), access_mte_regs },
+ { SYS_DESC(SYS_TFSRE0_EL1), access_mte_regs },
+
{ SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
{ SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
raw_spin_lock_irqsave(&irq->irq_lock, flags);
/*
- * An access targetting Group0 SGIs can only generate
- * those, while an access targetting Group1 SGIs can
+ * An access targeting Group0 SGIs can only generate
+ * those, while an access targeting Group1 SGIs can
* generate interrupts of either group.
*/
if (!irq->group || allow_group1) {
obj-$(CONFIG_CRC32) += crc32.o
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
+
+obj-$(CONFIG_ARM64_MTE) += mte.o
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 ARM Ltd.
+ */
+#include <linux/linkage.h>
+
+#include <asm/alternative.h>
+#include <asm/assembler.h>
+#include <asm/mte.h>
+#include <asm/page.h>
+#include <asm/sysreg.h>
+
+ .arch armv8.5-a+memtag
+
+/*
+ * multitag_transfer_size - set \reg to the block size that is accessed by the
+ * LDGM/STGM instructions.
+ */
+ .macro multitag_transfer_size, reg, tmp
+ mrs_s \reg, SYS_GMID_EL1
+ ubfx \reg, \reg, #SYS_GMID_EL1_BS_SHIFT, #SYS_GMID_EL1_BS_SIZE
+ mov \tmp, #4
+ lsl \reg, \tmp, \reg
+ .endm
+
+/*
+ * Clear the tags in a page
+ * x0 - address of the page to be cleared
+ */
+SYM_FUNC_START(mte_clear_page_tags)
+ multitag_transfer_size x1, x2
+1: stgm xzr, [x0]
+ add x0, x0, x1
+ tst x0, #(PAGE_SIZE - 1)
+ b.ne 1b
+ ret
+SYM_FUNC_END(mte_clear_page_tags)
+
+/*
+ * Copy the tags from the source page to the destination one
+ * x0 - address of the destination page
+ * x1 - address of the source page
+ */
+SYM_FUNC_START(mte_copy_page_tags)
+ mov x2, x0
+ mov x3, x1
+ multitag_transfer_size x5, x6
+1: ldgm x4, [x3]
+ stgm x4, [x2]
+ add x2, x2, x5
+ add x3, x3, x5
+ tst x2, #(PAGE_SIZE - 1)
+ b.ne 1b
+ ret
+SYM_FUNC_END(mte_copy_page_tags)
+
+/*
+ * Read tags from a user buffer (one tag per byte) and set the corresponding
+ * tags at the given kernel address. Used by PTRACE_POKEMTETAGS.
+ * x0 - kernel address (to)
+ * x1 - user buffer (from)
+ * x2 - number of tags/bytes (n)
+ * Returns:
+ * x0 - number of tags read/set
+ */
+SYM_FUNC_START(mte_copy_tags_from_user)
+ mov x3, x1
+ cbz x2, 2f
+1:
+ uao_user_alternative 2f, ldrb, ldtrb, w4, x1, 0
+ lsl x4, x4, #MTE_TAG_SHIFT
+ stg x4, [x0], #MTE_GRANULE_SIZE
+ add x1, x1, #1
+ subs x2, x2, #1
+ b.ne 1b
+
+ // exception handling and function return
+2: sub x0, x1, x3 // update the number of tags set
+ ret
+SYM_FUNC_END(mte_copy_tags_from_user)
+
+/*
+ * Get the tags from a kernel address range and write the tag values to the
+ * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS.
+ * x0 - user buffer (to)
+ * x1 - kernel address (from)
+ * x2 - number of tags/bytes (n)
+ * Returns:
+ * x0 - number of tags read/set
+ */
+SYM_FUNC_START(mte_copy_tags_to_user)
+ mov x3, x0
+ cbz x2, 2f
+1:
+ ldg x4, [x1]
+ ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
+ uao_user_alternative 2f, strb, sttrb, w4, x0, 0
+ add x0, x0, #1
+ add x1, x1, #MTE_GRANULE_SIZE
+ subs x2, x2, #1
+ b.ne 1b
+
+ // exception handling and function return
+2: sub x0, x0, x3 // update the number of tags copied
+ ret
+SYM_FUNC_END(mte_copy_tags_to_user)
+
+/*
+ * Save the tags in a page
+ * x0 - page address
+ * x1 - tag storage
+ */
+SYM_FUNC_START(mte_save_page_tags)
+ multitag_transfer_size x7, x5
+1:
+ mov x2, #0
+2:
+ ldgm x5, [x0]
+ orr x2, x2, x5
+ add x0, x0, x7
+ tst x0, #0xFF // 16 tag values fit in a register,
+ b.ne 2b // which is 16*16=256 bytes
+
+ str x2, [x1], #8
+
+ tst x0, #(PAGE_SIZE - 1)
+ b.ne 1b
+
+ ret
+SYM_FUNC_END(mte_save_page_tags)
+
+/*
+ * Restore the tags in a page
+ * x0 - page address
+ * x1 - tag storage
+ */
+SYM_FUNC_START(mte_restore_page_tags)
+ multitag_transfer_size x7, x5
+1:
+ ldr x2, [x1], #8
+2:
+ stgm x2, [x0]
+ add x0, x0, x7
+ tst x0, #0xFF
+ b.ne 2b
+
+ tst x0, #(PAGE_SIZE - 1)
+ b.ne 1b
+
+ ret
+SYM_FUNC_END(mte_restore_page_tags)
ioremap.o mmap.o pgd.o mmu.o \
context.o proc.o pageattr.o
obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
-obj-$(CONFIG_PTDUMP_CORE) += dump.o
+obj-$(CONFIG_PTDUMP_CORE) += ptdump.o
obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o
obj-$(CONFIG_NUMA) += numa.o
obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o
+obj-$(CONFIG_ARM64_MTE) += mteswap.o
KASAN_SANITIZE_physaddr.o += n
obj-$(CONFIG_KASAN) += kasan_init.o
static DEFINE_PER_CPU(u64, reserved_asids);
static cpumask_t tlb_flush_pending;
+static unsigned long max_pinned_asids;
+static unsigned long nr_pinned_asids;
+static unsigned long *pinned_asid_map;
+
#define ASID_MASK (~GENMASK(asid_bits - 1, 0))
#define ASID_FIRST_VERSION (1UL << asid_bits)
}
}
-static void set_kpti_asid_bits(void)
+static void set_kpti_asid_bits(unsigned long *map)
{
unsigned int len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long);
/*
* is set, then the ASID will map only userspace. Thus
* mark even as reserved for kernel.
*/
- memset(asid_map, 0xaa, len);
+ memset(map, 0xaa, len);
}
static void set_reserved_asid_bits(void)
{
- if (arm64_kernel_unmapped_at_el0())
- set_kpti_asid_bits();
+ if (pinned_asid_map)
+ bitmap_copy(asid_map, pinned_asid_map, NUM_USER_ASIDS);
+ else if (arm64_kernel_unmapped_at_el0())
+ set_kpti_asid_bits(asid_map);
else
bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
}
if (check_update_reserved_asid(asid, newasid))
return newasid;
+ /*
+ * If it is pinned, we can keep using it. Note that reserved
+ * takes priority, because even if it is also pinned, we need to
+ * update the generation into the reserved_asids.
+ */
+ if (refcount_read(&mm->context.pinned))
+ return newasid;
+
/*
* We had a valid ASID in a previous life, so try to re-use
* it if possible.
cpu_switch_mm(mm->pgd, mm);
}
+unsigned long arm64_mm_context_get(struct mm_struct *mm)
+{
+ unsigned long flags;
+ u64 asid;
+
+ if (!pinned_asid_map)
+ return 0;
+
+ raw_spin_lock_irqsave(&cpu_asid_lock, flags);
+
+ asid = atomic64_read(&mm->context.id);
+
+ if (refcount_inc_not_zero(&mm->context.pinned))
+ goto out_unlock;
+
+ if (nr_pinned_asids >= max_pinned_asids) {
+ asid = 0;
+ goto out_unlock;
+ }
+
+ if (!asid_gen_match(asid)) {
+ /*
+ * We went through one or more rollover since that ASID was
+ * used. Ensure that it is still valid, or generate a new one.
+ */
+ asid = new_context(mm);
+ atomic64_set(&mm->context.id, asid);
+ }
+
+ nr_pinned_asids++;
+ __set_bit(asid2idx(asid), pinned_asid_map);
+ refcount_set(&mm->context.pinned, 1);
+
+out_unlock:
+ raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
+
+ asid &= ~ASID_MASK;
+
+ /* Set the equivalent of USER_ASID_BIT */
+ if (asid && arm64_kernel_unmapped_at_el0())
+ asid |= 1;
+
+ return asid;
+}
+EXPORT_SYMBOL_GPL(arm64_mm_context_get);
+
+void arm64_mm_context_put(struct mm_struct *mm)
+{
+ unsigned long flags;
+ u64 asid = atomic64_read(&mm->context.id);
+
+ if (!pinned_asid_map)
+ return;
+
+ raw_spin_lock_irqsave(&cpu_asid_lock, flags);
+
+ if (refcount_dec_and_test(&mm->context.pinned)) {
+ __clear_bit(asid2idx(asid), pinned_asid_map);
+ nr_pinned_asids--;
+ }
+
+ raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
+}
+EXPORT_SYMBOL_GPL(arm64_mm_context_put);
+
/* Errata workaround post TTBRx_EL1 update. */
asmlinkage void post_ttbr_update_workaround(void)
{
{
unsigned long num_available_asids = NUM_USER_ASIDS;
- if (arm64_kernel_unmapped_at_el0())
+ if (arm64_kernel_unmapped_at_el0()) {
num_available_asids /= 2;
+ if (pinned_asid_map)
+ set_kpti_asid_bits(pinned_asid_map);
+ }
/*
* Expect allocation after rollover to fail if we don't have at least
* one more ASID than CPUs. ASID #0 is reserved for init_mm.
WARN_ON(num_available_asids - 1 <= num_possible_cpus());
pr_info("ASID allocator initialised with %lu entries\n",
num_available_asids);
+
+ /*
+ * There must always be an ASID available after rollover. Ensure that,
+ * even if all CPUs have a reserved ASID and the maximum number of ASIDs
+ * are pinned, there still is at least one empty slot in the ASID map.
+ */
+ max_pinned_asids = num_available_asids - num_possible_cpus() - 2;
return 0;
}
arch_initcall(asids_update_limit);
panic("Failed to allocate bitmap for %lu ASIDs\n",
NUM_USER_ASIDS);
+ pinned_asid_map = kcalloc(BITS_TO_LONGS(NUM_USER_ASIDS),
+ sizeof(*pinned_asid_map), GFP_KERNEL);
+ nr_pinned_asids = 0;
+
/*
* We cannot call set_reserved_asid_bits() here because CPU
* caps are not finalized yet, so it is safer to assume KPTI
* and reserve kernel ASID's from beginning.
*/
if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0))
- set_kpti_asid_bits();
+ set_kpti_asid_bits(asid_map);
return 0;
}
early_initcall(asids_init);
* Copyright (C) 2012 ARM Ltd.
*/
+#include <linux/bitops.h>
#include <linux/mm.h>
#include <asm/page.h>
#include <asm/cacheflush.h>
+#include <asm/cpufeature.h>
+#include <asm/mte.h>
-void __cpu_copy_user_page(void *kto, const void *kfrom, unsigned long vaddr)
+void copy_highpage(struct page *to, struct page *from)
{
- struct page *page = virt_to_page(kto);
+ struct page *kto = page_address(to);
+ struct page *kfrom = page_address(from);
+
copy_page(kto, kfrom);
- flush_dcache_page(page);
+
+ if (system_supports_mte() && test_bit(PG_mte_tagged, &from->flags)) {
+ set_bit(PG_mte_tagged, &to->flags);
+ mte_copy_page_tags(kto, kfrom);
+ }
}
-EXPORT_SYMBOL_GPL(__cpu_copy_user_page);
+EXPORT_SYMBOL(copy_highpage);
-void __cpu_clear_user_page(void *kaddr, unsigned long vaddr)
+void copy_user_highpage(struct page *to, struct page *from,
+ unsigned long vaddr, struct vm_area_struct *vma)
{
- clear_page(kaddr);
+ copy_highpage(to, from);
+ flush_dcache_page(to);
}
-EXPORT_SYMBOL_GPL(__cpu_clear_user_page);
+EXPORT_SYMBOL_GPL(copy_user_highpage);
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2014, The Linux Foundation. All rights reserved.
- * Debug helper to dump the current kernel pagetables of the system
- * so that we can see what the various memory ranges are set to.
- *
- * Derived from x86 and arm implementation:
- * (C) Copyright 2008 Intel Corporation
- *
- * Author: Arjan van de Ven <arjan@linux.intel.com>
- */
-#include <linux/debugfs.h>
-#include <linux/errno.h>
-#include <linux/fs.h>
-#include <linux/io.h>
-#include <linux/init.h>
-#include <linux/mm.h>
-#include <linux/ptdump.h>
-#include <linux/sched.h>
-#include <linux/seq_file.h>
-
-#include <asm/fixmap.h>
-#include <asm/kasan.h>
-#include <asm/memory.h>
-#include <asm/pgtable-hwdef.h>
-#include <asm/ptdump.h>
-
-
-enum address_markers_idx {
- PAGE_OFFSET_NR = 0,
- PAGE_END_NR,
-#ifdef CONFIG_KASAN
- KASAN_START_NR,
-#endif
-};
-
-static struct addr_marker address_markers[] = {
- { PAGE_OFFSET, "Linear Mapping start" },
- { 0 /* PAGE_END */, "Linear Mapping end" },
-#ifdef CONFIG_KASAN
- { 0 /* KASAN_SHADOW_START */, "Kasan shadow start" },
- { KASAN_SHADOW_END, "Kasan shadow end" },
-#endif
- { MODULES_VADDR, "Modules start" },
- { MODULES_END, "Modules end" },
- { VMALLOC_START, "vmalloc() area" },
- { VMALLOC_END, "vmalloc() end" },
- { FIXADDR_START, "Fixmap start" },
- { FIXADDR_TOP, "Fixmap end" },
- { PCI_IO_START, "PCI I/O start" },
- { PCI_IO_END, "PCI I/O end" },
-#ifdef CONFIG_SPARSEMEM_VMEMMAP
- { VMEMMAP_START, "vmemmap start" },
- { VMEMMAP_START + VMEMMAP_SIZE, "vmemmap end" },
-#endif
- { -1, NULL },
-};
-
-#define pt_dump_seq_printf(m, fmt, args...) \
-({ \
- if (m) \
- seq_printf(m, fmt, ##args); \
-})
-
-#define pt_dump_seq_puts(m, fmt) \
-({ \
- if (m) \
- seq_printf(m, fmt); \
-})
-
-/*
- * The page dumper groups page table entries of the same type into a single
- * description. It uses pg_state to track the range information while
- * iterating over the pte entries. When the continuity is broken it then
- * dumps out a description of the range.
- */
-struct pg_state {
- struct ptdump_state ptdump;
- struct seq_file *seq;
- const struct addr_marker *marker;
- unsigned long start_address;
- int level;
- u64 current_prot;
- bool check_wx;
- unsigned long wx_pages;
- unsigned long uxn_pages;
-};
-
-struct prot_bits {
- u64 mask;
- u64 val;
- const char *set;
- const char *clear;
-};
-
-static const struct prot_bits pte_bits[] = {
- {
- .mask = PTE_VALID,
- .val = PTE_VALID,
- .set = " ",
- .clear = "F",
- }, {
- .mask = PTE_USER,
- .val = PTE_USER,
- .set = "USR",
- .clear = " ",
- }, {
- .mask = PTE_RDONLY,
- .val = PTE_RDONLY,
- .set = "ro",
- .clear = "RW",
- }, {
- .mask = PTE_PXN,
- .val = PTE_PXN,
- .set = "NX",
- .clear = "x ",
- }, {
- .mask = PTE_SHARED,
- .val = PTE_SHARED,
- .set = "SHD",
- .clear = " ",
- }, {
- .mask = PTE_AF,
- .val = PTE_AF,
- .set = "AF",
- .clear = " ",
- }, {
- .mask = PTE_NG,
- .val = PTE_NG,
- .set = "NG",
- .clear = " ",
- }, {
- .mask = PTE_CONT,
- .val = PTE_CONT,
- .set = "CON",
- .clear = " ",
- }, {
- .mask = PTE_TABLE_BIT,
- .val = PTE_TABLE_BIT,
- .set = " ",
- .clear = "BLK",
- }, {
- .mask = PTE_UXN,
- .val = PTE_UXN,
- .set = "UXN",
- .clear = " ",
- }, {
- .mask = PTE_GP,
- .val = PTE_GP,
- .set = "GP",
- .clear = " ",
- }, {
- .mask = PTE_ATTRINDX_MASK,
- .val = PTE_ATTRINDX(MT_DEVICE_nGnRnE),
- .set = "DEVICE/nGnRnE",
- }, {
- .mask = PTE_ATTRINDX_MASK,
- .val = PTE_ATTRINDX(MT_DEVICE_nGnRE),
- .set = "DEVICE/nGnRE",
- }, {
- .mask = PTE_ATTRINDX_MASK,
- .val = PTE_ATTRINDX(MT_DEVICE_GRE),
- .set = "DEVICE/GRE",
- }, {
- .mask = PTE_ATTRINDX_MASK,
- .val = PTE_ATTRINDX(MT_NORMAL_NC),
- .set = "MEM/NORMAL-NC",
- }, {
- .mask = PTE_ATTRINDX_MASK,
- .val = PTE_ATTRINDX(MT_NORMAL),
- .set = "MEM/NORMAL",
- }
-};
-
-struct pg_level {
- const struct prot_bits *bits;
- const char *name;
- size_t num;
- u64 mask;
-};
-
-static struct pg_level pg_level[] = {
- { /* pgd */
- .name = "PGD",
- .bits = pte_bits,
- .num = ARRAY_SIZE(pte_bits),
- }, { /* p4d */
- .name = "P4D",
- .bits = pte_bits,
- .num = ARRAY_SIZE(pte_bits),
- }, { /* pud */
- .name = (CONFIG_PGTABLE_LEVELS > 3) ? "PUD" : "PGD",
- .bits = pte_bits,
- .num = ARRAY_SIZE(pte_bits),
- }, { /* pmd */
- .name = (CONFIG_PGTABLE_LEVELS > 2) ? "PMD" : "PGD",
- .bits = pte_bits,
- .num = ARRAY_SIZE(pte_bits),
- }, { /* pte */
- .name = "PTE",
- .bits = pte_bits,
- .num = ARRAY_SIZE(pte_bits),
- },
-};
-
-static void dump_prot(struct pg_state *st, const struct prot_bits *bits,
- size_t num)
-{
- unsigned i;
-
- for (i = 0; i < num; i++, bits++) {
- const char *s;
-
- if ((st->current_prot & bits->mask) == bits->val)
- s = bits->set;
- else
- s = bits->clear;
-
- if (s)
- pt_dump_seq_printf(st->seq, " %s", s);
- }
-}
-
-static void note_prot_uxn(struct pg_state *st, unsigned long addr)
-{
- if (!st->check_wx)
- return;
-
- if ((st->current_prot & PTE_UXN) == PTE_UXN)
- return;
-
- WARN_ONCE(1, "arm64/mm: Found non-UXN mapping at address %p/%pS\n",
- (void *)st->start_address, (void *)st->start_address);
-
- st->uxn_pages += (addr - st->start_address) / PAGE_SIZE;
-}
-
-static void note_prot_wx(struct pg_state *st, unsigned long addr)
-{
- if (!st->check_wx)
- return;
- if ((st->current_prot & PTE_RDONLY) == PTE_RDONLY)
- return;
- if ((st->current_prot & PTE_PXN) == PTE_PXN)
- return;
-
- WARN_ONCE(1, "arm64/mm: Found insecure W+X mapping at address %p/%pS\n",
- (void *)st->start_address, (void *)st->start_address);
-
- st->wx_pages += (addr - st->start_address) / PAGE_SIZE;
-}
-
-static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level,
- u64 val)
-{
- struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
- static const char units[] = "KMGTPE";
- u64 prot = 0;
-
- if (level >= 0)
- prot = val & pg_level[level].mask;
-
- if (st->level == -1) {
- st->level = level;
- st->current_prot = prot;
- st->start_address = addr;
- pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
- } else if (prot != st->current_prot || level != st->level ||
- addr >= st->marker[1].start_address) {
- const char *unit = units;
- unsigned long delta;
-
- if (st->current_prot) {
- note_prot_uxn(st, addr);
- note_prot_wx(st, addr);
- }
-
- pt_dump_seq_printf(st->seq, "0x%016lx-0x%016lx ",
- st->start_address, addr);
-
- delta = (addr - st->start_address) >> 10;
- while (!(delta & 1023) && unit[1]) {
- delta >>= 10;
- unit++;
- }
- pt_dump_seq_printf(st->seq, "%9lu%c %s", delta, *unit,
- pg_level[st->level].name);
- if (st->current_prot && pg_level[st->level].bits)
- dump_prot(st, pg_level[st->level].bits,
- pg_level[st->level].num);
- pt_dump_seq_puts(st->seq, "\n");
-
- if (addr >= st->marker[1].start_address) {
- st->marker++;
- pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
- }
-
- st->start_address = addr;
- st->current_prot = prot;
- st->level = level;
- }
-
- if (addr >= st->marker[1].start_address) {
- st->marker++;
- pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
- }
-
-}
-
-void ptdump_walk(struct seq_file *s, struct ptdump_info *info)
-{
- unsigned long end = ~0UL;
- struct pg_state st;
-
- if (info->base_addr < TASK_SIZE_64)
- end = TASK_SIZE_64;
-
- st = (struct pg_state){
- .seq = s,
- .marker = info->markers,
- .ptdump = {
- .note_page = note_page,
- .range = (struct ptdump_range[]){
- {info->base_addr, end},
- {0, 0}
- }
- }
- };
-
- ptdump_walk_pgd(&st.ptdump, info->mm, NULL);
-}
-
-static void ptdump_initialize(void)
-{
- unsigned i, j;
-
- for (i = 0; i < ARRAY_SIZE(pg_level); i++)
- if (pg_level[i].bits)
- for (j = 0; j < pg_level[i].num; j++)
- pg_level[i].mask |= pg_level[i].bits[j].mask;
-}
-
-static struct ptdump_info kernel_ptdump_info = {
- .mm = &init_mm,
- .markers = address_markers,
- .base_addr = PAGE_OFFSET,
-};
-
-void ptdump_check_wx(void)
-{
- struct pg_state st = {
- .seq = NULL,
- .marker = (struct addr_marker[]) {
- { 0, NULL},
- { -1, NULL},
- },
- .level = -1,
- .check_wx = true,
- .ptdump = {
- .note_page = note_page,
- .range = (struct ptdump_range[]) {
- {PAGE_OFFSET, ~0UL},
- {0, 0}
- }
- }
- };
-
- ptdump_walk_pgd(&st.ptdump, &init_mm, NULL);
-
- if (st.wx_pages || st.uxn_pages)
- pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found, %lu non-UXN pages found\n",
- st.wx_pages, st.uxn_pages);
- else
- pr_info("Checked W+X mappings: passed, no W+X pages found\n");
-}
-
-static int ptdump_init(void)
-{
- address_markers[PAGE_END_NR].start_address = PAGE_END;
-#ifdef CONFIG_KASAN
- address_markers[KASAN_START_NR].start_address = KASAN_SHADOW_START;
-#endif
- ptdump_initialize();
- ptdump_debugfs_register(&kernel_ptdump_info, "kernel_page_tables");
- return 0;
-}
-device_initcall(ptdump_init);
if (!fixup)
return 0;
- if (IS_ENABLED(CONFIG_BPF_JIT) &&
- regs->pc >= BPF_JIT_REGION_START &&
- regs->pc < BPF_JIT_REGION_END)
+ if (in_bpf_jit(regs))
return arm64_bpf_fixup_exception(fixup, regs);
regs->pc = (unsigned long)&fixup->fixup + fixup->fixup;
pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
} while (pteval != old_pteval);
- flush_tlb_fix_spurious_fault(vma, address);
+ /* Invalidate a stale read-only entry */
+ if (dirty)
+ flush_tlb_page(vma, address);
return 1;
}
return 0;
}
+static int do_tag_check_fault(unsigned long addr, unsigned int esr,
+ struct pt_regs *regs)
+{
+ do_bad_area(addr, esr, regs);
+ return 0;
+}
+
static const struct fault_info fault_info[] = {
{ do_bad, SIGKILL, SI_KERNEL, "ttbr address size fault" },
{ do_bad, SIGKILL, SI_KERNEL, "level 1 address size fault" },
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
{ do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" },
{ do_sea, SIGBUS, BUS_OBJERR, "synchronous external abort" },
- { do_bad, SIGKILL, SI_KERNEL, "unknown 17" },
+ { do_tag_check_fault, SIGSEGV, SEGV_MTESERR, "synchronous tag check fault" },
{ do_bad, SIGKILL, SI_KERNEL, "unknown 18" },
{ do_bad, SIGKILL, SI_KERNEL, "unknown 19" },
{ do_sea, SIGKILL, SI_KERNEL, "level 0 (translation table walk)" },
u64 idmap_t0sz = TCR_T0SZ(VA_BITS);
u64 idmap_ptrs_per_pgd = PTRS_PER_PGD;
-u64 __section(".mmuoff.data.write") vabits_actual;
+u64 __section(.mmuoff.data.write) vabits_actual;
EXPORT_SYMBOL(vabits_actual);
u64 kimage_voffset __ro_after_init;
* The following mapping attributes may be updated in live
* kernel mappings without the need for break-before-make.
*/
- static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE | PTE_NG;
+ pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE | PTE_NG;
/* creating or taking down mappings is always safe */
if (old == 0 || new == 0)
if (old & ~new & PTE_NG)
return false;
+ /*
+ * Changing the memory type between Normal and Normal-Tagged is safe
+ * since Tagged is considered a permission attribute from the
+ * mismatched attribute aliases perspective.
+ */
+ if (((old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) ||
+ (old & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)) &&
+ ((new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL) ||
+ (new & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)))
+ mask |= PTE_ATTRINDX_MASK;
+
return ((old ^ new) & ~mask) == 0;
}
if (memblock_is_nomap(reg))
continue;
- __map_memblock(pgdp, start, end, PAGE_KERNEL, flags);
+ /*
+ * The linear map must allow allocation tags reading/writing
+ * if MTE is present. Otherwise, it has the same attributes as
+ * PAGE_KERNEL.
+ */
+ __map_memblock(pgdp, start, end, PAGE_KERNEL_TAGGED, flags);
}
/*
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/pagemap.h>
+#include <linux/xarray.h>
+#include <linux/slab.h>
+#include <linux/swap.h>
+#include <linux/swapops.h>
+#include <asm/mte.h>
+
+static DEFINE_XARRAY(mte_pages);
+
+void *mte_allocate_tag_storage(void)
+{
+ /* tags granule is 16 bytes, 2 tags stored per byte */
+ return kmalloc(PAGE_SIZE / 16 / 2, GFP_KERNEL);
+}
+
+void mte_free_tag_storage(char *storage)
+{
+ kfree(storage);
+}
+
+int mte_save_tags(struct page *page)
+{
+ void *tag_storage, *ret;
+
+ if (!test_bit(PG_mte_tagged, &page->flags))
+ return 0;
+
+ tag_storage = mte_allocate_tag_storage();
+ if (!tag_storage)
+ return -ENOMEM;
+
+ mte_save_page_tags(page_address(page), tag_storage);
+
+ /* page_private contains the swap entry.val set in do_swap_page */
+ ret = xa_store(&mte_pages, page_private(page), tag_storage, GFP_KERNEL);
+ if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
+ mte_free_tag_storage(tag_storage);
+ return xa_err(ret);
+ } else if (ret) {
+ /* Entry is being replaced, free the old entry */
+ mte_free_tag_storage(ret);
+ }
+
+ return 0;
+}
+
+bool mte_restore_tags(swp_entry_t entry, struct page *page)
+{
+ void *tags = xa_load(&mte_pages, entry.val);
+
+ if (!tags)
+ return false;
+
+ mte_restore_page_tags(page_address(page), tags);
+
+ return true;
+}
+
+void mte_invalidate_tags(int type, pgoff_t offset)
+{
+ swp_entry_t entry = swp_entry(type, offset);
+ void *tags = xa_erase(&mte_pages, entry.val);
+
+ mte_free_tag_storage(tags);
+}
+
+void mte_invalidate_tags_area(int type)
+{
+ swp_entry_t entry = swp_entry(type, 0);
+ swp_entry_t last_entry = swp_entry(type + 1, 0);
+ void *tags;
+
+ XA_STATE(xa_state, &mte_pages, entry.val);
+
+ xa_lock(&mte_pages);
+ xas_for_each(&xa_state, tags, last_entry.val - 1) {
+ __xa_erase(&mte_pages, xa_state.xa_index);
+ mte_free_tag_storage(tags);
+ }
+ xa_unlock(&mte_pages);
+}
*/
const struct cpumask *cpumask_of_node(int node)
{
- if (WARN_ON(node >= nr_node_ids))
+
+ if (node == NUMA_NO_NODE)
+ return cpu_all_mask;
+
+ if (WARN_ON(node < 0 || node >= nr_node_ids))
return cpu_none_mask;
if (WARN_ON(node_to_cpumask_map[node] == NULL))
* arm64_numa_init() - Initialize NUMA
*
* Try each configured NUMA initialization method until one succeeds. The
- * last fallback is dummy single node config encomapssing whole memory.
+ * last fallback is dummy single node config encompassing whole memory.
*/
void __init arm64_numa_init(void)
{
#include <linux/sched.h>
#include <linux/vmalloc.h>
+#include <asm/cacheflush.h>
#include <asm/set_memory.h>
#include <asm/tlbflush.h>
#include <asm/cpufeature.h>
#include <asm/alternative.h>
#include <asm/smp.h>
+#include <asm/sysreg.h>
#ifdef CONFIG_ARM64_64K_PAGES
#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
#define TCR_KASAN_FLAGS 0
#endif
-/* Default MAIR_EL1 */
+/*
+ * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and
+ * changed during __cpu_setup to Normal Tagged if the system supports MTE.
+ */
#define MAIR_EL1_SET \
(MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \
MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \
MAIR_ATTRIDX(MAIR_ATTR_DEVICE_GRE, MT_DEVICE_GRE) | \
MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \
MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \
- MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT))
+ MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT) | \
+ MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED))
#ifdef CONFIG_CPU_PM
/**
* Memory region attributes
*/
mov_q x5, MAIR_EL1_SET
+#ifdef CONFIG_ARM64_MTE
+ /*
+ * Update MAIR_EL1, GCR_EL1 and TFSR*_EL1 if MTE is supported
+ * (ID_AA64PFR1_EL1[11:8] > 1).
+ */
+ mrs x10, ID_AA64PFR1_EL1
+ ubfx x10, x10, #ID_AA64PFR1_MTE_SHIFT, #4
+ cmp x10, #ID_AA64PFR1_MTE
+ b.lt 1f
+
+ /* Normal Tagged memory type at the corresponding MAIR index */
+ mov x10, #MAIR_ATTR_NORMAL_TAGGED
+ bfi x5, x10, #(8 * MT_NORMAL_TAGGED), #8
+
+ /* initialize GCR_EL1: all non-zero tags excluded by default */
+ mov x10, #(SYS_GCR_EL1_RRND | SYS_GCR_EL1_EXCL_MASK)
+ msr_s SYS_GCR_EL1, x10
+
+ /* clear any pending tag check faults in TFSR*_EL1 */
+ msr_s SYS_TFSR_EL1, xzr
+ msr_s SYS_TFSRE0_EL1, xzr
+1:
+#endif
msr mair_el1, x5
/*
* Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2014, The Linux Foundation. All rights reserved.
+ * Debug helper to dump the current kernel pagetables of the system
+ * so that we can see what the various memory ranges are set to.
+ *
+ * Derived from x86 and arm implementation:
+ * (C) Copyright 2008 Intel Corporation
+ *
+ * Author: Arjan van de Ven <arjan@linux.intel.com>
+ */
+#include <linux/debugfs.h>
+#include <linux/errno.h>
+#include <linux/fs.h>
+#include <linux/io.h>
+#include <linux/init.h>
+#include <linux/mm.h>
+#include <linux/ptdump.h>
+#include <linux/sched.h>
+#include <linux/seq_file.h>
+
+#include <asm/fixmap.h>
+#include <asm/kasan.h>
+#include <asm/memory.h>
+#include <asm/pgtable-hwdef.h>
+#include <asm/ptdump.h>
+
+
+enum address_markers_idx {
+ PAGE_OFFSET_NR = 0,
+ PAGE_END_NR,
+#ifdef CONFIG_KASAN
+ KASAN_START_NR,
+#endif
+};
+
+static struct addr_marker address_markers[] = {
+ { PAGE_OFFSET, "Linear Mapping start" },
+ { 0 /* PAGE_END */, "Linear Mapping end" },
+#ifdef CONFIG_KASAN
+ { 0 /* KASAN_SHADOW_START */, "Kasan shadow start" },
+ { KASAN_SHADOW_END, "Kasan shadow end" },
+#endif
+ { BPF_JIT_REGION_START, "BPF start" },
+ { BPF_JIT_REGION_END, "BPF end" },
+ { MODULES_VADDR, "Modules start" },
+ { MODULES_END, "Modules end" },
+ { VMALLOC_START, "vmalloc() area" },
+ { VMALLOC_END, "vmalloc() end" },
+ { FIXADDR_START, "Fixmap start" },
+ { FIXADDR_TOP, "Fixmap end" },
+ { PCI_IO_START, "PCI I/O start" },
+ { PCI_IO_END, "PCI I/O end" },
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
+ { VMEMMAP_START, "vmemmap start" },
+ { VMEMMAP_START + VMEMMAP_SIZE, "vmemmap end" },
+#endif
+ { -1, NULL },
+};
+
+#define pt_dump_seq_printf(m, fmt, args...) \
+({ \
+ if (m) \
+ seq_printf(m, fmt, ##args); \
+})
+
+#define pt_dump_seq_puts(m, fmt) \
+({ \
+ if (m) \
+ seq_printf(m, fmt); \
+})
+
+/*
+ * The page dumper groups page table entries of the same type into a single
+ * description. It uses pg_state to track the range information while
+ * iterating over the pte entries. When the continuity is broken it then
+ * dumps out a description of the range.
+ */
+struct pg_state {
+ struct ptdump_state ptdump;
+ struct seq_file *seq;
+ const struct addr_marker *marker;
+ unsigned long start_address;
+ int level;
+ u64 current_prot;
+ bool check_wx;
+ unsigned long wx_pages;
+ unsigned long uxn_pages;
+};
+
+struct prot_bits {
+ u64 mask;
+ u64 val;
+ const char *set;
+ const char *clear;
+};
+
+static const struct prot_bits pte_bits[] = {
+ {
+ .mask = PTE_VALID,
+ .val = PTE_VALID,
+ .set = " ",
+ .clear = "F",
+ }, {
+ .mask = PTE_USER,
+ .val = PTE_USER,
+ .set = "USR",
+ .clear = " ",
+ }, {
+ .mask = PTE_RDONLY,
+ .val = PTE_RDONLY,
+ .set = "ro",
+ .clear = "RW",
+ }, {
+ .mask = PTE_PXN,
+ .val = PTE_PXN,
+ .set = "NX",
+ .clear = "x ",
+ }, {
+ .mask = PTE_SHARED,
+ .val = PTE_SHARED,
+ .set = "SHD",
+ .clear = " ",
+ }, {
+ .mask = PTE_AF,
+ .val = PTE_AF,
+ .set = "AF",
+ .clear = " ",
+ }, {
+ .mask = PTE_NG,
+ .val = PTE_NG,
+ .set = "NG",
+ .clear = " ",
+ }, {
+ .mask = PTE_CONT,
+ .val = PTE_CONT,
+ .set = "CON",
+ .clear = " ",
+ }, {
+ .mask = PTE_TABLE_BIT,
+ .val = PTE_TABLE_BIT,
+ .set = " ",
+ .clear = "BLK",
+ }, {
+ .mask = PTE_UXN,
+ .val = PTE_UXN,
+ .set = "UXN",
+ .clear = " ",
+ }, {
+ .mask = PTE_GP,
+ .val = PTE_GP,
+ .set = "GP",
+ .clear = " ",
+ }, {
+ .mask = PTE_ATTRINDX_MASK,
+ .val = PTE_ATTRINDX(MT_DEVICE_nGnRnE),
+ .set = "DEVICE/nGnRnE",
+ }, {
+ .mask = PTE_ATTRINDX_MASK,
+ .val = PTE_ATTRINDX(MT_DEVICE_nGnRE),
+ .set = "DEVICE/nGnRE",
+ }, {
+ .mask = PTE_ATTRINDX_MASK,
+ .val = PTE_ATTRINDX(MT_DEVICE_GRE),
+ .set = "DEVICE/GRE",
+ }, {
+ .mask = PTE_ATTRINDX_MASK,
+ .val = PTE_ATTRINDX(MT_NORMAL_NC),
+ .set = "MEM/NORMAL-NC",
+ }, {
+ .mask = PTE_ATTRINDX_MASK,
+ .val = PTE_ATTRINDX(MT_NORMAL),
+ .set = "MEM/NORMAL",
+ }, {
+ .mask = PTE_ATTRINDX_MASK,
+ .val = PTE_ATTRINDX(MT_NORMAL_TAGGED),
+ .set = "MEM/NORMAL-TAGGED",
+ }
+};
+
+struct pg_level {
+ const struct prot_bits *bits;
+ const char *name;
+ size_t num;
+ u64 mask;
+};
+
+static struct pg_level pg_level[] = {
+ { /* pgd */
+ .name = "PGD",
+ .bits = pte_bits,
+ .num = ARRAY_SIZE(pte_bits),
+ }, { /* p4d */
+ .name = "P4D",
+ .bits = pte_bits,
+ .num = ARRAY_SIZE(pte_bits),
+ }, { /* pud */
+ .name = (CONFIG_PGTABLE_LEVELS > 3) ? "PUD" : "PGD",
+ .bits = pte_bits,
+ .num = ARRAY_SIZE(pte_bits),
+ }, { /* pmd */
+ .name = (CONFIG_PGTABLE_LEVELS > 2) ? "PMD" : "PGD",
+ .bits = pte_bits,
+ .num = ARRAY_SIZE(pte_bits),
+ }, { /* pte */
+ .name = "PTE",
+ .bits = pte_bits,
+ .num = ARRAY_SIZE(pte_bits),
+ },
+};
+
+static void dump_prot(struct pg_state *st, const struct prot_bits *bits,
+ size_t num)
+{
+ unsigned i;
+
+ for (i = 0; i < num; i++, bits++) {
+ const char *s;
+
+ if ((st->current_prot & bits->mask) == bits->val)
+ s = bits->set;
+ else
+ s = bits->clear;
+
+ if (s)
+ pt_dump_seq_printf(st->seq, " %s", s);
+ }
+}
+
+static void note_prot_uxn(struct pg_state *st, unsigned long addr)
+{
+ if (!st->check_wx)
+ return;
+
+ if ((st->current_prot & PTE_UXN) == PTE_UXN)
+ return;
+
+ WARN_ONCE(1, "arm64/mm: Found non-UXN mapping at address %p/%pS\n",
+ (void *)st->start_address, (void *)st->start_address);
+
+ st->uxn_pages += (addr - st->start_address) / PAGE_SIZE;
+}
+
+static void note_prot_wx(struct pg_state *st, unsigned long addr)
+{
+ if (!st->check_wx)
+ return;
+ if ((st->current_prot & PTE_RDONLY) == PTE_RDONLY)
+ return;
+ if ((st->current_prot & PTE_PXN) == PTE_PXN)
+ return;
+
+ WARN_ONCE(1, "arm64/mm: Found insecure W+X mapping at address %p/%pS\n",
+ (void *)st->start_address, (void *)st->start_address);
+
+ st->wx_pages += (addr - st->start_address) / PAGE_SIZE;
+}
+
+static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level,
+ u64 val)
+{
+ struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
+ static const char units[] = "KMGTPE";
+ u64 prot = 0;
+
+ if (level >= 0)
+ prot = val & pg_level[level].mask;
+
+ if (st->level == -1) {
+ st->level = level;
+ st->current_prot = prot;
+ st->start_address = addr;
+ pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+ } else if (prot != st->current_prot || level != st->level ||
+ addr >= st->marker[1].start_address) {
+ const char *unit = units;
+ unsigned long delta;
+
+ if (st->current_prot) {
+ note_prot_uxn(st, addr);
+ note_prot_wx(st, addr);
+ }
+
+ pt_dump_seq_printf(st->seq, "0x%016lx-0x%016lx ",
+ st->start_address, addr);
+
+ delta = (addr - st->start_address) >> 10;
+ while (!(delta & 1023) && unit[1]) {
+ delta >>= 10;
+ unit++;
+ }
+ pt_dump_seq_printf(st->seq, "%9lu%c %s", delta, *unit,
+ pg_level[st->level].name);
+ if (st->current_prot && pg_level[st->level].bits)
+ dump_prot(st, pg_level[st->level].bits,
+ pg_level[st->level].num);
+ pt_dump_seq_puts(st->seq, "\n");
+
+ if (addr >= st->marker[1].start_address) {
+ st->marker++;
+ pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+ }
+
+ st->start_address = addr;
+ st->current_prot = prot;
+ st->level = level;
+ }
+
+ if (addr >= st->marker[1].start_address) {
+ st->marker++;
+ pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+ }
+
+}
+
+void ptdump_walk(struct seq_file *s, struct ptdump_info *info)
+{
+ unsigned long end = ~0UL;
+ struct pg_state st;
+
+ if (info->base_addr < TASK_SIZE_64)
+ end = TASK_SIZE_64;
+
+ st = (struct pg_state){
+ .seq = s,
+ .marker = info->markers,
+ .ptdump = {
+ .note_page = note_page,
+ .range = (struct ptdump_range[]){
+ {info->base_addr, end},
+ {0, 0}
+ }
+ }
+ };
+
+ ptdump_walk_pgd(&st.ptdump, info->mm, NULL);
+}
+
+static void ptdump_initialize(void)
+{
+ unsigned i, j;
+
+ for (i = 0; i < ARRAY_SIZE(pg_level); i++)
+ if (pg_level[i].bits)
+ for (j = 0; j < pg_level[i].num; j++)
+ pg_level[i].mask |= pg_level[i].bits[j].mask;
+}
+
+static struct ptdump_info kernel_ptdump_info = {
+ .mm = &init_mm,
+ .markers = address_markers,
+ .base_addr = PAGE_OFFSET,
+};
+
+void ptdump_check_wx(void)
+{
+ struct pg_state st = {
+ .seq = NULL,
+ .marker = (struct addr_marker[]) {
+ { 0, NULL},
+ { -1, NULL},
+ },
+ .level = -1,
+ .check_wx = true,
+ .ptdump = {
+ .note_page = note_page,
+ .range = (struct ptdump_range[]) {
+ {PAGE_OFFSET, ~0UL},
+ {0, 0}
+ }
+ }
+ };
+
+ ptdump_walk_pgd(&st.ptdump, &init_mm, NULL);
+
+ if (st.wx_pages || st.uxn_pages)
+ pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found, %lu non-UXN pages found\n",
+ st.wx_pages, st.uxn_pages);
+ else
+ pr_info("Checked W+X mappings: passed, no W+X pages found\n");
+}
+
+static int ptdump_init(void)
+{
+ address_markers[PAGE_END_NR].start_address = PAGE_END;
+#ifdef CONFIG_KASAN
+ address_markers[KASAN_START_NR].start_address = KASAN_SHADOW_START;
+#endif
+ ptdump_initialize();
+ ptdump_debugfs_register(&kernel_ptdump_info, "kernel_page_tables");
+ return 0;
+}
+device_initcall(ptdump_init);
}
#define csum_tcpudp_nofold csum_tcpudp_nofold
+#define _HAVE_ARCH_CSUM_AND_COPY
+extern __wsum csum_partial_copy_nocheck(const void *src, void *dst, int len);
+
#include <asm-generic/checksum.h>
#endif /* _ASM_C6X_CHECKSUM_H */
ENTRY(csum_partial_copy_nocheck)
MVC .S2 ILC,B30
- MV .D1X B6,A31 ; given csum
ZERO .D1 A9 ; csum (a side)
|| ZERO .D2 B9 ; csum (b side)
|| SHRU .S2X A6,2,B5 ; len / 4
SHRU .S1 A9,16,A0
[A0] BNOP .S1 L91,5
-L10: ADD .D1 A31,A9,A9
- MV .D1 A9,A4
+L10: MV .D1 A9,A4
BNOP .S2 B3,4
MVC .S2 B30,ILC
void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)
{
- struct kretprobe_instance *ri = NULL;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address = 0;
- unsigned long trampoline_address =
- (unsigned long)&kretprobe_trampoline;
- kprobe_opcode_t *correct_ret_addr = NULL;
-
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
-
- /*
- * It is possible to have multiple instances associated with a given
- * task either because multiple functions in the call path have
- * return probes installed on them, and/or more than one
- * return probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always pushed into the head of the list
- * - when multiple return probes are registered for the same
- * function, the (chronologically) first instance's ret_addr
- * will be the real return address, and all the rest will
- * point to kretprobe_trampoline.
- */
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- orig_ret_address = (unsigned long)ri->ret_addr;
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
-
- correct_ret_addr = ri->ret_addr;
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- if (ri->rp && ri->rp->handler) {
- __this_cpu_write(current_kprobe, &ri->rp->kp);
- get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
- ri->ret_addr = correct_ret_addr;
- ri->rp->handler(ri, regs);
- __this_cpu_write(current_kprobe, NULL);
- }
-
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_hash_unlock(current, &flags);
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
- return (void *)orig_ret_address;
+ return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL);
}
void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *)regs->lr;
+ ri->fp = NULL;
regs->lr = (unsigned long) &kretprobe_trampoline;
}
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
DISCARDS
}
#define do_csum do_csum
unsigned int do_csum(const void *voidptr, int len);
-/*
- * the same as csum_partial, but copies from src while it
- * checksums
- *
- * here even more important to align src and dst on a 32-bit (or even
- * better 64-bit) boundary
- */
-#define csum_partial_copy_nocheck csum_partial_copy_nocheck
-__wsum csum_partial_copy_nocheck(const void *src, void *dst,
- int len, __wsum sum);
-
/*
* computes the checksum of the TCP/UDP pseudo-header
* returns a 16-bit checksum, already complemented
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
}
return 0xFFFF & sum0;
}
-
-/*
- * copy from ds while checksumming, otherwise like csum_partial
- */
-__wsum
-csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
-{
- memcpy(dst, src, len);
- return csum_partial(dst, len, sum);
-}
-EXPORT_SYMBOL(csum_partial_copy_nocheck);
select NEED_DMA_MAP_STATE
select NEED_SG_DMA_LENGTH
select NUMA if !FLATMEM
+ select PCI_MSI_ARCH_FALLBACKS if PCI_MSI
default y
help
The Itanium Processor Family is Intel's 64-bit successor to
config IA64_MCA_RECOVERY
tristate "MCA recovery from errors other than TLB."
-config PERFMON
- bool "Performance monitor support"
- depends on BROKEN
- help
- Selects whether support for the IA-64 performance monitor hardware
- is included in the kernel. This makes some kernel data-structures a
- little bigger and slows down execution a bit, but it is generally
- a good idea to turn this on. If you're unsure, say Y.
-
config IA64_PALINFO
tristate "/proc/pal support"
help
CONFIG_NR_CPUS=2
CONFIG_PREEMPT=y
# CONFIG_VIRTUAL_MEM_MAP is not set
-CONFIG_PERFMON=y
CONFIG_IA64_PALINFO=y
CONFIG_EFI_VARS=y
CONFIG_BINFMT_MISC=m
CONFIG_BLK_DEV_CRYPTOLOOP=m
CONFIG_BLK_DEV_NBD=m
CONFIG_BLK_DEV_RAM=m
-CONFIG_IDE=m
-CONFIG_BLK_DEV_IDECD=m
-CONFIG_BLK_DEV_GENERIC=m
-CONFIG_BLK_DEV_PIIX=m
+CONFIG_ATA=m
+CONFIG_ATA_GENERIC=m
+CONFIG_ATA_PIIX=m
CONFIG_SCSI=y
CONFIG_BLK_DEV_SD=y
CONFIG_SCSI_CONSTANTS=y
CONFIG_SMP=y
CONFIG_HOTPLUG_CPU=y
CONFIG_IA64_MCA_RECOVERY=y
-CONFIG_PERFMON=y
CONFIG_IA64_PALINFO=y
CONFIG_KEXEC=y
CONFIG_CRASH_DUMP=y
CONFIG_BLK_DEV_NBD=m
CONFIG_BLK_DEV_RAM=y
CONFIG_SGI_XP=m
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_CMD64X=y
-CONFIG_BLK_DEV_PIIX=y
+CONFIG_ATA=y
+CONFIG_ATA_GENERIC=y
+CONFIG_PATA_CMD64X=y
+CONFIG_ATA_PIIX=y
CONFIG_BLK_DEV_SD=y
CONFIG_CHR_DEV_ST=m
CONFIG_BLK_DEV_SR=m
CONFIG_HOTPLUG_CPU=y
CONFIG_SPARSEMEM_MANUAL=y
CONFIG_IA64_MCA_RECOVERY=y
-CONFIG_PERFMON=y
CONFIG_IA64_PALINFO=y
CONFIG_EFI_VARS=y
CONFIG_BINFMT_MISC=m
CONFIG_BLK_DEV_CRYPTOLOOP=m
CONFIG_BLK_DEV_NBD=m
CONFIG_BLK_DEV_RAM=y
-CONFIG_IDE=y
+CONFIG_ATA=y
CONFIG_BLK_DEV_IDECD=y
-CONFIG_IDE_GENERIC=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_CMD64X=y
-CONFIG_BLK_DEV_PIIX=y
+CONFIG_ATA_GENERIC=y
+CONFIG_PATA_CMD64X=y
+CONFIG_ATA_PIIX=y
CONFIG_SCSI=y
CONFIG_BLK_DEV_SD=y
CONFIG_CHR_DEV_ST=m
CONFIG_PERMIT_BSP_REMOVE=y
CONFIG_FORCE_CPEI_RETARGET=y
CONFIG_IA64_MCA_RECOVERY=y
-CONFIG_PERFMON=y
CONFIG_IA64_PALINFO=y
CONFIG_KEXEC=y
CONFIG_EFI_VARS=y
CONFIG_BLK_DEV_CRYPTOLOOP=m
CONFIG_BLK_DEV_NBD=m
CONFIG_BLK_DEV_RAM=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_CMD64X=y
-CONFIG_BLK_DEV_PIIX=y
+CONFIG_ATA=y
+CONFIG_ATA_GENERIC=y
+CONFIG_PATA_CMD64X=y
+CONFIG_ATA_PIIX=y
CONFIG_SCSI=y
CONFIG_BLK_DEV_SD=y
CONFIG_CHR_DEV_ST=m
CONFIG_HOTPLUG_CPU=y
CONFIG_FLATMEM_MANUAL=y
CONFIG_IA64_MCA_RECOVERY=y
-CONFIG_PERFMON=y
CONFIG_IA64_PALINFO=y
CONFIG_CRASH_DUMP=y
CONFIG_EFI_VARS=y
CONFIG_NETFILTER=y
CONFIG_BLK_DEV_LOOP=y
CONFIG_BLK_DEV_RAM=y
-CONFIG_IDE=y
-CONFIG_BLK_DEV_IDECD=y
-CONFIG_BLK_DEV_GENERIC=y
-CONFIG_BLK_DEV_CMD64X=y
+CONFIG_ATA=y
+CONFIG_ATA_GENERIC=y
+CONFIG_PATA_CMD64X=y
CONFIG_SCSI=y
CONFIG_BLK_DEV_SD=y
CONFIG_CHR_DEV_ST=y
*/
extern __wsum csum_partial(const void *buff, int len, __wsum sum);
-extern __wsum csum_partial_copy_nocheck(const void *src, void *dst,
- int len, __wsum sum);
-
/*
* This routine is used for miscellaneous IP-like checksums, mainly in
* icmp.c
__u64 map_base; /* base address for get_unmapped_area() */
__u64 rbs_bot; /* the base address for the RBS */
int last_fph_cpu; /* CPU that may hold the contents of f32-f127 */
-
-#ifdef CONFIG_PERFMON
- void *pfm_context; /* pointer to detailed PMU context */
- unsigned long pfm_needs_checking; /* when >0, pending perfmon work on kernel exit */
-# define INIT_THREAD_PM .pfm_context = NULL, \
- .pfm_needs_checking = 0UL,
-#else
-# define INIT_THREAD_PM
-#endif
unsigned long dbr[IA64_NUM_DBG_REGS];
unsigned long ibr[IA64_NUM_DBG_REGS];
struct ia64_fpreg fph[96]; /* saved/loaded on demand */
.map_base = DEFAULT_MAP_BASE, \
.rbs_bot = STACK_TOP - DEFAULT_USER_STACK_SIZE, \
.last_fph_cpu = -1, \
- INIT_THREAD_PM \
.dbr = {0, }, \
.ibr = {0, }, \
.fph = {{{{0}}}, } \
extern void ia64_save_extra (struct task_struct *task);
extern void ia64_load_extra (struct task_struct *task);
-#ifdef CONFIG_PERFMON
- DECLARE_PER_CPU(unsigned long, pfm_syst_info);
-# define PERFMON_IS_SYSWIDE() (__this_cpu_read(pfm_syst_info) & 0x1)
-#else
-# define PERFMON_IS_SYSWIDE() (0)
-#endif
-
#define IA64_HAS_EXTRA_STATE(t) \
- ((t)->thread.flags & (IA64_THREAD_DBG_VALID|IA64_THREAD_PM_VALID) \
- || PERFMON_IS_SYSWIDE())
+ ((t)->thread.flags & (IA64_THREAD_DBG_VALID|IA64_THREAD_PM_VALID))
#define __switch_to(prev,next,last) do { \
if (IA64_HAS_EXTRA_STATE(prev)) \
extra-y := head.o vmlinux.lds
obj-y := entry.o efi.o efi_stub.o gate-data.o fsys.o ia64_ksyms.o irq.o irq_ia64.o \
- irq_lsapic.o ivt.o pal.o patch.o process.o perfmon.o ptrace.o sal.o \
+ irq_lsapic.o ivt.o pal.o patch.o process.o ptrace.o sal.o \
salinfo.o setup.o signal.o sys_ia64.o time.o traps.o unaligned.o \
unwind.o mca.o mca_asm.o topology.o dma-mapping.o iosapic.o acpi.o \
acpi-ext.o
obj-$(CONFIG_MODULES) += module.o
obj-$(CONFIG_SMP) += smp.o smpboot.o
obj-$(CONFIG_NUMA) += numa.o
-obj-$(CONFIG_PERFMON) += perfmon_default_smpl.o
obj-$(CONFIG_IA64_CYCLONE) += cyclone.o
obj-$(CONFIG_IA64_MCA_RECOVERY) += mca_recovery.o
obj-$(CONFIG_KPROBES) += kprobes.o
#include <asm/hw_irq.h>
#include <asm/tlbflush.h>
-#ifdef CONFIG_PERFMON
-# include <asm/perfmon.h>
-#endif
-
#define IRQ_DEBUG 0
#define IRQ_VECTOR_UNASSIGNED (0)
"irq_move");
}
#endif
-#ifdef CONFIG_PERFMON
- pfm_init_percpu();
-#endif
}
void
{
}
-/*
- * At this point the target function has been tricked into
- * returning into our trampoline. Lookup the associated instance
- * and then:
- * - call the handler function
- * - cleanup by marking the instance as unused
- * - long jump back to the original return address
- */
int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
{
- struct kretprobe_instance *ri = NULL;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address = 0;
- unsigned long trampoline_address =
- ((struct fnptr *)kretprobe_trampoline)->ip;
-
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
-
- /*
- * It is possible to have multiple instances associated with a given
- * task either because an multiple functions in the call path
- * have a return probe installed on them, and/or more than one return
- * return probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always inserted at the head of the list
- * - when multiple return probes are registered for the same
- * function, the first instance's ret_addr will point to the
- * real return address, and all the rest will point to
- * kretprobe_trampoline
- */
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- regs->cr_iip = orig_ret_address;
-
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- if (ri->rp && ri->rp->handler)
- ri->rp->handler(ri, regs);
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
-
- kretprobe_hash_unlock(current, &flags);
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
+ regs->cr_iip = __kretprobe_trampoline_handler(regs, kretprobe_trampoline, NULL);
/*
* By returning a non-zero value, we are telling
* kprobe_handler() that we don't want the post_handler
struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *)regs->b0;
+ ri->fp = NULL;
/* Replace the return addr with trampoline addr */
regs->b0 = ((struct fnptr *)kretprobe_trampoline)->ip;
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * This file implements the perfmon-2 subsystem which is used
- * to program the IA-64 Performance Monitoring Unit (PMU).
- *
- * The initial version of perfmon.c was written by
- * Ganesh Venkitachalam, IBM Corp.
- *
- * Then it was modified for perfmon-1.x by Stephane Eranian and
- * David Mosberger, Hewlett Packard Co.
- *
- * Version Perfmon-2.x is a rewrite of perfmon-1.x
- * by Stephane Eranian, Hewlett Packard Co.
- *
- * Copyright (C) 1999-2005 Hewlett Packard Co
- * Stephane Eranian <eranian@hpl.hp.com>
- * David Mosberger-Tang <davidm@hpl.hp.com>
- *
- * More information about perfmon available at:
- * http://www.hpl.hp.com/research/linux/perfmon
- */
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/sched.h>
-#include <linux/sched/task.h>
-#include <linux/sched/task_stack.h>
-#include <linux/interrupt.h>
-#include <linux/proc_fs.h>
-#include <linux/seq_file.h>
-#include <linux/init.h>
-#include <linux/vmalloc.h>
-#include <linux/mm.h>
-#include <linux/sysctl.h>
-#include <linux/list.h>
-#include <linux/file.h>
-#include <linux/poll.h>
-#include <linux/vfs.h>
-#include <linux/smp.h>
-#include <linux/pagemap.h>
-#include <linux/mount.h>
-#include <linux/pseudo_fs.h>
-#include <linux/bitops.h>
-#include <linux/capability.h>
-#include <linux/rcupdate.h>
-#include <linux/completion.h>
-#include <linux/tracehook.h>
-#include <linux/slab.h>
-#include <linux/cpu.h>
-
-#include <asm/errno.h>
-#include <asm/intrinsics.h>
-#include <asm/page.h>
-#include <asm/perfmon.h>
-#include <asm/processor.h>
-#include <asm/signal.h>
-#include <linux/uaccess.h>
-#include <asm/delay.h>
-
-#include "irq.h"
-
-#ifdef CONFIG_PERFMON
-/*
- * perfmon context state
- */
-#define PFM_CTX_UNLOADED 1 /* context is not loaded onto any task */
-#define PFM_CTX_LOADED 2 /* context is loaded onto a task */
-#define PFM_CTX_MASKED 3 /* context is loaded but monitoring is masked due to overflow */
-#define PFM_CTX_ZOMBIE 4 /* owner of the context is closing it */
-
-#define PFM_INVALID_ACTIVATION (~0UL)
-
-#define PFM_NUM_PMC_REGS 64 /* PMC save area for ctxsw */
-#define PFM_NUM_PMD_REGS 64 /* PMD save area for ctxsw */
-
-/*
- * depth of message queue
- */
-#define PFM_MAX_MSGS 32
-#define PFM_CTXQ_EMPTY(g) ((g)->ctx_msgq_head == (g)->ctx_msgq_tail)
-
-/*
- * type of a PMU register (bitmask).
- * bitmask structure:
- * bit0 : register implemented
- * bit1 : end marker
- * bit2-3 : reserved
- * bit4 : pmc has pmc.pm
- * bit5 : pmc controls a counter (has pmc.oi), pmd is used as counter
- * bit6-7 : register type
- * bit8-31: reserved
- */
-#define PFM_REG_NOTIMPL 0x0 /* not implemented at all */
-#define PFM_REG_IMPL 0x1 /* register implemented */
-#define PFM_REG_END 0x2 /* end marker */
-#define PFM_REG_MONITOR (0x1<<4|PFM_REG_IMPL) /* a PMC with a pmc.pm field only */
-#define PFM_REG_COUNTING (0x2<<4|PFM_REG_MONITOR) /* a monitor + pmc.oi+ PMD used as a counter */
-#define PFM_REG_CONTROL (0x4<<4|PFM_REG_IMPL) /* PMU control register */
-#define PFM_REG_CONFIG (0x8<<4|PFM_REG_IMPL) /* configuration register */
-#define PFM_REG_BUFFER (0xc<<4|PFM_REG_IMPL) /* PMD used as buffer */
-
-#define PMC_IS_LAST(i) (pmu_conf->pmc_desc[i].type & PFM_REG_END)
-#define PMD_IS_LAST(i) (pmu_conf->pmd_desc[i].type & PFM_REG_END)
-
-#define PMC_OVFL_NOTIFY(ctx, i) ((ctx)->ctx_pmds[i].flags & PFM_REGFL_OVFL_NOTIFY)
-
-/* i assumed unsigned */
-#define PMC_IS_IMPL(i) (i< PMU_MAX_PMCS && (pmu_conf->pmc_desc[i].type & PFM_REG_IMPL))
-#define PMD_IS_IMPL(i) (i< PMU_MAX_PMDS && (pmu_conf->pmd_desc[i].type & PFM_REG_IMPL))
-
-/* XXX: these assume that register i is implemented */
-#define PMD_IS_COUNTING(i) ((pmu_conf->pmd_desc[i].type & PFM_REG_COUNTING) == PFM_REG_COUNTING)
-#define PMC_IS_COUNTING(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_COUNTING) == PFM_REG_COUNTING)
-#define PMC_IS_MONITOR(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_MONITOR) == PFM_REG_MONITOR)
-#define PMC_IS_CONTROL(i) ((pmu_conf->pmc_desc[i].type & PFM_REG_CONTROL) == PFM_REG_CONTROL)
-
-#define PMC_DFL_VAL(i) pmu_conf->pmc_desc[i].default_value
-#define PMC_RSVD_MASK(i) pmu_conf->pmc_desc[i].reserved_mask
-#define PMD_PMD_DEP(i) pmu_conf->pmd_desc[i].dep_pmd[0]
-#define PMC_PMD_DEP(i) pmu_conf->pmc_desc[i].dep_pmd[0]
-
-#define PFM_NUM_IBRS IA64_NUM_DBG_REGS
-#define PFM_NUM_DBRS IA64_NUM_DBG_REGS
-
-#define CTX_OVFL_NOBLOCK(c) ((c)->ctx_fl_block == 0)
-#define CTX_HAS_SMPL(c) ((c)->ctx_fl_is_sampling)
-#define PFM_CTX_TASK(h) (h)->ctx_task
-
-#define PMU_PMC_OI 5 /* position of pmc.oi bit */
-
-/* XXX: does not support more than 64 PMDs */
-#define CTX_USED_PMD(ctx, mask) (ctx)->ctx_used_pmds[0] |= (mask)
-#define CTX_IS_USED_PMD(ctx, c) (((ctx)->ctx_used_pmds[0] & (1UL << (c))) != 0UL)
-
-#define CTX_USED_MONITOR(ctx, mask) (ctx)->ctx_used_monitors[0] |= (mask)
-
-#define CTX_USED_IBR(ctx,n) (ctx)->ctx_used_ibrs[(n)>>6] |= 1UL<< ((n) % 64)
-#define CTX_USED_DBR(ctx,n) (ctx)->ctx_used_dbrs[(n)>>6] |= 1UL<< ((n) % 64)
-#define CTX_USES_DBREGS(ctx) (((pfm_context_t *)(ctx))->ctx_fl_using_dbreg==1)
-#define PFM_CODE_RR 0 /* requesting code range restriction */
-#define PFM_DATA_RR 1 /* requestion data range restriction */
-
-#define PFM_CPUINFO_CLEAR(v) pfm_get_cpu_var(pfm_syst_info) &= ~(v)
-#define PFM_CPUINFO_SET(v) pfm_get_cpu_var(pfm_syst_info) |= (v)
-#define PFM_CPUINFO_GET() pfm_get_cpu_var(pfm_syst_info)
-
-#define RDEP(x) (1UL<<(x))
-
-/*
- * context protection macros
- * in SMP:
- * - we need to protect against CPU concurrency (spin_lock)
- * - we need to protect against PMU overflow interrupts (local_irq_disable)
- * in UP:
- * - we need to protect against PMU overflow interrupts (local_irq_disable)
- *
- * spin_lock_irqsave()/spin_unlock_irqrestore():
- * in SMP: local_irq_disable + spin_lock
- * in UP : local_irq_disable
- *
- * spin_lock()/spin_lock():
- * in UP : removed automatically
- * in SMP: protect against context accesses from other CPU. interrupts
- * are not masked. This is useful for the PMU interrupt handler
- * because we know we will not get PMU concurrency in that code.
- */
-#define PROTECT_CTX(c, f) \
- do { \
- DPRINT(("spinlock_irq_save ctx %p by [%d]\n", c, task_pid_nr(current))); \
- spin_lock_irqsave(&(c)->ctx_lock, f); \
- DPRINT(("spinlocked ctx %p by [%d]\n", c, task_pid_nr(current))); \
- } while(0)
-
-#define UNPROTECT_CTX(c, f) \
- do { \
- DPRINT(("spinlock_irq_restore ctx %p by [%d]\n", c, task_pid_nr(current))); \
- spin_unlock_irqrestore(&(c)->ctx_lock, f); \
- } while(0)
-
-#define PROTECT_CTX_NOPRINT(c, f) \
- do { \
- spin_lock_irqsave(&(c)->ctx_lock, f); \
- } while(0)
-
-
-#define UNPROTECT_CTX_NOPRINT(c, f) \
- do { \
- spin_unlock_irqrestore(&(c)->ctx_lock, f); \
- } while(0)
-
-
-#define PROTECT_CTX_NOIRQ(c) \
- do { \
- spin_lock(&(c)->ctx_lock); \
- } while(0)
-
-#define UNPROTECT_CTX_NOIRQ(c) \
- do { \
- spin_unlock(&(c)->ctx_lock); \
- } while(0)
-
-
-#ifdef CONFIG_SMP
-
-#define GET_ACTIVATION() pfm_get_cpu_var(pmu_activation_number)
-#define INC_ACTIVATION() pfm_get_cpu_var(pmu_activation_number)++
-#define SET_ACTIVATION(c) (c)->ctx_last_activation = GET_ACTIVATION()
-
-#else /* !CONFIG_SMP */
-#define SET_ACTIVATION(t) do {} while(0)
-#define GET_ACTIVATION(t) do {} while(0)
-#define INC_ACTIVATION(t) do {} while(0)
-#endif /* CONFIG_SMP */
-
-#define SET_PMU_OWNER(t, c) do { pfm_get_cpu_var(pmu_owner) = (t); pfm_get_cpu_var(pmu_ctx) = (c); } while(0)
-#define GET_PMU_OWNER() pfm_get_cpu_var(pmu_owner)
-#define GET_PMU_CTX() pfm_get_cpu_var(pmu_ctx)
-
-#define LOCK_PFS(g) spin_lock_irqsave(&pfm_sessions.pfs_lock, g)
-#define UNLOCK_PFS(g) spin_unlock_irqrestore(&pfm_sessions.pfs_lock, g)
-
-#define PFM_REG_RETFLAG_SET(flags, val) do { flags &= ~PFM_REG_RETFL_MASK; flags |= (val); } while(0)
-
-/*
- * cmp0 must be the value of pmc0
- */
-#define PMC0_HAS_OVFL(cmp0) (cmp0 & ~0x1UL)
-
-#define PFMFS_MAGIC 0xa0b4d889
-
-/*
- * debugging
- */
-#define PFM_DEBUGGING 1
-#ifdef PFM_DEBUGGING
-#define DPRINT(a) \
- do { \
- if (unlikely(pfm_sysctl.debug >0)) { printk("%s.%d: CPU%d [%d] ", __func__, __LINE__, smp_processor_id(), task_pid_nr(current)); printk a; } \
- } while (0)
-
-#define DPRINT_ovfl(a) \
- do { \
- if (unlikely(pfm_sysctl.debug > 0 && pfm_sysctl.debug_ovfl >0)) { printk("%s.%d: CPU%d [%d] ", __func__, __LINE__, smp_processor_id(), task_pid_nr(current)); printk a; } \
- } while (0)
-#endif
-
-/*
- * 64-bit software counter structure
- *
- * the next_reset_type is applied to the next call to pfm_reset_regs()
- */
-typedef struct {
- unsigned long val; /* virtual 64bit counter value */
- unsigned long lval; /* last reset value */
- unsigned long long_reset; /* reset value on sampling overflow */
- unsigned long short_reset; /* reset value on overflow */
- unsigned long reset_pmds[4]; /* which other pmds to reset when this counter overflows */
- unsigned long smpl_pmds[4]; /* which pmds are accessed when counter overflow */
- unsigned long seed; /* seed for random-number generator */
- unsigned long mask; /* mask for random-number generator */
- unsigned int flags; /* notify/do not notify */
- unsigned long eventid; /* overflow event identifier */
-} pfm_counter_t;
-
-/*
- * context flags
- */
-typedef struct {
- unsigned int block:1; /* when 1, task will blocked on user notifications */
- unsigned int system:1; /* do system wide monitoring */
- unsigned int using_dbreg:1; /* using range restrictions (debug registers) */
- unsigned int is_sampling:1; /* true if using a custom format */
- unsigned int excl_idle:1; /* exclude idle task in system wide session */
- unsigned int going_zombie:1; /* context is zombie (MASKED+blocking) */
- unsigned int trap_reason:2; /* reason for going into pfm_handle_work() */
- unsigned int no_msg:1; /* no message sent on overflow */
- unsigned int can_restart:1; /* allowed to issue a PFM_RESTART */
- unsigned int reserved:22;
-} pfm_context_flags_t;
-
-#define PFM_TRAP_REASON_NONE 0x0 /* default value */
-#define PFM_TRAP_REASON_BLOCK 0x1 /* we need to block on overflow */
-#define PFM_TRAP_REASON_RESET 0x2 /* we need to reset PMDs */
-
-
-/*
- * perfmon context: encapsulates all the state of a monitoring session
- */
-
-typedef struct pfm_context {
- spinlock_t ctx_lock; /* context protection */
-
- pfm_context_flags_t ctx_flags; /* bitmask of flags (block reason incl.) */
- unsigned int ctx_state; /* state: active/inactive (no bitfield) */
-
- struct task_struct *ctx_task; /* task to which context is attached */
-
- unsigned long ctx_ovfl_regs[4]; /* which registers overflowed (notification) */
-
- struct completion ctx_restart_done; /* use for blocking notification mode */
-
- unsigned long ctx_used_pmds[4]; /* bitmask of PMD used */
- unsigned long ctx_all_pmds[4]; /* bitmask of all accessible PMDs */
- unsigned long ctx_reload_pmds[4]; /* bitmask of force reload PMD on ctxsw in */
-
- unsigned long ctx_all_pmcs[4]; /* bitmask of all accessible PMCs */
- unsigned long ctx_reload_pmcs[4]; /* bitmask of force reload PMC on ctxsw in */
- unsigned long ctx_used_monitors[4]; /* bitmask of monitor PMC being used */
-
- unsigned long ctx_pmcs[PFM_NUM_PMC_REGS]; /* saved copies of PMC values */
-
- unsigned int ctx_used_ibrs[1]; /* bitmask of used IBR (speedup ctxsw in) */
- unsigned int ctx_used_dbrs[1]; /* bitmask of used DBR (speedup ctxsw in) */
- unsigned long ctx_dbrs[IA64_NUM_DBG_REGS]; /* DBR values (cache) when not loaded */
- unsigned long ctx_ibrs[IA64_NUM_DBG_REGS]; /* IBR values (cache) when not loaded */
-
- pfm_counter_t ctx_pmds[PFM_NUM_PMD_REGS]; /* software state for PMDS */
-
- unsigned long th_pmcs[PFM_NUM_PMC_REGS]; /* PMC thread save state */
- unsigned long th_pmds[PFM_NUM_PMD_REGS]; /* PMD thread save state */
-
- unsigned long ctx_saved_psr_up; /* only contains psr.up value */
-
- unsigned long ctx_last_activation; /* context last activation number for last_cpu */
- unsigned int ctx_last_cpu; /* CPU id of current or last CPU used (SMP only) */
- unsigned int ctx_cpu; /* cpu to which perfmon is applied (system wide) */
-
- int ctx_fd; /* file descriptor used my this context */
- pfm_ovfl_arg_t ctx_ovfl_arg; /* argument to custom buffer format handler */
-
- pfm_buffer_fmt_t *ctx_buf_fmt; /* buffer format callbacks */
- void *ctx_smpl_hdr; /* points to sampling buffer header kernel vaddr */
- unsigned long ctx_smpl_size; /* size of sampling buffer */
- void *ctx_smpl_vaddr; /* user level virtual address of smpl buffer */
-
- wait_queue_head_t ctx_msgq_wait;
- pfm_msg_t ctx_msgq[PFM_MAX_MSGS];
- int ctx_msgq_head;
- int ctx_msgq_tail;
- struct fasync_struct *ctx_async_queue;
-
- wait_queue_head_t ctx_zombieq; /* termination cleanup wait queue */
-} pfm_context_t;
-
-/*
- * magic number used to verify that structure is really
- * a perfmon context
- */
-#define PFM_IS_FILE(f) ((f)->f_op == &pfm_file_ops)
-
-#define PFM_GET_CTX(t) ((pfm_context_t *)(t)->thread.pfm_context)
-
-#ifdef CONFIG_SMP
-#define SET_LAST_CPU(ctx, v) (ctx)->ctx_last_cpu = (v)
-#define GET_LAST_CPU(ctx) (ctx)->ctx_last_cpu
-#else
-#define SET_LAST_CPU(ctx, v) do {} while(0)
-#define GET_LAST_CPU(ctx) do {} while(0)
-#endif
-
-
-#define ctx_fl_block ctx_flags.block
-#define ctx_fl_system ctx_flags.system
-#define ctx_fl_using_dbreg ctx_flags.using_dbreg
-#define ctx_fl_is_sampling ctx_flags.is_sampling
-#define ctx_fl_excl_idle ctx_flags.excl_idle
-#define ctx_fl_going_zombie ctx_flags.going_zombie
-#define ctx_fl_trap_reason ctx_flags.trap_reason
-#define ctx_fl_no_msg ctx_flags.no_msg
-#define ctx_fl_can_restart ctx_flags.can_restart
-
-#define PFM_SET_WORK_PENDING(t, v) do { (t)->thread.pfm_needs_checking = v; } while(0);
-#define PFM_GET_WORK_PENDING(t) (t)->thread.pfm_needs_checking
-
-/*
- * global information about all sessions
- * mostly used to synchronize between system wide and per-process
- */
-typedef struct {
- spinlock_t pfs_lock; /* lock the structure */
-
- unsigned int pfs_task_sessions; /* number of per task sessions */
- unsigned int pfs_sys_sessions; /* number of per system wide sessions */
- unsigned int pfs_sys_use_dbregs; /* incremented when a system wide session uses debug regs */
- unsigned int pfs_ptrace_use_dbregs; /* incremented when a process uses debug regs */
- struct task_struct *pfs_sys_session[NR_CPUS]; /* point to task owning a system-wide session */
-} pfm_session_t;
-
-/*
- * information about a PMC or PMD.
- * dep_pmd[]: a bitmask of dependent PMD registers
- * dep_pmc[]: a bitmask of dependent PMC registers
- */
-typedef int (*pfm_reg_check_t)(struct task_struct *task, pfm_context_t *ctx, unsigned int cnum, unsigned long *val, struct pt_regs *regs);
-typedef struct {
- unsigned int type;
- int pm_pos;
- unsigned long default_value; /* power-on default value */
- unsigned long reserved_mask; /* bitmask of reserved bits */
- pfm_reg_check_t read_check;
- pfm_reg_check_t write_check;
- unsigned long dep_pmd[4];
- unsigned long dep_pmc[4];
-} pfm_reg_desc_t;
-
-/* assume cnum is a valid monitor */
-#define PMC_PM(cnum, val) (((val) >> (pmu_conf->pmc_desc[cnum].pm_pos)) & 0x1)
-
-/*
- * This structure is initialized at boot time and contains
- * a description of the PMU main characteristics.
- *
- * If the probe function is defined, detection is based
- * on its return value:
- * - 0 means recognized PMU
- * - anything else means not supported
- * When the probe function is not defined, then the pmu_family field
- * is used and it must match the host CPU family such that:
- * - cpu->family & config->pmu_family != 0
- */
-typedef struct {
- unsigned long ovfl_val; /* overflow value for counters */
-
- pfm_reg_desc_t *pmc_desc; /* detailed PMC register dependencies descriptions */
- pfm_reg_desc_t *pmd_desc; /* detailed PMD register dependencies descriptions */
-
- unsigned int num_pmcs; /* number of PMCS: computed at init time */
- unsigned int num_pmds; /* number of PMDS: computed at init time */
- unsigned long impl_pmcs[4]; /* bitmask of implemented PMCS */
- unsigned long impl_pmds[4]; /* bitmask of implemented PMDS */
-
- char *pmu_name; /* PMU family name */
- unsigned int pmu_family; /* cpuid family pattern used to identify pmu */
- unsigned int flags; /* pmu specific flags */
- unsigned int num_ibrs; /* number of IBRS: computed at init time */
- unsigned int num_dbrs; /* number of DBRS: computed at init time */
- unsigned int num_counters; /* PMC/PMD counting pairs : computed at init time */
- int (*probe)(void); /* customized probe routine */
- unsigned int use_rr_dbregs:1; /* set if debug registers used for range restriction */
-} pmu_config_t;
-/*
- * PMU specific flags
- */
-#define PFM_PMU_IRQ_RESEND 1 /* PMU needs explicit IRQ resend */
-
-/*
- * debug register related type definitions
- */
-typedef struct {
- unsigned long ibr_mask:56;
- unsigned long ibr_plm:4;
- unsigned long ibr_ig:3;
- unsigned long ibr_x:1;
-} ibr_mask_reg_t;
-
-typedef struct {
- unsigned long dbr_mask:56;
- unsigned long dbr_plm:4;
- unsigned long dbr_ig:2;
- unsigned long dbr_w:1;
- unsigned long dbr_r:1;
-} dbr_mask_reg_t;
-
-typedef union {
- unsigned long val;
- ibr_mask_reg_t ibr;
- dbr_mask_reg_t dbr;
-} dbreg_t;
-
-
-/*
- * perfmon command descriptions
- */
-typedef struct {
- int (*cmd_func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
- char *cmd_name;
- int cmd_flags;
- unsigned int cmd_narg;
- size_t cmd_argsize;
- int (*cmd_getsize)(void *arg, size_t *sz);
-} pfm_cmd_desc_t;
-
-#define PFM_CMD_FD 0x01 /* command requires a file descriptor */
-#define PFM_CMD_ARG_READ 0x02 /* command must read argument(s) */
-#define PFM_CMD_ARG_RW 0x04 /* command must read/write argument(s) */
-#define PFM_CMD_STOP 0x08 /* command does not work on zombie context */
-
-
-#define PFM_CMD_NAME(cmd) pfm_cmd_tab[(cmd)].cmd_name
-#define PFM_CMD_READ_ARG(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_ARG_READ)
-#define PFM_CMD_RW_ARG(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_ARG_RW)
-#define PFM_CMD_USE_FD(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_FD)
-#define PFM_CMD_STOPPED(cmd) (pfm_cmd_tab[(cmd)].cmd_flags & PFM_CMD_STOP)
-
-#define PFM_CMD_ARG_MANY -1 /* cannot be zero */
-
-typedef struct {
- unsigned long pfm_spurious_ovfl_intr_count; /* keep track of spurious ovfl interrupts */
- unsigned long pfm_replay_ovfl_intr_count; /* keep track of replayed ovfl interrupts */
- unsigned long pfm_ovfl_intr_count; /* keep track of ovfl interrupts */
- unsigned long pfm_ovfl_intr_cycles; /* cycles spent processing ovfl interrupts */
- unsigned long pfm_ovfl_intr_cycles_min; /* min cycles spent processing ovfl interrupts */
- unsigned long pfm_ovfl_intr_cycles_max; /* max cycles spent processing ovfl interrupts */
- unsigned long pfm_smpl_handler_calls;
- unsigned long pfm_smpl_handler_cycles;
- char pad[SMP_CACHE_BYTES] ____cacheline_aligned;
-} pfm_stats_t;
-
-/*
- * perfmon internal variables
- */
-static pfm_stats_t pfm_stats[NR_CPUS];
-static pfm_session_t pfm_sessions; /* global sessions information */
-
-static DEFINE_SPINLOCK(pfm_alt_install_check);
-static pfm_intr_handler_desc_t *pfm_alt_intr_handler;
-
-static struct proc_dir_entry *perfmon_dir;
-static pfm_uuid_t pfm_null_uuid = {0,};
-
-static spinlock_t pfm_buffer_fmt_lock;
-static LIST_HEAD(pfm_buffer_fmt_list);
-
-static pmu_config_t *pmu_conf;
-
-/* sysctl() controls */
-pfm_sysctl_t pfm_sysctl;
-EXPORT_SYMBOL(pfm_sysctl);
-
-static struct ctl_table pfm_ctl_table[] = {
- {
- .procname = "debug",
- .data = &pfm_sysctl.debug,
- .maxlen = sizeof(int),
- .mode = 0666,
- .proc_handler = proc_dointvec,
- },
- {
- .procname = "debug_ovfl",
- .data = &pfm_sysctl.debug_ovfl,
- .maxlen = sizeof(int),
- .mode = 0666,
- .proc_handler = proc_dointvec,
- },
- {
- .procname = "fastctxsw",
- .data = &pfm_sysctl.fastctxsw,
- .maxlen = sizeof(int),
- .mode = 0600,
- .proc_handler = proc_dointvec,
- },
- {
- .procname = "expert_mode",
- .data = &pfm_sysctl.expert_mode,
- .maxlen = sizeof(int),
- .mode = 0600,
- .proc_handler = proc_dointvec,
- },
- {}
-};
-static struct ctl_table pfm_sysctl_dir[] = {
- {
- .procname = "perfmon",
- .mode = 0555,
- .child = pfm_ctl_table,
- },
- {}
-};
-static struct ctl_table pfm_sysctl_root[] = {
- {
- .procname = "kernel",
- .mode = 0555,
- .child = pfm_sysctl_dir,
- },
- {}
-};
-static struct ctl_table_header *pfm_sysctl_header;
-
-static int pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
-
-#define pfm_get_cpu_var(v) __ia64_per_cpu_var(v)
-#define pfm_get_cpu_data(a,b) per_cpu(a, b)
-
-static inline void
-pfm_put_task(struct task_struct *task)
-{
- if (task != current) put_task_struct(task);
-}
-
-static inline unsigned long
-pfm_protect_ctx_ctxsw(pfm_context_t *x)
-{
- spin_lock(&(x)->ctx_lock);
- return 0UL;
-}
-
-static inline void
-pfm_unprotect_ctx_ctxsw(pfm_context_t *x, unsigned long f)
-{
- spin_unlock(&(x)->ctx_lock);
-}
-
-/* forward declaration */
-static const struct dentry_operations pfmfs_dentry_operations;
-
-static int pfmfs_init_fs_context(struct fs_context *fc)
-{
- struct pseudo_fs_context *ctx = init_pseudo(fc, PFMFS_MAGIC);
- if (!ctx)
- return -ENOMEM;
- ctx->dops = &pfmfs_dentry_operations;
- return 0;
-}
-
-static struct file_system_type pfm_fs_type = {
- .name = "pfmfs",
- .init_fs_context = pfmfs_init_fs_context,
- .kill_sb = kill_anon_super,
-};
-MODULE_ALIAS_FS("pfmfs");
-
-DEFINE_PER_CPU(unsigned long, pfm_syst_info);
-DEFINE_PER_CPU(struct task_struct *, pmu_owner);
-DEFINE_PER_CPU(pfm_context_t *, pmu_ctx);
-DEFINE_PER_CPU(unsigned long, pmu_activation_number);
-EXPORT_PER_CPU_SYMBOL_GPL(pfm_syst_info);
-
-
-/* forward declaration */
-static const struct file_operations pfm_file_ops;
-
-/*
- * forward declarations
- */
-#ifndef CONFIG_SMP
-static void pfm_lazy_save_regs (struct task_struct *ta);
-#endif
-
-void dump_pmu_state(const char *);
-static int pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
-
-#include "perfmon_itanium.h"
-#include "perfmon_mckinley.h"
-#include "perfmon_montecito.h"
-#include "perfmon_generic.h"
-
-static pmu_config_t *pmu_confs[]={
- &pmu_conf_mont,
- &pmu_conf_mck,
- &pmu_conf_ita,
- &pmu_conf_gen, /* must be last */
- NULL
-};
-
-
-static int pfm_end_notify_user(pfm_context_t *ctx);
-
-static inline void
-pfm_clear_psr_pp(void)
-{
- ia64_rsm(IA64_PSR_PP);
- ia64_srlz_i();
-}
-
-static inline void
-pfm_set_psr_pp(void)
-{
- ia64_ssm(IA64_PSR_PP);
- ia64_srlz_i();
-}
-
-static inline void
-pfm_clear_psr_up(void)
-{
- ia64_rsm(IA64_PSR_UP);
- ia64_srlz_i();
-}
-
-static inline void
-pfm_set_psr_up(void)
-{
- ia64_ssm(IA64_PSR_UP);
- ia64_srlz_i();
-}
-
-static inline unsigned long
-pfm_get_psr(void)
-{
- unsigned long tmp;
- tmp = ia64_getreg(_IA64_REG_PSR);
- ia64_srlz_i();
- return tmp;
-}
-
-static inline void
-pfm_set_psr_l(unsigned long val)
-{
- ia64_setreg(_IA64_REG_PSR_L, val);
- ia64_srlz_i();
-}
-
-static inline void
-pfm_freeze_pmu(void)
-{
- ia64_set_pmc(0,1UL);
- ia64_srlz_d();
-}
-
-static inline void
-pfm_unfreeze_pmu(void)
-{
- ia64_set_pmc(0,0UL);
- ia64_srlz_d();
-}
-
-static inline void
-pfm_restore_ibrs(unsigned long *ibrs, unsigned int nibrs)
-{
- int i;
-
- for (i=0; i < nibrs; i++) {
- ia64_set_ibr(i, ibrs[i]);
- ia64_dv_serialize_instruction();
- }
- ia64_srlz_i();
-}
-
-static inline void
-pfm_restore_dbrs(unsigned long *dbrs, unsigned int ndbrs)
-{
- int i;
-
- for (i=0; i < ndbrs; i++) {
- ia64_set_dbr(i, dbrs[i]);
- ia64_dv_serialize_data();
- }
- ia64_srlz_d();
-}
-
-/*
- * PMD[i] must be a counter. no check is made
- */
-static inline unsigned long
-pfm_read_soft_counter(pfm_context_t *ctx, int i)
-{
- return ctx->ctx_pmds[i].val + (ia64_get_pmd(i) & pmu_conf->ovfl_val);
-}
-
-/*
- * PMD[i] must be a counter. no check is made
- */
-static inline void
-pfm_write_soft_counter(pfm_context_t *ctx, int i, unsigned long val)
-{
- unsigned long ovfl_val = pmu_conf->ovfl_val;
-
- ctx->ctx_pmds[i].val = val & ~ovfl_val;
- /*
- * writing to unimplemented part is ignore, so we do not need to
- * mask off top part
- */
- ia64_set_pmd(i, val & ovfl_val);
-}
-
-static pfm_msg_t *
-pfm_get_new_msg(pfm_context_t *ctx)
-{
- int idx, next;
-
- next = (ctx->ctx_msgq_tail+1) % PFM_MAX_MSGS;
-
- DPRINT(("ctx_fd=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
- if (next == ctx->ctx_msgq_head) return NULL;
-
- idx = ctx->ctx_msgq_tail;
- ctx->ctx_msgq_tail = next;
-
- DPRINT(("ctx=%p head=%d tail=%d msg=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, idx));
-
- return ctx->ctx_msgq+idx;
-}
-
-static pfm_msg_t *
-pfm_get_next_msg(pfm_context_t *ctx)
-{
- pfm_msg_t *msg;
-
- DPRINT(("ctx=%p head=%d tail=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
-
- if (PFM_CTXQ_EMPTY(ctx)) return NULL;
-
- /*
- * get oldest message
- */
- msg = ctx->ctx_msgq+ctx->ctx_msgq_head;
-
- /*
- * and move forward
- */
- ctx->ctx_msgq_head = (ctx->ctx_msgq_head+1) % PFM_MAX_MSGS;
-
- DPRINT(("ctx=%p head=%d tail=%d type=%d\n", ctx, ctx->ctx_msgq_head, ctx->ctx_msgq_tail, msg->pfm_gen_msg.msg_type));
-
- return msg;
-}
-
-static void
-pfm_reset_msgq(pfm_context_t *ctx)
-{
- ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0;
- DPRINT(("ctx=%p msgq reset\n", ctx));
-}
-
-static pfm_context_t *
-pfm_context_alloc(int ctx_flags)
-{
- pfm_context_t *ctx;
-
- /*
- * allocate context descriptor
- * must be able to free with interrupts disabled
- */
- ctx = kzalloc(sizeof(pfm_context_t), GFP_KERNEL);
- if (ctx) {
- DPRINT(("alloc ctx @%p\n", ctx));
-
- /*
- * init context protection lock
- */
- spin_lock_init(&ctx->ctx_lock);
-
- /*
- * context is unloaded
- */
- ctx->ctx_state = PFM_CTX_UNLOADED;
-
- /*
- * initialization of context's flags
- */
- ctx->ctx_fl_block = (ctx_flags & PFM_FL_NOTIFY_BLOCK) ? 1 : 0;
- ctx->ctx_fl_system = (ctx_flags & PFM_FL_SYSTEM_WIDE) ? 1: 0;
- ctx->ctx_fl_no_msg = (ctx_flags & PFM_FL_OVFL_NO_MSG) ? 1: 0;
- /*
- * will move to set properties
- * ctx->ctx_fl_excl_idle = (ctx_flags & PFM_FL_EXCL_IDLE) ? 1: 0;
- */
-
- /*
- * init restart semaphore to locked
- */
- init_completion(&ctx->ctx_restart_done);
-
- /*
- * activation is used in SMP only
- */
- ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
- SET_LAST_CPU(ctx, -1);
-
- /*
- * initialize notification message queue
- */
- ctx->ctx_msgq_head = ctx->ctx_msgq_tail = 0;
- init_waitqueue_head(&ctx->ctx_msgq_wait);
- init_waitqueue_head(&ctx->ctx_zombieq);
-
- }
- return ctx;
-}
-
-static void
-pfm_context_free(pfm_context_t *ctx)
-{
- if (ctx) {
- DPRINT(("free ctx @%p\n", ctx));
- kfree(ctx);
- }
-}
-
-static void
-pfm_mask_monitoring(struct task_struct *task)
-{
- pfm_context_t *ctx = PFM_GET_CTX(task);
- unsigned long mask, val, ovfl_mask;
- int i;
-
- DPRINT_ovfl(("masking monitoring for [%d]\n", task_pid_nr(task)));
-
- ovfl_mask = pmu_conf->ovfl_val;
- /*
- * monitoring can only be masked as a result of a valid
- * counter overflow. In UP, it means that the PMU still
- * has an owner. Note that the owner can be different
- * from the current task. However the PMU state belongs
- * to the owner.
- * In SMP, a valid overflow only happens when task is
- * current. Therefore if we come here, we know that
- * the PMU state belongs to the current task, therefore
- * we can access the live registers.
- *
- * So in both cases, the live register contains the owner's
- * state. We can ONLY touch the PMU registers and NOT the PSR.
- *
- * As a consequence to this call, the ctx->th_pmds[] array
- * contains stale information which must be ignored
- * when context is reloaded AND monitoring is active (see
- * pfm_restart).
- */
- mask = ctx->ctx_used_pmds[0];
- for (i = 0; mask; i++, mask>>=1) {
- /* skip non used pmds */
- if ((mask & 0x1) == 0) continue;
- val = ia64_get_pmd(i);
-
- if (PMD_IS_COUNTING(i)) {
- /*
- * we rebuild the full 64 bit value of the counter
- */
- ctx->ctx_pmds[i].val += (val & ovfl_mask);
- } else {
- ctx->ctx_pmds[i].val = val;
- }
- DPRINT_ovfl(("pmd[%d]=0x%lx hw_pmd=0x%lx\n",
- i,
- ctx->ctx_pmds[i].val,
- val & ovfl_mask));
- }
- /*
- * mask monitoring by setting the privilege level to 0
- * we cannot use psr.pp/psr.up for this, it is controlled by
- * the user
- *
- * if task is current, modify actual registers, otherwise modify
- * thread save state, i.e., what will be restored in pfm_load_regs()
- */
- mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER;
- for(i= PMU_FIRST_COUNTER; mask; i++, mask>>=1) {
- if ((mask & 0x1) == 0UL) continue;
- ia64_set_pmc(i, ctx->th_pmcs[i] & ~0xfUL);
- ctx->th_pmcs[i] &= ~0xfUL;
- DPRINT_ovfl(("pmc[%d]=0x%lx\n", i, ctx->th_pmcs[i]));
- }
- /*
- * make all of this visible
- */
- ia64_srlz_d();
-}
-
-/*
- * must always be done with task == current
- *
- * context must be in MASKED state when calling
- */
-static void
-pfm_restore_monitoring(struct task_struct *task)
-{
- pfm_context_t *ctx = PFM_GET_CTX(task);
- unsigned long mask, ovfl_mask;
- unsigned long psr, val;
- int i, is_system;
-
- is_system = ctx->ctx_fl_system;
- ovfl_mask = pmu_conf->ovfl_val;
-
- if (task != current) {
- printk(KERN_ERR "perfmon.%d: invalid task[%d] current[%d]\n", __LINE__, task_pid_nr(task), task_pid_nr(current));
- return;
- }
- if (ctx->ctx_state != PFM_CTX_MASKED) {
- printk(KERN_ERR "perfmon.%d: task[%d] current[%d] invalid state=%d\n", __LINE__,
- task_pid_nr(task), task_pid_nr(current), ctx->ctx_state);
- return;
- }
- psr = pfm_get_psr();
- /*
- * monitoring is masked via the PMC.
- * As we restore their value, we do not want each counter to
- * restart right away. We stop monitoring using the PSR,
- * restore the PMC (and PMD) and then re-establish the psr
- * as it was. Note that there can be no pending overflow at
- * this point, because monitoring was MASKED.
- *
- * system-wide session are pinned and self-monitoring
- */
- if (is_system && (PFM_CPUINFO_GET() & PFM_CPUINFO_DCR_PP)) {
- /* disable dcr pp */
- ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) & ~IA64_DCR_PP);
- pfm_clear_psr_pp();
- } else {
- pfm_clear_psr_up();
- }
- /*
- * first, we restore the PMD
- */
- mask = ctx->ctx_used_pmds[0];
- for (i = 0; mask; i++, mask>>=1) {
- /* skip non used pmds */
- if ((mask & 0x1) == 0) continue;
-
- if (PMD_IS_COUNTING(i)) {
- /*
- * we split the 64bit value according to
- * counter width
- */
- val = ctx->ctx_pmds[i].val & ovfl_mask;
- ctx->ctx_pmds[i].val &= ~ovfl_mask;
- } else {
- val = ctx->ctx_pmds[i].val;
- }
- ia64_set_pmd(i, val);
-
- DPRINT(("pmd[%d]=0x%lx hw_pmd=0x%lx\n",
- i,
- ctx->ctx_pmds[i].val,
- val));
- }
- /*
- * restore the PMCs
- */
- mask = ctx->ctx_used_monitors[0] >> PMU_FIRST_COUNTER;
- for(i= PMU_FIRST_COUNTER; mask; i++, mask>>=1) {
- if ((mask & 0x1) == 0UL) continue;
- ctx->th_pmcs[i] = ctx->ctx_pmcs[i];
- ia64_set_pmc(i, ctx->th_pmcs[i]);
- DPRINT(("[%d] pmc[%d]=0x%lx\n",
- task_pid_nr(task), i, ctx->th_pmcs[i]));
- }
- ia64_srlz_d();
-
- /*
- * must restore DBR/IBR because could be modified while masked
- * XXX: need to optimize
- */
- if (ctx->ctx_fl_using_dbreg) {
- pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
- pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
- }
-
- /*
- * now restore PSR
- */
- if (is_system && (PFM_CPUINFO_GET() & PFM_CPUINFO_DCR_PP)) {
- /* enable dcr pp */
- ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) | IA64_DCR_PP);
- ia64_srlz_i();
- }
- pfm_set_psr_l(psr);
-}
-
-static inline void
-pfm_save_pmds(unsigned long *pmds, unsigned long mask)
-{
- int i;
-
- ia64_srlz_d();
-
- for (i=0; mask; i++, mask>>=1) {
- if (mask & 0x1) pmds[i] = ia64_get_pmd(i);
- }
-}
-
-/*
- * reload from thread state (used for ctxw only)
- */
-static inline void
-pfm_restore_pmds(unsigned long *pmds, unsigned long mask)
-{
- int i;
- unsigned long val, ovfl_val = pmu_conf->ovfl_val;
-
- for (i=0; mask; i++, mask>>=1) {
- if ((mask & 0x1) == 0) continue;
- val = PMD_IS_COUNTING(i) ? pmds[i] & ovfl_val : pmds[i];
- ia64_set_pmd(i, val);
- }
- ia64_srlz_d();
-}
-
-/*
- * propagate PMD from context to thread-state
- */
-static inline void
-pfm_copy_pmds(struct task_struct *task, pfm_context_t *ctx)
-{
- unsigned long ovfl_val = pmu_conf->ovfl_val;
- unsigned long mask = ctx->ctx_all_pmds[0];
- unsigned long val;
- int i;
-
- DPRINT(("mask=0x%lx\n", mask));
-
- for (i=0; mask; i++, mask>>=1) {
-
- val = ctx->ctx_pmds[i].val;
-
- /*
- * We break up the 64 bit value into 2 pieces
- * the lower bits go to the machine state in the
- * thread (will be reloaded on ctxsw in).
- * The upper part stays in the soft-counter.
- */
- if (PMD_IS_COUNTING(i)) {
- ctx->ctx_pmds[i].val = val & ~ovfl_val;
- val &= ovfl_val;
- }
- ctx->th_pmds[i] = val;
-
- DPRINT(("pmd[%d]=0x%lx soft_val=0x%lx\n",
- i,
- ctx->th_pmds[i],
- ctx->ctx_pmds[i].val));
- }
-}
-
-/*
- * propagate PMC from context to thread-state
- */
-static inline void
-pfm_copy_pmcs(struct task_struct *task, pfm_context_t *ctx)
-{
- unsigned long mask = ctx->ctx_all_pmcs[0];
- int i;
-
- DPRINT(("mask=0x%lx\n", mask));
-
- for (i=0; mask; i++, mask>>=1) {
- /* masking 0 with ovfl_val yields 0 */
- ctx->th_pmcs[i] = ctx->ctx_pmcs[i];
- DPRINT(("pmc[%d]=0x%lx\n", i, ctx->th_pmcs[i]));
- }
-}
-
-
-
-static inline void
-pfm_restore_pmcs(unsigned long *pmcs, unsigned long mask)
-{
- int i;
-
- for (i=0; mask; i++, mask>>=1) {
- if ((mask & 0x1) == 0) continue;
- ia64_set_pmc(i, pmcs[i]);
- }
- ia64_srlz_d();
-}
-
-static inline int
-pfm_uuid_cmp(pfm_uuid_t a, pfm_uuid_t b)
-{
- return memcmp(a, b, sizeof(pfm_uuid_t));
-}
-
-static inline int
-pfm_buf_fmt_exit(pfm_buffer_fmt_t *fmt, struct task_struct *task, void *buf, struct pt_regs *regs)
-{
- int ret = 0;
- if (fmt->fmt_exit) ret = (*fmt->fmt_exit)(task, buf, regs);
- return ret;
-}
-
-static inline int
-pfm_buf_fmt_getsize(pfm_buffer_fmt_t *fmt, struct task_struct *task, unsigned int flags, int cpu, void *arg, unsigned long *size)
-{
- int ret = 0;
- if (fmt->fmt_getsize) ret = (*fmt->fmt_getsize)(task, flags, cpu, arg, size);
- return ret;
-}
-
-
-static inline int
-pfm_buf_fmt_validate(pfm_buffer_fmt_t *fmt, struct task_struct *task, unsigned int flags,
- int cpu, void *arg)
-{
- int ret = 0;
- if (fmt->fmt_validate) ret = (*fmt->fmt_validate)(task, flags, cpu, arg);
- return ret;
-}
-
-static inline int
-pfm_buf_fmt_init(pfm_buffer_fmt_t *fmt, struct task_struct *task, void *buf, unsigned int flags,
- int cpu, void *arg)
-{
- int ret = 0;
- if (fmt->fmt_init) ret = (*fmt->fmt_init)(task, buf, flags, cpu, arg);
- return ret;
-}
-
-static inline int
-pfm_buf_fmt_restart(pfm_buffer_fmt_t *fmt, struct task_struct *task, pfm_ovfl_ctrl_t *ctrl, void *buf, struct pt_regs *regs)
-{
- int ret = 0;
- if (fmt->fmt_restart) ret = (*fmt->fmt_restart)(task, ctrl, buf, regs);
- return ret;
-}
-
-static inline int
-pfm_buf_fmt_restart_active(pfm_buffer_fmt_t *fmt, struct task_struct *task, pfm_ovfl_ctrl_t *ctrl, void *buf, struct pt_regs *regs)
-{
- int ret = 0;
- if (fmt->fmt_restart_active) ret = (*fmt->fmt_restart_active)(task, ctrl, buf, regs);
- return ret;
-}
-
-static pfm_buffer_fmt_t *
-__pfm_find_buffer_fmt(pfm_uuid_t uuid)
-{
- struct list_head * pos;
- pfm_buffer_fmt_t * entry;
-
- list_for_each(pos, &pfm_buffer_fmt_list) {
- entry = list_entry(pos, pfm_buffer_fmt_t, fmt_list);
- if (pfm_uuid_cmp(uuid, entry->fmt_uuid) == 0)
- return entry;
- }
- return NULL;
-}
-
-/*
- * find a buffer format based on its uuid
- */
-static pfm_buffer_fmt_t *
-pfm_find_buffer_fmt(pfm_uuid_t uuid)
-{
- pfm_buffer_fmt_t * fmt;
- spin_lock(&pfm_buffer_fmt_lock);
- fmt = __pfm_find_buffer_fmt(uuid);
- spin_unlock(&pfm_buffer_fmt_lock);
- return fmt;
-}
-
-int
-pfm_register_buffer_fmt(pfm_buffer_fmt_t *fmt)
-{
- int ret = 0;
-
- /* some sanity checks */
- if (fmt == NULL || fmt->fmt_name == NULL) return -EINVAL;
-
- /* we need at least a handler */
- if (fmt->fmt_handler == NULL) return -EINVAL;
-
- /*
- * XXX: need check validity of fmt_arg_size
- */
-
- spin_lock(&pfm_buffer_fmt_lock);
-
- if (__pfm_find_buffer_fmt(fmt->fmt_uuid)) {
- printk(KERN_ERR "perfmon: duplicate sampling format: %s\n", fmt->fmt_name);
- ret = -EBUSY;
- goto out;
- }
- list_add(&fmt->fmt_list, &pfm_buffer_fmt_list);
- printk(KERN_INFO "perfmon: added sampling format %s\n", fmt->fmt_name);
-
-out:
- spin_unlock(&pfm_buffer_fmt_lock);
- return ret;
-}
-EXPORT_SYMBOL(pfm_register_buffer_fmt);
-
-int
-pfm_unregister_buffer_fmt(pfm_uuid_t uuid)
-{
- pfm_buffer_fmt_t *fmt;
- int ret = 0;
-
- spin_lock(&pfm_buffer_fmt_lock);
-
- fmt = __pfm_find_buffer_fmt(uuid);
- if (!fmt) {
- printk(KERN_ERR "perfmon: cannot unregister format, not found\n");
- ret = -EINVAL;
- goto out;
- }
- list_del_init(&fmt->fmt_list);
- printk(KERN_INFO "perfmon: removed sampling format: %s\n", fmt->fmt_name);
-
-out:
- spin_unlock(&pfm_buffer_fmt_lock);
- return ret;
-
-}
-EXPORT_SYMBOL(pfm_unregister_buffer_fmt);
-
-static int
-pfm_reserve_session(struct task_struct *task, int is_syswide, unsigned int cpu)
-{
- unsigned long flags;
- /*
- * validity checks on cpu_mask have been done upstream
- */
- LOCK_PFS(flags);
-
- DPRINT(("in sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
- pfm_sessions.pfs_sys_sessions,
- pfm_sessions.pfs_task_sessions,
- pfm_sessions.pfs_sys_use_dbregs,
- is_syswide,
- cpu));
-
- if (is_syswide) {
- /*
- * cannot mix system wide and per-task sessions
- */
- if (pfm_sessions.pfs_task_sessions > 0UL) {
- DPRINT(("system wide not possible, %u conflicting task_sessions\n",
- pfm_sessions.pfs_task_sessions));
- goto abort;
- }
-
- if (pfm_sessions.pfs_sys_session[cpu]) goto error_conflict;
-
- DPRINT(("reserving system wide session on CPU%u currently on CPU%u\n", cpu, smp_processor_id()));
-
- pfm_sessions.pfs_sys_session[cpu] = task;
-
- pfm_sessions.pfs_sys_sessions++ ;
-
- } else {
- if (pfm_sessions.pfs_sys_sessions) goto abort;
- pfm_sessions.pfs_task_sessions++;
- }
-
- DPRINT(("out sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
- pfm_sessions.pfs_sys_sessions,
- pfm_sessions.pfs_task_sessions,
- pfm_sessions.pfs_sys_use_dbregs,
- is_syswide,
- cpu));
-
- /*
- * Force idle() into poll mode
- */
- cpu_idle_poll_ctrl(true);
-
- UNLOCK_PFS(flags);
-
- return 0;
-
-error_conflict:
- DPRINT(("system wide not possible, conflicting session [%d] on CPU%d\n",
- task_pid_nr(pfm_sessions.pfs_sys_session[cpu]),
- cpu));
-abort:
- UNLOCK_PFS(flags);
-
- return -EBUSY;
-
-}
-
-static int
-pfm_unreserve_session(pfm_context_t *ctx, int is_syswide, unsigned int cpu)
-{
- unsigned long flags;
- /*
- * validity checks on cpu_mask have been done upstream
- */
- LOCK_PFS(flags);
-
- DPRINT(("in sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
- pfm_sessions.pfs_sys_sessions,
- pfm_sessions.pfs_task_sessions,
- pfm_sessions.pfs_sys_use_dbregs,
- is_syswide,
- cpu));
-
-
- if (is_syswide) {
- pfm_sessions.pfs_sys_session[cpu] = NULL;
- /*
- * would not work with perfmon+more than one bit in cpu_mask
- */
- if (ctx && ctx->ctx_fl_using_dbreg) {
- if (pfm_sessions.pfs_sys_use_dbregs == 0) {
- printk(KERN_ERR "perfmon: invalid release for ctx %p sys_use_dbregs=0\n", ctx);
- } else {
- pfm_sessions.pfs_sys_use_dbregs--;
- }
- }
- pfm_sessions.pfs_sys_sessions--;
- } else {
- pfm_sessions.pfs_task_sessions--;
- }
- DPRINT(("out sys_sessions=%u task_sessions=%u dbregs=%u syswide=%d cpu=%u\n",
- pfm_sessions.pfs_sys_sessions,
- pfm_sessions.pfs_task_sessions,
- pfm_sessions.pfs_sys_use_dbregs,
- is_syswide,
- cpu));
-
- /* Undo forced polling. Last session reenables pal_halt */
- cpu_idle_poll_ctrl(false);
-
- UNLOCK_PFS(flags);
-
- return 0;
-}
-
-/*
- * removes virtual mapping of the sampling buffer.
- * IMPORTANT: cannot be called with interrupts disable, e.g. inside
- * a PROTECT_CTX() section.
- */
-static int
-pfm_remove_smpl_mapping(void *vaddr, unsigned long size)
-{
- struct task_struct *task = current;
- int r;
-
- /* sanity checks */
- if (task->mm == NULL || size == 0UL || vaddr == NULL) {
- printk(KERN_ERR "perfmon: pfm_remove_smpl_mapping [%d] invalid context mm=%p\n", task_pid_nr(task), task->mm);
- return -EINVAL;
- }
-
- DPRINT(("smpl_vaddr=%p size=%lu\n", vaddr, size));
-
- /*
- * does the actual unmapping
- */
- r = vm_munmap((unsigned long)vaddr, size);
-
- if (r !=0) {
- printk(KERN_ERR "perfmon: [%d] unable to unmap sampling buffer @%p size=%lu\n", task_pid_nr(task), vaddr, size);
- }
-
- DPRINT(("do_unmap(%p, %lu)=%d\n", vaddr, size, r));
-
- return 0;
-}
-
-/*
- * free actual physical storage used by sampling buffer
- */
-#if 0
-static int
-pfm_free_smpl_buffer(pfm_context_t *ctx)
-{
- pfm_buffer_fmt_t *fmt;
-
- if (ctx->ctx_smpl_hdr == NULL) goto invalid_free;
-
- /*
- * we won't use the buffer format anymore
- */
- fmt = ctx->ctx_buf_fmt;
-
- DPRINT(("sampling buffer @%p size %lu vaddr=%p\n",
- ctx->ctx_smpl_hdr,
- ctx->ctx_smpl_size,
- ctx->ctx_smpl_vaddr));
-
- pfm_buf_fmt_exit(fmt, current, NULL, NULL);
-
- /*
- * free the buffer
- */
- vfree(ctx->ctx_smpl_hdr);
-
- ctx->ctx_smpl_hdr = NULL;
- ctx->ctx_smpl_size = 0UL;
-
- return 0;
-
-invalid_free:
- printk(KERN_ERR "perfmon: pfm_free_smpl_buffer [%d] no buffer\n", task_pid_nr(current));
- return -EINVAL;
-}
-#endif
-
-static inline void
-pfm_exit_smpl_buffer(pfm_buffer_fmt_t *fmt)
-{
- if (fmt == NULL) return;
-
- pfm_buf_fmt_exit(fmt, current, NULL, NULL);
-
-}
-
-/*
- * pfmfs should _never_ be mounted by userland - too much of security hassle,
- * no real gain from having the whole whorehouse mounted. So we don't need
- * any operations on the root directory. However, we need a non-trivial
- * d_name - pfm: will go nicely and kill the special-casing in procfs.
- */
-static struct vfsmount *pfmfs_mnt __read_mostly;
-
-static int __init
-init_pfm_fs(void)
-{
- int err = register_filesystem(&pfm_fs_type);
- if (!err) {
- pfmfs_mnt = kern_mount(&pfm_fs_type);
- err = PTR_ERR(pfmfs_mnt);
- if (IS_ERR(pfmfs_mnt))
- unregister_filesystem(&pfm_fs_type);
- else
- err = 0;
- }
- return err;
-}
-
-static ssize_t
-pfm_read(struct file *filp, char __user *buf, size_t size, loff_t *ppos)
-{
- pfm_context_t *ctx;
- pfm_msg_t *msg;
- ssize_t ret;
- unsigned long flags;
- DECLARE_WAITQUEUE(wait, current);
- if (PFM_IS_FILE(filp) == 0) {
- printk(KERN_ERR "perfmon: pfm_poll: bad magic [%d]\n", task_pid_nr(current));
- return -EINVAL;
- }
-
- ctx = filp->private_data;
- if (ctx == NULL) {
- printk(KERN_ERR "perfmon: pfm_read: NULL ctx [%d]\n", task_pid_nr(current));
- return -EINVAL;
- }
-
- /*
- * check even when there is no message
- */
- if (size < sizeof(pfm_msg_t)) {
- DPRINT(("message is too small ctx=%p (>=%ld)\n", ctx, sizeof(pfm_msg_t)));
- return -EINVAL;
- }
-
- PROTECT_CTX(ctx, flags);
-
- /*
- * put ourselves on the wait queue
- */
- add_wait_queue(&ctx->ctx_msgq_wait, &wait);
-
-
- for(;;) {
- /*
- * check wait queue
- */
-
- set_current_state(TASK_INTERRUPTIBLE);
-
- DPRINT(("head=%d tail=%d\n", ctx->ctx_msgq_head, ctx->ctx_msgq_tail));
-
- ret = 0;
- if(PFM_CTXQ_EMPTY(ctx) == 0) break;
-
- UNPROTECT_CTX(ctx, flags);
-
- /*
- * check non-blocking read
- */
- ret = -EAGAIN;
- if(filp->f_flags & O_NONBLOCK) break;
-
- /*
- * check pending signals
- */
- if(signal_pending(current)) {
- ret = -EINTR;
- break;
- }
- /*
- * no message, so wait
- */
- schedule();
-
- PROTECT_CTX(ctx, flags);
- }
- DPRINT(("[%d] back to running ret=%ld\n", task_pid_nr(current), ret));
- set_current_state(TASK_RUNNING);
- remove_wait_queue(&ctx->ctx_msgq_wait, &wait);
-
- if (ret < 0) goto abort;
-
- ret = -EINVAL;
- msg = pfm_get_next_msg(ctx);
- if (msg == NULL) {
- printk(KERN_ERR "perfmon: pfm_read no msg for ctx=%p [%d]\n", ctx, task_pid_nr(current));
- goto abort_locked;
- }
-
- DPRINT(("fd=%d type=%d\n", msg->pfm_gen_msg.msg_ctx_fd, msg->pfm_gen_msg.msg_type));
-
- ret = -EFAULT;
- if(copy_to_user(buf, msg, sizeof(pfm_msg_t)) == 0) ret = sizeof(pfm_msg_t);
-
-abort_locked:
- UNPROTECT_CTX(ctx, flags);
-abort:
- return ret;
-}
-
-static ssize_t
-pfm_write(struct file *file, const char __user *ubuf,
- size_t size, loff_t *ppos)
-{
- DPRINT(("pfm_write called\n"));
- return -EINVAL;
-}
-
-static __poll_t
-pfm_poll(struct file *filp, poll_table * wait)
-{
- pfm_context_t *ctx;
- unsigned long flags;
- __poll_t mask = 0;
-
- if (PFM_IS_FILE(filp) == 0) {
- printk(KERN_ERR "perfmon: pfm_poll: bad magic [%d]\n", task_pid_nr(current));
- return 0;
- }
-
- ctx = filp->private_data;
- if (ctx == NULL) {
- printk(KERN_ERR "perfmon: pfm_poll: NULL ctx [%d]\n", task_pid_nr(current));
- return 0;
- }
-
-
- DPRINT(("pfm_poll ctx_fd=%d before poll_wait\n", ctx->ctx_fd));
-
- poll_wait(filp, &ctx->ctx_msgq_wait, wait);
-
- PROTECT_CTX(ctx, flags);
-
- if (PFM_CTXQ_EMPTY(ctx) == 0)
- mask = EPOLLIN | EPOLLRDNORM;
-
- UNPROTECT_CTX(ctx, flags);
-
- DPRINT(("pfm_poll ctx_fd=%d mask=0x%x\n", ctx->ctx_fd, mask));
-
- return mask;
-}
-
-static long
-pfm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
-{
- DPRINT(("pfm_ioctl called\n"));
- return -EINVAL;
-}
-
-/*
- * interrupt cannot be masked when coming here
- */
-static inline int
-pfm_do_fasync(int fd, struct file *filp, pfm_context_t *ctx, int on)
-{
- int ret;
-
- ret = fasync_helper (fd, filp, on, &ctx->ctx_async_queue);
-
- DPRINT(("pfm_fasync called by [%d] on ctx_fd=%d on=%d async_queue=%p ret=%d\n",
- task_pid_nr(current),
- fd,
- on,
- ctx->ctx_async_queue, ret));
-
- return ret;
-}
-
-static int
-pfm_fasync(int fd, struct file *filp, int on)
-{
- pfm_context_t *ctx;
- int ret;
-
- if (PFM_IS_FILE(filp) == 0) {
- printk(KERN_ERR "perfmon: pfm_fasync bad magic [%d]\n", task_pid_nr(current));
- return -EBADF;
- }
-
- ctx = filp->private_data;
- if (ctx == NULL) {
- printk(KERN_ERR "perfmon: pfm_fasync NULL ctx [%d]\n", task_pid_nr(current));
- return -EBADF;
- }
- /*
- * we cannot mask interrupts during this call because this may
- * may go to sleep if memory is not readily avalaible.
- *
- * We are protected from the conetxt disappearing by the get_fd()/put_fd()
- * done in caller. Serialization of this function is ensured by caller.
- */
- ret = pfm_do_fasync(fd, filp, ctx, on);
-
-
- DPRINT(("pfm_fasync called on ctx_fd=%d on=%d async_queue=%p ret=%d\n",
- fd,
- on,
- ctx->ctx_async_queue, ret));
-
- return ret;
-}
-
-#ifdef CONFIG_SMP
-/*
- * this function is exclusively called from pfm_close().
- * The context is not protected at that time, nor are interrupts
- * on the remote CPU. That's necessary to avoid deadlocks.
- */
-static void
-pfm_syswide_force_stop(void *info)
-{
- pfm_context_t *ctx = (pfm_context_t *)info;
- struct pt_regs *regs = task_pt_regs(current);
- struct task_struct *owner;
- unsigned long flags;
- int ret;
-
- if (ctx->ctx_cpu != smp_processor_id()) {
- printk(KERN_ERR "perfmon: pfm_syswide_force_stop for CPU%d but on CPU%d\n",
- ctx->ctx_cpu,
- smp_processor_id());
- return;
- }
- owner = GET_PMU_OWNER();
- if (owner != ctx->ctx_task) {
- printk(KERN_ERR "perfmon: pfm_syswide_force_stop CPU%d unexpected owner [%d] instead of [%d]\n",
- smp_processor_id(),
- task_pid_nr(owner), task_pid_nr(ctx->ctx_task));
- return;
- }
- if (GET_PMU_CTX() != ctx) {
- printk(KERN_ERR "perfmon: pfm_syswide_force_stop CPU%d unexpected ctx %p instead of %p\n",
- smp_processor_id(),
- GET_PMU_CTX(), ctx);
- return;
- }
-
- DPRINT(("on CPU%d forcing system wide stop for [%d]\n", smp_processor_id(), task_pid_nr(ctx->ctx_task)));
- /*
- * the context is already protected in pfm_close(), we simply
- * need to mask interrupts to avoid a PMU interrupt race on
- * this CPU
- */
- local_irq_save(flags);
-
- ret = pfm_context_unload(ctx, NULL, 0, regs);
- if (ret) {
- DPRINT(("context_unload returned %d\n", ret));
- }
-
- /*
- * unmask interrupts, PMU interrupts are now spurious here
- */
- local_irq_restore(flags);
-}
-
-static void
-pfm_syswide_cleanup_other_cpu(pfm_context_t *ctx)
-{
- int ret;
-
- DPRINT(("calling CPU%d for cleanup\n", ctx->ctx_cpu));
- ret = smp_call_function_single(ctx->ctx_cpu, pfm_syswide_force_stop, ctx, 1);
- DPRINT(("called CPU%d for cleanup ret=%d\n", ctx->ctx_cpu, ret));
-}
-#endif /* CONFIG_SMP */
-
-/*
- * called for each close(). Partially free resources.
- * When caller is self-monitoring, the context is unloaded.
- */
-static int
-pfm_flush(struct file *filp, fl_owner_t id)
-{
- pfm_context_t *ctx;
- struct task_struct *task;
- struct pt_regs *regs;
- unsigned long flags;
- unsigned long smpl_buf_size = 0UL;
- void *smpl_buf_vaddr = NULL;
- int state, is_system;
-
- if (PFM_IS_FILE(filp) == 0) {
- DPRINT(("bad magic for\n"));
- return -EBADF;
- }
-
- ctx = filp->private_data;
- if (ctx == NULL) {
- printk(KERN_ERR "perfmon: pfm_flush: NULL ctx [%d]\n", task_pid_nr(current));
- return -EBADF;
- }
-
- /*
- * remove our file from the async queue, if we use this mode.
- * This can be done without the context being protected. We come
- * here when the context has become unreachable by other tasks.
- *
- * We may still have active monitoring at this point and we may
- * end up in pfm_overflow_handler(). However, fasync_helper()
- * operates with interrupts disabled and it cleans up the
- * queue. If the PMU handler is called prior to entering
- * fasync_helper() then it will send a signal. If it is
- * invoked after, it will find an empty queue and no
- * signal will be sent. In both case, we are safe
- */
- PROTECT_CTX(ctx, flags);
-
- state = ctx->ctx_state;
- is_system = ctx->ctx_fl_system;
-
- task = PFM_CTX_TASK(ctx);
- regs = task_pt_regs(task);
-
- DPRINT(("ctx_state=%d is_current=%d\n",
- state,
- task == current ? 1 : 0));
-
- /*
- * if state == UNLOADED, then task is NULL
- */
-
- /*
- * we must stop and unload because we are losing access to the context.
- */
- if (task == current) {
-#ifdef CONFIG_SMP
- /*
- * the task IS the owner but it migrated to another CPU: that's bad
- * but we must handle this cleanly. Unfortunately, the kernel does
- * not provide a mechanism to block migration (while the context is loaded).
- *
- * We need to release the resource on the ORIGINAL cpu.
- */
- if (is_system && ctx->ctx_cpu != smp_processor_id()) {
-
- DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
- /*
- * keep context protected but unmask interrupt for IPI
- */
- local_irq_restore(flags);
-
- pfm_syswide_cleanup_other_cpu(ctx);
-
- /*
- * restore interrupt masking
- */
- local_irq_save(flags);
-
- /*
- * context is unloaded at this point
- */
- } else
-#endif /* CONFIG_SMP */
- {
-
- DPRINT(("forcing unload\n"));
- /*
- * stop and unload, returning with state UNLOADED
- * and session unreserved.
- */
- pfm_context_unload(ctx, NULL, 0, regs);
-
- DPRINT(("ctx_state=%d\n", ctx->ctx_state));
- }
- }
-
- /*
- * remove virtual mapping, if any, for the calling task.
- * cannot reset ctx field until last user is calling close().
- *
- * ctx_smpl_vaddr must never be cleared because it is needed
- * by every task with access to the context
- *
- * When called from do_exit(), the mm context is gone already, therefore
- * mm is NULL, i.e., the VMA is already gone and we do not have to
- * do anything here
- */
- if (ctx->ctx_smpl_vaddr && current->mm) {
- smpl_buf_vaddr = ctx->ctx_smpl_vaddr;
- smpl_buf_size = ctx->ctx_smpl_size;
- }
-
- UNPROTECT_CTX(ctx, flags);
-
- /*
- * if there was a mapping, then we systematically remove it
- * at this point. Cannot be done inside critical section
- * because some VM function reenables interrupts.
- *
- */
- if (smpl_buf_vaddr) pfm_remove_smpl_mapping(smpl_buf_vaddr, smpl_buf_size);
-
- return 0;
-}
-/*
- * called either on explicit close() or from exit_files().
- * Only the LAST user of the file gets to this point, i.e., it is
- * called only ONCE.
- *
- * IMPORTANT: we get called ONLY when the refcnt on the file gets to zero
- * (fput()),i.e, last task to access the file. Nobody else can access the
- * file at this point.
- *
- * When called from exit_files(), the VMA has been freed because exit_mm()
- * is executed before exit_files().
- *
- * When called from exit_files(), the current task is not yet ZOMBIE but we
- * flush the PMU state to the context.
- */
-static int
-pfm_close(struct inode *inode, struct file *filp)
-{
- pfm_context_t *ctx;
- struct task_struct *task;
- struct pt_regs *regs;
- DECLARE_WAITQUEUE(wait, current);
- unsigned long flags;
- unsigned long smpl_buf_size = 0UL;
- void *smpl_buf_addr = NULL;
- int free_possible = 1;
- int state, is_system;
-
- DPRINT(("pfm_close called private=%p\n", filp->private_data));
-
- if (PFM_IS_FILE(filp) == 0) {
- DPRINT(("bad magic\n"));
- return -EBADF;
- }
-
- ctx = filp->private_data;
- if (ctx == NULL) {
- printk(KERN_ERR "perfmon: pfm_close: NULL ctx [%d]\n", task_pid_nr(current));
- return -EBADF;
- }
-
- PROTECT_CTX(ctx, flags);
-
- state = ctx->ctx_state;
- is_system = ctx->ctx_fl_system;
-
- task = PFM_CTX_TASK(ctx);
- regs = task_pt_regs(task);
-
- DPRINT(("ctx_state=%d is_current=%d\n",
- state,
- task == current ? 1 : 0));
-
- /*
- * if task == current, then pfm_flush() unloaded the context
- */
- if (state == PFM_CTX_UNLOADED) goto doit;
-
- /*
- * context is loaded/masked and task != current, we need to
- * either force an unload or go zombie
- */
-
- /*
- * The task is currently blocked or will block after an overflow.
- * we must force it to wakeup to get out of the
- * MASKED state and transition to the unloaded state by itself.
- *
- * This situation is only possible for per-task mode
- */
- if (state == PFM_CTX_MASKED && CTX_OVFL_NOBLOCK(ctx) == 0) {
-
- /*
- * set a "partial" zombie state to be checked
- * upon return from down() in pfm_handle_work().
- *
- * We cannot use the ZOMBIE state, because it is checked
- * by pfm_load_regs() which is called upon wakeup from down().
- * In such case, it would free the context and then we would
- * return to pfm_handle_work() which would access the
- * stale context. Instead, we set a flag invisible to pfm_load_regs()
- * but visible to pfm_handle_work().
- *
- * For some window of time, we have a zombie context with
- * ctx_state = MASKED and not ZOMBIE
- */
- ctx->ctx_fl_going_zombie = 1;
-
- /*
- * force task to wake up from MASKED state
- */
- complete(&ctx->ctx_restart_done);
-
- DPRINT(("waking up ctx_state=%d\n", state));
-
- /*
- * put ourself to sleep waiting for the other
- * task to report completion
- *
- * the context is protected by mutex, therefore there
- * is no risk of being notified of completion before
- * begin actually on the waitq.
- */
- set_current_state(TASK_INTERRUPTIBLE);
- add_wait_queue(&ctx->ctx_zombieq, &wait);
-
- UNPROTECT_CTX(ctx, flags);
-
- /*
- * XXX: check for signals :
- * - ok for explicit close
- * - not ok when coming from exit_files()
- */
- schedule();
-
-
- PROTECT_CTX(ctx, flags);
-
-
- remove_wait_queue(&ctx->ctx_zombieq, &wait);
- set_current_state(TASK_RUNNING);
-
- /*
- * context is unloaded at this point
- */
- DPRINT(("after zombie wakeup ctx_state=%d for\n", state));
- }
- else if (task != current) {
-#ifdef CONFIG_SMP
- /*
- * switch context to zombie state
- */
- ctx->ctx_state = PFM_CTX_ZOMBIE;
-
- DPRINT(("zombie ctx for [%d]\n", task_pid_nr(task)));
- /*
- * cannot free the context on the spot. deferred until
- * the task notices the ZOMBIE state
- */
- free_possible = 0;
-#else
- pfm_context_unload(ctx, NULL, 0, regs);
-#endif
- }
-
-doit:
- /* reload state, may have changed during opening of critical section */
- state = ctx->ctx_state;
-
- /*
- * the context is still attached to a task (possibly current)
- * we cannot destroy it right now
- */
-
- /*
- * we must free the sampling buffer right here because
- * we cannot rely on it being cleaned up later by the
- * monitored task. It is not possible to free vmalloc'ed
- * memory in pfm_load_regs(). Instead, we remove the buffer
- * now. should there be subsequent PMU overflow originally
- * meant for sampling, the will be converted to spurious
- * and that's fine because the monitoring tools is gone anyway.
- */
- if (ctx->ctx_smpl_hdr) {
- smpl_buf_addr = ctx->ctx_smpl_hdr;
- smpl_buf_size = ctx->ctx_smpl_size;
- /* no more sampling */
- ctx->ctx_smpl_hdr = NULL;
- ctx->ctx_fl_is_sampling = 0;
- }
-
- DPRINT(("ctx_state=%d free_possible=%d addr=%p size=%lu\n",
- state,
- free_possible,
- smpl_buf_addr,
- smpl_buf_size));
-
- if (smpl_buf_addr) pfm_exit_smpl_buffer(ctx->ctx_buf_fmt);
-
- /*
- * UNLOADED that the session has already been unreserved.
- */
- if (state == PFM_CTX_ZOMBIE) {
- pfm_unreserve_session(ctx, ctx->ctx_fl_system , ctx->ctx_cpu);
- }
-
- /*
- * disconnect file descriptor from context must be done
- * before we unlock.
- */
- filp->private_data = NULL;
-
- /*
- * if we free on the spot, the context is now completely unreachable
- * from the callers side. The monitored task side is also cut, so we
- * can freely cut.
- *
- * If we have a deferred free, only the caller side is disconnected.
- */
- UNPROTECT_CTX(ctx, flags);
-
- /*
- * All memory free operations (especially for vmalloc'ed memory)
- * MUST be done with interrupts ENABLED.
- */
- vfree(smpl_buf_addr);
-
- /*
- * return the memory used by the context
- */
- if (free_possible) pfm_context_free(ctx);
-
- return 0;
-}
-
-static const struct file_operations pfm_file_ops = {
- .llseek = no_llseek,
- .read = pfm_read,
- .write = pfm_write,
- .poll = pfm_poll,
- .unlocked_ioctl = pfm_ioctl,
- .fasync = pfm_fasync,
- .release = pfm_close,
- .flush = pfm_flush
-};
-
-static char *pfmfs_dname(struct dentry *dentry, char *buffer, int buflen)
-{
- return dynamic_dname(dentry, buffer, buflen, "pfm:[%lu]",
- d_inode(dentry)->i_ino);
-}
-
-static const struct dentry_operations pfmfs_dentry_operations = {
- .d_delete = always_delete_dentry,
- .d_dname = pfmfs_dname,
-};
-
-
-static struct file *
-pfm_alloc_file(pfm_context_t *ctx)
-{
- struct file *file;
- struct inode *inode;
- struct path path;
- struct qstr this = { .name = "" };
-
- /*
- * allocate a new inode
- */
- inode = new_inode(pfmfs_mnt->mnt_sb);
- if (!inode)
- return ERR_PTR(-ENOMEM);
-
- DPRINT(("new inode ino=%ld @%p\n", inode->i_ino, inode));
-
- inode->i_mode = S_IFCHR|S_IRUGO;
- inode->i_uid = current_fsuid();
- inode->i_gid = current_fsgid();
-
- /*
- * allocate a new dcache entry
- */
- path.dentry = d_alloc(pfmfs_mnt->mnt_root, &this);
- if (!path.dentry) {
- iput(inode);
- return ERR_PTR(-ENOMEM);
- }
- path.mnt = mntget(pfmfs_mnt);
-
- d_add(path.dentry, inode);
-
- file = alloc_file(&path, FMODE_READ, &pfm_file_ops);
- if (IS_ERR(file)) {
- path_put(&path);
- return file;
- }
-
- file->f_flags = O_RDONLY;
- file->private_data = ctx;
-
- return file;
-}
-
-static int
-pfm_remap_buffer(struct vm_area_struct *vma, unsigned long buf, unsigned long addr, unsigned long size)
-{
- DPRINT(("CPU%d buf=0x%lx addr=0x%lx size=%ld\n", smp_processor_id(), buf, addr, size));
-
- while (size > 0) {
- unsigned long pfn = ia64_tpa(buf) >> PAGE_SHIFT;
-
-
- if (remap_pfn_range(vma, addr, pfn, PAGE_SIZE, PAGE_READONLY))
- return -ENOMEM;
-
- addr += PAGE_SIZE;
- buf += PAGE_SIZE;
- size -= PAGE_SIZE;
- }
- return 0;
-}
-
-/*
- * allocate a sampling buffer and remaps it into the user address space of the task
- */
-static int
-pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t *ctx, unsigned long rsize, void **user_vaddr)
-{
- struct mm_struct *mm = task->mm;
- struct vm_area_struct *vma = NULL;
- unsigned long size;
- void *smpl_buf;
-
-
- /*
- * the fixed header + requested size and align to page boundary
- */
- size = PAGE_ALIGN(rsize);
-
- DPRINT(("sampling buffer rsize=%lu size=%lu bytes\n", rsize, size));
-
- /*
- * check requested size to avoid Denial-of-service attacks
- * XXX: may have to refine this test
- * Check against address space limit.
- *
- * if ((mm->total_vm << PAGE_SHIFT) + len> task->rlim[RLIMIT_AS].rlim_cur)
- * return -ENOMEM;
- */
- if (size > task_rlimit(task, RLIMIT_MEMLOCK))
- return -ENOMEM;
-
- /*
- * We do the easy to undo allocations first.
- */
- smpl_buf = vzalloc(size);
- if (smpl_buf == NULL) {
- DPRINT(("Can't allocate sampling buffer\n"));
- return -ENOMEM;
- }
-
- DPRINT(("smpl_buf @%p\n", smpl_buf));
-
- /* allocate vma */
- vma = vm_area_alloc(mm);
- if (!vma) {
- DPRINT(("Cannot allocate vma\n"));
- goto error_kmem;
- }
-
- /*
- * partially initialize the vma for the sampling buffer
- */
- vma->vm_file = get_file(filp);
- vma->vm_flags = VM_READ|VM_MAYREAD|VM_DONTEXPAND|VM_DONTDUMP;
- vma->vm_page_prot = PAGE_READONLY; /* XXX may need to change */
-
- /*
- * Now we have everything we need and we can initialize
- * and connect all the data structures
- */
-
- ctx->ctx_smpl_hdr = smpl_buf;
- ctx->ctx_smpl_size = size; /* aligned size */
-
- /*
- * Let's do the difficult operations next.
- *
- * now we atomically find some area in the address space and
- * remap the buffer in it.
- */
- mmap_write_lock(task->mm);
-
- /* find some free area in address space, must have mmap sem held */
- vma->vm_start = get_unmapped_area(NULL, 0, size, 0, MAP_PRIVATE|MAP_ANONYMOUS);
- if (IS_ERR_VALUE(vma->vm_start)) {
- DPRINT(("Cannot find unmapped area for size %ld\n", size));
- mmap_write_unlock(task->mm);
- goto error;
- }
- vma->vm_end = vma->vm_start + size;
- vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
-
- DPRINT(("aligned size=%ld, hdr=%p mapped @0x%lx\n", size, ctx->ctx_smpl_hdr, vma->vm_start));
-
- /* can only be applied to current task, need to have the mm semaphore held when called */
- if (pfm_remap_buffer(vma, (unsigned long)smpl_buf, vma->vm_start, size)) {
- DPRINT(("Can't remap buffer\n"));
- mmap_write_unlock(task->mm);
- goto error;
- }
-
- /*
- * now insert the vma in the vm list for the process, must be
- * done with mmap lock held
- */
- insert_vm_struct(mm, vma);
-
- vm_stat_account(vma->vm_mm, vma->vm_flags, vma_pages(vma));
- mmap_write_unlock(task->mm);
-
- /*
- * keep track of user level virtual address
- */
- ctx->ctx_smpl_vaddr = (void *)vma->vm_start;
- *(unsigned long *)user_vaddr = vma->vm_start;
-
- return 0;
-
-error:
- vm_area_free(vma);
-error_kmem:
- vfree(smpl_buf);
-
- return -ENOMEM;
-}
-
-/*
- * XXX: do something better here
- */
-static int
-pfm_bad_permissions(struct task_struct *task)
-{
- const struct cred *tcred;
- kuid_t uid = current_uid();
- kgid_t gid = current_gid();
- int ret;
-
- rcu_read_lock();
- tcred = __task_cred(task);
-
- /* inspired by ptrace_attach() */
- DPRINT(("cur: uid=%d gid=%d task: euid=%d suid=%d uid=%d egid=%d sgid=%d\n",
- from_kuid(&init_user_ns, uid),
- from_kgid(&init_user_ns, gid),
- from_kuid(&init_user_ns, tcred->euid),
- from_kuid(&init_user_ns, tcred->suid),
- from_kuid(&init_user_ns, tcred->uid),
- from_kgid(&init_user_ns, tcred->egid),
- from_kgid(&init_user_ns, tcred->sgid)));
-
- ret = ((!uid_eq(uid, tcred->euid))
- || (!uid_eq(uid, tcred->suid))
- || (!uid_eq(uid, tcred->uid))
- || (!gid_eq(gid, tcred->egid))
- || (!gid_eq(gid, tcred->sgid))
- || (!gid_eq(gid, tcred->gid))) && !capable(CAP_SYS_PTRACE);
-
- rcu_read_unlock();
- return ret;
-}
-
-static int
-pfarg_is_sane(struct task_struct *task, pfarg_context_t *pfx)
-{
- int ctx_flags;
-
- /* valid signal */
-
- ctx_flags = pfx->ctx_flags;
-
- if (ctx_flags & PFM_FL_SYSTEM_WIDE) {
-
- /*
- * cannot block in this mode
- */
- if (ctx_flags & PFM_FL_NOTIFY_BLOCK) {
- DPRINT(("cannot use blocking mode when in system wide monitoring\n"));
- return -EINVAL;
- }
- } else {
- }
- /* probably more to add here */
-
- return 0;
-}
-
-static int
-pfm_setup_buffer_fmt(struct task_struct *task, struct file *filp, pfm_context_t *ctx, unsigned int ctx_flags,
- unsigned int cpu, pfarg_context_t *arg)
-{
- pfm_buffer_fmt_t *fmt = NULL;
- unsigned long size = 0UL;
- void *uaddr = NULL;
- void *fmt_arg = NULL;
- int ret = 0;
-#define PFM_CTXARG_BUF_ARG(a) (pfm_buffer_fmt_t *)(a+1)
-
- /* invoke and lock buffer format, if found */
- fmt = pfm_find_buffer_fmt(arg->ctx_smpl_buf_id);
- if (fmt == NULL) {
- DPRINT(("[%d] cannot find buffer format\n", task_pid_nr(task)));
- return -EINVAL;
- }
-
- /*
- * buffer argument MUST be contiguous to pfarg_context_t
- */
- if (fmt->fmt_arg_size) fmt_arg = PFM_CTXARG_BUF_ARG(arg);
-
- ret = pfm_buf_fmt_validate(fmt, task, ctx_flags, cpu, fmt_arg);
-
- DPRINT(("[%d] after validate(0x%x,%d,%p)=%d\n", task_pid_nr(task), ctx_flags, cpu, fmt_arg, ret));
-
- if (ret) goto error;
-
- /* link buffer format and context */
- ctx->ctx_buf_fmt = fmt;
- ctx->ctx_fl_is_sampling = 1; /* assume record() is defined */
-
- /*
- * check if buffer format wants to use perfmon buffer allocation/mapping service
- */
- ret = pfm_buf_fmt_getsize(fmt, task, ctx_flags, cpu, fmt_arg, &size);
- if (ret) goto error;
-
- if (size) {
- /*
- * buffer is always remapped into the caller's address space
- */
- ret = pfm_smpl_buffer_alloc(current, filp, ctx, size, &uaddr);
- if (ret) goto error;
-
- /* keep track of user address of buffer */
- arg->ctx_smpl_vaddr = uaddr;
- }
- ret = pfm_buf_fmt_init(fmt, task, ctx->ctx_smpl_hdr, ctx_flags, cpu, fmt_arg);
-
-error:
- return ret;
-}
-
-static void
-pfm_reset_pmu_state(pfm_context_t *ctx)
-{
- int i;
-
- /*
- * install reset values for PMC.
- */
- for (i=1; PMC_IS_LAST(i) == 0; i++) {
- if (PMC_IS_IMPL(i) == 0) continue;
- ctx->ctx_pmcs[i] = PMC_DFL_VAL(i);
- DPRINT(("pmc[%d]=0x%lx\n", i, ctx->ctx_pmcs[i]));
- }
- /*
- * PMD registers are set to 0UL when the context in memset()
- */
-
- /*
- * On context switched restore, we must restore ALL pmc and ALL pmd even
- * when they are not actively used by the task. In UP, the incoming process
- * may otherwise pick up left over PMC, PMD state from the previous process.
- * As opposed to PMD, stale PMC can cause harm to the incoming
- * process because they may change what is being measured.
- * Therefore, we must systematically reinstall the entire
- * PMC state. In SMP, the same thing is possible on the
- * same CPU but also on between 2 CPUs.
- *
- * The problem with PMD is information leaking especially
- * to user level when psr.sp=0
- *
- * There is unfortunately no easy way to avoid this problem
- * on either UP or SMP. This definitively slows down the
- * pfm_load_regs() function.
- */
-
- /*
- * bitmask of all PMCs accessible to this context
- *
- * PMC0 is treated differently.
- */
- ctx->ctx_all_pmcs[0] = pmu_conf->impl_pmcs[0] & ~0x1;
-
- /*
- * bitmask of all PMDs that are accessible to this context
- */
- ctx->ctx_all_pmds[0] = pmu_conf->impl_pmds[0];
-
- DPRINT(("<%d> all_pmcs=0x%lx all_pmds=0x%lx\n", ctx->ctx_fd, ctx->ctx_all_pmcs[0],ctx->ctx_all_pmds[0]));
-
- /*
- * useful in case of re-enable after disable
- */
- ctx->ctx_used_ibrs[0] = 0UL;
- ctx->ctx_used_dbrs[0] = 0UL;
-}
-
-static int
-pfm_ctx_getsize(void *arg, size_t *sz)
-{
- pfarg_context_t *req = (pfarg_context_t *)arg;
- pfm_buffer_fmt_t *fmt;
-
- *sz = 0;
-
- if (!pfm_uuid_cmp(req->ctx_smpl_buf_id, pfm_null_uuid)) return 0;
-
- fmt = pfm_find_buffer_fmt(req->ctx_smpl_buf_id);
- if (fmt == NULL) {
- DPRINT(("cannot find buffer format\n"));
- return -EINVAL;
- }
- /* get just enough to copy in user parameters */
- *sz = fmt->fmt_arg_size;
- DPRINT(("arg_size=%lu\n", *sz));
-
- return 0;
-}
-
-
-
-/*
- * cannot attach if :
- * - kernel task
- * - task not owned by caller
- * - task incompatible with context mode
- */
-static int
-pfm_task_incompatible(pfm_context_t *ctx, struct task_struct *task)
-{
- /*
- * no kernel task or task not owner by caller
- */
- if (task->mm == NULL) {
- DPRINT(("task [%d] has not memory context (kernel thread)\n", task_pid_nr(task)));
- return -EPERM;
- }
- if (pfm_bad_permissions(task)) {
- DPRINT(("no permission to attach to [%d]\n", task_pid_nr(task)));
- return -EPERM;
- }
- /*
- * cannot block in self-monitoring mode
- */
- if (CTX_OVFL_NOBLOCK(ctx) == 0 && task == current) {
- DPRINT(("cannot load a blocking context on self for [%d]\n", task_pid_nr(task)));
- return -EINVAL;
- }
-
- if (task->exit_state == EXIT_ZOMBIE) {
- DPRINT(("cannot attach to zombie task [%d]\n", task_pid_nr(task)));
- return -EBUSY;
- }
-
- /*
- * always ok for self
- */
- if (task == current) return 0;
-
- if (!task_is_stopped_or_traced(task)) {
- DPRINT(("cannot attach to non-stopped task [%d] state=%ld\n", task_pid_nr(task), task->state));
- return -EBUSY;
- }
- /*
- * make sure the task is off any CPU
- */
- wait_task_inactive(task, 0);
-
- /* more to come... */
-
- return 0;
-}
-
-static int
-pfm_get_task(pfm_context_t *ctx, pid_t pid, struct task_struct **task)
-{
- struct task_struct *p = current;
- int ret;
-
- /* XXX: need to add more checks here */
- if (pid < 2) return -EPERM;
-
- if (pid != task_pid_vnr(current)) {
- /* make sure task cannot go away while we operate on it */
- p = find_get_task_by_vpid(pid);
- if (!p)
- return -ESRCH;
- }
-
- ret = pfm_task_incompatible(ctx, p);
- if (ret == 0) {
- *task = p;
- } else if (p != current) {
- pfm_put_task(p);
- }
- return ret;
-}
-
-
-
-static int
-pfm_context_create(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- pfarg_context_t *req = (pfarg_context_t *)arg;
- struct file *filp;
- struct path path;
- int ctx_flags;
- int fd;
- int ret;
-
- /* let's check the arguments first */
- ret = pfarg_is_sane(current, req);
- if (ret < 0)
- return ret;
-
- ctx_flags = req->ctx_flags;
-
- ret = -ENOMEM;
-
- fd = get_unused_fd_flags(0);
- if (fd < 0)
- return fd;
-
- ctx = pfm_context_alloc(ctx_flags);
- if (!ctx)
- goto error;
-
- filp = pfm_alloc_file(ctx);
- if (IS_ERR(filp)) {
- ret = PTR_ERR(filp);
- goto error_file;
- }
-
- req->ctx_fd = ctx->ctx_fd = fd;
-
- /*
- * does the user want to sample?
- */
- if (pfm_uuid_cmp(req->ctx_smpl_buf_id, pfm_null_uuid)) {
- ret = pfm_setup_buffer_fmt(current, filp, ctx, ctx_flags, 0, req);
- if (ret)
- goto buffer_error;
- }
-
- DPRINT(("ctx=%p flags=0x%x system=%d notify_block=%d excl_idle=%d no_msg=%d ctx_fd=%d\n",
- ctx,
- ctx_flags,
- ctx->ctx_fl_system,
- ctx->ctx_fl_block,
- ctx->ctx_fl_excl_idle,
- ctx->ctx_fl_no_msg,
- ctx->ctx_fd));
-
- /*
- * initialize soft PMU state
- */
- pfm_reset_pmu_state(ctx);
-
- fd_install(fd, filp);
-
- return 0;
-
-buffer_error:
- path = filp->f_path;
- put_filp(filp);
- path_put(&path);
-
- if (ctx->ctx_buf_fmt) {
- pfm_buf_fmt_exit(ctx->ctx_buf_fmt, current, NULL, regs);
- }
-error_file:
- pfm_context_free(ctx);
-
-error:
- put_unused_fd(fd);
- return ret;
-}
-
-static inline unsigned long
-pfm_new_counter_value (pfm_counter_t *reg, int is_long_reset)
-{
- unsigned long val = is_long_reset ? reg->long_reset : reg->short_reset;
- unsigned long new_seed, old_seed = reg->seed, mask = reg->mask;
- extern unsigned long carta_random32 (unsigned long seed);
-
- if (reg->flags & PFM_REGFL_RANDOM) {
- new_seed = carta_random32(old_seed);
- val -= (old_seed & mask); /* counter values are negative numbers! */
- if ((mask >> 32) != 0)
- /* construct a full 64-bit random value: */
- new_seed |= carta_random32(old_seed >> 32) << 32;
- reg->seed = new_seed;
- }
- reg->lval = val;
- return val;
-}
-
-static void
-pfm_reset_regs_masked(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset)
-{
- unsigned long mask = ovfl_regs[0];
- unsigned long reset_others = 0UL;
- unsigned long val;
- int i;
-
- /*
- * now restore reset value on sampling overflowed counters
- */
- mask >>= PMU_FIRST_COUNTER;
- for(i = PMU_FIRST_COUNTER; mask; i++, mask >>= 1) {
-
- if ((mask & 0x1UL) == 0UL) continue;
-
- ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset);
- reset_others |= ctx->ctx_pmds[i].reset_pmds[0];
-
- DPRINT_ovfl((" %s reset ctx_pmds[%d]=%lx\n", is_long_reset ? "long" : "short", i, val));
- }
-
- /*
- * Now take care of resetting the other registers
- */
- for(i = 0; reset_others; i++, reset_others >>= 1) {
-
- if ((reset_others & 0x1) == 0) continue;
-
- ctx->ctx_pmds[i].val = val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset);
-
- DPRINT_ovfl(("%s reset_others pmd[%d]=%lx\n",
- is_long_reset ? "long" : "short", i, val));
- }
-}
-
-static void
-pfm_reset_regs(pfm_context_t *ctx, unsigned long *ovfl_regs, int is_long_reset)
-{
- unsigned long mask = ovfl_regs[0];
- unsigned long reset_others = 0UL;
- unsigned long val;
- int i;
-
- DPRINT_ovfl(("ovfl_regs=0x%lx is_long_reset=%d\n", ovfl_regs[0], is_long_reset));
-
- if (ctx->ctx_state == PFM_CTX_MASKED) {
- pfm_reset_regs_masked(ctx, ovfl_regs, is_long_reset);
- return;
- }
-
- /*
- * now restore reset value on sampling overflowed counters
- */
- mask >>= PMU_FIRST_COUNTER;
- for(i = PMU_FIRST_COUNTER; mask; i++, mask >>= 1) {
-
- if ((mask & 0x1UL) == 0UL) continue;
-
- val = pfm_new_counter_value(ctx->ctx_pmds+ i, is_long_reset);
- reset_others |= ctx->ctx_pmds[i].reset_pmds[0];
-
- DPRINT_ovfl((" %s reset ctx_pmds[%d]=%lx\n", is_long_reset ? "long" : "short", i, val));
-
- pfm_write_soft_counter(ctx, i, val);
- }
-
- /*
- * Now take care of resetting the other registers
- */
- for(i = 0; reset_others; i++, reset_others >>= 1) {
-
- if ((reset_others & 0x1) == 0) continue;
-
- val = pfm_new_counter_value(ctx->ctx_pmds + i, is_long_reset);
-
- if (PMD_IS_COUNTING(i)) {
- pfm_write_soft_counter(ctx, i, val);
- } else {
- ia64_set_pmd(i, val);
- }
- DPRINT_ovfl(("%s reset_others pmd[%d]=%lx\n",
- is_long_reset ? "long" : "short", i, val));
- }
- ia64_srlz_d();
-}
-
-static int
-pfm_write_pmcs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- struct task_struct *task;
- pfarg_reg_t *req = (pfarg_reg_t *)arg;
- unsigned long value, pmc_pm;
- unsigned long smpl_pmds, reset_pmds, impl_pmds;
- unsigned int cnum, reg_flags, flags, pmc_type;
- int i, can_access_pmu = 0, is_loaded, is_system, expert_mode;
- int is_monitor, is_counting, state;
- int ret = -EINVAL;
- pfm_reg_check_t wr_func;
-#define PFM_CHECK_PMC_PM(x, y, z) ((x)->ctx_fl_system ^ PMC_PM(y, z))
-
- state = ctx->ctx_state;
- is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
- is_system = ctx->ctx_fl_system;
- task = ctx->ctx_task;
- impl_pmds = pmu_conf->impl_pmds[0];
-
- if (state == PFM_CTX_ZOMBIE) return -EINVAL;
-
- if (is_loaded) {
- /*
- * In system wide and when the context is loaded, access can only happen
- * when the caller is running on the CPU being monitored by the session.
- * It does not have to be the owner (ctx_task) of the context per se.
- */
- if (is_system && ctx->ctx_cpu != smp_processor_id()) {
- DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
- return -EBUSY;
- }
- can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
- }
- expert_mode = pfm_sysctl.expert_mode;
-
- for (i = 0; i < count; i++, req++) {
-
- cnum = req->reg_num;
- reg_flags = req->reg_flags;
- value = req->reg_value;
- smpl_pmds = req->reg_smpl_pmds[0];
- reset_pmds = req->reg_reset_pmds[0];
- flags = 0;
-
-
- if (cnum >= PMU_MAX_PMCS) {
- DPRINT(("pmc%u is invalid\n", cnum));
- goto error;
- }
-
- pmc_type = pmu_conf->pmc_desc[cnum].type;
- pmc_pm = (value >> pmu_conf->pmc_desc[cnum].pm_pos) & 0x1;
- is_counting = (pmc_type & PFM_REG_COUNTING) == PFM_REG_COUNTING ? 1 : 0;
- is_monitor = (pmc_type & PFM_REG_MONITOR) == PFM_REG_MONITOR ? 1 : 0;
-
- /*
- * we reject all non implemented PMC as well
- * as attempts to modify PMC[0-3] which are used
- * as status registers by the PMU
- */
- if ((pmc_type & PFM_REG_IMPL) == 0 || (pmc_type & PFM_REG_CONTROL) == PFM_REG_CONTROL) {
- DPRINT(("pmc%u is unimplemented or no-access pmc_type=%x\n", cnum, pmc_type));
- goto error;
- }
- wr_func = pmu_conf->pmc_desc[cnum].write_check;
- /*
- * If the PMC is a monitor, then if the value is not the default:
- * - system-wide session: PMCx.pm=1 (privileged monitor)
- * - per-task : PMCx.pm=0 (user monitor)
- */
- if (is_monitor && value != PMC_DFL_VAL(cnum) && is_system ^ pmc_pm) {
- DPRINT(("pmc%u pmc_pm=%lu is_system=%d\n",
- cnum,
- pmc_pm,
- is_system));
- goto error;
- }
-
- if (is_counting) {
- /*
- * enforce generation of overflow interrupt. Necessary on all
- * CPUs.
- */
- value |= 1 << PMU_PMC_OI;
-
- if (reg_flags & PFM_REGFL_OVFL_NOTIFY) {
- flags |= PFM_REGFL_OVFL_NOTIFY;
- }
-
- if (reg_flags & PFM_REGFL_RANDOM) flags |= PFM_REGFL_RANDOM;
-
- /* verify validity of smpl_pmds */
- if ((smpl_pmds & impl_pmds) != smpl_pmds) {
- DPRINT(("invalid smpl_pmds 0x%lx for pmc%u\n", smpl_pmds, cnum));
- goto error;
- }
-
- /* verify validity of reset_pmds */
- if ((reset_pmds & impl_pmds) != reset_pmds) {
- DPRINT(("invalid reset_pmds 0x%lx for pmc%u\n", reset_pmds, cnum));
- goto error;
- }
- } else {
- if (reg_flags & (PFM_REGFL_OVFL_NOTIFY|PFM_REGFL_RANDOM)) {
- DPRINT(("cannot set ovfl_notify or random on pmc%u\n", cnum));
- goto error;
- }
- /* eventid on non-counting monitors are ignored */
- }
-
- /*
- * execute write checker, if any
- */
- if (likely(expert_mode == 0 && wr_func)) {
- ret = (*wr_func)(task, ctx, cnum, &value, regs);
- if (ret) goto error;
- ret = -EINVAL;
- }
-
- /*
- * no error on this register
- */
- PFM_REG_RETFLAG_SET(req->reg_flags, 0);
-
- /*
- * Now we commit the changes to the software state
- */
-
- /*
- * update overflow information
- */
- if (is_counting) {
- /*
- * full flag update each time a register is programmed
- */
- ctx->ctx_pmds[cnum].flags = flags;
-
- ctx->ctx_pmds[cnum].reset_pmds[0] = reset_pmds;
- ctx->ctx_pmds[cnum].smpl_pmds[0] = smpl_pmds;
- ctx->ctx_pmds[cnum].eventid = req->reg_smpl_eventid;
-
- /*
- * Mark all PMDS to be accessed as used.
- *
- * We do not keep track of PMC because we have to
- * systematically restore ALL of them.
- *
- * We do not update the used_monitors mask, because
- * if we have not programmed them, then will be in
- * a quiescent state, therefore we will not need to
- * mask/restore then when context is MASKED.
- */
- CTX_USED_PMD(ctx, reset_pmds);
- CTX_USED_PMD(ctx, smpl_pmds);
- /*
- * make sure we do not try to reset on
- * restart because we have established new values
- */
- if (state == PFM_CTX_MASKED) ctx->ctx_ovfl_regs[0] &= ~1UL << cnum;
- }
- /*
- * Needed in case the user does not initialize the equivalent
- * PMD. Clearing is done indirectly via pfm_reset_pmu_state() so there is no
- * possible leak here.
- */
- CTX_USED_PMD(ctx, pmu_conf->pmc_desc[cnum].dep_pmd[0]);
-
- /*
- * keep track of the monitor PMC that we are using.
- * we save the value of the pmc in ctx_pmcs[] and if
- * the monitoring is not stopped for the context we also
- * place it in the saved state area so that it will be
- * picked up later by the context switch code.
- *
- * The value in ctx_pmcs[] can only be changed in pfm_write_pmcs().
- *
- * The value in th_pmcs[] may be modified on overflow, i.e., when
- * monitoring needs to be stopped.
- */
- if (is_monitor) CTX_USED_MONITOR(ctx, 1UL << cnum);
-
- /*
- * update context state
- */
- ctx->ctx_pmcs[cnum] = value;
-
- if (is_loaded) {
- /*
- * write thread state
- */
- if (is_system == 0) ctx->th_pmcs[cnum] = value;
-
- /*
- * write hardware register if we can
- */
- if (can_access_pmu) {
- ia64_set_pmc(cnum, value);
- }
-#ifdef CONFIG_SMP
- else {
- /*
- * per-task SMP only here
- *
- * we are guaranteed that the task is not running on the other CPU,
- * we indicate that this PMD will need to be reloaded if the task
- * is rescheduled on the CPU it ran last on.
- */
- ctx->ctx_reload_pmcs[0] |= 1UL << cnum;
- }
-#endif
- }
-
- DPRINT(("pmc[%u]=0x%lx ld=%d apmu=%d flags=0x%x all_pmcs=0x%lx used_pmds=0x%lx eventid=%ld smpl_pmds=0x%lx reset_pmds=0x%lx reloads_pmcs=0x%lx used_monitors=0x%lx ovfl_regs=0x%lx\n",
- cnum,
- value,
- is_loaded,
- can_access_pmu,
- flags,
- ctx->ctx_all_pmcs[0],
- ctx->ctx_used_pmds[0],
- ctx->ctx_pmds[cnum].eventid,
- smpl_pmds,
- reset_pmds,
- ctx->ctx_reload_pmcs[0],
- ctx->ctx_used_monitors[0],
- ctx->ctx_ovfl_regs[0]));
- }
-
- /*
- * make sure the changes are visible
- */
- if (can_access_pmu) ia64_srlz_d();
-
- return 0;
-error:
- PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
- return ret;
-}
-
-static int
-pfm_write_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- struct task_struct *task;
- pfarg_reg_t *req = (pfarg_reg_t *)arg;
- unsigned long value, hw_value, ovfl_mask;
- unsigned int cnum;
- int i, can_access_pmu = 0, state;
- int is_counting, is_loaded, is_system, expert_mode;
- int ret = -EINVAL;
- pfm_reg_check_t wr_func;
-
-
- state = ctx->ctx_state;
- is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
- is_system = ctx->ctx_fl_system;
- ovfl_mask = pmu_conf->ovfl_val;
- task = ctx->ctx_task;
-
- if (unlikely(state == PFM_CTX_ZOMBIE)) return -EINVAL;
-
- /*
- * on both UP and SMP, we can only write to the PMC when the task is
- * the owner of the local PMU.
- */
- if (likely(is_loaded)) {
- /*
- * In system wide and when the context is loaded, access can only happen
- * when the caller is running on the CPU being monitored by the session.
- * It does not have to be the owner (ctx_task) of the context per se.
- */
- if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
- DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
- return -EBUSY;
- }
- can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
- }
- expert_mode = pfm_sysctl.expert_mode;
-
- for (i = 0; i < count; i++, req++) {
-
- cnum = req->reg_num;
- value = req->reg_value;
-
- if (!PMD_IS_IMPL(cnum)) {
- DPRINT(("pmd[%u] is unimplemented or invalid\n", cnum));
- goto abort_mission;
- }
- is_counting = PMD_IS_COUNTING(cnum);
- wr_func = pmu_conf->pmd_desc[cnum].write_check;
-
- /*
- * execute write checker, if any
- */
- if (unlikely(expert_mode == 0 && wr_func)) {
- unsigned long v = value;
-
- ret = (*wr_func)(task, ctx, cnum, &v, regs);
- if (ret) goto abort_mission;
-
- value = v;
- ret = -EINVAL;
- }
-
- /*
- * no error on this register
- */
- PFM_REG_RETFLAG_SET(req->reg_flags, 0);
-
- /*
- * now commit changes to software state
- */
- hw_value = value;
-
- /*
- * update virtualized (64bits) counter
- */
- if (is_counting) {
- /*
- * write context state
- */
- ctx->ctx_pmds[cnum].lval = value;
-
- /*
- * when context is load we use the split value
- */
- if (is_loaded) {
- hw_value = value & ovfl_mask;
- value = value & ~ovfl_mask;
- }
- }
- /*
- * update reset values (not just for counters)
- */
- ctx->ctx_pmds[cnum].long_reset = req->reg_long_reset;
- ctx->ctx_pmds[cnum].short_reset = req->reg_short_reset;
-
- /*
- * update randomization parameters (not just for counters)
- */
- ctx->ctx_pmds[cnum].seed = req->reg_random_seed;
- ctx->ctx_pmds[cnum].mask = req->reg_random_mask;
-
- /*
- * update context value
- */
- ctx->ctx_pmds[cnum].val = value;
-
- /*
- * Keep track of what we use
- *
- * We do not keep track of PMC because we have to
- * systematically restore ALL of them.
- */
- CTX_USED_PMD(ctx, PMD_PMD_DEP(cnum));
-
- /*
- * mark this PMD register used as well
- */
- CTX_USED_PMD(ctx, RDEP(cnum));
-
- /*
- * make sure we do not try to reset on
- * restart because we have established new values
- */
- if (is_counting && state == PFM_CTX_MASKED) {
- ctx->ctx_ovfl_regs[0] &= ~1UL << cnum;
- }
-
- if (is_loaded) {
- /*
- * write thread state
- */
- if (is_system == 0) ctx->th_pmds[cnum] = hw_value;
-
- /*
- * write hardware register if we can
- */
- if (can_access_pmu) {
- ia64_set_pmd(cnum, hw_value);
- } else {
-#ifdef CONFIG_SMP
- /*
- * we are guaranteed that the task is not running on the other CPU,
- * we indicate that this PMD will need to be reloaded if the task
- * is rescheduled on the CPU it ran last on.
- */
- ctx->ctx_reload_pmds[0] |= 1UL << cnum;
-#endif
- }
- }
-
- DPRINT(("pmd[%u]=0x%lx ld=%d apmu=%d, hw_value=0x%lx ctx_pmd=0x%lx short_reset=0x%lx "
- "long_reset=0x%lx notify=%c seed=0x%lx mask=0x%lx used_pmds=0x%lx reset_pmds=0x%lx reload_pmds=0x%lx all_pmds=0x%lx ovfl_regs=0x%lx\n",
- cnum,
- value,
- is_loaded,
- can_access_pmu,
- hw_value,
- ctx->ctx_pmds[cnum].val,
- ctx->ctx_pmds[cnum].short_reset,
- ctx->ctx_pmds[cnum].long_reset,
- PMC_OVFL_NOTIFY(ctx, cnum) ? 'Y':'N',
- ctx->ctx_pmds[cnum].seed,
- ctx->ctx_pmds[cnum].mask,
- ctx->ctx_used_pmds[0],
- ctx->ctx_pmds[cnum].reset_pmds[0],
- ctx->ctx_reload_pmds[0],
- ctx->ctx_all_pmds[0],
- ctx->ctx_ovfl_regs[0]));
- }
-
- /*
- * make changes visible
- */
- if (can_access_pmu) ia64_srlz_d();
-
- return 0;
-
-abort_mission:
- /*
- * for now, we have only one possibility for error
- */
- PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
- return ret;
-}
-
-/*
- * By the way of PROTECT_CONTEXT(), interrupts are masked while we are in this function.
- * Therefore we know, we do not have to worry about the PMU overflow interrupt. If an
- * interrupt is delivered during the call, it will be kept pending until we leave, making
- * it appears as if it had been generated at the UNPROTECT_CONTEXT(). At least we are
- * guaranteed to return consistent data to the user, it may simply be old. It is not
- * trivial to treat the overflow while inside the call because you may end up in
- * some module sampling buffer code causing deadlocks.
- */
-static int
-pfm_read_pmds(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- struct task_struct *task;
- unsigned long val = 0UL, lval, ovfl_mask, sval;
- pfarg_reg_t *req = (pfarg_reg_t *)arg;
- unsigned int cnum, reg_flags = 0;
- int i, can_access_pmu = 0, state;
- int is_loaded, is_system, is_counting, expert_mode;
- int ret = -EINVAL;
- pfm_reg_check_t rd_func;
-
- /*
- * access is possible when loaded only for
- * self-monitoring tasks or in UP mode
- */
-
- state = ctx->ctx_state;
- is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
- is_system = ctx->ctx_fl_system;
- ovfl_mask = pmu_conf->ovfl_val;
- task = ctx->ctx_task;
-
- if (state == PFM_CTX_ZOMBIE) return -EINVAL;
-
- if (likely(is_loaded)) {
- /*
- * In system wide and when the context is loaded, access can only happen
- * when the caller is running on the CPU being monitored by the session.
- * It does not have to be the owner (ctx_task) of the context per se.
- */
- if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
- DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
- return -EBUSY;
- }
- /*
- * this can be true when not self-monitoring only in UP
- */
- can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
-
- if (can_access_pmu) ia64_srlz_d();
- }
- expert_mode = pfm_sysctl.expert_mode;
-
- DPRINT(("ld=%d apmu=%d ctx_state=%d\n",
- is_loaded,
- can_access_pmu,
- state));
-
- /*
- * on both UP and SMP, we can only read the PMD from the hardware register when
- * the task is the owner of the local PMU.
- */
-
- for (i = 0; i < count; i++, req++) {
-
- cnum = req->reg_num;
- reg_flags = req->reg_flags;
-
- if (unlikely(!PMD_IS_IMPL(cnum))) goto error;
- /*
- * we can only read the register that we use. That includes
- * the one we explicitly initialize AND the one we want included
- * in the sampling buffer (smpl_regs).
- *
- * Having this restriction allows optimization in the ctxsw routine
- * without compromising security (leaks)
- */
- if (unlikely(!CTX_IS_USED_PMD(ctx, cnum))) goto error;
-
- sval = ctx->ctx_pmds[cnum].val;
- lval = ctx->ctx_pmds[cnum].lval;
- is_counting = PMD_IS_COUNTING(cnum);
-
- /*
- * If the task is not the current one, then we check if the
- * PMU state is still in the local live register due to lazy ctxsw.
- * If true, then we read directly from the registers.
- */
- if (can_access_pmu){
- val = ia64_get_pmd(cnum);
- } else {
- /*
- * context has been saved
- * if context is zombie, then task does not exist anymore.
- * In this case, we use the full value saved in the context (pfm_flush_regs()).
- */
- val = is_loaded ? ctx->th_pmds[cnum] : 0UL;
- }
- rd_func = pmu_conf->pmd_desc[cnum].read_check;
-
- if (is_counting) {
- /*
- * XXX: need to check for overflow when loaded
- */
- val &= ovfl_mask;
- val += sval;
- }
-
- /*
- * execute read checker, if any
- */
- if (unlikely(expert_mode == 0 && rd_func)) {
- unsigned long v = val;
- ret = (*rd_func)(ctx->ctx_task, ctx, cnum, &v, regs);
- if (ret) goto error;
- val = v;
- ret = -EINVAL;
- }
-
- PFM_REG_RETFLAG_SET(reg_flags, 0);
-
- DPRINT(("pmd[%u]=0x%lx\n", cnum, val));
-
- /*
- * update register return value, abort all if problem during copy.
- * we only modify the reg_flags field. no check mode is fine because
- * access has been verified upfront in sys_perfmonctl().
- */
- req->reg_value = val;
- req->reg_flags = reg_flags;
- req->reg_last_reset_val = lval;
- }
-
- return 0;
-
-error:
- PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
- return ret;
-}
-
-int
-pfm_mod_write_pmcs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
-{
- pfm_context_t *ctx;
-
- if (req == NULL) return -EINVAL;
-
- ctx = GET_PMU_CTX();
-
- if (ctx == NULL) return -EINVAL;
-
- /*
- * for now limit to current task, which is enough when calling
- * from overflow handler
- */
- if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
-
- return pfm_write_pmcs(ctx, req, nreq, regs);
-}
-EXPORT_SYMBOL(pfm_mod_write_pmcs);
-
-int
-pfm_mod_read_pmds(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
-{
- pfm_context_t *ctx;
-
- if (req == NULL) return -EINVAL;
-
- ctx = GET_PMU_CTX();
-
- if (ctx == NULL) return -EINVAL;
-
- /*
- * for now limit to current task, which is enough when calling
- * from overflow handler
- */
- if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
-
- return pfm_read_pmds(ctx, req, nreq, regs);
-}
-EXPORT_SYMBOL(pfm_mod_read_pmds);
-
-/*
- * Only call this function when a process it trying to
- * write the debug registers (reading is always allowed)
- */
-int
-pfm_use_debug_registers(struct task_struct *task)
-{
- pfm_context_t *ctx = task->thread.pfm_context;
- unsigned long flags;
- int ret = 0;
-
- if (pmu_conf->use_rr_dbregs == 0) return 0;
-
- DPRINT(("called for [%d]\n", task_pid_nr(task)));
-
- /*
- * do it only once
- */
- if (task->thread.flags & IA64_THREAD_DBG_VALID) return 0;
-
- /*
- * Even on SMP, we do not need to use an atomic here because
- * the only way in is via ptrace() and this is possible only when the
- * process is stopped. Even in the case where the ctxsw out is not totally
- * completed by the time we come here, there is no way the 'stopped' process
- * could be in the middle of fiddling with the pfm_write_ibr_dbr() routine.
- * So this is always safe.
- */
- if (ctx && ctx->ctx_fl_using_dbreg == 1) return -1;
-
- LOCK_PFS(flags);
-
- /*
- * We cannot allow setting breakpoints when system wide monitoring
- * sessions are using the debug registers.
- */
- if (pfm_sessions.pfs_sys_use_dbregs> 0)
- ret = -1;
- else
- pfm_sessions.pfs_ptrace_use_dbregs++;
-
- DPRINT(("ptrace_use_dbregs=%u sys_use_dbregs=%u by [%d] ret = %d\n",
- pfm_sessions.pfs_ptrace_use_dbregs,
- pfm_sessions.pfs_sys_use_dbregs,
- task_pid_nr(task), ret));
-
- UNLOCK_PFS(flags);
-
- return ret;
-}
-
-/*
- * This function is called for every task that exits with the
- * IA64_THREAD_DBG_VALID set. This indicates a task which was
- * able to use the debug registers for debugging purposes via
- * ptrace(). Therefore we know it was not using them for
- * performance monitoring, so we only decrement the number
- * of "ptraced" debug register users to keep the count up to date
- */
-int
-pfm_release_debug_registers(struct task_struct *task)
-{
- unsigned long flags;
- int ret;
-
- if (pmu_conf->use_rr_dbregs == 0) return 0;
-
- LOCK_PFS(flags);
- if (pfm_sessions.pfs_ptrace_use_dbregs == 0) {
- printk(KERN_ERR "perfmon: invalid release for [%d] ptrace_use_dbregs=0\n", task_pid_nr(task));
- ret = -1;
- } else {
- pfm_sessions.pfs_ptrace_use_dbregs--;
- ret = 0;
- }
- UNLOCK_PFS(flags);
-
- return ret;
-}
-
-static int
-pfm_restart(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- struct task_struct *task;
- pfm_buffer_fmt_t *fmt;
- pfm_ovfl_ctrl_t rst_ctrl;
- int state, is_system;
- int ret = 0;
-
- state = ctx->ctx_state;
- fmt = ctx->ctx_buf_fmt;
- is_system = ctx->ctx_fl_system;
- task = PFM_CTX_TASK(ctx);
-
- switch(state) {
- case PFM_CTX_MASKED:
- break;
- case PFM_CTX_LOADED:
- if (CTX_HAS_SMPL(ctx) && fmt->fmt_restart_active) break;
- fallthrough;
- case PFM_CTX_UNLOADED:
- case PFM_CTX_ZOMBIE:
- DPRINT(("invalid state=%d\n", state));
- return -EBUSY;
- default:
- DPRINT(("state=%d, cannot operate (no active_restart handler)\n", state));
- return -EINVAL;
- }
-
- /*
- * In system wide and when the context is loaded, access can only happen
- * when the caller is running on the CPU being monitored by the session.
- * It does not have to be the owner (ctx_task) of the context per se.
- */
- if (is_system && ctx->ctx_cpu != smp_processor_id()) {
- DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
- return -EBUSY;
- }
-
- /* sanity check */
- if (unlikely(task == NULL)) {
- printk(KERN_ERR "perfmon: [%d] pfm_restart no task\n", task_pid_nr(current));
- return -EINVAL;
- }
-
- if (task == current || is_system) {
-
- fmt = ctx->ctx_buf_fmt;
-
- DPRINT(("restarting self %d ovfl=0x%lx\n",
- task_pid_nr(task),
- ctx->ctx_ovfl_regs[0]));
-
- if (CTX_HAS_SMPL(ctx)) {
-
- prefetch(ctx->ctx_smpl_hdr);
-
- rst_ctrl.bits.mask_monitoring = 0;
- rst_ctrl.bits.reset_ovfl_pmds = 0;
-
- if (state == PFM_CTX_LOADED)
- ret = pfm_buf_fmt_restart_active(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
- else
- ret = pfm_buf_fmt_restart(fmt, task, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
- } else {
- rst_ctrl.bits.mask_monitoring = 0;
- rst_ctrl.bits.reset_ovfl_pmds = 1;
- }
-
- if (ret == 0) {
- if (rst_ctrl.bits.reset_ovfl_pmds)
- pfm_reset_regs(ctx, ctx->ctx_ovfl_regs, PFM_PMD_LONG_RESET);
-
- if (rst_ctrl.bits.mask_monitoring == 0) {
- DPRINT(("resuming monitoring for [%d]\n", task_pid_nr(task)));
-
- if (state == PFM_CTX_MASKED) pfm_restore_monitoring(task);
- } else {
- DPRINT(("keeping monitoring stopped for [%d]\n", task_pid_nr(task)));
-
- // cannot use pfm_stop_monitoring(task, regs);
- }
- }
- /*
- * clear overflowed PMD mask to remove any stale information
- */
- ctx->ctx_ovfl_regs[0] = 0UL;
-
- /*
- * back to LOADED state
- */
- ctx->ctx_state = PFM_CTX_LOADED;
-
- /*
- * XXX: not really useful for self monitoring
- */
- ctx->ctx_fl_can_restart = 0;
-
- return 0;
- }
-
- /*
- * restart another task
- */
-
- /*
- * When PFM_CTX_MASKED, we cannot issue a restart before the previous
- * one is seen by the task.
- */
- if (state == PFM_CTX_MASKED) {
- if (ctx->ctx_fl_can_restart == 0) return -EINVAL;
- /*
- * will prevent subsequent restart before this one is
- * seen by other task
- */
- ctx->ctx_fl_can_restart = 0;
- }
-
- /*
- * if blocking, then post the semaphore is PFM_CTX_MASKED, i.e.
- * the task is blocked or on its way to block. That's the normal
- * restart path. If the monitoring is not masked, then the task
- * can be actively monitoring and we cannot directly intervene.
- * Therefore we use the trap mechanism to catch the task and
- * force it to reset the buffer/reset PMDs.
- *
- * if non-blocking, then we ensure that the task will go into
- * pfm_handle_work() before returning to user mode.
- *
- * We cannot explicitly reset another task, it MUST always
- * be done by the task itself. This works for system wide because
- * the tool that is controlling the session is logically doing
- * "self-monitoring".
- */
- if (CTX_OVFL_NOBLOCK(ctx) == 0 && state == PFM_CTX_MASKED) {
- DPRINT(("unblocking [%d]\n", task_pid_nr(task)));
- complete(&ctx->ctx_restart_done);
- } else {
- DPRINT(("[%d] armed exit trap\n", task_pid_nr(task)));
-
- ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_RESET;
-
- PFM_SET_WORK_PENDING(task, 1);
-
- set_notify_resume(task);
-
- /*
- * XXX: send reschedule if task runs on another CPU
- */
- }
- return 0;
-}
-
-static int
-pfm_debug(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- unsigned int m = *(unsigned int *)arg;
-
- pfm_sysctl.debug = m == 0 ? 0 : 1;
-
- printk(KERN_INFO "perfmon debugging %s (timing reset)\n", pfm_sysctl.debug ? "on" : "off");
-
- if (m == 0) {
- memset(pfm_stats, 0, sizeof(pfm_stats));
- for(m=0; m < NR_CPUS; m++) pfm_stats[m].pfm_ovfl_intr_cycles_min = ~0UL;
- }
- return 0;
-}
-
-/*
- * arg can be NULL and count can be zero for this function
- */
-static int
-pfm_write_ibr_dbr(int mode, pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- struct thread_struct *thread = NULL;
- struct task_struct *task;
- pfarg_dbreg_t *req = (pfarg_dbreg_t *)arg;
- unsigned long flags;
- dbreg_t dbreg;
- unsigned int rnum;
- int first_time;
- int ret = 0, state;
- int i, can_access_pmu = 0;
- int is_system, is_loaded;
-
- if (pmu_conf->use_rr_dbregs == 0) return -EINVAL;
-
- state = ctx->ctx_state;
- is_loaded = state == PFM_CTX_LOADED ? 1 : 0;
- is_system = ctx->ctx_fl_system;
- task = ctx->ctx_task;
-
- if (state == PFM_CTX_ZOMBIE) return -EINVAL;
-
- /*
- * on both UP and SMP, we can only write to the PMC when the task is
- * the owner of the local PMU.
- */
- if (is_loaded) {
- thread = &task->thread;
- /*
- * In system wide and when the context is loaded, access can only happen
- * when the caller is running on the CPU being monitored by the session.
- * It does not have to be the owner (ctx_task) of the context per se.
- */
- if (unlikely(is_system && ctx->ctx_cpu != smp_processor_id())) {
- DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
- return -EBUSY;
- }
- can_access_pmu = GET_PMU_OWNER() == task || is_system ? 1 : 0;
- }
-
- /*
- * we do not need to check for ipsr.db because we do clear ibr.x, dbr.r, and dbr.w
- * ensuring that no real breakpoint can be installed via this call.
- *
- * IMPORTANT: regs can be NULL in this function
- */
-
- first_time = ctx->ctx_fl_using_dbreg == 0;
-
- /*
- * don't bother if we are loaded and task is being debugged
- */
- if (is_loaded && (thread->flags & IA64_THREAD_DBG_VALID) != 0) {
- DPRINT(("debug registers already in use for [%d]\n", task_pid_nr(task)));
- return -EBUSY;
- }
-
- /*
- * check for debug registers in system wide mode
- *
- * If though a check is done in pfm_context_load(),
- * we must repeat it here, in case the registers are
- * written after the context is loaded
- */
- if (is_loaded) {
- LOCK_PFS(flags);
-
- if (first_time && is_system) {
- if (pfm_sessions.pfs_ptrace_use_dbregs)
- ret = -EBUSY;
- else
- pfm_sessions.pfs_sys_use_dbregs++;
- }
- UNLOCK_PFS(flags);
- }
-
- if (ret != 0) return ret;
-
- /*
- * mark ourself as user of the debug registers for
- * perfmon purposes.
- */
- ctx->ctx_fl_using_dbreg = 1;
-
- /*
- * clear hardware registers to make sure we don't
- * pick up stale state.
- *
- * for a system wide session, we do not use
- * thread.dbr, thread.ibr because this process
- * never leaves the current CPU and the state
- * is shared by all processes running on it
- */
- if (first_time && can_access_pmu) {
- DPRINT(("[%d] clearing ibrs, dbrs\n", task_pid_nr(task)));
- for (i=0; i < pmu_conf->num_ibrs; i++) {
- ia64_set_ibr(i, 0UL);
- ia64_dv_serialize_instruction();
- }
- ia64_srlz_i();
- for (i=0; i < pmu_conf->num_dbrs; i++) {
- ia64_set_dbr(i, 0UL);
- ia64_dv_serialize_data();
- }
- ia64_srlz_d();
- }
-
- /*
- * Now install the values into the registers
- */
- for (i = 0; i < count; i++, req++) {
-
- rnum = req->dbreg_num;
- dbreg.val = req->dbreg_value;
-
- ret = -EINVAL;
-
- if ((mode == PFM_CODE_RR && rnum >= PFM_NUM_IBRS) || ((mode == PFM_DATA_RR) && rnum >= PFM_NUM_DBRS)) {
- DPRINT(("invalid register %u val=0x%lx mode=%d i=%d count=%d\n",
- rnum, dbreg.val, mode, i, count));
-
- goto abort_mission;
- }
-
- /*
- * make sure we do not install enabled breakpoint
- */
- if (rnum & 0x1) {
- if (mode == PFM_CODE_RR)
- dbreg.ibr.ibr_x = 0;
- else
- dbreg.dbr.dbr_r = dbreg.dbr.dbr_w = 0;
- }
-
- PFM_REG_RETFLAG_SET(req->dbreg_flags, 0);
-
- /*
- * Debug registers, just like PMC, can only be modified
- * by a kernel call. Moreover, perfmon() access to those
- * registers are centralized in this routine. The hardware
- * does not modify the value of these registers, therefore,
- * if we save them as they are written, we can avoid having
- * to save them on context switch out. This is made possible
- * by the fact that when perfmon uses debug registers, ptrace()
- * won't be able to modify them concurrently.
- */
- if (mode == PFM_CODE_RR) {
- CTX_USED_IBR(ctx, rnum);
-
- if (can_access_pmu) {
- ia64_set_ibr(rnum, dbreg.val);
- ia64_dv_serialize_instruction();
- }
-
- ctx->ctx_ibrs[rnum] = dbreg.val;
-
- DPRINT(("write ibr%u=0x%lx used_ibrs=0x%x ld=%d apmu=%d\n",
- rnum, dbreg.val, ctx->ctx_used_ibrs[0], is_loaded, can_access_pmu));
- } else {
- CTX_USED_DBR(ctx, rnum);
-
- if (can_access_pmu) {
- ia64_set_dbr(rnum, dbreg.val);
- ia64_dv_serialize_data();
- }
- ctx->ctx_dbrs[rnum] = dbreg.val;
-
- DPRINT(("write dbr%u=0x%lx used_dbrs=0x%x ld=%d apmu=%d\n",
- rnum, dbreg.val, ctx->ctx_used_dbrs[0], is_loaded, can_access_pmu));
- }
- }
-
- return 0;
-
-abort_mission:
- /*
- * in case it was our first attempt, we undo the global modifications
- */
- if (first_time) {
- LOCK_PFS(flags);
- if (ctx->ctx_fl_system) {
- pfm_sessions.pfs_sys_use_dbregs--;
- }
- UNLOCK_PFS(flags);
- ctx->ctx_fl_using_dbreg = 0;
- }
- /*
- * install error return flag
- */
- PFM_REG_RETFLAG_SET(req->dbreg_flags, PFM_REG_RETFL_EINVAL);
-
- return ret;
-}
-
-static int
-pfm_write_ibrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- return pfm_write_ibr_dbr(PFM_CODE_RR, ctx, arg, count, regs);
-}
-
-static int
-pfm_write_dbrs(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- return pfm_write_ibr_dbr(PFM_DATA_RR, ctx, arg, count, regs);
-}
-
-int
-pfm_mod_write_ibrs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
-{
- pfm_context_t *ctx;
-
- if (req == NULL) return -EINVAL;
-
- ctx = GET_PMU_CTX();
-
- if (ctx == NULL) return -EINVAL;
-
- /*
- * for now limit to current task, which is enough when calling
- * from overflow handler
- */
- if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
-
- return pfm_write_ibrs(ctx, req, nreq, regs);
-}
-EXPORT_SYMBOL(pfm_mod_write_ibrs);
-
-int
-pfm_mod_write_dbrs(struct task_struct *task, void *req, unsigned int nreq, struct pt_regs *regs)
-{
- pfm_context_t *ctx;
-
- if (req == NULL) return -EINVAL;
-
- ctx = GET_PMU_CTX();
-
- if (ctx == NULL) return -EINVAL;
-
- /*
- * for now limit to current task, which is enough when calling
- * from overflow handler
- */
- if (task != current && ctx->ctx_fl_system == 0) return -EBUSY;
-
- return pfm_write_dbrs(ctx, req, nreq, regs);
-}
-EXPORT_SYMBOL(pfm_mod_write_dbrs);
-
-
-static int
-pfm_get_features(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- pfarg_features_t *req = (pfarg_features_t *)arg;
-
- req->ft_version = PFM_VERSION;
- return 0;
-}
-
-static int
-pfm_stop(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- struct pt_regs *tregs;
- struct task_struct *task = PFM_CTX_TASK(ctx);
- int state, is_system;
-
- state = ctx->ctx_state;
- is_system = ctx->ctx_fl_system;
-
- /*
- * context must be attached to issue the stop command (includes LOADED,MASKED,ZOMBIE)
- */
- if (state == PFM_CTX_UNLOADED) return -EINVAL;
-
- /*
- * In system wide and when the context is loaded, access can only happen
- * when the caller is running on the CPU being monitored by the session.
- * It does not have to be the owner (ctx_task) of the context per se.
- */
- if (is_system && ctx->ctx_cpu != smp_processor_id()) {
- DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
- return -EBUSY;
- }
- DPRINT(("task [%d] ctx_state=%d is_system=%d\n",
- task_pid_nr(PFM_CTX_TASK(ctx)),
- state,
- is_system));
- /*
- * in system mode, we need to update the PMU directly
- * and the user level state of the caller, which may not
- * necessarily be the creator of the context.
- */
- if (is_system) {
- /*
- * Update local PMU first
- *
- * disable dcr pp
- */
- ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) & ~IA64_DCR_PP);
- ia64_srlz_i();
-
- /*
- * update local cpuinfo
- */
- PFM_CPUINFO_CLEAR(PFM_CPUINFO_DCR_PP);
-
- /*
- * stop monitoring, does srlz.i
- */
- pfm_clear_psr_pp();
-
- /*
- * stop monitoring in the caller
- */
- ia64_psr(regs)->pp = 0;
-
- return 0;
- }
- /*
- * per-task mode
- */
-
- if (task == current) {
- /* stop monitoring at kernel level */
- pfm_clear_psr_up();
-
- /*
- * stop monitoring at the user level
- */
- ia64_psr(regs)->up = 0;
- } else {
- tregs = task_pt_regs(task);
-
- /*
- * stop monitoring at the user level
- */
- ia64_psr(tregs)->up = 0;
-
- /*
- * monitoring disabled in kernel at next reschedule
- */
- ctx->ctx_saved_psr_up = 0;
- DPRINT(("task=[%d]\n", task_pid_nr(task)));
- }
- return 0;
-}
-
-
-static int
-pfm_start(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- struct pt_regs *tregs;
- int state, is_system;
-
- state = ctx->ctx_state;
- is_system = ctx->ctx_fl_system;
-
- if (state != PFM_CTX_LOADED) return -EINVAL;
-
- /*
- * In system wide and when the context is loaded, access can only happen
- * when the caller is running on the CPU being monitored by the session.
- * It does not have to be the owner (ctx_task) of the context per se.
- */
- if (is_system && ctx->ctx_cpu != smp_processor_id()) {
- DPRINT(("should be running on CPU%d\n", ctx->ctx_cpu));
- return -EBUSY;
- }
-
- /*
- * in system mode, we need to update the PMU directly
- * and the user level state of the caller, which may not
- * necessarily be the creator of the context.
- */
- if (is_system) {
-
- /*
- * set user level psr.pp for the caller
- */
- ia64_psr(regs)->pp = 1;
-
- /*
- * now update the local PMU and cpuinfo
- */
- PFM_CPUINFO_SET(PFM_CPUINFO_DCR_PP);
-
- /*
- * start monitoring at kernel level
- */
- pfm_set_psr_pp();
-
- /* enable dcr pp */
- ia64_setreg(_IA64_REG_CR_DCR, ia64_getreg(_IA64_REG_CR_DCR) | IA64_DCR_PP);
- ia64_srlz_i();
-
- return 0;
- }
-
- /*
- * per-process mode
- */
-
- if (ctx->ctx_task == current) {
-
- /* start monitoring at kernel level */
- pfm_set_psr_up();
-
- /*
- * activate monitoring at user level
- */
- ia64_psr(regs)->up = 1;
-
- } else {
- tregs = task_pt_regs(ctx->ctx_task);
-
- /*
- * start monitoring at the kernel level the next
- * time the task is scheduled
- */
- ctx->ctx_saved_psr_up = IA64_PSR_UP;
-
- /*
- * activate monitoring at user level
- */
- ia64_psr(tregs)->up = 1;
- }
- return 0;
-}
-
-static int
-pfm_get_pmc_reset(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- pfarg_reg_t *req = (pfarg_reg_t *)arg;
- unsigned int cnum;
- int i;
- int ret = -EINVAL;
-
- for (i = 0; i < count; i++, req++) {
-
- cnum = req->reg_num;
-
- if (!PMC_IS_IMPL(cnum)) goto abort_mission;
-
- req->reg_value = PMC_DFL_VAL(cnum);
-
- PFM_REG_RETFLAG_SET(req->reg_flags, 0);
-
- DPRINT(("pmc_reset_val pmc[%u]=0x%lx\n", cnum, req->reg_value));
- }
- return 0;
-
-abort_mission:
- PFM_REG_RETFLAG_SET(req->reg_flags, PFM_REG_RETFL_EINVAL);
- return ret;
-}
-
-static int
-pfm_check_task_exist(pfm_context_t *ctx)
-{
- struct task_struct *g, *t;
- int ret = -ESRCH;
-
- read_lock(&tasklist_lock);
-
- do_each_thread (g, t) {
- if (t->thread.pfm_context == ctx) {
- ret = 0;
- goto out;
- }
- } while_each_thread (g, t);
-out:
- read_unlock(&tasklist_lock);
-
- DPRINT(("pfm_check_task_exist: ret=%d ctx=%p\n", ret, ctx));
-
- return ret;
-}
-
-static int
-pfm_context_load(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- struct task_struct *task;
- struct thread_struct *thread;
- struct pfm_context_t *old;
- unsigned long flags;
-#ifndef CONFIG_SMP
- struct task_struct *owner_task = NULL;
-#endif
- pfarg_load_t *req = (pfarg_load_t *)arg;
- unsigned long *pmcs_source, *pmds_source;
- int the_cpu;
- int ret = 0;
- int state, is_system, set_dbregs = 0;
-
- state = ctx->ctx_state;
- is_system = ctx->ctx_fl_system;
- /*
- * can only load from unloaded or terminated state
- */
- if (state != PFM_CTX_UNLOADED) {
- DPRINT(("cannot load to [%d], invalid ctx_state=%d\n",
- req->load_pid,
- ctx->ctx_state));
- return -EBUSY;
- }
-
- DPRINT(("load_pid [%d] using_dbreg=%d\n", req->load_pid, ctx->ctx_fl_using_dbreg));
-
- if (CTX_OVFL_NOBLOCK(ctx) == 0 && req->load_pid == current->pid) {
- DPRINT(("cannot use blocking mode on self\n"));
- return -EINVAL;
- }
-
- ret = pfm_get_task(ctx, req->load_pid, &task);
- if (ret) {
- DPRINT(("load_pid [%d] get_task=%d\n", req->load_pid, ret));
- return ret;
- }
-
- ret = -EINVAL;
-
- /*
- * system wide is self monitoring only
- */
- if (is_system && task != current) {
- DPRINT(("system wide is self monitoring only load_pid=%d\n",
- req->load_pid));
- goto error;
- }
-
- thread = &task->thread;
-
- ret = 0;
- /*
- * cannot load a context which is using range restrictions,
- * into a task that is being debugged.
- */
- if (ctx->ctx_fl_using_dbreg) {
- if (thread->flags & IA64_THREAD_DBG_VALID) {
- ret = -EBUSY;
- DPRINT(("load_pid [%d] task is debugged, cannot load range restrictions\n", req->load_pid));
- goto error;
- }
- LOCK_PFS(flags);
-
- if (is_system) {
- if (pfm_sessions.pfs_ptrace_use_dbregs) {
- DPRINT(("cannot load [%d] dbregs in use\n",
- task_pid_nr(task)));
- ret = -EBUSY;
- } else {
- pfm_sessions.pfs_sys_use_dbregs++;
- DPRINT(("load [%d] increased sys_use_dbreg=%u\n", task_pid_nr(task), pfm_sessions.pfs_sys_use_dbregs));
- set_dbregs = 1;
- }
- }
-
- UNLOCK_PFS(flags);
-
- if (ret) goto error;
- }
-
- /*
- * SMP system-wide monitoring implies self-monitoring.
- *
- * The programming model expects the task to
- * be pinned on a CPU throughout the session.
- * Here we take note of the current CPU at the
- * time the context is loaded. No call from
- * another CPU will be allowed.
- *
- * The pinning via shed_setaffinity()
- * must be done by the calling task prior
- * to this call.
- *
- * systemwide: keep track of CPU this session is supposed to run on
- */
- the_cpu = ctx->ctx_cpu = smp_processor_id();
-
- ret = -EBUSY;
- /*
- * now reserve the session
- */
- ret = pfm_reserve_session(current, is_system, the_cpu);
- if (ret) goto error;
-
- /*
- * task is necessarily stopped at this point.
- *
- * If the previous context was zombie, then it got removed in
- * pfm_save_regs(). Therefore we should not see it here.
- * If we see a context, then this is an active context
- *
- * XXX: needs to be atomic
- */
- DPRINT(("before cmpxchg() old_ctx=%p new_ctx=%p\n",
- thread->pfm_context, ctx));
-
- ret = -EBUSY;
- old = ia64_cmpxchg(acq, &thread->pfm_context, NULL, ctx, sizeof(pfm_context_t *));
- if (old != NULL) {
- DPRINT(("load_pid [%d] already has a context\n", req->load_pid));
- goto error_unres;
- }
-
- pfm_reset_msgq(ctx);
-
- ctx->ctx_state = PFM_CTX_LOADED;
-
- /*
- * link context to task
- */
- ctx->ctx_task = task;
-
- if (is_system) {
- /*
- * we load as stopped
- */
- PFM_CPUINFO_SET(PFM_CPUINFO_SYST_WIDE);
- PFM_CPUINFO_CLEAR(PFM_CPUINFO_DCR_PP);
-
- if (ctx->ctx_fl_excl_idle) PFM_CPUINFO_SET(PFM_CPUINFO_EXCL_IDLE);
- } else {
- thread->flags |= IA64_THREAD_PM_VALID;
- }
-
- /*
- * propagate into thread-state
- */
- pfm_copy_pmds(task, ctx);
- pfm_copy_pmcs(task, ctx);
-
- pmcs_source = ctx->th_pmcs;
- pmds_source = ctx->th_pmds;
-
- /*
- * always the case for system-wide
- */
- if (task == current) {
-
- if (is_system == 0) {
-
- /* allow user level control */
- ia64_psr(regs)->sp = 0;
- DPRINT(("clearing psr.sp for [%d]\n", task_pid_nr(task)));
-
- SET_LAST_CPU(ctx, smp_processor_id());
- INC_ACTIVATION();
- SET_ACTIVATION(ctx);
-#ifndef CONFIG_SMP
- /*
- * push the other task out, if any
- */
- owner_task = GET_PMU_OWNER();
- if (owner_task) pfm_lazy_save_regs(owner_task);
-#endif
- }
- /*
- * load all PMD from ctx to PMU (as opposed to thread state)
- * restore all PMC from ctx to PMU
- */
- pfm_restore_pmds(pmds_source, ctx->ctx_all_pmds[0]);
- pfm_restore_pmcs(pmcs_source, ctx->ctx_all_pmcs[0]);
-
- ctx->ctx_reload_pmcs[0] = 0UL;
- ctx->ctx_reload_pmds[0] = 0UL;
-
- /*
- * guaranteed safe by earlier check against DBG_VALID
- */
- if (ctx->ctx_fl_using_dbreg) {
- pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
- pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
- }
- /*
- * set new ownership
- */
- SET_PMU_OWNER(task, ctx);
-
- DPRINT(("context loaded on PMU for [%d]\n", task_pid_nr(task)));
- } else {
- /*
- * when not current, task MUST be stopped, so this is safe
- */
- regs = task_pt_regs(task);
-
- /* force a full reload */
- ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
- SET_LAST_CPU(ctx, -1);
-
- /* initial saved psr (stopped) */
- ctx->ctx_saved_psr_up = 0UL;
- ia64_psr(regs)->up = ia64_psr(regs)->pp = 0;
- }
-
- ret = 0;
-
-error_unres:
- if (ret) pfm_unreserve_session(ctx, ctx->ctx_fl_system, the_cpu);
-error:
- /*
- * we must undo the dbregs setting (for system-wide)
- */
- if (ret && set_dbregs) {
- LOCK_PFS(flags);
- pfm_sessions.pfs_sys_use_dbregs--;
- UNLOCK_PFS(flags);
- }
- /*
- * release task, there is now a link with the context
- */
- if (is_system == 0 && task != current) {
- pfm_put_task(task);
-
- if (ret == 0) {
- ret = pfm_check_task_exist(ctx);
- if (ret) {
- ctx->ctx_state = PFM_CTX_UNLOADED;
- ctx->ctx_task = NULL;
- }
- }
- }
- return ret;
-}
-
-/*
- * in this function, we do not need to increase the use count
- * for the task via get_task_struct(), because we hold the
- * context lock. If the task were to disappear while having
- * a context attached, it would go through pfm_exit_thread()
- * which also grabs the context lock and would therefore be blocked
- * until we are here.
- */
-static void pfm_flush_pmds(struct task_struct *, pfm_context_t *ctx);
-
-static int
-pfm_context_unload(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs)
-{
- struct task_struct *task = PFM_CTX_TASK(ctx);
- struct pt_regs *tregs;
- int prev_state, is_system;
- int ret;
-
- DPRINT(("ctx_state=%d task [%d]\n", ctx->ctx_state, task ? task_pid_nr(task) : -1));
-
- prev_state = ctx->ctx_state;
- is_system = ctx->ctx_fl_system;
-
- /*
- * unload only when necessary
- */
- if (prev_state == PFM_CTX_UNLOADED) {
- DPRINT(("ctx_state=%d, nothing to do\n", prev_state));
- return 0;
- }
-
- /*
- * clear psr and dcr bits
- */
- ret = pfm_stop(ctx, NULL, 0, regs);
- if (ret) return ret;
-
- ctx->ctx_state = PFM_CTX_UNLOADED;
-
- /*
- * in system mode, we need to update the PMU directly
- * and the user level state of the caller, which may not
- * necessarily be the creator of the context.
- */
- if (is_system) {
-
- /*
- * Update cpuinfo
- *
- * local PMU is taken care of in pfm_stop()
- */
- PFM_CPUINFO_CLEAR(PFM_CPUINFO_SYST_WIDE);
- PFM_CPUINFO_CLEAR(PFM_CPUINFO_EXCL_IDLE);
-
- /*
- * save PMDs in context
- * release ownership
- */
- pfm_flush_pmds(current, ctx);
-
- /*
- * at this point we are done with the PMU
- * so we can unreserve the resource.
- */
- if (prev_state != PFM_CTX_ZOMBIE)
- pfm_unreserve_session(ctx, 1 , ctx->ctx_cpu);
-
- /*
- * disconnect context from task
- */
- task->thread.pfm_context = NULL;
- /*
- * disconnect task from context
- */
- ctx->ctx_task = NULL;
-
- /*
- * There is nothing more to cleanup here.
- */
- return 0;
- }
-
- /*
- * per-task mode
- */
- tregs = task == current ? regs : task_pt_regs(task);
-
- if (task == current) {
- /*
- * cancel user level control
- */
- ia64_psr(regs)->sp = 1;
-
- DPRINT(("setting psr.sp for [%d]\n", task_pid_nr(task)));
- }
- /*
- * save PMDs to context
- * release ownership
- */
- pfm_flush_pmds(task, ctx);
-
- /*
- * at this point we are done with the PMU
- * so we can unreserve the resource.
- *
- * when state was ZOMBIE, we have already unreserved.
- */
- if (prev_state != PFM_CTX_ZOMBIE)
- pfm_unreserve_session(ctx, 0 , ctx->ctx_cpu);
-
- /*
- * reset activation counter and psr
- */
- ctx->ctx_last_activation = PFM_INVALID_ACTIVATION;
- SET_LAST_CPU(ctx, -1);
-
- /*
- * PMU state will not be restored
- */
- task->thread.flags &= ~IA64_THREAD_PM_VALID;
-
- /*
- * break links between context and task
- */
- task->thread.pfm_context = NULL;
- ctx->ctx_task = NULL;
-
- PFM_SET_WORK_PENDING(task, 0);
-
- ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_NONE;
- ctx->ctx_fl_can_restart = 0;
- ctx->ctx_fl_going_zombie = 0;
-
- DPRINT(("disconnected [%d] from context\n", task_pid_nr(task)));
-
- return 0;
-}
-
-
-/*
- * called only from exit_thread()
- * we come here only if the task has a context attached (loaded or masked)
- */
-void
-pfm_exit_thread(struct task_struct *task)
-{
- pfm_context_t *ctx;
- unsigned long flags;
- struct pt_regs *regs = task_pt_regs(task);
- int ret, state;
- int free_ok = 0;
-
- ctx = PFM_GET_CTX(task);
-
- PROTECT_CTX(ctx, flags);
-
- DPRINT(("state=%d task [%d]\n", ctx->ctx_state, task_pid_nr(task)));
-
- state = ctx->ctx_state;
- switch(state) {
- case PFM_CTX_UNLOADED:
- /*
- * only comes to this function if pfm_context is not NULL, i.e., cannot
- * be in unloaded state
- */
- printk(KERN_ERR "perfmon: pfm_exit_thread [%d] ctx unloaded\n", task_pid_nr(task));
- break;
- case PFM_CTX_LOADED:
- case PFM_CTX_MASKED:
- ret = pfm_context_unload(ctx, NULL, 0, regs);
- if (ret) {
- printk(KERN_ERR "perfmon: pfm_exit_thread [%d] state=%d unload failed %d\n", task_pid_nr(task), state, ret);
- }
- DPRINT(("ctx unloaded for current state was %d\n", state));
-
- pfm_end_notify_user(ctx);
- break;
- case PFM_CTX_ZOMBIE:
- ret = pfm_context_unload(ctx, NULL, 0, regs);
- if (ret) {
- printk(KERN_ERR "perfmon: pfm_exit_thread [%d] state=%d unload failed %d\n", task_pid_nr(task), state, ret);
- }
- free_ok = 1;
- break;
- default:
- printk(KERN_ERR "perfmon: pfm_exit_thread [%d] unexpected state=%d\n", task_pid_nr(task), state);
- break;
- }
- UNPROTECT_CTX(ctx, flags);
-
- { u64 psr = pfm_get_psr();
- BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
- BUG_ON(GET_PMU_OWNER());
- BUG_ON(ia64_psr(regs)->up);
- BUG_ON(ia64_psr(regs)->pp);
- }
-
- /*
- * All memory free operations (especially for vmalloc'ed memory)
- * MUST be done with interrupts ENABLED.
- */
- if (free_ok) pfm_context_free(ctx);
-}
-
-/*
- * functions MUST be listed in the increasing order of their index (see permfon.h)
- */
-#define PFM_CMD(name, flags, arg_count, arg_type, getsz) { name, #name, flags, arg_count, sizeof(arg_type), getsz }
-#define PFM_CMD_S(name, flags) { name, #name, flags, 0, 0, NULL }
-#define PFM_CMD_PCLRWS (PFM_CMD_FD|PFM_CMD_ARG_RW|PFM_CMD_STOP)
-#define PFM_CMD_PCLRW (PFM_CMD_FD|PFM_CMD_ARG_RW)
-#define PFM_CMD_NONE { NULL, "no-cmd", 0, 0, 0, NULL}
-
-static pfm_cmd_desc_t pfm_cmd_tab[]={
-/* 0 */PFM_CMD_NONE,
-/* 1 */PFM_CMD(pfm_write_pmcs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
-/* 2 */PFM_CMD(pfm_write_pmds, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
-/* 3 */PFM_CMD(pfm_read_pmds, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
-/* 4 */PFM_CMD_S(pfm_stop, PFM_CMD_PCLRWS),
-/* 5 */PFM_CMD_S(pfm_start, PFM_CMD_PCLRWS),
-/* 6 */PFM_CMD_NONE,
-/* 7 */PFM_CMD_NONE,
-/* 8 */PFM_CMD(pfm_context_create, PFM_CMD_ARG_RW, 1, pfarg_context_t, pfm_ctx_getsize),
-/* 9 */PFM_CMD_NONE,
-/* 10 */PFM_CMD_S(pfm_restart, PFM_CMD_PCLRW),
-/* 11 */PFM_CMD_NONE,
-/* 12 */PFM_CMD(pfm_get_features, PFM_CMD_ARG_RW, 1, pfarg_features_t, NULL),
-/* 13 */PFM_CMD(pfm_debug, 0, 1, unsigned int, NULL),
-/* 14 */PFM_CMD_NONE,
-/* 15 */PFM_CMD(pfm_get_pmc_reset, PFM_CMD_ARG_RW, PFM_CMD_ARG_MANY, pfarg_reg_t, NULL),
-/* 16 */PFM_CMD(pfm_context_load, PFM_CMD_PCLRWS, 1, pfarg_load_t, NULL),
-/* 17 */PFM_CMD_S(pfm_context_unload, PFM_CMD_PCLRWS),
-/* 18 */PFM_CMD_NONE,
-/* 19 */PFM_CMD_NONE,
-/* 20 */PFM_CMD_NONE,
-/* 21 */PFM_CMD_NONE,
-/* 22 */PFM_CMD_NONE,
-/* 23 */PFM_CMD_NONE,
-/* 24 */PFM_CMD_NONE,
-/* 25 */PFM_CMD_NONE,
-/* 26 */PFM_CMD_NONE,
-/* 27 */PFM_CMD_NONE,
-/* 28 */PFM_CMD_NONE,
-/* 29 */PFM_CMD_NONE,
-/* 30 */PFM_CMD_NONE,
-/* 31 */PFM_CMD_NONE,
-/* 32 */PFM_CMD(pfm_write_ibrs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_dbreg_t, NULL),
-/* 33 */PFM_CMD(pfm_write_dbrs, PFM_CMD_PCLRWS, PFM_CMD_ARG_MANY, pfarg_dbreg_t, NULL)
-};
-#define PFM_CMD_COUNT (sizeof(pfm_cmd_tab)/sizeof(pfm_cmd_desc_t))
-
-static int
-pfm_check_task_state(pfm_context_t *ctx, int cmd, unsigned long flags)
-{
- struct task_struct *task;
- int state, old_state;
-
-recheck:
- state = ctx->ctx_state;
- task = ctx->ctx_task;
-
- if (task == NULL) {
- DPRINT(("context %d no task, state=%d\n", ctx->ctx_fd, state));
- return 0;
- }
-
- DPRINT(("context %d state=%d [%d] task_state=%ld must_stop=%d\n",
- ctx->ctx_fd,
- state,
- task_pid_nr(task),
- task->state, PFM_CMD_STOPPED(cmd)));
-
- /*
- * self-monitoring always ok.
- *
- * for system-wide the caller can either be the creator of the
- * context (to one to which the context is attached to) OR
- * a task running on the same CPU as the session.
- */
- if (task == current || ctx->ctx_fl_system) return 0;
-
- /*
- * we are monitoring another thread
- */
- switch(state) {
- case PFM_CTX_UNLOADED:
- /*
- * if context is UNLOADED we are safe to go
- */
- return 0;
- case PFM_CTX_ZOMBIE:
- /*
- * no command can operate on a zombie context
- */
- DPRINT(("cmd %d state zombie cannot operate on context\n", cmd));
- return -EINVAL;
- case PFM_CTX_MASKED:
- /*
- * PMU state has been saved to software even though
- * the thread may still be running.
- */
- if (cmd != PFM_UNLOAD_CONTEXT) return 0;
- }
-
- /*
- * context is LOADED or MASKED. Some commands may need to have
- * the task stopped.
- *
- * We could lift this restriction for UP but it would mean that
- * the user has no guarantee the task would not run between
- * two successive calls to perfmonctl(). That's probably OK.
- * If this user wants to ensure the task does not run, then
- * the task must be stopped.
- */
- if (PFM_CMD_STOPPED(cmd)) {
- if (!task_is_stopped_or_traced(task)) {
- DPRINT(("[%d] task not in stopped state\n", task_pid_nr(task)));
- return -EBUSY;
- }
- /*
- * task is now stopped, wait for ctxsw out
- *
- * This is an interesting point in the code.
- * We need to unprotect the context because
- * the pfm_save_regs() routines needs to grab
- * the same lock. There are danger in doing
- * this because it leaves a window open for
- * another task to get access to the context
- * and possibly change its state. The one thing
- * that is not possible is for the context to disappear
- * because we are protected by the VFS layer, i.e.,
- * get_fd()/put_fd().
- */
- old_state = state;
-
- UNPROTECT_CTX(ctx, flags);
-
- wait_task_inactive(task, 0);
-
- PROTECT_CTX(ctx, flags);
-
- /*
- * we must recheck to verify if state has changed
- */
- if (ctx->ctx_state != old_state) {
- DPRINT(("old_state=%d new_state=%d\n", old_state, ctx->ctx_state));
- goto recheck;
- }
- }
- return 0;
-}
-
-/*
- * system-call entry point (must return long)
- */
-asmlinkage long
-sys_perfmonctl (int fd, int cmd, void __user *arg, int count)
-{
- struct fd f = {NULL, 0};
- pfm_context_t *ctx = NULL;
- unsigned long flags = 0UL;
- void *args_k = NULL;
- long ret; /* will expand int return types */
- size_t base_sz, sz, xtra_sz = 0;
- int narg, completed_args = 0, call_made = 0, cmd_flags;
- int (*func)(pfm_context_t *ctx, void *arg, int count, struct pt_regs *regs);
- int (*getsize)(void *arg, size_t *sz);
-#define PFM_MAX_ARGSIZE 4096
-
- /*
- * reject any call if perfmon was disabled at initialization
- */
- if (unlikely(pmu_conf == NULL)) return -ENOSYS;
-
- if (unlikely(cmd < 0 || cmd >= PFM_CMD_COUNT)) {
- DPRINT(("invalid cmd=%d\n", cmd));
- return -EINVAL;
- }
-
- func = pfm_cmd_tab[cmd].cmd_func;
- narg = pfm_cmd_tab[cmd].cmd_narg;
- base_sz = pfm_cmd_tab[cmd].cmd_argsize;
- getsize = pfm_cmd_tab[cmd].cmd_getsize;
- cmd_flags = pfm_cmd_tab[cmd].cmd_flags;
-
- if (unlikely(func == NULL)) {
- DPRINT(("invalid cmd=%d\n", cmd));
- return -EINVAL;
- }
-
- DPRINT(("cmd=%s idx=%d narg=0x%x argsz=%lu count=%d\n",
- PFM_CMD_NAME(cmd),
- cmd,
- narg,
- base_sz,
- count));
-
- /*
- * check if number of arguments matches what the command expects
- */
- if (unlikely((narg == PFM_CMD_ARG_MANY && count <= 0) || (narg > 0 && narg != count)))
- return -EINVAL;
-
-restart_args:
- sz = xtra_sz + base_sz*count;
- /*
- * limit abuse to min page size
- */
- if (unlikely(sz > PFM_MAX_ARGSIZE)) {
- printk(KERN_ERR "perfmon: [%d] argument too big %lu\n", task_pid_nr(current), sz);
- return -E2BIG;
- }
-
- /*
- * allocate default-sized argument buffer
- */
- if (likely(count && args_k == NULL)) {
- args_k = kmalloc(PFM_MAX_ARGSIZE, GFP_KERNEL);
- if (args_k == NULL) return -ENOMEM;
- }
-
- ret = -EFAULT;
-
- /*
- * copy arguments
- *
- * assume sz = 0 for command without parameters
- */
- if (sz && copy_from_user(args_k, arg, sz)) {
- DPRINT(("cannot copy_from_user %lu bytes @%p\n", sz, arg));
- goto error_args;
- }
-
- /*
- * check if command supports extra parameters
- */
- if (completed_args == 0 && getsize) {
- /*
- * get extra parameters size (based on main argument)
- */
- ret = (*getsize)(args_k, &xtra_sz);
- if (ret) goto error_args;
-
- completed_args = 1;
-
- DPRINT(("restart_args sz=%lu xtra_sz=%lu\n", sz, xtra_sz));
-
- /* retry if necessary */
- if (likely(xtra_sz)) goto restart_args;
- }
-
- if (unlikely((cmd_flags & PFM_CMD_FD) == 0)) goto skip_fd;
-
- ret = -EBADF;
-
- f = fdget(fd);
- if (unlikely(f.file == NULL)) {
- DPRINT(("invalid fd %d\n", fd));
- goto error_args;
- }
- if (unlikely(PFM_IS_FILE(f.file) == 0)) {
- DPRINT(("fd %d not related to perfmon\n", fd));
- goto error_args;
- }
-
- ctx = f.file->private_data;
- if (unlikely(ctx == NULL)) {
- DPRINT(("no context for fd %d\n", fd));
- goto error_args;
- }
- prefetch(&ctx->ctx_state);
-
- PROTECT_CTX(ctx, flags);
-
- /*
- * check task is stopped
- */
- ret = pfm_check_task_state(ctx, cmd, flags);
- if (unlikely(ret)) goto abort_locked;
-
-skip_fd:
- ret = (*func)(ctx, args_k, count, task_pt_regs(current));
-
- call_made = 1;
-
-abort_locked:
- if (likely(ctx)) {
- DPRINT(("context unlocked\n"));
- UNPROTECT_CTX(ctx, flags);
- }
-
- /* copy argument back to user, if needed */
- if (call_made && PFM_CMD_RW_ARG(cmd) && copy_to_user(arg, args_k, base_sz*count)) ret = -EFAULT;
-
-error_args:
- if (f.file)
- fdput(f);
-
- kfree(args_k);
-
- DPRINT(("cmd=%s ret=%ld\n", PFM_CMD_NAME(cmd), ret));
-
- return ret;
-}
-
-static void
-pfm_resume_after_ovfl(pfm_context_t *ctx, unsigned long ovfl_regs, struct pt_regs *regs)
-{
- pfm_buffer_fmt_t *fmt = ctx->ctx_buf_fmt;
- pfm_ovfl_ctrl_t rst_ctrl;
- int state;
- int ret = 0;
-
- state = ctx->ctx_state;
- /*
- * Unlock sampling buffer and reset index atomically
- * XXX: not really needed when blocking
- */
- if (CTX_HAS_SMPL(ctx)) {
-
- rst_ctrl.bits.mask_monitoring = 0;
- rst_ctrl.bits.reset_ovfl_pmds = 0;
-
- if (state == PFM_CTX_LOADED)
- ret = pfm_buf_fmt_restart_active(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
- else
- ret = pfm_buf_fmt_restart(fmt, current, &rst_ctrl, ctx->ctx_smpl_hdr, regs);
- } else {
- rst_ctrl.bits.mask_monitoring = 0;
- rst_ctrl.bits.reset_ovfl_pmds = 1;
- }
-
- if (ret == 0) {
- if (rst_ctrl.bits.reset_ovfl_pmds) {
- pfm_reset_regs(ctx, &ovfl_regs, PFM_PMD_LONG_RESET);
- }
- if (rst_ctrl.bits.mask_monitoring == 0) {
- DPRINT(("resuming monitoring\n"));
- if (ctx->ctx_state == PFM_CTX_MASKED) pfm_restore_monitoring(current);
- } else {
- DPRINT(("stopping monitoring\n"));
- //pfm_stop_monitoring(current, regs);
- }
- ctx->ctx_state = PFM_CTX_LOADED;
- }
-}
-
-/*
- * context MUST BE LOCKED when calling
- * can only be called for current
- */
-static void
-pfm_context_force_terminate(pfm_context_t *ctx, struct pt_regs *regs)
-{
- int ret;
-
- DPRINT(("entering for [%d]\n", task_pid_nr(current)));
-
- ret = pfm_context_unload(ctx, NULL, 0, regs);
- if (ret) {
- printk(KERN_ERR "pfm_context_force_terminate: [%d] unloaded failed with %d\n", task_pid_nr(current), ret);
- }
-
- /*
- * and wakeup controlling task, indicating we are now disconnected
- */
- wake_up_interruptible(&ctx->ctx_zombieq);
-
- /*
- * given that context is still locked, the controlling
- * task will only get access when we return from
- * pfm_handle_work().
- */
-}
-
-static int pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds);
-
- /*
- * pfm_handle_work() can be called with interrupts enabled
- * (TIF_NEED_RESCHED) or disabled. The down_interruptible
- * call may sleep, therefore we must re-enable interrupts
- * to avoid deadlocks. It is safe to do so because this function
- * is called ONLY when returning to user level (pUStk=1), in which case
- * there is no risk of kernel stack overflow due to deep
- * interrupt nesting.
- */
-void
-pfm_handle_work(void)
-{
- pfm_context_t *ctx;
- struct pt_regs *regs;
- unsigned long flags, dummy_flags;
- unsigned long ovfl_regs;
- unsigned int reason;
- int ret;
-
- ctx = PFM_GET_CTX(current);
- if (ctx == NULL) {
- printk(KERN_ERR "perfmon: [%d] has no PFM context\n",
- task_pid_nr(current));
- return;
- }
-
- PROTECT_CTX(ctx, flags);
-
- PFM_SET_WORK_PENDING(current, 0);
-
- regs = task_pt_regs(current);
-
- /*
- * extract reason for being here and clear
- */
- reason = ctx->ctx_fl_trap_reason;
- ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_NONE;
- ovfl_regs = ctx->ctx_ovfl_regs[0];
-
- DPRINT(("reason=%d state=%d\n", reason, ctx->ctx_state));
-
- /*
- * must be done before we check for simple-reset mode
- */
- if (ctx->ctx_fl_going_zombie || ctx->ctx_state == PFM_CTX_ZOMBIE)
- goto do_zombie;
-
- //if (CTX_OVFL_NOBLOCK(ctx)) goto skip_blocking;
- if (reason == PFM_TRAP_REASON_RESET)
- goto skip_blocking;
-
- /*
- * restore interrupt mask to what it was on entry.
- * Could be enabled/diasbled.
- */
- UNPROTECT_CTX(ctx, flags);
-
- /*
- * force interrupt enable because of down_interruptible()
- */
- local_irq_enable();
-
- DPRINT(("before block sleeping\n"));
-
- /*
- * may go through without blocking on SMP systems
- * if restart has been received already by the time we call down()
- */
- ret = wait_for_completion_interruptible(&ctx->ctx_restart_done);
-
- DPRINT(("after block sleeping ret=%d\n", ret));
-
- /*
- * lock context and mask interrupts again
- * We save flags into a dummy because we may have
- * altered interrupts mask compared to entry in this
- * function.
- */
- PROTECT_CTX(ctx, dummy_flags);
-
- /*
- * we need to read the ovfl_regs only after wake-up
- * because we may have had pfm_write_pmds() in between
- * and that can changed PMD values and therefore
- * ovfl_regs is reset for these new PMD values.
- */
- ovfl_regs = ctx->ctx_ovfl_regs[0];
-
- if (ctx->ctx_fl_going_zombie) {
-do_zombie:
- DPRINT(("context is zombie, bailing out\n"));
- pfm_context_force_terminate(ctx, regs);
- goto nothing_to_do;
- }
- /*
- * in case of interruption of down() we don't restart anything
- */
- if (ret < 0)
- goto nothing_to_do;
-
-skip_blocking:
- pfm_resume_after_ovfl(ctx, ovfl_regs, regs);
- ctx->ctx_ovfl_regs[0] = 0UL;
-
-nothing_to_do:
- /*
- * restore flags as they were upon entry
- */
- UNPROTECT_CTX(ctx, flags);
-}
-
-static int
-pfm_notify_user(pfm_context_t *ctx, pfm_msg_t *msg)
-{
- if (ctx->ctx_state == PFM_CTX_ZOMBIE) {
- DPRINT(("ignoring overflow notification, owner is zombie\n"));
- return 0;
- }
-
- DPRINT(("waking up somebody\n"));
-
- if (msg) wake_up_interruptible(&ctx->ctx_msgq_wait);
-
- /*
- * safe, we are not in intr handler, nor in ctxsw when
- * we come here
- */
- kill_fasync (&ctx->ctx_async_queue, SIGIO, POLL_IN);
-
- return 0;
-}
-
-static int
-pfm_ovfl_notify_user(pfm_context_t *ctx, unsigned long ovfl_pmds)
-{
- pfm_msg_t *msg = NULL;
-
- if (ctx->ctx_fl_no_msg == 0) {
- msg = pfm_get_new_msg(ctx);
- if (msg == NULL) {
- printk(KERN_ERR "perfmon: pfm_ovfl_notify_user no more notification msgs\n");
- return -1;
- }
-
- msg->pfm_ovfl_msg.msg_type = PFM_MSG_OVFL;
- msg->pfm_ovfl_msg.msg_ctx_fd = ctx->ctx_fd;
- msg->pfm_ovfl_msg.msg_active_set = 0;
- msg->pfm_ovfl_msg.msg_ovfl_pmds[0] = ovfl_pmds;
- msg->pfm_ovfl_msg.msg_ovfl_pmds[1] = 0UL;
- msg->pfm_ovfl_msg.msg_ovfl_pmds[2] = 0UL;
- msg->pfm_ovfl_msg.msg_ovfl_pmds[3] = 0UL;
- msg->pfm_ovfl_msg.msg_tstamp = 0UL;
- }
-
- DPRINT(("ovfl msg: msg=%p no_msg=%d fd=%d ovfl_pmds=0x%lx\n",
- msg,
- ctx->ctx_fl_no_msg,
- ctx->ctx_fd,
- ovfl_pmds));
-
- return pfm_notify_user(ctx, msg);
-}
-
-static int
-pfm_end_notify_user(pfm_context_t *ctx)
-{
- pfm_msg_t *msg;
-
- msg = pfm_get_new_msg(ctx);
- if (msg == NULL) {
- printk(KERN_ERR "perfmon: pfm_end_notify_user no more notification msgs\n");
- return -1;
- }
- /* no leak */
- memset(msg, 0, sizeof(*msg));
-
- msg->pfm_end_msg.msg_type = PFM_MSG_END;
- msg->pfm_end_msg.msg_ctx_fd = ctx->ctx_fd;
- msg->pfm_ovfl_msg.msg_tstamp = 0UL;
-
- DPRINT(("end msg: msg=%p no_msg=%d ctx_fd=%d\n",
- msg,
- ctx->ctx_fl_no_msg,
- ctx->ctx_fd));
-
- return pfm_notify_user(ctx, msg);
-}
-
-/*
- * main overflow processing routine.
- * it can be called from the interrupt path or explicitly during the context switch code
- */
-static void pfm_overflow_handler(struct task_struct *task, pfm_context_t *ctx,
- unsigned long pmc0, struct pt_regs *regs)
-{
- pfm_ovfl_arg_t *ovfl_arg;
- unsigned long mask;
- unsigned long old_val, ovfl_val, new_val;
- unsigned long ovfl_notify = 0UL, ovfl_pmds = 0UL, smpl_pmds = 0UL, reset_pmds;
- unsigned long tstamp;
- pfm_ovfl_ctrl_t ovfl_ctrl;
- unsigned int i, has_smpl;
- int must_notify = 0;
-
- if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) goto stop_monitoring;
-
- /*
- * sanity test. Should never happen
- */
- if (unlikely((pmc0 & 0x1) == 0)) goto sanity_check;
-
- tstamp = ia64_get_itc();
- mask = pmc0 >> PMU_FIRST_COUNTER;
- ovfl_val = pmu_conf->ovfl_val;
- has_smpl = CTX_HAS_SMPL(ctx);
-
- DPRINT_ovfl(("pmc0=0x%lx pid=%d iip=0x%lx, %s "
- "used_pmds=0x%lx\n",
- pmc0,
- task ? task_pid_nr(task): -1,
- (regs ? regs->cr_iip : 0),
- CTX_OVFL_NOBLOCK(ctx) ? "nonblocking" : "blocking",
- ctx->ctx_used_pmds[0]));
-
-
- /*
- * first we update the virtual counters
- * assume there was a prior ia64_srlz_d() issued
- */
- for (i = PMU_FIRST_COUNTER; mask ; i++, mask >>= 1) {
-
- /* skip pmd which did not overflow */
- if ((mask & 0x1) == 0) continue;
-
- /*
- * Note that the pmd is not necessarily 0 at this point as qualified events
- * may have happened before the PMU was frozen. The residual count is not
- * taken into consideration here but will be with any read of the pmd via
- * pfm_read_pmds().
- */
- old_val = new_val = ctx->ctx_pmds[i].val;
- new_val += 1 + ovfl_val;
- ctx->ctx_pmds[i].val = new_val;
-
- /*
- * check for overflow condition
- */
- if (likely(old_val > new_val)) {
- ovfl_pmds |= 1UL << i;
- if (PMC_OVFL_NOTIFY(ctx, i)) ovfl_notify |= 1UL << i;
- }
-
- DPRINT_ovfl(("ctx_pmd[%d].val=0x%lx old_val=0x%lx pmd=0x%lx ovfl_pmds=0x%lx ovfl_notify=0x%lx\n",
- i,
- new_val,
- old_val,
- ia64_get_pmd(i) & ovfl_val,
- ovfl_pmds,
- ovfl_notify));
- }
-
- /*
- * there was no 64-bit overflow, nothing else to do
- */
- if (ovfl_pmds == 0UL) return;
-
- /*
- * reset all control bits
- */
- ovfl_ctrl.val = 0;
- reset_pmds = 0UL;
-
- /*
- * if a sampling format module exists, then we "cache" the overflow by
- * calling the module's handler() routine.
- */
- if (has_smpl) {
- unsigned long start_cycles, end_cycles;
- unsigned long pmd_mask;
- int j, k, ret = 0;
- int this_cpu = smp_processor_id();
-
- pmd_mask = ovfl_pmds >> PMU_FIRST_COUNTER;
- ovfl_arg = &ctx->ctx_ovfl_arg;
-
- prefetch(ctx->ctx_smpl_hdr);
-
- for(i=PMU_FIRST_COUNTER; pmd_mask && ret == 0; i++, pmd_mask >>=1) {
-
- mask = 1UL << i;
-
- if ((pmd_mask & 0x1) == 0) continue;
-
- ovfl_arg->ovfl_pmd = (unsigned char )i;
- ovfl_arg->ovfl_notify = ovfl_notify & mask ? 1 : 0;
- ovfl_arg->active_set = 0;
- ovfl_arg->ovfl_ctrl.val = 0; /* module must fill in all fields */
- ovfl_arg->smpl_pmds[0] = smpl_pmds = ctx->ctx_pmds[i].smpl_pmds[0];
-
- ovfl_arg->pmd_value = ctx->ctx_pmds[i].val;
- ovfl_arg->pmd_last_reset = ctx->ctx_pmds[i].lval;
- ovfl_arg->pmd_eventid = ctx->ctx_pmds[i].eventid;
-
- /*
- * copy values of pmds of interest. Sampling format may copy them
- * into sampling buffer.
- */
- if (smpl_pmds) {
- for(j=0, k=0; smpl_pmds; j++, smpl_pmds >>=1) {
- if ((smpl_pmds & 0x1) == 0) continue;
- ovfl_arg->smpl_pmds_values[k++] = PMD_IS_COUNTING(j) ? pfm_read_soft_counter(ctx, j) : ia64_get_pmd(j);
- DPRINT_ovfl(("smpl_pmd[%d]=pmd%u=0x%lx\n", k-1, j, ovfl_arg->smpl_pmds_values[k-1]));
- }
- }
-
- pfm_stats[this_cpu].pfm_smpl_handler_calls++;
-
- start_cycles = ia64_get_itc();
-
- /*
- * call custom buffer format record (handler) routine
- */
- ret = (*ctx->ctx_buf_fmt->fmt_handler)(task, ctx->ctx_smpl_hdr, ovfl_arg, regs, tstamp);
-
- end_cycles = ia64_get_itc();
-
- /*
- * For those controls, we take the union because they have
- * an all or nothing behavior.
- */
- ovfl_ctrl.bits.notify_user |= ovfl_arg->ovfl_ctrl.bits.notify_user;
- ovfl_ctrl.bits.block_task |= ovfl_arg->ovfl_ctrl.bits.block_task;
- ovfl_ctrl.bits.mask_monitoring |= ovfl_arg->ovfl_ctrl.bits.mask_monitoring;
- /*
- * build the bitmask of pmds to reset now
- */
- if (ovfl_arg->ovfl_ctrl.bits.reset_ovfl_pmds) reset_pmds |= mask;
-
- pfm_stats[this_cpu].pfm_smpl_handler_cycles += end_cycles - start_cycles;
- }
- /*
- * when the module cannot handle the rest of the overflows, we abort right here
- */
- if (ret && pmd_mask) {
- DPRINT(("handler aborts leftover ovfl_pmds=0x%lx\n",
- pmd_mask<<PMU_FIRST_COUNTER));
- }
- /*
- * remove the pmds we reset now from the set of pmds to reset in pfm_restart()
- */
- ovfl_pmds &= ~reset_pmds;
- } else {
- /*
- * when no sampling module is used, then the default
- * is to notify on overflow if requested by user
- */
- ovfl_ctrl.bits.notify_user = ovfl_notify ? 1 : 0;
- ovfl_ctrl.bits.block_task = ovfl_notify ? 1 : 0;
- ovfl_ctrl.bits.mask_monitoring = ovfl_notify ? 1 : 0; /* XXX: change for saturation */
- ovfl_ctrl.bits.reset_ovfl_pmds = ovfl_notify ? 0 : 1;
- /*
- * if needed, we reset all overflowed pmds
- */
- if (ovfl_notify == 0) reset_pmds = ovfl_pmds;
- }
-
- DPRINT_ovfl(("ovfl_pmds=0x%lx reset_pmds=0x%lx\n", ovfl_pmds, reset_pmds));
-
- /*
- * reset the requested PMD registers using the short reset values
- */
- if (reset_pmds) {
- unsigned long bm = reset_pmds;
- pfm_reset_regs(ctx, &bm, PFM_PMD_SHORT_RESET);
- }
-
- if (ovfl_notify && ovfl_ctrl.bits.notify_user) {
- /*
- * keep track of what to reset when unblocking
- */
- ctx->ctx_ovfl_regs[0] = ovfl_pmds;
-
- /*
- * check for blocking context
- */
- if (CTX_OVFL_NOBLOCK(ctx) == 0 && ovfl_ctrl.bits.block_task) {
-
- ctx->ctx_fl_trap_reason = PFM_TRAP_REASON_BLOCK;
-
- /*
- * set the perfmon specific checking pending work for the task
- */
- PFM_SET_WORK_PENDING(task, 1);
-
- /*
- * when coming from ctxsw, current still points to the
- * previous task, therefore we must work with task and not current.
- */
- set_notify_resume(task);
- }
- /*
- * defer until state is changed (shorten spin window). the context is locked
- * anyway, so the signal receiver would come spin for nothing.
- */
- must_notify = 1;
- }
-
- DPRINT_ovfl(("owner [%d] pending=%ld reason=%u ovfl_pmds=0x%lx ovfl_notify=0x%lx masked=%d\n",
- GET_PMU_OWNER() ? task_pid_nr(GET_PMU_OWNER()) : -1,
- PFM_GET_WORK_PENDING(task),
- ctx->ctx_fl_trap_reason,
- ovfl_pmds,
- ovfl_notify,
- ovfl_ctrl.bits.mask_monitoring ? 1 : 0));
- /*
- * in case monitoring must be stopped, we toggle the psr bits
- */
- if (ovfl_ctrl.bits.mask_monitoring) {
- pfm_mask_monitoring(task);
- ctx->ctx_state = PFM_CTX_MASKED;
- ctx->ctx_fl_can_restart = 1;
- }
-
- /*
- * send notification now
- */
- if (must_notify) pfm_ovfl_notify_user(ctx, ovfl_notify);
-
- return;
-
-sanity_check:
- printk(KERN_ERR "perfmon: CPU%d overflow handler [%d] pmc0=0x%lx\n",
- smp_processor_id(),
- task ? task_pid_nr(task) : -1,
- pmc0);
- return;
-
-stop_monitoring:
- /*
- * in SMP, zombie context is never restored but reclaimed in pfm_load_regs().
- * Moreover, zombies are also reclaimed in pfm_save_regs(). Therefore we can
- * come here as zombie only if the task is the current task. In which case, we
- * can access the PMU hardware directly.
- *
- * Note that zombies do have PM_VALID set. So here we do the minimal.
- *
- * In case the context was zombified it could not be reclaimed at the time
- * the monitoring program exited. At this point, the PMU reservation has been
- * returned, the sampiing buffer has been freed. We must convert this call
- * into a spurious interrupt. However, we must also avoid infinite overflows
- * by stopping monitoring for this task. We can only come here for a per-task
- * context. All we need to do is to stop monitoring using the psr bits which
- * are always task private. By re-enabling secure montioring, we ensure that
- * the monitored task will not be able to re-activate monitoring.
- * The task will eventually be context switched out, at which point the context
- * will be reclaimed (that includes releasing ownership of the PMU).
- *
- * So there might be a window of time where the number of per-task session is zero
- * yet one PMU might have a owner and get at most one overflow interrupt for a zombie
- * context. This is safe because if a per-task session comes in, it will push this one
- * out and by the virtue on pfm_save_regs(), this one will disappear. If a system wide
- * session is force on that CPU, given that we use task pinning, pfm_save_regs() will
- * also push our zombie context out.
- *
- * Overall pretty hairy stuff....
- */
- DPRINT(("ctx is zombie for [%d], converted to spurious\n", task ? task_pid_nr(task): -1));
- pfm_clear_psr_up();
- ia64_psr(regs)->up = 0;
- ia64_psr(regs)->sp = 1;
- return;
-}
-
-static int
-pfm_do_interrupt_handler(void *arg, struct pt_regs *regs)
-{
- struct task_struct *task;
- pfm_context_t *ctx;
- unsigned long flags;
- u64 pmc0;
- int this_cpu = smp_processor_id();
- int retval = 0;
-
- pfm_stats[this_cpu].pfm_ovfl_intr_count++;
-
- /*
- * srlz.d done before arriving here
- */
- pmc0 = ia64_get_pmc(0);
-
- task = GET_PMU_OWNER();
- ctx = GET_PMU_CTX();
-
- /*
- * if we have some pending bits set
- * assumes : if any PMC0.bit[63-1] is set, then PMC0.fr = 1
- */
- if (PMC0_HAS_OVFL(pmc0) && task) {
- /*
- * we assume that pmc0.fr is always set here
- */
-
- /* sanity check */
- if (!ctx) goto report_spurious1;
-
- if (ctx->ctx_fl_system == 0 && (task->thread.flags & IA64_THREAD_PM_VALID) == 0)
- goto report_spurious2;
-
- PROTECT_CTX_NOPRINT(ctx, flags);
-
- pfm_overflow_handler(task, ctx, pmc0, regs);
-
- UNPROTECT_CTX_NOPRINT(ctx, flags);
-
- } else {
- pfm_stats[this_cpu].pfm_spurious_ovfl_intr_count++;
- retval = -1;
- }
- /*
- * keep it unfrozen at all times
- */
- pfm_unfreeze_pmu();
-
- return retval;
-
-report_spurious1:
- printk(KERN_INFO "perfmon: spurious overflow interrupt on CPU%d: process %d has no PFM context\n",
- this_cpu, task_pid_nr(task));
- pfm_unfreeze_pmu();
- return -1;
-report_spurious2:
- printk(KERN_INFO "perfmon: spurious overflow interrupt on CPU%d: process %d, invalid flag\n",
- this_cpu,
- task_pid_nr(task));
- pfm_unfreeze_pmu();
- return -1;
-}
-
-static irqreturn_t
-pfm_interrupt_handler(int irq, void *arg)
-{
- unsigned long start_cycles, total_cycles;
- unsigned long min, max;
- int this_cpu;
- int ret;
- struct pt_regs *regs = get_irq_regs();
-
- this_cpu = get_cpu();
- if (likely(!pfm_alt_intr_handler)) {
- min = pfm_stats[this_cpu].pfm_ovfl_intr_cycles_min;
- max = pfm_stats[this_cpu].pfm_ovfl_intr_cycles_max;
-
- start_cycles = ia64_get_itc();
-
- ret = pfm_do_interrupt_handler(arg, regs);
-
- total_cycles = ia64_get_itc();
-
- /*
- * don't measure spurious interrupts
- */
- if (likely(ret == 0)) {
- total_cycles -= start_cycles;
-
- if (total_cycles < min) pfm_stats[this_cpu].pfm_ovfl_intr_cycles_min = total_cycles;
- if (total_cycles > max) pfm_stats[this_cpu].pfm_ovfl_intr_cycles_max = total_cycles;
-
- pfm_stats[this_cpu].pfm_ovfl_intr_cycles += total_cycles;
- }
- }
- else {
- (*pfm_alt_intr_handler->handler)(irq, arg, regs);
- }
-
- put_cpu();
- return IRQ_HANDLED;
-}
-
-/*
- * /proc/perfmon interface, for debug only
- */
-
-#define PFM_PROC_SHOW_HEADER ((void *)(long)nr_cpu_ids+1)
-
-static void *
-pfm_proc_start(struct seq_file *m, loff_t *pos)
-{
- if (*pos == 0) {
- return PFM_PROC_SHOW_HEADER;
- }
-
- while (*pos <= nr_cpu_ids) {
- if (cpu_online(*pos - 1)) {
- return (void *)*pos;
- }
- ++*pos;
- }
- return NULL;
-}
-
-static void *
-pfm_proc_next(struct seq_file *m, void *v, loff_t *pos)
-{
- ++*pos;
- return pfm_proc_start(m, pos);
-}
-
-static void
-pfm_proc_stop(struct seq_file *m, void *v)
-{
-}
-
-static void
-pfm_proc_show_header(struct seq_file *m)
-{
- struct list_head * pos;
- pfm_buffer_fmt_t * entry;
- unsigned long flags;
-
- seq_printf(m,
- "perfmon version : %u.%u\n"
- "model : %s\n"
- "fastctxsw : %s\n"
- "expert mode : %s\n"
- "ovfl_mask : 0x%lx\n"
- "PMU flags : 0x%x\n",
- PFM_VERSION_MAJ, PFM_VERSION_MIN,
- pmu_conf->pmu_name,
- pfm_sysctl.fastctxsw > 0 ? "Yes": "No",
- pfm_sysctl.expert_mode > 0 ? "Yes": "No",
- pmu_conf->ovfl_val,
- pmu_conf->flags);
-
- LOCK_PFS(flags);
-
- seq_printf(m,
- "proc_sessions : %u\n"
- "sys_sessions : %u\n"
- "sys_use_dbregs : %u\n"
- "ptrace_use_dbregs : %u\n",
- pfm_sessions.pfs_task_sessions,
- pfm_sessions.pfs_sys_sessions,
- pfm_sessions.pfs_sys_use_dbregs,
- pfm_sessions.pfs_ptrace_use_dbregs);
-
- UNLOCK_PFS(flags);
-
- spin_lock(&pfm_buffer_fmt_lock);
-
- list_for_each(pos, &pfm_buffer_fmt_list) {
- entry = list_entry(pos, pfm_buffer_fmt_t, fmt_list);
- seq_printf(m, "format : %16phD %s\n",
- entry->fmt_uuid, entry->fmt_name);
- }
- spin_unlock(&pfm_buffer_fmt_lock);
-
-}
-
-static int
-pfm_proc_show(struct seq_file *m, void *v)
-{
- unsigned long psr;
- unsigned int i;
- int cpu;
-
- if (v == PFM_PROC_SHOW_HEADER) {
- pfm_proc_show_header(m);
- return 0;
- }
-
- /* show info for CPU (v - 1) */
-
- cpu = (long)v - 1;
- seq_printf(m,
- "CPU%-2d overflow intrs : %lu\n"
- "CPU%-2d overflow cycles : %lu\n"
- "CPU%-2d overflow min : %lu\n"
- "CPU%-2d overflow max : %lu\n"
- "CPU%-2d smpl handler calls : %lu\n"
- "CPU%-2d smpl handler cycles : %lu\n"
- "CPU%-2d spurious intrs : %lu\n"
- "CPU%-2d replay intrs : %lu\n"
- "CPU%-2d syst_wide : %d\n"
- "CPU%-2d dcr_pp : %d\n"
- "CPU%-2d exclude idle : %d\n"
- "CPU%-2d owner : %d\n"
- "CPU%-2d context : %p\n"
- "CPU%-2d activations : %lu\n",
- cpu, pfm_stats[cpu].pfm_ovfl_intr_count,
- cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles,
- cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles_min,
- cpu, pfm_stats[cpu].pfm_ovfl_intr_cycles_max,
- cpu, pfm_stats[cpu].pfm_smpl_handler_calls,
- cpu, pfm_stats[cpu].pfm_smpl_handler_cycles,
- cpu, pfm_stats[cpu].pfm_spurious_ovfl_intr_count,
- cpu, pfm_stats[cpu].pfm_replay_ovfl_intr_count,
- cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_SYST_WIDE ? 1 : 0,
- cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_DCR_PP ? 1 : 0,
- cpu, pfm_get_cpu_data(pfm_syst_info, cpu) & PFM_CPUINFO_EXCL_IDLE ? 1 : 0,
- cpu, pfm_get_cpu_data(pmu_owner, cpu) ? pfm_get_cpu_data(pmu_owner, cpu)->pid: -1,
- cpu, pfm_get_cpu_data(pmu_ctx, cpu),
- cpu, pfm_get_cpu_data(pmu_activation_number, cpu));
-
- if (num_online_cpus() == 1 && pfm_sysctl.debug > 0) {
-
- psr = pfm_get_psr();
-
- ia64_srlz_d();
-
- seq_printf(m,
- "CPU%-2d psr : 0x%lx\n"
- "CPU%-2d pmc0 : 0x%lx\n",
- cpu, psr,
- cpu, ia64_get_pmc(0));
-
- for (i=0; PMC_IS_LAST(i) == 0; i++) {
- if (PMC_IS_COUNTING(i) == 0) continue;
- seq_printf(m,
- "CPU%-2d pmc%u : 0x%lx\n"
- "CPU%-2d pmd%u : 0x%lx\n",
- cpu, i, ia64_get_pmc(i),
- cpu, i, ia64_get_pmd(i));
- }
- }
- return 0;
-}
-
-const struct seq_operations pfm_seq_ops = {
- .start = pfm_proc_start,
- .next = pfm_proc_next,
- .stop = pfm_proc_stop,
- .show = pfm_proc_show
-};
-
-/*
- * we come here as soon as local_cpu_data->pfm_syst_wide is set. this happens
- * during pfm_enable() hence before pfm_start(). We cannot assume monitoring
- * is active or inactive based on mode. We must rely on the value in
- * local_cpu_data->pfm_syst_info
- */
-void
-pfm_syst_wide_update_task(struct task_struct *task, unsigned long info, int is_ctxswin)
-{
- struct pt_regs *regs;
- unsigned long dcr;
- unsigned long dcr_pp;
-
- dcr_pp = info & PFM_CPUINFO_DCR_PP ? 1 : 0;
-
- /*
- * pid 0 is guaranteed to be the idle task. There is one such task with pid 0
- * on every CPU, so we can rely on the pid to identify the idle task.
- */
- if ((info & PFM_CPUINFO_EXCL_IDLE) == 0 || task->pid) {
- regs = task_pt_regs(task);
- ia64_psr(regs)->pp = is_ctxswin ? dcr_pp : 0;
- return;
- }
- /*
- * if monitoring has started
- */
- if (dcr_pp) {
- dcr = ia64_getreg(_IA64_REG_CR_DCR);
- /*
- * context switching in?
- */
- if (is_ctxswin) {
- /* mask monitoring for the idle task */
- ia64_setreg(_IA64_REG_CR_DCR, dcr & ~IA64_DCR_PP);
- pfm_clear_psr_pp();
- ia64_srlz_i();
- return;
- }
- /*
- * context switching out
- * restore monitoring for next task
- *
- * Due to inlining this odd if-then-else construction generates
- * better code.
- */
- ia64_setreg(_IA64_REG_CR_DCR, dcr |IA64_DCR_PP);
- pfm_set_psr_pp();
- ia64_srlz_i();
- }
-}
-
-#ifdef CONFIG_SMP
-
-static void
-pfm_force_cleanup(pfm_context_t *ctx, struct pt_regs *regs)
-{
- struct task_struct *task = ctx->ctx_task;
-
- ia64_psr(regs)->up = 0;
- ia64_psr(regs)->sp = 1;
-
- if (GET_PMU_OWNER() == task) {
- DPRINT(("cleared ownership for [%d]\n",
- task_pid_nr(ctx->ctx_task)));
- SET_PMU_OWNER(NULL, NULL);
- }
-
- /*
- * disconnect the task from the context and vice-versa
- */
- PFM_SET_WORK_PENDING(task, 0);
-
- task->thread.pfm_context = NULL;
- task->thread.flags &= ~IA64_THREAD_PM_VALID;
-
- DPRINT(("force cleanup for [%d]\n", task_pid_nr(task)));
-}
-
-
-/*
- * in 2.6, interrupts are masked when we come here and the runqueue lock is held
- */
-void
-pfm_save_regs(struct task_struct *task)
-{
- pfm_context_t *ctx;
- unsigned long flags;
- u64 psr;
-
-
- ctx = PFM_GET_CTX(task);
- if (ctx == NULL) return;
-
- /*
- * we always come here with interrupts ALREADY disabled by
- * the scheduler. So we simply need to protect against concurrent
- * access, not CPU concurrency.
- */
- flags = pfm_protect_ctx_ctxsw(ctx);
-
- if (ctx->ctx_state == PFM_CTX_ZOMBIE) {
- struct pt_regs *regs = task_pt_regs(task);
-
- pfm_clear_psr_up();
-
- pfm_force_cleanup(ctx, regs);
-
- BUG_ON(ctx->ctx_smpl_hdr);
-
- pfm_unprotect_ctx_ctxsw(ctx, flags);
-
- pfm_context_free(ctx);
- return;
- }
-
- /*
- * save current PSR: needed because we modify it
- */
- ia64_srlz_d();
- psr = pfm_get_psr();
-
- BUG_ON(psr & (IA64_PSR_I));
-
- /*
- * stop monitoring:
- * This is the last instruction which may generate an overflow
- *
- * We do not need to set psr.sp because, it is irrelevant in kernel.
- * It will be restored from ipsr when going back to user level
- */
- pfm_clear_psr_up();
-
- /*
- * keep a copy of psr.up (for reload)
- */
- ctx->ctx_saved_psr_up = psr & IA64_PSR_UP;
-
- /*
- * release ownership of this PMU.
- * PM interrupts are masked, so nothing
- * can happen.
- */
- SET_PMU_OWNER(NULL, NULL);
-
- /*
- * we systematically save the PMD as we have no
- * guarantee we will be schedule at that same
- * CPU again.
- */
- pfm_save_pmds(ctx->th_pmds, ctx->ctx_used_pmds[0]);
-
- /*
- * save pmc0 ia64_srlz_d() done in pfm_save_pmds()
- * we will need it on the restore path to check
- * for pending overflow.
- */
- ctx->th_pmcs[0] = ia64_get_pmc(0);
-
- /*
- * unfreeze PMU if had pending overflows
- */
- if (ctx->th_pmcs[0] & ~0x1UL) pfm_unfreeze_pmu();
-
- /*
- * finally, allow context access.
- * interrupts will still be masked after this call.
- */
- pfm_unprotect_ctx_ctxsw(ctx, flags);
-}
-
-#else /* !CONFIG_SMP */
-void
-pfm_save_regs(struct task_struct *task)
-{
- pfm_context_t *ctx;
- u64 psr;
-
- ctx = PFM_GET_CTX(task);
- if (ctx == NULL) return;
-
- /*
- * save current PSR: needed because we modify it
- */
- psr = pfm_get_psr();
-
- BUG_ON(psr & (IA64_PSR_I));
-
- /*
- * stop monitoring:
- * This is the last instruction which may generate an overflow
- *
- * We do not need to set psr.sp because, it is irrelevant in kernel.
- * It will be restored from ipsr when going back to user level
- */
- pfm_clear_psr_up();
-
- /*
- * keep a copy of psr.up (for reload)
- */
- ctx->ctx_saved_psr_up = psr & IA64_PSR_UP;
-}
-
-static void
-pfm_lazy_save_regs (struct task_struct *task)
-{
- pfm_context_t *ctx;
- unsigned long flags;
-
- { u64 psr = pfm_get_psr();
- BUG_ON(psr & IA64_PSR_UP);
- }
-
- ctx = PFM_GET_CTX(task);
-
- /*
- * we need to mask PMU overflow here to
- * make sure that we maintain pmc0 until
- * we save it. overflow interrupts are
- * treated as spurious if there is no
- * owner.
- *
- * XXX: I don't think this is necessary
- */
- PROTECT_CTX(ctx,flags);
-
- /*
- * release ownership of this PMU.
- * must be done before we save the registers.
- *
- * after this call any PMU interrupt is treated
- * as spurious.
- */
- SET_PMU_OWNER(NULL, NULL);
-
- /*
- * save all the pmds we use
- */
- pfm_save_pmds(ctx->th_pmds, ctx->ctx_used_pmds[0]);
-
- /*
- * save pmc0 ia64_srlz_d() done in pfm_save_pmds()
- * it is needed to check for pended overflow
- * on the restore path
- */
- ctx->th_pmcs[0] = ia64_get_pmc(0);
-
- /*
- * unfreeze PMU if had pending overflows
- */
- if (ctx->th_pmcs[0] & ~0x1UL) pfm_unfreeze_pmu();
-
- /*
- * now get can unmask PMU interrupts, they will
- * be treated as purely spurious and we will not
- * lose any information
- */
- UNPROTECT_CTX(ctx,flags);
-}
-#endif /* CONFIG_SMP */
-
-#ifdef CONFIG_SMP
-/*
- * in 2.6, interrupts are masked when we come here and the runqueue lock is held
- */
-void
-pfm_load_regs (struct task_struct *task)
-{
- pfm_context_t *ctx;
- unsigned long pmc_mask = 0UL, pmd_mask = 0UL;
- unsigned long flags;
- u64 psr, psr_up;
- int need_irq_resend;
-
- ctx = PFM_GET_CTX(task);
- if (unlikely(ctx == NULL)) return;
-
- BUG_ON(GET_PMU_OWNER());
-
- /*
- * possible on unload
- */
- if (unlikely((task->thread.flags & IA64_THREAD_PM_VALID) == 0)) return;
-
- /*
- * we always come here with interrupts ALREADY disabled by
- * the scheduler. So we simply need to protect against concurrent
- * access, not CPU concurrency.
- */
- flags = pfm_protect_ctx_ctxsw(ctx);
- psr = pfm_get_psr();
-
- need_irq_resend = pmu_conf->flags & PFM_PMU_IRQ_RESEND;
-
- BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
- BUG_ON(psr & IA64_PSR_I);
-
- if (unlikely(ctx->ctx_state == PFM_CTX_ZOMBIE)) {
- struct pt_regs *regs = task_pt_regs(task);
-
- BUG_ON(ctx->ctx_smpl_hdr);
-
- pfm_force_cleanup(ctx, regs);
-
- pfm_unprotect_ctx_ctxsw(ctx, flags);
-
- /*
- * this one (kmalloc'ed) is fine with interrupts disabled
- */
- pfm_context_free(ctx);
-
- return;
- }
-
- /*
- * we restore ALL the debug registers to avoid picking up
- * stale state.
- */
- if (ctx->ctx_fl_using_dbreg) {
- pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
- pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
- }
- /*
- * retrieve saved psr.up
- */
- psr_up = ctx->ctx_saved_psr_up;
-
- /*
- * if we were the last user of the PMU on that CPU,
- * then nothing to do except restore psr
- */
- if (GET_LAST_CPU(ctx) == smp_processor_id() && ctx->ctx_last_activation == GET_ACTIVATION()) {
-
- /*
- * retrieve partial reload masks (due to user modifications)
- */
- pmc_mask = ctx->ctx_reload_pmcs[0];
- pmd_mask = ctx->ctx_reload_pmds[0];
-
- } else {
- /*
- * To avoid leaking information to the user level when psr.sp=0,
- * we must reload ALL implemented pmds (even the ones we don't use).
- * In the kernel we only allow PFM_READ_PMDS on registers which
- * we initialized or requested (sampling) so there is no risk there.
- */
- pmd_mask = pfm_sysctl.fastctxsw ? ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0];
-
- /*
- * ALL accessible PMCs are systematically reloaded, unused registers
- * get their default (from pfm_reset_pmu_state()) values to avoid picking
- * up stale configuration.
- *
- * PMC0 is never in the mask. It is always restored separately.
- */
- pmc_mask = ctx->ctx_all_pmcs[0];
- }
- /*
- * when context is MASKED, we will restore PMC with plm=0
- * and PMD with stale information, but that's ok, nothing
- * will be captured.
- *
- * XXX: optimize here
- */
- if (pmd_mask) pfm_restore_pmds(ctx->th_pmds, pmd_mask);
- if (pmc_mask) pfm_restore_pmcs(ctx->th_pmcs, pmc_mask);
-
- /*
- * check for pending overflow at the time the state
- * was saved.
- */
- if (unlikely(PMC0_HAS_OVFL(ctx->th_pmcs[0]))) {
- /*
- * reload pmc0 with the overflow information
- * On McKinley PMU, this will trigger a PMU interrupt
- */
- ia64_set_pmc(0, ctx->th_pmcs[0]);
- ia64_srlz_d();
- ctx->th_pmcs[0] = 0UL;
-
- /*
- * will replay the PMU interrupt
- */
- if (need_irq_resend) ia64_resend_irq(IA64_PERFMON_VECTOR);
-
- pfm_stats[smp_processor_id()].pfm_replay_ovfl_intr_count++;
- }
-
- /*
- * we just did a reload, so we reset the partial reload fields
- */
- ctx->ctx_reload_pmcs[0] = 0UL;
- ctx->ctx_reload_pmds[0] = 0UL;
-
- SET_LAST_CPU(ctx, smp_processor_id());
-
- /*
- * dump activation value for this PMU
- */
- INC_ACTIVATION();
- /*
- * record current activation for this context
- */
- SET_ACTIVATION(ctx);
-
- /*
- * establish new ownership.
- */
- SET_PMU_OWNER(task, ctx);
-
- /*
- * restore the psr.up bit. measurement
- * is active again.
- * no PMU interrupt can happen at this point
- * because we still have interrupts disabled.
- */
- if (likely(psr_up)) pfm_set_psr_up();
-
- /*
- * allow concurrent access to context
- */
- pfm_unprotect_ctx_ctxsw(ctx, flags);
-}
-#else /* !CONFIG_SMP */
-/*
- * reload PMU state for UP kernels
- * in 2.5 we come here with interrupts disabled
- */
-void
-pfm_load_regs (struct task_struct *task)
-{
- pfm_context_t *ctx;
- struct task_struct *owner;
- unsigned long pmd_mask, pmc_mask;
- u64 psr, psr_up;
- int need_irq_resend;
-
- owner = GET_PMU_OWNER();
- ctx = PFM_GET_CTX(task);
- psr = pfm_get_psr();
-
- BUG_ON(psr & (IA64_PSR_UP|IA64_PSR_PP));
- BUG_ON(psr & IA64_PSR_I);
-
- /*
- * we restore ALL the debug registers to avoid picking up
- * stale state.
- *
- * This must be done even when the task is still the owner
- * as the registers may have been modified via ptrace()
- * (not perfmon) by the previous task.
- */
- if (ctx->ctx_fl_using_dbreg) {
- pfm_restore_ibrs(ctx->ctx_ibrs, pmu_conf->num_ibrs);
- pfm_restore_dbrs(ctx->ctx_dbrs, pmu_conf->num_dbrs);
- }
-
- /*
- * retrieved saved psr.up
- */
- psr_up = ctx->ctx_saved_psr_up;
- need_irq_resend = pmu_conf->flags & PFM_PMU_IRQ_RESEND;
-
- /*
- * short path, our state is still there, just
- * need to restore psr and we go
- *
- * we do not touch either PMC nor PMD. the psr is not touched
- * by the overflow_handler. So we are safe w.r.t. to interrupt
- * concurrency even without interrupt masking.
- */
- if (likely(owner == task)) {
- if (likely(psr_up)) pfm_set_psr_up();
- return;
- }
-
- /*
- * someone else is still using the PMU, first push it out and
- * then we'll be able to install our stuff !
- *
- * Upon return, there will be no owner for the current PMU
- */
- if (owner) pfm_lazy_save_regs(owner);
-
- /*
- * To avoid leaking information to the user level when psr.sp=0,
- * we must reload ALL implemented pmds (even the ones we don't use).
- * In the kernel we only allow PFM_READ_PMDS on registers which
- * we initialized or requested (sampling) so there is no risk there.
- */
- pmd_mask = pfm_sysctl.fastctxsw ? ctx->ctx_used_pmds[0] : ctx->ctx_all_pmds[0];
-
- /*
- * ALL accessible PMCs are systematically reloaded, unused registers
- * get their default (from pfm_reset_pmu_state()) values to avoid picking
- * up stale configuration.
- *
- * PMC0 is never in the mask. It is always restored separately
- */
- pmc_mask = ctx->ctx_all_pmcs[0];
-
- pfm_restore_pmds(ctx->th_pmds, pmd_mask);
- pfm_restore_pmcs(ctx->th_pmcs, pmc_mask);
-
- /*
- * check for pending overflow at the time the state
- * was saved.
- */
- if (unlikely(PMC0_HAS_OVFL(ctx->th_pmcs[0]))) {
- /*
- * reload pmc0 with the overflow information
- * On McKinley PMU, this will trigger a PMU interrupt
- */
- ia64_set_pmc(0, ctx->th_pmcs[0]);
- ia64_srlz_d();
-
- ctx->th_pmcs[0] = 0UL;
-
- /*
- * will replay the PMU interrupt
- */
- if (need_irq_resend) ia64_resend_irq(IA64_PERFMON_VECTOR);
-
- pfm_stats[smp_processor_id()].pfm_replay_ovfl_intr_count++;
- }
-
- /*
- * establish new ownership.
- */
- SET_PMU_OWNER(task, ctx);
-
- /*
- * restore the psr.up bit. measurement
- * is active again.
- * no PMU interrupt can happen at this point
- * because we still have interrupts disabled.
- */
- if (likely(psr_up)) pfm_set_psr_up();
-}
-#endif /* CONFIG_SMP */
-
-/*
- * this function assumes monitoring is stopped
- */
-static void
-pfm_flush_pmds(struct task_struct *task, pfm_context_t *ctx)
-{
- u64 pmc0;
- unsigned long mask2, val, pmd_val, ovfl_val;
- int i, can_access_pmu = 0;
- int is_self;
-
- /*
- * is the caller the task being monitored (or which initiated the
- * session for system wide measurements)
- */
- is_self = ctx->ctx_task == task ? 1 : 0;
-
- /*
- * can access PMU is task is the owner of the PMU state on the current CPU
- * or if we are running on the CPU bound to the context in system-wide mode
- * (that is not necessarily the task the context is attached to in this mode).
- * In system-wide we always have can_access_pmu true because a task running on an
- * invalid processor is flagged earlier in the call stack (see pfm_stop).
- */
- can_access_pmu = (GET_PMU_OWNER() == task) || (ctx->ctx_fl_system && ctx->ctx_cpu == smp_processor_id());
- if (can_access_pmu) {
- /*
- * Mark the PMU as not owned
- * This will cause the interrupt handler to do nothing in case an overflow
- * interrupt was in-flight
- * This also guarantees that pmc0 will contain the final state
- * It virtually gives us full control on overflow processing from that point
- * on.
- */
- SET_PMU_OWNER(NULL, NULL);
- DPRINT(("releasing ownership\n"));
-
- /*
- * read current overflow status:
- *
- * we are guaranteed to read the final stable state
- */
- ia64_srlz_d();
- pmc0 = ia64_get_pmc(0); /* slow */
-
- /*
- * reset freeze bit, overflow status information destroyed
- */
- pfm_unfreeze_pmu();
- } else {
- pmc0 = ctx->th_pmcs[0];
- /*
- * clear whatever overflow status bits there were
- */
- ctx->th_pmcs[0] = 0;
- }
- ovfl_val = pmu_conf->ovfl_val;
- /*
- * we save all the used pmds
- * we take care of overflows for counting PMDs
- *
- * XXX: sampling situation is not taken into account here
- */
- mask2 = ctx->ctx_used_pmds[0];
-
- DPRINT(("is_self=%d ovfl_val=0x%lx mask2=0x%lx\n", is_self, ovfl_val, mask2));
-
- for (i = 0; mask2; i++, mask2>>=1) {
-
- /* skip non used pmds */
- if ((mask2 & 0x1) == 0) continue;
-
- /*
- * can access PMU always true in system wide mode
- */
- val = pmd_val = can_access_pmu ? ia64_get_pmd(i) : ctx->th_pmds[i];
-
- if (PMD_IS_COUNTING(i)) {
- DPRINT(("[%d] pmd[%d] ctx_pmd=0x%lx hw_pmd=0x%lx\n",
- task_pid_nr(task),
- i,
- ctx->ctx_pmds[i].val,
- val & ovfl_val));
-
- /*
- * we rebuild the full 64 bit value of the counter
- */
- val = ctx->ctx_pmds[i].val + (val & ovfl_val);
-
- /*
- * now everything is in ctx_pmds[] and we need
- * to clear the saved context from save_regs() such that
- * pfm_read_pmds() gets the correct value
- */
- pmd_val = 0UL;
-
- /*
- * take care of overflow inline
- */
- if (pmc0 & (1UL << i)) {
- val += 1 + ovfl_val;
- DPRINT(("[%d] pmd[%d] overflowed\n", task_pid_nr(task), i));
- }
- }
-
- DPRINT(("[%d] ctx_pmd[%d]=0x%lx pmd_val=0x%lx\n", task_pid_nr(task), i, val, pmd_val));
-
- if (is_self) ctx->th_pmds[i] = pmd_val;
-
- ctx->ctx_pmds[i].val = val;
- }
-}
-
-static void
-pfm_alt_save_pmu_state(void *data)
-{
- struct pt_regs *regs;
-
- regs = task_pt_regs(current);
-
- DPRINT(("called\n"));
-
- /*
- * should not be necessary but
- * let's take not risk
- */
- pfm_clear_psr_up();
- pfm_clear_psr_pp();
- ia64_psr(regs)->pp = 0;
-
- /*
- * This call is required
- * May cause a spurious interrupt on some processors
- */
- pfm_freeze_pmu();
-
- ia64_srlz_d();
-}
-
-void
-pfm_alt_restore_pmu_state(void *data)
-{
- struct pt_regs *regs;
-
- regs = task_pt_regs(current);
-
- DPRINT(("called\n"));
-
- /*
- * put PMU back in state expected
- * by perfmon
- */
- pfm_clear_psr_up();
- pfm_clear_psr_pp();
- ia64_psr(regs)->pp = 0;
-
- /*
- * perfmon runs with PMU unfrozen at all times
- */
- pfm_unfreeze_pmu();
-
- ia64_srlz_d();
-}
-
-int
-pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
-{
- int ret, i;
- int reserve_cpu;
-
- /* some sanity checks */
- if (hdl == NULL || hdl->handler == NULL) return -EINVAL;
-
- /* do the easy test first */
- if (pfm_alt_intr_handler) return -EBUSY;
-
- /* one at a time in the install or remove, just fail the others */
- if (!spin_trylock(&pfm_alt_install_check)) {
- return -EBUSY;
- }
-
- /* reserve our session */
- for_each_online_cpu(reserve_cpu) {
- ret = pfm_reserve_session(NULL, 1, reserve_cpu);
- if (ret) goto cleanup_reserve;
- }
-
- /* save the current system wide pmu states */
- on_each_cpu(pfm_alt_save_pmu_state, NULL, 1);
-
- /* officially change to the alternate interrupt handler */
- pfm_alt_intr_handler = hdl;
-
- spin_unlock(&pfm_alt_install_check);
-
- return 0;
-
-cleanup_reserve:
- for_each_online_cpu(i) {
- /* don't unreserve more than we reserved */
- if (i >= reserve_cpu) break;
-
- pfm_unreserve_session(NULL, 1, i);
- }
-
- spin_unlock(&pfm_alt_install_check);
-
- return ret;
-}
-EXPORT_SYMBOL_GPL(pfm_install_alt_pmu_interrupt);
-
-int
-pfm_remove_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
-{
- int i;
-
- if (hdl == NULL) return -EINVAL;
-
- /* cannot remove someone else's handler! */
- if (pfm_alt_intr_handler != hdl) return -EINVAL;
-
- /* one at a time in the install or remove, just fail the others */
- if (!spin_trylock(&pfm_alt_install_check)) {
- return -EBUSY;
- }
-
- pfm_alt_intr_handler = NULL;
-
- on_each_cpu(pfm_alt_restore_pmu_state, NULL, 1);
-
- for_each_online_cpu(i) {
- pfm_unreserve_session(NULL, 1, i);
- }
-
- spin_unlock(&pfm_alt_install_check);
-
- return 0;
-}
-EXPORT_SYMBOL_GPL(pfm_remove_alt_pmu_interrupt);
-
-/*
- * perfmon initialization routine, called from the initcall() table
- */
-static int init_pfm_fs(void);
-
-static int __init
-pfm_probe_pmu(void)
-{
- pmu_config_t **p;
- int family;
-
- family = local_cpu_data->family;
- p = pmu_confs;
-
- while(*p) {
- if ((*p)->probe) {
- if ((*p)->probe() == 0) goto found;
- } else if ((*p)->pmu_family == family || (*p)->pmu_family == 0xff) {
- goto found;
- }
- p++;
- }
- return -1;
-found:
- pmu_conf = *p;
- return 0;
-}
-
-int __init
-pfm_init(void)
-{
- unsigned int n, n_counters, i;
-
- printk("perfmon: version %u.%u IRQ %u\n",
- PFM_VERSION_MAJ,
- PFM_VERSION_MIN,
- IA64_PERFMON_VECTOR);
-
- if (pfm_probe_pmu()) {
- printk(KERN_INFO "perfmon: disabled, there is no support for processor family %d\n",
- local_cpu_data->family);
- return -ENODEV;
- }
-
- /*
- * compute the number of implemented PMD/PMC from the
- * description tables
- */
- n = 0;
- for (i=0; PMC_IS_LAST(i) == 0; i++) {
- if (PMC_IS_IMPL(i) == 0) continue;
- pmu_conf->impl_pmcs[i>>6] |= 1UL << (i&63);
- n++;
- }
- pmu_conf->num_pmcs = n;
-
- n = 0; n_counters = 0;
- for (i=0; PMD_IS_LAST(i) == 0; i++) {
- if (PMD_IS_IMPL(i) == 0) continue;
- pmu_conf->impl_pmds[i>>6] |= 1UL << (i&63);
- n++;
- if (PMD_IS_COUNTING(i)) n_counters++;
- }
- pmu_conf->num_pmds = n;
- pmu_conf->num_counters = n_counters;
-
- /*
- * sanity checks on the number of debug registers
- */
- if (pmu_conf->use_rr_dbregs) {
- if (pmu_conf->num_ibrs > IA64_NUM_DBG_REGS) {
- printk(KERN_INFO "perfmon: unsupported number of code debug registers (%u)\n", pmu_conf->num_ibrs);
- pmu_conf = NULL;
- return -1;
- }
- if (pmu_conf->num_dbrs > IA64_NUM_DBG_REGS) {
- printk(KERN_INFO "perfmon: unsupported number of data debug registers (%u)\n", pmu_conf->num_ibrs);
- pmu_conf = NULL;
- return -1;
- }
- }
-
- printk("perfmon: %s PMU detected, %u PMCs, %u PMDs, %u counters (%lu bits)\n",
- pmu_conf->pmu_name,
- pmu_conf->num_pmcs,
- pmu_conf->num_pmds,
- pmu_conf->num_counters,
- ffz(pmu_conf->ovfl_val));
-
- /* sanity check */
- if (pmu_conf->num_pmds >= PFM_NUM_PMD_REGS || pmu_conf->num_pmcs >= PFM_NUM_PMC_REGS) {
- printk(KERN_ERR "perfmon: not enough pmc/pmd, perfmon disabled\n");
- pmu_conf = NULL;
- return -1;
- }
-
- /*
- * create /proc/perfmon (mostly for debugging purposes)
- */
- perfmon_dir = proc_create_seq("perfmon", S_IRUGO, NULL, &pfm_seq_ops);
- if (perfmon_dir == NULL) {
- printk(KERN_ERR "perfmon: cannot create /proc entry, perfmon disabled\n");
- pmu_conf = NULL;
- return -1;
- }
-
- /*
- * create /proc/sys/kernel/perfmon (for debugging purposes)
- */
- pfm_sysctl_header = register_sysctl_table(pfm_sysctl_root);
-
- /*
- * initialize all our spinlocks
- */
- spin_lock_init(&pfm_sessions.pfs_lock);
- spin_lock_init(&pfm_buffer_fmt_lock);
-
- init_pfm_fs();
-
- for(i=0; i < NR_CPUS; i++) pfm_stats[i].pfm_ovfl_intr_cycles_min = ~0UL;
-
- return 0;
-}
-
-__initcall(pfm_init);
-
-/*
- * this function is called before pfm_init()
- */
-void
-pfm_init_percpu (void)
-{
- static int first_time=1;
- /*
- * make sure no measurement is active
- * (may inherit programmed PMCs from EFI).
- */
- pfm_clear_psr_pp();
- pfm_clear_psr_up();
-
- /*
- * we run with the PMU not frozen at all times
- */
- pfm_unfreeze_pmu();
-
- if (first_time) {
- register_percpu_irq(IA64_PERFMON_VECTOR, pfm_interrupt_handler,
- 0, "perfmon");
- first_time=0;
- }
-
- ia64_setreg(_IA64_REG_CR_PMV, IA64_PERFMON_VECTOR);
- ia64_srlz_d();
-}
-
-/*
- * used for debug purposes only
- */
-void
-dump_pmu_state(const char *from)
-{
- struct task_struct *task;
- struct pt_regs *regs;
- pfm_context_t *ctx;
- unsigned long psr, dcr, info, flags;
- int i, this_cpu;
-
- local_irq_save(flags);
-
- this_cpu = smp_processor_id();
- regs = task_pt_regs(current);
- info = PFM_CPUINFO_GET();
- dcr = ia64_getreg(_IA64_REG_CR_DCR);
-
- if (info == 0 && ia64_psr(regs)->pp == 0 && (dcr & IA64_DCR_PP) == 0) {
- local_irq_restore(flags);
- return;
- }
-
- printk("CPU%d from %s() current [%d] iip=0x%lx %s\n",
- this_cpu,
- from,
- task_pid_nr(current),
- regs->cr_iip,
- current->comm);
-
- task = GET_PMU_OWNER();
- ctx = GET_PMU_CTX();
-
- printk("->CPU%d owner [%d] ctx=%p\n", this_cpu, task ? task_pid_nr(task) : -1, ctx);
-
- psr = pfm_get_psr();
-
- printk("->CPU%d pmc0=0x%lx psr.pp=%d psr.up=%d dcr.pp=%d syst_info=0x%lx user_psr.up=%d user_psr.pp=%d\n",
- this_cpu,
- ia64_get_pmc(0),
- psr & IA64_PSR_PP ? 1 : 0,
- psr & IA64_PSR_UP ? 1 : 0,
- dcr & IA64_DCR_PP ? 1 : 0,
- info,
- ia64_psr(regs)->up,
- ia64_psr(regs)->pp);
-
- ia64_psr(regs)->up = 0;
- ia64_psr(regs)->pp = 0;
-
- for (i=1; PMC_IS_LAST(i) == 0; i++) {
- if (PMC_IS_IMPL(i) == 0) continue;
- printk("->CPU%d pmc[%d]=0x%lx thread_pmc[%d]=0x%lx\n", this_cpu, i, ia64_get_pmc(i), i, ctx->th_pmcs[i]);
- }
-
- for (i=1; PMD_IS_LAST(i) == 0; i++) {
- if (PMD_IS_IMPL(i) == 0) continue;
- printk("->CPU%d pmd[%d]=0x%lx thread_pmd[%d]=0x%lx\n", this_cpu, i, ia64_get_pmd(i), i, ctx->th_pmds[i]);
- }
-
- if (ctx) {
- printk("->CPU%d ctx_state=%d vaddr=%p addr=%p fd=%d ctx_task=[%d] saved_psr_up=0x%lx\n",
- this_cpu,
- ctx->ctx_state,
- ctx->ctx_smpl_vaddr,
- ctx->ctx_smpl_hdr,
- ctx->ctx_msgq_head,
- ctx->ctx_msgq_tail,
- ctx->ctx_saved_psr_up);
- }
- local_irq_restore(flags);
-}
-
-/*
- * called from process.c:copy_thread(). task is new child.
- */
-void
-pfm_inherit(struct task_struct *task, struct pt_regs *regs)
-{
- struct thread_struct *thread;
-
- DPRINT(("perfmon: pfm_inherit clearing state for [%d]\n", task_pid_nr(task)));
-
- thread = &task->thread;
-
- /*
- * cut links inherited from parent (current)
- */
- thread->pfm_context = NULL;
-
- PFM_SET_WORK_PENDING(task, 0);
-
- /*
- * the psr bits are already set properly in copy_threads()
- */
-}
-#else /* !CONFIG_PERFMON */
-asmlinkage long
-sys_perfmonctl (int fd, int cmd, void *arg, int count)
-{
- return -ENOSYS;
-}
-#endif /* CONFIG_PERFMON */
#include "entry.h"
-#ifdef CONFIG_PERFMON
-# include <asm/perfmon.h>
-#endif
-
#include "sigframe.h"
void (*ia64_mark_idle)(int);
return;
}
-#ifdef CONFIG_PERFMON
- if (current->thread.pfm_needs_checking)
- /*
- * Note: pfm_handle_work() allow us to call it with interrupts
- * disabled, and may enable interrupts within the function.
- */
- pfm_handle_work();
-#endif
-
/* deal with pending signal delivery */
if (test_thread_flag(TIF_SIGPENDING)) {
local_irq_enable(); /* force interrupt enable */
void
ia64_save_extra (struct task_struct *task)
{
-#ifdef CONFIG_PERFMON
- unsigned long info;
-#endif
-
if ((task->thread.flags & IA64_THREAD_DBG_VALID) != 0)
ia64_save_debug_regs(&task->thread.dbr[0]);
-
-#ifdef CONFIG_PERFMON
- if ((task->thread.flags & IA64_THREAD_PM_VALID) != 0)
- pfm_save_regs(task);
-
- info = __this_cpu_read(pfm_syst_info);
- if (info & PFM_CPUINFO_SYST_WIDE)
- pfm_syst_wide_update_task(task, info, 0);
-#endif
}
void
ia64_load_extra (struct task_struct *task)
{
-#ifdef CONFIG_PERFMON
- unsigned long info;
-#endif
-
if ((task->thread.flags & IA64_THREAD_DBG_VALID) != 0)
ia64_load_debug_regs(&task->thread.dbr[0]);
-
-#ifdef CONFIG_PERFMON
- if ((task->thread.flags & IA64_THREAD_PM_VALID) != 0)
- pfm_load_regs(task);
-
- info = __this_cpu_read(pfm_syst_info);
- if (info & PFM_CPUINFO_SYST_WIDE)
- pfm_syst_wide_update_task(task, info, 1);
-#endif
}
/*
*/
child_ptregs->cr_ipsr = ((child_ptregs->cr_ipsr | IA64_PSR_BITS_TO_SET)
& ~(IA64_PSR_BITS_TO_CLEAR | IA64_PSR_PP | IA64_PSR_UP));
-
-#ifdef CONFIG_PERFMON
- if (current->thread.pfm_context)
- pfm_inherit(p, child_ptregs);
-#endif
return retval;
}
{
ia64_drop_fpu(tsk);
-#ifdef CONFIG_PERFMON
- /* if needed, stop monitoring and flush state to perfmon context */
- if (tsk->thread.pfm_context)
- pfm_exit_thread(tsk);
-
- /* free debug register resources */
- if (tsk->thread.flags & IA64_THREAD_DBG_VALID)
- pfm_release_debug_registers(tsk);
-#endif
}
unsigned long
#include <asm/rse.h>
#include <linux/uaccess.h>
#include <asm/unwind.h>
-#ifdef CONFIG_PERFMON
-#include <asm/perfmon.h>
-#endif
#include "entry.h"
"address 0x%lx\n", addr);
return -1;
}
-#ifdef CONFIG_PERFMON
- /*
- * Check if debug registers are used by perfmon. This
- * test must be done once we know that we can do the
- * operation, i.e. the arguments are all valid, but
- * before we start modifying the state.
- *
- * Perfmon needs to keep a count of how many processes
- * are trying to modify the debug registers for system
- * wide monitoring sessions.
- *
- * We also include read access here, because they may
- * cause the PMU-installed debug register state
- * (dbr[], ibr[]) to be reset. The two arrays are also
- * used by perfmon, but we do not use
- * IA64_THREAD_DBG_VALID. The registers are restored
- * by the PMU context switch code.
- */
- if (pfm_use_debug_registers(child))
- return -1;
-#endif
if (!(child->thread.flags & IA64_THREAD_DBG_VALID)) {
child->thread.flags |= IA64_THREAD_DBG_VALID;
extern void ia64_init_itm(void);
extern volatile int time_keeper_id;
-#ifdef CONFIG_PERFMON
- extern void pfm_init_percpu(void);
-#endif
-
cpuid = smp_processor_id();
phys_id = hard_smp_processor_id();
itc_master = time_keeper_id;
ia64_mca_cmc_vector_setup(); /* Setup vector on AP */
-#ifdef CONFIG_PERFMON
- pfm_init_percpu();
-#endif
-
local_irq_enable();
if (!(sal_platform_features & IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT)) {
148 common mmap2 sys_mmap2
149 common pciconfig_read sys_pciconfig_read
150 common pciconfig_write sys_pciconfig_write
-151 common perfmonctl sys_perfmonctl
+151 common perfmonctl sys_ni_syscall
152 common sigaltstack sys_sigaltstack
153 common rt_sigaction sys_rt_sigaction
154 common rt_sigpending sys_rt_sigpending
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
/* Default discards */
DISCARDS
lib-$(CONFIG_ITANIUM) += copy_page.o copy_user.o memcpy.o
lib-$(CONFIG_MCKINLEY) += copy_page_mck.o memcpy_mck.o
-lib-$(CONFIG_PERFMON) += carta_random.o
AFLAGS___divdi3.o =
AFLAGS___udivdi3.o = -DUNSIGNED
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Fast, simple, yet decent quality random number generator based on
- * a paper by David G. Carta ("Two Fast Implementations of the
- * `Minimal Standard' Random Number Generator," Communications of the
- * ACM, January, 1990).
- *
- * Copyright (C) 2002 Hewlett-Packard Co
- * David Mosberger-Tang <davidm@hpl.hp.com>
- */
-
-#include <asm/asmmacro.h>
-
-#define a r2
-#define m r3
-#define lo r8
-#define hi r9
-#define t0 r16
-#define t1 r17
-#define seed r32
-
-GLOBAL_ENTRY(carta_random32)
- movl a = (16807 << 16) | 16807
- ;;
- pmpyshr2.u t0 = a, seed, 0
- pmpyshr2.u t1 = a, seed, 16
- ;;
- unpack2.l t0 = t1, t0
- dep m = -1, r0, 0, 31
- ;;
- zxt4 lo = t0
- shr.u hi = t0, 32
- ;;
- dep t0 = 0, hi, 15, 49 // t0 = (hi & 0x7fff)
- ;;
- shl t0 = t0, 16 // t0 = (hi & 0x7fff) << 16
- shr t1 = hi, 15 // t1 = (hi >> 15)
- ;;
- add lo = lo, t0
- ;;
- cmp.gtu p6, p0 = lo, m
- ;;
-(p6) and lo = lo, m
- ;;
-(p6) add lo = 1, lo
- ;;
- add lo = lo, t1
- ;;
- cmp.gtu p6, p0 = lo, m
- ;;
-(p6) and lo = lo, m
- ;;
-(p6) add lo = 1, lo
- br.ret.sptk.many rp
-END(carta_random32)
out:
return result;
}
-
-/*
- * XXX Fixme
- *
- * This is very ugly but temporary. THIS NEEDS SERIOUS ENHANCEMENTS.
- * But it's very tricky to get right even in C.
- */
-__wsum
-csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
-{
- memcpy(dst, src, len);
- return csum_partial(dst, len, sum);
-}
-
-EXPORT_SYMBOL(csum_partial_copy_nocheck);
if (map_start < map_end)
memmap_init_zone((unsigned long)(map_end - map_start),
args->nid, args->zone, page_to_pfn(map_start),
- MEMMAP_EARLY, NULL);
+ MEMINIT_EARLY, NULL);
return 0;
}
unsigned long start_pfn)
{
if (!vmem_map) {
- memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY,
- NULL);
+ memmap_init_zone(size, nid, zone, start_pfn,
+ MEMINIT_EARLY, NULL);
} else {
struct page *start;
struct memmap_init_callback_data args;
timer_int.o )
oprofile-y := $(DRIVER_OBJS) init.o backtrace.o
-oprofile-$(CONFIG_PERFMON) += perfmon.o
int __init oprofile_arch_init(struct oprofile_operations *ops)
{
- int ret = -ENODEV;
-
-#ifdef CONFIG_PERFMON
- /* perfmon_init() can fail, but we have no way to report it */
- ret = perfmon_init(ops);
-#endif
ops->backtrace = ia64_backtrace;
-
- return ret;
+ return -ENODEV;
}
void oprofile_arch_exit(void)
{
-#ifdef CONFIG_PERFMON
- perfmon_exit();
-#endif
}
+++ /dev/null
-/**
- * @file perfmon.c
- *
- * @remark Copyright 2003 OProfile authors
- * @remark Read the file COPYING
- *
- * @author John Levon <levon@movementarian.org>
- */
-
-#include <linux/kernel.h>
-#include <linux/oprofile.h>
-#include <linux/sched.h>
-#include <asm/perfmon.h>
-#include <asm/ptrace.h>
-#include <asm/errno.h>
-
-static int allow_ints;
-
-static int
-perfmon_handler(struct task_struct *task, void *buf, pfm_ovfl_arg_t *arg,
- struct pt_regs *regs, unsigned long stamp)
-{
- int event = arg->pmd_eventid;
-
- arg->ovfl_ctrl.bits.reset_ovfl_pmds = 1;
-
- /* the owner of the oprofile event buffer may have exited
- * without perfmon being shutdown (e.g. SIGSEGV)
- */
- if (allow_ints)
- oprofile_add_sample(regs, event);
- return 0;
-}
-
-
-static int perfmon_start(void)
-{
- allow_ints = 1;
- return 0;
-}
-
-
-static void perfmon_stop(void)
-{
- allow_ints = 0;
-}
-
-
-#define OPROFILE_FMT_UUID { \
- 0x77, 0x7a, 0x6e, 0x61, 0x20, 0x65, 0x73, 0x69, 0x74, 0x6e, 0x72, 0x20, 0x61, 0x65, 0x0a, 0x6c }
-
-static pfm_buffer_fmt_t oprofile_fmt = {
- .fmt_name = "oprofile_format",
- .fmt_uuid = OPROFILE_FMT_UUID,
- .fmt_handler = perfmon_handler,
-};
-
-
-static char *get_cpu_type(void)
-{
- __u8 family = local_cpu_data->family;
-
- switch (family) {
- case 0x07:
- return "ia64/itanium";
- case 0x1f:
- return "ia64/itanium2";
- default:
- return "ia64/ia64";
- }
-}
-
-
-/* all the ops are handled via userspace for IA64 perfmon */
-
-static int using_perfmon;
-
-int perfmon_init(struct oprofile_operations *ops)
-{
- int ret = pfm_register_buffer_fmt(&oprofile_fmt);
- if (ret)
- return -ENODEV;
-
- ops->cpu_type = get_cpu_type();
- ops->start = perfmon_start;
- ops->stop = perfmon_stop;
- using_perfmon = 1;
- printk(KERN_INFO "oprofile: using perfmon.\n");
- return 0;
-}
-
-
-void perfmon_exit(void)
-{
- if (!using_perfmon)
- return;
-
- pfm_unregister_buffer_fmt(oprofile_fmt.fmt_uuid);
-}
select ARCH_HAS_BINFMT_FLAT
select ARCH_HAS_DMA_PREP_COHERENT if HAS_DMA && MMU && !COLDFIRE
select ARCH_HAS_SYNC_DMA_FOR_DEVICE if HAS_DMA
+ select ARCH_HAVE_NMI_SAFE_CMPXCHG if RMW_INSNS
select ARCH_MIGHT_HAVE_PC_PARPORT if ISA
select ARCH_NO_PREEMPT if !COLDFIRE
+ select ARCH_WANT_IPC_PARSE_VERSION
select BINFMT_FLAT_ARGVP_ENVP_ON_STACK
select DMA_DIRECT_REMAP if HAS_DMA && MMU && !COLDFIRE
- select HAVE_IDE
- select HAVE_AOUT if MMU
- select HAVE_ASM_MODVERSIONS
- select HAVE_DEBUG_BUGVERBOSE
- select GENERIC_IRQ_SHOW
select GENERIC_ATOMIC64
- select NO_DMA if !MMU && !COLDFIRE
- select HAVE_UID16
- select VIRT_TO_BUS
- select ARCH_HAVE_NMI_SAFE_CMPXCHG if RMW_INSNS
select GENERIC_CPU_DEVICES
select GENERIC_IOMAP
+ select GENERIC_IRQ_SHOW
select GENERIC_STRNCPY_FROM_USER if MMU
select GENERIC_STRNLEN_USER if MMU
- select ARCH_WANT_IPC_PARSE_VERSION
+ select HAVE_AOUT if MMU
+ select HAVE_ASM_MODVERSIONS
+ select HAVE_DEBUG_BUGVERBOSE
select HAVE_FUTEX_CMPXCHG if MMU && FUTEX
+ select HAVE_IDE
select HAVE_MOD_ARCH_SPECIFIC
+ select HAVE_UID16
+ select MMU_GATHER_NO_RANGE if MMU
select MODULES_USE_ELF_REL
select MODULES_USE_ELF_RELA
- select OLD_SIGSUSPEND3
+ select NO_DMA if !MMU && !COLDFIRE
select OLD_SIGACTION
- select MMU_GATHER_NO_RANGE if MMU
+ select OLD_SIGSUSPEND3
+ select VIRT_TO_BUS
config CPU_BIG_ENDIAN
def_bool y
switch (amiga_model) {
case AMI_UNKNOWN:
- goto Generic;
+ break;
case AMI_600:
case AMI_1200:
case AMI_2000:
case AMI_2500:
AMIGAHW_SET(A2000_CLK); /* Is this correct for all models? */
- goto Generic;
+ break;
case AMI_3000:
case AMI_3000T:
AMIGAHW_SET(A3000_SCSI);
AMIGAHW_SET(A3000_CLK);
AMIGAHW_SET(ZORRO3);
- goto Generic;
+ break;
case AMI_4000T:
AMIGAHW_SET(A4000_SCSI);
AMIGAHW_SET(A4000_IDE);
AMIGAHW_SET(A3000_CLK);
AMIGAHW_SET(ZORRO3);
- goto Generic;
+ break;
case AMI_CDTV:
case AMI_CD32:
AMIGAHW_SET(CD_ROM);
AMIGAHW_SET(A2000_CLK); /* Is this correct? */
- goto Generic;
-
- Generic:
- AMIGAHW_SET(AMI_VIDEO);
- AMIGAHW_SET(AMI_BLITTER);
- AMIGAHW_SET(AMI_AUDIO);
- AMIGAHW_SET(AMI_FLOPPY);
- AMIGAHW_SET(AMI_KEYBOARD);
- AMIGAHW_SET(AMI_MOUSE);
- AMIGAHW_SET(AMI_SERIAL);
- AMIGAHW_SET(AMI_PARALLEL);
- AMIGAHW_SET(CHIP_RAM);
- AMIGAHW_SET(PAULA);
-
- switch (amiga_chipset) {
- case CS_OCS:
- case CS_ECS:
- case CS_AGA:
- switch (amiga_custom.deniseid & 0xf) {
- case 0x0c:
- AMIGAHW_SET(DENISE_HR);
- break;
- case 0x08:
- AMIGAHW_SET(LISA);
- break;
- }
- break;
- default:
- AMIGAHW_SET(DENISE);
- break;
- }
- switch ((amiga_custom.vposr>>8) & 0x7f) {
- case 0x00:
- AMIGAHW_SET(AGNUS_PAL);
- break;
- case 0x10:
- AMIGAHW_SET(AGNUS_NTSC);
- break;
- case 0x20:
- case 0x21:
- AMIGAHW_SET(AGNUS_HR_PAL);
- break;
- case 0x30:
- case 0x31:
- AMIGAHW_SET(AGNUS_HR_NTSC);
- break;
- case 0x22:
- case 0x23:
- AMIGAHW_SET(ALICE_PAL);
- break;
- case 0x32:
- case 0x33:
- AMIGAHW_SET(ALICE_NTSC);
- break;
- }
- AMIGAHW_SET(ZORRO);
break;
case AMI_DRACO:
panic("Unknown Amiga Model");
}
+ AMIGAHW_SET(AMI_VIDEO);
+ AMIGAHW_SET(AMI_BLITTER);
+ AMIGAHW_SET(AMI_AUDIO);
+ AMIGAHW_SET(AMI_FLOPPY);
+ AMIGAHW_SET(AMI_KEYBOARD);
+ AMIGAHW_SET(AMI_MOUSE);
+ AMIGAHW_SET(AMI_SERIAL);
+ AMIGAHW_SET(AMI_PARALLEL);
+ AMIGAHW_SET(CHIP_RAM);
+ AMIGAHW_SET(PAULA);
+
+ switch (amiga_chipset) {
+ case CS_OCS:
+ case CS_ECS:
+ case CS_AGA:
+ switch (amiga_custom.deniseid & 0xf) {
+ case 0x0c:
+ AMIGAHW_SET(DENISE_HR);
+ break;
+ case 0x08:
+ AMIGAHW_SET(LISA);
+ break;
+ default:
+ AMIGAHW_SET(DENISE);
+ break;
+ }
+ break;
+ }
+ switch ((amiga_custom.vposr>>8) & 0x7f) {
+ case 0x00:
+ AMIGAHW_SET(AGNUS_PAL);
+ break;
+ case 0x10:
+ AMIGAHW_SET(AGNUS_NTSC);
+ break;
+ case 0x20:
+ case 0x21:
+ AMIGAHW_SET(AGNUS_HR_PAL);
+ break;
+ case 0x30:
+ case 0x31:
+ AMIGAHW_SET(AGNUS_HR_NTSC);
+ break;
+ case 0x22:
+ case 0x23:
+ AMIGAHW_SET(ALICE_PAL);
+ break;
+ case 0x32:
+ case 0x33:
+ AMIGAHW_SET(ALICE_NTSC);
+ break;
+ }
+ AMIGAHW_SET(ZORRO);
+
#define AMIGAHW_ANNOUNCE(name, str) \
if (AMIGAHW_PRESENT(name)) \
pr_cont(str)
CONFIG_IDE=y
CONFIG_IDE_GD_ATAPI=y
CONFIG_BLK_DEV_IDECD=y
+CONFIG_BLK_DEV_PLATFORM=y
CONFIG_BLK_DEV_MAC_IDE=y
CONFIG_RAID_ATTRS=m
CONFIG_SCSI=y
CONFIG_IDE=y
CONFIG_IDE_GD_ATAPI=y
CONFIG_BLK_DEV_IDECD=y
+CONFIG_BLK_DEV_PLATFORM=y
CONFIG_BLK_DEV_GAYLE=y
CONFIG_BLK_DEV_BUDDHA=y
CONFIG_BLK_DEV_FALCON_IDE=y
*/
#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER
+#define _HAVE_ARCH_CSUM_AND_COPY
extern __wsum csum_and_copy_from_user(const void __user *src,
void *dst,
- int len, __wsum sum,
- int *csum_err);
+ int len);
extern __wsum csum_partial_copy_nocheck(const void *src,
- void *dst, int len,
- __wsum sum);
+ void *dst, int len);
/*
* This is a version of ip_fast_csum() optimized for IP headers,
#define TIF_MEMDIE 16 /* is terminating due to OOM killer */
#define TIF_RESTORE_SIGMASK 18 /* restore signal mask in do_signal */
+#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
+#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
+#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
+#define _TIF_DELAYED_TRACE (1 << TIF_DELAYED_TRACE)
+#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
+#define _TIF_MEMDIE (1 << TIF_MEMDIE)
+#define _TIF_RESTORE_SIGMASK (1 << TIF_RESTORE_SIGMASK)
+
#endif /* _ASM_M68K_THREAD_INFO_H */
* Of course, readability is a subjective issue, so it will never be
* argued that that goal was accomplished. It was merely a goal.
* A key way to help make code more readable is to give good
- * documentation. So, the first thing you will find is exaustive
+ * documentation. So, the first thing you will find is exhaustive
* write-ups on the structure of the file, and the features of the
* functional subroutines.
*
* mmu_engage
*
* This chunk of code performs the gruesome task of engaging the MMU.
- * The reason its gruesome is because when the MMU becomes engaged it
+ * The reason it's gruesome is because when the MMU becomes engaged it
* maps logical addresses to physical addresses. The Program Counter
* register is then passed through the MMU before the next instruction
* is fetched (the instruction following the engage MMU instruction).
/*
* After this point no new memory is allocated and
* the start of available memory is stored in availmem.
- * (The bootmem allocator requires now the physicall address.)
+ * (The bootmem allocator requires now the physical address.)
*/
movel L(memory_start),availmem
* seven bits of the logical address (LA) are used as an
* index into the "root table." Each entry in the root
* table has a bit which specifies if it's a valid pointer to a
- * pointer table. Each entry defines a 32KMeg range of memory.
+ * pointer table. Each entry defines a 32Meg range of memory.
* If an entry is invalid then that logical range of 32M is
* invalid and references to that range of memory (when the MMU
* is enabled) will fault. If the entry is valid, then it does
* bits 17..12 - index into the Page Table
* bits 11..0 - offset into a particular 4K page
*
- * The algorithms which follows do one thing: they abstract
+ * The algorithms which follow do one thing: they abstract
* the MMU hardware. For example, there are three kinds of
* cache settings that are relevant. Either, memory is
* being mapped in which case it is either Kernel Code (or
* mmu_map
*
* This routine will map a range of memory using a pointer
- * table and allocating the pages on the fly from the kernel.
+ * table and allocate the pages on the fly from the kernel.
* The pointer table does not have to be already linked into
* the root table, this routine will do that if necessary.
*
/* Find the start of free memory, get_bi_record does this for us,
* as the bootinfo structure is located directly behind the kernel
- * and and we simply search for the last entry.
+ * we simply search for the last entry.
*/
get_bi_record BI_LAST
addw #PAGESIZE-1,%a0
jne 2f
/* If the page table entry doesn't exist, we allocate a complete new
- * page and use it as one continues big page table which can cover
+ * page and use it as one continuous big page table which can cover
* 4MB of memory, nearly almost all mappings have that alignment.
*/
get_new_page
void show_registers(struct pt_regs *regs)
{
struct frame *fp = (struct frame *)regs;
- mm_segment_t old_fs = get_fs();
u16 c, *cp;
unsigned long addr;
int i;
show_stack(NULL, (unsigned long *)addr, KERN_INFO);
pr_info("Code:");
- set_fs(KERNEL_DS);
cp = (u16 *)regs->pc;
for (i = -8; i < 16; i++) {
- if (get_user(c, cp + i) && i >= 0) {
+ if (get_kernel_nofault(c, cp + i) && i >= 0) {
pr_cont(" Bad PC value.");
break;
}
else
pr_cont(" <%04x>", c);
}
- set_fs(old_fs);
pr_cont("\n");
}
*/
__wsum
-csum_and_copy_from_user(const void __user *src, void *dst,
- int len, __wsum sum, int *csum_err)
+csum_and_copy_from_user(const void __user *src, void *dst, int len)
{
/*
* GCC doesn't like more than 10 operands for the asm
* code.
*/
unsigned long tmp1, tmp2;
+ __wsum sum = ~0U;
__asm__("movel %2,%4\n\t"
"btst #1,%4\n\t" /* Check alignment */
"clrl %5\n\t"
"addxl %5,%0\n\t" /* add X bit */
"7:\t"
- "clrl %5\n" /* no error - clear return value */
- "8:\n"
".section .fixup,\"ax\"\n"
".even\n"
- /* If any exception occurs zero out the rest.
- Similarities with the code above are intentional :-) */
+ /* If any exception occurs, return 0 */
"90:\t"
- "clrw %3@+\n\t"
- "movel %1,%4\n\t"
- "lsrl #5,%1\n\t"
- "jeq 1f\n\t"
- "subql #1,%1\n"
- "91:\t"
- "clrl %3@+\n"
- "92:\t"
- "clrl %3@+\n"
- "93:\t"
- "clrl %3@+\n"
- "94:\t"
- "clrl %3@+\n"
- "95:\t"
- "clrl %3@+\n"
- "96:\t"
- "clrl %3@+\n"
- "97:\t"
- "clrl %3@+\n"
- "98:\t"
- "clrl %3@+\n\t"
- "dbra %1,91b\n\t"
- "clrw %1\n\t"
- "subql #1,%1\n\t"
- "jcc 91b\n"
- "1:\t"
- "movel %4,%1\n\t"
- "andw #0x1c,%4\n\t"
- "jeq 1f\n\t"
- "lsrw #2,%4\n\t"
- "subqw #1,%4\n"
- "99:\t"
- "clrl %3@+\n\t"
- "dbra %4,99b\n\t"
- "1:\t"
- "andw #3,%1\n\t"
- "jeq 9f\n"
- "100:\t"
- "clrw %3@+\n\t"
- "tstw %1\n\t"
- "jeq 9f\n"
- "101:\t"
- "clrb %3@+\n"
- "9:\t"
-#define STR(X) STR1(X)
-#define STR1(X) #X
- "moveq #-" STR(EFAULT) ",%5\n\t"
- "jra 8b\n"
+ "clrl %0\n"
+ "jra 7b\n"
".previous\n"
".section __ex_table,\"a\"\n"
".long 10b,90b\n"
- ".long 11b,91b\n"
- ".long 12b,92b\n"
- ".long 13b,93b\n"
- ".long 14b,94b\n"
- ".long 15b,95b\n"
- ".long 16b,96b\n"
- ".long 17b,97b\n"
- ".long 18b,98b\n"
- ".long 19b,99b\n"
- ".long 20b,100b\n"
- ".long 21b,101b\n"
+ ".long 11b,90b\n"
+ ".long 12b,90b\n"
+ ".long 13b,90b\n"
+ ".long 14b,90b\n"
+ ".long 15b,90b\n"
+ ".long 16b,90b\n"
+ ".long 17b,90b\n"
+ ".long 18b,90b\n"
+ ".long 19b,90b\n"
+ ".long 20b,90b\n"
+ ".long 21b,90b\n"
".previous"
: "=d" (sum), "=d" (len), "=a" (src), "=a" (dst),
"=&d" (tmp1), "=d" (tmp2)
: "0" (sum), "1" (len), "2" (src), "3" (dst)
);
- *csum_err = tmp2;
-
- return(sum);
+ return sum;
}
EXPORT_SYMBOL(csum_and_copy_from_user);
*/
__wsum
-csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
+csum_partial_copy_nocheck(const void *src, void *dst, int len)
{
unsigned long tmp1, tmp2;
+ __wsum sum = 0;
__asm__("movel %2,%4\n\t"
"btst #1,%4\n\t" /* Check alignment */
"jeq 2f\n\t"
#include <linux/init.h>
#include <linux/vt_kern.h>
#include <linux/platform_device.h>
+#include <linux/ata_platform.h>
#include <linux/adb.h>
#include <linux/cuda.h>
#include <linux/pmu.h>
},
};
+static const struct resource mac_ide_quadra_rsrc[] __initconst = {
+ DEFINE_RES_MEM(0x50F1A000, 0x104),
+ DEFINE_RES_IRQ(IRQ_NUBUS_F),
+};
+
+static const struct resource mac_ide_pb_rsrc[] __initconst = {
+ DEFINE_RES_MEM(0x50F1A000, 0x104),
+ DEFINE_RES_IRQ(IRQ_NUBUS_C),
+};
+
+static const struct resource mac_pata_baboon_rsrc[] __initconst = {
+ DEFINE_RES_MEM(0x50F1A000, 0x38),
+ DEFINE_RES_MEM(0x50F1A038, 0x04),
+ DEFINE_RES_IRQ(IRQ_BABOON_1),
+};
+
+static const struct pata_platform_info mac_pata_baboon_data __initconst = {
+ .ioport_shift = 2,
+};
+
int __init mac_platform_init(void)
{
phys_addr_t swim_base = 0;
break;
}
+ /*
+ * IDE device
+ */
+
+ switch (macintosh_config->ide_type) {
+ case MAC_IDE_QUADRA:
+ platform_device_register_simple("mac_ide", -1,
+ mac_ide_quadra_rsrc, ARRAY_SIZE(mac_ide_quadra_rsrc));
+ break;
+ case MAC_IDE_PB:
+ platform_device_register_simple("mac_ide", -1,
+ mac_ide_pb_rsrc, ARRAY_SIZE(mac_ide_pb_rsrc));
+ break;
+ case MAC_IDE_BABOON:
+ platform_device_register_resndata(NULL, "pata_platform", -1,
+ mac_pata_baboon_rsrc, ARRAY_SIZE(mac_pata_baboon_rsrc),
+ &mac_pata_baboon_data, sizeof(mac_pata_baboon_data));
+ break;
+ }
+
/*
* Ethernet device
*/
* support 16-bit stereo output, but only mono input."
*
* Technical Information Library (TIL) article number 16405.
- * http://support.apple.com/kb/TA32601
+ * https://support.apple.com/kb/TA32601
*
* --David Kilzer
*/
unsigned long max_zone_pfn[MAX_NR_ZONES] = { 0 };
int i;
- empty_zero_page = (void *) memblock_alloc(PAGE_SIZE, PAGE_SIZE);
+ empty_zero_page = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
if (!empty_zero_page)
panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
__func__, PAGE_SIZE, PAGE_SIZE);
{
pte_t *pte_table = last_pte_table;
- if (((unsigned long)last_pte_table & ~PAGE_MASK) == 0) {
- pte_table = (pte_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
+ if (PAGE_ALIGNED(last_pte_table)) {
+ pte_table = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
if (!pte_table) {
panic("%s: Failed to allocate %lu bytes align=%lx\n",
__func__, PAGE_SIZE, PAGE_SIZE);
}
last_pmd_table += PTRS_PER_PMD;
- if (((unsigned long)last_pmd_table & ~PAGE_MASK) == 0) {
- last_pmd_table = (pmd_t *)memblock_alloc_low(PAGE_SIZE,
- PAGE_SIZE);
+ if (PAGE_ALIGNED(last_pmd_table)) {
+ last_pmd_table = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
if (!last_pmd_table)
panic("%s: Failed to allocate %lu bytes align=%lx\n",
__func__, PAGE_SIZE, PAGE_SIZE);
# SPDX-License-Identifier: GPL-2.0
generated-y += syscall_table.h
generic-y += extable.h
-generic-y += hw_irq.h
generic-y += kvm_para.h
generic-y += local64.h
generic-y += mcs_spinlock.h
select MODULES_USE_ELF_REL if MODULES
select MODULES_USE_ELF_RELA if MODULES && 64BIT
select PERF_USE_VMALLOC
+ select PCI_MSI_ARCH_FALLBACKS if PCI_MSI
select RTC_LIB
select SYSCTL_EXCEPTION_TRACE
select VIRT_TO_BUS
{
struct cpuinfo_mips *c = ¤t_cpu_data;
- if ((c->cputype == CPU_74K) || (c->cputype == CPU_1074K)) {
+ if (c->cputype == CPU_74K) {
pr_info("Using bcma bus\n");
#ifdef CONFIG_BCM47XX_BCMA
bcm47xx_bus_type = BCM47XX_BUS_TYPE_BCMA;
*/
__wsum csum_partial(const void *buff, int len, __wsum sum);
-__wsum __csum_partial_copy_kernel(const void *src, void *dst,
- int len, __wsum sum, int *err_ptr);
-
-__wsum __csum_partial_copy_from_user(const void *src, void *dst,
- int len, __wsum sum, int *err_ptr);
-__wsum __csum_partial_copy_to_user(const void *src, void *dst,
- int len, __wsum sum, int *err_ptr);
-/*
- * this is a new version of the above that records errors it finds in *errp,
- * but continues and zeros the rest of the buffer.
- */
-static inline
-__wsum csum_partial_copy_from_user(const void __user *src, void *dst, int len,
- __wsum sum, int *err_ptr)
-{
- might_fault();
- if (uaccess_kernel())
- return __csum_partial_copy_kernel((__force void *)src, dst,
- len, sum, err_ptr);
- else
- return __csum_partial_copy_from_user((__force void *)src, dst,
- len, sum, err_ptr);
-}
+__wsum __csum_partial_copy_from_user(const void __user *src, void *dst, int len);
+__wsum __csum_partial_copy_to_user(const void *src, void __user *dst, int len);
#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER
static inline
-__wsum csum_and_copy_from_user(const void __user *src, void *dst,
- int len, __wsum sum, int *err_ptr)
+__wsum csum_and_copy_from_user(const void __user *src, void *dst, int len)
{
- if (access_ok(src, len))
- return csum_partial_copy_from_user(src, dst, len, sum,
- err_ptr);
- if (len)
- *err_ptr = -EFAULT;
-
- return sum;
+ might_fault();
+ if (!access_ok(src, len))
+ return 0;
+ return __csum_partial_copy_from_user(src, dst, len);
}
/*
*/
#define HAVE_CSUM_COPY_USER
static inline
-__wsum csum_and_copy_to_user(const void *src, void __user *dst, int len,
- __wsum sum, int *err_ptr)
+__wsum csum_and_copy_to_user(const void *src, void __user *dst, int len)
{
might_fault();
- if (access_ok(dst, len)) {
- if (uaccess_kernel())
- return __csum_partial_copy_kernel(src,
- (__force void *)dst,
- len, sum, err_ptr);
- else
- return __csum_partial_copy_to_user(src,
- (__force void *)dst,
- len, sum, err_ptr);
- }
- if (len)
- *err_ptr = -EFAULT;
-
- return (__force __wsum)-1; /* invalid checksum */
+ if (!access_ok(dst, len))
+ return 0;
+ return __csum_partial_copy_to_user(src, dst, len);
}
/*
* the same as csum_partial, but copies from user space (but on MIPS
* we have just one address space, so this is identical to the above)
*/
-__wsum csum_partial_copy_nocheck(const void *src, void *dst,
- int len, __wsum sum);
-#define csum_partial_copy_nocheck csum_partial_copy_nocheck
+#define _HAVE_ARCH_CSUM_AND_COPY
+__wsum __csum_partial_copy_nocheck(const void *src, void *dst, int len);
+static inline __wsum csum_partial_copy_nocheck(const void *src, void *dst, int len)
+{
+ return __csum_partial_copy_nocheck(src, dst, len);
+}
/*
* Fold a partial checksum without adding pseudo headers
typedef struct {
s32 val[2];
} compat_fsid_t;
-typedef s64 compat_s64;
-typedef u64 compat_u64;
struct compat_stat {
compat_dev_t st_dev;
case CPU_34K:
case CPU_1004K:
case CPU_74K:
+ case CPU_1074K:
case CPU_M14KC:
case CPU_M14KEC:
case CPU_INTERAPTIV:
struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *) regs->regs[31];
+ ri->fp = NULL;
/* Replace the return addr with trampoline addr */
regs->regs[31] = (unsigned long)kretprobe_trampoline;
static int __kprobes trampoline_probe_handler(struct kprobe *p,
struct pt_regs *regs)
{
- struct kretprobe_instance *ri = NULL;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address = 0;
- unsigned long trampoline_address = (unsigned long)kretprobe_trampoline;
-
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
-
- /*
- * It is possible to have multiple instances associated with a given
- * task either because an multiple functions in the call path
- * have a return probe installed on them, and/or more than one return
- * return probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always inserted at the head of the list
- * - when multiple return probes are registered for the same
- * function, the first instance's ret_addr will point to the
- * real return address, and all the rest will point to
- * kretprobe_trampoline
- */
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- if (ri->rp && ri->rp->handler)
- ri->rp->handler(ri, regs);
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
- instruction_pointer(regs) = orig_ret_address;
-
- kretprobe_hash_unlock(current, &flags);
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
+ instruction_pointer(regs) = __kretprobe_trampoline_handler(regs,
+ kretprobe_trampoline, NULL);
/*
* By returning a non-zero value, we are telling
* kprobe_handler() that we don't want the post_handler
15 n32 ioctl compat_sys_ioctl
16 n32 pread64 sys_pread64
17 n32 pwrite64 sys_pwrite64
-18 n32 readv compat_sys_readv
-19 n32 writev compat_sys_writev
+18 n32 readv sys_readv
+19 n32 writev sys_writev
20 n32 access sys_access
21 n32 pipe sysm_pipe
22 n32 _newselect compat_sys_select
157 n32 sync sys_sync
158 n32 acct sys_acct
159 n32 settimeofday compat_sys_settimeofday
-160 n32 mount compat_sys_mount
+160 n32 mount sys_mount
161 n32 umount2 sys_umount
162 n32 swapon sys_swapon
163 n32 swapoff sys_swapoff
267 n32 splice sys_splice
268 n32 sync_file_range sys_sync_file_range
269 n32 tee sys_tee
-270 n32 vmsplice compat_sys_vmsplice
+270 n32 vmsplice sys_vmsplice
271 n32 move_pages compat_sys_move_pages
272 n32 set_robust_list compat_sys_set_robust_list
273 n32 get_robust_list compat_sys_get_robust_list
306 n32 syncfs sys_syncfs
307 n32 sendmmsg compat_sys_sendmmsg
308 n32 setns sys_setns
-309 n32 process_vm_readv compat_sys_process_vm_readv
-310 n32 process_vm_writev compat_sys_process_vm_writev
+309 n32 process_vm_readv sys_process_vm_readv
+310 n32 process_vm_writev sys_process_vm_writev
311 n32 kcmp sys_kcmp
312 n32 finit_module sys_finit_module
313 n32 sched_setattr sys_sched_setattr
18 o32 unused18 sys_ni_syscall
19 o32 lseek sys_lseek
20 o32 getpid sys_getpid
-21 o32 mount sys_mount compat_sys_mount
+21 o32 mount sys_mount
22 o32 umount sys_oldumount
23 o32 setuid sys_setuid
24 o32 getuid sys_getuid
142 o32 _newselect sys_select compat_sys_select
143 o32 flock sys_flock
144 o32 msync sys_msync
-145 o32 readv sys_readv compat_sys_readv
-146 o32 writev sys_writev compat_sys_writev
+145 o32 readv sys_readv
+146 o32 writev sys_writev
147 o32 cacheflush sys_cacheflush
148 o32 cachectl sys_cachectl
149 o32 sysmips __sys_sysmips
304 o32 splice sys_splice
305 o32 sync_file_range sys_sync_file_range sys32_sync_file_range
306 o32 tee sys_tee
-307 o32 vmsplice sys_vmsplice compat_sys_vmsplice
+307 o32 vmsplice sys_vmsplice
308 o32 move_pages sys_move_pages compat_sys_move_pages
309 o32 set_robust_list sys_set_robust_list compat_sys_set_robust_list
310 o32 get_robust_list sys_get_robust_list compat_sys_get_robust_list
342 o32 syncfs sys_syncfs
343 o32 sendmmsg sys_sendmmsg compat_sys_sendmmsg
344 o32 setns sys_setns
-345 o32 process_vm_readv sys_process_vm_readv compat_sys_process_vm_readv
-346 o32 process_vm_writev sys_process_vm_writev compat_sys_process_vm_writev
+345 o32 process_vm_readv sys_process_vm_readv
+346 o32 process_vm_writev sys_process_vm_writev
347 o32 kcmp sys_kcmp
348 o32 finit_module sys_finit_module
349 o32 sched_setattr sys_sched_setattr
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
/* These must appear regardless of . */
.gptab.sdata : {
/*
* checksum and copy routines based on memcpy.S
*
- * csum_partial_copy_nocheck(src, dst, len, sum)
- * __csum_partial_copy_kernel(src, dst, len, sum, errp)
+ * csum_partial_copy_nocheck(src, dst, len)
+ * __csum_partial_copy_kernel(src, dst, len)
*
* See "Spec" in memcpy.S for details. Unlike __copy_user, all
* function in this file use the standard calling convention.
#define src a0
#define dst a1
#define len a2
-#define psum a3
#define sum v0
#define odd t8
-#define errptr t9
/*
- * The exception handler for loads requires that:
- * 1- AT contain the address of the byte just past the end of the source
- * of the copy,
- * 2- src_entry <= src < AT, and
- * 3- (dst - src) == (dst_entry - src_entry),
- * The _entry suffix denotes values when __copy_user was called.
- *
- * (1) is set up up by __csum_partial_copy_from_user and maintained by
- * not writing AT in __csum_partial_copy
- * (2) is met by incrementing src by the number of bytes copied
- * (3) is met by not doing loads between a pair of increments of dst and src
- *
- * The exception handlers for stores stores -EFAULT to errptr and return.
- * These handlers do not need to overwrite any data.
+ * All exception handlers simply return 0.
*/
/* Instruction type */
* addr : Address
* handler : Exception handler
*/
-#define EXC(insn, type, reg, addr, handler) \
+#define EXC(insn, type, reg, addr) \
.if \mode == LEGACY_MODE; \
9: insn reg, addr; \
.section __ex_table,"a"; \
- PTR 9b, handler; \
+ PTR 9b, .L_exc; \
.previous; \
/* This is enabled in EVA mode */ \
.else; \
((\to == USEROP) && (type == ST_INSN)); \
9: __BUILD_EVA_INSN(insn##e, reg, addr); \
.section __ex_table,"a"; \
- PTR 9b, handler; \
+ PTR 9b, .L_exc; \
.previous; \
.else; \
/* EVA without exception */ \
#ifdef USE_DOUBLE
#define LOADK ld /* No exception */
-#define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler)
-#define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
-#define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler)
-#define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler)
-#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
-#define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler)
-#define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler)
-#define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler)
+#define LOAD(reg, addr) EXC(ld, LD_INSN, reg, addr)
+#define LOADBU(reg, addr) EXC(lbu, LD_INSN, reg, addr)
+#define LOADL(reg, addr) EXC(ldl, LD_INSN, reg, addr)
+#define LOADR(reg, addr) EXC(ldr, LD_INSN, reg, addr)
+#define STOREB(reg, addr) EXC(sb, ST_INSN, reg, addr)
+#define STOREL(reg, addr) EXC(sdl, ST_INSN, reg, addr)
+#define STORER(reg, addr) EXC(sdr, ST_INSN, reg, addr)
+#define STORE(reg, addr) EXC(sd, ST_INSN, reg, addr)
#define ADD daddu
#define SUB dsubu
#define SRL dsrl
#else
#define LOADK lw /* No exception */
-#define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler)
-#define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
-#define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler)
-#define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler)
-#define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
-#define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler)
-#define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler)
-#define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler)
+#define LOAD(reg, addr) EXC(lw, LD_INSN, reg, addr)
+#define LOADBU(reg, addr) EXC(lbu, LD_INSN, reg, addr)
+#define LOADL(reg, addr) EXC(lwl, LD_INSN, reg, addr)
+#define LOADR(reg, addr) EXC(lwr, LD_INSN, reg, addr)
+#define STOREB(reg, addr) EXC(sb, ST_INSN, reg, addr)
+#define STOREL(reg, addr) EXC(swl, ST_INSN, reg, addr)
+#define STORER(reg, addr) EXC(swr, ST_INSN, reg, addr)
+#define STORE(reg, addr) EXC(sw, ST_INSN, reg, addr)
#define ADD addu
#define SUB subu
#define SRL srl
.set at=v1
#endif
- .macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to, __nocheck
+ .macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to
- PTR_ADDU AT, src, len /* See (1) above. */
- /* initialize __nocheck if this the first time we execute this
- * macro
- */
-#ifdef CONFIG_64BIT
- move errptr, a4
-#else
- lw errptr, 16(sp)
-#endif
- .if \__nocheck == 1
- FEXPORT(csum_partial_copy_nocheck)
- EXPORT_SYMBOL(csum_partial_copy_nocheck)
- .endif
- move sum, zero
+ li sum, -1
move odd, zero
/*
* Note: dst & src may be unaligned, len may be 0
SUB len, 8*NBYTES # subtract here for bgez loop
.align 4
1:
- LOAD(t0, UNIT(0)(src), .Ll_exc\@)
- LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
- LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
- LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
- LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@)
- LOAD(t5, UNIT(5)(src), .Ll_exc_copy\@)
- LOAD(t6, UNIT(6)(src), .Ll_exc_copy\@)
- LOAD(t7, UNIT(7)(src), .Ll_exc_copy\@)
+ LOAD(t0, UNIT(0)(src))
+ LOAD(t1, UNIT(1)(src))
+ LOAD(t2, UNIT(2)(src))
+ LOAD(t3, UNIT(3)(src))
+ LOAD(t4, UNIT(4)(src))
+ LOAD(t5, UNIT(5)(src))
+ LOAD(t6, UNIT(6)(src))
+ LOAD(t7, UNIT(7)(src))
SUB len, len, 8*NBYTES
ADD src, src, 8*NBYTES
- STORE(t0, UNIT(0)(dst), .Ls_exc\@)
+ STORE(t0, UNIT(0)(dst))
ADDC(t0, t1)
- STORE(t1, UNIT(1)(dst), .Ls_exc\@)
+ STORE(t1, UNIT(1)(dst))
ADDC(sum, t0)
- STORE(t2, UNIT(2)(dst), .Ls_exc\@)
+ STORE(t2, UNIT(2)(dst))
ADDC(t2, t3)
- STORE(t3, UNIT(3)(dst), .Ls_exc\@)
+ STORE(t3, UNIT(3)(dst))
ADDC(sum, t2)
- STORE(t4, UNIT(4)(dst), .Ls_exc\@)
+ STORE(t4, UNIT(4)(dst))
ADDC(t4, t5)
- STORE(t5, UNIT(5)(dst), .Ls_exc\@)
+ STORE(t5, UNIT(5)(dst))
ADDC(sum, t4)
- STORE(t6, UNIT(6)(dst), .Ls_exc\@)
+ STORE(t6, UNIT(6)(dst))
ADDC(t6, t7)
- STORE(t7, UNIT(7)(dst), .Ls_exc\@)
+ STORE(t7, UNIT(7)(dst))
ADDC(sum, t6)
.set reorder /* DADDI_WAR */
ADD dst, dst, 8*NBYTES
/*
* len >= 4*NBYTES
*/
- LOAD(t0, UNIT(0)(src), .Ll_exc\@)
- LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
- LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
- LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
+ LOAD(t0, UNIT(0)(src))
+ LOAD(t1, UNIT(1)(src))
+ LOAD(t2, UNIT(2)(src))
+ LOAD(t3, UNIT(3)(src))
SUB len, len, 4*NBYTES
ADD src, src, 4*NBYTES
- STORE(t0, UNIT(0)(dst), .Ls_exc\@)
+ STORE(t0, UNIT(0)(dst))
ADDC(t0, t1)
- STORE(t1, UNIT(1)(dst), .Ls_exc\@)
+ STORE(t1, UNIT(1)(dst))
ADDC(sum, t0)
- STORE(t2, UNIT(2)(dst), .Ls_exc\@)
+ STORE(t2, UNIT(2)(dst))
ADDC(t2, t3)
- STORE(t3, UNIT(3)(dst), .Ls_exc\@)
+ STORE(t3, UNIT(3)(dst))
ADDC(sum, t2)
.set reorder /* DADDI_WAR */
ADD dst, dst, 4*NBYTES
beq rem, len, .Lcopy_bytes\@
nop
1:
- LOAD(t0, 0(src), .Ll_exc\@)
+ LOAD(t0, 0(src))
ADD src, src, NBYTES
SUB len, len, NBYTES
- STORE(t0, 0(dst), .Ls_exc\@)
+ STORE(t0, 0(dst))
ADDC(sum, t0)
.set reorder /* DADDI_WAR */
ADD dst, dst, NBYTES
ADD t1, dst, len # t1 is just past last byte of dst
li bits, 8*NBYTES
SLL rem, len, 3 # rem = number of bits to keep
- LOAD(t0, 0(src), .Ll_exc\@)
+ LOAD(t0, 0(src))
SUB bits, bits, rem # bits = number of bits to discard
SHIFT_DISCARD t0, t0, bits
- STREST(t0, -1(t1), .Ls_exc\@)
+ STREST(t0, -1(t1))
SHIFT_DISCARD_REVERT t0, t0, bits
.set reorder
ADDC(sum, t0)
* Set match = (src and dst have same alignment)
*/
#define match rem
- LDFIRST(t3, FIRST(0)(src), .Ll_exc\@)
+ LDFIRST(t3, FIRST(0)(src))
ADD t2, zero, NBYTES
- LDREST(t3, REST(0)(src), .Ll_exc_copy\@)
+ LDREST(t3, REST(0)(src))
SUB t2, t2, t1 # t2 = number of bytes copied
xor match, t0, t1
- STFIRST(t3, FIRST(0)(dst), .Ls_exc\@)
+ STFIRST(t3, FIRST(0)(dst))
SLL t4, t1, 3 # t4 = number of bits to discard
SHIFT_DISCARD t3, t3, t4
/* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
* It's OK to load FIRST(N+1) before REST(N) because the two addresses
* are to the same unit (unless src is aligned, but it's not).
*/
- LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
- LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@)
+ LDFIRST(t0, FIRST(0)(src))
+ LDFIRST(t1, FIRST(1)(src))
SUB len, len, 4*NBYTES
- LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
- LDREST(t1, REST(1)(src), .Ll_exc_copy\@)
- LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@)
- LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@)
- LDREST(t2, REST(2)(src), .Ll_exc_copy\@)
- LDREST(t3, REST(3)(src), .Ll_exc_copy\@)
+ LDREST(t0, REST(0)(src))
+ LDREST(t1, REST(1)(src))
+ LDFIRST(t2, FIRST(2)(src))
+ LDFIRST(t3, FIRST(3)(src))
+ LDREST(t2, REST(2)(src))
+ LDREST(t3, REST(3)(src))
ADD src, src, 4*NBYTES
#ifdef CONFIG_CPU_SB1
nop # improves slotting
#endif
- STORE(t0, UNIT(0)(dst), .Ls_exc\@)
+ STORE(t0, UNIT(0)(dst))
ADDC(t0, t1)
- STORE(t1, UNIT(1)(dst), .Ls_exc\@)
+ STORE(t1, UNIT(1)(dst))
ADDC(sum, t0)
- STORE(t2, UNIT(2)(dst), .Ls_exc\@)
+ STORE(t2, UNIT(2)(dst))
ADDC(t2, t3)
- STORE(t3, UNIT(3)(dst), .Ls_exc\@)
+ STORE(t3, UNIT(3)(dst))
ADDC(sum, t2)
.set reorder /* DADDI_WAR */
ADD dst, dst, 4*NBYTES
beq rem, len, .Lcopy_bytes\@
nop
1:
- LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
- LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
+ LDFIRST(t0, FIRST(0)(src))
+ LDREST(t0, REST(0)(src))
ADD src, src, NBYTES
SUB len, len, NBYTES
- STORE(t0, 0(dst), .Ls_exc\@)
+ STORE(t0, 0(dst))
ADDC(sum, t0)
.set reorder /* DADDI_WAR */
ADD dst, dst, NBYTES
#endif
move t2, zero # partial word
li t3, SHIFT_START # shift
-/* use .Ll_exc_copy here to return correct sum on fault */
#define COPY_BYTE(N) \
- LOADBU(t0, N(src), .Ll_exc_copy\@); \
+ LOADBU(t0, N(src)); \
SUB len, len, 1; \
- STOREB(t0, N(dst), .Ls_exc\@); \
+ STOREB(t0, N(dst)); \
SLLV t0, t0, t3; \
addu t3, SHIFT_INC; \
beqz len, .Lcopy_bytes_done\@; \
COPY_BYTE(4)
COPY_BYTE(5)
#endif
- LOADBU(t0, NBYTES-2(src), .Ll_exc_copy\@)
+ LOADBU(t0, NBYTES-2(src))
SUB len, len, 1
- STOREB(t0, NBYTES-2(dst), .Ls_exc\@)
+ STOREB(t0, NBYTES-2(dst))
SLLV t0, t0, t3
or t2, t0
.Lcopy_bytes_done\@:
#endif
.set pop
.set reorder
- ADDC32(sum, psum)
jr ra
.set noreorder
+ .endm
-.Ll_exc_copy\@:
- /*
- * Copy bytes from src until faulting load address (or until a
- * lb faults)
- *
- * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
- * may be more than a byte beyond the last address.
- * Hence, the lb below may get an exception.
- *
- * Assumes src < THREAD_BUADDR($28)
- */
- LOADK t0, TI_TASK($28)
- li t2, SHIFT_START
- LOADK t0, THREAD_BUADDR(t0)
-1:
- LOADBU(t1, 0(src), .Ll_exc\@)
- ADD src, src, 1
- sb t1, 0(dst) # can't fault -- we're copy_from_user
- SLLV t1, t1, t2
- addu t2, SHIFT_INC
- ADDC(sum, t1)
- .set reorder /* DADDI_WAR */
- ADD dst, dst, 1
- bne src, t0, 1b
- .set noreorder
-.Ll_exc\@:
- LOADK t0, TI_TASK($28)
- nop
- LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
- nop
- SUB len, AT, t0 # len number of uncopied bytes
- /*
- * Here's where we rely on src and dst being incremented in tandem,
- * See (3) above.
- * dst += (fault addr - src) to put dst at first byte to clear
- */
- ADD dst, t0 # compute start address in a1
- SUB dst, src
- /*
- * Clear len bytes starting at dst. Can't call __bzero because it
- * might modify len. An inefficient loop for these rare times...
- */
- .set reorder /* DADDI_WAR */
- SUB src, len, 1
- beqz len, .Ldone\@
- .set noreorder
-1: sb zero, 0(dst)
- ADD dst, dst, 1
- .set push
- .set noat
-#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
- bnez src, 1b
- SUB src, src, 1
-#else
- li v1, 1
- bnez src, 1b
- SUB src, src, v1
-#endif
- li v1, -EFAULT
- b .Ldone\@
- sw v1, (errptr)
-
-.Ls_exc\@:
- li v0, -1 /* invalid checksum */
- li v1, -EFAULT
+ .set noreorder
+.L_exc:
jr ra
- sw v1, (errptr)
- .set pop
- .endm
+ li v0, 0
-LEAF(__csum_partial_copy_kernel)
-EXPORT_SYMBOL(__csum_partial_copy_kernel)
+FEXPORT(__csum_partial_copy_nocheck)
+EXPORT_SYMBOL(__csum_partial_copy_nocheck)
#ifndef CONFIG_EVA
FEXPORT(__csum_partial_copy_to_user)
EXPORT_SYMBOL(__csum_partial_copy_to_user)
FEXPORT(__csum_partial_copy_from_user)
EXPORT_SYMBOL(__csum_partial_copy_from_user)
#endif
-__BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP 1
-END(__csum_partial_copy_kernel)
+__BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP
#ifdef CONFIG_EVA
LEAF(__csum_partial_copy_to_user)
-__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE KERNELOP USEROP 0
+__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE KERNELOP USEROP
END(__csum_partial_copy_to_user)
LEAF(__csum_partial_copy_from_user)
-__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE USEROP KERNELOP 0
+__BUILD_CSUM_PARTIAL_COPY_USER EVA_MODE USEROP KERNELOP
END(__csum_partial_copy_from_user)
#endif
endif
endif
+# Some -march= flags enable MMI instructions, and GCC complains about that
+# support being enabled alongside -msoft-float. Thus explicitly disable MMI.
+cflags-y += $(call cc-option,-mno-loongson-mmi)
+
#
# Loongson Machines' Support
#
if (res)
goto fault;
- set_fpr64(current->thread.fpu.fpr,
- insn.loongson3_lswc2_format.rt, value);
- set_fpr64(current->thread.fpu.fpr,
- insn.loongson3_lswc2_format.rq, value_next);
+ set_fpr64(¤t->thread.fpu.fpr[insn.loongson3_lswc2_format.rt], 0, value);
+ set_fpr64(¤t->thread.fpu.fpr[insn.loongson3_lswc2_format.rq], 0, value_next);
compute_return_epc(regs);
own_fpu(1);
}
goto sigbus;
lose_fpu(1);
- value_next = get_fpr64(current->thread.fpu.fpr,
- insn.loongson3_lswc2_format.rq);
+ value_next = get_fpr64(¤t->thread.fpu.fpr[insn.loongson3_lswc2_format.rq], 0);
StoreDW(addr + 8, value_next, res);
if (res)
goto fault;
- value = get_fpr64(current->thread.fpu.fpr,
- insn.loongson3_lswc2_format.rt);
+ value = get_fpr64(¤t->thread.fpu.fpr[insn.loongson3_lswc2_format.rt], 0);
StoreDW(addr, value, res);
if (res)
if (res)
goto fault;
- set_fpr64(current->thread.fpu.fpr,
- insn.loongson3_lsdc2_format.rt, value);
+ set_fpr64(¤t->thread.fpu.fpr[insn.loongson3_lsdc2_format.rt], 0, value);
compute_return_epc(regs);
own_fpu(1);
if (res)
goto fault;
- set_fpr64(current->thread.fpu.fpr,
- insn.loongson3_lsdc2_format.rt, value);
+ set_fpr64(¤t->thread.fpu.fpr[insn.loongson3_lsdc2_format.rt], 0, value);
compute_return_epc(regs);
own_fpu(1);
break;
goto sigbus;
lose_fpu(1);
- value = get_fpr64(current->thread.fpu.fpr,
- insn.loongson3_lsdc2_format.rt);
+ value = get_fpr64(¤t->thread.fpu.fpr[insn.loongson3_lsdc2_format.rt], 0);
StoreW(addr, value, res);
if (res)
goto sigbus;
lose_fpu(1);
- value = get_fpr64(current->thread.fpu.fpr,
- insn.loongson3_lsdc2_format.rt);
+ value = get_fpr64(¤t->thread.fpu.fpr[insn.loongson3_lsdc2_format.rt], 0);
StoreDW(addr, value, res);
if (res)
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
DISCARDS
}
/* Take these from lib/checksum.c */
extern __wsum csum_partial(const void *buff, int len, __wsum sum);
-__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len,
- __wsum sum);
-#define csum_partial_copy_nocheck csum_partial_copy_nocheck
-
extern __sum16 ip_fast_csum(const void *iph, unsigned int ihl);
extern __sum16 ip_compute_csum(const void *buff, int len);
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
DISCARDS
}
/* Throw in the debugging sections */
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
/* Sections to be discarded -- must be last */
DISCARDS
}
STABS_DEBUG
+ ELF_DETAILS
.note 0 : { *(.note) }
/* Sections to be discarded */
*/
extern __wsum csum_partial(const void *, int, __wsum);
-/*
- * The same as csum_partial, but copies from src while it checksums.
- *
- * Here even more important to align src and dst on a 32-bit (or even
- * better 64-bit) boundary
- */
-extern __wsum csum_partial_copy_nocheck(const void *, void *, int, __wsum);
-
/*
* Optimized for IP headers, which always checksum on 4 octet boundaries.
*
return csum_fold(sum);
}
-/*
- * Copy and checksum to user
- */
-#define HAVE_CSUM_COPY_USER
-static __inline__ __wsum csum_and_copy_to_user(const void *src,
- void __user *dst,
- int len, __wsum sum,
- int *err_ptr)
-{
- /* code stolen from include/asm-mips64 */
- sum = csum_partial(src, len, sum);
-
- if (copy_to_user(dst, src, len)) {
- *err_ptr = -EFAULT;
- return (__force __wsum)-1;
- }
-
- return sum;
-}
-
#endif
typedef u16 compat_nlink_t;
typedef u16 compat_ipc_pid_t;
typedef u32 compat_caddr_t;
-typedef s64 compat_s64;
-typedef u64 compat_u64;
struct compat_stat {
compat_dev_t st_dev; /* dev_t is 32 bits on parisc */
static int __kprobes trampoline_probe_handler(struct kprobe *p,
struct pt_regs *regs)
{
- struct kretprobe_instance *ri = NULL;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address = 0;
- unsigned long trampoline_address = (unsigned long)trampoline_p.addr;
- kprobe_opcode_t *correct_ret_addr = NULL;
-
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
-
- /*
- * It is possible to have multiple instances associated with a given
- * task either because multiple functions in the call path have
- * a return probe installed on them, and/or more than one return
- * probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always inserted at the head of the list
- * - when multiple return probes are registered for the same
- * function, the first instance's ret_addr will point to the
- * real return address, and all the rest will point to
- * kretprobe_trampoline
- */
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- orig_ret_address = (unsigned long)ri->ret_addr;
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
-
- correct_ret_addr = ri->ret_addr;
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- if (ri->rp && ri->rp->handler) {
- __this_cpu_write(current_kprobe, &ri->rp->kp);
- get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
- ri->ret_addr = correct_ret_addr;
- ri->rp->handler(ri, regs);
- __this_cpu_write(current_kprobe, NULL);
- }
-
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
+ unsigned long orig_ret_address;
- kretprobe_hash_unlock(current, &flags);
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
+ orig_ret_address = __kretprobe_trampoline_handler(regs, trampoline_p.addr, NULL);
instruction_pointer_set(regs, orig_ret_address);
+
return 1;
}
struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *)regs->gr[2];
+ ri->fp = NULL;
/* Replace the return addr with trampoline addr. */
regs->gr[2] = (unsigned long)trampoline_p.addr;
18 common stat sys_newstat compat_sys_newstat
19 common lseek sys_lseek compat_sys_lseek
20 common getpid sys_getpid
-21 common mount sys_mount compat_sys_mount
+21 common mount sys_mount
22 common bind sys_bind
23 common setuid sys_setuid
24 common getuid sys_getuid
142 common _newselect sys_select compat_sys_select
143 common flock sys_flock
144 common msync sys_msync
-145 common readv sys_readv compat_sys_readv
-146 common writev sys_writev compat_sys_writev
+145 common readv sys_readv
+146 common writev sys_writev
147 common getsid sys_getsid
148 common fdatasync sys_fdatasync
149 common _sysctl sys_ni_syscall
292 32 sync_file_range parisc_sync_file_range
292 64 sync_file_range sys_sync_file_range
293 common tee sys_tee
-294 common vmsplice sys_vmsplice compat_sys_vmsplice
+294 common vmsplice sys_vmsplice
295 common move_pages sys_move_pages compat_sys_move_pages
296 common getcpu sys_getcpu
297 common epoll_pwait sys_epoll_pwait compat_sys_epoll_pwait
327 common syncfs sys_syncfs
328 common setns sys_setns
329 common sendmmsg sys_sendmmsg compat_sys_sendmmsg
-330 common process_vm_readv sys_process_vm_readv compat_sys_process_vm_readv
-331 common process_vm_writev sys_process_vm_writev compat_sys_process_vm_writev
+330 common process_vm_readv sys_process_vm_readv
+331 common process_vm_writev sys_process_vm_writev
332 common kcmp sys_kcmp
333 common finit_module sys_finit_module
334 common sched_setattr sys_sched_setattr
_end = . ;
STABS_DEBUG
+ ELF_DETAILS
.note 0 : { *(.note) }
/* Sections to be discarded */
}
EXPORT_SYMBOL(csum_partial);
-
-/*
- * copy while checksumming, otherwise like csum_partial
- */
-__wsum csum_partial_copy_nocheck(const void *src, void *dst,
- int len, __wsum sum)
-{
- /*
- * It's 2:30 am and I don't feel like doing it real ...
- * This is lots slower than the real thing (tm)
- */
- sum = csum_partial(src, len, sum);
- memcpy(dst, src, len);
-
- return sum;
-}
-EXPORT_SYMBOL(csum_partial_copy_nocheck);
select ARCH_HAS_STRICT_KERNEL_RWX if (PPC32 && !HIBERNATION)
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
select ARCH_HAS_UACCESS_FLUSHCACHE
- select ARCH_HAS_UACCESS_MCSAFE if PPC64
+ select ARCH_HAS_COPY_MC if PPC64
select ARCH_HAS_UBSAN_SANITIZE_ALL
select ARCH_HAVE_NMI_SAFE_CMPXCHG
select ARCH_KEEP_MEMBLOCK
select OLD_SIGACTION if PPC32
select OLD_SIGSUSPEND
select PCI_DOMAINS if PCI
+ select PCI_MSI_ARCH_FALLBACKS if PCI_MSI
select PCI_SYSCALL if PCI
select PPC_DAWR if PPC64
select RTC_LIB
#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
+#include <linux/random.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/cpufeature.h>
static int __init crc_test_init(void)
{
u16 crc16 = 0, verify16 = 0;
- u32 crc32 = 0, verify32 = 0;
__le32 verify32le = 0;
unsigned char *data;
+ u32 verify32 = 0;
unsigned long i;
+ __le32 crc32;
int ret;
struct crypto_shash *crct10dif_tfm;
crypto_shash_final(crc32c_shash, (u8 *)(&crc32));
verify32 = le32_to_cpu(verify32le);
verify32le = ~cpu_to_le32(__crc32c_le(~verify32, data+offset, len));
- if (crc32 != (u32)verify32le) {
+ if (crc32 != verify32le) {
pr_err("FAILURE in CRC32: got 0x%08x expected 0x%08x (len %lu)\n",
crc32, verify32, len);
break;
* Like csum_partial, this must be called with even lengths,
* except for the last fragment.
*/
-extern __wsum csum_partial_copy_generic(const void *src, void *dst,
- int len, __wsum sum,
- int *src_err, int *dst_err);
+extern __wsum csum_partial_copy_generic(const void *src, void *dst, int len);
#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER
extern __wsum csum_and_copy_from_user(const void __user *src, void *dst,
- int len, __wsum sum, int *err_ptr);
+ int len);
#define HAVE_CSUM_COPY_USER
extern __wsum csum_and_copy_to_user(const void *src, void __user *dst,
- int len, __wsum sum, int *err_ptr);
+ int len);
-#define csum_partial_copy_nocheck(src, dst, len, sum) \
- csum_partial_copy_generic((src), (dst), (len), (sum), NULL, NULL)
+#define _HAVE_ARCH_CSUM_AND_COPY
+#define csum_partial_copy_nocheck(src, dst, len) \
+ csum_partial_copy_generic((src), (dst), (len))
/*
typedef u16 compat_ipc_pid_t;
typedef u32 compat_caddr_t;
typedef __kernel_fsid_t compat_fsid_t;
-typedef s64 compat_s64;
-typedef u64 compat_u64;
struct compat_stat {
compat_dev_t st_dev;
#ifndef CONFIG_KASAN
#define __HAVE_ARCH_MEMSET32
#define __HAVE_ARCH_MEMSET64
-#define __HAVE_ARCH_MEMCPY_MCSAFE
-extern int memcpy_mcsafe(void *dst, const void *src, __kernel_size_t sz);
extern void *__memset16(uint16_t *, uint16_t v, __kernel_size_t);
extern void *__memset32(uint32_t *, uint32_t v, __kernel_size_t);
extern void *__memset64(uint64_t *, uint64_t v, __kernel_size_t);
extern unsigned long __copy_tofrom_user(void __user *to,
const void __user *from, unsigned long size);
+#ifdef CONFIG_ARCH_HAS_COPY_MC
+unsigned long __must_check
+copy_mc_generic(void *to, const void *from, unsigned long size);
+
+static inline unsigned long __must_check
+copy_mc_to_kernel(void *to, const void *from, unsigned long size)
+{
+ return copy_mc_generic(to, from, size);
+}
+#define copy_mc_to_kernel copy_mc_to_kernel
+
+static inline unsigned long __must_check
+copy_mc_to_user(void __user *to, const void *from, unsigned long n)
+{
+ if (likely(check_copy_size(from, n, true))) {
+ if (access_ok(to, n)) {
+ allow_write_to_user(to, n);
+ n = copy_mc_generic((void *)to, from, n);
+ prevent_write_to_user(to, n);
+ }
+ }
+
+ return n;
+}
+#endif
+
#ifdef __powerpc64__
static inline unsigned long
raw_copy_in_user(void __user *to, const void __user *from, unsigned long n)
return ret;
}
-static __always_inline unsigned long __must_check
-copy_to_user_mcsafe(void __user *to, const void *from, unsigned long n)
-{
- if (likely(check_copy_size(from, n, true))) {
- if (access_ok(to, n)) {
- allow_write_to_user(to, n);
- n = memcpy_mcsafe((void *)to, from, n);
- prevent_write_to_user(to, n);
- }
- }
-
- return n;
-}
-
unsigned long __arch_clear_user(void __user *addr, unsigned long size);
static inline unsigned long clear_user(void __user *addr, unsigned long size)
void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *)regs->link;
+ ri->fp = NULL;
/* Replace the return addr with trampoline addr */
regs->link = (unsigned long)kretprobe_trampoline;
*/
static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
{
- struct kretprobe_instance *ri = NULL;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address = 0;
- unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline;
-
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
-
- /*
- * It is possible to have multiple instances associated with a given
- * task either because an multiple functions in the call path
- * have a return probe installed on them, and/or more than one return
- * return probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always inserted at the head of the list
- * - when multiple return probes are registered for the same
- * function, the first instance's ret_addr will point to the
- * real return address, and all the rest will point to
- * kretprobe_trampoline
- */
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- if (ri->rp && ri->rp->handler)
- ri->rp->handler(ri, regs);
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
+ unsigned long orig_ret_address;
+ orig_ret_address = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL);
/*
* We get here through one of two paths:
* 1. by taking a trap -> kprobe_handler() -> here
regs->nip = orig_ret_address - 4;
regs->link = orig_ret_address;
- kretprobe_hash_unlock(current, &flags);
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
-
return 0;
}
NOKPROBE_SYMBOL(trampoline_probe_handler);
18 spu oldstat sys_ni_syscall
19 common lseek sys_lseek compat_sys_lseek
20 common getpid sys_getpid
-21 nospu mount sys_mount compat_sys_mount
+21 nospu mount sys_mount
22 32 umount sys_oldumount
22 64 umount sys_ni_syscall
22 spu umount sys_ni_syscall
142 common _newselect sys_select compat_sys_select
143 common flock sys_flock
144 common msync sys_msync
-145 common readv sys_readv compat_sys_readv
-146 common writev sys_writev compat_sys_writev
+145 common readv sys_readv
+146 common writev sys_writev
147 common getsid sys_getsid
148 common fdatasync sys_fdatasync
149 nospu _sysctl sys_ni_syscall
282 common unshare sys_unshare
283 common splice sys_splice
284 common tee sys_tee
-285 common vmsplice sys_vmsplice compat_sys_vmsplice
+285 common vmsplice sys_vmsplice
286 common openat sys_openat compat_sys_openat
287 common mkdirat sys_mkdirat
288 common mknodat sys_mknodat
348 common syncfs sys_syncfs
349 common sendmmsg sys_sendmmsg compat_sys_sendmmsg
350 common setns sys_setns
-351 nospu process_vm_readv sys_process_vm_readv compat_sys_process_vm_readv
-352 nospu process_vm_writev sys_process_vm_writev compat_sys_process_vm_writev
+351 nospu process_vm_readv sys_process_vm_readv
+352 nospu process_vm_writev sys_process_vm_writev
353 nospu finit_module sys_finit_module
354 nospu kcmp sys_kcmp
355 common sched_setattr sys_sched_setattr
PROVIDE32 (end = .);
STABS_DEBUG
-
DWARF_DEBUG
+ ELF_DETAILS
DISCARDS
/DISCARD/ : {
memcpy_power7.o
obj64-y += copypage_64.o copyuser_64.o mem_64.o hweight_64.o \
- memcpy_64.o memcpy_mcsafe_64.o
+ memcpy_64.o copy_mc_64.o
ifndef CONFIG_PPC_QUEUED_SPINLOCKS
obj64-$(CONFIG_SMP) += locks.o
/*
* Computes the checksum of a memory block at src, length len,
- * and adds in "sum" (32-bit), while copying the block to dst.
- * If an access exception occurs on src or dst, it stores -EFAULT
- * to *src_err or *dst_err respectively, and (for an error on
- * src) zeroes the rest of dst.
+ * and adds in 0xffffffff, while copying the block to dst.
+ * If an access exception occurs it returns zero.
*
- * csum_partial_copy_generic(src, dst, len, sum, src_err, dst_err)
+ * csum_partial_copy_generic(src, dst, len)
*/
#define CSUM_COPY_16_BYTES_WITHEX(n) \
8 ## n ## 0: \
adde r12,r12,r10
#define CSUM_COPY_16_BYTES_EXCODE(n) \
- EX_TABLE(8 ## n ## 0b, src_error); \
- EX_TABLE(8 ## n ## 1b, src_error); \
- EX_TABLE(8 ## n ## 2b, src_error); \
- EX_TABLE(8 ## n ## 3b, src_error); \
- EX_TABLE(8 ## n ## 4b, dst_error); \
- EX_TABLE(8 ## n ## 5b, dst_error); \
- EX_TABLE(8 ## n ## 6b, dst_error); \
- EX_TABLE(8 ## n ## 7b, dst_error);
+ EX_TABLE(8 ## n ## 0b, fault); \
+ EX_TABLE(8 ## n ## 1b, fault); \
+ EX_TABLE(8 ## n ## 2b, fault); \
+ EX_TABLE(8 ## n ## 3b, fault); \
+ EX_TABLE(8 ## n ## 4b, fault); \
+ EX_TABLE(8 ## n ## 5b, fault); \
+ EX_TABLE(8 ## n ## 6b, fault); \
+ EX_TABLE(8 ## n ## 7b, fault);
.text
.stabs "arch/powerpc/lib/",N_SO,0,0,0f
CACHELINE_MASK = (L1_CACHE_BYTES-1)
_GLOBAL(csum_partial_copy_generic)
- stwu r1,-16(r1)
- stw r7,12(r1)
- stw r8,8(r1)
-
- addic r12,r6,0
+ li r12,-1
+ addic r0,r0,0 /* clear carry */
addi r6,r4,-4
neg r0,r4
addi r4,r3,-4
rlwinm r3,r3,8,0,31 /* odd destination address: rotate one byte */
blr
-/* read fault */
-src_error:
- lwz r7,12(r1)
- addi r1,r1,16
- cmpwi cr0,r7,0
- beqlr
- li r0,-EFAULT
- stw r0,0(r7)
- blr
-/* write fault */
-dst_error:
- lwz r8,8(r1)
- addi r1,r1,16
- cmpwi cr0,r8,0
- beqlr
- li r0,-EFAULT
- stw r0,0(r8)
+fault:
+ li r3,0
blr
- EX_TABLE(70b, src_error);
- EX_TABLE(71b, dst_error);
- EX_TABLE(72b, src_error);
- EX_TABLE(73b, dst_error);
- EX_TABLE(54b, dst_error);
+ EX_TABLE(70b, fault);
+ EX_TABLE(71b, fault);
+ EX_TABLE(72b, fault);
+ EX_TABLE(73b, fault);
+ EX_TABLE(54b, fault);
/*
* this stuff handles faults in the cacheline loop and branches to either
- * src_error (if in read part) or dst_error (if in write part)
+ * fault (if in read part) or fault (if in write part)
*/
CSUM_COPY_16_BYTES_EXCODE(0)
#if L1_CACHE_BYTES >= 32
#endif
#endif
- EX_TABLE(30b, src_error);
- EX_TABLE(31b, dst_error);
- EX_TABLE(40b, src_error);
- EX_TABLE(41b, dst_error);
- EX_TABLE(50b, src_error);
- EX_TABLE(51b, dst_error);
+ EX_TABLE(30b, fault);
+ EX_TABLE(31b, fault);
+ EX_TABLE(40b, fault);
+ EX_TABLE(41b, fault);
+ EX_TABLE(50b, fault);
+ EX_TABLE(51b, fault);
EXPORT_SYMBOL(csum_partial_copy_generic)
.macro srcnr
100:
- EX_TABLE(100b,.Lsrc_error_nr)
+ EX_TABLE(100b,.Lerror_nr)
.endm
.macro source
150:
- EX_TABLE(150b,.Lsrc_error)
+ EX_TABLE(150b,.Lerror)
.endm
.macro dstnr
200:
- EX_TABLE(200b,.Ldest_error_nr)
+ EX_TABLE(200b,.Lerror_nr)
.endm
.macro dest
250:
- EX_TABLE(250b,.Ldest_error)
+ EX_TABLE(250b,.Lerror)
.endm
/*
* Computes the checksum of a memory block at src, length len,
- * and adds in "sum" (32-bit), while copying the block to dst.
- * If an access exception occurs on src or dst, it stores -EFAULT
- * to *src_err or *dst_err respectively. The caller must take any action
- * required in this case (zeroing memory, recalculating partial checksum etc).
+ * and adds in 0xffffffff (32-bit), while copying the block to dst.
+ * If an access exception occurs, it returns 0.
*
- * csum_partial_copy_generic(r3=src, r4=dst, r5=len, r6=sum, r7=src_err, r8=dst_err)
+ * csum_partial_copy_generic(r3=src, r4=dst, r5=len)
*/
_GLOBAL(csum_partial_copy_generic)
+ li r6,-1
addic r0,r6,0 /* clear carry */
srdi. r6,r5,3 /* less than 8 bytes? */
srdi r3,r3,32
blr
-.Lsrc_error:
+.Lerror:
ld r14,STK_REG(R14)(r1)
ld r15,STK_REG(R15)(r1)
ld r16,STK_REG(R16)(r1)
addi r1,r1,STACKFRAMESIZE
-.Lsrc_error_nr:
- cmpdi 0,r7,0
- beqlr
- li r6,-EFAULT
- stw r6,0(r7)
+.Lerror_nr:
+ li r3,0
blr
-.Ldest_error:
- ld r14,STK_REG(R14)(r1)
- ld r15,STK_REG(R15)(r1)
- ld r16,STK_REG(R16)(r1)
- addi r1,r1,STACKFRAMESIZE
-.Ldest_error_nr:
- cmpdi 0,r8,0
- beqlr
- li r6,-EFAULT
- stw r6,0(r8)
- blr
EXPORT_SYMBOL(csum_partial_copy_generic)
/*
#include <linux/uaccess.h>
__wsum csum_and_copy_from_user(const void __user *src, void *dst,
- int len, __wsum sum, int *err_ptr)
+ int len)
{
- unsigned int csum;
+ __wsum csum;
might_sleep();
- allow_read_from_user(src, len);
-
- *err_ptr = 0;
-
- if (!len) {
- csum = 0;
- goto out;
- }
- if (unlikely((len < 0) || !access_ok(src, len))) {
- *err_ptr = -EFAULT;
- csum = (__force unsigned int)sum;
- goto out;
- }
+ if (unlikely(!access_ok(src, len)))
+ return 0;
- csum = csum_partial_copy_generic((void __force *)src, dst,
- len, sum, err_ptr, NULL);
-
- if (unlikely(*err_ptr)) {
- int missing = __copy_from_user(dst, src, len);
-
- if (missing) {
- memset(dst + len - missing, 0, missing);
- *err_ptr = -EFAULT;
- } else {
- *err_ptr = 0;
- }
+ allow_read_from_user(src, len);
- csum = csum_partial(dst, len, sum);
- }
+ csum = csum_partial_copy_generic((void __force *)src, dst, len);
-out:
prevent_read_from_user(src, len);
- return (__force __wsum)csum;
+ return csum;
}
EXPORT_SYMBOL(csum_and_copy_from_user);
-__wsum csum_and_copy_to_user(const void *src, void __user *dst, int len,
- __wsum sum, int *err_ptr)
+__wsum csum_and_copy_to_user(const void *src, void __user *dst, int len)
{
- unsigned int csum;
+ __wsum csum;
might_sleep();
- allow_write_to_user(dst, len);
-
- *err_ptr = 0;
-
- if (!len) {
- csum = 0;
- goto out;
- }
+ if (unlikely(!access_ok(dst, len)))
+ return 0;
- if (unlikely((len < 0) || !access_ok(dst, len))) {
- *err_ptr = -EFAULT;
- csum = -1; /* invalid checksum */
- goto out;
- }
-
- csum = csum_partial_copy_generic(src, (void __force *)dst,
- len, sum, NULL, err_ptr);
-
- if (unlikely(*err_ptr)) {
- csum = csum_partial(src, len, sum);
+ allow_write_to_user(dst, len);
- if (copy_to_user(dst, src, len)) {
- *err_ptr = -EFAULT;
- csum = -1; /* invalid checksum */
- }
- }
+ csum = csum_partial_copy_generic(src, (void __force *)dst, len);
-out:
prevent_write_to_user(dst, len);
- return (__force __wsum)csum;
+ return csum;
}
EXPORT_SYMBOL(csum_and_copy_to_user);
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) IBM Corporation, 2011
+ * Derived from copyuser_power7.s by Anton Blanchard <anton@au.ibm.com>
+ * Author - Balbir Singh <bsingharora@gmail.com>
+ */
+#include <asm/ppc_asm.h>
+#include <asm/errno.h>
+#include <asm/export.h>
+
+ .macro err1
+100:
+ EX_TABLE(100b,.Ldo_err1)
+ .endm
+
+ .macro err2
+200:
+ EX_TABLE(200b,.Ldo_err2)
+ .endm
+
+ .macro err3
+300: EX_TABLE(300b,.Ldone)
+ .endm
+
+.Ldo_err2:
+ ld r22,STK_REG(R22)(r1)
+ ld r21,STK_REG(R21)(r1)
+ ld r20,STK_REG(R20)(r1)
+ ld r19,STK_REG(R19)(r1)
+ ld r18,STK_REG(R18)(r1)
+ ld r17,STK_REG(R17)(r1)
+ ld r16,STK_REG(R16)(r1)
+ ld r15,STK_REG(R15)(r1)
+ ld r14,STK_REG(R14)(r1)
+ addi r1,r1,STACKFRAMESIZE
+.Ldo_err1:
+ /* Do a byte by byte copy to get the exact remaining size */
+ mtctr r7
+46:
+err3; lbz r0,0(r4)
+ addi r4,r4,1
+err3; stb r0,0(r3)
+ addi r3,r3,1
+ bdnz 46b
+ li r3,0
+ blr
+
+.Ldone:
+ mfctr r3
+ blr
+
+
+_GLOBAL(copy_mc_generic)
+ mr r7,r5
+ cmpldi r5,16
+ blt .Lshort_copy
+
+.Lcopy:
+ /* Get the source 8B aligned */
+ neg r6,r4
+ mtocrf 0x01,r6
+ clrldi r6,r6,(64-3)
+
+ bf cr7*4+3,1f
+err1; lbz r0,0(r4)
+ addi r4,r4,1
+err1; stb r0,0(r3)
+ addi r3,r3,1
+ subi r7,r7,1
+
+1: bf cr7*4+2,2f
+err1; lhz r0,0(r4)
+ addi r4,r4,2
+err1; sth r0,0(r3)
+ addi r3,r3,2
+ subi r7,r7,2
+
+2: bf cr7*4+1,3f
+err1; lwz r0,0(r4)
+ addi r4,r4,4
+err1; stw r0,0(r3)
+ addi r3,r3,4
+ subi r7,r7,4
+
+3: sub r5,r5,r6
+ cmpldi r5,128
+
+ mflr r0
+ stdu r1,-STACKFRAMESIZE(r1)
+ std r14,STK_REG(R14)(r1)
+ std r15,STK_REG(R15)(r1)
+ std r16,STK_REG(R16)(r1)
+ std r17,STK_REG(R17)(r1)
+ std r18,STK_REG(R18)(r1)
+ std r19,STK_REG(R19)(r1)
+ std r20,STK_REG(R20)(r1)
+ std r21,STK_REG(R21)(r1)
+ std r22,STK_REG(R22)(r1)
+ std r0,STACKFRAMESIZE+16(r1)
+
+ blt 5f
+ srdi r6,r5,7
+ mtctr r6
+
+ /* Now do cacheline (128B) sized loads and stores. */
+ .align 5
+4:
+err2; ld r0,0(r4)
+err2; ld r6,8(r4)
+err2; ld r8,16(r4)
+err2; ld r9,24(r4)
+err2; ld r10,32(r4)
+err2; ld r11,40(r4)
+err2; ld r12,48(r4)
+err2; ld r14,56(r4)
+err2; ld r15,64(r4)
+err2; ld r16,72(r4)
+err2; ld r17,80(r4)
+err2; ld r18,88(r4)
+err2; ld r19,96(r4)
+err2; ld r20,104(r4)
+err2; ld r21,112(r4)
+err2; ld r22,120(r4)
+ addi r4,r4,128
+err2; std r0,0(r3)
+err2; std r6,8(r3)
+err2; std r8,16(r3)
+err2; std r9,24(r3)
+err2; std r10,32(r3)
+err2; std r11,40(r3)
+err2; std r12,48(r3)
+err2; std r14,56(r3)
+err2; std r15,64(r3)
+err2; std r16,72(r3)
+err2; std r17,80(r3)
+err2; std r18,88(r3)
+err2; std r19,96(r3)
+err2; std r20,104(r3)
+err2; std r21,112(r3)
+err2; std r22,120(r3)
+ addi r3,r3,128
+ subi r7,r7,128
+ bdnz 4b
+
+ clrldi r5,r5,(64-7)
+
+ /* Up to 127B to go */
+5: srdi r6,r5,4
+ mtocrf 0x01,r6
+
+6: bf cr7*4+1,7f
+err2; ld r0,0(r4)
+err2; ld r6,8(r4)
+err2; ld r8,16(r4)
+err2; ld r9,24(r4)
+err2; ld r10,32(r4)
+err2; ld r11,40(r4)
+err2; ld r12,48(r4)
+err2; ld r14,56(r4)
+ addi r4,r4,64
+err2; std r0,0(r3)
+err2; std r6,8(r3)
+err2; std r8,16(r3)
+err2; std r9,24(r3)
+err2; std r10,32(r3)
+err2; std r11,40(r3)
+err2; std r12,48(r3)
+err2; std r14,56(r3)
+ addi r3,r3,64
+ subi r7,r7,64
+
+7: ld r14,STK_REG(R14)(r1)
+ ld r15,STK_REG(R15)(r1)
+ ld r16,STK_REG(R16)(r1)
+ ld r17,STK_REG(R17)(r1)
+ ld r18,STK_REG(R18)(r1)
+ ld r19,STK_REG(R19)(r1)
+ ld r20,STK_REG(R20)(r1)
+ ld r21,STK_REG(R21)(r1)
+ ld r22,STK_REG(R22)(r1)
+ addi r1,r1,STACKFRAMESIZE
+
+ /* Up to 63B to go */
+ bf cr7*4+2,8f
+err1; ld r0,0(r4)
+err1; ld r6,8(r4)
+err1; ld r8,16(r4)
+err1; ld r9,24(r4)
+ addi r4,r4,32
+err1; std r0,0(r3)
+err1; std r6,8(r3)
+err1; std r8,16(r3)
+err1; std r9,24(r3)
+ addi r3,r3,32
+ subi r7,r7,32
+
+ /* Up to 31B to go */
+8: bf cr7*4+3,9f
+err1; ld r0,0(r4)
+err1; ld r6,8(r4)
+ addi r4,r4,16
+err1; std r0,0(r3)
+err1; std r6,8(r3)
+ addi r3,r3,16
+ subi r7,r7,16
+
+9: clrldi r5,r5,(64-4)
+
+ /* Up to 15B to go */
+.Lshort_copy:
+ mtocrf 0x01,r5
+ bf cr7*4+0,12f
+err1; lwz r0,0(r4) /* Less chance of a reject with word ops */
+err1; lwz r6,4(r4)
+ addi r4,r4,8
+err1; stw r0,0(r3)
+err1; stw r6,4(r3)
+ addi r3,r3,8
+ subi r7,r7,8
+
+12: bf cr7*4+1,13f
+err1; lwz r0,0(r4)
+ addi r4,r4,4
+err1; stw r0,0(r3)
+ addi r3,r3,4
+ subi r7,r7,4
+
+13: bf cr7*4+2,14f
+err1; lhz r0,0(r4)
+ addi r4,r4,2
+err1; sth r0,0(r3)
+ addi r3,r3,2
+ subi r7,r7,2
+
+14: bf cr7*4+3,15f
+err1; lbz r0,0(r4)
+err1; stb r0,0(r3)
+
+15: li r3,0
+ blr
+
+EXPORT_SYMBOL_GPL(copy_mc_generic);
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (C) IBM Corporation, 2011
- * Derived from copyuser_power7.s by Anton Blanchard <anton@au.ibm.com>
- * Author - Balbir Singh <bsingharora@gmail.com>
- */
-#include <asm/ppc_asm.h>
-#include <asm/errno.h>
-#include <asm/export.h>
-
- .macro err1
-100:
- EX_TABLE(100b,.Ldo_err1)
- .endm
-
- .macro err2
-200:
- EX_TABLE(200b,.Ldo_err2)
- .endm
-
- .macro err3
-300: EX_TABLE(300b,.Ldone)
- .endm
-
-.Ldo_err2:
- ld r22,STK_REG(R22)(r1)
- ld r21,STK_REG(R21)(r1)
- ld r20,STK_REG(R20)(r1)
- ld r19,STK_REG(R19)(r1)
- ld r18,STK_REG(R18)(r1)
- ld r17,STK_REG(R17)(r1)
- ld r16,STK_REG(R16)(r1)
- ld r15,STK_REG(R15)(r1)
- ld r14,STK_REG(R14)(r1)
- addi r1,r1,STACKFRAMESIZE
-.Ldo_err1:
- /* Do a byte by byte copy to get the exact remaining size */
- mtctr r7
-46:
-err3; lbz r0,0(r4)
- addi r4,r4,1
-err3; stb r0,0(r3)
- addi r3,r3,1
- bdnz 46b
- li r3,0
- blr
-
-.Ldone:
- mfctr r3
- blr
-
-
-_GLOBAL(memcpy_mcsafe)
- mr r7,r5
- cmpldi r5,16
- blt .Lshort_copy
-
-.Lcopy:
- /* Get the source 8B aligned */
- neg r6,r4
- mtocrf 0x01,r6
- clrldi r6,r6,(64-3)
-
- bf cr7*4+3,1f
-err1; lbz r0,0(r4)
- addi r4,r4,1
-err1; stb r0,0(r3)
- addi r3,r3,1
- subi r7,r7,1
-
-1: bf cr7*4+2,2f
-err1; lhz r0,0(r4)
- addi r4,r4,2
-err1; sth r0,0(r3)
- addi r3,r3,2
- subi r7,r7,2
-
-2: bf cr7*4+1,3f
-err1; lwz r0,0(r4)
- addi r4,r4,4
-err1; stw r0,0(r3)
- addi r3,r3,4
- subi r7,r7,4
-
-3: sub r5,r5,r6
- cmpldi r5,128
-
- mflr r0
- stdu r1,-STACKFRAMESIZE(r1)
- std r14,STK_REG(R14)(r1)
- std r15,STK_REG(R15)(r1)
- std r16,STK_REG(R16)(r1)
- std r17,STK_REG(R17)(r1)
- std r18,STK_REG(R18)(r1)
- std r19,STK_REG(R19)(r1)
- std r20,STK_REG(R20)(r1)
- std r21,STK_REG(R21)(r1)
- std r22,STK_REG(R22)(r1)
- std r0,STACKFRAMESIZE+16(r1)
-
- blt 5f
- srdi r6,r5,7
- mtctr r6
-
- /* Now do cacheline (128B) sized loads and stores. */
- .align 5
-4:
-err2; ld r0,0(r4)
-err2; ld r6,8(r4)
-err2; ld r8,16(r4)
-err2; ld r9,24(r4)
-err2; ld r10,32(r4)
-err2; ld r11,40(r4)
-err2; ld r12,48(r4)
-err2; ld r14,56(r4)
-err2; ld r15,64(r4)
-err2; ld r16,72(r4)
-err2; ld r17,80(r4)
-err2; ld r18,88(r4)
-err2; ld r19,96(r4)
-err2; ld r20,104(r4)
-err2; ld r21,112(r4)
-err2; ld r22,120(r4)
- addi r4,r4,128
-err2; std r0,0(r3)
-err2; std r6,8(r3)
-err2; std r8,16(r3)
-err2; std r9,24(r3)
-err2; std r10,32(r3)
-err2; std r11,40(r3)
-err2; std r12,48(r3)
-err2; std r14,56(r3)
-err2; std r15,64(r3)
-err2; std r16,72(r3)
-err2; std r17,80(r3)
-err2; std r18,88(r3)
-err2; std r19,96(r3)
-err2; std r20,104(r3)
-err2; std r21,112(r3)
-err2; std r22,120(r3)
- addi r3,r3,128
- subi r7,r7,128
- bdnz 4b
-
- clrldi r5,r5,(64-7)
-
- /* Up to 127B to go */
-5: srdi r6,r5,4
- mtocrf 0x01,r6
-
-6: bf cr7*4+1,7f
-err2; ld r0,0(r4)
-err2; ld r6,8(r4)
-err2; ld r8,16(r4)
-err2; ld r9,24(r4)
-err2; ld r10,32(r4)
-err2; ld r11,40(r4)
-err2; ld r12,48(r4)
-err2; ld r14,56(r4)
- addi r4,r4,64
-err2; std r0,0(r3)
-err2; std r6,8(r3)
-err2; std r8,16(r3)
-err2; std r9,24(r3)
-err2; std r10,32(r3)
-err2; std r11,40(r3)
-err2; std r12,48(r3)
-err2; std r14,56(r3)
- addi r3,r3,64
- subi r7,r7,64
-
-7: ld r14,STK_REG(R14)(r1)
- ld r15,STK_REG(R15)(r1)
- ld r16,STK_REG(R16)(r1)
- ld r17,STK_REG(R17)(r1)
- ld r18,STK_REG(R18)(r1)
- ld r19,STK_REG(R19)(r1)
- ld r20,STK_REG(R20)(r1)
- ld r21,STK_REG(R21)(r1)
- ld r22,STK_REG(R22)(r1)
- addi r1,r1,STACKFRAMESIZE
-
- /* Up to 63B to go */
- bf cr7*4+2,8f
-err1; ld r0,0(r4)
-err1; ld r6,8(r4)
-err1; ld r8,16(r4)
-err1; ld r9,24(r4)
- addi r4,r4,32
-err1; std r0,0(r3)
-err1; std r6,8(r3)
-err1; std r8,16(r3)
-err1; std r9,24(r3)
- addi r3,r3,32
- subi r7,r7,32
-
- /* Up to 31B to go */
-8: bf cr7*4+3,9f
-err1; ld r0,0(r4)
-err1; ld r6,8(r4)
- addi r4,r4,16
-err1; std r0,0(r3)
-err1; std r6,8(r3)
- addi r3,r3,16
- subi r7,r7,16
-
-9: clrldi r5,r5,(64-4)
-
- /* Up to 15B to go */
-.Lshort_copy:
- mtocrf 0x01,r5
- bf cr7*4+0,12f
-err1; lwz r0,0(r4) /* Less chance of a reject with word ops */
-err1; lwz r6,4(r4)
- addi r4,r4,8
-err1; stw r0,0(r3)
-err1; stw r6,4(r3)
- addi r3,r3,8
- subi r7,r7,8
-
-12: bf cr7*4+1,13f
-err1; lwz r0,0(r4)
- addi r4,r4,4
-err1; stw r0,0(r3)
- addi r3,r3,4
- subi r7,r7,4
-
-13: bf cr7*4+2,14f
-err1; lhz r0,0(r4)
- addi r4,r4,2
-err1; sth r0,0(r3)
- addi r3,r3,2
- subi r7,r7,2
-
-14: bf cr7*4+3,15f
-err1; lbz r0,0(r4)
-err1; stb r0,0(r3)
-
-15: li r3,0
- blr
-
-EXPORT_SYMBOL_GPL(memcpy_mcsafe);
case BPF_JMP | BPF_JSET | BPF_K:
case BPF_JMP | BPF_JSET | BPF_X:
true_cond = COND_NE;
- fallthrough;
cond_branch:
/* same targets, can avoid doing the test :) */
if (filter[i].jt == filter[i].jf) {
#include <linux/random.h>
#include <linux/version.h>
-#include <asm/timex.h>
extern unsigned long __stack_chk_guard;
static __always_inline void boot_init_stack_canary(void)
{
unsigned long canary;
- unsigned long tsc;
/* Try to get a semi random initial value. */
get_random_bytes(&canary, sizeof(canary));
- tsc = get_cycles();
- canary += tsc + (tsc << BITS_PER_LONG/2);
canary ^= LINUX_VERSION_CODE;
canary &= CANARY_MASK;
#define get_cycles_hi get_cycles_hi
#endif /* CONFIG_64BIT */
+/*
+ * Much like MIPS, we may not have a viable counter to use at an early point
+ * in the boot process. Unfortunately we don't have a fallback, so instead
+ * we just return 0.
+ */
+static inline unsigned long random_get_entropy(void)
+{
+ if (unlikely(clint_time_val == NULL))
+ return 0;
+ return get_cycles();
+}
+#define random_get_entropy() random_get_entropy()
+
#else /* CONFIG_RISCV_M_MODE */
static inline cycles_t get_cycles(void)
/* Beginning of code and text segment */
. = LOAD_OFFSET;
_start = .;
- _stext = .;
HEAD_TEXT_SECTION
. = ALIGN(PAGE_SIZE);
__init_begin = .;
INIT_TEXT_SECTION(PAGE_SIZE)
- INIT_DATA_SECTION(16)
. = ALIGN(8);
__soc_early_init_table : {
__soc_early_init_table_start = .;
. = ALIGN(SECTION_ALIGN);
.text : {
_text = .;
+ _stext = .;
TEXT_TEXT
SCHED_TEXT
CPUIDLE_TEXT
_etext = .;
}
+ INIT_DATA_SECTION(16)
+
/* Start of data section */
_sdata = .;
RO_DATA(SECTION_ALIGN)
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
DISCARDS
}
#else
dtb_early_va = (void *)dtb_pa;
#endif
+ dtb_early_pa = dtb_pa;
}
static inline void setup_vm_final(void)
select OLD_SIGSUSPEND3
select PCI_DOMAINS if PCI
select PCI_MSI if PCI
+ select PCI_MSI_ARCH_FALLBACKS if PCI_MSI
select SPARSE_IRQ
select SYSCTL_EXCEPTION_TRACE
select THREAD_INFO_IN_TASK
return sum;
}
-static inline __wsum
-csum_partial_copy_nocheck (const void *src, void *dst, int len, __wsum sum)
-{
- memcpy(dst,src,len);
- return csum_partial(dst, len, sum);
-}
-
/*
* Fold a partial checksum without adding pseudo headers
*/
typedef u16 compat_ipc_pid_t;
typedef u32 compat_caddr_t;
typedef __kernel_fsid_t compat_fsid_t;
-typedef s64 compat_s64;
-typedef u64 compat_u64;
typedef struct {
u32 mask;
#define pgd_offset(mm, address) pgd_offset_raw(READ_ONCE((mm)->pgd), address)
-static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
+static inline p4d_t *p4d_offset_lockless(pgd_t *pgdp, pgd_t pgd, unsigned long address)
{
- if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R1)
- return (p4d_t *) pgd_deref(*pgd) + p4d_index(address);
- return (p4d_t *) pgd;
+ if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R1)
+ return (p4d_t *) pgd_deref(pgd) + p4d_index(address);
+ return (p4d_t *) pgdp;
}
+#define p4d_offset_lockless p4d_offset_lockless
-static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
+static inline p4d_t *p4d_offset(pgd_t *pgdp, unsigned long address)
{
- if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R2)
- return (pud_t *) p4d_deref(*p4d) + pud_index(address);
- return (pud_t *) p4d;
+ return p4d_offset_lockless(pgdp, *pgdp, address);
+}
+
+static inline pud_t *pud_offset_lockless(p4d_t *p4dp, p4d_t p4d, unsigned long address)
+{
+ if ((p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R2)
+ return (pud_t *) p4d_deref(p4d) + pud_index(address);
+ return (pud_t *) p4dp;
+}
+#define pud_offset_lockless pud_offset_lockless
+
+static inline pud_t *pud_offset(p4d_t *p4dp, unsigned long address)
+{
+ return pud_offset_lockless(p4dp, *p4dp, address);
}
#define pud_offset pud_offset
-static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
+static inline pmd_t *pmd_offset_lockless(pud_t *pudp, pud_t pud, unsigned long address)
+{
+ if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R3)
+ return (pmd_t *) pud_deref(pud) + pmd_index(address);
+ return (pmd_t *) pudp;
+}
+#define pmd_offset_lockless pmd_offset_lockless
+
+static inline pmd_t *pmd_offset(pud_t *pudp, unsigned long address)
{
- if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R3)
- return (pmd_t *) pud_deref(*pud) + pmd_index(address);
- return (pmd_t *) pud;
+ return pmd_offset_lockless(pudp, *pudp, address);
}
#define pmd_offset pmd_offset
void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14];
+ ri->fp = NULL;
/* Replace the return addr with trampoline addr */
regs->gprs[14] = (unsigned long) &kretprobe_trampoline;
*/
static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
{
- struct kretprobe_instance *ri;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address;
- unsigned long trampoline_address;
- kprobe_opcode_t *correct_ret_addr;
-
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
-
- /*
- * It is possible to have multiple instances associated with a given
- * task either because an multiple functions in the call path
- * have a return probe installed on them, and/or more than one return
- * return probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always inserted at the head of the list
- * - when multiple return probes are registered for the same
- * function, the first instance's ret_addr will point to the
- * real return address, and all the rest will point to
- * kretprobe_trampoline
- */
- ri = NULL;
- orig_ret_address = 0;
- correct_ret_addr = NULL;
- trampoline_address = (unsigned long) &kretprobe_trampoline;
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- orig_ret_address = (unsigned long) ri->ret_addr;
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
-
- correct_ret_addr = ri->ret_addr;
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- orig_ret_address = (unsigned long) ri->ret_addr;
-
- if (ri->rp && ri->rp->handler) {
- ri->ret_addr = correct_ret_addr;
- ri->rp->handler(ri, regs);
- }
-
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- regs->psw.addr = orig_ret_address;
-
- kretprobe_hash_unlock(current, &flags);
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
+ regs->psw.addr = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL);
/*
* By returning a non-zero value, we are telling
* kprobe_handler() that we don't want the post_handler
unwind_for_each_frame(&state, task, regs, 0) {
addr = unwind_get_return_address(&state);
- if (!addr || !consume_entry(cookie, addr, false))
+ if (!addr || !consume_entry(cookie, addr))
break;
}
}
return -EINVAL;
#endif
- if (!consume_entry(cookie, addr, false))
+ if (!consume_entry(cookie, addr))
return -EINVAL;
}
16 32 lchown - sys_lchown16
19 common lseek sys_lseek compat_sys_lseek
20 common getpid sys_getpid sys_getpid
-21 common mount sys_mount compat_sys_mount
+21 common mount sys_mount sys_mount
22 common umount sys_oldumount sys_oldumount
23 32 setuid - sys_setuid16
24 32 getuid - sys_getuid16
142 64 select sys_select -
143 common flock sys_flock sys_flock
144 common msync sys_msync sys_msync
-145 common readv sys_readv compat_sys_readv
-146 common writev sys_writev compat_sys_writev
+145 common readv sys_readv sys_readv
+146 common writev sys_writev sys_writev
147 common getsid sys_getsid sys_getsid
148 common fdatasync sys_fdatasync sys_fdatasync
149 common _sysctl - -
306 common splice sys_splice sys_splice
307 common sync_file_range sys_sync_file_range compat_sys_s390_sync_file_range
308 common tee sys_tee sys_tee
-309 common vmsplice sys_vmsplice compat_sys_vmsplice
+309 common vmsplice sys_vmsplice sys_vmsplice
310 common move_pages sys_move_pages compat_sys_move_pages
311 common getcpu sys_getcpu sys_getcpu
312 common epoll_pwait sys_epoll_pwait compat_sys_epoll_pwait
337 common clock_adjtime sys_clock_adjtime sys_clock_adjtime32
338 common syncfs sys_syncfs sys_syncfs
339 common setns sys_setns sys_setns
-340 common process_vm_readv sys_process_vm_readv compat_sys_process_vm_readv
-341 common process_vm_writev sys_process_vm_writev compat_sys_process_vm_writev
+340 common process_vm_readv sys_process_vm_readv sys_process_vm_readv
+341 common process_vm_writev sys_process_vm_writev sys_process_vm_writev
342 common s390_runtime_instr sys_s390_runtime_instr sys_s390_runtime_instr
343 common kcmp sys_kcmp sys_kcmp
344 common finit_module sys_finit_module sys_finit_module
/* Debugging sections. */
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
/* Sections to be discarded */
DISCARDS
* better 64-bit) boundary
*/
-asmlinkage __wsum csum_partial_copy_generic(const void *src, void *dst,
- int len, __wsum sum,
- int *src_err_ptr, int *dst_err_ptr);
+asmlinkage __wsum csum_partial_copy_generic(const void *src, void *dst, int len);
+#define _HAVE_ARCH_CSUM_AND_COPY
/*
* Note: when you get a NULL pointer exception here this means someone
* passed in an incorrect kernel address to one of these functions.
* access_ok().
*/
static inline
-__wsum csum_partial_copy_nocheck(const void *src, void *dst,
- int len, __wsum sum)
+__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len)
{
- return csum_partial_copy_generic(src, dst, len, sum, NULL, NULL);
+ return csum_partial_copy_generic(src, dst, len);
}
#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER
static inline
-__wsum csum_and_copy_from_user(const void __user *src, void *dst,
- int len, __wsum sum, int *err_ptr)
+__wsum csum_and_copy_from_user(const void __user *src, void *dst, int len)
{
- if (access_ok(src, len))
- return csum_partial_copy_generic((__force const void *)src, dst,
- len, sum, err_ptr, NULL);
- if (len)
- *err_ptr = -EFAULT;
- return sum;
+ if (!access_ok(src, len))
+ return 0;
+ return csum_partial_copy_generic((__force const void *)src, dst, len);
}
/*
#define HAVE_CSUM_COPY_USER
static inline __wsum csum_and_copy_to_user(const void *src,
void __user *dst,
- int len, __wsum sum,
- int *err_ptr)
+ int len)
{
- if (access_ok(dst, len))
- return csum_partial_copy_generic((__force const void *)src,
- dst, len, sum, NULL, err_ptr);
-
- if (len)
- *err_ptr = -EFAULT;
-
- return (__force __wsum)-1; /* invalid checksum */
+ if (!access_ok(dst, len))
+ return 0;
+ return csum_partial_copy_generic((__force const void *)src, dst, len);
}
#endif /* __ASM_SH_CHECKSUM_H */
struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *) regs->pr;
+ ri->fp = NULL;
/* Replace the return addr with trampoline addr */
regs->pr = (unsigned long)kretprobe_trampoline;
*/
int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
{
- struct kretprobe_instance *ri = NULL;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address = 0;
- unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
+ regs->pc = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL);
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
-
- /*
- * It is possible to have multiple instances associated with a given
- * task either because an multiple functions in the call path
- * have a return probe installed on them, and/or more then one return
- * return probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always inserted at the head of the list
- * - when multiple return probes are registered for the same
- * function, the first instance's ret_addr will point to the
- * real return address, and all the rest will point to
- * kretprobe_trampoline
- */
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- if (ri->rp && ri->rp->handler) {
- __this_cpu_write(current_kprobe, &ri->rp->kp);
- ri->rp->handler(ri, regs);
- __this_cpu_write(current_kprobe, NULL);
- }
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
-
- regs->pc = orig_ret_address;
- kretprobe_hash_unlock(current, &flags);
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
-
- return orig_ret_address;
+ return 1;
}
static int __kprobes post_kprobe_handler(struct pt_regs *regs)
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
DISCARDS
}
mov r6, r0
/*
-unsigned int csum_partial_copy_generic (const char *src, char *dst, int len,
- int sum, int *src_err_ptr, int *dst_err_ptr)
+unsigned int csum_partial_copy_generic (const char *src, char *dst, int len)
*/
/*
- * Copy from ds while checksumming, otherwise like csum_partial
- *
- * The macros SRC and DST specify the type of access for the instruction.
- * thus we can call a custom exception handler for all access types.
- *
- * FIXME: could someone double-check whether I haven't mixed up some SRC and
- * DST definitions? It's damn hard to trigger all cases. I hope I got
- * them all but there's no guarantee.
+ * Copy from ds while checksumming, otherwise like csum_partial with initial
+ * sum being ~0U
*/
-#define SRC(...) \
+#define EXC(...) \
9999: __VA_ARGS__ ; \
.section __ex_table, "a"; \
.long 9999b, 6001f ; \
.previous
-#define DST(...) \
- 9999: __VA_ARGS__ ; \
- .section __ex_table, "a"; \
- .long 9999b, 6002f ; \
- .previous
-
!
! r4: const char *SRC
! r5: char *DST
! r6: int LEN
-! r7: int SUM
-!
-! on stack:
-! int *SRC_ERR_PTR
-! int *DST_ERR_PTR
!
ENTRY(csum_partial_copy_generic)
- mov.l r5,@-r15
- mov.l r6,@-r15
-
+ mov #-1,r7
mov #3,r0 ! Check src and dest are equally aligned
mov r4,r1
and r0,r1
clrt
.align 2
5:
-SRC( mov.b @r4+,r1 )
-SRC( mov.b @r4+,r0 )
+EXC( mov.b @r4+,r1 )
+EXC( mov.b @r4+,r0 )
extu.b r1,r1
-DST( mov.b r1,@r5 )
-DST( mov.b r0,@(1,r5) )
+EXC( mov.b r1,@r5 )
+EXC( mov.b r0,@(1,r5) )
extu.b r0,r0
add #2,r5
! Handle first two bytes as a special case
.align 2
1:
-SRC( mov.w @r4+,r0 )
-DST( mov.w r0,@r5 )
+EXC( mov.w @r4+,r0 )
+EXC( mov.w r0,@r5 )
add #2,r5
extu.w r0,r0
addc r0,r7
clrt
.align 2
1:
-SRC( mov.l @r4+,r0 )
-SRC( mov.l @r4+,r1 )
+EXC( mov.l @r4+,r0 )
+EXC( mov.l @r4+,r1 )
addc r0,r7
-DST( mov.l r0,@r5 )
-DST( mov.l r1,@(4,r5) )
+EXC( mov.l r0,@r5 )
+EXC( mov.l r1,@(4,r5) )
addc r1,r7
-SRC( mov.l @r4+,r0 )
-SRC( mov.l @r4+,r1 )
+EXC( mov.l @r4+,r0 )
+EXC( mov.l @r4+,r1 )
addc r0,r7
-DST( mov.l r0,@(8,r5) )
-DST( mov.l r1,@(12,r5) )
+EXC( mov.l r0,@(8,r5) )
+EXC( mov.l r1,@(12,r5) )
addc r1,r7
-SRC( mov.l @r4+,r0 )
-SRC( mov.l @r4+,r1 )
+EXC( mov.l @r4+,r0 )
+EXC( mov.l @r4+,r1 )
addc r0,r7
-DST( mov.l r0,@(16,r5) )
-DST( mov.l r1,@(20,r5) )
+EXC( mov.l r0,@(16,r5) )
+EXC( mov.l r1,@(20,r5) )
addc r1,r7
-SRC( mov.l @r4+,r0 )
-SRC( mov.l @r4+,r1 )
+EXC( mov.l @r4+,r0 )
+EXC( mov.l @r4+,r1 )
addc r0,r7
-DST( mov.l r0,@(24,r5) )
-DST( mov.l r1,@(28,r5) )
+EXC( mov.l r0,@(24,r5) )
+EXC( mov.l r1,@(28,r5) )
addc r1,r7
add #32,r5
movt r0
clrt
shlr2 r6
3:
-SRC( mov.l @r4+,r0 )
+EXC( mov.l @r4+,r0 )
addc r0,r7
-DST( mov.l r0,@r5 )
+EXC( mov.l r0,@r5 )
add #4,r5
movt r0
dt r6
mov #2,r1
cmp/hs r1,r6
bf 5f
-SRC( mov.w @r4+,r0 )
-DST( mov.w r0,@r5 )
+EXC( mov.w @r4+,r0 )
+EXC( mov.w r0,@r5 )
extu.w r0,r0
add #2,r5
cmp/eq r1,r6
shll16 r0
addc r0,r7
5:
-SRC( mov.b @r4+,r0 )
-DST( mov.b r0,@r5 )
+EXC( mov.b @r4+,r0 )
+EXC( mov.b r0,@r5 )
extu.b r0,r0
#ifndef __LITTLE_ENDIAN__
shll8 r0
mov #0,r0
addc r0,r7
7:
-5000:
# Exception handler:
.section .fixup, "ax"
6001:
- mov.l @(8,r15),r0 ! src_err_ptr
- mov #-EFAULT,r1
- mov.l r1,@r0
-
- ! zero the complete destination - computing the rest
- ! is too much work
- mov.l @(4,r15),r5 ! dst
- mov.l @r15,r6 ! len
- mov #0,r7
-1: mov.b r7,@r5
- dt r6
- bf/s 1b
- add #1,r5
- mov.l 8000f,r0
- jmp @r0
- nop
- .align 2
-8000: .long 5000b
-
-6002:
- mov.l @(12,r15),r0 ! dst_err_ptr
- mov #-EFAULT,r1
- mov.l r1,@r0
- mov.l 8001f,r0
- jmp @r0
- nop
- .align 2
-8001: .long 5000b
-
+ rts
+ mov #0,r0
.previous
- add #8,r15
rts
mov r7,r0
select GENERIC_STRNLEN_USER
select MODULES_USE_ELF_RELA
select PCI_SYSCALL if PCI
+ select PCI_MSI_ARCH_FALLBACKS if PCI_MSI
select ODD_RT_SIGACTION
select OLD_SIGSUSPEND
select CPU_NO_EFFICIENT_FFS
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef ___ASM_SPARC_CHECKSUM_H
#define ___ASM_SPARC_CHECKSUM_H
+#define _HAVE_ARCH_CSUM_AND_COPY
#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER
+#define HAVE_CSUM_COPY_USER
#if defined(__sparc__) && defined(__arch64__)
#include <asm/checksum_64.h>
#else
unsigned int __csum_partial_copy_sparc_generic (const unsigned char *, unsigned char *);
static inline __wsum
-csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
+csum_partial_copy_nocheck(const void *src, void *dst, int len)
{
register unsigned int ret asm("o0") = (unsigned int)src;
register char *d asm("o1") = dst;
__asm__ __volatile__ (
"call __csum_partial_copy_sparc_generic\n\t"
- " mov %6, %%g7\n"
+ " mov -1, %%g7\n"
: "=&r" (ret), "=&r" (d), "=&r" (l)
- : "0" (ret), "1" (d), "2" (l), "r" (sum)
+ : "0" (ret), "1" (d), "2" (l)
: "o2", "o3", "o4", "o5", "o7",
"g2", "g3", "g4", "g5", "g7",
"memory", "cc");
}
static inline __wsum
-csum_and_copy_from_user(const void __user *src, void *dst, int len,
- __wsum sum, int *err)
- {
- register unsigned long ret asm("o0") = (unsigned long)src;
- register char *d asm("o1") = dst;
- register int l asm("g1") = len;
- register __wsum s asm("g7") = sum;
-
- if (unlikely(!access_ok(src, len))) {
- if (len)
- *err = -EFAULT;
- return sum;
- }
-
- __asm__ __volatile__ (
- ".section __ex_table,#alloc\n\t"
- ".align 4\n\t"
- ".word 1f,2\n\t"
- ".previous\n"
- "1:\n\t"
- "call __csum_partial_copy_sparc_generic\n\t"
- " st %8, [%%sp + 64]\n"
- : "=&r" (ret), "=&r" (d), "=&r" (l), "=&r" (s)
- : "0" (ret), "1" (d), "2" (l), "3" (s), "r" (err)
- : "o2", "o3", "o4", "o5", "o7", "g2", "g3", "g4", "g5",
- "cc", "memory");
- return (__force __wsum)ret;
+csum_and_copy_from_user(const void __user *src, void *dst, int len)
+{
+ if (unlikely(!access_ok(src, len)))
+ return 0;
+ return csum_partial_copy_nocheck((__force void *)src, dst, len);
}
-#define HAVE_CSUM_COPY_USER
-
static inline __wsum
-csum_and_copy_to_user(const void *src, void __user *dst, int len,
- __wsum sum, int *err)
+csum_and_copy_to_user(const void *src, void __user *dst, int len)
{
- if (!access_ok(dst, len)) {
- *err = -EFAULT;
- return sum;
- } else {
- register unsigned long ret asm("o0") = (unsigned long)src;
- register char __user *d asm("o1") = dst;
- register int l asm("g1") = len;
- register __wsum s asm("g7") = sum;
-
- __asm__ __volatile__ (
- ".section __ex_table,#alloc\n\t"
- ".align 4\n\t"
- ".word 1f,1\n\t"
- ".previous\n"
- "1:\n\t"
- "call __csum_partial_copy_sparc_generic\n\t"
- " st %8, [%%sp + 64]\n"
- : "=&r" (ret), "=&r" (d), "=&r" (l), "=&r" (s)
- : "0" (ret), "1" (d), "2" (l), "3" (s), "r" (err)
- : "o2", "o3", "o4", "o5", "o7",
- "g2", "g3", "g4", "g5",
- "cc", "memory");
- return (__force __wsum)ret;
- }
+ if (!access_ok(dst, len))
+ return 0;
+ return csum_partial_copy_nocheck(src, (__force void *)dst, len);
}
/* ihl is always 5 or greater, almost always is 5, and iph is word aligned
* here even more important to align src and dst on a 32-bit (or even
* better 64-bit) boundary
*/
-__wsum csum_partial_copy_nocheck(const void *src, void *dst,
- int len, __wsum sum);
-
-long __csum_partial_copy_from_user(const void __user *src,
- void *dst, int len,
- __wsum sum);
-
-static inline __wsum
-csum_and_copy_from_user(const void __user *src,
- void *dst, int len,
- __wsum sum, int *err)
-{
- long ret = __csum_partial_copy_from_user(src, dst, len, sum);
- if (ret < 0)
- *err = -EFAULT;
- return (__force __wsum) ret;
-}
-
-/*
- * Copy and checksum to user
- */
-#define HAVE_CSUM_COPY_USER
-long __csum_partial_copy_to_user(const void *src,
- void __user *dst, int len,
- __wsum sum);
-
-static inline __wsum
-csum_and_copy_to_user(const void *src,
- void __user *dst, int len,
- __wsum sum, int *err)
-{
- long ret = __csum_partial_copy_to_user(src, dst, len, sum);
- if (ret < 0)
- *err = -EFAULT;
- return (__force __wsum) ret;
-}
+__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len);
+__wsum csum_and_copy_from_user(const void __user *src, void *dst, int len);
+__wsum csum_and_copy_to_user(const void *src, void __user *dst, int len);
/* ihl is always 5 or greater, almost always is 5, and iph is word aligned
* the majority of the time.
typedef u16 compat_ipc_pid_t;
typedef u32 compat_caddr_t;
typedef __kernel_fsid_t compat_fsid_t;
-typedef s64 compat_s64;
-typedef u64 compat_u64;
+
struct compat_stat {
compat_dev_t st_dev;
compat_ino_t st_ino;
struct pt_regs *regs)
{
ri->ret_addr = (kprobe_opcode_t *)(regs->u_regs[UREG_RETPC] + 8);
+ ri->fp = NULL;
/* Replace the return addr with trampoline addr */
regs->u_regs[UREG_RETPC] =
static int __kprobes trampoline_probe_handler(struct kprobe *p,
struct pt_regs *regs)
{
- struct kretprobe_instance *ri = NULL;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address = 0;
- unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline;
+ unsigned long orig_ret_address = 0;
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
-
- /*
- * It is possible to have multiple instances associated with a given
- * task either because an multiple functions in the call path
- * have a return probe installed on them, and/or more than one return
- * return probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always inserted at the head of the list
- * - when multiple return probes are registered for the same
- * function, the first instance's ret_addr will point to the
- * real return address, and all the rest will point to
- * kretprobe_trampoline
- */
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
-
- if (ri->rp && ri->rp->handler)
- ri->rp->handler(ri, regs);
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
+ orig_ret_address = __kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL);
regs->tpc = orig_ret_address;
regs->tnpc = orig_ret_address + 4;
- kretprobe_hash_unlock(current, &flags);
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
/*
* By returning a non-zero value, we are telling
* kprobe_handler() that we don't want the post_handler
23 64 setuid sys_setuid
24 32 getuid sys_getuid16
24 64 getuid sys_getuid
-25 common vmsplice sys_vmsplice compat_sys_vmsplice
+25 common vmsplice sys_vmsplice
26 common ptrace sys_ptrace compat_sys_ptrace
27 common alarm sys_alarm
28 common sigaltstack sys_sigaltstack compat_sys_sigaltstack
117 common getrusage sys_getrusage compat_sys_getrusage
118 common getsockopt sys_getsockopt sys_getsockopt
119 common getcwd sys_getcwd
-120 common readv sys_readv compat_sys_readv
-121 common writev sys_writev compat_sys_writev
+120 common readv sys_readv
+121 common writev sys_writev
122 common settimeofday sys_settimeofday compat_sys_settimeofday
123 32 fchown sys_fchown16
123 64 fchown sys_fchown
164 64 utrap_install sys_utrap_install
165 common quotactl sys_quotactl
166 common set_tid_address sys_set_tid_address
-167 common mount sys_mount compat_sys_mount
+167 common mount sys_mount
168 common ustat sys_ustat compat_sys_ustat
169 common setxattr sys_setxattr
170 common lsetxattr sys_lsetxattr
335 common syncfs sys_syncfs
336 common sendmmsg sys_sendmmsg compat_sys_sendmmsg
337 common setns sys_setns
-338 common process_vm_readv sys_process_vm_readv compat_sys_process_vm_readv
-339 common process_vm_writev sys_process_vm_writev compat_sys_process_vm_writev
+338 common process_vm_readv sys_process_vm_readv
+339 common process_vm_writev sys_process_vm_writev
340 32 kern_features sys_ni_syscall sys_kern_features
340 64 kern_features sys_kern_features
341 common kcmp sys_kcmp
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
DISCARDS
}
cpout: retl ! get outta here
mov %o2, %o0 ! return computed csum
- .globl __csum_partial_copy_start, __csum_partial_copy_end
-__csum_partial_copy_start:
-
/* Work around cpp -rob */
#define ALLOC #alloc
#define EXECINSTR #execinstr
-#define EX(x,y,a,b) \
-98: x,y; \
- .section .fixup,ALLOC,EXECINSTR; \
- .align 4; \
-99: ba 30f; \
- a, b, %o3; \
- .section __ex_table,ALLOC; \
- .align 4; \
- .word 98b, 99b; \
- .text; \
- .align 4
-
-#define EX2(x,y) \
-98: x,y; \
- .section __ex_table,ALLOC; \
- .align 4; \
- .word 98b, 30f; \
- .text; \
- .align 4
-
-#define EX3(x,y) \
+#define EX(x,y) \
98: x,y; \
.section __ex_table,ALLOC; \
.align 4; \
- .word 98b, 96f; \
+ .word 98b, cc_fault; \
.text; \
.align 4
-#define EXT(start,end,handler) \
+#define EXT(start,end) \
.section __ex_table,ALLOC; \
.align 4; \
- .word start, 0, end, handler; \
+ .word start, 0, end, cc_fault; \
.text; \
.align 4
cc_end_cruft:
be 1f
andcc %o3, 4, %g0
- EX(ldd [%o0 + 0x00], %g2, and %o3, 0xf)
+ EX(ldd [%o0 + 0x00], %g2)
add %o1, 8, %o1
addcc %g2, %g7, %g7
add %o0, 8, %o0
addxcc %g3, %g7, %g7
- EX2(st %g2, [%o1 - 0x08])
+ EX(st %g2, [%o1 - 0x08])
addx %g0, %g7, %g7
andcc %o3, 4, %g0
- EX2(st %g3, [%o1 - 0x04])
+ EX(st %g3, [%o1 - 0x04])
1: be 1f
andcc %o3, 3, %o3
- EX(ld [%o0 + 0x00], %g2, add %o3, 4)
+ EX(ld [%o0 + 0x00], %g2)
add %o1, 4, %o1
addcc %g2, %g7, %g7
- EX2(st %g2, [%o1 - 0x04])
+ EX(st %g2, [%o1 - 0x04])
addx %g0, %g7, %g7
andcc %o3, 3, %g0
add %o0, 4, %o0
subcc %o3, 2, %o3
b 4f
or %g0, %g0, %o4
-2: EX(lduh [%o0 + 0x00], %o4, add %o3, 2)
+2: EX(lduh [%o0 + 0x00], %o4)
add %o0, 2, %o0
- EX2(sth %o4, [%o1 + 0x00])
+ EX(sth %o4, [%o1 + 0x00])
be 6f
add %o1, 2, %o1
sll %o4, 16, %o4
-4: EX(ldub [%o0 + 0x00], %o5, add %g0, 1)
- EX2(stb %o5, [%o1 + 0x00])
+4: EX(ldub [%o0 + 0x00], %o5)
+ EX(stb %o5, [%o1 + 0x00])
sll %o5, 8, %o5
or %o5, %o4, %o4
6: addcc %o4, %g7, %g7
andcc %o0, 0x2, %g0
be 1f
andcc %o0, 0x4, %g0
- EX(lduh [%o0 + 0x00], %g4, add %g1, 0)
+ EX(lduh [%o0 + 0x00], %g4)
sub %g1, 2, %g1
- EX2(sth %g4, [%o1 + 0x00])
+ EX(sth %g4, [%o1 + 0x00])
add %o0, 2, %o0
sll %g4, 16, %g4
addcc %g4, %g7, %g7
or %g3, %g7, %g7
1: be 3f
andcc %g1, 0xffffff80, %g0
- EX(ld [%o0 + 0x00], %g4, add %g1, 0)
+ EX(ld [%o0 + 0x00], %g4)
sub %g1, 4, %g1
- EX2(st %g4, [%o1 + 0x00])
+ EX(st %g4, [%o1 + 0x00])
add %o0, 4, %o0
addcc %g4, %g7, %g7
add %o1, 4, %o1
CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
-10: EXT(5b, 10b, 20f) ! note for exception handling
+10: EXT(5b, 10b) ! note for exception handling
sub %g1, 128, %g1 ! detract from length
addx %g0, %g7, %g7 ! add in last carry bit
andcc %g1, 0xffffff80, %g0 ! more to csum?
CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x28,%g2,%g3,%g4,%g5)
CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x18,%g2,%g3,%g4,%g5)
CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x08,%g2,%g3,%g4,%g5)
-12: EXT(cctbl, 12b, 22f) ! note for exception table handling
+12: EXT(cctbl, 12b) ! note for exception table handling
addx %g0, %g7, %g7
andcc %o3, 0xf, %g0 ! check for low bits set
ccte: bne cc_end_cruft ! something left, handle it out of band
CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3)
-11: EXT(ccdbl, 11b, 21f) ! note for exception table handling
+11: EXT(ccdbl, 11b) ! note for exception table handling
sub %g1, 128, %g1 ! detract from length
addx %g0, %g7, %g7 ! add in last carry bit
andcc %g1, 0xffffff80, %g0 ! more to csum?
be,a 1f
srl %g1, 1, %g4
sub %g1, 1, %g1
- EX(ldub [%o0], %g5, add %g1, 1)
+ EX(ldub [%o0], %g5)
add %o0, 1, %o0
- EX2(stb %g5, [%o1])
+ EX(stb %g5, [%o1])
srl %g1, 1, %g4
add %o1, 1, %o1
1: cmp %g4, 0
andcc %o0, 2, %g0
be,a 1f
srl %g4, 1, %g4
- EX(lduh [%o0], %o4, add %g1, 0)
+ EX(lduh [%o0], %o4)
sub %g1, 2, %g1
srl %o4, 8, %g2
sub %g4, 1, %g4
- EX2(stb %g2, [%o1])
+ EX(stb %g2, [%o1])
add %o4, %g5, %g5
- EX2(stb %o4, [%o1 + 1])
+ EX(stb %o4, [%o1 + 1])
add %o0, 2, %o0
srl %g4, 1, %g4
add %o1, 2, %o1
1: cmp %g4, 0
be,a 2f
andcc %g1, 2, %g0
- EX3(ld [%o0], %o4)
+ EX(ld [%o0], %o4)
5: srl %o4, 24, %g2
srl %o4, 16, %g3
- EX2(stb %g2, [%o1])
+ EX(stb %g2, [%o1])
srl %o4, 8, %g2
- EX2(stb %g3, [%o1 + 1])
+ EX(stb %g3, [%o1 + 1])
add %o0, 4, %o0
- EX2(stb %g2, [%o1 + 2])
+ EX(stb %g2, [%o1 + 2])
addcc %o4, %g5, %g5
- EX2(stb %o4, [%o1 + 3])
+ EX(stb %o4, [%o1 + 3])
addx %g5, %g0, %g5 ! I am now to lazy to optimize this (question it
add %o1, 4, %o1 ! is worthy). Maybe some day - with the sll/srl
subcc %g4, 1, %g4 ! tricks
bne,a 5b
- EX3(ld [%o0], %o4)
+ EX(ld [%o0], %o4)
sll %g5, 16, %g2
srl %g5, 16, %g5
srl %g2, 16, %g2
add %g2, %g5, %g5
2: be,a 3f
andcc %g1, 1, %g0
- EX(lduh [%o0], %o4, and %g1, 3)
+ EX(lduh [%o0], %o4)
andcc %g1, 1, %g0
srl %o4, 8, %g2
add %o0, 2, %o0
- EX2(stb %g2, [%o1])
+ EX(stb %g2, [%o1])
add %g5, %o4, %g5
- EX2(stb %o4, [%o1 + 1])
+ EX(stb %o4, [%o1 + 1])
add %o1, 2, %o1
3: be,a 1f
sll %g5, 16, %o4
- EX(ldub [%o0], %g2, add %g0, 1)
+ EX(ldub [%o0], %g2)
sll %g2, 8, %o4
- EX2(stb %g2, [%o1])
+ EX(stb %g2, [%o1])
add %g5, %o4, %g5
sll %g5, 16, %o4
1: addcc %o4, %g5, %g5
4: addcc %g7, %g5, %g7
retl
addx %g0, %g7, %o0
-__csum_partial_copy_end:
/* We do these strange calculations for the csum_*_from_user case only, ie.
* we only bother with faults on loads... */
-/* o2 = ((g2%20)&3)*8
- * o3 = g1 - (g2/20)*32 - o2 */
-20:
- cmp %g2, 20
- blu,a 1f
- and %g2, 3, %o2
- sub %g1, 32, %g1
- b 20b
- sub %g2, 20, %g2
-1:
- sll %o2, 3, %o2
- b 31f
- sub %g1, %o2, %o3
-
-/* o2 = (!(g2 & 15) ? 0 : (((g2 & 15) + 1) & ~1)*8)
- * o3 = g1 - (g2/16)*32 - o2 */
-21:
- andcc %g2, 15, %o3
- srl %g2, 4, %g2
- be,a 1f
- clr %o2
- add %o3, 1, %o3
- and %o3, 14, %o3
- sll %o3, 3, %o2
-1:
- sll %g2, 5, %g2
- sub %g1, %g2, %o3
- b 31f
- sub %o3, %o2, %o3
-
-/* o0 += (g2/10)*16 - 0x70
- * 01 += (g2/10)*16 - 0x70
- * o2 = (g2 % 10) ? 8 : 0
- * o3 += 0x70 - (g2/10)*16 - o2 */
-22:
- cmp %g2, 10
- blu,a 1f
- sub %o0, 0x70, %o0
- add %o0, 16, %o0
- add %o1, 16, %o1
- sub %o3, 16, %o3
- b 22b
- sub %g2, 10, %g2
-1:
- sub %o1, 0x70, %o1
- add %o3, 0x70, %o3
- clr %o2
- tst %g2
- bne,a 1f
- mov 8, %o2
-1:
- b 31f
- sub %o3, %o2, %o3
-96:
- and %g1, 3, %g1
- sll %g4, 2, %g4
- add %g1, %g4, %o3
-30:
-/* %o1 is dst
- * %o3 is # bytes to zero out
- * %o4 is faulting address
- * %o5 is %pc where fault occurred */
- clr %o2
-31:
-/* %o0 is src
- * %o1 is dst
- * %o2 is # of bytes to copy from src to dst
- * %o3 is # bytes to zero out
- * %o4 is faulting address
- * %o5 is %pc where fault occurred */
- save %sp, -104, %sp
- mov %i5, %o0
- mov %i7, %o1
- mov %i4, %o2
- call lookup_fault
- mov %g7, %i4
- cmp %o0, 2
- bne 1f
- add %g0, -EFAULT, %i5
- tst %i2
- be 2f
- mov %i0, %o1
- mov %i1, %o0
-5:
- call memcpy
- mov %i2, %o2
- tst %o0
- bne,a 2f
- add %i3, %i2, %i3
- add %i1, %i2, %i1
-2:
- mov %i1, %o0
-6:
- call __bzero
- mov %i3, %o1
-1:
- ld [%sp + 168], %o2 ! struct_ptr of parent
- st %i5, [%o2]
+cc_fault:
ret
- restore
-
- .section __ex_table,#alloc
- .align 4
- .word 5b,2
- .word 6b,2
+ clr %o0
.globl FUNC_NAME
.type FUNC_NAME,#function
EXPORT_SYMBOL(FUNC_NAME)
-FUNC_NAME: /* %o0=src, %o1=dst, %o2=len, %o3=sum */
+FUNC_NAME: /* %o0=src, %o1=dst, %o2=len */
LOAD(prefetch, %o0 + 0x000, #n_reads)
xor %o0, %o1, %g1
+ mov 1, %o3
clr %o4
andcc %g1, 0x3, %g0
bne,pn %icc, 95f
.section .fixup, "ax"; \
.align 4; \
99: retl; \
- mov -1, %o0; \
+ mov 0, %o0; \
.section __ex_table,"a";\
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
-#define FUNC_NAME __csum_partial_copy_from_user
+#define FUNC_NAME csum_and_copy_from_user
#define LOAD(type,addr,dest) type##a [addr] %asi, dest
#include "csum_copy.S"
.section .fixup,"ax"; \
.align 4; \
99: retl; \
- mov -1, %o0; \
+ mov 0, %o0; \
.section __ex_table,"a";\
.align 4; \
.word 98b, 99b; \
.text; \
.align 4;
-#define FUNC_NAME __csum_partial_copy_to_user
+#define FUNC_NAME csum_and_copy_to_user
#define STORE(type,src,addr) type##a src, [addr] %asi
#include "csum_copy.S"
if (fixup > 10) {
extern const unsigned int __memset_start[];
extern const unsigned int __memset_end[];
- extern const unsigned int __csum_partial_copy_start[];
- extern const unsigned int __csum_partial_copy_end[];
#ifdef DEBUG_EXCEPTIONS
printk("Exception: PC<%08lx> faddr<%08lx>\n",
regs->pc, fixup, g2);
#endif
if ((regs->pc >= (unsigned long)__memset_start &&
- regs->pc < (unsigned long)__memset_end) ||
- (regs->pc >= (unsigned long)__csum_partial_copy_start &&
- regs->pc < (unsigned long)__csum_partial_copy_end)) {
+ regs->pc < (unsigned long)__memset_end)) {
regs->u_regs[UREG_I4] = address;
regs->u_regs[UREG_I5] = regs->pc;
}
PROVIDE (end = .);
STABS_DEBUG
-
DWARF_DEBUG
+ ELF_DETAILS
DISCARDS
}
PROVIDE (end = .);
STABS_DEBUG
-
DWARF_DEBUG
+ ELF_DETAILS
DISCARDS
}
select ARCH_HAS_PTE_DEVMAP if X86_64
select ARCH_HAS_PTE_SPECIAL
select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64
- select ARCH_HAS_UACCESS_MCSAFE if X86_64 && X86_MCE
+ select ARCH_HAS_COPY_MC if X86_64
select ARCH_HAS_SET_MEMORY
select ARCH_HAS_SET_DIRECT_MAP
select ARCH_HAS_STRICT_KERNEL_RWX
select HAVE_FUNCTION_ARG_ACCESS_API
select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR
select HAVE_STACK_VALIDATION if X86_64
+ select HAVE_STATIC_CALL
+ select HAVE_STATIC_CALL_INLINE if HAVE_STACK_VALIDATION
select HAVE_RSEQ
select HAVE_SYSCALL_TRACEPOINTS
select HAVE_UNSTABLE_SCHED_CLOCK
select RTC_MC146818_LIB
select SPARSE_IRQ
select SRCU
+ select STACK_VALIDATION if HAVE_STACK_VALIDATION && (HAVE_STATIC_CALL_INLINE || RETPOLINE)
select SYSCTL_EXCEPTION_TRACE
select THREAD_INFO_IN_TASK
select USER_STACKTRACE_SUPPORT
config RETPOLINE
bool "Avoid speculative indirect branches in kernel"
default y
- select STACK_VALIDATION if HAVE_STACK_VALIDATION
help
Compile kernel with the retpoline compiler options to guard against
kernel-to-user data leaks by avoiding speculative indirect
You should normally say N here, unless you want to debug early
crashes or need a very simple printk logging facility.
-config MCSAFE_TEST
+config COPY_MC_TEST
def_bool n
config EFI_PGT_DUMP
LDFLAGS_vmlinux += -z max-page-size=0x200000
endif
+# We never want expected sections to be placed heuristically by the
+# linker. All sections should be explicitly named in the linker script.
+LDFLAGS_vmlinux += $(call ld-option, --orphan-handling=warn)
+
archscripts: scripts_basic
$(Q)$(MAKE) $(build)=arch/x86/tools relocs
vmlinux.bin.xz vmlinux.bin.lzo vmlinux.bin.lz4 vmlinux.bin.zst
KBUILD_CFLAGS := -m$(BITS) -O2
-KBUILD_CFLAGS += -fno-strict-aliasing $(call cc-option, -fPIE, -fPIC)
+KBUILD_CFLAGS += -fno-strict-aliasing -fPIE
KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
cflags-$(CONFIG_X86_32) := -march=i386
cflags-$(CONFIG_X86_64) := -mcmodel=small
KBUILD_CFLAGS += -D__DISABLE_EXPORTS
# Disable relocation relaxation in case the link is not PIE.
KBUILD_CFLAGS += $(call as-option,-Wa$(comma)-mrelax-relocations=no)
+KBUILD_CFLAGS += -include $(srctree)/include/linux/hidden.h
KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__
GCOV_PROFILE := n
UBSAN_SANITIZE :=n
KBUILD_LDFLAGS := -m elf_$(UTS_MACHINE)
+KBUILD_LDFLAGS += $(call ld-option,--no-ld-generated-unwind-info)
# Compressed kernel should be built as PIE since it may be loaded at any
# address by the bootloader.
-ifeq ($(CONFIG_X86_32),y)
-KBUILD_LDFLAGS += $(call ld-option, -pie) $(call ld-option, --no-dynamic-linker)
-else
-# To build 64-bit compressed kernel as PIE, we disable relocation
-# overflow check to avoid relocation overflow error with a new linker
-# command-line option, -z noreloc-overflow.
-KBUILD_LDFLAGS += $(shell $(LD) --help 2>&1 | grep -q "\-z noreloc-overflow" \
- && echo "-z noreloc-overflow -pie --no-dynamic-linker")
-endif
-LDFLAGS_vmlinux := -T
+LDFLAGS_vmlinux := -pie $(call ld-option, --no-dynamic-linker)
+LDFLAGS_vmlinux += $(call ld-option, --orphan-handling=warn)
+LDFLAGS_vmlinux += -T
hostprogs := mkpiggy
HOST_EXTRACFLAGS += -I$(srctree)/tools/include
vmlinux-objs-$(CONFIG_EFI_MIXED) += $(obj)/efi_thunk_$(BITS).o
efi-obj-$(CONFIG_EFI_STUB) = $(objtree)/drivers/firmware/efi/libstub/lib.a
-# The compressed kernel is built with -fPIC/-fPIE so that a boot loader
-# can place it anywhere in memory and it will still run. However, since
-# it is executed as-is without any ELF relocation processing performed
-# (and has already had all relocation sections stripped from the binary),
-# none of the code can use data relocations (e.g. static assignments of
-# pointer values), since they will be meaningless at runtime. This check
-# will refuse to link the vmlinux if any of these relocations are found.
-quiet_cmd_check_data_rel = DATAREL $@
-define cmd_check_data_rel
- for obj in $(filter %.o,$^); do \
- $(READELF) -S $$obj | grep -qF .rel.local && { \
- echo "error: $$obj has data relocations!" >&2; \
- exit 1; \
- } || true; \
- done
-endef
-
-# We need to run two commands under "if_changed", so merge them into a
-# single invocation.
-quiet_cmd_check-and-link-vmlinux = LD $@
- cmd_check-and-link-vmlinux = $(cmd_check_data_rel); $(cmd_ld)
-
$(obj)/vmlinux: $(vmlinux-objs-y) $(efi-obj-y) FORCE
- $(call if_changed,check-and-link-vmlinux)
+ $(call if_changed,ld)
OBJCOPYFLAGS_vmlinux.bin := -R .comment -S
$(obj)/vmlinux.bin: vmlinux FORCE
#include <asm/bootparam.h>
/*
- * The 32-bit x86 assembler in binutils 2.26 will generate R_386_GOT32X
- * relocation to get the symbol address in PIC. When the compressed x86
- * kernel isn't built as PIC, the linker optimizes R_386_GOT32X
- * relocations to their fixed symbol addresses. However, when the
- * compressed x86 kernel is loaded at a different address, it leads
- * to the following load failure:
- *
- * Failed to allocate space for phdrs
- *
- * during the decompression stage.
- *
- * If the compressed x86 kernel is relocatable at run-time, it should be
- * compiled with -fPIE, instead of -fPIC, if possible and should be built as
- * Position Independent Executable (PIE) so that linker won't optimize
- * R_386_GOT32X relocation to its fixed symbol address. Older
- * linkers generate R_386_32 relocations against locally defined symbols,
- * _bss, _ebss, _got, _egot and _end, in PIE. It isn't wrong, just less
- * optimal than R_386_RELATIVE. But the x86 kernel fails to properly handle
- * R_386_32 relocations when relocating the kernel. To generate
- * R_386_RELATIVE relocations, we mark _bss, _ebss, _got, _egot and _end as
- * hidden:
+ * These symbols needed to be marked as .hidden to prevent the BFD linker from
+ * generating R_386_32 (rather than R_386_RELATIVE) relocations for them when
+ * the 32-bit compressed kernel is linked as PIE. This is no longer necessary,
+ * but it doesn't hurt to keep them .hidden.
*/
.hidden _bss
.hidden _ebss
- .hidden _got
- .hidden _egot
.hidden _end
__HEAD
leal (BP_scratch+4)(%esi), %esp
call 1f
1: popl %edx
- subl $1b, %edx
+ addl $_GLOBAL_OFFSET_TABLE_+(.-1b), %edx
/* Load new GDT */
- leal gdt(%edx), %eax
+ leal gdt@GOTOFF(%edx), %eax
movl %eax, 2(%eax)
lgdt (%eax)
movl %eax, %ss
/*
- * %edx contains the address we are loaded at by the boot loader and %ebx
- * contains the address where we should move the kernel image temporarily
- * for safe in-place decompression. %ebp contains the address that the kernel
- * will be decompressed to.
+ * %edx contains the address we are loaded at by the boot loader (plus the
+ * offset to the GOT). The below code calculates %ebx to be the address where
+ * we should move the kernel image temporarily for safe in-place decompression
+ * (again, plus the offset to the GOT).
+ *
+ * %ebp is calculated to be the address that the kernel will be decompressed to.
*/
#ifdef CONFIG_RELOCATABLE
- movl %edx, %ebx
+ leal startup_32@GOTOFF(%edx), %ebx
#ifdef CONFIG_EFI_STUB
/*
* image_offset = startup_32 - image_base
* Otherwise image_offset will be zero and has no effect on the calculations.
*/
- subl image_offset(%edx), %ebx
+ subl image_offset@GOTOFF(%edx), %ebx
#endif
movl BP_kernel_alignment(%esi), %eax
movl %ebx, %ebp // Save the output address for later
/* Target address to relocate to for decompression */
addl BP_init_size(%esi), %ebx
- subl $_end, %ebx
+ subl $_end@GOTOFF, %ebx
/* Set up the stack */
- leal boot_stack_end(%ebx), %esp
+ leal boot_stack_end@GOTOFF(%ebx), %esp
/* Zero EFLAGS */
pushl $0
* where decompression in place becomes safe.
*/
pushl %esi
- leal (_bss-4)(%edx), %esi
- leal (_bss-4)(%ebx), %edi
+ leal (_bss@GOTOFF-4)(%edx), %esi
+ leal (_bss@GOTOFF-4)(%ebx), %edi
movl $(_bss - startup_32), %ecx
shrl $2, %ecx
std
* during extract_kernel below. To avoid any issues, repoint the GDTR
* to the new copy of the GDT.
*/
- leal gdt(%ebx), %eax
+ leal gdt@GOTOFF(%ebx), %eax
movl %eax, 2(%eax)
lgdt (%eax)
/*
* Jump to the relocated address.
*/
- leal .Lrelocated(%ebx), %eax
+ leal .Lrelocated@GOTOFF(%ebx), %eax
jmp *%eax
SYM_FUNC_END(startup_32)
add $0x4, %esp
movl 8(%esp), %esi /* save boot_params pointer */
call efi_main
- leal startup_32(%eax), %eax
+ /* efi_main returns the possibly relocated address of startup_32 */
jmp *%eax
SYM_FUNC_END(efi32_stub_entry)
SYM_FUNC_END_ALIAS(efi_stub_entry)
* Clear BSS (stack is currently empty)
*/
xorl %eax, %eax
- leal _bss(%ebx), %edi
- leal _ebss(%ebx), %ecx
+ leal _bss@GOTOFF(%ebx), %edi
+ leal _ebss@GOTOFF(%ebx), %ecx
subl %edi, %ecx
shrl $2, %ecx
rep stosl
-/*
- * Adjust our own GOT
- */
- leal _got(%ebx), %edx
- leal _egot(%ebx), %ecx
-1:
- cmpl %ecx, %edx
- jae 2f
- addl %ebx, (%edx)
- addl $4, %edx
- jmp 1b
-2:
-
/*
* Do the extraction, and jump to the new kernel..
*/
- /* push arguments for extract_kernel: */
- pushl $z_output_len /* decompressed length, end of relocs */
-
- pushl %ebp /* output address */
-
- pushl $z_input_len /* input_len */
- leal input_data(%ebx), %eax
- pushl %eax /* input_data */
- leal boot_heap(%ebx), %eax
- pushl %eax /* heap area */
- pushl %esi /* real mode pointer */
- call extract_kernel /* returns kernel location in %eax */
+ /* push arguments for extract_kernel: */
+
+ pushl output_len@GOTOFF(%ebx) /* decompressed length, end of relocs */
+ pushl %ebp /* output address */
+ pushl input_len@GOTOFF(%ebx) /* input_len */
+ leal input_data@GOTOFF(%ebx), %eax
+ pushl %eax /* input_data */
+ leal boot_heap@GOTOFF(%ebx), %eax
+ pushl %eax /* heap area */
+ pushl %esi /* real mode pointer */
+ call extract_kernel /* returns kernel location in %eax */
addl $24, %esp
/*
*/
.hidden _bss
.hidden _ebss
- .hidden _got
- .hidden _egot
.hidden _end
__HEAD
+
+/*
+ * This macro gives the relative virtual address of X, i.e. the offset of X
+ * from startup_32. This is the same as the link-time virtual address of X,
+ * since startup_32 is at 0, but defining it this way tells the
+ * assembler/linker that we do not want the actual run-time address of X. This
+ * prevents the linker from trying to create unwanted run-time relocation
+ * entries for the reference when the compressed kernel is linked as PIE.
+ *
+ * A reference X(%reg) will result in the link-time VA of X being stored with
+ * the instruction, and a run-time R_X86_64_RELATIVE relocation entry that
+ * adds the 64-bit base address where the kernel is loaded.
+ *
+ * Replacing it with (X-startup_32)(%reg) results in the offset being stored,
+ * and no run-time relocation.
+ *
+ * The macro should be used as a displacement with a base register containing
+ * the run-time address of startup_32 [i.e. rva(X)(%reg)], or as an immediate
+ * [$ rva(X)].
+ *
+ * This macro can only be used from within the .head.text section, since the
+ * expression requires startup_32 to be in the same section as the code being
+ * assembled.
+ */
+#define rva(X) ((X) - startup_32)
+
.code32
SYM_FUNC_START(startup_32)
/*
leal (BP_scratch+4)(%esi), %esp
call 1f
1: popl %ebp
- subl $1b, %ebp
+ subl $ rva(1b), %ebp
/* Load new GDT with the 64bit segments using 32bit descriptor */
- leal gdt(%ebp), %eax
+ leal rva(gdt)(%ebp), %eax
movl %eax, 2(%eax)
lgdt (%eax)
movl %eax, %ss
/* setup a stack and make sure cpu supports long mode. */
- leal boot_stack_end(%ebp), %esp
+ leal rva(boot_stack_end)(%ebp), %esp
call verify_cpu
testl %eax, %eax
* image_offset = startup_32 - image_base
* Otherwise image_offset will be zero and has no effect on the calculations.
*/
- subl image_offset(%ebp), %ebx
+ subl rva(image_offset)(%ebp), %ebx
#endif
movl BP_kernel_alignment(%esi), %eax
/* Target address to relocate to for decompression */
addl BP_init_size(%esi), %ebx
- subl $_end, %ebx
+ subl $ rva(_end), %ebx
/*
* Prepare for entering 64 bit mode
1:
/* Initialize Page tables to 0 */
- leal pgtable(%ebx), %edi
+ leal rva(pgtable)(%ebx), %edi
xorl %eax, %eax
movl $(BOOT_INIT_PGT_SIZE/4), %ecx
rep stosl
/* Build Level 4 */
- leal pgtable + 0(%ebx), %edi
+ leal rva(pgtable + 0)(%ebx), %edi
leal 0x1007 (%edi), %eax
movl %eax, 0(%edi)
addl %edx, 4(%edi)
/* Build Level 3 */
- leal pgtable + 0x1000(%ebx), %edi
+ leal rva(pgtable + 0x1000)(%ebx), %edi
leal 0x1007(%edi), %eax
movl $4, %ecx
1: movl %eax, 0x00(%edi)
jnz 1b
/* Build Level 2 */
- leal pgtable + 0x2000(%ebx), %edi
+ leal rva(pgtable + 0x2000)(%ebx), %edi
movl $0x00000183, %eax
movl $2048, %ecx
1: movl %eax, 0(%edi)
jnz 1b
/* Enable the boot page tables */
- leal pgtable(%ebx), %eax
+ leal rva(pgtable)(%ebx), %eax
movl %eax, %cr3
/* Enable Long mode in EFER (Extended Feature Enable Register) */
* We place all of the values on our mini stack so lret can
* used to perform that far jump.
*/
- leal startup_64(%ebp), %eax
+ leal rva(startup_64)(%ebp), %eax
#ifdef CONFIG_EFI_MIXED
- movl efi32_boot_args(%ebp), %edi
+ movl rva(efi32_boot_args)(%ebp), %edi
cmp $0, %edi
jz 1f
- leal efi64_stub_entry(%ebp), %eax
- movl efi32_boot_args+4(%ebp), %esi
- movl efi32_boot_args+8(%ebp), %edx // saved bootparams pointer
+ leal rva(efi64_stub_entry)(%ebp), %eax
+ movl rva(efi32_boot_args+4)(%ebp), %esi
+ movl rva(efi32_boot_args+8)(%ebp), %edx // saved bootparams pointer
cmpl $0, %edx
jnz 1f
/*
* the correct stack alignment for entry.
*/
subl $40, %esp
- leal efi_pe_entry(%ebp), %eax
+ leal rva(efi_pe_entry)(%ebp), %eax
movl %edi, %ecx // MS calling convention
movl %esi, %edx
1:
call 1f
1: pop %ebp
- subl $1b, %ebp
+ subl $ rva(1b), %ebp
- movl %esi, efi32_boot_args+8(%ebp)
+ movl %esi, rva(efi32_boot_args+8)(%ebp)
SYM_INNER_LABEL(efi32_pe_stub_entry, SYM_L_LOCAL)
- movl %ecx, efi32_boot_args(%ebp)
- movl %edx, efi32_boot_args+4(%ebp)
- movb $0, efi_is64(%ebp)
+ movl %ecx, rva(efi32_boot_args)(%ebp)
+ movl %edx, rva(efi32_boot_args+4)(%ebp)
+ movb $0, rva(efi_is64)(%ebp)
/* Save firmware GDTR and code/data selectors */
- sgdtl efi32_boot_gdt(%ebp)
- movw %cs, efi32_boot_cs(%ebp)
- movw %ds, efi32_boot_ds(%ebp)
+ sgdtl rva(efi32_boot_gdt)(%ebp)
+ movw %cs, rva(efi32_boot_cs)(%ebp)
+ movw %ds, rva(efi32_boot_ds)(%ebp)
/* Disable paging */
movl %cr0, %eax
/* Target address to relocate to for decompression */
movl BP_init_size(%rsi), %ebx
- subl $_end, %ebx
+ subl $ rva(_end), %ebx
addq %rbp, %rbx
/* Set up the stack */
- leaq boot_stack_end(%rbx), %rsp
-
- /*
- * paging_prepare() and cleanup_trampoline() below can have GOT
- * references. Adjust the table with address we are running at.
- *
- * Zero RAX for adjust_got: the GOT was not adjusted before;
- * there's no adjustment to undo.
- */
- xorq %rax, %rax
-
- /*
- * Calculate the address the binary is loaded at and use it as
- * a GOT adjustment.
- */
- call 1f
-1: popq %rdi
- subq $1b, %rdi
-
- call .Ladjust_got
+ leaq rva(boot_stack_end)(%rbx), %rsp
/*
* At this point we are in long mode with 4-level paging enabled,
lretq
trampoline_return:
/* Restore the stack, the 32-bit trampoline uses its own stack */
- leaq boot_stack_end(%rbx), %rsp
+ leaq rva(boot_stack_end)(%rbx), %rsp
/*
* cleanup_trampoline() would restore trampoline memory.
* this function call.
*/
pushq %rsi
- leaq top_pgtable(%rbx), %rdi
+ leaq rva(top_pgtable)(%rbx), %rdi
call cleanup_trampoline
popq %rsi
pushq $0
popfq
- /*
- * Previously we've adjusted the GOT with address the binary was
- * loaded at. Now we need to re-adjust for relocation address.
- *
- * Calculate the address the binary is loaded at, so that we can
- * undo the previous GOT adjustment.
- */
- call 1f
-1: popq %rax
- subq $1b, %rax
-
- /* The new adjustment is the relocation address */
- movq %rbx, %rdi
- call .Ladjust_got
-
/*
* Copy the compressed kernel to the end of our buffer
* where decompression in place becomes safe.
*/
pushq %rsi
leaq (_bss-8)(%rip), %rsi
- leaq (_bss-8)(%rbx), %rdi
- movq $_bss /* - $startup_32 */, %rcx
- shrq $3, %rcx
+ leaq rva(_bss-8)(%rbx), %rdi
+ movl $(_bss - startup_32), %ecx
+ shrl $3, %ecx
std
rep movsq
cld
* during extract_kernel below. To avoid any issues, repoint the GDTR
* to the new copy of the GDT.
*/
- leaq gdt64(%rbx), %rax
- leaq gdt(%rbx), %rdx
+ leaq rva(gdt64)(%rbx), %rax
+ leaq rva(gdt)(%rbx), %rdx
movq %rdx, 2(%rax)
lgdt (%rax)
/*
* Jump to the relocated address.
*/
- leaq .Lrelocated(%rbx), %rax
+ leaq rva(.Lrelocated)(%rbx), %rax
jmp *%rax
SYM_CODE_END(startup_64)
movq %rdx, %rbx /* save boot_params pointer */
call efi_main
movq %rbx,%rsi
- leaq startup_64(%rax), %rax
+ leaq rva(startup_64)(%rax), %rax
jmp *%rax
SYM_FUNC_END(efi64_stub_entry)
SYM_FUNC_END_ALIAS(efi_stub_entry)
movq %rsi, %rdi /* real mode address */
leaq boot_heap(%rip), %rsi /* malloc area for uncompression */
leaq input_data(%rip), %rdx /* input_data */
- movl $z_input_len, %ecx /* input_len */
+ movl input_len(%rip), %ecx /* input_len */
movq %rbp, %r8 /* output target address */
- movl $z_output_len, %r9d /* decompressed length, end of relocs */
+ movl output_len(%rip), %r9d /* decompressed length, end of relocs */
call extract_kernel /* returns kernel location in %rax */
popq %rsi
jmp *%rax
SYM_FUNC_END(.Lrelocated)
-/*
- * Adjust the global offset table
- *
- * RAX is the previous adjustment of the table to undo (use 0 if it's the
- * first time we touch GOT).
- * RDI is the new adjustment to apply.
- */
-.Ladjust_got:
- /* Walk through the GOT adding the address to the entries */
- leaq _got(%rip), %rdx
- leaq _egot(%rip), %rcx
-1:
- cmpq %rcx, %rdx
- jae 2f
- subq %rax, (%rdx) /* Undo previous adjustment */
- addq %rdi, (%rdx) /* Apply the new adjustment */
- addq $8, %rdx
- jmp 1b
-2:
- ret
-
.code32
/*
* This is the 32-bit trampoline that will be copied over to low memory.
#define BS32_handle_protocol 88 // offsetof(efi_boot_services_32_t, handle_protocol)
#define LI32_image_base 32 // offsetof(efi_loaded_image_32_t, image_base)
- .text
+ __HEAD
.code32
SYM_FUNC_START(efi32_pe_entry)
/*
call 1f
1: pop %ebx
- subl $1b, %ebx
+ subl $ rva(1b), %ebx
/* Get the loaded image protocol pointer from the image handle */
leal -4(%ebp), %eax
pushl %eax // &loaded_image
- leal loaded_image_proto(%ebx), %eax
+ leal rva(loaded_image_proto)(%ebx), %eax
pushl %eax // pass the GUID address
pushl 8(%ebp) // pass the image handle
* use it before we get to the 64-bit efi_pe_entry() in C code.
*/
subl %esi, %ebx
- movl %ebx, image_offset(%ebp) // save image_offset
+ movl %ebx, rva(image_offset)(%ebp) // save image_offset
jmp efi32_pe_stub_entry
2: popl %edi // restore callee-save registers
#define STATIC
#include <linux/decompress/mm.h>
+#define _SETUP
+#include <asm/setup.h> /* For COMMAND_LINE_SIZE */
+#undef _SETUP
+
#ifdef CONFIG_X86_5LEVEL
unsigned int __pgtable_l5_enabled;
unsigned int pgdir_shift __ro_after_init = 39;
static bool memmap_too_large;
-/* Store memory limit specified by "mem=nn[KMG]" or "memmap=nn[KMG]" */
-static unsigned long long mem_limit = ULLONG_MAX;
+/*
+ * Store memory limit: MAXMEM on 64-bit and KERNEL_IMAGE_SIZE on 32-bit.
+ * It may be reduced by "mem=nn[KMG]" or "memmap=nn[KMG]" command line options.
+ */
+static u64 mem_limit;
/* Number of immovable memory regions */
static int num_immovable_mem;
};
static int
-parse_memmap(char *p, unsigned long long *start, unsigned long long *size,
- enum parse_mode mode)
+parse_memmap(char *p, u64 *start, u64 *size, enum parse_mode mode)
{
char *oldp;
*/
*size = 0;
} else {
- unsigned long long flags;
+ u64 flags;
/*
* efi_fake_mem=nn@ss:attr the attr specifies
while (str && (i < MAX_MEMMAP_REGIONS)) {
int rc;
- unsigned long long start, size;
+ u64 start, size;
char *k = strchr(str, ',');
if (k)
if (start == 0) {
/* Store the specified memory limit if size > 0 */
- if (size > 0)
+ if (size > 0 && size < mem_limit)
mem_limit = size;
continue;
static void handle_mem_options(void)
{
char *args = (char *)get_cmd_line_ptr();
- size_t len = strlen((char *)args);
+ size_t len;
char *tmp_cmdline;
char *param, *val;
u64 mem_size;
- if (!strstr(args, "memmap=") && !strstr(args, "mem=") &&
- !strstr(args, "hugepages"))
+ if (!args)
return;
+ len = strnlen(args, COMMAND_LINE_SIZE-1);
tmp_cmdline = malloc(len + 1);
if (!tmp_cmdline)
error("Failed to allocate space for tmp_cmdline");
while (*args) {
args = next_arg(args, ¶m, &val);
/* Stop at -- */
- if (!val && strcmp(param, "--") == 0) {
- warn("Only '--' specified in cmdline");
- goto out;
- }
+ if (!val && strcmp(param, "--") == 0)
+ break;
if (!strcmp(param, "memmap")) {
mem_avoid_memmap(PARSE_MEMMAP, val);
- } else if (strstr(param, "hugepages")) {
+ } else if (IS_ENABLED(CONFIG_X86_64) && strstr(param, "hugepages")) {
parse_gb_huge_pages(param, val);
} else if (!strcmp(param, "mem")) {
char *p = val;
continue;
mem_size = memparse(p, &p);
if (mem_size == 0)
- goto out;
+ break;
- mem_limit = mem_size;
+ if (mem_size < mem_limit)
+ mem_limit = mem_size;
} else if (!strcmp(param, "efi_fake_mem")) {
mem_avoid_memmap(PARSE_EFI, val);
}
}
-out:
free(tmp_cmdline);
return;
}
/*
- * In theory, KASLR can put the kernel anywhere in the range of [16M, 64T).
+ * In theory, KASLR can put the kernel anywhere in the range of [16M, MAXMEM)
+ * on 64-bit, and [16M, KERNEL_IMAGE_SIZE) on 32-bit.
+ *
* The mem_avoid array is used to store the ranges that need to be avoided
* when KASLR searches for an appropriate random address. We must avoid any
* regions that are unsafe to overlap with during decompression, and other
{
unsigned long init_size = boot_params->hdr.init_size;
u64 initrd_start, initrd_size;
- u64 cmd_line, cmd_line_size;
- char *ptr;
+ unsigned long cmd_line, cmd_line_size;
/*
* Avoid the region that is unsafe to overlap during
/* No need to set mapping for initrd, it will be handled in VO. */
/* Avoid kernel command line. */
- cmd_line = (u64)boot_params->ext_cmd_line_ptr << 32;
- cmd_line |= boot_params->hdr.cmd_line_ptr;
+ cmd_line = get_cmd_line_ptr();
/* Calculate size of cmd_line. */
- ptr = (char *)(unsigned long)cmd_line;
- for (cmd_line_size = 0; ptr[cmd_line_size++];)
- ;
- mem_avoid[MEM_AVOID_CMDLINE].start = cmd_line;
- mem_avoid[MEM_AVOID_CMDLINE].size = cmd_line_size;
- add_identity_map(mem_avoid[MEM_AVOID_CMDLINE].start,
- mem_avoid[MEM_AVOID_CMDLINE].size);
+ if (cmd_line) {
+ cmd_line_size = strnlen((char *)cmd_line, COMMAND_LINE_SIZE-1) + 1;
+ mem_avoid[MEM_AVOID_CMDLINE].start = cmd_line;
+ mem_avoid[MEM_AVOID_CMDLINE].size = cmd_line_size;
+ add_identity_map(mem_avoid[MEM_AVOID_CMDLINE].start,
+ mem_avoid[MEM_AVOID_CMDLINE].size);
+ }
/* Avoid boot parameters. */
mem_avoid[MEM_AVOID_BOOTPARAMS].start = (unsigned long)boot_params;
{
int i;
struct setup_data *ptr;
- unsigned long earliest = img->start + img->size;
+ u64 earliest = img->start + img->size;
bool is_overlapping = false;
for (i = 0; i < MEM_AVOID_MAX; i++) {
}
struct slot_area {
- unsigned long addr;
- int num;
+ u64 addr;
+ unsigned long num;
};
#define MAX_SLOT_AREA 100
static struct slot_area slot_areas[MAX_SLOT_AREA];
-
+static unsigned int slot_area_index;
static unsigned long slot_max;
-static unsigned long slot_area_index;
-
static void store_slot_info(struct mem_vector *region, unsigned long image_size)
{
struct slot_area slot_area;
return;
slot_area.addr = region->start;
- slot_area.num = (region->size - image_size) /
- CONFIG_PHYSICAL_ALIGN + 1;
+ slot_area.num = 1 + (region->size - image_size) / CONFIG_PHYSICAL_ALIGN;
- if (slot_area.num > 0) {
- slot_areas[slot_area_index++] = slot_area;
- slot_max += slot_area.num;
- }
+ slot_areas[slot_area_index++] = slot_area;
+ slot_max += slot_area.num;
}
/*
static void
process_gb_huge_pages(struct mem_vector *region, unsigned long image_size)
{
- unsigned long addr, size = 0;
+ u64 pud_start, pud_end;
+ unsigned long gb_huge_pages;
struct mem_vector tmp;
- int i = 0;
- if (!max_gb_huge_pages) {
+ if (!IS_ENABLED(CONFIG_X86_64) || !max_gb_huge_pages) {
store_slot_info(region, image_size);
return;
}
- addr = ALIGN(region->start, PUD_SIZE);
- /* Did we raise the address above the passed in memory entry? */
- if (addr < region->start + region->size)
- size = region->size - (addr - region->start);
-
- /* Check how many 1GB huge pages can be filtered out: */
- while (size > PUD_SIZE && max_gb_huge_pages) {
- size -= PUD_SIZE;
- max_gb_huge_pages--;
- i++;
- }
+ /* Are there any 1GB pages in the region? */
+ pud_start = ALIGN(region->start, PUD_SIZE);
+ pud_end = ALIGN_DOWN(region->start + region->size, PUD_SIZE);
/* No good 1GB huge pages found: */
- if (!i) {
+ if (pud_start >= pud_end) {
store_slot_info(region, image_size);
return;
}
- /*
- * Skip those 'i'*1GB good huge pages, and continue checking and
- * processing the remaining head or tail part of the passed region
- * if available.
- */
-
- if (addr >= region->start + image_size) {
+ /* Check if the head part of the region is usable. */
+ if (pud_start >= region->start + image_size) {
tmp.start = region->start;
- tmp.size = addr - region->start;
+ tmp.size = pud_start - region->start;
store_slot_info(&tmp, image_size);
}
- size = region->size - (addr - region->start) - i * PUD_SIZE;
- if (size >= image_size) {
- tmp.start = addr + i * PUD_SIZE;
- tmp.size = size;
+ /* Skip the good 1GB pages. */
+ gb_huge_pages = (pud_end - pud_start) >> PUD_SHIFT;
+ if (gb_huge_pages > max_gb_huge_pages) {
+ pud_end = pud_start + (max_gb_huge_pages << PUD_SHIFT);
+ max_gb_huge_pages = 0;
+ } else {
+ max_gb_huge_pages -= gb_huge_pages;
+ }
+
+ /* Check if the tail part of the region is usable. */
+ if (region->start + region->size >= pud_end + image_size) {
+ tmp.start = pud_end;
+ tmp.size = region->start + region->size - pud_end;
store_slot_info(&tmp, image_size);
}
}
-static unsigned long slots_fetch_random(void)
+static u64 slots_fetch_random(void)
{
unsigned long slot;
- int i;
+ unsigned int i;
/* Handle case of no slots stored. */
if (slot_max == 0)
slot -= slot_areas[i].num;
continue;
}
- return slot_areas[i].addr + slot * CONFIG_PHYSICAL_ALIGN;
+ return slot_areas[i].addr + ((u64)slot * CONFIG_PHYSICAL_ALIGN);
}
if (i == slot_area_index)
unsigned long image_size)
{
struct mem_vector region, overlap;
- unsigned long start_orig, end;
- struct mem_vector cur_entry;
-
- /* On 32-bit, ignore entries entirely above our maximum. */
- if (IS_ENABLED(CONFIG_X86_32) && entry->start >= KERNEL_IMAGE_SIZE)
- return;
+ u64 region_end;
- /* Ignore entries entirely below our minimum. */
- if (entry->start + entry->size < minimum)
- return;
-
- /* Ignore entries above memory limit */
- end = min(entry->size + entry->start, mem_limit);
- if (entry->start >= end)
- return;
- cur_entry.start = entry->start;
- cur_entry.size = end - entry->start;
-
- region.start = cur_entry.start;
- region.size = cur_entry.size;
+ /* Enforce minimum and memory limit. */
+ region.start = max_t(u64, entry->start, minimum);
+ region_end = min(entry->start + entry->size, mem_limit);
/* Give up if slot area array is full. */
while (slot_area_index < MAX_SLOT_AREA) {
- start_orig = region.start;
-
- /* Potentially raise address to minimum location. */
- if (region.start < minimum)
- region.start = minimum;
-
/* Potentially raise address to meet alignment needs. */
region.start = ALIGN(region.start, CONFIG_PHYSICAL_ALIGN);
/* Did we raise the address above the passed in memory entry? */
- if (region.start > cur_entry.start + cur_entry.size)
+ if (region.start > region_end)
return;
/* Reduce size by any delta from the original address. */
- region.size -= region.start - start_orig;
-
- /* On 32-bit, reduce region size to fit within max size. */
- if (IS_ENABLED(CONFIG_X86_32) &&
- region.start + region.size > KERNEL_IMAGE_SIZE)
- region.size = KERNEL_IMAGE_SIZE - region.start;
+ region.size = region_end - region.start;
/* Return if region can't contain decompressed kernel */
if (region.size < image_size)
}
/* Store beginning of region if holds at least image_size. */
- if (overlap.start > region.start + image_size) {
- struct mem_vector beginning;
-
- beginning.start = region.start;
- beginning.size = overlap.start - region.start;
- process_gb_huge_pages(&beginning, image_size);
+ if (overlap.start >= region.start + image_size) {
+ region.size = overlap.start - region.start;
+ process_gb_huge_pages(®ion, image_size);
}
- /* Return if overlap extends to or past end of region. */
- if (overlap.start + overlap.size >= region.start + region.size)
- return;
-
/* Clip off the overlapping region and start over. */
- region.size -= overlap.start - region.start + overlap.size;
region.start = overlap.start + overlap.size;
}
}
static bool process_mem_region(struct mem_vector *region,
- unsigned long long minimum,
- unsigned long long image_size)
+ unsigned long minimum,
+ unsigned long image_size)
{
int i;
/*
* immovable memory and @region.
*/
for (i = 0; i < num_immovable_mem; i++) {
- unsigned long long start, end, entry_end, region_end;
+ u64 start, end, entry_end, region_end;
struct mem_vector entry;
if (!mem_overlaps(region, &immovable_mem[i]))
#ifdef CONFIG_EFI
/*
- * Returns true if mirror region found (and must have been processed
- * for slots adding)
+ * Returns true if we processed the EFI memmap, which we prefer over the E820
+ * table if it is available.
*/
static bool
process_efi_entries(unsigned long minimum, unsigned long image_size)
static unsigned long find_random_phys_addr(unsigned long minimum,
unsigned long image_size)
{
+ u64 phys_addr;
+
+ /* Bail out early if it's impossible to succeed. */
+ if (minimum + image_size > mem_limit)
+ return 0;
+
/* Check if we had too many memmaps. */
if (memmap_too_large) {
debug_putstr("Aborted memory entries scan (more than 4 memmap= args)!\n");
return 0;
}
- /* Make sure minimum is aligned. */
- minimum = ALIGN(minimum, CONFIG_PHYSICAL_ALIGN);
+ if (!process_efi_entries(minimum, image_size))
+ process_e820_entries(minimum, image_size);
- if (process_efi_entries(minimum, image_size))
- return slots_fetch_random();
+ phys_addr = slots_fetch_random();
- process_e820_entries(minimum, image_size);
- return slots_fetch_random();
+ /* Perform a final check to make sure the address is in range. */
+ if (phys_addr < minimum || phys_addr + image_size > mem_limit) {
+ warn("Invalid physical address chosen!\n");
+ return 0;
+ }
+
+ return (unsigned long)phys_addr;
}
static unsigned long find_random_virt_addr(unsigned long minimum,
{
unsigned long slots, random_addr;
- /* Make sure minimum is aligned. */
- minimum = ALIGN(minimum, CONFIG_PHYSICAL_ALIGN);
- /* Align image_size for easy slot calculations. */
- image_size = ALIGN(image_size, CONFIG_PHYSICAL_ALIGN);
-
/*
* There are how many CONFIG_PHYSICAL_ALIGN-sized slots
* that can hold image_size within the range of minimum to
* KERNEL_IMAGE_SIZE?
*/
- slots = (KERNEL_IMAGE_SIZE - minimum - image_size) /
- CONFIG_PHYSICAL_ALIGN + 1;
+ slots = 1 + (KERNEL_IMAGE_SIZE - minimum - image_size) / CONFIG_PHYSICAL_ALIGN;
random_addr = kaslr_get_random_long("Virtual") % slots;
/* Prepare to add new identity pagetables on demand. */
initialize_identity_maps();
+ if (IS_ENABLED(CONFIG_X86_32))
+ mem_limit = KERNEL_IMAGE_SIZE;
+ else
+ mem_limit = MAXMEM;
+
/* Record the various known unsafe memory ranges. */
mem_avoid_init(input, input_size, *output);
* location:
*/
min_addr = min(*output, 512UL << 20);
+ /* Make sure minimum is aligned. */
+ min_addr = ALIGN(min_addr, CONFIG_PHYSICAL_ALIGN);
/* Walk available memory entries to find a random address. */
random_addr = find_random_phys_addr(min_addr, output_size);
int cmdline_find_option_bool(const char *option);
struct mem_vector {
- unsigned long long start;
- unsigned long long size;
+ u64 start;
+ u64 size;
};
#if CONFIG_RANDOMIZE_BASE
printf(".incbin \"%s\"\n", argv[1]);
printf("input_data_end:\n");
+ printf(".section \".rodata\",\"a\",@progbits\n");
+ printf(".globl input_len\n");
+ printf("input_len:\n\t.long %lu\n", ilen);
+ printf(".globl output_len\n");
+ printf("output_len:\n\t.long %lu\n", (unsigned long)olen);
+
retval = 0;
bail:
if (f)
*(.rodata.*)
_erodata = . ;
}
- .got : {
- _got = .;
- KEEP(*(.got.plt))
- KEEP(*(.got))
- _egot = .;
- }
.data : {
_data = . ;
*(.data)
. = ALIGN(PAGE_SIZE); /* keep ZO size page aligned */
_end = .;
+ STABS_DEBUG
+ DWARF_DEBUG
+ ELF_DETAILS
+
DISCARDS
+ /DISCARD/ : {
+ *(.dynamic) *(.dynsym) *(.dynstr) *(.dynbss)
+ *(.hash) *(.gnu.hash)
+ *(.note.*)
+ }
+
+ .got.plt (INFO) : {
+ *(.got.plt)
+ }
+ ASSERT(SIZEOF(.got.plt) == 0 ||
+#ifdef CONFIG_X86_64
+ SIZEOF(.got.plt) == 0x18,
+#else
+ SIZEOF(.got.plt) == 0xc,
+#endif
+ "Unexpected GOT/PLT entries detected!")
+
+ /*
+ * Sections that should stay zero sized, which is safer to
+ * explicitly check instead of blindly discarding.
+ */
+ .got : {
+ *(.got)
+ }
+ ASSERT(SIZEOF(.got) == 0, "Unexpected GOT entries detected!")
+
+ .plt : {
+ *(.plt) *(.plt.*)
+ }
+ ASSERT(SIZEOF(.plt) == 0, "Unexpected run-time procedure linkages detected!")
+
+ .rel.dyn : {
+ *(.rel.*) *(.rel_*)
+ }
+ ASSERT(SIZEOF(.rel.dyn) == 0, "Unexpected run-time relocations (.rel) detected!")
+
+ .rela.dyn : {
+ *(.rela.*) *(.rela_*)
+ }
+ ASSERT(SIZEOF(.rela.dyn) == 0, "Unexpected run-time relocations (.rela) detected!")
}
.initdata : { *(.initdata) }
__end_init = .;
- .text : { *(.text) }
+ .text : { *(.text .text.*) }
.text32 : { *(.text32) }
. = ALIGN(16);
/* Set the default root device */
put_unaligned_le16(DEFAULT_ROOT_DEV, &buf[508]);
- printf("Setup is %d bytes (padded to %d bytes).\n", c, i);
-
/* Open and stat the kernel file */
fd = open(argv[2], O_RDONLY);
if (fd < 0)
if (fstat(fd, &sb))
die("Unable to stat `%s': %m", argv[2]);
sz = sb.st_size;
- printf("System is %d kB\n", (sz+1023)/1024);
kernel = mmap(NULL, sz, PROT_READ, MAP_SHARED, fd, 0);
if (kernel == MAP_FAILED)
die("Unable to mmap '%s': %m", argv[2]);
}
/* Write the CRC */
- printf("CRC %x\n", crc);
put_unaligned_le32(crc, buf);
if (fwrite(buf, 1, 4, dest) != 4)
die("Writing CRC failed");
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
+#include <linux/sizes.h>
#include <asm/cpufeature.h>
#include <asm/fpu/api.h>
#include <crypto/internal/skcipher.h>
#include <linux/kernel.h>
#include <linux/module.h>
+#include <linux/sizes.h>
#include <asm/simd.h>
asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
#define SCALE_F sizeof(unsigned long)
#ifdef CONFIG_X86_64
-#define REX_PRE "0x48, "
+#define CRC32_INST "crc32q %1, %q0"
#else
-#define REX_PRE
+#define CRC32_INST "crc32l %1, %0"
#endif
#ifdef CONFIG_X86_64
static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length)
{
while (length--) {
- __asm__ __volatile__(
- ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
- :"=S"(crc)
- :"0"(crc), "c"(*data)
- );
+ asm("crc32b %1, %0"
+ : "+r" (crc) : "rm" (*data));
data++;
}
unsigned long *ptmp = (unsigned long *)p;
while (iquotient--) {
- __asm__ __volatile__(
- ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;"
- :"=S"(crc)
- :"0"(crc), "c"(*ptmp)
- );
+ asm(CRC32_INST
+ : "+r" (crc) : "rm" (*ptmp));
ptmp++;
}
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
+#include <linux/scatterlist.h>
#include <asm/cpufeature.h>
#include <asm/processor.h>
asm volatile(
/* Clear registers to propagate the carry bit */
- " xor %%r8, %%r8;"
- " xor %%r9, %%r9;"
- " xor %%r10, %%r10;"
- " xor %%r11, %%r11;"
- " xor %1, %1;"
+ " xor %%r8d, %%r8d;"
+ " xor %%r9d, %%r9d;"
+ " xor %%r10d, %%r10d;"
+ " xor %%r11d, %%r11d;"
+ " xor %k1, %k1;"
/* Begin addition chain */
" addq 0(%3), %0;"
" cmovc %0, %%rax;"
/* Step 2: Add carry*38 to the original sum */
- " xor %%rcx, %%rcx;"
+ " xor %%ecx, %%ecx;"
" add %%rax, %%r8;"
" adcx %%rcx, %%r9;"
" movq %%r9, 8(%1);"
/* Compute src1[0] * src2 */
" movq 0(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 0(%0);"
+ " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;"
/* Compute src1[1] * src2 */
" movq 8(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);"
+ " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
/* Compute src1[2] * src2 */
" movq 16(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);"
+ " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
/* Compute src1[3] * src2 */
" movq 24(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);"
+ " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 32(%1), %%r8, %%r13;"
- " xor %3, %3;"
+ " xor %k3, %k3;"
" adoxq 0(%1), %%r8;"
" mulxq 40(%1), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
/* Compute src1[0] * src2 */
" movq 0(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 0(%0);"
+ " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;"
/* Compute src1[1] * src2 */
" movq 8(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);"
+ " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
/* Compute src1[2] * src2 */
" movq 16(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);"
+ " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
/* Compute src1[3] * src2 */
" movq 24(%1), %%rdx;"
- " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);"
+ " mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;"
/* Compute src1[0] * src2 */
" movq 32(%1), %%rdx;"
- " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 64(%0);"
- " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 72(%0);"
+ " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 64(%0);"
+ " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 72(%0);"
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;"
/* Compute src1[1] * src2 */
" movq 40(%1), %%rdx;"
- " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 72(%0), %%r8;" " movq %%r8, 72(%0);"
- " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 80(%0);"
+ " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 72(%0), %%r8;" " movq %%r8, 72(%0);"
+ " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 80(%0);"
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
/* Compute src1[2] * src2 */
" movq 48(%1), %%rdx;"
- " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 80(%0), %%r8;" " movq %%r8, 80(%0);"
- " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 88(%0);"
+ " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 80(%0), %%r8;" " movq %%r8, 80(%0);"
+ " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 88(%0);"
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
/* Compute src1[3] * src2 */
" movq 56(%1), %%rdx;"
- " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 88(%0), %%r8;" " movq %%r8, 88(%0);"
- " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 96(%0);"
+ " mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 88(%0), %%r8;" " movq %%r8, 88(%0);"
+ " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 96(%0);"
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 104(%0);" " mov $0, %%r8;"
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 112(%0);" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 120(%0);"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 32(%1), %%r8, %%r13;"
- " xor %3, %3;"
+ " xor %k3, %k3;"
" adoxq 0(%1), %%r8;"
" mulxq 40(%1), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 96(%1), %%r8, %%r13;"
- " xor %3, %3;"
+ " xor %k3, %k3;"
" adoxq 64(%1), %%r8;"
" mulxq 104(%1), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
/* Step 1: Compute all partial products */
" movq 0(%1), %%rdx;" /* f[0] */
- " mulxq 8(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */
+ " mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
" mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
" mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
" movq 24(%1), %%rdx;" /* f[3] */
" mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
/* Step 2: Compute two parallel carry chains */
- " xor %%r15, %%r15;"
+ " xor %%r15d, %%r15d;"
" adox %%rax, %%r10;"
" adcx %%r8, %%r8;"
" adox %%rcx, %%r11;"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 32(%1), %%r8, %%r13;"
- " xor %%rcx, %%rcx;"
+ " xor %%ecx, %%ecx;"
" adoxq 0(%1), %%r8;"
" mulxq 40(%1), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
asm volatile(
/* Step 1: Compute all partial products */
" movq 0(%1), %%rdx;" /* f[0] */
- " mulxq 8(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */
+ " mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
" mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
" mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
" movq 24(%1), %%rdx;" /* f[3] */
" mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
/* Step 2: Compute two parallel carry chains */
- " xor %%r15, %%r15;"
+ " xor %%r15d, %%r15d;"
" adox %%rax, %%r10;"
" adcx %%r8, %%r8;"
" adox %%rcx, %%r11;"
/* Step 1: Compute all partial products */
" movq 32(%1), %%rdx;" /* f[0] */
- " mulxq 40(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */
+ " mulxq 40(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
" mulxq 48(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
" mulxq 56(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
" movq 56(%1), %%rdx;" /* f[3] */
" mulxq 48(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
/* Step 2: Compute two parallel carry chains */
- " xor %%r15, %%r15;"
+ " xor %%r15d, %%r15d;"
" adox %%rax, %%r10;"
" adcx %%r8, %%r8;"
" adox %%rcx, %%r11;"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 32(%1), %%r8, %%r13;"
- " xor %%rcx, %%rcx;"
+ " xor %%ecx, %%ecx;"
" adoxq 0(%1), %%r8;"
" mulxq 40(%1), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 96(%1), %%r8, %%r13;"
- " xor %%rcx, %%rcx;"
+ " xor %%ecx, %%ecx;"
" adoxq 64(%1), %%r8;"
" mulxq 104(%1), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
#include <crypto/internal/simd.h>
#include <crypto/nhpoly1305.h>
#include <linux/module.h>
+#include <linux/sizes.h>
#include <asm/simd.h>
asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
#include <crypto/internal/simd.h>
#include <crypto/nhpoly1305.h>
#include <linux/module.h>
+#include <linux/sizes.h>
#include <asm/simd.h>
asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
___
&declare_function("poly1305_init_x86_64", 32, 3);
$code.=<<___;
- xor %rax,%rax
+ xor %eax,%eax
mov %rax,0($ctx) # initialize hash value
mov %rax,8($ctx)
mov %rax,16($ctx)
.type poly1305_init_base2_44,\@function,3
.align 32
poly1305_init_base2_44:
- xor %rax,%rax
+ xor %eax,%eax
mov %rax,0($ctx) # initialize hash value
mov %rax,8($ctx)
mov %rax,16($ctx)
mov \$16,$len
sub %r10,$len
xor %eax,%eax
- xor %r11,%r11
+ xor %r11d,%r11d
.Loop_dec_byte:
mov ($inp,$otp),%r11b
mov ($otp),%al
.long 0xa548f3fc # cld; rep movsq
mov $disp,%rsi
- xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
+ xor %ecx,%ecx # arg1, UNW_FLAG_NHANDLER
mov 8(%rsi),%rdx # arg2, disp->ImageBase
mov 0(%rsi),%r8 # arg3, disp->ControlPc
mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/module.h>
+#include <linux/sizes.h>
#include <asm/intel-family.h>
#include <asm/simd.h>
dctx->s[1] = get_unaligned_le32(&inp[4]);
dctx->s[2] = get_unaligned_le32(&inp[8]);
dctx->s[3] = get_unaligned_le32(&inp[12]);
- inp += POLY1305_BLOCK_SIZE;
- len -= POLY1305_BLOCK_SIZE;
- acc += POLY1305_BLOCK_SIZE;
dctx->sset = true;
}
}
#include <asm/percpu.h>
#include <asm/asm-offsets.h>
#include <asm/processor-flags.h>
-#include <asm/inst.h>
/*
old_regs = set_irq_regs(regs);
instrumentation_begin();
- run_on_irqstack_cond(__xen_pv_evtchn_do_upcall, NULL, regs);
+ run_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs);
instrumentation_begin();
set_irq_regs(old_regs);
.code64
.section .entry.text, "ax"
-#ifdef CONFIG_PARAVIRT
+#ifdef CONFIG_PARAVIRT_XXL
SYM_CODE_START(native_usergs_sysret64)
UNWIND_HINT_EMPTY
swapgs
sysretq
SYM_CODE_END(native_usergs_sysret64)
-#endif /* CONFIG_PARAVIRT */
+#endif /* CONFIG_PARAVIRT_XXL */
/*
* 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
* rdx: Function argument (can be NULL if none)
*/
SYM_FUNC_START(asm_call_on_stack)
+SYM_INNER_LABEL(asm_call_sysvec_on_stack, SYM_L_GLOBAL)
+SYM_INNER_LABEL(asm_call_irq_on_stack, SYM_L_GLOBAL)
/*
* Save the frame pointer unconditionally. This allows the ORC
* unwinder to handle the stack switch.
* retrieve and set the current CPUs kernel GSBASE. The stored value
* has to be restored in paranoid_exit unconditionally.
*
- * The MSR write ensures that no subsequent load is based on a
- * mispredicted GSBASE. No extra FENCE required.
+ * The unconditional write to GS base below ensures that no subsequent
+ * loads based on a mispredicted GS base can happen, therefore no LFENCE
+ * is needed here.
*/
SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx
ret
* Reuse the 64-bit entry points for the x32 versions that occupy different
* slots in the syscall table.
*/
+#define __x32_sys_readv __x64_sys_readv
+#define __x32_sys_writev __x64_sys_writev
#define __x32_sys_getsockopt __x64_sys_getsockopt
#define __x32_sys_setsockopt __x64_sys_setsockopt
+#define __x32_sys_vmsplice __x64_sys_vmsplice
+#define __x32_sys_process_vm_readv __x64_sys_process_vm_readv
+#define __x32_sys_process_vm_writev __x64_sys_process_vm_writev
#define __SYSCALL_64(nr, sym)
18 i386 oldstat sys_stat
19 i386 lseek sys_lseek compat_sys_lseek
20 i386 getpid sys_getpid
-21 i386 mount sys_mount compat_sys_mount
+21 i386 mount sys_mount
22 i386 umount sys_oldumount
23 i386 setuid sys_setuid16
24 i386 getuid sys_getuid16
128 i386 init_module sys_init_module
129 i386 delete_module sys_delete_module
130 i386 get_kernel_syms
-131 i386 quotactl sys_quotactl compat_sys_quotactl32
+131 i386 quotactl sys_quotactl
132 i386 getpgid sys_getpgid
133 i386 fchdir sys_fchdir
134 i386 bdflush sys_bdflush
142 i386 _newselect sys_select compat_sys_select
143 i386 flock sys_flock
144 i386 msync sys_msync
-145 i386 readv sys_readv compat_sys_readv
-146 i386 writev sys_writev compat_sys_writev
+145 i386 readv sys_readv
+146 i386 writev sys_writev
147 i386 getsid sys_getsid
148 i386 fdatasync sys_fdatasync
149 i386 _sysctl sys_ni_syscall
313 i386 splice sys_splice
314 i386 sync_file_range sys_ia32_sync_file_range
315 i386 tee sys_tee
-316 i386 vmsplice sys_vmsplice compat_sys_vmsplice
+316 i386 vmsplice sys_vmsplice
317 i386 move_pages sys_move_pages compat_sys_move_pages
318 i386 getcpu sys_getcpu
319 i386 epoll_pwait sys_epoll_pwait
344 i386 syncfs sys_syncfs
345 i386 sendmmsg sys_sendmmsg compat_sys_sendmmsg
346 i386 setns sys_setns
-347 i386 process_vm_readv sys_process_vm_readv compat_sys_process_vm_readv
-348 i386 process_vm_writev sys_process_vm_writev compat_sys_process_vm_writev
+347 i386 process_vm_readv sys_process_vm_readv
+348 i386 process_vm_writev sys_process_vm_writev
349 i386 kcmp sys_kcmp
350 i386 finit_module sys_finit_module
351 i386 sched_setattr sys_sched_setattr
512 x32 rt_sigaction compat_sys_rt_sigaction
513 x32 rt_sigreturn compat_sys_x32_rt_sigreturn
514 x32 ioctl compat_sys_ioctl
-515 x32 readv compat_sys_readv
-516 x32 writev compat_sys_writev
+515 x32 readv sys_readv
+516 x32 writev sys_writev
517 x32 recvfrom compat_sys_recvfrom
518 x32 sendmsg compat_sys_sendmsg
519 x32 recvmsg compat_sys_recvmsg
529 x32 waitid compat_sys_waitid
530 x32 set_robust_list compat_sys_set_robust_list
531 x32 get_robust_list compat_sys_get_robust_list
-532 x32 vmsplice compat_sys_vmsplice
+532 x32 vmsplice sys_vmsplice
533 x32 move_pages compat_sys_move_pages
534 x32 preadv compat_sys_preadv64
535 x32 pwritev compat_sys_pwritev64
536 x32 rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo
537 x32 recvmmsg compat_sys_recvmmsg_time64
538 x32 sendmmsg compat_sys_sendmmsg
-539 x32 process_vm_readv compat_sys_process_vm_readv
-540 x32 process_vm_writev compat_sys_process_vm_writev
+539 x32 process_vm_readv sys_process_vm_readv
+540 x32 process_vm_writev sys_process_vm_writev
541 x32 setsockopt sys_setsockopt
542 x32 getsockopt sys_getsockopt
543 x32 io_setup compat_sys_io_setup
#undef CONFIG_ILLEGAL_POINTER_VALUE
#undef CONFIG_SPARSEMEM_VMEMMAP
#undef CONFIG_NR_CPUS
+#undef CONFIG_PARAVIRT_XXL
#define CONFIG_X86_32 1
#define CONFIG_PGTABLE_LEVELS 2
u64 max_period;
unsigned long offset_mask[1];
int offset_max;
+ unsigned int fetch_count_reset_broken : 1;
struct cpu_perf_ibs __percpu *pcpu;
struct attribute **format_attrs;
{
u64 count = 0;
- if (config & IBS_OP_VAL)
- count += (config & IBS_OP_MAX_CNT) << 4; /* cnt rolled over */
-
- if (ibs_caps & IBS_CAPS_RDWROPCNT)
- count += (config & IBS_OP_CUR_CNT) >> 32;
+ /*
+ * If the internal 27-bit counter rolled over, the count is MaxCnt
+ * and the lower 7 bits of CurCnt are randomized.
+ * Otherwise CurCnt has the full 27-bit current counter value.
+ */
+ if (config & IBS_OP_VAL) {
+ count = (config & IBS_OP_MAX_CNT) << 4;
+ if (ibs_caps & IBS_CAPS_OPCNTEXT)
+ count += config & IBS_OP_MAX_CNT_EXT_MASK;
+ } else if (ibs_caps & IBS_CAPS_RDWROPCNT) {
+ count = (config & IBS_OP_CUR_CNT) >> 32;
+ }
return count;
}
static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs,
struct hw_perf_event *hwc, u64 config)
{
- wrmsrl(hwc->config_base, hwc->config | config | perf_ibs->enable_mask);
+ u64 tmp = hwc->config | config;
+
+ if (perf_ibs->fetch_count_reset_broken)
+ wrmsrl(hwc->config_base, tmp & ~perf_ibs->enable_mask);
+
+ wrmsrl(hwc->config_base, tmp | perf_ibs->enable_mask);
}
/*
struct hw_perf_event *hwc = &event->hw;
struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
- u64 period;
+ u64 period, config = 0;
if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
return;
hwc->state = 0;
perf_ibs_set_period(perf_ibs, hwc, &period);
+ if (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_OPCNTEXT)) {
+ config |= period & IBS_OP_MAX_CNT_EXT_MASK;
+ period &= ~IBS_OP_MAX_CNT_EXT_MASK;
+ }
+ config |= period >> 4;
+
/*
* Set STARTED before enabling the hardware, such that a subsequent NMI
* must observe it.
*/
set_bit(IBS_STARTED, pcpu->state);
clear_bit(IBS_STOPPING, pcpu->state);
- perf_ibs_enable_event(perf_ibs, hwc, period >> 4);
+ perf_ibs_enable_event(perf_ibs, hwc, config);
perf_event_update_userpage(event);
}
struct perf_ibs_data ibs_data;
int offset, size, check_rip, offset_max, throttle = 0;
unsigned int msr;
- u64 *buf, *config, period;
+ u64 *buf, *config, period, new_config = 0;
if (!test_bit(IBS_STARTED, pcpu->state)) {
fail:
perf_ibs->offset_max,
offset + 1);
} while (offset < offset_max);
+ /*
+ * Read IbsBrTarget, IbsOpData4, and IbsExtdCtl separately
+ * depending on their availability.
+ * Can't add to offset_max as they are staggered
+ */
if (event->attr.sample_type & PERF_SAMPLE_RAW) {
- /*
- * Read IbsBrTarget and IbsOpData4 separately
- * depending on their availability.
- * Can't add to offset_max as they are staggered
- */
- if (ibs_caps & IBS_CAPS_BRNTRGT) {
- rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++);
- size++;
+ if (perf_ibs == &perf_ibs_op) {
+ if (ibs_caps & IBS_CAPS_BRNTRGT) {
+ rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++);
+ size++;
+ }
+ if (ibs_caps & IBS_CAPS_OPDATA4) {
+ rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++);
+ size++;
+ }
}
- if (ibs_caps & IBS_CAPS_OPDATA4) {
- rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++);
+ if (perf_ibs == &perf_ibs_fetch && (ibs_caps & IBS_CAPS_FETCHCTLEXTD)) {
+ rdmsrl(MSR_AMD64_ICIBSEXTDCTL, *buf++);
size++;
}
}
if (throttle) {
perf_ibs_stop(event, 0);
} else {
- period >>= 4;
-
- if ((ibs_caps & IBS_CAPS_RDWROPCNT) &&
- (*config & IBS_OP_CNT_CTL))
- period |= *config & IBS_OP_CUR_CNT_RAND;
+ if (perf_ibs == &perf_ibs_op) {
+ if (ibs_caps & IBS_CAPS_OPCNTEXT) {
+ new_config = period & IBS_OP_MAX_CNT_EXT_MASK;
+ period &= ~IBS_OP_MAX_CNT_EXT_MASK;
+ }
+ if ((ibs_caps & IBS_CAPS_RDWROPCNT) && (*config & IBS_OP_CNT_CTL))
+ new_config |= *config & IBS_OP_CUR_CNT_RAND;
+ }
+ new_config |= period >> 4;
- perf_ibs_enable_event(perf_ibs, hwc, period);
+ perf_ibs_enable_event(perf_ibs, hwc, new_config);
}
perf_event_update_userpage(event);
{
struct attribute **attr = ibs_op_format_attrs;
+ /*
+ * Some chips fail to reset the fetch count when it is written; instead
+ * they need a 0-1 transition of IbsFetchEn.
+ */
+ if (boot_cpu_data.x86 >= 0x16 && boot_cpu_data.x86 <= 0x18)
+ perf_ibs_fetch.fetch_count_reset_broken = 1;
+
perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch");
if (ibs_caps & IBS_CAPS_OPCNT) {
perf_ibs_op.config_mask |= IBS_OP_CNT_CTL;
*attr++ = &format_attr_cnt_ctl.attr;
}
+
+ if (ibs_caps & IBS_CAPS_OPCNTEXT) {
+ perf_ibs_op.max_period |= IBS_OP_MAX_CNT_EXT_MASK;
+ perf_ibs_op.config_mask |= IBS_OP_MAX_CNT_EXT_MASK;
+ perf_ibs_op.cnt_mask |= IBS_OP_MAX_CNT_EXT_MASK;
+ }
+
perf_ibs_pmu_init(&perf_ibs_op, "ibs_op");
register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs");
while (amd_iommu_v2_event_descs[i].attr.attr.name)
i++;
- attrs = kcalloc(i + 1, sizeof(struct attribute **), GFP_KERNEL);
+ attrs = kcalloc(i + 1, sizeof(*attrs), GFP_KERNEL);
if (!attrs)
return -ENOMEM;
}
/*
- * Convert logical CPU number to L3 PMC Config ThreadMask format
+ * Return a full thread and slice mask unless user
+ * has provided them
*/
-static u64 l3_thread_slice_mask(int cpu)
+static u64 l3_thread_slice_mask(u64 config)
{
- u64 thread_mask, core = topology_core_id(cpu);
- unsigned int shift, thread = 0;
+ if (boot_cpu_data.x86 <= 0x18)
+ return ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) |
+ ((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK);
- if (topology_smt_supported() && !topology_is_primary_thread(cpu))
- thread = 1;
-
- if (boot_cpu_data.x86 <= 0x18) {
- shift = AMD64_L3_THREAD_SHIFT + 2 * (core % 4) + thread;
- thread_mask = BIT_ULL(shift);
-
- return AMD64_L3_SLICE_MASK | thread_mask;
- }
-
- core = (core << AMD64_L3_COREID_SHIFT) & AMD64_L3_COREID_MASK;
- shift = AMD64_L3_THREAD_SHIFT + thread;
- thread_mask = BIT_ULL(shift);
+ /*
+ * If the user doesn't specify a threadmask, they're not trying to
+ * count core 0, so we enable all cores & threads.
+ * We'll also assume that they want to count slice 0 if they specify
+ * a threadmask and leave sliceid and enallslices unpopulated.
+ */
+ if (!(config & AMD64_L3_F19H_THREAD_MASK))
+ return AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES |
+ AMD64_L3_EN_ALL_CORES;
- return AMD64_L3_EN_ALL_SLICES | core | thread_mask;
+ return config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK |
+ AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES |
+ AMD64_L3_COREID_MASK);
}
static int amd_uncore_event_init(struct perf_event *event)
* For other events, the two fields do not affect the count.
*/
if (l3_mask && is_llc_event(event))
- hwc->config |= l3_thread_slice_mask(event->cpu);
+ hwc->config |= l3_thread_slice_mask(event->attr.config);
uncore = event_to_amd_uncore(event);
if (!uncore)
.attrs = amd_uncore_attrs,
};
-/*
- * Similar to PMU_FORMAT_ATTR but allowing for format_attr to be assigned based
- * on family
- */
-#define AMD_FORMAT_ATTR(_dev, _name, _format) \
-static ssize_t \
-_dev##_show##_name(struct device *dev, \
- struct device_attribute *attr, \
- char *page) \
-{ \
- BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
- return sprintf(page, _format "\n"); \
-} \
-static struct device_attribute format_attr_##_dev##_name = __ATTR_RO(_dev);
-
-/* Used for each uncore counter type */
-#define AMD_ATTRIBUTE(_name) \
-static struct attribute *amd_uncore_format_attr_##_name[] = { \
- &format_attr_event_##_name.attr, \
- &format_attr_umask.attr, \
- NULL, \
-}; \
-static struct attribute_group amd_uncore_format_group_##_name = { \
- .name = "format", \
- .attrs = amd_uncore_format_attr_##_name, \
-}; \
-static const struct attribute_group *amd_uncore_attr_groups_##_name[] = { \
- &amd_uncore_attr_group, \
- &amd_uncore_format_group_##_name, \
- NULL, \
+#define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \
+static ssize_t __uncore_##_var##_show(struct kobject *kobj, \
+ struct kobj_attribute *attr, \
+ char *page) \
+{ \
+ BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
+ return sprintf(page, _format "\n"); \
+} \
+static struct kobj_attribute format_attr_##_var = \
+ __ATTR(_name, 0444, __uncore_##_var##_show, NULL)
+
+DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35");
+DEFINE_UNCORE_FORMAT_ATTR(event14, event, "config:0-7,32-35,59-60"); /* F17h+ DF */
+DEFINE_UNCORE_FORMAT_ATTR(event8, event, "config:0-7"); /* F17h+ L3 */
+DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
+DEFINE_UNCORE_FORMAT_ATTR(coreid, coreid, "config:42-44"); /* F19h L3 */
+DEFINE_UNCORE_FORMAT_ATTR(slicemask, slicemask, "config:48-51"); /* F17h L3 */
+DEFINE_UNCORE_FORMAT_ATTR(threadmask8, threadmask, "config:56-63"); /* F17h L3 */
+DEFINE_UNCORE_FORMAT_ATTR(threadmask2, threadmask, "config:56-57"); /* F19h L3 */
+DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 */
+DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */
+DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */
+
+static struct attribute *amd_uncore_df_format_attr[] = {
+ &format_attr_event12.attr, /* event14 if F17h+ */
+ &format_attr_umask.attr,
+ NULL,
+};
+
+static struct attribute *amd_uncore_l3_format_attr[] = {
+ &format_attr_event12.attr, /* event8 if F17h+ */
+ &format_attr_umask.attr,
+ NULL, /* slicemask if F17h, coreid if F19h */
+ NULL, /* threadmask8 if F17h, enallslices if F19h */
+ NULL, /* enallcores if F19h */
+ NULL, /* sliceid if F19h */
+ NULL, /* threadmask2 if F19h */
+ NULL,
+};
+
+static struct attribute_group amd_uncore_df_format_group = {
+ .name = "format",
+ .attrs = amd_uncore_df_format_attr,
};
-AMD_FORMAT_ATTR(event, , "config:0-7,32-35");
-AMD_FORMAT_ATTR(umask, , "config:8-15");
-AMD_FORMAT_ATTR(event, _df, "config:0-7,32-35,59-60");
-AMD_FORMAT_ATTR(event, _l3, "config:0-7");
-AMD_ATTRIBUTE(df);
-AMD_ATTRIBUTE(l3);
+static struct attribute_group amd_uncore_l3_format_group = {
+ .name = "format",
+ .attrs = amd_uncore_l3_format_attr,
+};
+
+static const struct attribute_group *amd_uncore_df_attr_groups[] = {
+ &amd_uncore_attr_group,
+ &amd_uncore_df_format_group,
+ NULL,
+};
+
+static const struct attribute_group *amd_uncore_l3_attr_groups[] = {
+ &amd_uncore_attr_group,
+ &amd_uncore_l3_format_group,
+ NULL,
+};
static struct pmu amd_nb_pmu = {
.task_ctx_nr = perf_invalid_context,
+ .attr_groups = amd_uncore_df_attr_groups,
+ .name = "amd_nb",
.event_init = amd_uncore_event_init,
.add = amd_uncore_add,
.del = amd_uncore_del,
static struct pmu amd_llc_pmu = {
.task_ctx_nr = perf_invalid_context,
+ .attr_groups = amd_uncore_l3_attr_groups,
+ .name = "amd_l2",
.event_init = amd_uncore_event_init,
.add = amd_uncore_add,
.del = amd_uncore_del,
static int __init amd_uncore_init(void)
{
+ struct attribute **df_attr = amd_uncore_df_format_attr;
+ struct attribute **l3_attr = amd_uncore_l3_format_attr;
int ret = -ENODEV;
if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
return -ENODEV;
+ num_counters_nb = NUM_COUNTERS_NB;
+ num_counters_llc = NUM_COUNTERS_L2;
if (boot_cpu_data.x86 >= 0x17) {
/*
* For F17h and above, the Northbridge counters are
* counters are supported too. The PMUs are exported
* based on family as either L2 or L3 and NB or DF.
*/
- num_counters_nb = NUM_COUNTERS_NB;
num_counters_llc = NUM_COUNTERS_L3;
amd_nb_pmu.name = "amd_df";
amd_llc_pmu.name = "amd_l3";
- format_attr_event_df.show = &event_show_df;
- format_attr_event_l3.show = &event_show_l3;
l3_mask = true;
- } else {
- num_counters_nb = NUM_COUNTERS_NB;
- num_counters_llc = NUM_COUNTERS_L2;
- amd_nb_pmu.name = "amd_nb";
- amd_llc_pmu.name = "amd_l2";
- format_attr_event_df = format_attr_event;
- format_attr_event_l3 = format_attr_event;
- l3_mask = false;
}
- amd_nb_pmu.attr_groups = amd_uncore_attr_groups_df;
- amd_llc_pmu.attr_groups = amd_uncore_attr_groups_l3;
-
if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) {
+ if (boot_cpu_data.x86 >= 0x17)
+ *df_attr = &format_attr_event14.attr;
+
amd_uncore_nb = alloc_percpu(struct amd_uncore *);
if (!amd_uncore_nb) {
ret = -ENOMEM;
if (ret)
goto fail_nb;
- pr_info("%s NB counters detected\n",
- boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ?
- "HYGON" : "AMD");
+ pr_info("%d %s %s counters detected\n", num_counters_nb,
+ boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON" : "",
+ amd_nb_pmu.name);
+
ret = 0;
}
if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) {
+ if (boot_cpu_data.x86 >= 0x19) {
+ *l3_attr++ = &format_attr_event8.attr;
+ *l3_attr++ = &format_attr_umask.attr;
+ *l3_attr++ = &format_attr_coreid.attr;
+ *l3_attr++ = &format_attr_enallslices.attr;
+ *l3_attr++ = &format_attr_enallcores.attr;
+ *l3_attr++ = &format_attr_sliceid.attr;
+ *l3_attr++ = &format_attr_threadmask2.attr;
+ } else if (boot_cpu_data.x86 >= 0x17) {
+ *l3_attr++ = &format_attr_event8.attr;
+ *l3_attr++ = &format_attr_umask.attr;
+ *l3_attr++ = &format_attr_slicemask.attr;
+ *l3_attr++ = &format_attr_threadmask8.attr;
+ }
+
amd_uncore_llc = alloc_percpu(struct amd_uncore *);
if (!amd_uncore_llc) {
ret = -ENOMEM;
if (ret)
goto fail_llc;
- pr_info("%s LLC counters detected\n",
- boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ?
- "HYGON" : "AMD");
+ pr_info("%d %s %s counters detected\n", num_counters_llc,
+ boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON" : "",
+ amd_llc_pmu.name);
ret = 0;
}
#include <linux/bitops.h>
#include <linux/device.h>
#include <linux/nospec.h>
+#include <linux/static_call.h>
#include <asm/apic.h>
#include <asm/stacktrace.h>
DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key);
DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key);
+/*
+ * This here uses DEFINE_STATIC_CALL_NULL() to get a static_call defined
+ * from just a typename, as opposed to an actual function.
+ */
+DEFINE_STATIC_CALL_NULL(x86_pmu_handle_irq, *x86_pmu.handle_irq);
+DEFINE_STATIC_CALL_NULL(x86_pmu_disable_all, *x86_pmu.disable_all);
+DEFINE_STATIC_CALL_NULL(x86_pmu_enable_all, *x86_pmu.enable_all);
+DEFINE_STATIC_CALL_NULL(x86_pmu_enable, *x86_pmu.enable);
+DEFINE_STATIC_CALL_NULL(x86_pmu_disable, *x86_pmu.disable);
+
+DEFINE_STATIC_CALL_NULL(x86_pmu_add, *x86_pmu.add);
+DEFINE_STATIC_CALL_NULL(x86_pmu_del, *x86_pmu.del);
+DEFINE_STATIC_CALL_NULL(x86_pmu_read, *x86_pmu.read);
+
+DEFINE_STATIC_CALL_NULL(x86_pmu_schedule_events, *x86_pmu.schedule_events);
+DEFINE_STATIC_CALL_NULL(x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints);
+DEFINE_STATIC_CALL_NULL(x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints);
+
+DEFINE_STATIC_CALL_NULL(x86_pmu_start_scheduling, *x86_pmu.start_scheduling);
+DEFINE_STATIC_CALL_NULL(x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling);
+DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling, *x86_pmu.stop_scheduling);
+
+DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task, *x86_pmu.sched_task);
+DEFINE_STATIC_CALL_NULL(x86_pmu_swap_task_ctx, *x86_pmu.swap_task_ctx);
+
+DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs, *x86_pmu.drain_pebs);
+DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases);
+
u64 __read_mostly hw_cache_event_ids
[PERF_COUNT_HW_CACHE_MAX]
[PERF_COUNT_HW_CACHE_OP_MAX]
if (unlikely(!hwc->event_base))
return 0;
+ if (unlikely(is_topdown_count(event)) && x86_pmu.update_topdown_event)
+ return x86_pmu.update_topdown_event(event);
+
/*
* Careful: an NMI might modify the previous event value.
*
cpuc->enabled = 0;
barrier();
- x86_pmu.disable_all();
+ static_call(x86_pmu_disable_all)();
}
void x86_pmu_enable_all(int added)
if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
n0 -= cpuc->n_txn;
- if (x86_pmu.start_scheduling)
- x86_pmu.start_scheduling(cpuc);
+ static_call_cond(x86_pmu_start_scheduling)(cpuc);
for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
c = cpuc->event_constraint[i];
* change due to external factors (sibling state, allow_tfa).
*/
if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) {
- c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]);
+ c = static_call(x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]);
cpuc->event_constraint[i] = c;
}
if (!unsched && assign) {
for (i = 0; i < n; i++) {
e = cpuc->event_list[i];
- if (x86_pmu.commit_scheduling)
- x86_pmu.commit_scheduling(cpuc, i, assign[i]);
+ static_call_cond(x86_pmu_commit_scheduling)(cpuc, i, assign[i]);
}
} else {
for (i = n0; i < n; i++) {
/*
* release events that failed scheduling
*/
- if (x86_pmu.put_event_constraints)
- x86_pmu.put_event_constraints(cpuc, e);
+ static_call_cond(x86_pmu_put_event_constraints)(cpuc, e);
cpuc->event_constraint[i] = NULL;
}
}
- if (x86_pmu.stop_scheduling)
- x86_pmu.stop_scheduling(cpuc);
+ static_call_cond(x86_pmu_stop_scheduling)(cpuc);
return unsched ? -EINVAL : 0;
}
+static int add_nr_metric_event(struct cpu_hw_events *cpuc,
+ struct perf_event *event)
+{
+ if (is_metric_event(event)) {
+ if (cpuc->n_metric == INTEL_TD_METRIC_NUM)
+ return -EINVAL;
+ cpuc->n_metric++;
+ cpuc->n_txn_metric++;
+ }
+
+ return 0;
+}
+
+static void del_nr_metric_event(struct cpu_hw_events *cpuc,
+ struct perf_event *event)
+{
+ if (is_metric_event(event))
+ cpuc->n_metric--;
+}
+
+static int collect_event(struct cpu_hw_events *cpuc, struct perf_event *event,
+ int max_count, int n)
+{
+
+ if (x86_pmu.intel_cap.perf_metrics && add_nr_metric_event(cpuc, event))
+ return -EINVAL;
+
+ if (n >= max_count + cpuc->n_metric)
+ return -EINVAL;
+
+ cpuc->event_list[n] = event;
+ if (is_counter_pair(&event->hw)) {
+ cpuc->n_pair++;
+ cpuc->n_txn_pair++;
+ }
+
+ return 0;
+}
+
/*
* dogrp: true if must collect siblings events (group)
* returns total number of events and error code
}
if (is_x86_event(leader)) {
- if (n >= max_count)
+ if (collect_event(cpuc, leader, max_count, n))
return -EINVAL;
- cpuc->event_list[n] = leader;
n++;
- if (is_counter_pair(&leader->hw))
- cpuc->n_pair++;
}
+
if (!dogrp)
return n;
for_each_sibling_event(event, leader) {
- if (!is_x86_event(event) ||
- event->state <= PERF_EVENT_STATE_OFF)
+ if (!is_x86_event(event) || event->state <= PERF_EVENT_STATE_OFF)
continue;
- if (n >= max_count)
+ if (collect_event(cpuc, event, max_count, n))
return -EINVAL;
- cpuc->event_list[n] = event;
n++;
- if (is_counter_pair(&event->hw))
- cpuc->n_pair++;
}
return n;
}
hwc->event_base = 0;
break;
+ case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
+ /* All the metric events are mapped onto the fixed counter 3. */
+ idx = INTEL_PMC_IDX_FIXED_SLOTS;
+ /* fall through */
case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS-1:
hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 +
(idx - INTEL_PMC_IDX_FIXED);
- hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) | 1<<30;
+ hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) |
+ INTEL_PMC_FIXED_RDPMC_BASE;
break;
default:
cpuc->enabled = 1;
barrier();
- x86_pmu.enable_all(added);
+ static_call(x86_pmu_enable_all)(added);
}
static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
if (unlikely(!hwc->event_base))
return 0;
+ if (unlikely(is_topdown_count(event)) &&
+ x86_pmu.set_topdown_event_period)
+ return x86_pmu.set_topdown_event_period(event);
+
/*
* If we are way outside a reasonable range then just skip forward:
*/
wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
/*
- * Clear the Merge event counter's upper 16 bits since
+ * Sign extend the Merge event counter's upper 16 bits since
* we currently declare a 48-bit counter width
*/
if (is_counter_pair(hwc))
- wrmsrl(x86_pmu_event_addr(idx + 1), 0);
+ wrmsrl(x86_pmu_event_addr(idx + 1), 0xffff);
/*
* Due to erratum on certan cpu we need
if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
goto done_collect;
- ret = x86_pmu.schedule_events(cpuc, n, assign);
+ ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign);
if (ret)
goto out;
/*
cpuc->n_added += n - n0;
cpuc->n_txn += n - n0;
- if (x86_pmu.add) {
- /*
- * This is before x86_pmu_enable() will call x86_pmu_start(),
- * so we enable LBRs before an event needs them etc..
- */
- x86_pmu.add(event);
- }
+ /*
+ * This is before x86_pmu_enable() will call x86_pmu_start(),
+ * so we enable LBRs before an event needs them etc..
+ */
+ static_call_cond(x86_pmu_add)(event);
ret = 0;
out:
cpuc->events[idx] = event;
__set_bit(idx, cpuc->active_mask);
__set_bit(idx, cpuc->running);
- x86_pmu.enable(event);
+ static_call(x86_pmu_enable)(event);
perf_event_update_userpage(event);
}
struct hw_perf_event *hwc = &event->hw;
if (test_bit(hwc->idx, cpuc->active_mask)) {
- x86_pmu.disable(event);
+ static_call(x86_pmu_disable)(event);
__clear_bit(hwc->idx, cpuc->active_mask);
cpuc->events[hwc->idx] = NULL;
WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
if (i >= cpuc->n_events - cpuc->n_added)
--cpuc->n_added;
- if (x86_pmu.put_event_constraints)
- x86_pmu.put_event_constraints(cpuc, event);
+ static_call_cond(x86_pmu_put_event_constraints)(cpuc, event);
/* Delete the array entry. */
while (++i < cpuc->n_events) {
}
cpuc->event_constraint[i-1] = NULL;
--cpuc->n_events;
+ if (x86_pmu.intel_cap.perf_metrics)
+ del_nr_metric_event(cpuc, event);
perf_event_update_userpage(event);
do_del:
- if (x86_pmu.del) {
- /*
- * This is after x86_pmu_stop(); so we disable LBRs after any
- * event can need them etc..
- */
- x86_pmu.del(event);
- }
+
+ /*
+ * This is after x86_pmu_stop(); so we disable LBRs after any
+ * event can need them etc..
+ */
+ static_call_cond(x86_pmu_del)(event);
}
int x86_pmu_handle_irq(struct pt_regs *regs)
return NMI_DONE;
start_clock = sched_clock();
- ret = x86_pmu.handle_irq(regs);
+ ret = static_call(x86_pmu_handle_irq)(regs);
finish_clock = sched_clock();
perf_sample_event_took(finish_clock - start_clock);
static struct attribute_group x86_pmu_attr_group;
static struct attribute_group x86_pmu_caps_group;
+static void x86_pmu_static_call_update(void)
+{
+ static_call_update(x86_pmu_handle_irq, x86_pmu.handle_irq);
+ static_call_update(x86_pmu_disable_all, x86_pmu.disable_all);
+ static_call_update(x86_pmu_enable_all, x86_pmu.enable_all);
+ static_call_update(x86_pmu_enable, x86_pmu.enable);
+ static_call_update(x86_pmu_disable, x86_pmu.disable);
+
+ static_call_update(x86_pmu_add, x86_pmu.add);
+ static_call_update(x86_pmu_del, x86_pmu.del);
+ static_call_update(x86_pmu_read, x86_pmu.read);
+
+ static_call_update(x86_pmu_schedule_events, x86_pmu.schedule_events);
+ static_call_update(x86_pmu_get_event_constraints, x86_pmu.get_event_constraints);
+ static_call_update(x86_pmu_put_event_constraints, x86_pmu.put_event_constraints);
+
+ static_call_update(x86_pmu_start_scheduling, x86_pmu.start_scheduling);
+ static_call_update(x86_pmu_commit_scheduling, x86_pmu.commit_scheduling);
+ static_call_update(x86_pmu_stop_scheduling, x86_pmu.stop_scheduling);
+
+ static_call_update(x86_pmu_sched_task, x86_pmu.sched_task);
+ static_call_update(x86_pmu_swap_task_ctx, x86_pmu.swap_task_ctx);
+
+ static_call_update(x86_pmu_drain_pebs, x86_pmu.drain_pebs);
+ static_call_update(x86_pmu_pebs_aliases, x86_pmu.pebs_aliases);
+}
+
+static void _x86_pmu_read(struct perf_event *event)
+{
+ x86_perf_event_update(event);
+}
+
static int __init init_hw_perf_events(void)
{
struct x86_pmu_quirk *quirk;
pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed);
pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl);
+ if (!x86_pmu.read)
+ x86_pmu.read = _x86_pmu_read;
+
+ x86_pmu_static_call_update();
+
/*
* Install callbacks. Core will call them for each online
* cpu.
}
early_initcall(init_hw_perf_events);
-static inline void x86_pmu_read(struct perf_event *event)
+static void x86_pmu_read(struct perf_event *event)
{
- if (x86_pmu.read)
- return x86_pmu.read(event);
- x86_perf_event_update(event);
+ static_call(x86_pmu_read)(event);
}
/*
perf_pmu_disable(pmu);
__this_cpu_write(cpu_hw_events.n_txn, 0);
+ __this_cpu_write(cpu_hw_events.n_txn_pair, 0);
+ __this_cpu_write(cpu_hw_events.n_txn_metric, 0);
}
/*
*/
__this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
__this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
+ __this_cpu_sub(cpu_hw_events.n_pair, __this_cpu_read(cpu_hw_events.n_txn_pair));
+ __this_cpu_sub(cpu_hw_events.n_metric, __this_cpu_read(cpu_hw_events.n_txn_metric));
perf_pmu_enable(pmu);
}
if (!x86_pmu_initialized())
return -EAGAIN;
- ret = x86_pmu.schedule_events(cpuc, n, assign);
+ ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign);
if (ret)
return ret;
static int x86_pmu_event_idx(struct perf_event *event)
{
- int idx = event->hw.idx;
+ struct hw_perf_event *hwc = &event->hw;
- if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
+ if (!(hwc->flags & PERF_X86_EVENT_RDPMC_ALLOWED))
return 0;
- if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
- idx -= INTEL_PMC_IDX_FIXED;
- idx |= 1 << 30;
- }
-
- return idx + 1;
+ if (is_metric_idx(hwc->idx))
+ return INTEL_PMC_FIXED_RDPMC_METRICS + 1;
+ else
+ return hwc->event_base_rdpmc + 1;
}
static ssize_t get_attr_rdpmc(struct device *cdev,
static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
{
- if (x86_pmu.sched_task)
- x86_pmu.sched_task(ctx, sched_in);
+ static_call_cond(x86_pmu_sched_task)(ctx, sched_in);
}
static void x86_pmu_swap_task_ctx(struct perf_event_context *prev,
struct perf_event_context *next)
{
- if (x86_pmu.swap_task_ctx)
- x86_pmu.swap_task_ctx(prev, next);
+ static_call_cond(x86_pmu_swap_task_ctx)(prev, next);
}
void perf_check_microcode(void)
static struct event_constraint intel_icl_event_constraints[] = {
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
- INTEL_UEVENT_CONSTRAINT(0x1c0, 0), /* INST_RETIRED.PREC_DIST */
+ FIXED_EVENT_CONSTRAINT(0x01c0, 0), /* INST_RETIRED.PREC_DIST */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */
+ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0),
+ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1),
+ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2),
+ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BE_BOUND, 3),
INTEL_EVENT_CONSTRAINT_RANGE(0x03, 0x0a, 0xf),
INTEL_EVENT_CONSTRAINT_RANGE(0x1f, 0x28, 0xf),
INTEL_EVENT_CONSTRAINT(0x32, 0xf), /* SW_PREFETCH_ACCESS.* */
EVENT_ATTR_STR_HT(topdown-recovery-bubbles.scale, td_recovery_bubbles_scale,
"4", "2");
+EVENT_ATTR_STR(slots, slots, "event=0x00,umask=0x4");
+EVENT_ATTR_STR(topdown-retiring, td_retiring, "event=0x00,umask=0x80");
+EVENT_ATTR_STR(topdown-bad-spec, td_bad_spec, "event=0x00,umask=0x81");
+EVENT_ATTR_STR(topdown-fe-bound, td_fe_bound, "event=0x00,umask=0x82");
+EVENT_ATTR_STR(topdown-be-bound, td_be_bound, "event=0x00,umask=0x83");
+
static struct attribute *snb_events_attrs[] = {
EVENT_PTR(td_slots_issued),
EVENT_PTR(td_slots_retired),
static void intel_pmu_disable_fixed(struct perf_event *event)
{
struct hw_perf_event *hwc = &event->hw;
- int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
u64 ctrl_val, mask;
+ int idx = hwc->idx;
- mask = 0xfULL << (idx * 4);
+ if (is_topdown_idx(idx)) {
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ /*
+ * When there are other active TopDown events,
+ * don't disable the fixed counter 3.
+ */
+ if (*(u64 *)cpuc->active_mask & INTEL_PMC_OTHER_TOPDOWN_BITS(idx))
+ return;
+ idx = INTEL_PMC_IDX_FIXED_SLOTS;
+ }
+ intel_clear_masks(event, idx);
+
+ mask = 0xfULL << ((idx - INTEL_PMC_IDX_FIXED) * 4);
rdmsrl(hwc->config_base, ctrl_val);
ctrl_val &= ~mask;
wrmsrl(hwc->config_base, ctrl_val);
struct hw_perf_event *hwc = &event->hw;
int idx = hwc->idx;
- if (idx < INTEL_PMC_IDX_FIXED) {
+ switch (idx) {
+ case 0 ... INTEL_PMC_IDX_FIXED - 1:
intel_clear_masks(event, idx);
x86_pmu_disable_event(event);
- } else if (idx < INTEL_PMC_IDX_FIXED_BTS) {
- intel_clear_masks(event, idx);
+ break;
+ case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
+ case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
intel_pmu_disable_fixed(event);
- } else if (idx == INTEL_PMC_IDX_FIXED_BTS) {
+ break;
+ case INTEL_PMC_IDX_FIXED_BTS:
intel_pmu_disable_bts();
intel_pmu_drain_bts_buffer();
- } else if (idx == INTEL_PMC_IDX_FIXED_VLBR)
+ return;
+ case INTEL_PMC_IDX_FIXED_VLBR:
intel_clear_masks(event, idx);
+ break;
+ default:
+ intel_clear_masks(event, idx);
+ pr_warn("Failed to disable the event with invalid index %d\n",
+ idx);
+ return;
+ }
/*
* Needs to be called after x86_pmu_disable_event,
intel_pmu_pebs_del(event);
}
+static int icl_set_topdown_event_period(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ s64 left = local64_read(&hwc->period_left);
+
+ /*
+ * The values in PERF_METRICS MSR are derived from fixed counter 3.
+ * Software should start both registers, PERF_METRICS and fixed
+ * counter 3, from zero.
+ * Clear PERF_METRICS and Fixed counter 3 in initialization.
+ * After that, both MSRs will be cleared for each read.
+ * Don't need to clear them again.
+ */
+ if (left == x86_pmu.max_period) {
+ wrmsrl(MSR_CORE_PERF_FIXED_CTR3, 0);
+ wrmsrl(MSR_PERF_METRICS, 0);
+ hwc->saved_slots = 0;
+ hwc->saved_metric = 0;
+ }
+
+ if ((hwc->saved_slots) && is_slots_event(event)) {
+ wrmsrl(MSR_CORE_PERF_FIXED_CTR3, hwc->saved_slots);
+ wrmsrl(MSR_PERF_METRICS, hwc->saved_metric);
+ }
+
+ perf_event_update_userpage(event);
+
+ return 0;
+}
+
+static inline u64 icl_get_metrics_event_value(u64 metric, u64 slots, int idx)
+{
+ u32 val;
+
+ /*
+ * The metric is reported as an 8bit integer fraction
+ * suming up to 0xff.
+ * slots-in-metric = (Metric / 0xff) * slots
+ */
+ val = (metric >> ((idx - INTEL_PMC_IDX_METRIC_BASE) * 8)) & 0xff;
+ return mul_u64_u32_div(slots, val, 0xff);
+}
+
+static u64 icl_get_topdown_value(struct perf_event *event,
+ u64 slots, u64 metrics)
+{
+ int idx = event->hw.idx;
+ u64 delta;
+
+ if (is_metric_idx(idx))
+ delta = icl_get_metrics_event_value(metrics, slots, idx);
+ else
+ delta = slots;
+
+ return delta;
+}
+
+static void __icl_update_topdown_event(struct perf_event *event,
+ u64 slots, u64 metrics,
+ u64 last_slots, u64 last_metrics)
+{
+ u64 delta, last = 0;
+
+ delta = icl_get_topdown_value(event, slots, metrics);
+ if (last_slots)
+ last = icl_get_topdown_value(event, last_slots, last_metrics);
+
+ /*
+ * The 8bit integer fraction of metric may be not accurate,
+ * especially when the changes is very small.
+ * For example, if only a few bad_spec happens, the fraction
+ * may be reduced from 1 to 0. If so, the bad_spec event value
+ * will be 0 which is definitely less than the last value.
+ * Avoid update event->count for this case.
+ */
+ if (delta > last) {
+ delta -= last;
+ local64_add(delta, &event->count);
+ }
+}
+
+static void update_saved_topdown_regs(struct perf_event *event,
+ u64 slots, u64 metrics)
+{
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+ struct perf_event *other;
+ int idx;
+
+ event->hw.saved_slots = slots;
+ event->hw.saved_metric = metrics;
+
+ for_each_set_bit(idx, cpuc->active_mask, INTEL_PMC_IDX_TD_BE_BOUND + 1) {
+ if (!is_topdown_idx(idx))
+ continue;
+ other = cpuc->events[idx];
+ other->hw.saved_slots = slots;
+ other->hw.saved_metric = metrics;
+ }
+}
+
+/*
+ * Update all active Topdown events.
+ *
+ * The PERF_METRICS and Fixed counter 3 are read separately. The values may be
+ * modify by a NMI. PMU has to be disabled before calling this function.
+ */
+static u64 icl_update_topdown_event(struct perf_event *event)
+{
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+ struct perf_event *other;
+ u64 slots, metrics;
+ bool reset = true;
+ int idx;
+
+ /* read Fixed counter 3 */
+ rdpmcl((3 | INTEL_PMC_FIXED_RDPMC_BASE), slots);
+ if (!slots)
+ return 0;
+
+ /* read PERF_METRICS */
+ rdpmcl(INTEL_PMC_FIXED_RDPMC_METRICS, metrics);
+
+ for_each_set_bit(idx, cpuc->active_mask, INTEL_PMC_IDX_TD_BE_BOUND + 1) {
+ if (!is_topdown_idx(idx))
+ continue;
+ other = cpuc->events[idx];
+ __icl_update_topdown_event(other, slots, metrics,
+ event ? event->hw.saved_slots : 0,
+ event ? event->hw.saved_metric : 0);
+ }
+
+ /*
+ * Check and update this event, which may have been cleared
+ * in active_mask e.g. x86_pmu_stop()
+ */
+ if (event && !test_bit(event->hw.idx, cpuc->active_mask)) {
+ __icl_update_topdown_event(event, slots, metrics,
+ event->hw.saved_slots,
+ event->hw.saved_metric);
+
+ /*
+ * In x86_pmu_stop(), the event is cleared in active_mask first,
+ * then drain the delta, which indicates context switch for
+ * counting.
+ * Save metric and slots for context switch.
+ * Don't need to reset the PERF_METRICS and Fixed counter 3.
+ * Because the values will be restored in next schedule in.
+ */
+ update_saved_topdown_regs(event, slots, metrics);
+ reset = false;
+ }
+
+ if (reset) {
+ /* The fixed counter 3 has to be written before the PERF_METRICS. */
+ wrmsrl(MSR_CORE_PERF_FIXED_CTR3, 0);
+ wrmsrl(MSR_PERF_METRICS, 0);
+ if (event)
+ update_saved_topdown_regs(event, 0, 0);
+ }
+
+ return slots;
+}
+
+static void intel_pmu_read_topdown_event(struct perf_event *event)
+{
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ /* Only need to call update_topdown_event() once for group read. */
+ if ((cpuc->txn_flags & PERF_PMU_TXN_READ) &&
+ !is_slots_event(event))
+ return;
+
+ perf_pmu_disable(event->pmu);
+ x86_pmu.update_topdown_event(event);
+ perf_pmu_enable(event->pmu);
+}
+
static void intel_pmu_read_event(struct perf_event *event)
{
if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
intel_pmu_auto_reload_read(event);
+ else if (is_topdown_count(event) && x86_pmu.update_topdown_event)
+ intel_pmu_read_topdown_event(event);
else
x86_perf_event_update(event);
}
static void intel_pmu_enable_fixed(struct perf_event *event)
{
struct hw_perf_event *hwc = &event->hw;
- int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
u64 ctrl_val, mask, bits = 0;
+ int idx = hwc->idx;
+
+ if (is_topdown_idx(idx)) {
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+ /*
+ * When there are other active TopDown events,
+ * don't enable the fixed counter 3 again.
+ */
+ if (*(u64 *)cpuc->active_mask & INTEL_PMC_OTHER_TOPDOWN_BITS(idx))
+ return;
+
+ idx = INTEL_PMC_IDX_FIXED_SLOTS;
+ }
+
+ intel_set_masks(event, idx);
/*
* Enable IRQ generation (0x8), if not PEBS,
if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)
bits |= 0x4;
+ idx -= INTEL_PMC_IDX_FIXED;
bits <<= (idx * 4);
mask = 0xfULL << (idx * 4);
if (unlikely(event->attr.precise_ip))
intel_pmu_pebs_enable(event);
- if (idx < INTEL_PMC_IDX_FIXED) {
+ switch (idx) {
+ case 0 ... INTEL_PMC_IDX_FIXED - 1:
intel_set_masks(event, idx);
__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
- } else if (idx < INTEL_PMC_IDX_FIXED_BTS) {
- intel_set_masks(event, idx);
+ break;
+ case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
+ case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
intel_pmu_enable_fixed(event);
- } else if (idx == INTEL_PMC_IDX_FIXED_BTS) {
+ break;
+ case INTEL_PMC_IDX_FIXED_BTS:
if (!__this_cpu_read(cpu_hw_events.enabled))
return;
intel_pmu_enable_bts(hwc->config);
- } else if (idx == INTEL_PMC_IDX_FIXED_VLBR)
+ break;
+ case INTEL_PMC_IDX_FIXED_VLBR:
intel_set_masks(event, idx);
+ break;
+ default:
+ pr_warn("Failed to enable the event with invalid index %d\n",
+ idx);
+ }
}
static void intel_pmu_add_event(struct perf_event *event)
/*
* PEBS overflow sets bit 62 in the global status register
*/
- if (__test_and_clear_bit(62, (unsigned long *)&status)) {
+ if (__test_and_clear_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, (unsigned long *)&status)) {
u64 pebs_enabled = cpuc->pebs_enabled;
handled++;
/*
* Intel PT
*/
- if (__test_and_clear_bit(55, (unsigned long *)&status)) {
+ if (__test_and_clear_bit(GLOBAL_STATUS_TRACE_TOPAPMI_BIT, (unsigned long *)&status)) {
handled++;
if (unlikely(perf_guest_cbs && perf_guest_cbs->is_in_guest() &&
perf_guest_cbs->handle_intel_pt_intr))
intel_pt_interrupt();
}
+ /*
+ * Intel Perf mertrics
+ */
+ if (__test_and_clear_bit(GLOBAL_STATUS_PERF_METRICS_OVF_BIT, (unsigned long *)&status)) {
+ handled++;
+ if (x86_pmu.update_topdown_event)
+ x86_pmu.update_topdown_event(NULL);
+ }
+
/*
* Checkpointed counters can lead to 'spurious' PMIs because the
* rollback caused by the PMI will have cleared the overflow status
if (event->attr.type != PERF_TYPE_RAW)
return 0;
+ /*
+ * Config Topdown slots and metric events
+ *
+ * The slots event on Fixed Counter 3 can support sampling,
+ * which will be handled normally in x86_perf_event_update().
+ *
+ * Metric events don't support sampling and require being paired
+ * with a slots event as group leader. When the slots event
+ * is used in a metrics group, it too cannot support sampling.
+ */
+ if (x86_pmu.intel_cap.perf_metrics && is_topdown_event(event)) {
+ if (event->attr.config1 || event->attr.config2)
+ return -EINVAL;
+
+ /*
+ * The TopDown metrics events and slots event don't
+ * support any filters.
+ */
+ if (event->attr.config & X86_ALL_EVENT_FLAGS)
+ return -EINVAL;
+
+ if (is_metric_event(event)) {
+ struct perf_event *leader = event->group_leader;
+
+ /* The metric events don't support sampling. */
+ if (is_sampling_event(event))
+ return -EINVAL;
+
+ /* The metric events require a slots group leader. */
+ if (!is_slots_event(leader))
+ return -EINVAL;
+
+ /*
+ * The leader/SLOTS must not be a sampling event for
+ * metric use; hardware requires it starts at 0 when used
+ * in conjunction with MSR_PERF_METRICS.
+ */
+ if (is_sampling_event(leader))
+ return -EINVAL;
+
+ event->event_caps |= PERF_EV_CAP_SIBLING;
+ /*
+ * Only once we have a METRICs sibling do we
+ * need TopDown magic.
+ */
+ leader->hw.flags |= PERF_X86_EVENT_TOPDOWN;
+ event->hw.flags |= PERF_X86_EVENT_TOPDOWN;
+ }
+ }
+
if (!(event->attr.config & ARCH_PERFMON_EVENTSEL_ANY))
return 0;
if (x86_pmu.counter_freezing)
enable_counter_freeze();
+ /* Disable perf metrics if any added CPU doesn't support it. */
+ if (x86_pmu.intel_cap.perf_metrics) {
+ union perf_capabilities perf_cap;
+
+ rdmsrl(MSR_IA32_PERF_CAPABILITIES, perf_cap.capabilities);
+ if (!perf_cap.perf_metrics) {
+ x86_pmu.intel_cap.perf_metrics = 0;
+ x86_pmu.intel_ctrl &= ~(1ULL << GLOBAL_CTRL_EN_PERF_METRICS);
+ }
+ }
+
if (!cpuc->shared_regs)
return;
NULL,
};
+static struct attribute *icl_td_events_attrs[] = {
+ EVENT_PTR(slots),
+ EVENT_PTR(td_retiring),
+ EVENT_PTR(td_bad_spec),
+ EVENT_PTR(td_fe_bound),
+ EVENT_PTR(td_be_bound),
+ NULL,
+};
+
static struct attribute *icl_tsx_events_attrs[] = {
EVENT_PTR(tx_start),
EVENT_PTR(tx_abort),
case INTEL_FAM6_ATOM_TREMONT_D:
case INTEL_FAM6_ATOM_TREMONT:
+ case INTEL_FAM6_ATOM_TREMONT_L:
x86_pmu.late_ack = true;
memcpy(hw_cache_event_ids, glp_hw_cache_event_ids,
sizeof(hw_cache_event_ids));
hsw_format_attr : nhm_format_attr;
extra_skl_attr = skl_format_attr;
mem_attr = icl_events_attrs;
+ td_attr = icl_td_events_attrs;
tsx_attr = icl_tsx_events_attrs;
x86_pmu.rtm_abort_event = X86_CONFIG(.event=0xca, .umask=0x02);
x86_pmu.lbr_pt_coexist = true;
intel_pmu_pebs_data_source_skl(pmem);
+ x86_pmu.update_topdown_event = icl_update_topdown_event;
+ x86_pmu.set_topdown_event_period = icl_set_topdown_event_period;
pr_cont("Icelake events, ");
name = "icelake";
break;
* counter, so do not extend mask to generic counters
*/
for_each_event_constraint(c, x86_pmu.event_constraints) {
+ /*
+ * Don't extend the topdown slots and metrics
+ * events to the generic counters.
+ */
+ if (c->idxmsk64 & INTEL_PMC_MSK_TOPDOWN) {
+ c->weight = hweight64(c->idxmsk64);
+ continue;
+ }
+
if (c->cmask == FIXED_EVENT_FLAGS
&& c->idxmsk64 != INTEL_PMC_MSK_FIXED_REF_CYCLES) {
c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1;
if (x86_pmu.counter_freezing)
x86_pmu.handle_irq = intel_pmu_handle_irq_v4;
+ if (x86_pmu.intel_cap.perf_metrics)
+ x86_pmu.intel_ctrl |= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS;
+
return 0;
}
static inline void intel_pmu_drain_pebs_buffer(void)
{
- struct pt_regs regs;
-
- x86_pmu.drain_pebs(®s);
+ x86_pmu.drain_pebs(NULL);
}
/*
struct x86_perf_regs perf_regs;
struct pt_regs *regs = &perf_regs.regs;
void *at = get_next_pebs_record_by_bit(base, top, bit);
+ struct pt_regs dummy_iregs;
if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
/*
} else if (!intel_pmu_save_and_restart(event))
return;
+ if (!iregs)
+ iregs = &dummy_iregs;
+
while (count > 1) {
setup_sample(event, iregs, at, &data, regs);
perf_event_output(event, &data, regs);
}
setup_sample(event, iregs, at, &data, regs);
-
- /*
- * All but the last records are processed.
- * The last one is left to be able to call the overflow handler.
- */
- if (perf_event_overflow(event, &data, regs)) {
- x86_pmu_stop(event, 0);
- return;
+ if (iregs == &dummy_iregs) {
+ /*
+ * The PEBS records may be drained in the non-overflow context,
+ * e.g., large PEBS + context switch. Perf should treat the
+ * last record the same as other PEBS records, and doesn't
+ * invoke the generic overflow handler.
+ */
+ perf_event_output(event, &data, regs);
+ } else {
+ /*
+ * All but the last records are processed.
+ * The last one is left to be able to call the overflow handler.
+ */
+ if (perf_event_overflow(event, &data, regs))
+ x86_pmu_stop(event, 0);
}
-
}
static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
static bool pcidrv_registered;
struct pci_driver *uncore_pci_driver;
+/* The PCI driver for the device which the uncore doesn't own. */
+struct pci_driver *uncore_pci_sub_driver;
/* pci bus to socket mapping */
DEFINE_RAW_SPINLOCK(pci2phy_map_lock);
struct list_head pci2phy_map_head = LIST_HEAD_INIT(pci2phy_map_head);
}
/*
- * add a pci uncore device
+ * Get the die information of a PCI device.
+ * @pdev: The PCI device.
+ * @phys_id: The physical socket id which the device maps to.
+ * @die: The die id which the device maps to.
*/
-static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+static int uncore_pci_get_dev_die_info(struct pci_dev *pdev,
+ int *phys_id, int *die)
{
- struct intel_uncore_type *type;
- struct intel_uncore_pmu *pmu = NULL;
- struct intel_uncore_box *box;
- int phys_id, die, ret;
-
- phys_id = uncore_pcibus_to_physid(pdev->bus);
- if (phys_id < 0)
+ *phys_id = uncore_pcibus_to_physid(pdev->bus);
+ if (*phys_id < 0)
return -ENODEV;
- die = (topology_max_die_per_package() > 1) ? phys_id :
- topology_phys_to_logical_pkg(phys_id);
- if (die < 0)
+ *die = (topology_max_die_per_package() > 1) ? *phys_id :
+ topology_phys_to_logical_pkg(*phys_id);
+ if (*die < 0)
return -EINVAL;
- if (UNCORE_PCI_DEV_TYPE(id->driver_data) == UNCORE_EXTRA_PCI_DEV) {
- int idx = UNCORE_PCI_DEV_IDX(id->driver_data);
-
- uncore_extra_pci_dev[die].dev[idx] = pdev;
- pci_set_drvdata(pdev, NULL);
- return 0;
- }
-
- type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)];
+ return 0;
+}
- /*
- * Some platforms, e.g. Knights Landing, use a common PCI device ID
- * for multiple instances of an uncore PMU device type. We should check
- * PCI slot and func to indicate the uncore box.
- */
- if (id->driver_data & ~0xffff) {
- struct pci_driver *pci_drv = pdev->driver;
- const struct pci_device_id *ids = pci_drv->id_table;
- unsigned int devfn;
-
- while (ids && ids->vendor) {
- if ((ids->vendor == pdev->vendor) &&
- (ids->device == pdev->device)) {
- devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(ids->driver_data),
- UNCORE_PCI_DEV_FUNC(ids->driver_data));
- if (devfn == pdev->devfn) {
- pmu = &type->pmus[UNCORE_PCI_DEV_IDX(ids->driver_data)];
- break;
- }
+/*
+ * Find the PMU of a PCI device.
+ * @pdev: The PCI device.
+ * @ids: The ID table of the available PCI devices with a PMU.
+ */
+static struct intel_uncore_pmu *
+uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids)
+{
+ struct intel_uncore_pmu *pmu = NULL;
+ struct intel_uncore_type *type;
+ kernel_ulong_t data;
+ unsigned int devfn;
+
+ while (ids && ids->vendor) {
+ if ((ids->vendor == pdev->vendor) &&
+ (ids->device == pdev->device)) {
+ data = ids->driver_data;
+ devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(data),
+ UNCORE_PCI_DEV_FUNC(data));
+ if (devfn == pdev->devfn) {
+ type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(data)];
+ pmu = &type->pmus[UNCORE_PCI_DEV_IDX(data)];
+ break;
}
- ids++;
}
- if (pmu == NULL)
- return -ENODEV;
- } else {
- /*
- * for performance monitoring unit with multiple boxes,
- * each box has a different function id.
- */
- pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)];
+ ids++;
}
+ return pmu;
+}
+
+/*
+ * Register the PMU for a PCI device
+ * @pdev: The PCI device.
+ * @type: The corresponding PMU type of the device.
+ * @pmu: The corresponding PMU of the device.
+ * @phys_id: The physical socket id which the device maps to.
+ * @die: The die id which the device maps to.
+ */
+static int uncore_pci_pmu_register(struct pci_dev *pdev,
+ struct intel_uncore_type *type,
+ struct intel_uncore_pmu *pmu,
+ int phys_id, int die)
+{
+ struct intel_uncore_box *box;
+ int ret;
if (WARN_ON_ONCE(pmu->boxes[die] != NULL))
return -EINVAL;
box->pci_dev = pdev;
box->pmu = pmu;
uncore_box_init(box);
- pci_set_drvdata(pdev, box);
pmu->boxes[die] = box;
if (atomic_inc_return(&pmu->activeboxes) > 1)
/* First active box registers the pmu */
ret = uncore_pmu_register(pmu);
if (ret) {
- pci_set_drvdata(pdev, NULL);
pmu->boxes[die] = NULL;
uncore_box_exit(box);
kfree(box);
return ret;
}
+/*
+ * add a pci uncore device
+ */
+static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+ struct intel_uncore_type *type;
+ struct intel_uncore_pmu *pmu = NULL;
+ int phys_id, die, ret;
+
+ ret = uncore_pci_get_dev_die_info(pdev, &phys_id, &die);
+ if (ret)
+ return ret;
+
+ if (UNCORE_PCI_DEV_TYPE(id->driver_data) == UNCORE_EXTRA_PCI_DEV) {
+ int idx = UNCORE_PCI_DEV_IDX(id->driver_data);
+
+ uncore_extra_pci_dev[die].dev[idx] = pdev;
+ pci_set_drvdata(pdev, NULL);
+ return 0;
+ }
+
+ type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)];
+
+ /*
+ * Some platforms, e.g. Knights Landing, use a common PCI device ID
+ * for multiple instances of an uncore PMU device type. We should check
+ * PCI slot and func to indicate the uncore box.
+ */
+ if (id->driver_data & ~0xffff) {
+ struct pci_driver *pci_drv = pdev->driver;
+
+ pmu = uncore_pci_find_dev_pmu(pdev, pci_drv->id_table);
+ if (pmu == NULL)
+ return -ENODEV;
+ } else {
+ /*
+ * for performance monitoring unit with multiple boxes,
+ * each box has a different function id.
+ */
+ pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)];
+ }
+
+ ret = uncore_pci_pmu_register(pdev, type, pmu, phys_id, die);
+
+ pci_set_drvdata(pdev, pmu->boxes[die]);
+
+ return ret;
+}
+
+/*
+ * Unregister the PMU of a PCI device
+ * @pmu: The corresponding PMU is unregistered.
+ * @phys_id: The physical socket id which the device maps to.
+ * @die: The die id which the device maps to.
+ */
+static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu,
+ int phys_id, int die)
+{
+ struct intel_uncore_box *box = pmu->boxes[die];
+
+ if (WARN_ON_ONCE(phys_id != box->pci_phys_id))
+ return;
+
+ pmu->boxes[die] = NULL;
+ if (atomic_dec_return(&pmu->activeboxes) == 0)
+ uncore_pmu_unregister(pmu);
+ uncore_box_exit(box);
+ kfree(box);
+}
+
static void uncore_pci_remove(struct pci_dev *pdev)
{
struct intel_uncore_box *box;
struct intel_uncore_pmu *pmu;
int i, phys_id, die;
- phys_id = uncore_pcibus_to_physid(pdev->bus);
+ if (uncore_pci_get_dev_die_info(pdev, &phys_id, &die))
+ return;
box = pci_get_drvdata(pdev);
if (!box) {
- die = (topology_max_die_per_package() > 1) ? phys_id :
- topology_phys_to_logical_pkg(phys_id);
for (i = 0; i < UNCORE_EXTRA_PCI_DEV_MAX; i++) {
if (uncore_extra_pci_dev[die].dev[i] == pdev) {
uncore_extra_pci_dev[die].dev[i] = NULL;
}
pmu = box->pmu;
- if (WARN_ON_ONCE(phys_id != box->pci_phys_id))
- return;
pci_set_drvdata(pdev, NULL);
- pmu->boxes[box->dieid] = NULL;
- if (atomic_dec_return(&pmu->activeboxes) == 0)
- uncore_pmu_unregister(pmu);
- uncore_box_exit(box);
- kfree(box);
+
+ uncore_pci_pmu_unregister(pmu, phys_id, die);
+}
+
+static int uncore_bus_notify(struct notifier_block *nb,
+ unsigned long action, void *data)
+{
+ struct device *dev = data;
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct intel_uncore_pmu *pmu;
+ int phys_id, die;
+
+ /* Unregister the PMU when the device is going to be deleted. */
+ if (action != BUS_NOTIFY_DEL_DEVICE)
+ return NOTIFY_DONE;
+
+ pmu = uncore_pci_find_dev_pmu(pdev, uncore_pci_sub_driver->id_table);
+ if (!pmu)
+ return NOTIFY_DONE;
+
+ if (uncore_pci_get_dev_die_info(pdev, &phys_id, &die))
+ return NOTIFY_DONE;
+
+ uncore_pci_pmu_unregister(pmu, phys_id, die);
+
+ return NOTIFY_OK;
+}
+
+static struct notifier_block uncore_notifier = {
+ .notifier_call = uncore_bus_notify,
+};
+
+static void uncore_pci_sub_driver_init(void)
+{
+ const struct pci_device_id *ids = uncore_pci_sub_driver->id_table;
+ struct intel_uncore_type *type;
+ struct intel_uncore_pmu *pmu;
+ struct pci_dev *pci_sub_dev;
+ bool notify = false;
+ unsigned int devfn;
+ int phys_id, die;
+
+ while (ids && ids->vendor) {
+ pci_sub_dev = NULL;
+ type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(ids->driver_data)];
+ /*
+ * Search the available device, and register the
+ * corresponding PMU.
+ */
+ while ((pci_sub_dev = pci_get_device(PCI_VENDOR_ID_INTEL,
+ ids->device, pci_sub_dev))) {
+ devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(ids->driver_data),
+ UNCORE_PCI_DEV_FUNC(ids->driver_data));
+ if (devfn != pci_sub_dev->devfn)
+ continue;
+
+ pmu = &type->pmus[UNCORE_PCI_DEV_IDX(ids->driver_data)];
+ if (!pmu)
+ continue;
+
+ if (uncore_pci_get_dev_die_info(pci_sub_dev,
+ &phys_id, &die))
+ continue;
+
+ if (!uncore_pci_pmu_register(pci_sub_dev, type, pmu,
+ phys_id, die))
+ notify = true;
+ }
+ ids++;
+ }
+
+ if (notify && bus_register_notifier(&pci_bus_type, &uncore_notifier))
+ notify = false;
+
+ if (!notify)
+ uncore_pci_sub_driver = NULL;
}
static int __init uncore_pci_init(void)
if (ret)
goto errtype;
+ if (uncore_pci_sub_driver)
+ uncore_pci_sub_driver_init();
+
pcidrv_registered = true;
return 0;
{
if (pcidrv_registered) {
pcidrv_registered = false;
+ if (uncore_pci_sub_driver)
+ bus_unregister_notifier(&pci_bus_type, &uncore_notifier);
pci_unregister_driver(uncore_pci_driver);
uncore_types_exit(uncore_pci_uncores);
kfree(uncore_extra_pci_dev);
};
static const struct intel_uncore_init_fun tgl_uncore_init __initconst = {
- .cpu_init = icl_uncore_cpu_init,
+ .cpu_init = tgl_uncore_cpu_init,
.mmio_init = tgl_uncore_mmio_init,
};
static const struct intel_uncore_init_fun tgl_l_uncore_init __initconst = {
- .cpu_init = icl_uncore_cpu_init,
+ .cpu_init = tgl_uncore_cpu_init,
.mmio_init = tgl_l_uncore_mmio_init,
};
extern struct intel_uncore_type **uncore_pci_uncores;
extern struct intel_uncore_type **uncore_mmio_uncores;
extern struct pci_driver *uncore_pci_driver;
+extern struct pci_driver *uncore_pci_sub_driver;
extern raw_spinlock_t pci2phy_map_lock;
extern struct list_head pci2phy_map_head;
extern struct pci_extra_dev *uncore_extra_pci_dev;
void nhm_uncore_cpu_init(void);
void skl_uncore_cpu_init(void);
void icl_uncore_cpu_init(void);
+void tgl_uncore_cpu_init(void);
void tgl_uncore_mmio_init(void);
void tgl_l_uncore_mmio_init(void);
int snb_pci2phy_map_init(int devid);
#define ICL_UNC_CBO_0_PER_CTR0 0x702
#define ICL_UNC_CBO_MSR_OFFSET 0x8
+/* ICL ARB register */
+#define ICL_UNC_ARB_PER_CTR 0x3b1
+#define ICL_UNC_ARB_PERFEVTSEL 0x3b3
+
DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
snb_uncore_arb.ops = &skl_uncore_msr_ops;
}
+static struct intel_uncore_ops icl_uncore_msr_ops = {
+ .disable_event = snb_uncore_msr_disable_event,
+ .enable_event = snb_uncore_msr_enable_event,
+ .read_counter = uncore_msr_read_counter,
+};
+
static struct intel_uncore_type icl_uncore_cbox = {
.name = "cbox",
- .num_counters = 4,
+ .num_counters = 2,
.perf_ctr_bits = 44,
.perf_ctr = ICL_UNC_CBO_0_PER_CTR0,
.event_ctl = SNB_UNC_CBO_0_PERFEVTSEL0,
.event_mask = SNB_UNC_RAW_EVENT_MASK,
.msr_offset = ICL_UNC_CBO_MSR_OFFSET,
- .ops = &skl_uncore_msr_ops,
+ .ops = &icl_uncore_msr_ops,
.format_group = &snb_uncore_format_group,
};
.single_fixed = 1,
.event_mask = SNB_UNC_CTL_EV_SEL_MASK,
.format_group = &icl_uncore_clock_format_group,
- .ops = &skl_uncore_msr_ops,
+ .ops = &icl_uncore_msr_ops,
.event_descs = icl_uncore_events,
};
+static struct intel_uncore_type icl_uncore_arb = {
+ .name = "arb",
+ .num_counters = 1,
+ .num_boxes = 1,
+ .perf_ctr_bits = 44,
+ .perf_ctr = ICL_UNC_ARB_PER_CTR,
+ .event_ctl = ICL_UNC_ARB_PERFEVTSEL,
+ .event_mask = SNB_UNC_RAW_EVENT_MASK,
+ .ops = &icl_uncore_msr_ops,
+ .format_group = &snb_uncore_format_group,
+};
+
static struct intel_uncore_type *icl_msr_uncores[] = {
&icl_uncore_cbox,
- &snb_uncore_arb,
+ &icl_uncore_arb,
&icl_uncore_clockbox,
NULL,
};
{
uncore_msr_uncores = icl_msr_uncores;
icl_uncore_cbox.num_boxes = icl_get_cbox_num();
+}
+
+static struct intel_uncore_type *tgl_msr_uncores[] = {
+ &icl_uncore_cbox,
+ &snb_uncore_arb,
+ &icl_uncore_clockbox,
+ NULL,
+};
+
+void tgl_uncore_cpu_init(void)
+{
+ uncore_msr_uncores = tgl_msr_uncores;
+ icl_uncore_cbox.num_boxes = icl_get_cbox_num();
+ icl_uncore_cbox.ops = &skl_uncore_msr_ops;
+ icl_uncore_clockbox.ops = &skl_uncore_msr_ops;
snb_uncore_arb.ops = &skl_uncore_msr_ops;
}
#define SNR_M2M_PCI_PMON_BOX_CTL 0x438
#define SNR_M2M_PCI_PMON_UMASK_EXT 0xff
+/* SNR PCIE3 */
+#define SNR_PCIE3_PCI_PMON_CTL0 0x508
+#define SNR_PCIE3_PCI_PMON_CTR0 0x4e8
+#define SNR_PCIE3_PCI_PMON_BOX_CTL 0x4e0
+
/* SNR IMC */
#define SNR_IMC_MMIO_PMON_FIXED_CTL 0x54
#define SNR_IMC_MMIO_PMON_FIXED_CTR 0x38
ret = skx_iio_get_topology(type);
if (ret)
- return ret;
+ goto clear_attr_update;
+
+ ret = -ENOMEM;
/* One more for NULL. */
attrs = kcalloc((uncore_max_dies() + 1), sizeof(*attrs), GFP_KERNEL);
kfree(eas);
kfree(attrs);
kfree(type->topology);
+clear_attr_update:
type->attr_update = NULL;
- return -ENOMEM;
+ return ret;
}
static void skx_iio_cleanup_mapping(struct intel_uncore_type *type)
.format_group = &snr_m2m_uncore_format_group,
};
+static void snr_uncore_pci_enable_event(struct intel_uncore_box *box, struct perf_event *event)
+{
+ struct pci_dev *pdev = box->pci_dev;
+ struct hw_perf_event *hwc = &event->hw;
+
+ pci_write_config_dword(pdev, hwc->config_base, (u32)(hwc->config | SNBEP_PMON_CTL_EN));
+ pci_write_config_dword(pdev, hwc->config_base + 4, (u32)(hwc->config >> 32));
+}
+
+static struct intel_uncore_ops snr_pcie3_uncore_pci_ops = {
+ .init_box = snr_m2m_uncore_pci_init_box,
+ .disable_box = snbep_uncore_pci_disable_box,
+ .enable_box = snbep_uncore_pci_enable_box,
+ .disable_event = snbep_uncore_pci_disable_event,
+ .enable_event = snr_uncore_pci_enable_event,
+ .read_counter = snbep_uncore_pci_read_counter,
+};
+
+static struct intel_uncore_type snr_uncore_pcie3 = {
+ .name = "pcie3",
+ .num_counters = 4,
+ .num_boxes = 1,
+ .perf_ctr_bits = 48,
+ .perf_ctr = SNR_PCIE3_PCI_PMON_CTR0,
+ .event_ctl = SNR_PCIE3_PCI_PMON_CTL0,
+ .event_mask = SKX_IIO_PMON_RAW_EVENT_MASK,
+ .event_mask_ext = SKX_IIO_PMON_RAW_EVENT_MASK_EXT,
+ .box_ctl = SNR_PCIE3_PCI_PMON_BOX_CTL,
+ .ops = &snr_pcie3_uncore_pci_ops,
+ .format_group = &skx_uncore_iio_format_group,
+};
+
enum {
SNR_PCI_UNCORE_M2M,
+ SNR_PCI_UNCORE_PCIE3,
};
static struct intel_uncore_type *snr_pci_uncores[] = {
[SNR_PCI_UNCORE_M2M] = &snr_uncore_m2m,
+ [SNR_PCI_UNCORE_PCIE3] = &snr_uncore_pcie3,
NULL,
};
.id_table = snr_uncore_pci_ids,
};
+static const struct pci_device_id snr_uncore_pci_sub_ids[] = {
+ { /* PCIe3 RP */
+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x334a),
+ .driver_data = UNCORE_PCI_DEV_FULL_DATA(4, 0, SNR_PCI_UNCORE_PCIE3, 0),
+ },
+ { /* end: all zeroes */ }
+};
+
+static struct pci_driver snr_uncore_pci_sub_driver = {
+ .name = "snr_uncore_sub",
+ .id_table = snr_uncore_pci_sub_ids,
+};
+
int snr_uncore_pci_init(void)
{
/* SNR UBOX DID */
uncore_pci_uncores = snr_pci_uncores;
uncore_pci_driver = &snr_uncore_pci_driver;
+ uncore_pci_sub_driver = &snr_uncore_pci_sub_driver;
return 0;
}
INTEL_UNCORE_EVENT_DESC(dclk, "event=0xff,umask=0x10"),
INTEL_UNCORE_EVENT_DESC(read, "event=0xff,umask=0x20"),
- INTEL_UNCORE_EVENT_DESC(read.scale, "3.814697266e-6"),
+ INTEL_UNCORE_EVENT_DESC(read.scale, "6.103515625e-5"),
INTEL_UNCORE_EVENT_DESC(read.unit, "MiB"),
INTEL_UNCORE_EVENT_DESC(write, "event=0xff,umask=0x21"),
- INTEL_UNCORE_EVENT_DESC(write.scale, "3.814697266e-6"),
+ INTEL_UNCORE_EVENT_DESC(write.scale, "6.103515625e-5"),
INTEL_UNCORE_EVENT_DESC(write.unit, "MiB"),
{ /* end: all zeroes */ },
};
INTEL_UNCORE_EVENT_DESC(dclk, "event=0xff,umask=0x10"),
INTEL_UNCORE_EVENT_DESC(read, "event=0xff,umask=0x20"),
- INTEL_UNCORE_EVENT_DESC(read.scale, "3.814697266e-6"),
+ INTEL_UNCORE_EVENT_DESC(read.scale, "6.103515625e-5"),
INTEL_UNCORE_EVENT_DESC(read.unit, "MiB"),
INTEL_UNCORE_EVENT_DESC(write, "event=0xff,umask=0x21"),
- INTEL_UNCORE_EVENT_DESC(write.scale, "3.814697266e-6"),
+ INTEL_UNCORE_EVENT_DESC(write.scale, "6.103515625e-5"),
INTEL_UNCORE_EVENT_DESC(write.unit, "MiB"),
INTEL_UNCORE_EVENT_DESC(ddrt_read, "event=0xff,umask=0x30"),
- INTEL_UNCORE_EVENT_DESC(ddrt_read.scale, "3.814697266e-6"),
+ INTEL_UNCORE_EVENT_DESC(ddrt_read.scale, "6.103515625e-5"),
INTEL_UNCORE_EVENT_DESC(ddrt_read.unit, "MiB"),
INTEL_UNCORE_EVENT_DESC(ddrt_write, "event=0xff,umask=0x31"),
- INTEL_UNCORE_EVENT_DESC(ddrt_write.scale, "3.814697266e-6"),
+ INTEL_UNCORE_EVENT_DESC(ddrt_write.scale, "6.103515625e-5"),
INTEL_UNCORE_EVENT_DESC(ddrt_write.unit, "MiB"),
{ /* end: all zeroes */ },
};
case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
case INTEL_FAM6_ATOM_TREMONT_D:
case INTEL_FAM6_ATOM_TREMONT:
+ case INTEL_FAM6_ATOM_TREMONT_L:
case INTEL_FAM6_XEON_PHI_KNL:
case INTEL_FAM6_XEON_PHI_KNM:
#define PERF_X86_EVENT_PEBS_VIA_PT 0x0800 /* use PT buffer for PEBS */
#define PERF_X86_EVENT_PAIR 0x1000 /* Large Increment per Cycle */
#define PERF_X86_EVENT_LBR_SELECT 0x2000 /* Save/Restore MSR_LBR_SELECT */
+#define PERF_X86_EVENT_TOPDOWN 0x4000 /* Count Topdown slots/metrics events */
+
+static inline bool is_topdown_count(struct perf_event *event)
+{
+ return event->hw.flags & PERF_X86_EVENT_TOPDOWN;
+}
+
+static inline bool is_metric_event(struct perf_event *event)
+{
+ u64 config = event->attr.config;
+
+ return ((config & ARCH_PERFMON_EVENTSEL_EVENT) == 0) &&
+ ((config & INTEL_ARCH_EVENT_MASK) >= INTEL_TD_METRIC_RETIRING) &&
+ ((config & INTEL_ARCH_EVENT_MASK) <= INTEL_TD_METRIC_MAX);
+}
+
+static inline bool is_slots_event(struct perf_event *event)
+{
+ return (event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_TD_SLOTS;
+}
+
+static inline bool is_topdown_event(struct perf_event *event)
+{
+ return is_metric_event(event) || is_slots_event(event);
+}
struct amd_nb {
int nb_id; /* NorthBridge id */
they've never been enabled yet */
int n_txn; /* the # last events in the below arrays;
added in the current transaction */
+ int n_txn_pair;
+ int n_txn_metric;
int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
u64 tags[X86_PMC_IDX_MAX];
*/
u64 tfa_shadow;
+ /*
+ * Perf Metrics
+ */
+ /* number of accepted metrics events */
+ int n_metric;
+
/*
* AMD specific bits
*/
#define FIXED_EVENT_CONSTRAINT(c, n) \
EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
+/*
+ * The special metric counters do not actually exist. They are calculated from
+ * the combination of the FxCtr3 + MSR_PERF_METRICS.
+ *
+ * The special metric counters are mapped to a dummy offset for the scheduler.
+ * The sharing between multiple users of the same metric without multiplexing
+ * is not allowed, even though the hardware supports that in principle.
+ */
+
+#define METRIC_EVENT_CONSTRAINT(c, n) \
+ EVENT_CONSTRAINT(c, (1ULL << (INTEL_PMC_IDX_METRIC_BASE + n)), \
+ INTEL_ARCH_EVENT_MASK)
+
/*
* Constraint on the Event code + UMask
*/
*/
u64 full_width_write:1;
u64 pebs_baseline:1;
- u64 pebs_metrics_available:1;
+ u64 perf_metrics:1;
u64 pebs_output_pt_available:1;
};
u64 capabilities;
*/
atomic_t lbr_exclusive[x86_lbr_exclusive_max];
+ /*
+ * Intel perf metrics
+ */
+ u64 (*update_topdown_event)(struct perf_event *event);
+ int (*set_topdown_event_period)(struct perf_event *event);
+
/*
* perf task context (i.e. struct perf_event_context::task_ctx_data)
* switch helper to bridge calls from perf/core to perf/x86.
X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &model_spr),
X86_MATCH_VENDOR_FAM(AMD, 0x17, &model_amd_fam17h),
X86_MATCH_VENDOR_FAM(HYGON, 0x18, &model_amd_fam17h),
+ X86_MATCH_VENDOR_FAM(AMD, 0x19, &model_amd_fam17h),
{},
};
MODULE_DEVICE_TABLE(x86cpu, rapl_model_match);
* Check for required features and priviliges to make TSC frequency
* change notifications work.
*/
- return ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS &&
+ return ms_hyperv.features & HV_ACCESS_FREQUENCY_MSRS &&
ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE &&
- ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT;
+ ms_hyperv.features & HV_ACCESS_REENLIGHTENMENT;
}
DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment)
return;
/* Absolutely required MSRs */
- required_msrs = HV_X64_MSR_HYPERCALL_AVAILABLE |
- HV_X64_MSR_VP_INDEX_AVAILABLE;
+ required_msrs = HV_MSR_HYPERCALL_AVAILABLE |
+ HV_MSR_VP_INDEX_AVAILABLE;
if ((ms_hyperv.features & required_msrs) != required_msrs)
return;
{
if (!hv_pvspin || !apic ||
!(ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) ||
- !(ms_hyperv.features & HV_X64_MSR_GUEST_IDLE_AVAILABLE)) {
+ !(ms_hyperv.features & HV_MSR_GUEST_IDLE_AVAILABLE)) {
pr_info("PV spinlocks disabled\n");
return;
}
static inline void apic_smt_update(void) { }
#endif
+struct msi_msg;
+
+#ifdef CONFIG_PCI_MSI
+void x86_vector_msi_compose_msg(struct irq_data *data, struct msi_msg *msg);
+#else
+# define x86_vector_msi_compose_msg NULL
+#endif
+
extern void ioapic_zap_locks(void);
#endif /* _ASM_X86_APIC_H */
#include <asm/string.h>
#include <asm/page.h>
#include <asm/checksum.h>
+#include <asm/mce.h>
#include <asm-generic/asm-prototypes.h>
# define _ASM_EXTABLE_UA(from, to) \
_ASM_EXTABLE_HANDLE(from, to, ex_handler_uaccess)
+# define _ASM_EXTABLE_CPY(from, to) \
+ _ASM_EXTABLE_HANDLE(from, to, ex_handler_copy)
+
# define _ASM_EXTABLE_FAULT(from, to) \
_ASM_EXTABLE_HANDLE(from, to, ex_handler_fault)
-# define _ASM_NOKPROBE(entry) \
+# ifdef CONFIG_KPROBES
+# define _ASM_NOKPROBE(entry) \
.pushsection "_kprobe_blacklist","aw" ; \
_ASM_ALIGN ; \
_ASM_PTR (entry); \
.popsection
+# else
+# define _ASM_NOKPROBE(entry)
+# endif
#else /* ! __ASSEMBLY__ */
# define _EXPAND_EXTABLE_HANDLE(x) #x
# define _ASM_EXTABLE_UA(from, to) \
_ASM_EXTABLE_HANDLE(from, to, ex_handler_uaccess)
+# define _ASM_EXTABLE_CPY(from, to) \
+ _ASM_EXTABLE_HANDLE(from, to, ex_handler_copy)
+
# define _ASM_EXTABLE_FAULT(from, to) \
_ASM_EXTABLE_HANDLE(from, to, ex_handler_fault)
/* SPDX-License-Identifier: GPL-2.0 */
#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER 1
#define HAVE_CSUM_COPY_USER
+#define _HAVE_ARCH_CSUM_AND_COPY
#ifdef CONFIG_X86_32
# include <asm/checksum_32.h>
#else
* better 64-bit) boundary
*/
-asmlinkage __wsum csum_partial_copy_generic(const void *src, void *dst,
- int len, __wsum sum,
- int *src_err_ptr, int *dst_err_ptr);
+asmlinkage __wsum csum_partial_copy_generic(const void *src, void *dst, int len);
/*
* Note: when you get a NULL pointer exception here this means someone
* If you use these functions directly please don't forget the
* access_ok().
*/
-static inline __wsum csum_partial_copy_nocheck(const void *src, void *dst,
- int len, __wsum sum)
+static inline __wsum csum_partial_copy_nocheck(const void *src, void *dst, int len)
{
- return csum_partial_copy_generic(src, dst, len, sum, NULL, NULL);
+ return csum_partial_copy_generic(src, dst, len);
}
static inline __wsum csum_and_copy_from_user(const void __user *src,
- void *dst, int len,
- __wsum sum, int *err_ptr)
+ void *dst, int len)
{
__wsum ret;
might_sleep();
- if (!user_access_begin(src, len)) {
- if (len)
- *err_ptr = -EFAULT;
- return sum;
- }
- ret = csum_partial_copy_generic((__force void *)src, dst,
- len, sum, err_ptr, NULL);
+ if (!user_access_begin(src, len))
+ return 0;
+ ret = csum_partial_copy_generic((__force void *)src, dst, len);
user_access_end();
return ret;
*/
static inline __wsum csum_and_copy_to_user(const void *src,
void __user *dst,
- int len, __wsum sum,
- int *err_ptr)
+ int len)
{
__wsum ret;
might_sleep();
- if (user_access_begin(dst, len)) {
- ret = csum_partial_copy_generic(src, (__force void *)dst,
- len, sum, NULL, err_ptr);
- user_access_end();
- return ret;
- }
+ if (!user_access_begin(dst, len))
+ return 0;
- if (len)
- *err_ptr = -EFAULT;
-
- return (__force __wsum)-1; /* invalid checksum */
+ ret = csum_partial_copy_generic(src, (__force void *)dst, len);
+ user_access_end();
+ return ret;
}
#endif /* _ASM_X86_CHECKSUM_32_H */
extern __wsum csum_partial(const void *buff, int len, __wsum sum);
/* Do not call this directly. Use the wrappers below */
-extern __visible __wsum csum_partial_copy_generic(const void *src, const void *dst,
- int len, __wsum sum,
- int *src_err_ptr, int *dst_err_ptr);
+extern __visible __wsum csum_partial_copy_generic(const void *src, void *dst, int len);
-
-extern __wsum csum_and_copy_from_user(const void __user *src, void *dst,
- int len, __wsum isum, int *errp);
-extern __wsum csum_and_copy_to_user(const void *src, void __user *dst,
- int len, __wsum isum, int *errp);
-extern __wsum csum_partial_copy_nocheck(const void *src, void *dst,
- int len, __wsum sum);
+extern __wsum csum_and_copy_from_user(const void __user *src, void *dst, int len);
+extern __wsum csum_and_copy_to_user(const void *src, void __user *dst, int len);
+extern __wsum csum_partial_copy_nocheck(const void *src, void *dst, int len);
/**
* ip_compute_csum - Compute an 16bit IP checksum.
typedef u16 compat_ipc_pid_t;
typedef u32 compat_caddr_t;
typedef __kernel_fsid_t compat_fsid_t;
-typedef s64 __attribute__((aligned(4))) compat_s64;
-typedef u64 __attribute__((aligned(4))) compat_u64;
struct compat_stat {
compat_dev_t st_dev;
return in_32bit_syscall();
}
#define in_compat_syscall in_compat_syscall /* override the generic impl */
+#define compat_need_64bit_alignment_fixup in_ia32_syscall
#endif
struct compat_siginfo;
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _COPY_MC_TEST_H_
+#define _COPY_MC_TEST_H_
+
+#ifndef __ASSEMBLY__
+#ifdef CONFIG_COPY_MC_TEST
+extern unsigned long copy_mc_test_src;
+extern unsigned long copy_mc_test_dst;
+
+static inline void copy_mc_inject_src(void *addr)
+{
+ if (addr)
+ copy_mc_test_src = (unsigned long) addr;
+ else
+ copy_mc_test_src = ~0UL;
+}
+
+static inline void copy_mc_inject_dst(void *addr)
+{
+ if (addr)
+ copy_mc_test_dst = (unsigned long) addr;
+ else
+ copy_mc_test_dst = ~0UL;
+}
+#else /* CONFIG_COPY_MC_TEST */
+static inline void copy_mc_inject_src(void *addr)
+{
+}
+
+static inline void copy_mc_inject_dst(void *addr)
+{
+}
+#endif /* CONFIG_COPY_MC_TEST */
+
+#else /* __ASSEMBLY__ */
+#include <asm/export.h>
+
+#ifdef CONFIG_COPY_MC_TEST
+.macro COPY_MC_TEST_CTL
+ .pushsection .data
+ .align 8
+ .globl copy_mc_test_src
+ copy_mc_test_src:
+ .quad 0
+ EXPORT_SYMBOL_GPL(copy_mc_test_src)
+ .globl copy_mc_test_dst
+ copy_mc_test_dst:
+ .quad 0
+ EXPORT_SYMBOL_GPL(copy_mc_test_dst)
+ .popsection
+.endm
+
+.macro COPY_MC_TEST_SRC reg count target
+ leaq \count(\reg), %r9
+ cmp copy_mc_test_src, %r9
+ ja \target
+.endm
+
+.macro COPY_MC_TEST_DST reg count target
+ leaq \count(\reg), %r9
+ cmp copy_mc_test_dst, %r9
+ ja \target
+.endm
+#else
+.macro COPY_MC_TEST_CTL
+.endm
+
+.macro COPY_MC_TEST_SRC reg count target
+.endm
+
+.macro COPY_MC_TEST_DST reg count target
+.endm
+#endif /* CONFIG_COPY_MC_TEST */
+#endif /* __ASSEMBLY__ */
+#endif /* _COPY_MC_TEST_H_ */
#define X86_FEATURE_SYSCALL32 ( 3*32+14) /* "" syscall in IA32 userspace */
#define X86_FEATURE_SYSENTER32 ( 3*32+15) /* "" sysenter in IA32 userspace */
#define X86_FEATURE_REP_GOOD ( 3*32+16) /* REP microcode works well */
-/* free ( 3*32+17) */
+#define X86_FEATURE_SME_COHERENT ( 3*32+17) /* "" AMD hardware-enforced cache coherency */
#define X86_FEATURE_LFENCE_RDTSC ( 3*32+18) /* "" LFENCE synchronizes RDTSC */
#define X86_FEATURE_ACC_POWER ( 3*32+19) /* AMD Accumulated Power Mechanism */
#define X86_FEATURE_NOPL ( 3*32+20) /* The NOPL (0F 1F) instructions */
#define X86_FEATURE_FENCE_SWAPGS_USER (11*32+ 4) /* "" LFENCE in user entry SWAPGS path */
#define X86_FEATURE_FENCE_SWAPGS_KERNEL (11*32+ 5) /* "" LFENCE in kernel entry SWAPGS path */
#define X86_FEATURE_SPLIT_LOCK_DETECT (11*32+ 6) /* #AC for split lock */
+#define X86_FEATURE_PER_THREAD_MBA (11*32+ 7) /* "" Per-thread Memory Bandwidth Allocation */
/* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
#define X86_FEATURE_AVX512_BF16 (12*32+ 5) /* AVX512 BFLOAT16 instructions */
#define X86_FEATURE_CLDEMOTE (16*32+25) /* CLDEMOTE instruction */
#define X86_FEATURE_MOVDIRI (16*32+27) /* MOVDIRI instruction */
#define X86_FEATURE_MOVDIR64B (16*32+28) /* MOVDIR64B instruction */
+#define X86_FEATURE_ENQCMD (16*32+29) /* ENQCMD and ENQCMDS instructions */
/* AMD-defined CPU features, CPUID level 0x80000007 (EBX), word 17 */
#define X86_FEATURE_OVERFLOW_RECOV (17*32+ 0) /* MCA overflow recovery support */
#define X86_FEATURE_MD_CLEAR (18*32+10) /* VERW clears CPU buffers */
#define X86_FEATURE_TSX_FORCE_ABORT (18*32+13) /* "" TSX_FORCE_ABORT */
#define X86_FEATURE_SERIALIZE (18*32+14) /* SERIALIZE instruction */
+#define X86_FEATURE_TSXLDTRK (18*32+16) /* TSX Suspend Load Address Tracking */
#define X86_FEATURE_PCONFIG (18*32+18) /* Intel PCONFIG */
#define X86_FEATURE_ARCH_LBR (18*32+19) /* Intel ARCH LBR */
#define X86_FEATURE_SPEC_CTRL (18*32+26) /* "" Speculation Control (IBRS + IBPB) */
return __this_cpu_read(cpu_dr7) & DR_GLOBAL_ENABLE_MASK;
}
-extern void aout_dump_debugregs(struct user *dump);
-
extern void hw_breakpoint_restore(void);
static __always_inline unsigned long local_db_save(void)
# define DISABLE_PTI (1 << (X86_FEATURE_PTI & 31))
#endif
+#ifdef CONFIG_IOMMU_SUPPORT
+# define DISABLE_ENQCMD 0
+#else
+# define DISABLE_ENQCMD (1 << (X86_FEATURE_ENQCMD & 31))
+#endif
+
/*
* Make sure to add features to the correct mask
*/
#define DISABLED_MASK13 0
#define DISABLED_MASK14 0
#define DISABLED_MASK15 0
-#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP)
+#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \
+ DISABLE_ENQCMD)
#define DISABLED_MASK17 0
#define DISABLED_MASK18 0
#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)
(b)->handler = (tmp).handler - (delta); \
} while (0)
+enum handler_type {
+ EX_HANDLER_NONE,
+ EX_HANDLER_FAULT,
+ EX_HANDLER_UACCESS,
+ EX_HANDLER_OTHER
+};
+
extern int fixup_exception(struct pt_regs *regs, int trapnr,
unsigned long error_code, unsigned long fault_addr);
extern int fixup_bug(struct pt_regs *regs, int trapnr);
-extern bool ex_has_fault_handler(unsigned long ip);
+extern enum handler_type ex_get_fault_handler_type(unsigned long ip);
extern void early_fixup_exception(struct pt_regs *regs, int trapnr);
#endif
FIX_PCIE_MCFG,
#endif
#endif
-#ifdef CONFIG_PARAVIRT
+#ifdef CONFIG_PARAVIRT_XXL
FIX_PARAVIRT_BOOTMAP,
#endif
#ifdef CONFIG_X86_INTEL_MID
*/
extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name);
+/*
+ * Tasks that are not using SVA have mm->pasid set to zero to note that they
+ * will not have the valid bit set in MSR_IA32_PASID while they are running.
+ */
+#define PASID_DISABLED 0
+
+#ifdef CONFIG_IOMMU_SUPPORT
+/* Update current's PASID MSR/state by mm's PASID. */
+void update_pasid(void);
+#else
+static inline void update_pasid(void) { }
+#endif
#endif /* _ASM_X86_FPU_API_H */
pkru_val = pk->pkru;
}
__write_pkru(pkru_val);
+
+ /*
+ * Expensive PASID MSR write will be avoided in update_pasid() because
+ * TIF_NEED_FPU_LOAD was set. And the PASID state won't be updated
+ * unless it's different from mm->pasid to reduce overhead.
+ */
+ update_pasid();
}
/*
{
u32 eax, edx;
- asm volatile(".byte 0x0f,0x01,0xd0" /* xgetbv */
- : "=a" (eax), "=d" (edx)
- : "c" (index));
+ asm volatile("xgetbv" : "=a" (eax), "=d" (edx) : "c" (index));
return eax + ((u64)edx << 32);
}
u32 eax = value;
u32 edx = value >> 32;
- asm volatile(".byte 0x0f,0x01,0xd1" /* xsetbv */
- : : "a" (eax), "d" (edx), "c" (index));
+ asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index));
}
#endif /* _ASM_X86_FPU_INTERNAL_H */
XFEATURE_Hi16_ZMM,
XFEATURE_PT_UNIMPLEMENTED_SO_FAR,
XFEATURE_PKRU,
- XFEATURE_RSRVD_COMP_10,
+ XFEATURE_PASID,
XFEATURE_RSRVD_COMP_11,
XFEATURE_RSRVD_COMP_12,
XFEATURE_RSRVD_COMP_13,
#define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM)
#define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)
#define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU)
+#define XFEATURE_MASK_PASID (1 << XFEATURE_PASID)
#define XFEATURE_MASK_LBR (1 << XFEATURE_LBR)
#define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE)
struct lbr_entry entries[];
} __packed;
+/*
+ * State component 10 is supervisor state used for context-switching the
+ * PASID state.
+ */
+struct ia32_pasid_state {
+ u64 pasid;
+} __packed;
+
struct xstate_header {
u64 xfeatures;
u64 xcomp_bv;
XFEATURE_MASK_BNDCSR)
/* All currently supported supervisor features */
-#define XFEATURE_MASK_SUPERVISOR_SUPPORTED (0)
+#define XFEATURE_MASK_SUPERVISOR_SUPPORTED (XFEATURE_MASK_PASID)
/*
* A supervisor state component may not always contain valuable information,
{
unsigned long fsbase;
- if (static_cpu_has(X86_FEATURE_FSGSBASE))
+ if (boot_cpu_has(X86_FEATURE_FSGSBASE))
fsbase = rdfsbase();
else
rdmsrl(MSR_FS_BASE, fsbase);
static inline void x86_fsbase_write_cpu(unsigned long fsbase)
{
- if (static_cpu_has(X86_FEATURE_FSGSBASE))
+ if (boot_cpu_has(X86_FEATURE_FSGSBASE))
wrfsbase(fsbase);
else
wrmsrl(MSR_FS_BASE, fsbase);
enum irq_alloc_type {
X86_IRQ_ALLOC_TYPE_IOAPIC = 1,
X86_IRQ_ALLOC_TYPE_HPET,
- X86_IRQ_ALLOC_TYPE_MSI,
- X86_IRQ_ALLOC_TYPE_MSIX,
+ X86_IRQ_ALLOC_TYPE_PCI_MSI,
+ X86_IRQ_ALLOC_TYPE_PCI_MSIX,
X86_IRQ_ALLOC_TYPE_DMAR,
X86_IRQ_ALLOC_TYPE_UV,
+ X86_IRQ_ALLOC_TYPE_IOAPIC_GET_PARENT,
+ X86_IRQ_ALLOC_TYPE_HPET_GET_PARENT,
};
+struct ioapic_alloc_info {
+ int pin;
+ int node;
+ u32 trigger : 1;
+ u32 polarity : 1;
+ u32 valid : 1;
+ struct IO_APIC_route_entry *entry;
+};
+
+struct uv_alloc_info {
+ int limit;
+ int blade;
+ unsigned long offset;
+ char *name;
+
+};
+
+/**
+ * irq_alloc_info - X86 specific interrupt allocation info
+ * @type: X86 specific allocation type
+ * @flags: Flags for allocation tweaks
+ * @devid: Device ID for allocations
+ * @hwirq: Associated hw interrupt number in the domain
+ * @mask: CPU mask for vector allocation
+ * @desc: Pointer to msi descriptor
+ * @data: Allocation specific data
+ *
+ * @ioapic: IOAPIC specific allocation data
+ * @uv: UV specific allocation data
+*/
struct irq_alloc_info {
enum irq_alloc_type type;
u32 flags;
- const struct cpumask *mask; /* CPU mask for vector allocation */
+ u32 devid;
+ irq_hw_number_t hwirq;
+ const struct cpumask *mask;
+ struct msi_desc *desc;
+ void *data;
+
union {
- int unused;
-#ifdef CONFIG_HPET_TIMER
- struct {
- int hpet_id;
- int hpet_index;
- void *hpet_data;
- };
-#endif
-#ifdef CONFIG_PCI_MSI
- struct {
- struct pci_dev *msi_dev;
- irq_hw_number_t msi_hwirq;
- };
-#endif
-#ifdef CONFIG_X86_IO_APIC
- struct {
- int ioapic_id;
- int ioapic_pin;
- int ioapic_node;
- u32 ioapic_trigger : 1;
- u32 ioapic_polarity : 1;
- u32 ioapic_valid : 1;
- struct IO_APIC_route_entry *ioapic_entry;
- };
-#endif
-#ifdef CONFIG_DMAR_TABLE
- struct {
- int dmar_id;
- void *dmar_data;
- };
-#endif
-#ifdef CONFIG_X86_UV
- struct {
- int uv_limit;
- int uv_blade;
- unsigned long uv_offset;
- char *uv_name;
- };
-#endif
-#if IS_ENABLED(CONFIG_VMD)
- struct {
- struct msi_desc *desc;
- };
-#endif
+ struct ioapic_alloc_info ioapic;
+ struct uv_alloc_info uv;
};
};
#define HYPERV_CPUID_MIN 0x40000005
#define HYPERV_CPUID_MAX 0x4000ffff
-/*
- * Aliases for Group A features that have X64 in the name.
- * On x86/x64 these are HYPERV_CPUID_FEATURES.EAX bits.
- */
-
-#define HV_X64_MSR_VP_RUNTIME_AVAILABLE \
- HV_MSR_VP_RUNTIME_AVAILABLE
-#define HV_X64_MSR_SYNIC_AVAILABLE \
- HV_MSR_SYNIC_AVAILABLE
-#define HV_X64_MSR_APIC_ACCESS_AVAILABLE \
- HV_MSR_APIC_ACCESS_AVAILABLE
-#define HV_X64_MSR_HYPERCALL_AVAILABLE \
- HV_MSR_HYPERCALL_AVAILABLE
-#define HV_X64_MSR_VP_INDEX_AVAILABLE \
- HV_MSR_VP_INDEX_AVAILABLE
-#define HV_X64_MSR_RESET_AVAILABLE \
- HV_MSR_RESET_AVAILABLE
-#define HV_X64_MSR_GUEST_IDLE_AVAILABLE \
- HV_MSR_GUEST_IDLE_AVAILABLE
-#define HV_X64_ACCESS_FREQUENCY_MSRS \
- HV_ACCESS_FREQUENCY_MSRS
-#define HV_X64_ACCESS_REENLIGHTENMENT \
- HV_ACCESS_REENLIGHTENMENT
-#define HV_X64_ACCESS_TSC_INVARIANT \
- HV_ACCESS_TSC_INVARIANT
-
-/*
- * Aliases for Group B features that have X64 in the name.
- * On x86/x64 these are HYPERV_CPUID_FEATURES.EBX bits.
- */
-#define HV_X64_POST_MESSAGES HV_POST_MESSAGES
-#define HV_X64_SIGNAL_EVENTS HV_SIGNAL_EVENTS
-
/*
* Group D Features. The bit assignments are custom to each architecture.
* On x86/x64 these are HYPERV_CPUID_FEATURES.EDX bits.
instrumentation_begin(); \
irq_enter_rcu(); \
kvm_set_cpu_l1tf_flush_l1d(); \
- run_on_irqstack_cond(__##func, regs, regs); \
+ run_sysvec_on_irqstack_cond(__##func, regs); \
irq_exit_rcu(); \
instrumentation_end(); \
irqentry_exit(regs, state); \
/* NMI */
DECLARE_IDTENTRY_NMI(X86_TRAP_NMI, exc_nmi);
-#if defined(CONFIG_XEN_PV) && defined(CONFIG_X86_64)
+#ifdef CONFIG_XEN_PV
DECLARE_IDTENTRY_RAW(X86_TRAP_NMI, xenpv_exc_nmi);
#endif
#else
DECLARE_IDTENTRY_RAW(X86_TRAP_DB, exc_debug);
#endif
-#if defined(CONFIG_XEN_PV) && defined(CONFIG_X86_64)
+#ifdef CONFIG_XEN_PV
DECLARE_IDTENTRY_RAW(X86_TRAP_DB, xenpv_exc_debug);
#endif
#endif
#ifdef CONFIG_X86_LOCAL_APIC
-# ifdef CONFIG_X86_UV
-DECLARE_IDTENTRY_SYSVEC(UV_BAU_MESSAGE, sysvec_uv_bau_message);
-# endif
-
# ifdef CONFIG_X86_MCE_THRESHOLD
DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR, sysvec_threshold);
# endif
/**
* iosubmit_cmds512 - copy data to single MMIO location, in 512-bit units
- * @__dst: destination, in MMIO space (must be 512-bit aligned)
+ * @dst: destination, in MMIO space (must be 512-bit aligned)
* @src: source
* @count: number of 512 bits quantities to submit
*
* Warning: Do not use this helper unless your driver has checked that the CPU
* instruction is supported on the platform.
*/
-static inline void iosubmit_cmds512(void __iomem *__dst, const void *src,
+static inline void iosubmit_cmds512(void __iomem *dst, const void *src,
size_t count)
{
- /*
- * Note that this isn't an "on-stack copy", just definition of "dst"
- * as a pointer to 64-bytes of stuff that is going to be overwritten.
- * In the MOVDIR64B case that may be needed as you can use the
- * MOVDIR64B instruction to copy arbitrary memory around. This trick
- * lets the compiler know how much gets clobbered.
- */
- volatile struct { char _[64]; } *dst = __dst;
const u8 *from = src;
const u8 *end = from + count * 64;
while (from < end) {
- /* MOVDIR64B [rdx], rax */
- asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
- : "=m" (dst)
- : "d" (from), "a" (dst));
+ movdir64b(dst, from);
from += 64;
}
}
extern int irq_remap_enable_fault_handling(void);
extern void panic_if_irq_remap(const char *msg);
-extern struct irq_domain *
-irq_remapping_get_ir_irq_domain(struct irq_alloc_info *info);
extern struct irq_domain *
irq_remapping_get_irq_domain(struct irq_alloc_info *info);
{
}
-static inline struct irq_domain *
-irq_remapping_get_ir_irq_domain(struct irq_alloc_info *info)
-{
- return NULL;
-}
-
static inline struct irq_domain *
irq_remapping_get_irq_domain(struct irq_alloc_info *info)
{
return __this_cpu_read(irq_count) != -1;
}
-void asm_call_on_stack(void *sp, void *func, void *arg);
+void asm_call_on_stack(void *sp, void (*func)(void), void *arg);
+void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs),
+ struct pt_regs *regs);
+void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc),
+ struct irq_desc *desc);
-static __always_inline void __run_on_irqstack(void *func, void *arg)
+static __always_inline void __run_on_irqstack(void (*func)(void))
{
void *tos = __this_cpu_read(hardirq_stack_ptr);
__this_cpu_add(irq_count, 1);
- asm_call_on_stack(tos - 8, func, arg);
+ asm_call_on_stack(tos - 8, func, NULL);
+ __this_cpu_sub(irq_count, 1);
+}
+
+static __always_inline void
+__run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
+ struct pt_regs *regs)
+{
+ void *tos = __this_cpu_read(hardirq_stack_ptr);
+
+ __this_cpu_add(irq_count, 1);
+ asm_call_sysvec_on_stack(tos - 8, func, regs);
+ __this_cpu_sub(irq_count, 1);
+}
+
+static __always_inline void
+__run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
+ struct irq_desc *desc)
+{
+ void *tos = __this_cpu_read(hardirq_stack_ptr);
+
+ __this_cpu_add(irq_count, 1);
+ asm_call_irq_on_stack(tos - 8, func, desc);
__this_cpu_sub(irq_count, 1);
}
#else /* CONFIG_X86_64 */
static inline bool irqstack_active(void) { return false; }
-static inline void __run_on_irqstack(void *func, void *arg) { }
+static inline void __run_on_irqstack(void (*func)(void)) { }
+static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
+ struct pt_regs *regs) { }
+static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
+ struct irq_desc *desc) { }
#endif /* !CONFIG_X86_64 */
static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs)
return !user_mode(regs) && !irqstack_active();
}
-static __always_inline void run_on_irqstack_cond(void *func, void *arg,
+
+static __always_inline void run_on_irqstack_cond(void (*func)(void),
struct pt_regs *regs)
{
- void (*__func)(void *arg) = func;
+ lockdep_assert_irqs_disabled();
+
+ if (irq_needs_irq_stack(regs))
+ __run_on_irqstack(func);
+ else
+ func();
+}
+
+static __always_inline void
+run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs),
+ struct pt_regs *regs)
+{
+ lockdep_assert_irqs_disabled();
+ if (irq_needs_irq_stack(regs))
+ __run_sysvec_on_irqstack(func, regs);
+ else
+ func(regs);
+}
+
+static __always_inline void
+run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc,
+ struct pt_regs *regs)
+{
lockdep_assert_irqs_disabled();
if (irq_needs_irq_stack(regs))
- __run_on_irqstack(__func, arg);
+ __run_irq_on_irqstack(func, desc);
else
- __func(arg);
+ func(desc);
}
#endif
#endif /* CONFIG_X86_IO_APIC */
#ifdef CONFIG_PCI_MSI
-extern void arch_init_msi_domain(struct irq_domain *domain);
+void x86_create_pci_msi_domain(void);
+struct irq_domain *native_create_pci_msi_domain(void);
+extern struct irq_domain *x86_pci_msi_default_domain;
#else
-static inline void arch_init_msi_domain(struct irq_domain *domain) { }
+static inline void x86_create_pci_msi_domain(void) { }
+#define native_create_pci_msi_domain NULL
+#define x86_pci_msi_default_domain NULL
#endif
#endif
extern int kprobe_int3_handler(struct pt_regs *regs);
extern int kprobe_debug_handler(struct pt_regs *regs);
+#else
+
+static inline int kprobe_debug_handler(struct pt_regs *regs) { return 0; }
+
#endif /* CONFIG_KPROBES */
#endif /* _ASM_X86_KPROBES_H */
#define MCE_HANDLED_NFIT BIT_ULL(3)
#define MCE_HANDLED_EDAC BIT_ULL(4)
#define MCE_HANDLED_MCELOG BIT_ULL(5)
+
+/*
+ * Indicates an MCE which has happened in kernel space but from
+ * which the kernel can recover simply by executing fixup_exception()
+ * so that an error is returned to the caller of the function that
+ * hit the machine check.
+ */
#define MCE_IN_KERNEL_RECOV BIT_ULL(6)
+/*
+ * Indicates an MCE that happened in kernel space while copying data
+ * from user. In this case fixup_exception() gets the kernel to the
+ * error exit for the copy function. Machine check handler can then
+ * treat it like a fault taken in user mode.
+ */
+#define MCE_IN_KERNEL_COPYIN BIT_ULL(7)
+
/*
* This structure contains all data related to the MCE log. Also
* carries a signature to make it easier to find from external
extern int mce_p5_enabled;
+#ifdef CONFIG_ARCH_HAS_COPY_MC
+extern void enable_copy_mc_fragile(void);
+unsigned long __must_check copy_mc_fragile(void *dst, const void *src, unsigned cnt);
+#else
+static inline void enable_copy_mc_fragile(void)
+{
+}
+#endif
+
#ifdef CONFIG_X86_MCE
int mcheck_init(void);
void mcheck_cpu_init(struct cpuinfo_x86 *c);
void mce_log(struct mce *m);
DECLARE_PER_CPU(struct device *, mce_device);
-/*
- * Maximum banks number.
- * This is the limit of the current register layout on
- * Intel CPUs.
- */
-#define MAX_NR_BANKS 32
+/* Maximum number of MCA banks per CPU. */
+#define MAX_NR_BANKS 64
#ifdef CONFIG_X86_MCE_INTEL
void mce_intel_feature_init(struct cpuinfo_x86 *c);
struct smca_hwid {
unsigned int bank_type; /* Use with smca_bank_types for easy indexing. */
u32 hwid_mcatype; /* (hwid,mcatype) tuple */
- u32 xec_bitmap; /* Bitmap of valid ExtErrorCodes; current max is 21. */
u8 count; /* Number of instances. */
};
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _MCSAFE_TEST_H_
-#define _MCSAFE_TEST_H_
-
-#ifndef __ASSEMBLY__
-#ifdef CONFIG_MCSAFE_TEST
-extern unsigned long mcsafe_test_src;
-extern unsigned long mcsafe_test_dst;
-
-static inline void mcsafe_inject_src(void *addr)
-{
- if (addr)
- mcsafe_test_src = (unsigned long) addr;
- else
- mcsafe_test_src = ~0UL;
-}
-
-static inline void mcsafe_inject_dst(void *addr)
-{
- if (addr)
- mcsafe_test_dst = (unsigned long) addr;
- else
- mcsafe_test_dst = ~0UL;
-}
-#else /* CONFIG_MCSAFE_TEST */
-static inline void mcsafe_inject_src(void *addr)
-{
-}
-
-static inline void mcsafe_inject_dst(void *addr)
-{
-}
-#endif /* CONFIG_MCSAFE_TEST */
-
-#else /* __ASSEMBLY__ */
-#include <asm/export.h>
-
-#ifdef CONFIG_MCSAFE_TEST
-.macro MCSAFE_TEST_CTL
- .pushsection .data
- .align 8
- .globl mcsafe_test_src
- mcsafe_test_src:
- .quad 0
- EXPORT_SYMBOL_GPL(mcsafe_test_src)
- .globl mcsafe_test_dst
- mcsafe_test_dst:
- .quad 0
- EXPORT_SYMBOL_GPL(mcsafe_test_dst)
- .popsection
-.endm
-
-.macro MCSAFE_TEST_SRC reg count target
- leaq \count(\reg), %r9
- cmp mcsafe_test_src, %r9
- ja \target
-.endm
-
-.macro MCSAFE_TEST_DST reg count target
- leaq \count(\reg), %r9
- cmp mcsafe_test_dst, %r9
- ja \target
-.endm
-#else
-.macro MCSAFE_TEST_CTL
-.endm
-
-.macro MCSAFE_TEST_SRC reg count target
-.endm
-
-.macro MCSAFE_TEST_DST reg count target
-.endm
-#endif /* CONFIG_MCSAFE_TEST */
-#endif /* __ASSEMBLY__ */
-#endif /* _MCSAFE_TEST_H_ */
#ifdef CONFIG_X86_MPPARSE
extern void e820__memblock_alloc_reserved_mpc_new(void);
extern int enable_update_mptable;
-extern int default_mpc_apic_id(struct mpc_cpu *m);
-extern void default_smp_read_mpc_oem(struct mpc_table *mpc);
-# ifdef CONFIG_X86_IO_APIC
-extern void default_mpc_oem_bus_info(struct mpc_bus *m, char *str);
-# else
-# define default_mpc_oem_bus_info NULL
-# endif
extern void default_find_smp_config(void);
extern void default_get_smp_config(unsigned int early);
#else
static inline void e820__memblock_alloc_reserved_mpc_new(void) { }
#define enable_update_mptable 0
-#define default_mpc_apic_id NULL
-#define default_smp_read_mpc_oem NULL
-#define default_mpc_oem_bus_info NULL
#define default_find_smp_config x86_init_noop
#define default_get_smp_config x86_init_uint_noop
#endif
int pci_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec,
msi_alloc_info_t *arg);
-void pci_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc);
-
#endif /* _ASM_X86_MSI_H */
#define MSR_IA32_LASTINTFROMIP 0x000001dd
#define MSR_IA32_LASTINTTOIP 0x000001de
+#define MSR_IA32_PASID 0x00000d93
+#define MSR_IA32_PASID_VALID BIT_ULL(31)
+
/* DEBUGCTLMSR bits (others vary by model): */
#define DEBUGCTLMSR_LBR (1UL << 0) /* last branch recording */
#define DEBUGCTLMSR_BTF_SHIFT 1
#define MSR_AMD64_IBSOP_REG_MASK ((1UL<<MSR_AMD64_IBSOP_REG_COUNT)-1)
#define MSR_AMD64_IBSCTL 0xc001103a
#define MSR_AMD64_IBSBRTARGET 0xc001103b
+#define MSR_AMD64_ICIBSEXTDCTL 0xc001103c
#define MSR_AMD64_IBSOPDATA4 0xc001103d
#define MSR_AMD64_IBS_REG_COUNT_MAX 8 /* includes MSR_AMD64_IBSBRTARGET */
#define MSR_AMD64_SEV 0xc0010131
#define MSR_CORE_PERF_FIXED_CTR0 0x00000309
#define MSR_CORE_PERF_FIXED_CTR1 0x0000030a
#define MSR_CORE_PERF_FIXED_CTR2 0x0000030b
+#define MSR_CORE_PERF_FIXED_CTR3 0x0000030c
#define MSR_CORE_PERF_FIXED_CTR_CTRL 0x0000038d
#define MSR_CORE_PERF_GLOBAL_STATUS 0x0000038e
#define MSR_CORE_PERF_GLOBAL_CTRL 0x0000038f
#define MSR_CORE_PERF_GLOBAL_OVF_CTRL 0x00000390
+#define MSR_PERF_METRICS 0x00000329
+
/* PERF_GLOBAL_OVF_CTL bits */
#define MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT 55
#define MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI (1ULL << MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI_BIT)
PVOP_VCALL0(cpu.wbinvd);
}
-#define get_kernel_rpl() (pv_info.kernel_rpl)
-
static inline u64 paravirt_read_msr(unsigned msr)
{
return PVOP_CALL1(u64, cpu.read_msr, msr);
PVOP_VCALL2(cpu.load_tls, t, cpu);
}
-#ifdef CONFIG_X86_64
static inline void load_gs_index(unsigned int gs)
{
PVOP_VCALL1(cpu.load_gs_index, gs);
}
-#endif
static inline void write_ldt_entry(struct desc_struct *dt, int entry,
const void *desc)
static inline pte_t __pte(pteval_t val)
{
- pteval_t ret;
-
- if (sizeof(pteval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pteval_t, mmu.make_pte, val, (u64)val >> 32);
- else
- ret = PVOP_CALLEE1(pteval_t, mmu.make_pte, val);
-
- return (pte_t) { .pte = ret };
+ return (pte_t) { PVOP_CALLEE1(pteval_t, mmu.make_pte, val) };
}
static inline pteval_t pte_val(pte_t pte)
{
- pteval_t ret;
-
- if (sizeof(pteval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pteval_t, mmu.pte_val,
- pte.pte, (u64)pte.pte >> 32);
- else
- ret = PVOP_CALLEE1(pteval_t, mmu.pte_val, pte.pte);
-
- return ret;
+ return PVOP_CALLEE1(pteval_t, mmu.pte_val, pte.pte);
}
static inline pgd_t __pgd(pgdval_t val)
{
- pgdval_t ret;
-
- if (sizeof(pgdval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pgdval_t, mmu.make_pgd, val, (u64)val >> 32);
- else
- ret = PVOP_CALLEE1(pgdval_t, mmu.make_pgd, val);
-
- return (pgd_t) { ret };
+ return (pgd_t) { PVOP_CALLEE1(pgdval_t, mmu.make_pgd, val) };
}
static inline pgdval_t pgd_val(pgd_t pgd)
{
- pgdval_t ret;
-
- if (sizeof(pgdval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pgdval_t, mmu.pgd_val,
- pgd.pgd, (u64)pgd.pgd >> 32);
- else
- ret = PVOP_CALLEE1(pgdval_t, mmu.pgd_val, pgd.pgd);
-
- return ret;
+ return PVOP_CALLEE1(pgdval_t, mmu.pgd_val, pgd.pgd);
}
#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
pte_t *ptep, pte_t old_pte, pte_t pte)
{
- if (sizeof(pteval_t) > sizeof(long))
- /* 5 arg words */
- pv_ops.mmu.ptep_modify_prot_commit(vma, addr, ptep, pte);
- else
- PVOP_VCALL4(mmu.ptep_modify_prot_commit,
- vma, addr, ptep, pte.pte);
+ PVOP_VCALL4(mmu.ptep_modify_prot_commit, vma, addr, ptep, pte.pte);
}
static inline void set_pte(pte_t *ptep, pte_t pte)
{
- if (sizeof(pteval_t) > sizeof(long))
- PVOP_VCALL3(mmu.set_pte, ptep, pte.pte, (u64)pte.pte >> 32);
- else
- PVOP_VCALL2(mmu.set_pte, ptep, pte.pte);
-}
-
-static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep, pte_t pte)
-{
- if (sizeof(pteval_t) > sizeof(long))
- /* 5 arg words */
- pv_ops.mmu.set_pte_at(mm, addr, ptep, pte);
- else
- PVOP_VCALL4(mmu.set_pte_at, mm, addr, ptep, pte.pte);
+ PVOP_VCALL2(mmu.set_pte, ptep, pte.pte);
}
static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
{
- pmdval_t val = native_pmd_val(pmd);
-
- if (sizeof(pmdval_t) > sizeof(long))
- PVOP_VCALL3(mmu.set_pmd, pmdp, val, (u64)val >> 32);
- else
- PVOP_VCALL2(mmu.set_pmd, pmdp, val);
+ PVOP_VCALL2(mmu.set_pmd, pmdp, native_pmd_val(pmd));
}
-#if CONFIG_PGTABLE_LEVELS >= 3
static inline pmd_t __pmd(pmdval_t val)
{
- pmdval_t ret;
-
- if (sizeof(pmdval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pmdval_t, mmu.make_pmd, val, (u64)val >> 32);
- else
- ret = PVOP_CALLEE1(pmdval_t, mmu.make_pmd, val);
-
- return (pmd_t) { ret };
+ return (pmd_t) { PVOP_CALLEE1(pmdval_t, mmu.make_pmd, val) };
}
static inline pmdval_t pmd_val(pmd_t pmd)
{
- pmdval_t ret;
-
- if (sizeof(pmdval_t) > sizeof(long))
- ret = PVOP_CALLEE2(pmdval_t, mmu.pmd_val,
- pmd.pmd, (u64)pmd.pmd >> 32);
- else
- ret = PVOP_CALLEE1(pmdval_t, mmu.pmd_val, pmd.pmd);
-
- return ret;
+ return PVOP_CALLEE1(pmdval_t, mmu.pmd_val, pmd.pmd);
}
static inline void set_pud(pud_t *pudp, pud_t pud)
{
- pudval_t val = native_pud_val(pud);
-
- if (sizeof(pudval_t) > sizeof(long))
- PVOP_VCALL3(mmu.set_pud, pudp, val, (u64)val >> 32);
- else
- PVOP_VCALL2(mmu.set_pud, pudp, val);
+ PVOP_VCALL2(mmu.set_pud, pudp, native_pud_val(pud));
}
-#if CONFIG_PGTABLE_LEVELS >= 4
+
static inline pud_t __pud(pudval_t val)
{
pudval_t ret;
static inline void pud_clear(pud_t *pudp)
{
- set_pud(pudp, __pud(0));
+ set_pud(pudp, native_make_pud(0));
}
static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
} while (0)
#define pgd_clear(pgdp) do { \
- if (pgtable_l5_enabled()) \
- set_pgd(pgdp, __pgd(0)); \
+ if (pgtable_l5_enabled()) \
+ set_pgd(pgdp, native_make_pgd(0)); \
} while (0)
#endif /* CONFIG_PGTABLE_LEVELS == 5 */
static inline void p4d_clear(p4d_t *p4dp)
{
- set_p4d(p4dp, __p4d(0));
+ set_p4d(p4dp, native_make_p4d(0));
}
-#endif /* CONFIG_PGTABLE_LEVELS == 4 */
-
-#endif /* CONFIG_PGTABLE_LEVELS >= 3 */
-
-#ifdef CONFIG_X86_PAE
-/* Special-case pte-setting operations for PAE, which can't update a
- 64-bit pte atomically */
-static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
-{
- PVOP_VCALL3(mmu.set_pte_atomic, ptep, pte.pte, pte.pte >> 32);
-}
-
-static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep)
-{
- PVOP_VCALL3(mmu.pte_clear, mm, addr, ptep);
-}
-
-static inline void pmd_clear(pmd_t *pmdp)
-{
- PVOP_VCALL1(mmu.pmd_clear, pmdp);
-}
-#else /* !CONFIG_X86_PAE */
static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
{
set_pte(ptep, pte);
static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
pte_t *ptep)
{
- set_pte_at(mm, addr, ptep, __pte(0));
+ set_pte(ptep, native_make_pte(0));
}
static inline void pmd_clear(pmd_t *pmdp)
{
- set_pmd(pmdp, __pmd(0));
+ set_pmd(pmdp, native_make_pmd(0));
}
-#endif /* CONFIG_X86_PAE */
#define __HAVE_ARCH_START_CONTEXT_SWITCH
static inline void arch_start_context_switch(struct task_struct *prev)
#endif /* SMP && PARAVIRT_SPINLOCKS */
#ifdef CONFIG_X86_32
-#define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
-#define PV_RESTORE_REGS "popl %edx; popl %ecx;"
-
/* save and restore all caller-save registers, except return value */
#define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;"
#define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;"
-
-#define PV_FLAGS_ARG "0"
-#define PV_EXTRA_CLOBBERS
-#define PV_VEXTRA_CLOBBERS
#else
/* save and restore all caller-save registers, except return value */
#define PV_SAVE_ALL_CALLER_REGS \
"pop %rsi;" \
"pop %rdx;" \
"pop %rcx;"
-
-/* We save some registers, but all of them, that's too much. We clobber all
- * caller saved registers but the argument parameter */
-#define PV_SAVE_REGS "pushq %%rdi;"
-#define PV_RESTORE_REGS "popq %%rdi;"
-#define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
-#define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
-#define PV_FLAGS_ARG "D"
#endif
/*
/* general info */
struct pv_info {
#ifdef CONFIG_PARAVIRT_XXL
- unsigned int kernel_rpl;
- int shared_kernel_pmd;
-
-#ifdef CONFIG_X86_64
u16 extra_user_64bit_cs; /* __USER_CS if none */
-#endif
#endif
const char *name;
void (*set_ldt)(const void *desc, unsigned entries);
unsigned long (*store_tr)(void);
void (*load_tls)(struct thread_struct *t, unsigned int cpu);
-#ifdef CONFIG_X86_64
void (*load_gs_index)(unsigned int idx);
-#endif
void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
const void *desc);
void (*write_gdt_entry)(struct desc_struct *,
/* Pagetable manipulation functions */
void (*set_pte)(pte_t *ptep, pte_t pteval);
- void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep, pte_t pteval);
void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
pte_t (*ptep_modify_prot_start)(struct vm_area_struct *vma, unsigned long addr,
struct paravirt_callee_save pgd_val;
struct paravirt_callee_save make_pgd;
-#if CONFIG_PGTABLE_LEVELS >= 3
-#ifdef CONFIG_X86_PAE
- void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
- void (*pte_clear)(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep);
- void (*pmd_clear)(pmd_t *pmdp);
-
-#endif /* CONFIG_X86_PAE */
-
void (*set_pud)(pud_t *pudp, pud_t pudval);
struct paravirt_callee_save pmd_val;
struct paravirt_callee_save make_pmd;
-#if CONFIG_PGTABLE_LEVELS >= 4
struct paravirt_callee_save pud_val;
struct paravirt_callee_save make_pud;
void (*set_pgd)(pgd_t *pgdp, pgd_t pgdval);
#endif /* CONFIG_PGTABLE_LEVELS >= 5 */
-#endif /* CONFIG_PGTABLE_LEVELS >= 4 */
-
-#endif /* CONFIG_PGTABLE_LEVELS >= 3 */
-
struct pv_lazy_ops lazy_mode;
/* dom0 ops */
extern void pci_iommu_alloc(void);
-#ifdef CONFIG_PCI_MSI
-/* implemented in arch/x86/kernel/apic/io_apic. */
-struct msi_desc;
-int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type);
-void native_teardown_msi_irq(unsigned int irq);
-void native_restore_msi_irqs(struct pci_dev *dev);
-#else
-#define native_setup_msi_irqs NULL
-#define native_teardown_msi_irq NULL
-#endif
-
/* generic pci stuff */
#include <asm-generic/pci.h>
extern bool port_cf9_safe;
/* arch_initcall level */
+#ifdef CONFIG_PCI_DIRECT
extern int pci_direct_probe(void);
extern void pci_direct_init(int type);
+#else
+static inline int pci_direct_probe(void) { return -1; }
+static inline void pci_direct_init(int type) { }
+#endif
+
+#ifdef CONFIG_PCI_BIOS
extern void pci_pcbios_init(void);
+#else
+static inline void pci_pcbios_init(void) { }
+#endif
+
extern void __init dmi_check_pciprobe(void);
extern void __init dmi_check_skip_isa_align(void);
* Fixed-purpose performance events:
*/
+/* RDPMC offset for Fixed PMCs */
+#define INTEL_PMC_FIXED_RDPMC_BASE (1 << 30)
+#define INTEL_PMC_FIXED_RDPMC_METRICS (1 << 29)
+
/*
- * All 3 fixed-mode PMCs are configured via this single MSR:
+ * All the fixed-mode PMCs are configured via this single MSR:
*/
#define MSR_ARCH_PERFMON_FIXED_CTR_CTRL 0x38d
/*
- * The counts are available in three separate MSRs:
+ * There is no event-code assigned to the fixed-mode PMCs.
+ *
+ * For a fixed-mode PMC, which has an equivalent event on a general-purpose
+ * PMC, the event-code of the equivalent event is used for the fixed-mode PMC,
+ * e.g., Instr_Retired.Any and CPU_CLK_Unhalted.Core.
+ *
+ * For a fixed-mode PMC, which doesn't have an equivalent event, a
+ * pseudo-encoding is used, e.g., CPU_CLK_Unhalted.Ref and TOPDOWN.SLOTS.
+ * The pseudo event-code for a fixed-mode PMC must be 0x00.
+ * The pseudo umask-code is 0xX. The X equals the index of the fixed
+ * counter + 1, e.g., the fixed counter 2 has the pseudo-encoding 0x0300.
+ *
+ * The counts are available in separate MSRs:
*/
/* Instr_Retired.Any: */
#define MSR_ARCH_PERFMON_FIXED_CTR1 0x30a
#define INTEL_PMC_IDX_FIXED_CPU_CYCLES (INTEL_PMC_IDX_FIXED + 1)
-/* CPU_CLK_Unhalted.Ref: */
+/* CPU_CLK_Unhalted.Ref: event=0x00,umask=0x3 (pseudo-encoding) */
#define MSR_ARCH_PERFMON_FIXED_CTR2 0x30b
#define INTEL_PMC_IDX_FIXED_REF_CYCLES (INTEL_PMC_IDX_FIXED + 2)
#define INTEL_PMC_MSK_FIXED_REF_CYCLES (1ULL << INTEL_PMC_IDX_FIXED_REF_CYCLES)
+/* TOPDOWN.SLOTS: event=0x00,umask=0x4 (pseudo-encoding) */
+#define MSR_ARCH_PERFMON_FIXED_CTR3 0x30c
+#define INTEL_PMC_IDX_FIXED_SLOTS (INTEL_PMC_IDX_FIXED + 3)
+#define INTEL_PMC_MSK_FIXED_SLOTS (1ULL << INTEL_PMC_IDX_FIXED_SLOTS)
+
/*
* We model BTS tracing as another fixed-mode PMC.
*
- * We choose a value in the middle of the fixed event range, since lower
+ * We choose the value 47 for the fixed index of BTS, since lower
* values are used by actual fixed events and higher values are used
* to indicate other overflow conditions in the PERF_GLOBAL_STATUS msr.
*/
-#define INTEL_PMC_IDX_FIXED_BTS (INTEL_PMC_IDX_FIXED + 16)
+#define INTEL_PMC_IDX_FIXED_BTS (INTEL_PMC_IDX_FIXED + 15)
+
+/*
+ * The PERF_METRICS MSR is modeled as several magic fixed-mode PMCs, one for
+ * each TopDown metric event.
+ *
+ * Internally the TopDown metric events are mapped to the FxCtr 3 (SLOTS).
+ */
+#define INTEL_PMC_IDX_METRIC_BASE (INTEL_PMC_IDX_FIXED + 16)
+#define INTEL_PMC_IDX_TD_RETIRING (INTEL_PMC_IDX_METRIC_BASE + 0)
+#define INTEL_PMC_IDX_TD_BAD_SPEC (INTEL_PMC_IDX_METRIC_BASE + 1)
+#define INTEL_PMC_IDX_TD_FE_BOUND (INTEL_PMC_IDX_METRIC_BASE + 2)
+#define INTEL_PMC_IDX_TD_BE_BOUND (INTEL_PMC_IDX_METRIC_BASE + 3)
+#define INTEL_PMC_IDX_METRIC_END INTEL_PMC_IDX_TD_BE_BOUND
+#define INTEL_PMC_MSK_TOPDOWN ((0xfull << INTEL_PMC_IDX_METRIC_BASE) | \
+ INTEL_PMC_MSK_FIXED_SLOTS)
-#define GLOBAL_STATUS_COND_CHG BIT_ULL(63)
-#define GLOBAL_STATUS_BUFFER_OVF BIT_ULL(62)
-#define GLOBAL_STATUS_UNC_OVF BIT_ULL(61)
-#define GLOBAL_STATUS_ASIF BIT_ULL(60)
-#define GLOBAL_STATUS_COUNTERS_FROZEN BIT_ULL(59)
-#define GLOBAL_STATUS_LBRS_FROZEN_BIT 58
-#define GLOBAL_STATUS_LBRS_FROZEN BIT_ULL(GLOBAL_STATUS_LBRS_FROZEN_BIT)
-#define GLOBAL_STATUS_TRACE_TOPAPMI BIT_ULL(55)
+/*
+ * There is no event-code assigned to the TopDown events.
+ *
+ * For the slots event, use the pseudo code of the fixed counter 3.
+ *
+ * For the metric events, the pseudo event-code is 0x00.
+ * The pseudo umask-code starts from the middle of the pseudo event
+ * space, 0x80.
+ */
+#define INTEL_TD_SLOTS 0x0400 /* TOPDOWN.SLOTS */
+/* Level 1 metrics */
+#define INTEL_TD_METRIC_RETIRING 0x8000 /* Retiring metric */
+#define INTEL_TD_METRIC_BAD_SPEC 0x8100 /* Bad speculation metric */
+#define INTEL_TD_METRIC_FE_BOUND 0x8200 /* FE bound metric */
+#define INTEL_TD_METRIC_BE_BOUND 0x8300 /* BE bound metric */
+#define INTEL_TD_METRIC_MAX INTEL_TD_METRIC_BE_BOUND
+#define INTEL_TD_METRIC_NUM 4
+
+static inline bool is_metric_idx(int idx)
+{
+ return (unsigned)(idx - INTEL_PMC_IDX_METRIC_BASE) < INTEL_TD_METRIC_NUM;
+}
+
+static inline bool is_topdown_idx(int idx)
+{
+ return is_metric_idx(idx) || idx == INTEL_PMC_IDX_FIXED_SLOTS;
+}
+#define INTEL_PMC_OTHER_TOPDOWN_BITS(bit) \
+ (~(0x1ull << bit) & INTEL_PMC_MSK_TOPDOWN)
+
+#define GLOBAL_STATUS_COND_CHG BIT_ULL(63)
+#define GLOBAL_STATUS_BUFFER_OVF_BIT 62
+#define GLOBAL_STATUS_BUFFER_OVF BIT_ULL(GLOBAL_STATUS_BUFFER_OVF_BIT)
+#define GLOBAL_STATUS_UNC_OVF BIT_ULL(61)
+#define GLOBAL_STATUS_ASIF BIT_ULL(60)
+#define GLOBAL_STATUS_COUNTERS_FROZEN BIT_ULL(59)
+#define GLOBAL_STATUS_LBRS_FROZEN_BIT 58
+#define GLOBAL_STATUS_LBRS_FROZEN BIT_ULL(GLOBAL_STATUS_LBRS_FROZEN_BIT)
+#define GLOBAL_STATUS_TRACE_TOPAPMI_BIT 55
+#define GLOBAL_STATUS_TRACE_TOPAPMI BIT_ULL(GLOBAL_STATUS_TRACE_TOPAPMI_BIT)
+#define GLOBAL_STATUS_PERF_METRICS_OVF_BIT 48
+
+#define GLOBAL_CTRL_EN_PERF_METRICS 48
/*
* We model guest LBR event tracing as another fixed-mode PMC like BTS.
*
#define IBS_OP_ENABLE (1ULL<<17)
#define IBS_OP_MAX_CNT 0x0000FFFFULL
#define IBS_OP_MAX_CNT_EXT 0x007FFFFFULL /* not a register bit mask */
+#define IBS_OP_MAX_CNT_EXT_MASK (0x7FULL<<20) /* separate upper 7 bits */
#define IBS_RIP_INVALID (1ULL<<38)
#ifdef CONFIG_X86_LOCAL_APIC
} pte_t;
#endif /* !__ASSEMBLY__ */
-#ifdef CONFIG_PARAVIRT_XXL
-#define SHARED_KERNEL_PMD ((!static_cpu_has(X86_FEATURE_PTI) && \
- (pv_info.shared_kernel_pmd)))
-#else
#define SHARED_KERNEL_PMD (!static_cpu_has(X86_FEATURE_PTI))
-#endif
#define ARCH_PAGE_TABLE_SYNC_MASK (SHARED_KERNEL_PMD ? 0 : PGTBL_PMD_MODIFIED)
#include <asm/paravirt.h>
#else /* !CONFIG_PARAVIRT_XXL */
#define set_pte(ptep, pte) native_set_pte(ptep, pte)
-#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
#define set_pte_atomic(ptep, pte) \
native_set_pte_atomic(ptep, pte)
return res;
}
-static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep , pte_t pte)
+static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
+ pte_t *ptep, pte_t pte)
{
- native_set_pte(ptep, pte);
+ set_pte(ptep, pte);
}
static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
#define PGD_KERNEL_START ((PAGE_SIZE / 2) / sizeof(pgd_t))
-#define ARCH_PAGE_TABLE_SYNC_MASK (pgtable_l5_enabled() ? PGTBL_PGD_MODIFIED : PGTBL_P4D_MODIFIED)
-
#endif /* _ASM_X86_PGTABLE_64_DEFS_H */
/* Save middle states of ptrace breakpoints */
struct perf_event *ptrace_bps[HBP_NUM];
/* Debug status used for traps, single steps, etc... */
- unsigned long debugreg6;
+ unsigned long virtual_dr6;
/* Keep track of the exact dr7 value set by the user */
unsigned long ptrace_dr7;
/* Fault info: */
#endif
#ifdef CONFIG_X86_64
-#ifdef CONFIG_PARAVIRT
+#ifdef CONFIG_PARAVIRT_XXL
/* Paravirtualized systems may not have PSE or PGE available */
#define NEED_PSE 0
#define NEED_PGE 0
#endif
-#ifndef CONFIG_PARAVIRT_XXL
-# define get_kernel_rpl() 0
-#endif
-
#define IDT_ENTRIES 256
#define NUM_EXCEPTION_VECTORS 32
#define nop() asm volatile ("nop")
+static inline void serialize(void)
+{
+ /* Instruction opcode for SERIALIZE; supported in binutils >= 2.35. */
+ asm volatile(".byte 0xf, 0x1, 0xe8" ::: "memory");
+}
+
+/* The dst parameter must be 64-bytes aligned */
+static inline void movdir64b(void *dst, const void *src)
+{
+ const struct { char _[64]; } *__src = src;
+ struct { char _[64]; } *__dst = dst;
+
+ /*
+ * MOVDIR64B %(rdx), rax.
+ *
+ * Both __src and __dst must be memory constraints in order to tell the
+ * compiler that no other memory accesses should be reordered around
+ * this one.
+ *
+ * Also, both must be supplied as lvalues because this tells
+ * the compiler what the object is (its size) the instruction accesses.
+ * I.e., not the pointers but what they point to, thus the deref'ing '*'.
+ */
+ asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+ : "+m" (*__dst)
+ : "m" (*__src), "a" (__dst), "d" (__src));
+}
+
+/**
+ * enqcmds - Enqueue a command in supervisor (CPL0) mode
+ * @dst: destination, in MMIO space (must be 512-bit aligned)
+ * @src: 512 bits memory operand
+ *
+ * The ENQCMDS instruction allows software to write a 512-bit command to
+ * a 512-bit-aligned special MMIO region that supports the instruction.
+ * A return status is loaded into the ZF flag in the RFLAGS register.
+ * ZF = 0 equates to success, and ZF = 1 indicates retry or error.
+ *
+ * This function issues the ENQCMDS instruction to submit data from
+ * kernel space to MMIO space, in a unit of 512 bits. Order of data access
+ * is not guaranteed, nor is a memory barrier performed afterwards. It
+ * returns 0 on success and -EAGAIN on failure.
+ *
+ * Warning: Do not use this helper unless your driver has checked that the
+ * ENQCMDS instruction is supported on the platform and the device accepts
+ * ENQCMDS.
+ */
+static inline int enqcmds(void __iomem *dst, const void *src)
+{
+ const struct { char _[64]; } *__src = src;
+ struct { char _[64]; } *__dst = dst;
+ int zf;
+
+ /*
+ * ENQCMDS %(rdx), rax
+ *
+ * See movdir64b()'s comment on operand specification.
+ */
+ asm volatile(".byte 0xf3, 0x0f, 0x38, 0xf8, 0x02, 0x66, 0x90"
+ CC_SET(z)
+ : CC_OUT(z) (zf), "+m" (*__dst)
+ : "m" (*__src), "a" (__dst), "d" (__src));
+
+ /* Submission failure is indicated via EFLAGS.ZF=1 */
+ if (zf)
+ return -EAGAIN;
+
+ return 0;
+}
+
#endif /* __KERNEL__ */
#endif /* _ASM_X86_SPECIAL_INSNS_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_STATIC_CALL_H
+#define _ASM_STATIC_CALL_H
+
+#include <asm/text-patching.h>
+
+/*
+ * For CONFIG_HAVE_STATIC_CALL_INLINE, this is a temporary trampoline which
+ * uses the current value of the key->func pointer to do an indirect jump to
+ * the function. This trampoline is only used during boot, before the call
+ * sites get patched by static_call_update(). The name of this trampoline has
+ * a magical aspect: objtool uses it to find static call sites so it can create
+ * the .static_call_sites section.
+ *
+ * For CONFIG_HAVE_STATIC_CALL, this is a permanent trampoline which
+ * does a direct jump to the function. The direct jump gets patched by
+ * static_call_update().
+ *
+ * Having the trampoline in a special section forces GCC to emit a JMP.d32 when
+ * it does tail-call optimization on the call; since you cannot compute the
+ * relative displacement across sections.
+ */
+
+#define __ARCH_DEFINE_STATIC_CALL_TRAMP(name, insns) \
+ asm(".pushsection .static_call.text, \"ax\" \n" \
+ ".align 4 \n" \
+ ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \
+ STATIC_CALL_TRAMP_STR(name) ": \n" \
+ insns " \n" \
+ ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \
+ ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \
+ ".popsection \n")
+
+#define ARCH_DEFINE_STATIC_CALL_TRAMP(name, func) \
+ __ARCH_DEFINE_STATIC_CALL_TRAMP(name, ".byte 0xe9; .long " #func " - (. + 4)")
+
+#define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) \
+ __ARCH_DEFINE_STATIC_CALL_TRAMP(name, "ret; nop; nop; nop; nop")
+
+#endif /* _ASM_STATIC_CALL_H */
#endif
-#define __HAVE_ARCH_MEMCPY_MCSAFE 1
-__must_check unsigned long __memcpy_mcsafe(void *dst, const void *src,
- size_t cnt);
-DECLARE_STATIC_KEY_FALSE(mcsafe_key);
-
-/**
- * memcpy_mcsafe - copy memory with indication if a machine check happened
- *
- * @dst: destination address
- * @src: source address
- * @cnt: number of bytes to copy
- *
- * Low level memory copy function that catches machine checks
- * We only call into the "safe" function on systems that can
- * actually do machine check recovery. Everyone else can just
- * use memcpy().
- *
- * Return 0 for success, or number of bytes not copied if there was an
- * exception.
- */
-static __always_inline __must_check unsigned long
-memcpy_mcsafe(void *dst, const void *src, size_t cnt)
-{
-#ifdef CONFIG_X86_MCE
- if (static_branch_unlikely(&mcsafe_key))
- return __memcpy_mcsafe(dst, src, cnt);
- else
-#endif
- memcpy(dst, src, cnt);
- return 0;
-}
-
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
#define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1
void __memcpy_flushcache(void *dst, const void *src, size_t cnt);
#include <linux/preempt.h>
#include <asm/processor.h>
#include <asm/cpufeature.h>
+#include <asm/special_insns.h>
#ifdef CONFIG_X86_32
static inline void iret_to_self(void)
*
* b) Text was modified on a different CPU, may subsequently be
* executed on this CPU, and you want to make sure the new version
- * gets executed. This generally means you're calling this in a IPI.
+ * gets executed. This generally means you're calling this in an IPI.
*
* If you're calling this for a different reason, you're probably doing
* it wrong.
+ *
+ * Like all of Linux's memory ordering operations, this is a
+ * compiler barrier as well.
*/
static inline void sync_core(void)
{
/*
- * There are quite a few ways to do this. IRET-to-self is nice
- * because it works on every CPU, at any CPL (so it's compatible
- * with paravirtualization), and it never exits to a hypervisor.
- * The only down sides are that it's a bit slow (it seems to be
- * a bit more than 2x slower than the fastest options) and that
- * it unmasks NMIs. The "push %cs" is needed because, in
- * paravirtual environments, __KERNEL_CS may not be a valid CS
- * value when we do IRET directly.
+ * The SERIALIZE instruction is the most straightforward way to
+ * do this, but it is not universally available.
+ */
+ if (static_cpu_has(X86_FEATURE_SERIALIZE)) {
+ serialize();
+ return;
+ }
+
+ /*
+ * For all other processors, there are quite a few ways to do this.
+ * IRET-to-self is nice because it works on every CPU, at any CPL
+ * (so it's compatible with paravirtualization), and it never exits
+ * to a hypervisor. The only downsides are that it's a bit slow
+ * (it seems to be a bit more than 2x slower than the fastest
+ * options) and that it unmasks NMIs. The "push %cs" is needed,
+ * because in paravirtual environments __KERNEL_CS may not be a
+ * valid CS value when we do IRET directly.
*
* In case NMI unmasking or performance ever becomes a problem,
* the next best option appears to be MOV-to-CR2 and an
* CPUID is the conventional way, but it's nasty: it doesn't
* exist on some 486-like CPUs, and it usually exits to a
* hypervisor.
- *
- * Like all of Linux's memory ordering operations, this is a
- * compiler barrier as well.
*/
iret_to_self();
}
#define INT3_INSN_SIZE 1
#define INT3_INSN_OPCODE 0xCC
+#define RET_INSN_SIZE 1
+#define RET_INSN_OPCODE 0xC3
+
#define CALL_INSN_SIZE 5
#define CALL_INSN_OPCODE 0xE8
switch(opcode) {
__CASE(INT3);
+ __CASE(RET);
__CASE(CALL);
__CASE(JMP32);
__CASE(JMP8);
*(unsigned long *)regs->sp = val;
}
+static __always_inline
+unsigned long int3_emulate_pop(struct pt_regs *regs)
+{
+ unsigned long val = *(unsigned long *)regs->sp;
+ regs->sp += sizeof(unsigned long);
+ return val;
+}
+
static __always_inline
void int3_emulate_call(struct pt_regs *regs, unsigned long func)
{
int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE);
int3_emulate_jmp(regs, func);
}
+
+static __always_inline
+void int3_emulate_ret(struct pt_regs *regs)
+{
+ unsigned long ip = int3_emulate_pop(regs);
+ int3_emulate_jmp(regs, ip);
+}
#endif /* !CONFIG_UML_X86 */
#endif /* _ASM_X86_TEXT_PATCHING_H */
void math_emulate(struct math_emu_info *);
+bool fault_in_kernel_space(unsigned long address);
+
#ifdef CONFIG_VMAP_STACK
void __noreturn handle_stack_overflow(const char *message,
struct pt_regs *regs,
likely(!__range_not_ok(addr, size, user_addr_max())); \
})
-/*
- * These are the main single-value transfer routines. They automatically
- * use the right size if we just have the right pointer type.
- *
- * This gets kind of ugly. We want to return _two_ values in "get_user()"
- * and yet we don't want to do any pointers, because that is too much
- * of a performance impact. Thus we have a few rather ugly macros here,
- * and hide all the ugliness from the user.
- *
- * The "__xxx" versions of the user access functions are versions that
- * do not verify the address space, that must have been done previously
- * with a separate "access_ok()" call (this is used when we do multiple
- * accesses to the same area of user memory).
- */
-
extern int __get_user_1(void);
extern int __get_user_2(void);
extern int __get_user_4(void);
extern int __get_user_8(void);
+extern int __get_user_nocheck_1(void);
+extern int __get_user_nocheck_2(void);
+extern int __get_user_nocheck_4(void);
+extern int __get_user_nocheck_8(void);
extern int __get_user_bad(void);
#define __uaccess_begin() stac()
#define __typefits(x,type,not) \
__builtin_choose_expr(sizeof(x)<=sizeof(type),(unsigned type)0,not)
-/**
- * get_user - Get a simple variable from user space.
- * @x: Variable to store result.
- * @ptr: Source address, in user space.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * This macro copies a single simple variable from user space to kernel
- * space. It supports simple types like char and int, but not larger
- * data types like structures or arrays.
- *
- * @ptr must have pointer-to-simple-variable type, and the result of
- * dereferencing @ptr must be assignable to @x without a cast.
- *
- * Return: zero on success, or -EFAULT on error.
- * On error, the variable @x is set to zero.
- */
/*
+ * This is used for both get_user() and __get_user() to expand to
+ * the proper special function call that has odd calling conventions
+ * due to returning both a value and an error, and that depends on
+ * the size of the pointer passed in.
+ *
* Careful: we have to cast the result to the type of the pointer
* for sign reasons.
*
* Clang/LLVM cares about the size of the register, but still wants
* the base register for something that ends up being a pair.
*/
-#define get_user(x, ptr) \
+#define do_get_user_call(fn,x,ptr) \
({ \
int __ret_gu; \
register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX); \
__chk_user_ptr(ptr); \
- might_fault(); \
- asm volatile("call __get_user_%P4" \
+ asm volatile("call __" #fn "_%P4" \
: "=a" (__ret_gu), "=r" (__val_gu), \
ASM_CALL_CONSTRAINT \
: "0" (ptr), "i" (sizeof(*(ptr)))); \
__builtin_expect(__ret_gu, 0); \
})
-#define __put_user_x(size, x, ptr, __ret_pu) \
- asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
- : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
+/**
+ * get_user - Get a simple variable from user space.
+ * @x: Variable to store result.
+ * @ptr: Source address, in user space.
+ *
+ * Context: User context only. This function may sleep if pagefaults are
+ * enabled.
+ *
+ * This macro copies a single simple variable from user space to kernel
+ * space. It supports simple types like char and int, but not larger
+ * data types like structures or arrays.
+ *
+ * @ptr must have pointer-to-simple-variable type, and the result of
+ * dereferencing @ptr must be assignable to @x without a cast.
+ *
+ * Return: zero on success, or -EFAULT on error.
+ * On error, the variable @x is set to zero.
+ */
+#define get_user(x,ptr) ({ might_fault(); do_get_user_call(get_user,x,ptr); })
+/**
+ * __get_user - Get a simple variable from user space, with less checking.
+ * @x: Variable to store result.
+ * @ptr: Source address, in user space.
+ *
+ * Context: User context only. This function may sleep if pagefaults are
+ * enabled.
+ *
+ * This macro copies a single simple variable from user space to kernel
+ * space. It supports simple types like char and int, but not larger
+ * data types like structures or arrays.
+ *
+ * @ptr must have pointer-to-simple-variable type, and the result of
+ * dereferencing @ptr must be assignable to @x without a cast.
+ *
+ * Caller must check the pointer with access_ok() before calling this
+ * function.
+ *
+ * Return: zero on success, or -EFAULT on error.
+ * On error, the variable @x is set to zero.
+ */
+#define __get_user(x,ptr) do_get_user_call(get_user_nocheck,x,ptr)
#ifdef CONFIG_X86_32
: : "A" (x), "r" (addr) \
: : label)
-#define __put_user_x8(x, ptr, __ret_pu) \
- asm volatile("call __put_user_8" : "=a" (__ret_pu) \
- : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
#else
#define __put_user_goto_u64(x, ptr, label) \
__put_user_goto(x, ptr, "q", "er", label)
-#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
#endif
extern void __put_user_bad(void);
/*
* Strange magic calling convention: pointer in %ecx,
- * value in %eax(:%edx), return value in %eax. clobbers %rbx
+ * value in %eax(:%edx), return value in %ecx. clobbers %rbx
*/
extern void __put_user_1(void);
extern void __put_user_2(void);
extern void __put_user_4(void);
extern void __put_user_8(void);
+extern void __put_user_nocheck_1(void);
+extern void __put_user_nocheck_2(void);
+extern void __put_user_nocheck_4(void);
+extern void __put_user_nocheck_8(void);
+
+#define do_put_user_call(fn,x,ptr) \
+({ \
+ int __ret_pu; \
+ register __typeof__(*(ptr)) __val_pu asm("%"_ASM_AX); \
+ __chk_user_ptr(ptr); \
+ __val_pu = (x); \
+ asm volatile("call __" #fn "_%P[size]" \
+ : "=c" (__ret_pu), \
+ ASM_CALL_CONSTRAINT \
+ : "0" (ptr), \
+ "r" (__val_pu), \
+ [size] "i" (sizeof(*(ptr))) \
+ :"ebx"); \
+ __builtin_expect(__ret_pu, 0); \
+})
/**
* put_user - Write a simple value into user space.
*
* Return: zero on success, or -EFAULT on error.
*/
-#define put_user(x, ptr) \
-({ \
- int __ret_pu; \
- __typeof__(*(ptr)) __pu_val; \
- __chk_user_ptr(ptr); \
- might_fault(); \
- __pu_val = x; \
- switch (sizeof(*(ptr))) { \
- case 1: \
- __put_user_x(1, __pu_val, ptr, __ret_pu); \
- break; \
- case 2: \
- __put_user_x(2, __pu_val, ptr, __ret_pu); \
- break; \
- case 4: \
- __put_user_x(4, __pu_val, ptr, __ret_pu); \
- break; \
- case 8: \
- __put_user_x8(__pu_val, ptr, __ret_pu); \
- break; \
- default: \
- __put_user_x(X, __pu_val, ptr, __ret_pu); \
- break; \
- } \
- __builtin_expect(__ret_pu, 0); \
-})
+#define put_user(x, ptr) ({ might_fault(); do_put_user_call(put_user,x,ptr); })
+
+/**
+ * __put_user - Write a simple value into user space, with less checking.
+ * @x: Value to copy to user space.
+ * @ptr: Destination address, in user space.
+ *
+ * Context: User context only. This function may sleep if pagefaults are
+ * enabled.
+ *
+ * This macro copies a single simple value from kernel space to user
+ * space. It supports simple types like char and int, but not larger
+ * data types like structures or arrays.
+ *
+ * @ptr must have pointer-to-simple-variable type, and @x must be assignable
+ * to the result of dereferencing @ptr.
+ *
+ * Caller must check the pointer with access_ok() before calling this
+ * function.
+ *
+ * Return: zero on success, or -EFAULT on error.
+ */
+#define __put_user(x, ptr) do_put_user_call(put_user_nocheck,x,ptr)
#define __put_user_size(x, ptr, size, label) \
do { \
} \
} while (0)
+#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
+
+#ifdef CONFIG_X86_32
+#define __get_user_asm_u64(x, ptr, label) do { \
+ unsigned int __gu_low, __gu_high; \
+ const unsigned int __user *__gu_ptr; \
+ __gu_ptr = (const void __user *)(ptr); \
+ __get_user_asm(__gu_low, ptr, "l", "=r", label); \
+ __get_user_asm(__gu_high, ptr+1, "l", "=r", label); \
+ (x) = ((unsigned long long)__gu_high << 32) | __gu_low; \
+} while (0)
+#else
+#define __get_user_asm_u64(x, ptr, label) \
+ __get_user_asm(x, ptr, "q", "=r", label)
+#endif
+
+#define __get_user_size(x, ptr, size, label) \
+do { \
+ __chk_user_ptr(ptr); \
+ switch (size) { \
+ unsigned char x_u8__; \
+ case 1: \
+ __get_user_asm(x_u8__, ptr, "b", "=q", label); \
+ (x) = x_u8__; \
+ break; \
+ case 2: \
+ __get_user_asm(x, ptr, "w", "=r", label); \
+ break; \
+ case 4: \
+ __get_user_asm(x, ptr, "l", "=r", label); \
+ break; \
+ case 8: \
+ __get_user_asm_u64(x, ptr, label); \
+ break; \
+ default: \
+ (x) = __get_user_bad(); \
+ } \
+} while (0)
+
+#define __get_user_asm(x, addr, itype, ltype, label) \
+ asm_volatile_goto("\n" \
+ "1: mov"itype" %[umem],%[output]\n" \
+ _ASM_EXTABLE_UA(1b, %l2) \
+ : [output] ltype(x) \
+ : [umem] "m" (__m(addr)) \
+ : : label)
+
+#else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
+
#ifdef CONFIG_X86_32
#define __get_user_asm_u64(x, ptr, retval) \
({ \
: [umem] "m" (__m(addr)), \
[efault] "i" (-EFAULT), "0" (err))
-#define __put_user_nocheck(x, ptr, size) \
-({ \
- __label__ __pu_label; \
- int __pu_err = -EFAULT; \
- __typeof__(*(ptr)) __pu_val = (x); \
- __typeof__(ptr) __pu_ptr = (ptr); \
- __typeof__(size) __pu_size = (size); \
- __uaccess_begin(); \
- __put_user_size(__pu_val, __pu_ptr, __pu_size, __pu_label); \
- __pu_err = 0; \
-__pu_label: \
- __uaccess_end(); \
- __builtin_expect(__pu_err, 0); \
-})
-
-#define __get_user_nocheck(x, ptr, size) \
-({ \
- int __gu_err; \
- __inttype(*(ptr)) __gu_val; \
- __typeof__(ptr) __gu_ptr = (ptr); \
- __typeof__(size) __gu_size = (size); \
- __uaccess_begin_nospec(); \
- __get_user_size(__gu_val, __gu_ptr, __gu_size, __gu_err); \
- __uaccess_end(); \
- (x) = (__force __typeof__(*(ptr)))__gu_val; \
- __builtin_expect(__gu_err, 0); \
-})
+#endif // CONFIG_CC_ASM_GOTO_OUTPUT
/* FIXME: this hack is definitely wrong -AK */
struct __large_struct { unsigned long buf[100]; };
: : ltype(x), "m" (__m(addr)) \
: : label)
-/**
- * __get_user - Get a simple variable from user space, with less checking.
- * @x: Variable to store result.
- * @ptr: Source address, in user space.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * This macro copies a single simple variable from user space to kernel
- * space. It supports simple types like char and int, but not larger
- * data types like structures or arrays.
- *
- * @ptr must have pointer-to-simple-variable type, and the result of
- * dereferencing @ptr must be assignable to @x without a cast.
- *
- * Caller must check the pointer with access_ok() before calling this
- * function.
- *
- * Return: zero on success, or -EFAULT on error.
- * On error, the variable @x is set to zero.
- */
-
-#define __get_user(x, ptr) \
- __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
-
-/**
- * __put_user - Write a simple value into user space, with less checking.
- * @x: Value to copy to user space.
- * @ptr: Destination address, in user space.
- *
- * Context: User context only. This function may sleep if pagefaults are
- * enabled.
- *
- * This macro copies a single simple value from kernel space to user
- * space. It supports simple types like char and int, but not larger
- * data types like structures or arrays.
- *
- * @ptr must have pointer-to-simple-variable type, and @x must be assignable
- * to the result of dereferencing @ptr.
- *
- * Caller must check the pointer with access_ok() before calling this
- * function.
- *
- * Return: zero on success, or -EFAULT on error.
- */
-
-#define __put_user(x, ptr) \
- __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
-
extern unsigned long
copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
extern __must_check long
unsigned long __must_check clear_user(void __user *mem, unsigned long len);
unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
+#ifdef CONFIG_ARCH_HAS_COPY_MC
+unsigned long __must_check
+copy_mc_to_kernel(void *to, const void *from, unsigned len);
+#define copy_mc_to_kernel copy_mc_to_kernel
+
+unsigned long __must_check
+copy_mc_to_user(void *to, const void *from, unsigned len);
+#endif
+
/*
* movsl can be slow when source and dest are not both 8-byte aligned
*/
#define unsafe_put_user(x, ptr, label) \
__put_user_size((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), label)
+#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
+#define unsafe_get_user(x, ptr, err_label) \
+do { \
+ __inttype(*(ptr)) __gu_val; \
+ __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), err_label); \
+ (x) = (__force __typeof__(*(ptr)))__gu_val; \
+} while (0)
+#else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define unsafe_get_user(x, ptr, err_label) \
do { \
int __gu_err; \
(x) = (__force __typeof__(*(ptr)))__gu_val; \
if (unlikely(__gu_err)) goto err_label; \
} while (0)
+#endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
/*
* We want the unsafe accessors to always be inlined and use
#define HAVE_GET_KERNEL_NOFAULT
+#ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
+#define __get_kernel_nofault(dst, src, type, err_label) \
+ __get_user_size(*((type *)(dst)), (__force type __user *)(src), \
+ sizeof(type), err_label)
+#else // !CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define __get_kernel_nofault(dst, src, type, err_label) \
do { \
int __kr_err; \
if (unlikely(__kr_err)) \
goto err_label; \
} while (0)
+#endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT
#define __put_kernel_nofault(dst, src, type, err_label) \
__put_user_size(*((type *)(src)), (__force type __user *)(dst), \
return ret;
}
-static __always_inline __must_check unsigned long
-copy_to_user_mcsafe(void *to, const void *from, unsigned len)
-{
- unsigned long ret;
-
- __uaccess_begin();
- /*
- * Note, __memcpy_mcsafe() is explicitly used since it can
- * handle exceptions / faults. memcpy_mcsafe() may fall back to
- * memcpy() which lacks this handling.
- */
- ret = __memcpy_mcsafe(to, from, len);
- __uaccess_end();
- return ret;
-}
-
static __always_inline __must_check unsigned long
raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
{
kasan_check_write(dst, size);
return __copy_user_flushcache(dst, src, size);
}
-
-unsigned long
-mcsafe_handle_tail(char *to, char *from, unsigned len);
-
#endif /* _ASM_X86_UACCESS_64_H */
/*
* UV BIOS layer definitions.
*
- * Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved.
- * Copyright (c) Russ Anderson <rja@sgi.com>
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
+ * Copyright (C) 2007-2017 Silicon Graphics, Inc. All rights reserved.
+ * Copyright (c) Russ Anderson <rja@sgi.com>
*/
#include <linux/rtc.h>
u32 limit; /* PA bits 56:26 (UV_GAM_RANGE_SHFT) */
};
+#define UV_AT_SIZE 8 /* 7 character arch type + NULL char */
+struct uv_arch_type_entry {
+ char archtype[UV_AT_SIZE];
+};
+
#define UV_SYSTAB_SIG "UVST"
#define UV_SYSTAB_VERSION_1 1 /* UV2/3 BIOS version */
#define UV_SYSTAB_VERSION_UV4 0x400 /* UV4 BIOS base version */
#define UV_SYSTAB_VERSION_UV4_3 0x403 /* - GAM Range PXM Value */
#define UV_SYSTAB_VERSION_UV4_LATEST UV_SYSTAB_VERSION_UV4_3
+#define UV_SYSTAB_VERSION_UV5 0x500 /* UV5 GAM base version */
+#define UV_SYSTAB_VERSION_UV5_LATEST UV_SYSTAB_VERSION_UV5
+
#define UV_SYSTAB_TYPE_UNUSED 0 /* End of table (offset == 0) */
#define UV_SYSTAB_TYPE_GAM_PARAMS 1 /* GAM PARAM conversions */
#define UV_SYSTAB_TYPE_GAM_RNG_TBL 2 /* GAM entry table */
-#define UV_SYSTAB_TYPE_MAX 3
+#define UV_SYSTAB_TYPE_ARCH_TYPE 3 /* UV arch type */
+#define UV_SYSTAB_TYPE_MAX 4
/*
* The UV system table describes specific firmware
extern int uv_bios_set_legacy_vga_target(bool decode, int domain, int bus);
extern int uv_bios_init(void);
+extern unsigned long get_uv_systab_phys(bool msg);
extern unsigned long sn_rtc_cycles_per_second;
extern int uv_type;
extern void uv_cpu_init(void);
extern void uv_nmi_init(void);
extern void uv_system_init(void);
-extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
- const struct flush_tlb_info *info);
-#else /* X86_UV */
+#else /* !X86_UV */
static inline enum uv_system_type get_uv_system_type(void) { return UV_NONE; }
static inline bool is_early_uv_system(void) { return 0; }
+++ /dev/null
-/*
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * SGI UV Broadcast Assist Unit definitions
- *
- * Copyright (C) 2008-2011 Silicon Graphics, Inc. All rights reserved.
- */
-
-#ifndef _ASM_X86_UV_UV_BAU_H
-#define _ASM_X86_UV_UV_BAU_H
-
-#include <linux/bitmap.h>
-#include <asm/idtentry.h>
-
-#define BITSPERBYTE 8
-
-/*
- * Broadcast Assist Unit messaging structures
- *
- * Selective Broadcast activations are induced by software action
- * specifying a particular 8-descriptor "set" via a 6-bit index written
- * to an MMR.
- * Thus there are 64 unique 512-byte sets of SB descriptors - one set for
- * each 6-bit index value. These descriptor sets are mapped in sequence
- * starting with set 0 located at the address specified in the
- * BAU_SB_DESCRIPTOR_BASE register, set 1 is located at BASE + 512,
- * set 2 is at BASE + 2*512, set 3 at BASE + 3*512, and so on.
- *
- * We will use one set for sending BAU messages from each of the
- * cpu's on the uvhub.
- *
- * TLB shootdown will use the first of the 8 descriptors of each set.
- * Each of the descriptors is 64 bytes in size (8*64 = 512 bytes in a set).
- */
-
-#define MAX_CPUS_PER_UVHUB 128
-#define MAX_CPUS_PER_SOCKET 64
-#define ADP_SZ 64 /* hardware-provided max. */
-#define UV_CPUS_PER_AS 32 /* hardware-provided max. */
-#define ITEMS_PER_DESC 8
-/* the 'throttle' to prevent the hardware stay-busy bug */
-#define MAX_BAU_CONCURRENT 3
-#define UV_ACT_STATUS_MASK 0x3
-#define UV_ACT_STATUS_SIZE 2
-#define UV_DISTRIBUTION_SIZE 256
-#define UV_SW_ACK_NPENDING 8
-#define UV_NET_ENDPOINT_INTD 0x28
-#define UV_PAYLOADQ_GNODE_SHIFT 49
-#define UV_PTC_BASENAME "sgi_uv/ptc_statistics"
-#define UV_BAU_BASENAME "sgi_uv/bau_tunables"
-#define UV_BAU_TUNABLES_DIR "sgi_uv"
-#define UV_BAU_TUNABLES_FILE "bau_tunables"
-#define WHITESPACE " \t\n"
-#define cpubit_isset(cpu, bau_local_cpumask) \
- test_bit((cpu), (bau_local_cpumask).bits)
-
-/* [19:16] SOFT_ACK timeout period 19: 1 is urgency 7 17:16 1 is multiplier */
-/*
- * UV2: Bit 19 selects between
- * (0): 10 microsecond timebase and
- * (1): 80 microseconds
- * we're using 560us
- */
-#define UV_INTD_SOFT_ACK_TIMEOUT_PERIOD (15UL)
-/* assuming UV3 is the same */
-
-#define BAU_MISC_CONTROL_MULT_MASK 3
-
-#define UVH_AGING_PRESCALE_SEL 0x000000b000UL
-/* [30:28] URGENCY_7 an index into a table of times */
-#define BAU_URGENCY_7_SHIFT 28
-#define BAU_URGENCY_7_MASK 7
-
-#define UVH_TRANSACTION_TIMEOUT 0x000000b200UL
-/* [45:40] BAU - BAU transaction timeout select - a multiplier */
-#define BAU_TRANS_SHIFT 40
-#define BAU_TRANS_MASK 0x3f
-
-/*
- * shorten some awkward names
- */
-#define AS_PUSH_SHIFT UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT
-#define SOFTACK_MSHIFT UVH_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT
-#define SOFTACK_PSHIFT UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT
-#define SOFTACK_TIMEOUT_PERIOD UV_INTD_SOFT_ACK_TIMEOUT_PERIOD
-#define PREFETCH_HINT_SHFT UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_PREFETCH_HINT_SHFT
-#define SB_STATUS_SHFT UV3H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT
-#define write_gmmr uv_write_global_mmr64
-#define write_lmmr uv_write_local_mmr
-#define read_lmmr uv_read_local_mmr
-#define read_gmmr uv_read_global_mmr64
-
-/*
- * bits in UVH_LB_BAU_SB_ACTIVATION_STATUS_0/1
- */
-#define DS_IDLE 0
-#define DS_ACTIVE 1
-#define DS_DESTINATION_TIMEOUT 2
-#define DS_SOURCE_TIMEOUT 3
-/*
- * bits put together from HRP_LB_BAU_SB_ACTIVATION_STATUS_0/1/2
- * values 1 and 3 will not occur
- * Decoded meaning ERROR BUSY AUX ERR
- * ------------------------------- ---- ----- -------
- * IDLE 0 0 0
- * BUSY (active) 0 1 0
- * SW Ack Timeout (destination) 1 0 0
- * SW Ack INTD rejected (strong NACK) 1 0 1
- * Source Side Time Out Detected 1 1 0
- * Destination Side PUT Failed 1 1 1
- */
-#define UV2H_DESC_IDLE 0
-#define UV2H_DESC_BUSY 2
-#define UV2H_DESC_DEST_TIMEOUT 4
-#define UV2H_DESC_DEST_STRONG_NACK 5
-#define UV2H_DESC_SOURCE_TIMEOUT 6
-#define UV2H_DESC_DEST_PUT_ERR 7
-
-/*
- * delay for 'plugged' timeout retries, in microseconds
- */
-#define PLUGGED_DELAY 10
-
-/*
- * threshholds at which to use IPI to free resources
- */
-/* after this # consecutive 'plugged' timeouts, use IPI to release resources */
-#define PLUGSB4RESET 100
-/* after this many consecutive timeouts, use IPI to release resources */
-#define TIMEOUTSB4RESET 1
-/* at this number uses of IPI to release resources, giveup the request */
-#define IPI_RESET_LIMIT 1
-/* after this # consecutive successes, bump up the throttle if it was lowered */
-#define COMPLETE_THRESHOLD 5
-/* after this # of giveups (fall back to kernel IPI's) disable the use of
- the BAU for a period of time */
-#define GIVEUP_LIMIT 100
-
-#define UV_LB_SUBNODEID 0x10
-
-#define UV_SA_SHFT UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT
-#define UV_SA_MASK UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK
-/* 4 bits of software ack period */
-#define UV2_ACK_MASK 0x7UL
-#define UV2_ACK_UNITS_SHFT 3
-#define UV2_EXT_SHFT UV2H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT
-
-/*
- * number of entries in the destination side payload queue
- */
-#define DEST_Q_SIZE 20
-/*
- * number of destination side software ack resources
- */
-#define DEST_NUM_RESOURCES 8
-/*
- * completion statuses for sending a TLB flush message
- */
-#define FLUSH_RETRY_PLUGGED 1
-#define FLUSH_RETRY_TIMEOUT 2
-#define FLUSH_GIVEUP 3
-#define FLUSH_COMPLETE 4
-
-/*
- * tuning the action when the numalink network is extremely delayed
- */
-#define CONGESTED_RESPONSE_US 1000 /* 'long' response time, in
- microseconds */
-#define CONGESTED_REPS 10 /* long delays averaged over
- this many broadcasts */
-#define DISABLED_PERIOD 10 /* time for the bau to be
- disabled, in seconds */
-/* see msg_type: */
-#define MSG_NOOP 0
-#define MSG_REGULAR 1
-#define MSG_RETRY 2
-
-#define BAU_DESC_QUALIFIER 0x534749
-
-enum uv_bau_version {
- UV_BAU_V2 = 2,
- UV_BAU_V3,
- UV_BAU_V4,
-};
-
-/*
- * Distribution: 32 bytes (256 bits) (bytes 0-0x1f of descriptor)
- * If the 'multilevel' flag in the header portion of the descriptor
- * has been set to 0, then endpoint multi-unicast mode is selected.
- * The distribution specification (32 bytes) is interpreted as a 256-bit
- * distribution vector. Adjacent bits correspond to consecutive even numbered
- * nodeIDs. The result of adding the index of a given bit to the 15-bit
- * 'base_dest_nasid' field of the header corresponds to the
- * destination nodeID associated with that specified bit.
- */
-struct pnmask {
- unsigned long bits[BITS_TO_LONGS(UV_DISTRIBUTION_SIZE)];
-};
-
-/*
- * mask of cpu's on a uvhub
- * (during initialization we need to check that unsigned long has
- * enough bits for max. cpu's per uvhub)
- */
-struct bau_local_cpumask {
- unsigned long bits;
-};
-
-/*
- * Payload: 16 bytes (128 bits) (bytes 0x20-0x2f of descriptor)
- * only 12 bytes (96 bits) of the payload area are usable.
- * An additional 3 bytes (bits 27:4) of the header address are carried
- * to the next bytes of the destination payload queue.
- * And an additional 2 bytes of the header Suppl_A field are also
- * carried to the destination payload queue.
- * But the first byte of the Suppl_A becomes bits 127:120 (the 16th byte)
- * of the destination payload queue, which is written by the hardware
- * with the s/w ack resource bit vector.
- * [ effective message contents (16 bytes (128 bits) maximum), not counting
- * the s/w ack bit vector ]
- */
-
-/**
- * struct uv2_3_bau_msg_payload - defines payload for INTD transactions
- * @address: Signifies a page or all TLB's of the cpu
- * @sending_cpu: CPU from which the message originates
- * @acknowledge_count: CPUs on the destination Hub that received the interrupt
- */
-struct uv2_3_bau_msg_payload {
- u64 address;
- u16 sending_cpu;
- u16 acknowledge_count;
-};
-
-/**
- * struct uv4_bau_msg_payload - defines payload for INTD transactions
- * @address: Signifies a page or all TLB's of the cpu
- * @sending_cpu: CPU from which the message originates
- * @acknowledge_count: CPUs on the destination Hub that received the interrupt
- * @qualifier: Set by source to verify origin of INTD broadcast
- */
-struct uv4_bau_msg_payload {
- u64 address;
- u16 sending_cpu;
- u16 acknowledge_count;
- u32 reserved:8;
- u32 qualifier:24;
-};
-
-/*
- * UV2 Message header: 16 bytes (128 bits) (bytes 0x30-0x3f of descriptor)
- * see figure 9-2 of harp_sys.pdf
- * assuming UV3 is the same
- */
-struct uv2_3_bau_msg_header {
- unsigned int base_dest_nasid:15; /* nasid of the first bit */
- /* bits 14:0 */ /* in uvhub map */
- unsigned int dest_subnodeid:5; /* must be 0x10, for the LB */
- /* bits 19:15 */
- unsigned int rsvd_1:1; /* must be zero */
- /* bit 20 */
- /* Address bits 59:21 */
- /* bits 25:2 of address (44:21) are payload */
- /* these next 24 bits become bytes 12-14 of msg */
- /* bits 28:21 land in byte 12 */
- unsigned int replied_to:1; /* sent as 0 by the source to
- byte 12 */
- /* bit 21 */
- unsigned int msg_type:3; /* software type of the
- message */
- /* bits 24:22 */
- unsigned int canceled:1; /* message canceled, resource
- is to be freed*/
- /* bit 25 */
- unsigned int payload_1:3; /* not currently used */
- /* bits 28:26 */
-
- /* bits 36:29 land in byte 13 */
- unsigned int payload_2a:3; /* not currently used */
- unsigned int payload_2b:5; /* not currently used */
- /* bits 36:29 */
-
- /* bits 44:37 land in byte 14 */
- unsigned int payload_3:8; /* not currently used */
- /* bits 44:37 */
-
- unsigned int rsvd_2:7; /* reserved */
- /* bits 51:45 */
- unsigned int swack_flag:1; /* software acknowledge flag */
- /* bit 52 */
- unsigned int rsvd_3a:3; /* must be zero */
- unsigned int rsvd_3b:8; /* must be zero */
- unsigned int rsvd_3c:8; /* must be zero */
- unsigned int rsvd_3d:3; /* must be zero */
- /* bits 74:53 */
- unsigned int fairness:3; /* usually zero */
- /* bits 77:75 */
-
- unsigned int sequence:16; /* message sequence number */
- /* bits 93:78 Suppl_A */
- unsigned int chaining:1; /* next descriptor is part of
- this activation*/
- /* bit 94 */
- unsigned int multilevel:1; /* multi-level multicast
- format */
- /* bit 95 */
- unsigned int rsvd_4:24; /* ordered / source node /
- source subnode / aging
- must be zero */
- /* bits 119:96 */
- unsigned int command:8; /* message type */
- /* bits 127:120 */
-};
-
-/*
- * The activation descriptor:
- * The format of the message to send, plus all accompanying control
- * Should be 64 bytes
- */
-struct bau_desc {
- struct pnmask distribution;
- /*
- * message template, consisting of header and payload:
- */
- union bau_msg_header {
- struct uv2_3_bau_msg_header uv2_3_hdr;
- } header;
-
- union bau_payload_header {
- struct uv2_3_bau_msg_payload uv2_3;
- struct uv4_bau_msg_payload uv4;
- } payload;
-};
-/* UV2:
- * -payload-- ---------header------
- * bytes 0-11 bits 70-78 bits 21-44
- * A B (2) C (3)
- *
- * A/B/C are moved to:
- * A C B
- * bytes 0-11 bytes 12-14 bytes 16-17 (byte 15 filled in by hw as vector)
- * ------------payload queue-----------
- */
-
-/*
- * The payload queue on the destination side is an array of these.
- * With BAU_MISC_CONTROL set for software acknowledge mode, the messages
- * are 32 bytes (2 micropackets) (256 bits) in length, but contain only 17
- * bytes of usable data, including the sw ack vector in byte 15 (bits 127:120)
- * (12 bytes come from bau_msg_payload, 3 from payload_1, 2 from
- * swack_vec and payload_2)
- * "Enabling Software Acknowledgment mode (see Section 4.3.3 Software
- * Acknowledge Processing) also selects 32 byte (17 bytes usable) payload
- * operation."
- */
-struct bau_pq_entry {
- unsigned long address; /* signifies a page or all TLB's
- of the cpu */
- /* 64 bits, bytes 0-7 */
- unsigned short sending_cpu; /* cpu that sent the message */
- /* 16 bits, bytes 8-9 */
- unsigned short acknowledge_count; /* filled in by destination */
- /* 16 bits, bytes 10-11 */
- /* these next 3 bytes come from bits 58-81 of the message header */
- unsigned short replied_to:1; /* sent as 0 by the source */
- unsigned short msg_type:3; /* software message type */
- unsigned short canceled:1; /* sent as 0 by the source */
- unsigned short unused1:3; /* not currently using */
- /* byte 12 */
- unsigned char unused2a; /* not currently using */
- /* byte 13 */
- unsigned char unused2; /* not currently using */
- /* byte 14 */
- unsigned char swack_vec; /* filled in by the hardware */
- /* byte 15 (bits 127:120) */
- unsigned short sequence; /* message sequence number */
- /* bytes 16-17 */
- unsigned char unused4[2]; /* not currently using bytes 18-19 */
- /* bytes 18-19 */
- int number_of_cpus; /* filled in at destination */
- /* 32 bits, bytes 20-23 (aligned) */
- unsigned char unused5[8]; /* not using */
- /* bytes 24-31 */
-};
-
-struct msg_desc {
- struct bau_pq_entry *msg;
- int msg_slot;
- struct bau_pq_entry *queue_first;
- struct bau_pq_entry *queue_last;
-};
-
-struct reset_args {
- int sender;
-};
-
-/*
- * This structure is allocated per_cpu for UV TLB shootdown statistics.
- */
-struct ptc_stats {
- /* sender statistics */
- unsigned long s_giveup; /* number of fall backs to
- IPI-style flushes */
- unsigned long s_requestor; /* number of shootdown
- requests */
- unsigned long s_stimeout; /* source side timeouts */
- unsigned long s_dtimeout; /* destination side timeouts */
- unsigned long s_strongnacks; /* number of strong nack's */
- unsigned long s_time; /* time spent in sending side */
- unsigned long s_retriesok; /* successful retries */
- unsigned long s_ntargcpu; /* total number of cpu's
- targeted */
- unsigned long s_ntargself; /* times the sending cpu was
- targeted */
- unsigned long s_ntarglocals; /* targets of cpus on the local
- blade */
- unsigned long s_ntargremotes; /* targets of cpus on remote
- blades */
- unsigned long s_ntarglocaluvhub; /* targets of the local hub */
- unsigned long s_ntargremoteuvhub; /* remotes hubs targeted */
- unsigned long s_ntarguvhub; /* total number of uvhubs
- targeted */
- unsigned long s_ntarguvhub16; /* number of times target
- hubs >= 16*/
- unsigned long s_ntarguvhub8; /* number of times target
- hubs >= 8 */
- unsigned long s_ntarguvhub4; /* number of times target
- hubs >= 4 */
- unsigned long s_ntarguvhub2; /* number of times target
- hubs >= 2 */
- unsigned long s_ntarguvhub1; /* number of times target
- hubs == 1 */
- unsigned long s_resets_plug; /* ipi-style resets from plug
- state */
- unsigned long s_resets_timeout; /* ipi-style resets from
- timeouts */
- unsigned long s_busy; /* status stayed busy past
- s/w timer */
- unsigned long s_throttles; /* waits in throttle */
- unsigned long s_retry_messages; /* retry broadcasts */
- unsigned long s_bau_reenabled; /* for bau enable/disable */
- unsigned long s_bau_disabled; /* for bau enable/disable */
- unsigned long s_uv2_wars; /* uv2 workaround, perm. busy */
- unsigned long s_uv2_wars_hw; /* uv2 workaround, hiwater */
- unsigned long s_uv2_war_waits; /* uv2 workaround, long waits */
- unsigned long s_overipilimit; /* over the ipi reset limit */
- unsigned long s_giveuplimit; /* disables, over giveup limit*/
- unsigned long s_enters; /* entries to the driver */
- unsigned long s_ipifordisabled; /* fall back to IPI; disabled */
- unsigned long s_plugged; /* plugged by h/w bug*/
- unsigned long s_congested; /* giveup on long wait */
- /* destination statistics */
- unsigned long d_alltlb; /* times all tlb's on this
- cpu were flushed */
- unsigned long d_onetlb; /* times just one tlb on this
- cpu was flushed */
- unsigned long d_multmsg; /* interrupts with multiple
- messages */
- unsigned long d_nomsg; /* interrupts with no message */
- unsigned long d_time; /* time spent on destination
- side */
- unsigned long d_requestee; /* number of messages
- processed */
- unsigned long d_retries; /* number of retry messages
- processed */
- unsigned long d_canceled; /* number of messages canceled
- by retries */
- unsigned long d_nocanceled; /* retries that found nothing
- to cancel */
- unsigned long d_resets; /* number of ipi-style requests
- processed */
- unsigned long d_rcanceled; /* number of messages canceled
- by resets */
-};
-
-struct tunables {
- int *tunp;
- int deflt;
-};
-
-struct hub_and_pnode {
- short uvhub;
- short pnode;
-};
-
-struct socket_desc {
- short num_cpus;
- short cpu_number[MAX_CPUS_PER_SOCKET];
-};
-
-struct uvhub_desc {
- unsigned short socket_mask;
- short num_cpus;
- short uvhub;
- short pnode;
- struct socket_desc socket[2];
-};
-
-/**
- * struct bau_control
- * @status_mmr: location of status mmr, determined by uvhub_cpu
- * @status_index: index of ERR|BUSY bits in status mmr, determined by uvhub_cpu
- *
- * Per-cpu control struct containing CPU topology information and BAU tuneables.
- */
-struct bau_control {
- struct bau_desc *descriptor_base;
- struct bau_pq_entry *queue_first;
- struct bau_pq_entry *queue_last;
- struct bau_pq_entry *bau_msg_head;
- struct bau_control *uvhub_master;
- struct bau_control *socket_master;
- struct ptc_stats *statp;
- cpumask_t *cpumask;
- unsigned long timeout_interval;
- unsigned long set_bau_on_time;
- atomic_t active_descriptor_count;
- int plugged_tries;
- int timeout_tries;
- int ipi_attempts;
- int conseccompletes;
- u64 status_mmr;
- int status_index;
- bool nobau;
- short baudisabled;
- short cpu;
- short osnode;
- short uvhub_cpu;
- short uvhub;
- short uvhub_version;
- short cpus_in_socket;
- short cpus_in_uvhub;
- short partition_base_pnode;
- short busy; /* all were busy (war) */
- unsigned short message_number;
- unsigned short uvhub_quiesce;
- short socket_acknowledge_count[DEST_Q_SIZE];
- cycles_t send_message;
- cycles_t period_end;
- cycles_t period_time;
- spinlock_t uvhub_lock;
- spinlock_t queue_lock;
- spinlock_t disable_lock;
- /* tunables */
- int max_concurr;
- int max_concurr_const;
- int plugged_delay;
- int plugsb4reset;
- int timeoutsb4reset;
- int ipi_reset_limit;
- int complete_threshold;
- int cong_response_us;
- int cong_reps;
- cycles_t disabled_period;
- int period_giveups;
- int giveup_limit;
- long period_requests;
- struct hub_and_pnode *thp;
-};
-
-/* Abstracted BAU functions */
-struct bau_operations {
- unsigned long (*read_l_sw_ack)(void);
- unsigned long (*read_g_sw_ack)(int pnode);
- unsigned long (*bau_gpa_to_offset)(unsigned long vaddr);
- void (*write_l_sw_ack)(unsigned long mmr);
- void (*write_g_sw_ack)(int pnode, unsigned long mmr);
- void (*write_payload_first)(int pnode, unsigned long mmr);
- void (*write_payload_last)(int pnode, unsigned long mmr);
- int (*wait_completion)(struct bau_desc*,
- struct bau_control*, long try);
-};
-
-static inline void write_mmr_data_broadcast(int pnode, unsigned long mmr_image)
-{
- write_gmmr(pnode, UVH_BAU_DATA_BROADCAST, mmr_image);
-}
-
-static inline void write_mmr_descriptor_base(int pnode, unsigned long mmr_image)
-{
- write_gmmr(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE, mmr_image);
-}
-
-static inline void write_mmr_activation(unsigned long index)
-{
- write_lmmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);
-}
-
-static inline void write_gmmr_activation(int pnode, unsigned long mmr_image)
-{
- write_gmmr(pnode, UVH_LB_BAU_SB_ACTIVATION_CONTROL, mmr_image);
-}
-
-static inline void write_mmr_proc_payload_first(int pnode, unsigned long mmr_image)
-{
- write_gmmr(pnode, UV4H_LB_PROC_INTD_QUEUE_FIRST, mmr_image);
-}
-
-static inline void write_mmr_proc_payload_last(int pnode, unsigned long mmr_image)
-{
- write_gmmr(pnode, UV4H_LB_PROC_INTD_QUEUE_LAST, mmr_image);
-}
-
-static inline void write_mmr_payload_first(int pnode, unsigned long mmr_image)
-{
- write_gmmr(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST, mmr_image);
-}
-
-static inline void write_mmr_payload_tail(int pnode, unsigned long mmr_image)
-{
- write_gmmr(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL, mmr_image);
-}
-
-static inline void write_mmr_payload_last(int pnode, unsigned long mmr_image)
-{
- write_gmmr(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST, mmr_image);
-}
-
-static inline void write_mmr_misc_control(int pnode, unsigned long mmr_image)
-{
- write_gmmr(pnode, UVH_LB_BAU_MISC_CONTROL, mmr_image);
-}
-
-static inline unsigned long read_mmr_misc_control(int pnode)
-{
- return read_gmmr(pnode, UVH_LB_BAU_MISC_CONTROL);
-}
-
-static inline void write_mmr_sw_ack(unsigned long mr)
-{
- uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, mr);
-}
-
-static inline void write_gmmr_sw_ack(int pnode, unsigned long mr)
-{
- write_gmmr(pnode, UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, mr);
-}
-
-static inline unsigned long read_mmr_sw_ack(void)
-{
- return read_lmmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE);
-}
-
-static inline unsigned long read_gmmr_sw_ack(int pnode)
-{
- return read_gmmr(pnode, UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE);
-}
-
-static inline void write_mmr_proc_sw_ack(unsigned long mr)
-{
- uv_write_local_mmr(UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR, mr);
-}
-
-static inline void write_gmmr_proc_sw_ack(int pnode, unsigned long mr)
-{
- write_gmmr(pnode, UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR, mr);
-}
-
-static inline unsigned long read_mmr_proc_sw_ack(void)
-{
- return read_lmmr(UV4H_LB_PROC_INTD_SOFT_ACK_PENDING);
-}
-
-static inline unsigned long read_gmmr_proc_sw_ack(int pnode)
-{
- return read_gmmr(pnode, UV4H_LB_PROC_INTD_SOFT_ACK_PENDING);
-}
-
-static inline void write_mmr_data_config(int pnode, unsigned long mr)
-{
- uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG, mr);
-}
-
-static inline int bau_uvhub_isset(int uvhub, struct pnmask *dstp)
-{
- return constant_test_bit(uvhub, &dstp->bits[0]);
-}
-static inline void bau_uvhub_set(int pnode, struct pnmask *dstp)
-{
- __set_bit(pnode, &dstp->bits[0]);
-}
-static inline void bau_uvhubs_clear(struct pnmask *dstp,
- int nbits)
-{
- bitmap_zero(&dstp->bits[0], nbits);
-}
-static inline int bau_uvhub_weight(struct pnmask *dstp)
-{
- return bitmap_weight((unsigned long *)&dstp->bits[0],
- UV_DISTRIBUTION_SIZE);
-}
-
-static inline void bau_cpubits_clear(struct bau_local_cpumask *dstp, int nbits)
-{
- bitmap_zero(&dstp->bits, nbits);
-}
-
-struct atomic_short {
- short counter;
-};
-
-/*
- * atomic_read_short - read a short atomic variable
- * @v: pointer of type atomic_short
- *
- * Atomically reads the value of @v.
- */
-static inline int atomic_read_short(const struct atomic_short *v)
-{
- return v->counter;
-}
-
-/*
- * atom_asr - add and return a short int
- * @i: short value to add
- * @v: pointer of type atomic_short
- *
- * Atomically adds @i to @v and returns @i + @v
- */
-static inline int atom_asr(short i, struct atomic_short *v)
-{
- short __i = i;
- asm volatile(LOCK_PREFIX "xaddw %0, %1"
- : "+r" (i), "+m" (v->counter)
- : : "memory");
- return i + __i;
-}
-
-/*
- * conditionally add 1 to *v, unless *v is >= u
- * return 0 if we cannot add 1 to *v because it is >= u
- * return 1 if we can add 1 to *v because it is < u
- * the add is atomic
- *
- * This is close to atomic_add_unless(), but this allows the 'u' value
- * to be lowered below the current 'v'. atomic_add_unless can only stop
- * on equal.
- */
-static inline int atomic_inc_unless_ge(spinlock_t *lock, atomic_t *v, int u)
-{
- spin_lock(lock);
- if (atomic_read(v) >= u) {
- spin_unlock(lock);
- return 0;
- }
- atomic_inc(v);
- spin_unlock(lock);
- return 1;
-}
-
-void uv_bau_message_interrupt(struct pt_regs *regs);
-
-#endif /* _ASM_X86_UV_UV_BAU_H */
*
* SGI UV architectural definitions
*
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
* Copyright (C) 2007-2014 Silicon Graphics, Inc. All rights reserved.
*/
*/
#define UV_MAX_NASID_VALUE (UV_MAX_NUMALINK_BLADES * 2)
-/* System Controller Interface Reg info */
-struct uv_scir_s {
- struct timer_list timer;
- unsigned long offset;
- unsigned long last;
- unsigned long idle_on;
- unsigned long idle_off;
- unsigned char state;
- unsigned char enabled;
-};
-
/* GAM (globally addressed memory) range table */
struct uv_gam_range_s {
u32 limit; /* PA bits 56:26 (GAM_RANGE_SHFT) */
* available in the L3 cache on the cpu socket for the node.
*/
struct uv_hub_info_s {
+ unsigned int hub_type;
+ unsigned char hub_revision;
unsigned long global_mmr_base;
unsigned long global_mmr_shift;
unsigned long gpa_mask;
unsigned char m_val;
unsigned char n_val;
unsigned char gr_table_len;
- unsigned char hub_revision;
unsigned char apic_pnode_shift;
unsigned char gpa_shift;
+ unsigned char nasid_shift;
unsigned char m_shift;
unsigned char n_lshift;
unsigned int gnode_extra;
struct uv_cpu_info_s {
void *p_uv_hub_info;
unsigned char blade_cpu_id;
- struct uv_scir_s scir;
+ void *reserved;
};
DECLARE_PER_CPU(struct uv_cpu_info_s, __uv_cpu_info);
#define uv_cpu_info this_cpu_ptr(&__uv_cpu_info)
#define uv_cpu_info_per(cpu) (&per_cpu(__uv_cpu_info, cpu))
-#define uv_scir_info (&uv_cpu_info->scir)
-#define uv_cpu_scir_info(cpu) (&uv_cpu_info_per(cpu)->scir)
-
/* Node specific hub common info struct */
extern void **__uv_hub_info_list;
static inline struct uv_hub_info_s *uv_hub_info_list(int node)
return (struct uv_hub_info_s *)uv_cpu_info_per(cpu)->p_uv_hub_info;
}
+static inline int uv_hub_type(void)
+{
+ return uv_hub_info->hub_type;
+}
+
+static inline __init void uv_hub_type_set(int uvmask)
+{
+ uv_hub_info->hub_type = uvmask;
+}
+
+
/*
* HUB revision ranges for each UV HUB architecture.
* This is a software convention - NOT the hardware revision numbers in
#define UV3_HUB_REVISION_BASE 5
#define UV4_HUB_REVISION_BASE 7
#define UV4A_HUB_REVISION_BASE 8 /* UV4 (fixed) rev 2 */
+#define UV5_HUB_REVISION_BASE 9
-static inline int is_uv2_hub(void)
-{
- return is_uv_hubbed(uv(2));
-}
-
-static inline int is_uv3_hub(void)
-{
- return is_uv_hubbed(uv(3));
-}
+static inline int is_uv(int uvmask) { return uv_hub_type() & uvmask; }
+static inline int is_uv1_hub(void) { return 0; }
+static inline int is_uv2_hub(void) { return is_uv(UV2); }
+static inline int is_uv3_hub(void) { return is_uv(UV3); }
+static inline int is_uv4a_hub(void) { return is_uv(UV4A); }
+static inline int is_uv4_hub(void) { return is_uv(UV4); }
+static inline int is_uv5_hub(void) { return is_uv(UV5); }
-/* First test "is UV4A", then "is UV4" */
-static inline int is_uv4a_hub(void)
-{
- if (is_uv_hubbed(uv(4)))
- return (uv_hub_info->hub_revision == UV4A_HUB_REVISION_BASE);
- return 0;
-}
+/*
+ * UV4A is a revision of UV4. So on UV4A, both is_uv4_hub() and
+ * is_uv4a_hub() return true, While on UV4, only is_uv4_hub()
+ * returns true. So to get true results, first test if is UV4A,
+ * then test if is UV4.
+ */
-static inline int is_uv4_hub(void)
-{
- return is_uv_hubbed(uv(4));
-}
+/* UVX class: UV2,3,4 */
+static inline int is_uvx_hub(void) { return is_uv(UVX); }
-static inline int is_uvx_hub(void)
-{
- return (is_uv_hubbed(-2) >= uv(2));
-}
+/* UVY class: UV5,..? */
+static inline int is_uvy_hub(void) { return is_uv(UVY); }
-static inline int is_uv_hub(void)
-{
- return is_uvx_hub();
-}
+/* Any UV Hubbed System */
+static inline int is_uv_hub(void) { return is_uv(UV_ANY); }
union uvh_apicid {
unsigned long v;
* g - GNODE (full 15-bit global nasid, right shifted 1)
* p - PNODE (local part of nsids, right shifted 1)
*/
-#define UV_NASID_TO_PNODE(n) (((n) >> 1) & uv_hub_info->pnode_mask)
+#define UV_NASID_TO_PNODE(n) \
+ (((n) >> uv_hub_info->nasid_shift) & uv_hub_info->pnode_mask)
#define UV_PNODE_TO_GNODE(p) ((p) |uv_hub_info->gnode_extra)
-#define UV_PNODE_TO_NASID(p) (UV_PNODE_TO_GNODE(p) << 1)
+#define UV_PNODE_TO_NASID(p) \
+ (UV_PNODE_TO_GNODE(p) << uv_hub_info->nasid_shift)
#define UV2_LOCAL_MMR_BASE 0xfa000000UL
#define UV2_GLOBAL_MMR32_BASE 0xfc000000UL
#define UV3_GLOBAL_MMR32_SIZE (32UL * 1024 * 1024)
#define UV4_LOCAL_MMR_BASE 0xfa000000UL
-#define UV4_GLOBAL_MMR32_BASE 0xfc000000UL
+#define UV4_GLOBAL_MMR32_BASE 0
#define UV4_LOCAL_MMR_SIZE (32UL * 1024 * 1024)
-#define UV4_GLOBAL_MMR32_SIZE (16UL * 1024 * 1024)
+#define UV4_GLOBAL_MMR32_SIZE 0
+
+#define UV5_LOCAL_MMR_BASE 0xfa000000UL
+#define UV5_GLOBAL_MMR32_BASE 0
+#define UV5_LOCAL_MMR_SIZE (32UL * 1024 * 1024)
+#define UV5_GLOBAL_MMR32_SIZE 0
#define UV_LOCAL_MMR_BASE ( \
- is_uv2_hub() ? UV2_LOCAL_MMR_BASE : \
- is_uv3_hub() ? UV3_LOCAL_MMR_BASE : \
- /*is_uv4_hub*/ UV4_LOCAL_MMR_BASE)
+ is_uv(UV2) ? UV2_LOCAL_MMR_BASE : \
+ is_uv(UV3) ? UV3_LOCAL_MMR_BASE : \
+ is_uv(UV4) ? UV4_LOCAL_MMR_BASE : \
+ is_uv(UV5) ? UV5_LOCAL_MMR_BASE : \
+ 0)
#define UV_GLOBAL_MMR32_BASE ( \
- is_uv2_hub() ? UV2_GLOBAL_MMR32_BASE : \
- is_uv3_hub() ? UV3_GLOBAL_MMR32_BASE : \
- /*is_uv4_hub*/ UV4_GLOBAL_MMR32_BASE)
+ is_uv(UV2) ? UV2_GLOBAL_MMR32_BASE : \
+ is_uv(UV3) ? UV3_GLOBAL_MMR32_BASE : \
+ is_uv(UV4) ? UV4_GLOBAL_MMR32_BASE : \
+ is_uv(UV5) ? UV5_GLOBAL_MMR32_BASE : \
+ 0)
#define UV_LOCAL_MMR_SIZE ( \
- is_uv2_hub() ? UV2_LOCAL_MMR_SIZE : \
- is_uv3_hub() ? UV3_LOCAL_MMR_SIZE : \
- /*is_uv4_hub*/ UV4_LOCAL_MMR_SIZE)
+ is_uv(UV2) ? UV2_LOCAL_MMR_SIZE : \
+ is_uv(UV3) ? UV3_LOCAL_MMR_SIZE : \
+ is_uv(UV4) ? UV4_LOCAL_MMR_SIZE : \
+ is_uv(UV5) ? UV5_LOCAL_MMR_SIZE : \
+ 0)
#define UV_GLOBAL_MMR32_SIZE ( \
- is_uv2_hub() ? UV2_GLOBAL_MMR32_SIZE : \
- is_uv3_hub() ? UV3_GLOBAL_MMR32_SIZE : \
- /*is_uv4_hub*/ UV4_GLOBAL_MMR32_SIZE)
+ is_uv(UV2) ? UV2_GLOBAL_MMR32_SIZE : \
+ is_uv(UV3) ? UV3_GLOBAL_MMR32_SIZE : \
+ is_uv(UV4) ? UV4_GLOBAL_MMR32_SIZE : \
+ is_uv(UV5) ? UV5_GLOBAL_MMR32_SIZE : \
+ 0)
#define UV_GLOBAL_MMR64_BASE (uv_hub_info->global_mmr_base)
#define UVH_TSC_SYNC_SHIFT_UV2K 16 /* UV2/3k have different bits */
#define UVH_TSC_SYNC_MASK 3 /* 0011 */
#define UVH_TSC_SYNC_VALID 3 /* 0011 */
-#define UVH_TSC_SYNC_INVALID 2 /* 0010 */
+#define UVH_TSC_SYNC_UNKNOWN 0 /* 0000 */
/* BMC sets a bit this MMR non-zero before sending an NMI */
#define UVH_NMI_MMR UVH_BIOS_KERNEL_MMR
#define UVH_NMI_MMR_SHIFT 63
#define UVH_NMI_MMR_TYPE "SCRATCH5"
-/* Newer SMM NMI handler, not present in all systems */
-#define UVH_NMI_MMRX UVH_EVENT_OCCURRED0
-#define UVH_NMI_MMRX_CLEAR UVH_EVENT_OCCURRED0_ALIAS
-#define UVH_NMI_MMRX_SHIFT UVH_EVENT_OCCURRED0_EXTIO_INT0_SHFT
-#define UVH_NMI_MMRX_TYPE "EXTIO_INT0"
-
-/* Non-zero indicates newer SMM NMI handler present */
-#define UVH_NMI_MMRX_SUPPORTED UVH_EXTIO_INT0_BROADCAST
-
-/* Indicates to BIOS that we want to use the newer SMM NMI handler */
-#define UVH_NMI_MMRX_REQ UVH_BIOS_KERNEL_MMR_ALIAS_2
-#define UVH_NMI_MMRX_REQ_SHIFT 62
-
struct uv_hub_nmi_s {
raw_spinlock_t nmi_lock;
atomic_t in_nmi; /* flag this node in UV NMI IRQ */
#define UV_NMI_STATE_DUMP 2
#define UV_NMI_STATE_DUMP_DONE 3
-/* Update SCIR state */
-static inline void uv_set_scir_bits(unsigned char value)
-{
- if (uv_scir_info->state != value) {
- uv_scir_info->state = value;
- uv_write_local_mmr8(uv_scir_info->offset, value);
- }
-}
-
-static inline unsigned long uv_scir_offset(int apicid)
-{
- return SCIR_LOCAL_MMR_BASE | (apicid & 0x3f);
-}
-
-static inline void uv_set_cpu_scir_bits(int cpu, unsigned char value)
-{
- if (uv_cpu_scir_info(cpu)->state != value) {
- uv_write_global_mmr8(uv_cpu_to_pnode(cpu),
- uv_cpu_scir_info(cpu)->offset, value);
- uv_cpu_scir_info(cpu)->state = value;
- }
-}
-
/*
* Get the minimum revision number of the hub chips within the partition.
* (See UVx_HUB_REVISION_BASE above for specific values.)
* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
- * SGI UV MMR definitions
+ * HPE UV MMR definitions
*
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
* Copyright (C) 2007-2016 Silicon Graphics, Inc. All rights reserved.
*/
* grouped by architecture types.
*
* UVH - definitions common to all UV hub types.
- * UVXH - definitions common to all UV eXtended hub types (currently 2, 3, 4).
- * UV2H - definitions specific to UV type 2 hub.
- * UV3H - definitions specific to UV type 3 hub.
+ * UVXH - definitions common to UVX class (2, 3, 4).
+ * UVYH - definitions common to UVY class (5).
+ * UV5H - definitions specific to UV type 5 hub.
+ * UV4AH - definitions specific to UV type 4A hub.
* UV4H - definitions specific to UV type 4 hub.
- *
- * So in general, MMR addresses and structures are identical on all hubs types.
- * These MMRs are identified as:
- * #define UVH_xxx <address>
- * union uvh_xxx {
- * unsigned long v;
- * struct uvh_int_cmpd_s {
- * } s;
- * };
+ * UV3H - definitions specific to UV type 3 hub.
+ * UV2H - definitions specific to UV type 2 hub.
*
* If the MMR exists on all hub types but have different addresses,
- * use a conditional operator to define the value at runtime.
- * #define UV2Hxxx b
- * #define UV3Hxxx c
- * #define UV4Hxxx d
- * #define UV4AHxxx e
- * #define UVHxxx (is_uv2_hub() ? UV2Hxxx :
- * (is_uv3_hub() ? UV3Hxxx :
- * (is_uv4a_hub() ? UV4AHxxx :
- * UV4Hxxx))
+ * use a conditional operator to define the value at runtime. Any
+ * that are not defined are blank.
+ * (UV4A variations only generated if different from uv4)
+ * #define UVHxxx (
+ * is_uv(UV5) ? UV5Hxxx value :
+ * is_uv(UV4A) ? UV4AHxxx value :
+ * is_uv(UV4) ? UV4Hxxx value :
+ * is_uv(UV3) ? UV3Hxxx value :
+ * is_uv(UV2) ? UV2Hxxx value :
+ * <ucv> or <undef value>)
+ *
+ * Class UVX has UVs (2|3|4|4A).
+ * Class UVY has UVs (5).
*
* union uvh_xxx {
* unsigned long v;
* struct uvh_xxx_s { # Common fields only
* } s;
- * struct uv2h_xxx_s { # Full UV2 definition (*)
- * } s2;
- * struct uv3h_xxx_s { # Full UV3 definition (*)
- * } s3;
- * (NOTE: No struct uv4ah_xxx_s members exist)
+ * struct uv5h_xxx_s { # Full UV5 definition (*)
+ * } s5;
+ * struct uv4ah_xxx_s { # Full UV4A definition (*)
+ * } s4a;
* struct uv4h_xxx_s { # Full UV4 definition (*)
* } s4;
+ * struct uv3h_xxx_s { # Full UV3 definition (*)
+ * } s3;
+ * struct uv2h_xxx_s { # Full UV2 definition (*)
+ * } s2;
* };
* (* - if present and different than the common struct)
*
* if the contents is the same for all hubs, only the "s" structure is
* generated.
*
- * If the MMR exists on ONLY 1 type of hub, no generic definition is
- * generated:
- * #define UVnH_xxx <uvn address>
- * union uvnh_xxx {
- * unsigned long v;
- * struct uvh_int_cmpd_s {
- * } sn;
- * };
- *
- * (GEN Flags: mflags_opt= undefs=function UV234=UVXH)
+ * (GEN Flags: undefs=function)
*/
+ /* UV bit masks */
+#define UV2 (1 << 0)
+#define UV3 (1 << 1)
+#define UV4 (1 << 2)
+#define UV4A (1 << 3)
+#define UV5 (1 << 4)
+#define UVX (UV2|UV3|UV4)
+#define UVY (UV5)
+#define UV_ANY (~0)
+
+
+
+
#define UV_MMR_ENABLE (1UL << 63)
+#define UV1_HUB_PART_NUMBER 0x88a5
#define UV2_HUB_PART_NUMBER 0x8eb8
#define UV2_HUB_PART_NUMBER_X 0x1111
#define UV3_HUB_PART_NUMBER 0x9578
#define UV3_HUB_PART_NUMBER_X 0x4321
#define UV4_HUB_PART_NUMBER 0x99a1
+#define UV5_HUB_PART_NUMBER 0xa171
/* Error function to catch undefined references */
extern unsigned long uv_undefined(char *str);
-/* ========================================================================= */
-/* UVH_BAU_DATA_BROADCAST */
-/* ========================================================================= */
-#define UVH_BAU_DATA_BROADCAST 0x61688UL
-
-#define UV2H_BAU_DATA_BROADCAST_32 0x440
-#define UV3H_BAU_DATA_BROADCAST_32 0x440
-#define UV4H_BAU_DATA_BROADCAST_32 0x360
-#define UVH_BAU_DATA_BROADCAST_32 ( \
- is_uv2_hub() ? UV2H_BAU_DATA_BROADCAST_32 : \
- is_uv3_hub() ? UV3H_BAU_DATA_BROADCAST_32 : \
- /*is_uv4_hub*/ UV4H_BAU_DATA_BROADCAST_32)
-
-#define UVH_BAU_DATA_BROADCAST_ENABLE_SHFT 0
-#define UVH_BAU_DATA_BROADCAST_ENABLE_MASK 0x0000000000000001UL
-
-
-union uvh_bau_data_broadcast_u {
- unsigned long v;
- struct uvh_bau_data_broadcast_s {
- unsigned long enable:1; /* RW */
- unsigned long rsvd_1_63:63;
- } s;
-};
-
-/* ========================================================================= */
-/* UVH_BAU_DATA_CONFIG */
-/* ========================================================================= */
-#define UVH_BAU_DATA_CONFIG 0x61680UL
-
-#define UV2H_BAU_DATA_CONFIG_32 0x438
-#define UV3H_BAU_DATA_CONFIG_32 0x438
-#define UV4H_BAU_DATA_CONFIG_32 0x358
-#define UVH_BAU_DATA_CONFIG_32 ( \
- is_uv2_hub() ? UV2H_BAU_DATA_CONFIG_32 : \
- is_uv3_hub() ? UV3H_BAU_DATA_CONFIG_32 : \
- /*is_uv4_hub*/ UV4H_BAU_DATA_CONFIG_32)
-
-#define UVH_BAU_DATA_CONFIG_VECTOR_SHFT 0
-#define UVH_BAU_DATA_CONFIG_DM_SHFT 8
-#define UVH_BAU_DATA_CONFIG_DESTMODE_SHFT 11
-#define UVH_BAU_DATA_CONFIG_STATUS_SHFT 12
-#define UVH_BAU_DATA_CONFIG_P_SHFT 13
-#define UVH_BAU_DATA_CONFIG_T_SHFT 15
-#define UVH_BAU_DATA_CONFIG_M_SHFT 16
-#define UVH_BAU_DATA_CONFIG_APIC_ID_SHFT 32
-#define UVH_BAU_DATA_CONFIG_VECTOR_MASK 0x00000000000000ffUL
-#define UVH_BAU_DATA_CONFIG_DM_MASK 0x0000000000000700UL
-#define UVH_BAU_DATA_CONFIG_DESTMODE_MASK 0x0000000000000800UL
-#define UVH_BAU_DATA_CONFIG_STATUS_MASK 0x0000000000001000UL
-#define UVH_BAU_DATA_CONFIG_P_MASK 0x0000000000002000UL
-#define UVH_BAU_DATA_CONFIG_T_MASK 0x0000000000008000UL
-#define UVH_BAU_DATA_CONFIG_M_MASK 0x0000000000010000UL
-#define UVH_BAU_DATA_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
-
-
-union uvh_bau_data_config_u {
- unsigned long v;
- struct uvh_bau_data_config_s {
- unsigned long vector_:8; /* RW */
- unsigned long dm:3; /* RW */
- unsigned long destmode:1; /* RW */
- unsigned long status:1; /* RO */
- unsigned long p:1; /* RO */
- unsigned long rsvd_14:1;
- unsigned long t:1; /* RO */
- unsigned long m:1; /* RW */
- unsigned long rsvd_17_31:15;
- unsigned long apic_id:32; /* RW */
- } s;
-};
-
/* ========================================================================= */
/* UVH_EVENT_OCCURRED0 */
/* ========================================================================= */
#define UVH_EVENT_OCCURRED0 0x70000UL
-#define UVH_EVENT_OCCURRED0_32 0x5e8
+/* UVH common defines*/
#define UVH_EVENT_OCCURRED0_LB_HCERR_SHFT 0
-#define UVH_EVENT_OCCURRED0_RH_AOERR0_SHFT 11
#define UVH_EVENT_OCCURRED0_LB_HCERR_MASK 0x0000000000000001UL
-#define UVH_EVENT_OCCURRED0_RH_AOERR0_MASK 0x0000000000000800UL
+/* UVXH common defines */
#define UVXH_EVENT_OCCURRED0_RH_HCERR_SHFT 2
-#define UVXH_EVENT_OCCURRED0_LH0_HCERR_SHFT 3
-#define UVXH_EVENT_OCCURRED0_LH1_HCERR_SHFT 4
-#define UVXH_EVENT_OCCURRED0_GR0_HCERR_SHFT 5
-#define UVXH_EVENT_OCCURRED0_GR1_HCERR_SHFT 6
-#define UVXH_EVENT_OCCURRED0_NI0_HCERR_SHFT 7
-#define UVXH_EVENT_OCCURRED0_NI1_HCERR_SHFT 8
-#define UVXH_EVENT_OCCURRED0_LB_AOERR0_SHFT 9
-#define UVXH_EVENT_OCCURRED0_LH0_AOERR0_SHFT 12
-#define UVXH_EVENT_OCCURRED0_LH1_AOERR0_SHFT 13
-#define UVXH_EVENT_OCCURRED0_GR0_AOERR0_SHFT 14
-#define UVXH_EVENT_OCCURRED0_GR1_AOERR0_SHFT 15
-#define UVXH_EVENT_OCCURRED0_XB_AOERR0_SHFT 16
#define UVXH_EVENT_OCCURRED0_RH_HCERR_MASK 0x0000000000000004UL
+#define UVXH_EVENT_OCCURRED0_LH0_HCERR_SHFT 3
#define UVXH_EVENT_OCCURRED0_LH0_HCERR_MASK 0x0000000000000008UL
+#define UVXH_EVENT_OCCURRED0_LH1_HCERR_SHFT 4
#define UVXH_EVENT_OCCURRED0_LH1_HCERR_MASK 0x0000000000000010UL
+#define UVXH_EVENT_OCCURRED0_GR0_HCERR_SHFT 5
#define UVXH_EVENT_OCCURRED0_GR0_HCERR_MASK 0x0000000000000020UL
+#define UVXH_EVENT_OCCURRED0_GR1_HCERR_SHFT 6
#define UVXH_EVENT_OCCURRED0_GR1_HCERR_MASK 0x0000000000000040UL
+#define UVXH_EVENT_OCCURRED0_NI0_HCERR_SHFT 7
#define UVXH_EVENT_OCCURRED0_NI0_HCERR_MASK 0x0000000000000080UL
+#define UVXH_EVENT_OCCURRED0_NI1_HCERR_SHFT 8
#define UVXH_EVENT_OCCURRED0_NI1_HCERR_MASK 0x0000000000000100UL
+#define UVXH_EVENT_OCCURRED0_LB_AOERR0_SHFT 9
#define UVXH_EVENT_OCCURRED0_LB_AOERR0_MASK 0x0000000000000200UL
+#define UVXH_EVENT_OCCURRED0_RH_AOERR0_SHFT 11
+#define UVXH_EVENT_OCCURRED0_RH_AOERR0_MASK 0x0000000000000800UL
+#define UVXH_EVENT_OCCURRED0_LH0_AOERR0_SHFT 12
#define UVXH_EVENT_OCCURRED0_LH0_AOERR0_MASK 0x0000000000001000UL
+#define UVXH_EVENT_OCCURRED0_LH1_AOERR0_SHFT 13
#define UVXH_EVENT_OCCURRED0_LH1_AOERR0_MASK 0x0000000000002000UL
+#define UVXH_EVENT_OCCURRED0_GR0_AOERR0_SHFT 14
#define UVXH_EVENT_OCCURRED0_GR0_AOERR0_MASK 0x0000000000004000UL
+#define UVXH_EVENT_OCCURRED0_GR1_AOERR0_SHFT 15
#define UVXH_EVENT_OCCURRED0_GR1_AOERR0_MASK 0x0000000000008000UL
+#define UVXH_EVENT_OCCURRED0_XB_AOERR0_SHFT 16
#define UVXH_EVENT_OCCURRED0_XB_AOERR0_MASK 0x0000000000010000UL
-#define UV2H_EVENT_OCCURRED0_QP_HCERR_SHFT 1
-#define UV2H_EVENT_OCCURRED0_QP_AOERR0_SHFT 10
-#define UV2H_EVENT_OCCURRED0_RT_AOERR0_SHFT 17
-#define UV2H_EVENT_OCCURRED0_NI0_AOERR0_SHFT 18
-#define UV2H_EVENT_OCCURRED0_NI1_AOERR0_SHFT 19
-#define UV2H_EVENT_OCCURRED0_LB_AOERR1_SHFT 20
-#define UV2H_EVENT_OCCURRED0_QP_AOERR1_SHFT 21
-#define UV2H_EVENT_OCCURRED0_RH_AOERR1_SHFT 22
-#define UV2H_EVENT_OCCURRED0_LH0_AOERR1_SHFT 23
-#define UV2H_EVENT_OCCURRED0_LH1_AOERR1_SHFT 24
-#define UV2H_EVENT_OCCURRED0_GR0_AOERR1_SHFT 25
-#define UV2H_EVENT_OCCURRED0_GR1_AOERR1_SHFT 26
-#define UV2H_EVENT_OCCURRED0_XB_AOERR1_SHFT 27
-#define UV2H_EVENT_OCCURRED0_RT_AOERR1_SHFT 28
-#define UV2H_EVENT_OCCURRED0_NI0_AOERR1_SHFT 29
-#define UV2H_EVENT_OCCURRED0_NI1_AOERR1_SHFT 30
-#define UV2H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_SHFT 31
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_0_SHFT 32
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_1_SHFT 33
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_2_SHFT 34
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_3_SHFT 35
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_4_SHFT 36
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_5_SHFT 37
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_6_SHFT 38
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_7_SHFT 39
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_8_SHFT 40
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_9_SHFT 41
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_10_SHFT 42
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_11_SHFT 43
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_12_SHFT 44
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_13_SHFT 45
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_14_SHFT 46
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_15_SHFT 47
-#define UV2H_EVENT_OCCURRED0_L1_NMI_INT_SHFT 48
-#define UV2H_EVENT_OCCURRED0_STOP_CLOCK_SHFT 49
-#define UV2H_EVENT_OCCURRED0_ASIC_TO_L1_SHFT 50
-#define UV2H_EVENT_OCCURRED0_L1_TO_ASIC_SHFT 51
-#define UV2H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_SHFT 52
-#define UV2H_EVENT_OCCURRED0_IPI_INT_SHFT 53
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT0_SHFT 54
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT1_SHFT 55
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT2_SHFT 56
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT3_SHFT 57
-#define UV2H_EVENT_OCCURRED0_PROFILE_INT_SHFT 58
-#define UV2H_EVENT_OCCURRED0_QP_HCERR_MASK 0x0000000000000002UL
-#define UV2H_EVENT_OCCURRED0_QP_AOERR0_MASK 0x0000000000000400UL
-#define UV2H_EVENT_OCCURRED0_RT_AOERR0_MASK 0x0000000000020000UL
-#define UV2H_EVENT_OCCURRED0_NI0_AOERR0_MASK 0x0000000000040000UL
-#define UV2H_EVENT_OCCURRED0_NI1_AOERR0_MASK 0x0000000000080000UL
-#define UV2H_EVENT_OCCURRED0_LB_AOERR1_MASK 0x0000000000100000UL
-#define UV2H_EVENT_OCCURRED0_QP_AOERR1_MASK 0x0000000000200000UL
-#define UV2H_EVENT_OCCURRED0_RH_AOERR1_MASK 0x0000000000400000UL
-#define UV2H_EVENT_OCCURRED0_LH0_AOERR1_MASK 0x0000000000800000UL
-#define UV2H_EVENT_OCCURRED0_LH1_AOERR1_MASK 0x0000000001000000UL
-#define UV2H_EVENT_OCCURRED0_GR0_AOERR1_MASK 0x0000000002000000UL
-#define UV2H_EVENT_OCCURRED0_GR1_AOERR1_MASK 0x0000000004000000UL
-#define UV2H_EVENT_OCCURRED0_XB_AOERR1_MASK 0x0000000008000000UL
-#define UV2H_EVENT_OCCURRED0_RT_AOERR1_MASK 0x0000000010000000UL
-#define UV2H_EVENT_OCCURRED0_NI0_AOERR1_MASK 0x0000000020000000UL
-#define UV2H_EVENT_OCCURRED0_NI1_AOERR1_MASK 0x0000000040000000UL
-#define UV2H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_MASK 0x0000000080000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_0_MASK 0x0000000100000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_1_MASK 0x0000000200000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_2_MASK 0x0000000400000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_3_MASK 0x0000000800000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_4_MASK 0x0000001000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_5_MASK 0x0000002000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_6_MASK 0x0000004000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_7_MASK 0x0000008000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_8_MASK 0x0000010000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_9_MASK 0x0000020000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_10_MASK 0x0000040000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_11_MASK 0x0000080000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_12_MASK 0x0000100000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_13_MASK 0x0000200000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_14_MASK 0x0000400000000000UL
-#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_15_MASK 0x0000800000000000UL
-#define UV2H_EVENT_OCCURRED0_L1_NMI_INT_MASK 0x0001000000000000UL
-#define UV2H_EVENT_OCCURRED0_STOP_CLOCK_MASK 0x0002000000000000UL
-#define UV2H_EVENT_OCCURRED0_ASIC_TO_L1_MASK 0x0004000000000000UL
-#define UV2H_EVENT_OCCURRED0_L1_TO_ASIC_MASK 0x0008000000000000UL
-#define UV2H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_MASK 0x0010000000000000UL
-#define UV2H_EVENT_OCCURRED0_IPI_INT_MASK 0x0020000000000000UL
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT0_MASK 0x0040000000000000UL
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT1_MASK 0x0080000000000000UL
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT2_MASK 0x0100000000000000UL
-#define UV2H_EVENT_OCCURRED0_EXTIO_INT3_MASK 0x0200000000000000UL
-#define UV2H_EVENT_OCCURRED0_PROFILE_INT_MASK 0x0400000000000000UL
-
-#define UV3H_EVENT_OCCURRED0_QP_HCERR_SHFT 1
-#define UV3H_EVENT_OCCURRED0_QP_AOERR0_SHFT 10
-#define UV3H_EVENT_OCCURRED0_RT_AOERR0_SHFT 17
-#define UV3H_EVENT_OCCURRED0_NI0_AOERR0_SHFT 18
-#define UV3H_EVENT_OCCURRED0_NI1_AOERR0_SHFT 19
-#define UV3H_EVENT_OCCURRED0_LB_AOERR1_SHFT 20
-#define UV3H_EVENT_OCCURRED0_QP_AOERR1_SHFT 21
-#define UV3H_EVENT_OCCURRED0_RH_AOERR1_SHFT 22
-#define UV3H_EVENT_OCCURRED0_LH0_AOERR1_SHFT 23
-#define UV3H_EVENT_OCCURRED0_LH1_AOERR1_SHFT 24
-#define UV3H_EVENT_OCCURRED0_GR0_AOERR1_SHFT 25
-#define UV3H_EVENT_OCCURRED0_GR1_AOERR1_SHFT 26
-#define UV3H_EVENT_OCCURRED0_XB_AOERR1_SHFT 27
-#define UV3H_EVENT_OCCURRED0_RT_AOERR1_SHFT 28
-#define UV3H_EVENT_OCCURRED0_NI0_AOERR1_SHFT 29
-#define UV3H_EVENT_OCCURRED0_NI1_AOERR1_SHFT 30
-#define UV3H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_SHFT 31
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_0_SHFT 32
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_1_SHFT 33
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_2_SHFT 34
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_3_SHFT 35
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_4_SHFT 36
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_5_SHFT 37
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_6_SHFT 38
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_7_SHFT 39
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_8_SHFT 40
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_9_SHFT 41
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_10_SHFT 42
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_11_SHFT 43
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_12_SHFT 44
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_13_SHFT 45
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_14_SHFT 46
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_15_SHFT 47
-#define UV3H_EVENT_OCCURRED0_L1_NMI_INT_SHFT 48
-#define UV3H_EVENT_OCCURRED0_STOP_CLOCK_SHFT 49
-#define UV3H_EVENT_OCCURRED0_ASIC_TO_L1_SHFT 50
-#define UV3H_EVENT_OCCURRED0_L1_TO_ASIC_SHFT 51
-#define UV3H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_SHFT 52
-#define UV3H_EVENT_OCCURRED0_IPI_INT_SHFT 53
-#define UV3H_EVENT_OCCURRED0_EXTIO_INT0_SHFT 54
-#define UV3H_EVENT_OCCURRED0_EXTIO_INT1_SHFT 55
-#define UV3H_EVENT_OCCURRED0_EXTIO_INT2_SHFT 56
-#define UV3H_EVENT_OCCURRED0_EXTIO_INT3_SHFT 57
-#define UV3H_EVENT_OCCURRED0_PROFILE_INT_SHFT 58
-#define UV3H_EVENT_OCCURRED0_QP_HCERR_MASK 0x0000000000000002UL
-#define UV3H_EVENT_OCCURRED0_QP_AOERR0_MASK 0x0000000000000400UL
-#define UV3H_EVENT_OCCURRED0_RT_AOERR0_MASK 0x0000000000020000UL
-#define UV3H_EVENT_OCCURRED0_NI0_AOERR0_MASK 0x0000000000040000UL
-#define UV3H_EVENT_OCCURRED0_NI1_AOERR0_MASK 0x0000000000080000UL
-#define UV3H_EVENT_OCCURRED0_LB_AOERR1_MASK 0x0000000000100000UL
-#define UV3H_EVENT_OCCURRED0_QP_AOERR1_MASK 0x0000000000200000UL
-#define UV3H_EVENT_OCCURRED0_RH_AOERR1_MASK 0x0000000000400000UL
-#define UV3H_EVENT_OCCURRED0_LH0_AOERR1_MASK 0x0000000000800000UL
-#define UV3H_EVENT_OCCURRED0_LH1_AOERR1_MASK 0x0000000001000000UL
-#define UV3H_EVENT_OCCURRED0_GR0_AOERR1_MASK 0x0000000002000000UL
-#define UV3H_EVENT_OCCURRED0_GR1_AOERR1_MASK 0x0000000004000000UL
-#define UV3H_EVENT_OCCURRED0_XB_AOERR1_MASK 0x0000000008000000UL
-#define UV3H_EVENT_OCCURRED0_RT_AOERR1_MASK 0x0000000010000000UL
-#define UV3H_EVENT_OCCURRED0_NI0_AOERR1_MASK 0x0000000020000000UL
-#define UV3H_EVENT_OCCURRED0_NI1_AOERR1_MASK 0x0000000040000000UL
-#define UV3H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_MASK 0x0000000080000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_0_MASK 0x0000000100000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_1_MASK 0x0000000200000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_2_MASK 0x0000000400000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_3_MASK 0x0000000800000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_4_MASK 0x0000001000000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_5_MASK 0x0000002000000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_6_MASK 0x0000004000000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_7_MASK 0x0000008000000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_8_MASK 0x0000010000000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_9_MASK 0x0000020000000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_10_MASK 0x0000040000000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_11_MASK 0x0000080000000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_12_MASK 0x0000100000000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_13_MASK 0x0000200000000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_14_MASK 0x0000400000000000UL
-#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_15_MASK 0x0000800000000000UL
-#define UV3H_EVENT_OCCURRED0_L1_NMI_INT_MASK 0x0001000000000000UL
-#define UV3H_EVENT_OCCURRED0_STOP_CLOCK_MASK 0x0002000000000000UL
-#define UV3H_EVENT_OCCURRED0_ASIC_TO_L1_MASK 0x0004000000000000UL
-#define UV3H_EVENT_OCCURRED0_L1_TO_ASIC_MASK 0x0008000000000000UL
-#define UV3H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_MASK 0x0010000000000000UL
-#define UV3H_EVENT_OCCURRED0_IPI_INT_MASK 0x0020000000000000UL
-#define UV3H_EVENT_OCCURRED0_EXTIO_INT0_MASK 0x0040000000000000UL
-#define UV3H_EVENT_OCCURRED0_EXTIO_INT1_MASK 0x0080000000000000UL
-#define UV3H_EVENT_OCCURRED0_EXTIO_INT2_MASK 0x0100000000000000UL
-#define UV3H_EVENT_OCCURRED0_EXTIO_INT3_MASK 0x0200000000000000UL
-#define UV3H_EVENT_OCCURRED0_PROFILE_INT_MASK 0x0400000000000000UL
-
+/* UVYH common defines */
+#define UVYH_EVENT_OCCURRED0_KT_HCERR_SHFT 1
+#define UVYH_EVENT_OCCURRED0_KT_HCERR_MASK 0x0000000000000002UL
+#define UVYH_EVENT_OCCURRED0_RH0_HCERR_SHFT 2
+#define UVYH_EVENT_OCCURRED0_RH0_HCERR_MASK 0x0000000000000004UL
+#define UVYH_EVENT_OCCURRED0_RH1_HCERR_SHFT 3
+#define UVYH_EVENT_OCCURRED0_RH1_HCERR_MASK 0x0000000000000008UL
+#define UVYH_EVENT_OCCURRED0_LH0_HCERR_SHFT 4
+#define UVYH_EVENT_OCCURRED0_LH0_HCERR_MASK 0x0000000000000010UL
+#define UVYH_EVENT_OCCURRED0_LH1_HCERR_SHFT 5
+#define UVYH_EVENT_OCCURRED0_LH1_HCERR_MASK 0x0000000000000020UL
+#define UVYH_EVENT_OCCURRED0_LH2_HCERR_SHFT 6
+#define UVYH_EVENT_OCCURRED0_LH2_HCERR_MASK 0x0000000000000040UL
+#define UVYH_EVENT_OCCURRED0_LH3_HCERR_SHFT 7
+#define UVYH_EVENT_OCCURRED0_LH3_HCERR_MASK 0x0000000000000080UL
+#define UVYH_EVENT_OCCURRED0_XB_HCERR_SHFT 8
+#define UVYH_EVENT_OCCURRED0_XB_HCERR_MASK 0x0000000000000100UL
+#define UVYH_EVENT_OCCURRED0_RDM_HCERR_SHFT 9
+#define UVYH_EVENT_OCCURRED0_RDM_HCERR_MASK 0x0000000000000200UL
+#define UVYH_EVENT_OCCURRED0_NI0_HCERR_SHFT 10
+#define UVYH_EVENT_OCCURRED0_NI0_HCERR_MASK 0x0000000000000400UL
+#define UVYH_EVENT_OCCURRED0_NI1_HCERR_SHFT 11
+#define UVYH_EVENT_OCCURRED0_NI1_HCERR_MASK 0x0000000000000800UL
+#define UVYH_EVENT_OCCURRED0_LB_AOERR0_SHFT 12
+#define UVYH_EVENT_OCCURRED0_LB_AOERR0_MASK 0x0000000000001000UL
+#define UVYH_EVENT_OCCURRED0_KT_AOERR0_SHFT 13
+#define UVYH_EVENT_OCCURRED0_KT_AOERR0_MASK 0x0000000000002000UL
+#define UVYH_EVENT_OCCURRED0_RH0_AOERR0_SHFT 14
+#define UVYH_EVENT_OCCURRED0_RH0_AOERR0_MASK 0x0000000000004000UL
+#define UVYH_EVENT_OCCURRED0_RH1_AOERR0_SHFT 15
+#define UVYH_EVENT_OCCURRED0_RH1_AOERR0_MASK 0x0000000000008000UL
+#define UVYH_EVENT_OCCURRED0_LH0_AOERR0_SHFT 16
+#define UVYH_EVENT_OCCURRED0_LH0_AOERR0_MASK 0x0000000000010000UL
+#define UVYH_EVENT_OCCURRED0_LH1_AOERR0_SHFT 17
+#define UVYH_EVENT_OCCURRED0_LH1_AOERR0_MASK 0x0000000000020000UL
+#define UVYH_EVENT_OCCURRED0_LH2_AOERR0_SHFT 18
+#define UVYH_EVENT_OCCURRED0_LH2_AOERR0_MASK 0x0000000000040000UL
+#define UVYH_EVENT_OCCURRED0_LH3_AOERR0_SHFT 19
+#define UVYH_EVENT_OCCURRED0_LH3_AOERR0_MASK 0x0000000000080000UL
+#define UVYH_EVENT_OCCURRED0_XB_AOERR0_SHFT 20
+#define UVYH_EVENT_OCCURRED0_XB_AOERR0_MASK 0x0000000000100000UL
+#define UVYH_EVENT_OCCURRED0_RDM_AOERR0_SHFT 21
+#define UVYH_EVENT_OCCURRED0_RDM_AOERR0_MASK 0x0000000000200000UL
+#define UVYH_EVENT_OCCURRED0_RT0_AOERR0_SHFT 22
+#define UVYH_EVENT_OCCURRED0_RT0_AOERR0_MASK 0x0000000000400000UL
+#define UVYH_EVENT_OCCURRED0_RT1_AOERR0_SHFT 23
+#define UVYH_EVENT_OCCURRED0_RT1_AOERR0_MASK 0x0000000000800000UL
+#define UVYH_EVENT_OCCURRED0_NI0_AOERR0_SHFT 24
+#define UVYH_EVENT_OCCURRED0_NI0_AOERR0_MASK 0x0000000001000000UL
+#define UVYH_EVENT_OCCURRED0_NI1_AOERR0_SHFT 25
+#define UVYH_EVENT_OCCURRED0_NI1_AOERR0_MASK 0x0000000002000000UL
+#define UVYH_EVENT_OCCURRED0_LB_AOERR1_SHFT 26
+#define UVYH_EVENT_OCCURRED0_LB_AOERR1_MASK 0x0000000004000000UL
+#define UVYH_EVENT_OCCURRED0_KT_AOERR1_SHFT 27
+#define UVYH_EVENT_OCCURRED0_KT_AOERR1_MASK 0x0000000008000000UL
+#define UVYH_EVENT_OCCURRED0_RH0_AOERR1_SHFT 28
+#define UVYH_EVENT_OCCURRED0_RH0_AOERR1_MASK 0x0000000010000000UL
+#define UVYH_EVENT_OCCURRED0_RH1_AOERR1_SHFT 29
+#define UVYH_EVENT_OCCURRED0_RH1_AOERR1_MASK 0x0000000020000000UL
+#define UVYH_EVENT_OCCURRED0_LH0_AOERR1_SHFT 30
+#define UVYH_EVENT_OCCURRED0_LH0_AOERR1_MASK 0x0000000040000000UL
+#define UVYH_EVENT_OCCURRED0_LH1_AOERR1_SHFT 31
+#define UVYH_EVENT_OCCURRED0_LH1_AOERR1_MASK 0x0000000080000000UL
+#define UVYH_EVENT_OCCURRED0_LH2_AOERR1_SHFT 32
+#define UVYH_EVENT_OCCURRED0_LH2_AOERR1_MASK 0x0000000100000000UL
+#define UVYH_EVENT_OCCURRED0_LH3_AOERR1_SHFT 33
+#define UVYH_EVENT_OCCURRED0_LH3_AOERR1_MASK 0x0000000200000000UL
+#define UVYH_EVENT_OCCURRED0_XB_AOERR1_SHFT 34
+#define UVYH_EVENT_OCCURRED0_XB_AOERR1_MASK 0x0000000400000000UL
+#define UVYH_EVENT_OCCURRED0_RDM_AOERR1_SHFT 35
+#define UVYH_EVENT_OCCURRED0_RDM_AOERR1_MASK 0x0000000800000000UL
+#define UVYH_EVENT_OCCURRED0_RT0_AOERR1_SHFT 36
+#define UVYH_EVENT_OCCURRED0_RT0_AOERR1_MASK 0x0000001000000000UL
+#define UVYH_EVENT_OCCURRED0_RT1_AOERR1_SHFT 37
+#define UVYH_EVENT_OCCURRED0_RT1_AOERR1_MASK 0x0000002000000000UL
+#define UVYH_EVENT_OCCURRED0_NI0_AOERR1_SHFT 38
+#define UVYH_EVENT_OCCURRED0_NI0_AOERR1_MASK 0x0000004000000000UL
+#define UVYH_EVENT_OCCURRED0_NI1_AOERR1_SHFT 39
+#define UVYH_EVENT_OCCURRED0_NI1_AOERR1_MASK 0x0000008000000000UL
+#define UVYH_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_SHFT 40
+#define UVYH_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_MASK 0x0000010000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_0_SHFT 41
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_0_MASK 0x0000020000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_1_SHFT 42
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_1_MASK 0x0000040000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_2_SHFT 43
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_2_MASK 0x0000080000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_3_SHFT 44
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_3_MASK 0x0000100000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_4_SHFT 45
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_4_MASK 0x0000200000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_5_SHFT 46
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_5_MASK 0x0000400000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_6_SHFT 47
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_6_MASK 0x0000800000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_7_SHFT 48
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_7_MASK 0x0001000000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_8_SHFT 49
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_8_MASK 0x0002000000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_9_SHFT 50
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_9_MASK 0x0004000000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_10_SHFT 51
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_10_MASK 0x0008000000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_11_SHFT 52
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_11_MASK 0x0010000000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_12_SHFT 53
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_12_MASK 0x0020000000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_13_SHFT 54
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_13_MASK 0x0040000000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_14_SHFT 55
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_14_MASK 0x0080000000000000UL
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_15_SHFT 56
+#define UVYH_EVENT_OCCURRED0_LB_IRQ_INT_15_MASK 0x0100000000000000UL
+#define UVYH_EVENT_OCCURRED0_L1_NMI_INT_SHFT 57
+#define UVYH_EVENT_OCCURRED0_L1_NMI_INT_MASK 0x0200000000000000UL
+#define UVYH_EVENT_OCCURRED0_STOP_CLOCK_SHFT 58
+#define UVYH_EVENT_OCCURRED0_STOP_CLOCK_MASK 0x0400000000000000UL
+#define UVYH_EVENT_OCCURRED0_ASIC_TO_L1_SHFT 59
+#define UVYH_EVENT_OCCURRED0_ASIC_TO_L1_MASK 0x0800000000000000UL
+#define UVYH_EVENT_OCCURRED0_L1_TO_ASIC_SHFT 60
+#define UVYH_EVENT_OCCURRED0_L1_TO_ASIC_MASK 0x1000000000000000UL
+#define UVYH_EVENT_OCCURRED0_LA_SEQ_TRIGGER_SHFT 61
+#define UVYH_EVENT_OCCURRED0_LA_SEQ_TRIGGER_MASK 0x2000000000000000UL
+
+/* UV4 unique defines */
#define UV4H_EVENT_OCCURRED0_KT_HCERR_SHFT 1
+#define UV4H_EVENT_OCCURRED0_KT_HCERR_MASK 0x0000000000000002UL
#define UV4H_EVENT_OCCURRED0_KT_AOERR0_SHFT 10
-#define UV4H_EVENT_OCCURRED0_RTQ0_AOERR0_SHFT 17
-#define UV4H_EVENT_OCCURRED0_RTQ1_AOERR0_SHFT 18
-#define UV4H_EVENT_OCCURRED0_RTQ2_AOERR0_SHFT 19
-#define UV4H_EVENT_OCCURRED0_RTQ3_AOERR0_SHFT 20
-#define UV4H_EVENT_OCCURRED0_NI0_AOERR0_SHFT 21
-#define UV4H_EVENT_OCCURRED0_NI1_AOERR0_SHFT 22
-#define UV4H_EVENT_OCCURRED0_LB_AOERR1_SHFT 23
-#define UV4H_EVENT_OCCURRED0_KT_AOERR1_SHFT 24
-#define UV4H_EVENT_OCCURRED0_RH_AOERR1_SHFT 25
-#define UV4H_EVENT_OCCURRED0_LH0_AOERR1_SHFT 26
-#define UV4H_EVENT_OCCURRED0_LH1_AOERR1_SHFT 27
-#define UV4H_EVENT_OCCURRED0_GR0_AOERR1_SHFT 28
-#define UV4H_EVENT_OCCURRED0_GR1_AOERR1_SHFT 29
-#define UV4H_EVENT_OCCURRED0_XB_AOERR1_SHFT 30
-#define UV4H_EVENT_OCCURRED0_RTQ0_AOERR1_SHFT 31
-#define UV4H_EVENT_OCCURRED0_RTQ1_AOERR1_SHFT 32
-#define UV4H_EVENT_OCCURRED0_RTQ2_AOERR1_SHFT 33
-#define UV4H_EVENT_OCCURRED0_RTQ3_AOERR1_SHFT 34
-#define UV4H_EVENT_OCCURRED0_NI0_AOERR1_SHFT 35
-#define UV4H_EVENT_OCCURRED0_NI1_AOERR1_SHFT 36
-#define UV4H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_SHFT 37
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_0_SHFT 38
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_1_SHFT 39
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_2_SHFT 40
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_3_SHFT 41
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_4_SHFT 42
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_5_SHFT 43
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_6_SHFT 44
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_7_SHFT 45
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_8_SHFT 46
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_9_SHFT 47
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_10_SHFT 48
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_11_SHFT 49
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_12_SHFT 50
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_13_SHFT 51
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_14_SHFT 52
-#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_15_SHFT 53
-#define UV4H_EVENT_OCCURRED0_L1_NMI_INT_SHFT 54
-#define UV4H_EVENT_OCCURRED0_STOP_CLOCK_SHFT 55
-#define UV4H_EVENT_OCCURRED0_ASIC_TO_L1_SHFT 56
-#define UV4H_EVENT_OCCURRED0_L1_TO_ASIC_SHFT 57
-#define UV4H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_SHFT 58
-#define UV4H_EVENT_OCCURRED0_IPI_INT_SHFT 59
-#define UV4H_EVENT_OCCURRED0_EXTIO_INT0_SHFT 60
-#define UV4H_EVENT_OCCURRED0_EXTIO_INT1_SHFT 61
-#define UV4H_EVENT_OCCURRED0_EXTIO_INT2_SHFT 62
-#define UV4H_EVENT_OCCURRED0_EXTIO_INT3_SHFT 63
-#define UV4H_EVENT_OCCURRED0_KT_HCERR_MASK 0x0000000000000002UL
#define UV4H_EVENT_OCCURRED0_KT_AOERR0_MASK 0x0000000000000400UL
+#define UV4H_EVENT_OCCURRED0_RTQ0_AOERR0_SHFT 17
#define UV4H_EVENT_OCCURRED0_RTQ0_AOERR0_MASK 0x0000000000020000UL
+#define UV4H_EVENT_OCCURRED0_RTQ1_AOERR0_SHFT 18
#define UV4H_EVENT_OCCURRED0_RTQ1_AOERR0_MASK 0x0000000000040000UL
+#define UV4H_EVENT_OCCURRED0_RTQ2_AOERR0_SHFT 19
#define UV4H_EVENT_OCCURRED0_RTQ2_AOERR0_MASK 0x0000000000080000UL
+#define UV4H_EVENT_OCCURRED0_RTQ3_AOERR0_SHFT 20
#define UV4H_EVENT_OCCURRED0_RTQ3_AOERR0_MASK 0x0000000000100000UL
+#define UV4H_EVENT_OCCURRED0_NI0_AOERR0_SHFT 21
#define UV4H_EVENT_OCCURRED0_NI0_AOERR0_MASK 0x0000000000200000UL
+#define UV4H_EVENT_OCCURRED0_NI1_AOERR0_SHFT 22
#define UV4H_EVENT_OCCURRED0_NI1_AOERR0_MASK 0x0000000000400000UL
+#define UV4H_EVENT_OCCURRED0_LB_AOERR1_SHFT 23
#define UV4H_EVENT_OCCURRED0_LB_AOERR1_MASK 0x0000000000800000UL
+#define UV4H_EVENT_OCCURRED0_KT_AOERR1_SHFT 24
#define UV4H_EVENT_OCCURRED0_KT_AOERR1_MASK 0x0000000001000000UL
+#define UV4H_EVENT_OCCURRED0_RH_AOERR1_SHFT 25
#define UV4H_EVENT_OCCURRED0_RH_AOERR1_MASK 0x0000000002000000UL
+#define UV4H_EVENT_OCCURRED0_LH0_AOERR1_SHFT 26
#define UV4H_EVENT_OCCURRED0_LH0_AOERR1_MASK 0x0000000004000000UL
+#define UV4H_EVENT_OCCURRED0_LH1_AOERR1_SHFT 27
#define UV4H_EVENT_OCCURRED0_LH1_AOERR1_MASK 0x0000000008000000UL
+#define UV4H_EVENT_OCCURRED0_GR0_AOERR1_SHFT 28
#define UV4H_EVENT_OCCURRED0_GR0_AOERR1_MASK 0x0000000010000000UL
+#define UV4H_EVENT_OCCURRED0_GR1_AOERR1_SHFT 29
#define UV4H_EVENT_OCCURRED0_GR1_AOERR1_MASK 0x0000000020000000UL
+#define UV4H_EVENT_OCCURRED0_XB_AOERR1_SHFT 30
#define UV4H_EVENT_OCCURRED0_XB_AOERR1_MASK 0x0000000040000000UL
+#define UV4H_EVENT_OCCURRED0_RTQ0_AOERR1_SHFT 31
#define UV4H_EVENT_OCCURRED0_RTQ0_AOERR1_MASK 0x0000000080000000UL
+#define UV4H_EVENT_OCCURRED0_RTQ1_AOERR1_SHFT 32
#define UV4H_EVENT_OCCURRED0_RTQ1_AOERR1_MASK 0x0000000100000000UL
+#define UV4H_EVENT_OCCURRED0_RTQ2_AOERR1_SHFT 33
#define UV4H_EVENT_OCCURRED0_RTQ2_AOERR1_MASK 0x0000000200000000UL
+#define UV4H_EVENT_OCCURRED0_RTQ3_AOERR1_SHFT 34
#define UV4H_EVENT_OCCURRED0_RTQ3_AOERR1_MASK 0x0000000400000000UL
+#define UV4H_EVENT_OCCURRED0_NI0_AOERR1_SHFT 35
#define UV4H_EVENT_OCCURRED0_NI0_AOERR1_MASK 0x0000000800000000UL
+#define UV4H_EVENT_OCCURRED0_NI1_AOERR1_SHFT 36
#define UV4H_EVENT_OCCURRED0_NI1_AOERR1_MASK 0x0000001000000000UL
+#define UV4H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_SHFT 37
#define UV4H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_MASK 0x0000002000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_0_SHFT 38
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_0_MASK 0x0000004000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_1_SHFT 39
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_1_MASK 0x0000008000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_2_SHFT 40
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_2_MASK 0x0000010000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_3_SHFT 41
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_3_MASK 0x0000020000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_4_SHFT 42
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_4_MASK 0x0000040000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_5_SHFT 43
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_5_MASK 0x0000080000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_6_SHFT 44
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_6_MASK 0x0000100000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_7_SHFT 45
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_7_MASK 0x0000200000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_8_SHFT 46
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_8_MASK 0x0000400000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_9_SHFT 47
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_9_MASK 0x0000800000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_10_SHFT 48
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_10_MASK 0x0001000000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_11_SHFT 49
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_11_MASK 0x0002000000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_12_SHFT 50
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_12_MASK 0x0004000000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_13_SHFT 51
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_13_MASK 0x0008000000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_14_SHFT 52
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_14_MASK 0x0010000000000000UL
+#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_15_SHFT 53
#define UV4H_EVENT_OCCURRED0_LB_IRQ_INT_15_MASK 0x0020000000000000UL
+#define UV4H_EVENT_OCCURRED0_L1_NMI_INT_SHFT 54
#define UV4H_EVENT_OCCURRED0_L1_NMI_INT_MASK 0x0040000000000000UL
+#define UV4H_EVENT_OCCURRED0_STOP_CLOCK_SHFT 55
#define UV4H_EVENT_OCCURRED0_STOP_CLOCK_MASK 0x0080000000000000UL
+#define UV4H_EVENT_OCCURRED0_ASIC_TO_L1_SHFT 56
#define UV4H_EVENT_OCCURRED0_ASIC_TO_L1_MASK 0x0100000000000000UL
+#define UV4H_EVENT_OCCURRED0_L1_TO_ASIC_SHFT 57
#define UV4H_EVENT_OCCURRED0_L1_TO_ASIC_MASK 0x0200000000000000UL
+#define UV4H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_SHFT 58
#define UV4H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_MASK 0x0400000000000000UL
+#define UV4H_EVENT_OCCURRED0_IPI_INT_SHFT 59
#define UV4H_EVENT_OCCURRED0_IPI_INT_MASK 0x0800000000000000UL
+#define UV4H_EVENT_OCCURRED0_EXTIO_INT0_SHFT 60
#define UV4H_EVENT_OCCURRED0_EXTIO_INT0_MASK 0x1000000000000000UL
+#define UV4H_EVENT_OCCURRED0_EXTIO_INT1_SHFT 61
#define UV4H_EVENT_OCCURRED0_EXTIO_INT1_MASK 0x2000000000000000UL
+#define UV4H_EVENT_OCCURRED0_EXTIO_INT2_SHFT 62
#define UV4H_EVENT_OCCURRED0_EXTIO_INT2_MASK 0x4000000000000000UL
+#define UV4H_EVENT_OCCURRED0_EXTIO_INT3_SHFT 63
#define UV4H_EVENT_OCCURRED0_EXTIO_INT3_MASK 0x8000000000000000UL
-#define UVH_EVENT_OCCURRED0_EXTIO_INT0_SHFT ( \
- is_uv2_hub() ? UV2H_EVENT_OCCURRED0_EXTIO_INT0_SHFT : \
- is_uv3_hub() ? UV3H_EVENT_OCCURRED0_EXTIO_INT0_SHFT : \
- /*is_uv4_hub*/ UV4H_EVENT_OCCURRED0_EXTIO_INT0_SHFT)
+/* UV3 unique defines */
+#define UV3H_EVENT_OCCURRED0_QP_HCERR_SHFT 1
+#define UV3H_EVENT_OCCURRED0_QP_HCERR_MASK 0x0000000000000002UL
+#define UV3H_EVENT_OCCURRED0_QP_AOERR0_SHFT 10
+#define UV3H_EVENT_OCCURRED0_QP_AOERR0_MASK 0x0000000000000400UL
+#define UV3H_EVENT_OCCURRED0_RT_AOERR0_SHFT 17
+#define UV3H_EVENT_OCCURRED0_RT_AOERR0_MASK 0x0000000000020000UL
+#define UV3H_EVENT_OCCURRED0_NI0_AOERR0_SHFT 18
+#define UV3H_EVENT_OCCURRED0_NI0_AOERR0_MASK 0x0000000000040000UL
+#define UV3H_EVENT_OCCURRED0_NI1_AOERR0_SHFT 19
+#define UV3H_EVENT_OCCURRED0_NI1_AOERR0_MASK 0x0000000000080000UL
+#define UV3H_EVENT_OCCURRED0_LB_AOERR1_SHFT 20
+#define UV3H_EVENT_OCCURRED0_LB_AOERR1_MASK 0x0000000000100000UL
+#define UV3H_EVENT_OCCURRED0_QP_AOERR1_SHFT 21
+#define UV3H_EVENT_OCCURRED0_QP_AOERR1_MASK 0x0000000000200000UL
+#define UV3H_EVENT_OCCURRED0_RH_AOERR1_SHFT 22
+#define UV3H_EVENT_OCCURRED0_RH_AOERR1_MASK 0x0000000000400000UL
+#define UV3H_EVENT_OCCURRED0_LH0_AOERR1_SHFT 23
+#define UV3H_EVENT_OCCURRED0_LH0_AOERR1_MASK 0x0000000000800000UL
+#define UV3H_EVENT_OCCURRED0_LH1_AOERR1_SHFT 24
+#define UV3H_EVENT_OCCURRED0_LH1_AOERR1_MASK 0x0000000001000000UL
+#define UV3H_EVENT_OCCURRED0_GR0_AOERR1_SHFT 25
+#define UV3H_EVENT_OCCURRED0_GR0_AOERR1_MASK 0x0000000002000000UL
+#define UV3H_EVENT_OCCURRED0_GR1_AOERR1_SHFT 26
+#define UV3H_EVENT_OCCURRED0_GR1_AOERR1_MASK 0x0000000004000000UL
+#define UV3H_EVENT_OCCURRED0_XB_AOERR1_SHFT 27
+#define UV3H_EVENT_OCCURRED0_XB_AOERR1_MASK 0x0000000008000000UL
+#define UV3H_EVENT_OCCURRED0_RT_AOERR1_SHFT 28
+#define UV3H_EVENT_OCCURRED0_RT_AOERR1_MASK 0x0000000010000000UL
+#define UV3H_EVENT_OCCURRED0_NI0_AOERR1_SHFT 29
+#define UV3H_EVENT_OCCURRED0_NI0_AOERR1_MASK 0x0000000020000000UL
+#define UV3H_EVENT_OCCURRED0_NI1_AOERR1_SHFT 30
+#define UV3H_EVENT_OCCURRED0_NI1_AOERR1_MASK 0x0000000040000000UL
+#define UV3H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_SHFT 31
+#define UV3H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_MASK 0x0000000080000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_0_SHFT 32
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_0_MASK 0x0000000100000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_1_SHFT 33
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_1_MASK 0x0000000200000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_2_SHFT 34
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_2_MASK 0x0000000400000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_3_SHFT 35
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_3_MASK 0x0000000800000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_4_SHFT 36
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_4_MASK 0x0000001000000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_5_SHFT 37
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_5_MASK 0x0000002000000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_6_SHFT 38
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_6_MASK 0x0000004000000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_7_SHFT 39
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_7_MASK 0x0000008000000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_8_SHFT 40
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_8_MASK 0x0000010000000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_9_SHFT 41
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_9_MASK 0x0000020000000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_10_SHFT 42
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_10_MASK 0x0000040000000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_11_SHFT 43
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_11_MASK 0x0000080000000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_12_SHFT 44
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_12_MASK 0x0000100000000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_13_SHFT 45
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_13_MASK 0x0000200000000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_14_SHFT 46
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_14_MASK 0x0000400000000000UL
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_15_SHFT 47
+#define UV3H_EVENT_OCCURRED0_LB_IRQ_INT_15_MASK 0x0000800000000000UL
+#define UV3H_EVENT_OCCURRED0_L1_NMI_INT_SHFT 48
+#define UV3H_EVENT_OCCURRED0_L1_NMI_INT_MASK 0x0001000000000000UL
+#define UV3H_EVENT_OCCURRED0_STOP_CLOCK_SHFT 49
+#define UV3H_EVENT_OCCURRED0_STOP_CLOCK_MASK 0x0002000000000000UL
+#define UV3H_EVENT_OCCURRED0_ASIC_TO_L1_SHFT 50
+#define UV3H_EVENT_OCCURRED0_ASIC_TO_L1_MASK 0x0004000000000000UL
+#define UV3H_EVENT_OCCURRED0_L1_TO_ASIC_SHFT 51
+#define UV3H_EVENT_OCCURRED0_L1_TO_ASIC_MASK 0x0008000000000000UL
+#define UV3H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_SHFT 52
+#define UV3H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_MASK 0x0010000000000000UL
+#define UV3H_EVENT_OCCURRED0_IPI_INT_SHFT 53
+#define UV3H_EVENT_OCCURRED0_IPI_INT_MASK 0x0020000000000000UL
+#define UV3H_EVENT_OCCURRED0_EXTIO_INT0_SHFT 54
+#define UV3H_EVENT_OCCURRED0_EXTIO_INT0_MASK 0x0040000000000000UL
+#define UV3H_EVENT_OCCURRED0_EXTIO_INT1_SHFT 55
+#define UV3H_EVENT_OCCURRED0_EXTIO_INT1_MASK 0x0080000000000000UL
+#define UV3H_EVENT_OCCURRED0_EXTIO_INT2_SHFT 56
+#define UV3H_EVENT_OCCURRED0_EXTIO_INT2_MASK 0x0100000000000000UL
+#define UV3H_EVENT_OCCURRED0_EXTIO_INT3_SHFT 57
+#define UV3H_EVENT_OCCURRED0_EXTIO_INT3_MASK 0x0200000000000000UL
+#define UV3H_EVENT_OCCURRED0_PROFILE_INT_SHFT 58
+#define UV3H_EVENT_OCCURRED0_PROFILE_INT_MASK 0x0400000000000000UL
-union uvh_event_occurred0_u {
- unsigned long v;
- struct uvh_event_occurred0_s {
- unsigned long lb_hcerr:1; /* RW, W1C */
- unsigned long rsvd_1_10:10;
- unsigned long rh_aoerr0:1; /* RW, W1C */
- unsigned long rsvd_12_63:52;
- } s;
- struct uvxh_event_occurred0_s {
- unsigned long lb_hcerr:1; /* RW */
- unsigned long rsvd_1:1;
- unsigned long rh_hcerr:1; /* RW */
- unsigned long lh0_hcerr:1; /* RW */
- unsigned long lh1_hcerr:1; /* RW */
- unsigned long gr0_hcerr:1; /* RW */
- unsigned long gr1_hcerr:1; /* RW */
- unsigned long ni0_hcerr:1; /* RW */
- unsigned long ni1_hcerr:1; /* RW */
- unsigned long lb_aoerr0:1; /* RW */
- unsigned long rsvd_10:1;
- unsigned long rh_aoerr0:1; /* RW */
- unsigned long lh0_aoerr0:1; /* RW */
- unsigned long lh1_aoerr0:1; /* RW */
- unsigned long gr0_aoerr0:1; /* RW */
- unsigned long gr1_aoerr0:1; /* RW */
- unsigned long xb_aoerr0:1; /* RW */
- unsigned long rsvd_17_63:47;
- } sx;
- struct uv4h_event_occurred0_s {
- unsigned long lb_hcerr:1; /* RW */
- unsigned long kt_hcerr:1; /* RW */
- unsigned long rh_hcerr:1; /* RW */
- unsigned long lh0_hcerr:1; /* RW */
- unsigned long lh1_hcerr:1; /* RW */
- unsigned long gr0_hcerr:1; /* RW */
- unsigned long gr1_hcerr:1; /* RW */
- unsigned long ni0_hcerr:1; /* RW */
- unsigned long ni1_hcerr:1; /* RW */
- unsigned long lb_aoerr0:1; /* RW */
- unsigned long kt_aoerr0:1; /* RW */
- unsigned long rh_aoerr0:1; /* RW */
- unsigned long lh0_aoerr0:1; /* RW */
- unsigned long lh1_aoerr0:1; /* RW */
- unsigned long gr0_aoerr0:1; /* RW */
- unsigned long gr1_aoerr0:1; /* RW */
- unsigned long xb_aoerr0:1; /* RW */
- unsigned long rtq0_aoerr0:1; /* RW */
- unsigned long rtq1_aoerr0:1; /* RW */
- unsigned long rtq2_aoerr0:1; /* RW */
- unsigned long rtq3_aoerr0:1; /* RW */
- unsigned long ni0_aoerr0:1; /* RW */
- unsigned long ni1_aoerr0:1; /* RW */
- unsigned long lb_aoerr1:1; /* RW */
- unsigned long kt_aoerr1:1; /* RW */
- unsigned long rh_aoerr1:1; /* RW */
- unsigned long lh0_aoerr1:1; /* RW */
- unsigned long lh1_aoerr1:1; /* RW */
- unsigned long gr0_aoerr1:1; /* RW */
- unsigned long gr1_aoerr1:1; /* RW */
- unsigned long xb_aoerr1:1; /* RW */
- unsigned long rtq0_aoerr1:1; /* RW */
- unsigned long rtq1_aoerr1:1; /* RW */
- unsigned long rtq2_aoerr1:1; /* RW */
- unsigned long rtq3_aoerr1:1; /* RW */
- unsigned long ni0_aoerr1:1; /* RW */
- unsigned long ni1_aoerr1:1; /* RW */
- unsigned long system_shutdown_int:1; /* RW */
- unsigned long lb_irq_int_0:1; /* RW */
- unsigned long lb_irq_int_1:1; /* RW */
- unsigned long lb_irq_int_2:1; /* RW */
- unsigned long lb_irq_int_3:1; /* RW */
- unsigned long lb_irq_int_4:1; /* RW */
- unsigned long lb_irq_int_5:1; /* RW */
- unsigned long lb_irq_int_6:1; /* RW */
- unsigned long lb_irq_int_7:1; /* RW */
+/* UV2 unique defines */
+#define UV2H_EVENT_OCCURRED0_QP_HCERR_SHFT 1
+#define UV2H_EVENT_OCCURRED0_QP_HCERR_MASK 0x0000000000000002UL
+#define UV2H_EVENT_OCCURRED0_QP_AOERR0_SHFT 10
+#define UV2H_EVENT_OCCURRED0_QP_AOERR0_MASK 0x0000000000000400UL
+#define UV2H_EVENT_OCCURRED0_RT_AOERR0_SHFT 17
+#define UV2H_EVENT_OCCURRED0_RT_AOERR0_MASK 0x0000000000020000UL
+#define UV2H_EVENT_OCCURRED0_NI0_AOERR0_SHFT 18
+#define UV2H_EVENT_OCCURRED0_NI0_AOERR0_MASK 0x0000000000040000UL
+#define UV2H_EVENT_OCCURRED0_NI1_AOERR0_SHFT 19
+#define UV2H_EVENT_OCCURRED0_NI1_AOERR0_MASK 0x0000000000080000UL
+#define UV2H_EVENT_OCCURRED0_LB_AOERR1_SHFT 20
+#define UV2H_EVENT_OCCURRED0_LB_AOERR1_MASK 0x0000000000100000UL
+#define UV2H_EVENT_OCCURRED0_QP_AOERR1_SHFT 21
+#define UV2H_EVENT_OCCURRED0_QP_AOERR1_MASK 0x0000000000200000UL
+#define UV2H_EVENT_OCCURRED0_RH_AOERR1_SHFT 22
+#define UV2H_EVENT_OCCURRED0_RH_AOERR1_MASK 0x0000000000400000UL
+#define UV2H_EVENT_OCCURRED0_LH0_AOERR1_SHFT 23
+#define UV2H_EVENT_OCCURRED0_LH0_AOERR1_MASK 0x0000000000800000UL
+#define UV2H_EVENT_OCCURRED0_LH1_AOERR1_SHFT 24
+#define UV2H_EVENT_OCCURRED0_LH1_AOERR1_MASK 0x0000000001000000UL
+#define UV2H_EVENT_OCCURRED0_GR0_AOERR1_SHFT 25
+#define UV2H_EVENT_OCCURRED0_GR0_AOERR1_MASK 0x0000000002000000UL
+#define UV2H_EVENT_OCCURRED0_GR1_AOERR1_SHFT 26
+#define UV2H_EVENT_OCCURRED0_GR1_AOERR1_MASK 0x0000000004000000UL
+#define UV2H_EVENT_OCCURRED0_XB_AOERR1_SHFT 27
+#define UV2H_EVENT_OCCURRED0_XB_AOERR1_MASK 0x0000000008000000UL
+#define UV2H_EVENT_OCCURRED0_RT_AOERR1_SHFT 28
+#define UV2H_EVENT_OCCURRED0_RT_AOERR1_MASK 0x0000000010000000UL
+#define UV2H_EVENT_OCCURRED0_NI0_AOERR1_SHFT 29
+#define UV2H_EVENT_OCCURRED0_NI0_AOERR1_MASK 0x0000000020000000UL
+#define UV2H_EVENT_OCCURRED0_NI1_AOERR1_SHFT 30
+#define UV2H_EVENT_OCCURRED0_NI1_AOERR1_MASK 0x0000000040000000UL
+#define UV2H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_SHFT 31
+#define UV2H_EVENT_OCCURRED0_SYSTEM_SHUTDOWN_INT_MASK 0x0000000080000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_0_SHFT 32
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_0_MASK 0x0000000100000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_1_SHFT 33
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_1_MASK 0x0000000200000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_2_SHFT 34
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_2_MASK 0x0000000400000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_3_SHFT 35
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_3_MASK 0x0000000800000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_4_SHFT 36
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_4_MASK 0x0000001000000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_5_SHFT 37
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_5_MASK 0x0000002000000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_6_SHFT 38
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_6_MASK 0x0000004000000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_7_SHFT 39
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_7_MASK 0x0000008000000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_8_SHFT 40
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_8_MASK 0x0000010000000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_9_SHFT 41
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_9_MASK 0x0000020000000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_10_SHFT 42
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_10_MASK 0x0000040000000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_11_SHFT 43
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_11_MASK 0x0000080000000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_12_SHFT 44
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_12_MASK 0x0000100000000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_13_SHFT 45
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_13_MASK 0x0000200000000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_14_SHFT 46
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_14_MASK 0x0000400000000000UL
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_15_SHFT 47
+#define UV2H_EVENT_OCCURRED0_LB_IRQ_INT_15_MASK 0x0000800000000000UL
+#define UV2H_EVENT_OCCURRED0_L1_NMI_INT_SHFT 48
+#define UV2H_EVENT_OCCURRED0_L1_NMI_INT_MASK 0x0001000000000000UL
+#define UV2H_EVENT_OCCURRED0_STOP_CLOCK_SHFT 49
+#define UV2H_EVENT_OCCURRED0_STOP_CLOCK_MASK 0x0002000000000000UL
+#define UV2H_EVENT_OCCURRED0_ASIC_TO_L1_SHFT 50
+#define UV2H_EVENT_OCCURRED0_ASIC_TO_L1_MASK 0x0004000000000000UL
+#define UV2H_EVENT_OCCURRED0_L1_TO_ASIC_SHFT 51
+#define UV2H_EVENT_OCCURRED0_L1_TO_ASIC_MASK 0x0008000000000000UL
+#define UV2H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_SHFT 52
+#define UV2H_EVENT_OCCURRED0_LA_SEQ_TRIGGER_MASK 0x0010000000000000UL
+#define UV2H_EVENT_OCCURRED0_IPI_INT_SHFT 53
+#define UV2H_EVENT_OCCURRED0_IPI_INT_MASK 0x0020000000000000UL
+#define UV2H_EVENT_OCCURRED0_EXTIO_INT0_SHFT 54
+#define UV2H_EVENT_OCCURRED0_EXTIO_INT0_MASK 0x0040000000000000UL
+#define UV2H_EVENT_OCCURRED0_EXTIO_INT1_SHFT 55
+#define UV2H_EVENT_OCCURRED0_EXTIO_INT1_MASK 0x0080000000000000UL
+#define UV2H_EVENT_OCCURRED0_EXTIO_INT2_SHFT 56
+#define UV2H_EVENT_OCCURRED0_EXTIO_INT2_MASK 0x0100000000000000UL
+#define UV2H_EVENT_OCCURRED0_EXTIO_INT3_SHFT 57
+#define UV2H_EVENT_OCCURRED0_EXTIO_INT3_MASK 0x0200000000000000UL
+#define UV2H_EVENT_OCCURRED0_PROFILE_INT_SHFT 58
+#define UV2H_EVENT_OCCURRED0_PROFILE_INT_MASK 0x0400000000000000UL
+
+#define UVH_EVENT_OCCURRED0_EXTIO_INT0_MASK ( \
+ is_uv(UV4) ? 0x1000000000000000UL : \
+ is_uv(UV3) ? 0x0040000000000000UL : \
+ is_uv(UV2) ? 0x0040000000000000UL : \
+ 0)
+#define UVH_EVENT_OCCURRED0_EXTIO_INT0_SHFT ( \
+ is_uv(UV4) ? 60 : \
+ is_uv(UV3) ? 54 : \
+ is_uv(UV2) ? 54 : \
+ -1)
+
+union uvh_event_occurred0_u {
+ unsigned long v;
+
+ /* UVH common struct */
+ struct uvh_event_occurred0_s {
+ unsigned long lb_hcerr:1; /* RW */
+ unsigned long rsvd_1_63:63;
+ } s;
+
+ /* UVXH common struct */
+ struct uvxh_event_occurred0_s {
+ unsigned long lb_hcerr:1; /* RW */
+ unsigned long rsvd_1:1;
+ unsigned long rh_hcerr:1; /* RW */
+ unsigned long lh0_hcerr:1; /* RW */
+ unsigned long lh1_hcerr:1; /* RW */
+ unsigned long gr0_hcerr:1; /* RW */
+ unsigned long gr1_hcerr:1; /* RW */
+ unsigned long ni0_hcerr:1; /* RW */
+ unsigned long ni1_hcerr:1; /* RW */
+ unsigned long lb_aoerr0:1; /* RW */
+ unsigned long rsvd_10:1;
+ unsigned long rh_aoerr0:1; /* RW */
+ unsigned long lh0_aoerr0:1; /* RW */
+ unsigned long lh1_aoerr0:1; /* RW */
+ unsigned long gr0_aoerr0:1; /* RW */
+ unsigned long gr1_aoerr0:1; /* RW */
+ unsigned long xb_aoerr0:1; /* RW */
+ unsigned long rsvd_17_63:47;
+ } sx;
+
+ /* UVYH common struct */
+ struct uvyh_event_occurred0_s {
+ unsigned long lb_hcerr:1; /* RW */
+ unsigned long kt_hcerr:1; /* RW */
+ unsigned long rh0_hcerr:1; /* RW */
+ unsigned long rh1_hcerr:1; /* RW */
+ unsigned long lh0_hcerr:1; /* RW */
+ unsigned long lh1_hcerr:1; /* RW */
+ unsigned long lh2_hcerr:1; /* RW */
+ unsigned long lh3_hcerr:1; /* RW */
+ unsigned long xb_hcerr:1; /* RW */
+ unsigned long rdm_hcerr:1; /* RW */
+ unsigned long ni0_hcerr:1; /* RW */
+ unsigned long ni1_hcerr:1; /* RW */
+ unsigned long lb_aoerr0:1; /* RW */
+ unsigned long kt_aoerr0:1; /* RW */
+ unsigned long rh0_aoerr0:1; /* RW */
+ unsigned long rh1_aoerr0:1; /* RW */
+ unsigned long lh0_aoerr0:1; /* RW */
+ unsigned long lh1_aoerr0:1; /* RW */
+ unsigned long lh2_aoerr0:1; /* RW */
+ unsigned long lh3_aoerr0:1; /* RW */
+ unsigned long xb_aoerr0:1; /* RW */
+ unsigned long rdm_aoerr0:1; /* RW */
+ unsigned long rt0_aoerr0:1; /* RW */
+ unsigned long rt1_aoerr0:1; /* RW */
+ unsigned long ni0_aoerr0:1; /* RW */
+ unsigned long ni1_aoerr0:1; /* RW */
+ unsigned long lb_aoerr1:1; /* RW */
+ unsigned long kt_aoerr1:1; /* RW */
+ unsigned long rh0_aoerr1:1; /* RW */
+ unsigned long rh1_aoerr1:1; /* RW */
+ unsigned long lh0_aoerr1:1; /* RW */
+ unsigned long lh1_aoerr1:1; /* RW */
+ unsigned long lh2_aoerr1:1; /* RW */
+ unsigned long lh3_aoerr1:1; /* RW */
+ unsigned long xb_aoerr1:1; /* RW */
+ unsigned long rdm_aoerr1:1; /* RW */
+ unsigned long rt0_aoerr1:1; /* RW */
+ unsigned long rt1_aoerr1:1; /* RW */
+ unsigned long ni0_aoerr1:1; /* RW */
+ unsigned long ni1_aoerr1:1; /* RW */
+ unsigned long system_shutdown_int:1; /* RW */
+ unsigned long lb_irq_int_0:1; /* RW */
+ unsigned long lb_irq_int_1:1; /* RW */
+ unsigned long lb_irq_int_2:1; /* RW */
+ unsigned long lb_irq_int_3:1; /* RW */
+ unsigned long lb_irq_int_4:1; /* RW */
+ unsigned long lb_irq_int_5:1; /* RW */
+ unsigned long lb_irq_int_6:1; /* RW */
+ unsigned long lb_irq_int_7:1; /* RW */
+ unsigned long lb_irq_int_8:1; /* RW */
+ unsigned long lb_irq_int_9:1; /* RW */
+ unsigned long lb_irq_int_10:1; /* RW */
+ unsigned long lb_irq_int_11:1; /* RW */
+ unsigned long lb_irq_int_12:1; /* RW */
+ unsigned long lb_irq_int_13:1; /* RW */
+ unsigned long lb_irq_int_14:1; /* RW */
+ unsigned long lb_irq_int_15:1; /* RW */
+ unsigned long l1_nmi_int:1; /* RW */
+ unsigned long stop_clock:1; /* RW */
+ unsigned long asic_to_l1:1; /* RW */
+ unsigned long l1_to_asic:1; /* RW */
+ unsigned long la_seq_trigger:1; /* RW */
+ unsigned long rsvd_62_63:2;
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_event_occurred0_s {
+ unsigned long lb_hcerr:1; /* RW */
+ unsigned long kt_hcerr:1; /* RW */
+ unsigned long rh0_hcerr:1; /* RW */
+ unsigned long rh1_hcerr:1; /* RW */
+ unsigned long lh0_hcerr:1; /* RW */
+ unsigned long lh1_hcerr:1; /* RW */
+ unsigned long lh2_hcerr:1; /* RW */
+ unsigned long lh3_hcerr:1; /* RW */
+ unsigned long xb_hcerr:1; /* RW */
+ unsigned long rdm_hcerr:1; /* RW */
+ unsigned long ni0_hcerr:1; /* RW */
+ unsigned long ni1_hcerr:1; /* RW */
+ unsigned long lb_aoerr0:1; /* RW */
+ unsigned long kt_aoerr0:1; /* RW */
+ unsigned long rh0_aoerr0:1; /* RW */
+ unsigned long rh1_aoerr0:1; /* RW */
+ unsigned long lh0_aoerr0:1; /* RW */
+ unsigned long lh1_aoerr0:1; /* RW */
+ unsigned long lh2_aoerr0:1; /* RW */
+ unsigned long lh3_aoerr0:1; /* RW */
+ unsigned long xb_aoerr0:1; /* RW */
+ unsigned long rdm_aoerr0:1; /* RW */
+ unsigned long rt0_aoerr0:1; /* RW */
+ unsigned long rt1_aoerr0:1; /* RW */
+ unsigned long ni0_aoerr0:1; /* RW */
+ unsigned long ni1_aoerr0:1; /* RW */
+ unsigned long lb_aoerr1:1; /* RW */
+ unsigned long kt_aoerr1:1; /* RW */
+ unsigned long rh0_aoerr1:1; /* RW */
+ unsigned long rh1_aoerr1:1; /* RW */
+ unsigned long lh0_aoerr1:1; /* RW */
+ unsigned long lh1_aoerr1:1; /* RW */
+ unsigned long lh2_aoerr1:1; /* RW */
+ unsigned long lh3_aoerr1:1; /* RW */
+ unsigned long xb_aoerr1:1; /* RW */
+ unsigned long rdm_aoerr1:1; /* RW */
+ unsigned long rt0_aoerr1:1; /* RW */
+ unsigned long rt1_aoerr1:1; /* RW */
+ unsigned long ni0_aoerr1:1; /* RW */
+ unsigned long ni1_aoerr1:1; /* RW */
+ unsigned long system_shutdown_int:1; /* RW */
+ unsigned long lb_irq_int_0:1; /* RW */
+ unsigned long lb_irq_int_1:1; /* RW */
+ unsigned long lb_irq_int_2:1; /* RW */
+ unsigned long lb_irq_int_3:1; /* RW */
+ unsigned long lb_irq_int_4:1; /* RW */
+ unsigned long lb_irq_int_5:1; /* RW */
+ unsigned long lb_irq_int_6:1; /* RW */
+ unsigned long lb_irq_int_7:1; /* RW */
+ unsigned long lb_irq_int_8:1; /* RW */
+ unsigned long lb_irq_int_9:1; /* RW */
+ unsigned long lb_irq_int_10:1; /* RW */
+ unsigned long lb_irq_int_11:1; /* RW */
+ unsigned long lb_irq_int_12:1; /* RW */
+ unsigned long lb_irq_int_13:1; /* RW */
+ unsigned long lb_irq_int_14:1; /* RW */
+ unsigned long lb_irq_int_15:1; /* RW */
+ unsigned long l1_nmi_int:1; /* RW */
+ unsigned long stop_clock:1; /* RW */
+ unsigned long asic_to_l1:1; /* RW */
+ unsigned long l1_to_asic:1; /* RW */
+ unsigned long la_seq_trigger:1; /* RW */
+ unsigned long rsvd_62_63:2;
+ } s5;
+
+ /* UV4 unique struct */
+ struct uv4h_event_occurred0_s {
+ unsigned long lb_hcerr:1; /* RW */
+ unsigned long kt_hcerr:1; /* RW */
+ unsigned long rh_hcerr:1; /* RW */
+ unsigned long lh0_hcerr:1; /* RW */
+ unsigned long lh1_hcerr:1; /* RW */
+ unsigned long gr0_hcerr:1; /* RW */
+ unsigned long gr1_hcerr:1; /* RW */
+ unsigned long ni0_hcerr:1; /* RW */
+ unsigned long ni1_hcerr:1; /* RW */
+ unsigned long lb_aoerr0:1; /* RW */
+ unsigned long kt_aoerr0:1; /* RW */
+ unsigned long rh_aoerr0:1; /* RW */
+ unsigned long lh0_aoerr0:1; /* RW */
+ unsigned long lh1_aoerr0:1; /* RW */
+ unsigned long gr0_aoerr0:1; /* RW */
+ unsigned long gr1_aoerr0:1; /* RW */
+ unsigned long xb_aoerr0:1; /* RW */
+ unsigned long rtq0_aoerr0:1; /* RW */
+ unsigned long rtq1_aoerr0:1; /* RW */
+ unsigned long rtq2_aoerr0:1; /* RW */
+ unsigned long rtq3_aoerr0:1; /* RW */
+ unsigned long ni0_aoerr0:1; /* RW */
+ unsigned long ni1_aoerr0:1; /* RW */
+ unsigned long lb_aoerr1:1; /* RW */
+ unsigned long kt_aoerr1:1; /* RW */
+ unsigned long rh_aoerr1:1; /* RW */
+ unsigned long lh0_aoerr1:1; /* RW */
+ unsigned long lh1_aoerr1:1; /* RW */
+ unsigned long gr0_aoerr1:1; /* RW */
+ unsigned long gr1_aoerr1:1; /* RW */
+ unsigned long xb_aoerr1:1; /* RW */
+ unsigned long rtq0_aoerr1:1; /* RW */
+ unsigned long rtq1_aoerr1:1; /* RW */
+ unsigned long rtq2_aoerr1:1; /* RW */
+ unsigned long rtq3_aoerr1:1; /* RW */
+ unsigned long ni0_aoerr1:1; /* RW */
+ unsigned long ni1_aoerr1:1; /* RW */
+ unsigned long system_shutdown_int:1; /* RW */
+ unsigned long lb_irq_int_0:1; /* RW */
+ unsigned long lb_irq_int_1:1; /* RW */
+ unsigned long lb_irq_int_2:1; /* RW */
+ unsigned long lb_irq_int_3:1; /* RW */
+ unsigned long lb_irq_int_4:1; /* RW */
+ unsigned long lb_irq_int_5:1; /* RW */
+ unsigned long lb_irq_int_6:1; /* RW */
+ unsigned long lb_irq_int_7:1; /* RW */
unsigned long lb_irq_int_8:1; /* RW */
unsigned long lb_irq_int_9:1; /* RW */
unsigned long lb_irq_int_10:1; /* RW */
unsigned long extio_int2:1; /* RW */
unsigned long extio_int3:1; /* RW */
} s4;
-};
-
-/* ========================================================================= */
-/* UVH_EVENT_OCCURRED0_ALIAS */
-/* ========================================================================= */
-#define UVH_EVENT_OCCURRED0_ALIAS 0x70008UL
-#define UVH_EVENT_OCCURRED0_ALIAS_32 0x5f0
-
-
-/* ========================================================================= */
-/* UVH_EXTIO_INT0_BROADCAST */
-/* ========================================================================= */
-#define UVH_EXTIO_INT0_BROADCAST 0x61448UL
-
-#define UV2H_EXTIO_INT0_BROADCAST_32 0x3f0
-#define UV3H_EXTIO_INT0_BROADCAST_32 0x3f0
-#define UV4H_EXTIO_INT0_BROADCAST_32 0x310
-#define UVH_EXTIO_INT0_BROADCAST_32 ( \
- is_uv2_hub() ? UV2H_EXTIO_INT0_BROADCAST_32 : \
- is_uv3_hub() ? UV3H_EXTIO_INT0_BROADCAST_32 : \
- /*is_uv4_hub*/ UV4H_EXTIO_INT0_BROADCAST_32)
-
-#define UVH_EXTIO_INT0_BROADCAST_ENABLE_SHFT 0
-#define UVH_EXTIO_INT0_BROADCAST_ENABLE_MASK 0x0000000000000001UL
+ /* UV3 unique struct */
+ struct uv3h_event_occurred0_s {
+ unsigned long lb_hcerr:1; /* RW */
+ unsigned long qp_hcerr:1; /* RW */
+ unsigned long rh_hcerr:1; /* RW */
+ unsigned long lh0_hcerr:1; /* RW */
+ unsigned long lh1_hcerr:1; /* RW */
+ unsigned long gr0_hcerr:1; /* RW */
+ unsigned long gr1_hcerr:1; /* RW */
+ unsigned long ni0_hcerr:1; /* RW */
+ unsigned long ni1_hcerr:1; /* RW */
+ unsigned long lb_aoerr0:1; /* RW */
+ unsigned long qp_aoerr0:1; /* RW */
+ unsigned long rh_aoerr0:1; /* RW */
+ unsigned long lh0_aoerr0:1; /* RW */
+ unsigned long lh1_aoerr0:1; /* RW */
+ unsigned long gr0_aoerr0:1; /* RW */
+ unsigned long gr1_aoerr0:1; /* RW */
+ unsigned long xb_aoerr0:1; /* RW */
+ unsigned long rt_aoerr0:1; /* RW */
+ unsigned long ni0_aoerr0:1; /* RW */
+ unsigned long ni1_aoerr0:1; /* RW */
+ unsigned long lb_aoerr1:1; /* RW */
+ unsigned long qp_aoerr1:1; /* RW */
+ unsigned long rh_aoerr1:1; /* RW */
+ unsigned long lh0_aoerr1:1; /* RW */
+ unsigned long lh1_aoerr1:1; /* RW */
+ unsigned long gr0_aoerr1:1; /* RW */
+ unsigned long gr1_aoerr1:1; /* RW */
+ unsigned long xb_aoerr1:1; /* RW */
+ unsigned long rt_aoerr1:1; /* RW */
+ unsigned long ni0_aoerr1:1; /* RW */
+ unsigned long ni1_aoerr1:1; /* RW */
+ unsigned long system_shutdown_int:1; /* RW */
+ unsigned long lb_irq_int_0:1; /* RW */
+ unsigned long lb_irq_int_1:1; /* RW */
+ unsigned long lb_irq_int_2:1; /* RW */
+ unsigned long lb_irq_int_3:1; /* RW */
+ unsigned long lb_irq_int_4:1; /* RW */
+ unsigned long lb_irq_int_5:1; /* RW */
+ unsigned long lb_irq_int_6:1; /* RW */
+ unsigned long lb_irq_int_7:1; /* RW */
+ unsigned long lb_irq_int_8:1; /* RW */
+ unsigned long lb_irq_int_9:1; /* RW */
+ unsigned long lb_irq_int_10:1; /* RW */
+ unsigned long lb_irq_int_11:1; /* RW */
+ unsigned long lb_irq_int_12:1; /* RW */
+ unsigned long lb_irq_int_13:1; /* RW */
+ unsigned long lb_irq_int_14:1; /* RW */
+ unsigned long lb_irq_int_15:1; /* RW */
+ unsigned long l1_nmi_int:1; /* RW */
+ unsigned long stop_clock:1; /* RW */
+ unsigned long asic_to_l1:1; /* RW */
+ unsigned long l1_to_asic:1; /* RW */
+ unsigned long la_seq_trigger:1; /* RW */
+ unsigned long ipi_int:1; /* RW */
+ unsigned long extio_int0:1; /* RW */
+ unsigned long extio_int1:1; /* RW */
+ unsigned long extio_int2:1; /* RW */
+ unsigned long extio_int3:1; /* RW */
+ unsigned long profile_int:1; /* RW */
+ unsigned long rsvd_59_63:5;
+ } s3;
-union uvh_extio_int0_broadcast_u {
- unsigned long v;
- struct uvh_extio_int0_broadcast_s {
- unsigned long enable:1; /* RW */
- unsigned long rsvd_1_63:63;
- } s;
+ /* UV2 unique struct */
+ struct uv2h_event_occurred0_s {
+ unsigned long lb_hcerr:1; /* RW */
+ unsigned long qp_hcerr:1; /* RW */
+ unsigned long rh_hcerr:1; /* RW */
+ unsigned long lh0_hcerr:1; /* RW */
+ unsigned long lh1_hcerr:1; /* RW */
+ unsigned long gr0_hcerr:1; /* RW */
+ unsigned long gr1_hcerr:1; /* RW */
+ unsigned long ni0_hcerr:1; /* RW */
+ unsigned long ni1_hcerr:1; /* RW */
+ unsigned long lb_aoerr0:1; /* RW */
+ unsigned long qp_aoerr0:1; /* RW */
+ unsigned long rh_aoerr0:1; /* RW */
+ unsigned long lh0_aoerr0:1; /* RW */
+ unsigned long lh1_aoerr0:1; /* RW */
+ unsigned long gr0_aoerr0:1; /* RW */
+ unsigned long gr1_aoerr0:1; /* RW */
+ unsigned long xb_aoerr0:1; /* RW */
+ unsigned long rt_aoerr0:1; /* RW */
+ unsigned long ni0_aoerr0:1; /* RW */
+ unsigned long ni1_aoerr0:1; /* RW */
+ unsigned long lb_aoerr1:1; /* RW */
+ unsigned long qp_aoerr1:1; /* RW */
+ unsigned long rh_aoerr1:1; /* RW */
+ unsigned long lh0_aoerr1:1; /* RW */
+ unsigned long lh1_aoerr1:1; /* RW */
+ unsigned long gr0_aoerr1:1; /* RW */
+ unsigned long gr1_aoerr1:1; /* RW */
+ unsigned long xb_aoerr1:1; /* RW */
+ unsigned long rt_aoerr1:1; /* RW */
+ unsigned long ni0_aoerr1:1; /* RW */
+ unsigned long ni1_aoerr1:1; /* RW */
+ unsigned long system_shutdown_int:1; /* RW */
+ unsigned long lb_irq_int_0:1; /* RW */
+ unsigned long lb_irq_int_1:1; /* RW */
+ unsigned long lb_irq_int_2:1; /* RW */
+ unsigned long lb_irq_int_3:1; /* RW */
+ unsigned long lb_irq_int_4:1; /* RW */
+ unsigned long lb_irq_int_5:1; /* RW */
+ unsigned long lb_irq_int_6:1; /* RW */
+ unsigned long lb_irq_int_7:1; /* RW */
+ unsigned long lb_irq_int_8:1; /* RW */
+ unsigned long lb_irq_int_9:1; /* RW */
+ unsigned long lb_irq_int_10:1; /* RW */
+ unsigned long lb_irq_int_11:1; /* RW */
+ unsigned long lb_irq_int_12:1; /* RW */
+ unsigned long lb_irq_int_13:1; /* RW */
+ unsigned long lb_irq_int_14:1; /* RW */
+ unsigned long lb_irq_int_15:1; /* RW */
+ unsigned long l1_nmi_int:1; /* RW */
+ unsigned long stop_clock:1; /* RW */
+ unsigned long asic_to_l1:1; /* RW */
+ unsigned long l1_to_asic:1; /* RW */
+ unsigned long la_seq_trigger:1; /* RW */
+ unsigned long ipi_int:1; /* RW */
+ unsigned long extio_int0:1; /* RW */
+ unsigned long extio_int1:1; /* RW */
+ unsigned long extio_int2:1; /* RW */
+ unsigned long extio_int3:1; /* RW */
+ unsigned long profile_int:1; /* RW */
+ unsigned long rsvd_59_63:5;
+ } s2;
};
/* ========================================================================= */
-/* UVH_GR0_TLB_INT0_CONFIG */
+/* UVH_EVENT_OCCURRED0_ALIAS */
/* ========================================================================= */
-#define UVH_GR0_TLB_INT0_CONFIG 0x61b00UL
-
-#define UVH_GR0_TLB_INT0_CONFIG_VECTOR_SHFT 0
-#define UVH_GR0_TLB_INT0_CONFIG_DM_SHFT 8
-#define UVH_GR0_TLB_INT0_CONFIG_DESTMODE_SHFT 11
-#define UVH_GR0_TLB_INT0_CONFIG_STATUS_SHFT 12
-#define UVH_GR0_TLB_INT0_CONFIG_P_SHFT 13
-#define UVH_GR0_TLB_INT0_CONFIG_T_SHFT 15
-#define UVH_GR0_TLB_INT0_CONFIG_M_SHFT 16
-#define UVH_GR0_TLB_INT0_CONFIG_APIC_ID_SHFT 32
-#define UVH_GR0_TLB_INT0_CONFIG_VECTOR_MASK 0x00000000000000ffUL
-#define UVH_GR0_TLB_INT0_CONFIG_DM_MASK 0x0000000000000700UL
-#define UVH_GR0_TLB_INT0_CONFIG_DESTMODE_MASK 0x0000000000000800UL
-#define UVH_GR0_TLB_INT0_CONFIG_STATUS_MASK 0x0000000000001000UL
-#define UVH_GR0_TLB_INT0_CONFIG_P_MASK 0x0000000000002000UL
-#define UVH_GR0_TLB_INT0_CONFIG_T_MASK 0x0000000000008000UL
-#define UVH_GR0_TLB_INT0_CONFIG_M_MASK 0x0000000000010000UL
-#define UVH_GR0_TLB_INT0_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
-
+#define UVH_EVENT_OCCURRED0_ALIAS 0x70008UL
-union uvh_gr0_tlb_int0_config_u {
- unsigned long v;
- struct uvh_gr0_tlb_int0_config_s {
- unsigned long vector_:8; /* RW */
- unsigned long dm:3; /* RW */
- unsigned long destmode:1; /* RW */
- unsigned long status:1; /* RO */
- unsigned long p:1; /* RO */
- unsigned long rsvd_14:1;
- unsigned long t:1; /* RO */
- unsigned long m:1; /* RW */
- unsigned long rsvd_17_31:15;
- unsigned long apic_id:32; /* RW */
- } s;
-};
/* ========================================================================= */
-/* UVH_GR0_TLB_INT1_CONFIG */
+/* UVH_EVENT_OCCURRED1 */
/* ========================================================================= */
-#define UVH_GR0_TLB_INT1_CONFIG 0x61b40UL
-
-#define UVH_GR0_TLB_INT1_CONFIG_VECTOR_SHFT 0
-#define UVH_GR0_TLB_INT1_CONFIG_DM_SHFT 8
-#define UVH_GR0_TLB_INT1_CONFIG_DESTMODE_SHFT 11
-#define UVH_GR0_TLB_INT1_CONFIG_STATUS_SHFT 12
-#define UVH_GR0_TLB_INT1_CONFIG_P_SHFT 13
-#define UVH_GR0_TLB_INT1_CONFIG_T_SHFT 15
-#define UVH_GR0_TLB_INT1_CONFIG_M_SHFT 16
-#define UVH_GR0_TLB_INT1_CONFIG_APIC_ID_SHFT 32
-#define UVH_GR0_TLB_INT1_CONFIG_VECTOR_MASK 0x00000000000000ffUL
-#define UVH_GR0_TLB_INT1_CONFIG_DM_MASK 0x0000000000000700UL
-#define UVH_GR0_TLB_INT1_CONFIG_DESTMODE_MASK 0x0000000000000800UL
-#define UVH_GR0_TLB_INT1_CONFIG_STATUS_MASK 0x0000000000001000UL
-#define UVH_GR0_TLB_INT1_CONFIG_P_MASK 0x0000000000002000UL
-#define UVH_GR0_TLB_INT1_CONFIG_T_MASK 0x0000000000008000UL
-#define UVH_GR0_TLB_INT1_CONFIG_M_MASK 0x0000000000010000UL
-#define UVH_GR0_TLB_INT1_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
-
-
-union uvh_gr0_tlb_int1_config_u {
+#define UVH_EVENT_OCCURRED1 0x70080UL
+
+
+
+/* UVYH common defines */
+#define UVYH_EVENT_OCCURRED1_IPI_INT_SHFT 0
+#define UVYH_EVENT_OCCURRED1_IPI_INT_MASK 0x0000000000000001UL
+#define UVYH_EVENT_OCCURRED1_EXTIO_INT0_SHFT 1
+#define UVYH_EVENT_OCCURRED1_EXTIO_INT0_MASK 0x0000000000000002UL
+#define UVYH_EVENT_OCCURRED1_EXTIO_INT1_SHFT 2
+#define UVYH_EVENT_OCCURRED1_EXTIO_INT1_MASK 0x0000000000000004UL
+#define UVYH_EVENT_OCCURRED1_EXTIO_INT2_SHFT 3
+#define UVYH_EVENT_OCCURRED1_EXTIO_INT2_MASK 0x0000000000000008UL
+#define UVYH_EVENT_OCCURRED1_EXTIO_INT3_SHFT 4
+#define UVYH_EVENT_OCCURRED1_EXTIO_INT3_MASK 0x0000000000000010UL
+#define UVYH_EVENT_OCCURRED1_PROFILE_INT_SHFT 5
+#define UVYH_EVENT_OCCURRED1_PROFILE_INT_MASK 0x0000000000000020UL
+#define UVYH_EVENT_OCCURRED1_BAU_DATA_SHFT 6
+#define UVYH_EVENT_OCCURRED1_BAU_DATA_MASK 0x0000000000000040UL
+#define UVYH_EVENT_OCCURRED1_PROC_GENERAL_SHFT 7
+#define UVYH_EVENT_OCCURRED1_PROC_GENERAL_MASK 0x0000000000000080UL
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT0_SHFT 8
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT0_MASK 0x0000000000000100UL
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT1_SHFT 9
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT1_MASK 0x0000000000000200UL
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT2_SHFT 10
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT2_MASK 0x0000000000000400UL
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT3_SHFT 11
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT3_MASK 0x0000000000000800UL
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT4_SHFT 12
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT4_MASK 0x0000000000001000UL
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT5_SHFT 13
+#define UVYH_EVENT_OCCURRED1_XH_TLB_INT5_MASK 0x0000000000002000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT0_SHFT 14
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT0_MASK 0x0000000000004000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT1_SHFT 15
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT1_MASK 0x0000000000008000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT2_SHFT 16
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT2_MASK 0x0000000000010000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT3_SHFT 17
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT3_MASK 0x0000000000020000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT4_SHFT 18
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT4_MASK 0x0000000000040000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT5_SHFT 19
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT5_MASK 0x0000000000080000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT6_SHFT 20
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT6_MASK 0x0000000000100000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT7_SHFT 21
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT7_MASK 0x0000000000200000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT8_SHFT 22
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT8_MASK 0x0000000000400000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT9_SHFT 23
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT9_MASK 0x0000000000800000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT10_SHFT 24
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT10_MASK 0x0000000001000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT11_SHFT 25
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT11_MASK 0x0000000002000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT12_SHFT 26
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT12_MASK 0x0000000004000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT13_SHFT 27
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT13_MASK 0x0000000008000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT14_SHFT 28
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT14_MASK 0x0000000010000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT15_SHFT 29
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT15_MASK 0x0000000020000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT16_SHFT 30
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT16_MASK 0x0000000040000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT17_SHFT 31
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT17_MASK 0x0000000080000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT18_SHFT 32
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT18_MASK 0x0000000100000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT19_SHFT 33
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT19_MASK 0x0000000200000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT20_SHFT 34
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT20_MASK 0x0000000400000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT21_SHFT 35
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT21_MASK 0x0000000800000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT22_SHFT 36
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT22_MASK 0x0000001000000000UL
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT23_SHFT 37
+#define UVYH_EVENT_OCCURRED1_RDM_TLB_INT23_MASK 0x0000002000000000UL
+
+/* UV4 unique defines */
+#define UV4H_EVENT_OCCURRED1_PROFILE_INT_SHFT 0
+#define UV4H_EVENT_OCCURRED1_PROFILE_INT_MASK 0x0000000000000001UL
+#define UV4H_EVENT_OCCURRED1_BAU_DATA_SHFT 1
+#define UV4H_EVENT_OCCURRED1_BAU_DATA_MASK 0x0000000000000002UL
+#define UV4H_EVENT_OCCURRED1_PROC_GENERAL_SHFT 2
+#define UV4H_EVENT_OCCURRED1_PROC_GENERAL_MASK 0x0000000000000004UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT0_SHFT 3
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT0_MASK 0x0000000000000008UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT1_SHFT 4
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT1_MASK 0x0000000000000010UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT2_SHFT 5
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT2_MASK 0x0000000000000020UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT3_SHFT 6
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT3_MASK 0x0000000000000040UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT4_SHFT 7
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT4_MASK 0x0000000000000080UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT5_SHFT 8
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT5_MASK 0x0000000000000100UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT6_SHFT 9
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT6_MASK 0x0000000000000200UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT7_SHFT 10
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT7_MASK 0x0000000000000400UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT8_SHFT 11
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT8_MASK 0x0000000000000800UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT9_SHFT 12
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT9_MASK 0x0000000000001000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT10_SHFT 13
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT10_MASK 0x0000000000002000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT11_SHFT 14
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT11_MASK 0x0000000000004000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT12_SHFT 15
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT12_MASK 0x0000000000008000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT13_SHFT 16
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT13_MASK 0x0000000000010000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT14_SHFT 17
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT14_MASK 0x0000000000020000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT15_SHFT 18
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT15_MASK 0x0000000000040000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT16_SHFT 19
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT16_MASK 0x0000000000080000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT17_SHFT 20
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT17_MASK 0x0000000000100000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT18_SHFT 21
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT18_MASK 0x0000000000200000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT19_SHFT 22
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT19_MASK 0x0000000000400000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT20_SHFT 23
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT20_MASK 0x0000000000800000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT21_SHFT 24
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT21_MASK 0x0000000001000000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT22_SHFT 25
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT22_MASK 0x0000000002000000UL
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT23_SHFT 26
+#define UV4H_EVENT_OCCURRED1_GR0_TLB_INT23_MASK 0x0000000004000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT0_SHFT 27
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT0_MASK 0x0000000008000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT1_SHFT 28
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT1_MASK 0x0000000010000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT2_SHFT 29
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT2_MASK 0x0000000020000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT3_SHFT 30
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT3_MASK 0x0000000040000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT4_SHFT 31
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT4_MASK 0x0000000080000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT5_SHFT 32
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT5_MASK 0x0000000100000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT6_SHFT 33
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT6_MASK 0x0000000200000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT7_SHFT 34
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT7_MASK 0x0000000400000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT8_SHFT 35
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT8_MASK 0x0000000800000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT9_SHFT 36
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT9_MASK 0x0000001000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT10_SHFT 37
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT10_MASK 0x0000002000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT11_SHFT 38
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT11_MASK 0x0000004000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT12_SHFT 39
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT12_MASK 0x0000008000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT13_SHFT 40
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT13_MASK 0x0000010000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT14_SHFT 41
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT14_MASK 0x0000020000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT15_SHFT 42
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT15_MASK 0x0000040000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT16_SHFT 43
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT16_MASK 0x0000080000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT17_SHFT 44
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT17_MASK 0x0000100000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT18_SHFT 45
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT18_MASK 0x0000200000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT19_SHFT 46
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT19_MASK 0x0000400000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT20_SHFT 47
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT20_MASK 0x0000800000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT21_SHFT 48
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT21_MASK 0x0001000000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT22_SHFT 49
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT22_MASK 0x0002000000000000UL
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT23_SHFT 50
+#define UV4H_EVENT_OCCURRED1_GR1_TLB_INT23_MASK 0x0004000000000000UL
+
+/* UV3 unique defines */
+#define UV3H_EVENT_OCCURRED1_BAU_DATA_SHFT 0
+#define UV3H_EVENT_OCCURRED1_BAU_DATA_MASK 0x0000000000000001UL
+#define UV3H_EVENT_OCCURRED1_POWER_MANAGEMENT_REQ_SHFT 1
+#define UV3H_EVENT_OCCURRED1_POWER_MANAGEMENT_REQ_MASK 0x0000000000000002UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT0_SHFT 2
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT0_MASK 0x0000000000000004UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT1_SHFT 3
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT1_MASK 0x0000000000000008UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT2_SHFT 4
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT2_MASK 0x0000000000000010UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT3_SHFT 5
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT3_MASK 0x0000000000000020UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT4_SHFT 6
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT4_MASK 0x0000000000000040UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT5_SHFT 7
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT5_MASK 0x0000000000000080UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT6_SHFT 8
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT6_MASK 0x0000000000000100UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT7_SHFT 9
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT7_MASK 0x0000000000000200UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT8_SHFT 10
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT8_MASK 0x0000000000000400UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT9_SHFT 11
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT9_MASK 0x0000000000000800UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT10_SHFT 12
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT10_MASK 0x0000000000001000UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT11_SHFT 13
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT11_MASK 0x0000000000002000UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT12_SHFT 14
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT12_MASK 0x0000000000004000UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT13_SHFT 15
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT13_MASK 0x0000000000008000UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT14_SHFT 16
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT14_MASK 0x0000000000010000UL
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT15_SHFT 17
+#define UV3H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT15_MASK 0x0000000000020000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT0_SHFT 18
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT0_MASK 0x0000000000040000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT1_SHFT 19
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT1_MASK 0x0000000000080000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT2_SHFT 20
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT2_MASK 0x0000000000100000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT3_SHFT 21
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT3_MASK 0x0000000000200000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT4_SHFT 22
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT4_MASK 0x0000000000400000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT5_SHFT 23
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT5_MASK 0x0000000000800000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT6_SHFT 24
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT6_MASK 0x0000000001000000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT7_SHFT 25
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT7_MASK 0x0000000002000000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT8_SHFT 26
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT8_MASK 0x0000000004000000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT9_SHFT 27
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT9_MASK 0x0000000008000000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT10_SHFT 28
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT10_MASK 0x0000000010000000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT11_SHFT 29
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT11_MASK 0x0000000020000000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT12_SHFT 30
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT12_MASK 0x0000000040000000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT13_SHFT 31
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT13_MASK 0x0000000080000000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT14_SHFT 32
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT14_MASK 0x0000000100000000UL
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT15_SHFT 33
+#define UV3H_EVENT_OCCURRED1_GR0_TLB_INT15_MASK 0x0000000200000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT0_SHFT 34
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT0_MASK 0x0000000400000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT1_SHFT 35
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT1_MASK 0x0000000800000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT2_SHFT 36
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT2_MASK 0x0000001000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT3_SHFT 37
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT3_MASK 0x0000002000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT4_SHFT 38
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT4_MASK 0x0000004000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT5_SHFT 39
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT5_MASK 0x0000008000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT6_SHFT 40
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT6_MASK 0x0000010000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT7_SHFT 41
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT7_MASK 0x0000020000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT8_SHFT 42
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT8_MASK 0x0000040000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT9_SHFT 43
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT9_MASK 0x0000080000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT10_SHFT 44
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT10_MASK 0x0000100000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT11_SHFT 45
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT11_MASK 0x0000200000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT12_SHFT 46
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT12_MASK 0x0000400000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT13_SHFT 47
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT13_MASK 0x0000800000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT14_SHFT 48
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT14_MASK 0x0001000000000000UL
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT15_SHFT 49
+#define UV3H_EVENT_OCCURRED1_GR1_TLB_INT15_MASK 0x0002000000000000UL
+#define UV3H_EVENT_OCCURRED1_RTC_INTERVAL_INT_SHFT 50
+#define UV3H_EVENT_OCCURRED1_RTC_INTERVAL_INT_MASK 0x0004000000000000UL
+#define UV3H_EVENT_OCCURRED1_BAU_DASHBOARD_INT_SHFT 51
+#define UV3H_EVENT_OCCURRED1_BAU_DASHBOARD_INT_MASK 0x0008000000000000UL
+
+/* UV2 unique defines */
+#define UV2H_EVENT_OCCURRED1_BAU_DATA_SHFT 0
+#define UV2H_EVENT_OCCURRED1_BAU_DATA_MASK 0x0000000000000001UL
+#define UV2H_EVENT_OCCURRED1_POWER_MANAGEMENT_REQ_SHFT 1
+#define UV2H_EVENT_OCCURRED1_POWER_MANAGEMENT_REQ_MASK 0x0000000000000002UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT0_SHFT 2
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT0_MASK 0x0000000000000004UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT1_SHFT 3
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT1_MASK 0x0000000000000008UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT2_SHFT 4
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT2_MASK 0x0000000000000010UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT3_SHFT 5
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT3_MASK 0x0000000000000020UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT4_SHFT 6
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT4_MASK 0x0000000000000040UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT5_SHFT 7
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT5_MASK 0x0000000000000080UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT6_SHFT 8
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT6_MASK 0x0000000000000100UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT7_SHFT 9
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT7_MASK 0x0000000000000200UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT8_SHFT 10
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT8_MASK 0x0000000000000400UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT9_SHFT 11
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT9_MASK 0x0000000000000800UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT10_SHFT 12
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT10_MASK 0x0000000000001000UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT11_SHFT 13
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT11_MASK 0x0000000000002000UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT12_SHFT 14
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT12_MASK 0x0000000000004000UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT13_SHFT 15
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT13_MASK 0x0000000000008000UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT14_SHFT 16
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT14_MASK 0x0000000000010000UL
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT15_SHFT 17
+#define UV2H_EVENT_OCCURRED1_MESSAGE_ACCELERATOR_INT15_MASK 0x0000000000020000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT0_SHFT 18
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT0_MASK 0x0000000000040000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT1_SHFT 19
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT1_MASK 0x0000000000080000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT2_SHFT 20
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT2_MASK 0x0000000000100000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT3_SHFT 21
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT3_MASK 0x0000000000200000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT4_SHFT 22
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT4_MASK 0x0000000000400000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT5_SHFT 23
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT5_MASK 0x0000000000800000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT6_SHFT 24
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT6_MASK 0x0000000001000000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT7_SHFT 25
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT7_MASK 0x0000000002000000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT8_SHFT 26
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT8_MASK 0x0000000004000000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT9_SHFT 27
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT9_MASK 0x0000000008000000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT10_SHFT 28
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT10_MASK 0x0000000010000000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT11_SHFT 29
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT11_MASK 0x0000000020000000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT12_SHFT 30
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT12_MASK 0x0000000040000000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT13_SHFT 31
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT13_MASK 0x0000000080000000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT14_SHFT 32
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT14_MASK 0x0000000100000000UL
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT15_SHFT 33
+#define UV2H_EVENT_OCCURRED1_GR0_TLB_INT15_MASK 0x0000000200000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT0_SHFT 34
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT0_MASK 0x0000000400000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT1_SHFT 35
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT1_MASK 0x0000000800000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT2_SHFT 36
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT2_MASK 0x0000001000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT3_SHFT 37
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT3_MASK 0x0000002000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT4_SHFT 38
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT4_MASK 0x0000004000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT5_SHFT 39
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT5_MASK 0x0000008000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT6_SHFT 40
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT6_MASK 0x0000010000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT7_SHFT 41
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT7_MASK 0x0000020000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT8_SHFT 42
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT8_MASK 0x0000040000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT9_SHFT 43
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT9_MASK 0x0000080000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT10_SHFT 44
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT10_MASK 0x0000100000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT11_SHFT 45
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT11_MASK 0x0000200000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT12_SHFT 46
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT12_MASK 0x0000400000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT13_SHFT 47
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT13_MASK 0x0000800000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT14_SHFT 48
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT14_MASK 0x0001000000000000UL
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT15_SHFT 49
+#define UV2H_EVENT_OCCURRED1_GR1_TLB_INT15_MASK 0x0002000000000000UL
+#define UV2H_EVENT_OCCURRED1_RTC_INTERVAL_INT_SHFT 50
+#define UV2H_EVENT_OCCURRED1_RTC_INTERVAL_INT_MASK 0x0004000000000000UL
+#define UV2H_EVENT_OCCURRED1_BAU_DASHBOARD_INT_SHFT 51
+#define UV2H_EVENT_OCCURRED1_BAU_DASHBOARD_INT_MASK 0x0008000000000000UL
+
+#define UVH_EVENT_OCCURRED1_EXTIO_INT0_MASK ( \
+ is_uv(UV5) ? 0x0000000000000002UL : \
+ 0)
+#define UVH_EVENT_OCCURRED1_EXTIO_INT0_SHFT ( \
+ is_uv(UV5) ? 1 : \
+ -1)
+
+union uvyh_event_occurred1_u {
unsigned long v;
- struct uvh_gr0_tlb_int1_config_s {
- unsigned long vector_:8; /* RW */
- unsigned long dm:3; /* RW */
- unsigned long destmode:1; /* RW */
- unsigned long status:1; /* RO */
- unsigned long p:1; /* RO */
- unsigned long rsvd_14:1;
- unsigned long t:1; /* RO */
- unsigned long m:1; /* RW */
- unsigned long rsvd_17_31:15;
- unsigned long apic_id:32; /* RW */
- } s;
-};
-/* ========================================================================= */
-/* UVH_GR0_TLB_MMR_CONTROL */
-/* ========================================================================= */
-#define UV2H_GR0_TLB_MMR_CONTROL 0xc01080UL
-#define UV3H_GR0_TLB_MMR_CONTROL 0xc01080UL
-#define UV4H_GR0_TLB_MMR_CONTROL 0x601080UL
-#define UVH_GR0_TLB_MMR_CONTROL ( \
- is_uv2_hub() ? UV2H_GR0_TLB_MMR_CONTROL : \
- is_uv3_hub() ? UV3H_GR0_TLB_MMR_CONTROL : \
- /*is_uv4_hub*/ UV4H_GR0_TLB_MMR_CONTROL)
-
-#define UVH_GR0_TLB_MMR_CONTROL_INDEX_SHFT 0
-#define UVH_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT 16
-#define UVH_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
-#define UVH_GR0_TLB_MMR_CONTROL_MMR_WRITE_SHFT 30
-#define UVH_GR0_TLB_MMR_CONTROL_MMR_READ_SHFT 31
-#define UVH_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK 0x0000000000010000UL
-#define UVH_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
-#define UVH_GR0_TLB_MMR_CONTROL_MMR_WRITE_MASK 0x0000000040000000UL
-#define UVH_GR0_TLB_MMR_CONTROL_MMR_READ_MASK 0x0000000080000000UL
-
-#define UVXH_GR0_TLB_MMR_CONTROL_INDEX_SHFT 0
-#define UVXH_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT 16
-#define UVXH_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
-#define UVXH_GR0_TLB_MMR_CONTROL_MMR_WRITE_SHFT 30
-#define UVXH_GR0_TLB_MMR_CONTROL_MMR_READ_SHFT 31
-#define UVXH_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT 32
-#define UVXH_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK 0x0000000000010000UL
-#define UVXH_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
-#define UVXH_GR0_TLB_MMR_CONTROL_MMR_WRITE_MASK 0x0000000040000000UL
-#define UVXH_GR0_TLB_MMR_CONTROL_MMR_READ_MASK 0x0000000080000000UL
-#define UVXH_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_MASK 0x0000000100000000UL
-
-#define UV2H_GR0_TLB_MMR_CONTROL_INDEX_SHFT 0
-#define UV2H_GR0_TLB_MMR_CONTROL_MEM_SEL_SHFT 12
-#define UV2H_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT 16
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_WRITE_SHFT 30
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_READ_SHFT 31
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT 32
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_INJ_CON_SHFT 48
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_INJ_TLBRAM_SHFT 52
-#define UV2H_GR0_TLB_MMR_CONTROL_INDEX_MASK 0x0000000000000fffUL
-#define UV2H_GR0_TLB_MMR_CONTROL_MEM_SEL_MASK 0x0000000000003000UL
-#define UV2H_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK 0x0000000000010000UL
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_WRITE_MASK 0x0000000040000000UL
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_READ_MASK 0x0000000080000000UL
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_MASK 0x0000000100000000UL
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_INJ_CON_MASK 0x0001000000000000UL
-#define UV2H_GR0_TLB_MMR_CONTROL_MMR_INJ_TLBRAM_MASK 0x0010000000000000UL
-
-#define UV3H_GR0_TLB_MMR_CONTROL_INDEX_SHFT 0
-#define UV3H_GR0_TLB_MMR_CONTROL_MEM_SEL_SHFT 12
-#define UV3H_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT 16
-#define UV3H_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
-#define UV3H_GR0_TLB_MMR_CONTROL_ECC_SEL_SHFT 21
-#define UV3H_GR0_TLB_MMR_CONTROL_MMR_WRITE_SHFT 30
-#define UV3H_GR0_TLB_MMR_CONTROL_MMR_READ_SHFT 31
-#define UV3H_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT 32
-#define UV3H_GR0_TLB_MMR_CONTROL_INDEX_MASK 0x0000000000000fffUL
-#define UV3H_GR0_TLB_MMR_CONTROL_MEM_SEL_MASK 0x0000000000003000UL
-#define UV3H_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK 0x0000000000010000UL
-#define UV3H_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
-#define UV3H_GR0_TLB_MMR_CONTROL_ECC_SEL_MASK 0x0000000000200000UL
-#define UV3H_GR0_TLB_MMR_CONTROL_MMR_WRITE_MASK 0x0000000040000000UL
-#define UV3H_GR0_TLB_MMR_CONTROL_MMR_READ_MASK 0x0000000080000000UL
-#define UV3H_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_MASK 0x0000000100000000UL
-
-#define UV4H_GR0_TLB_MMR_CONTROL_INDEX_SHFT 0
-#define UV4H_GR0_TLB_MMR_CONTROL_MEM_SEL_SHFT 13
-#define UV4H_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT 16
-#define UV4H_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
-#define UV4H_GR0_TLB_MMR_CONTROL_ECC_SEL_SHFT 21
-#define UV4H_GR0_TLB_MMR_CONTROL_MMR_WRITE_SHFT 30
-#define UV4H_GR0_TLB_MMR_CONTROL_MMR_READ_SHFT 31
-#define UV4H_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT 32
-#define UV4H_GR0_TLB_MMR_CONTROL_PAGE_SIZE_SHFT 59
-#define UV4H_GR0_TLB_MMR_CONTROL_INDEX_MASK 0x0000000000001fffUL
-#define UV4H_GR0_TLB_MMR_CONTROL_MEM_SEL_MASK 0x0000000000006000UL
-#define UV4H_GR0_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK 0x0000000000010000UL
-#define UV4H_GR0_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
-#define UV4H_GR0_TLB_MMR_CONTROL_ECC_SEL_MASK 0x0000000000200000UL
-#define UV4H_GR0_TLB_MMR_CONTROL_MMR_WRITE_MASK 0x0000000040000000UL
-#define UV4H_GR0_TLB_MMR_CONTROL_MMR_READ_MASK 0x0000000080000000UL
-#define UV4H_GR0_TLB_MMR_CONTROL_MMR_OP_DONE_MASK 0x0000000100000000UL
-#define UV4H_GR0_TLB_MMR_CONTROL_PAGE_SIZE_MASK 0xf800000000000000UL
-
-#define UVH_GR0_TLB_MMR_CONTROL_INDEX_MASK ( \
- is_uv2_hub() ? UV2H_GR0_TLB_MMR_CONTROL_INDEX_MASK : \
- is_uv3_hub() ? UV3H_GR0_TLB_MMR_CONTROL_INDEX_MASK : \
- /*is_uv4_hub*/ UV4H_GR0_TLB_MMR_CONTROL_INDEX_MASK)
-#define UVH_GR0_TLB_MMR_CONTROL_MEM_SEL_MASK ( \
- is_uv2_hub() ? UV2H_GR0_TLB_MMR_CONTROL_MEM_SEL_MASK : \
- is_uv3_hub() ? UV3H_GR0_TLB_MMR_CONTROL_MEM_SEL_MASK : \
- /*is_uv4_hub*/ UV4H_GR0_TLB_MMR_CONTROL_MEM_SEL_MASK)
-#define UVH_GR0_TLB_MMR_CONTROL_MEM_SEL_SHFT ( \
- is_uv2_hub() ? UV2H_GR0_TLB_MMR_CONTROL_MEM_SEL_SHFT : \
- is_uv3_hub() ? UV3H_GR0_TLB_MMR_CONTROL_MEM_SEL_SHFT : \
- /*is_uv4_hub*/ UV4H_GR0_TLB_MMR_CONTROL_MEM_SEL_SHFT)
-
-union uvh_gr0_tlb_mmr_control_u {
- unsigned long v;
- struct uvh_gr0_tlb_mmr_control_s {
- unsigned long rsvd_0_15:16;
- unsigned long auto_valid_en:1; /* RW */
- unsigned long rsvd_17_19:3;
- unsigned long mmr_hash_index_en:1; /* RW */
- unsigned long rsvd_21_29:9;
- unsigned long mmr_write:1; /* WP */
- unsigned long mmr_read:1; /* WP */
- unsigned long rsvd_32_48:17;
- unsigned long rsvd_49_51:3;
- unsigned long rsvd_52_63:12;
- } s;
- struct uvxh_gr0_tlb_mmr_control_s {
- unsigned long rsvd_0_15:16;
- unsigned long auto_valid_en:1; /* RW */
- unsigned long rsvd_17_19:3;
- unsigned long mmr_hash_index_en:1; /* RW */
- unsigned long rsvd_21_29:9;
- unsigned long mmr_write:1; /* WP */
- unsigned long mmr_read:1; /* WP */
- unsigned long mmr_op_done:1; /* RW */
- unsigned long rsvd_33_47:15;
- unsigned long rsvd_48:1;
- unsigned long rsvd_49_51:3;
- unsigned long rsvd_52_63:12;
- } sx;
- struct uv2h_gr0_tlb_mmr_control_s {
- unsigned long index:12; /* RW */
- unsigned long mem_sel:2; /* RW */
- unsigned long rsvd_14_15:2;
- unsigned long auto_valid_en:1; /* RW */
- unsigned long rsvd_17_19:3;
- unsigned long mmr_hash_index_en:1; /* RW */
- unsigned long rsvd_21_29:9;
- unsigned long mmr_write:1; /* WP */
- unsigned long mmr_read:1; /* WP */
- unsigned long mmr_op_done:1; /* RW */
- unsigned long rsvd_33_47:15;
- unsigned long mmr_inj_con:1; /* RW */
- unsigned long rsvd_49_51:3;
- unsigned long mmr_inj_tlbram:1; /* RW */
- unsigned long rsvd_53_63:11;
- } s2;
- struct uv3h_gr0_tlb_mmr_control_s {
- unsigned long index:12; /* RW */
- unsigned long mem_sel:2; /* RW */
- unsigned long rsvd_14_15:2;
- unsigned long auto_valid_en:1; /* RW */
- unsigned long rsvd_17_19:3;
- unsigned long mmr_hash_index_en:1; /* RW */
- unsigned long ecc_sel:1; /* RW */
- unsigned long rsvd_22_29:8;
- unsigned long mmr_write:1; /* WP */
- unsigned long mmr_read:1; /* WP */
- unsigned long mmr_op_done:1; /* RW */
- unsigned long rsvd_33_47:15;
- unsigned long undef_48:1; /* Undefined */
- unsigned long rsvd_49_51:3;
- unsigned long undef_52:1; /* Undefined */
- unsigned long rsvd_53_63:11;
- } s3;
- struct uv4h_gr0_tlb_mmr_control_s {
- unsigned long index:13; /* RW */
- unsigned long mem_sel:2; /* RW */
- unsigned long rsvd_15:1;
- unsigned long auto_valid_en:1; /* RW */
- unsigned long rsvd_17_19:3;
- unsigned long mmr_hash_index_en:1; /* RW */
- unsigned long ecc_sel:1; /* RW */
- unsigned long rsvd_22_29:8;
- unsigned long mmr_write:1; /* WP */
- unsigned long mmr_read:1; /* WP */
- unsigned long mmr_op_done:1; /* RW */
- unsigned long rsvd_33_47:15;
- unsigned long undef_48:1; /* Undefined */
- unsigned long rsvd_49_51:3;
- unsigned long rsvd_52_58:7;
- unsigned long page_size:5; /* RW */
+ /* UVYH common struct */
+ struct uvyh_event_occurred1_s {
+ unsigned long ipi_int:1; /* RW */
+ unsigned long extio_int0:1; /* RW */
+ unsigned long extio_int1:1; /* RW */
+ unsigned long extio_int2:1; /* RW */
+ unsigned long extio_int3:1; /* RW */
+ unsigned long profile_int:1; /* RW */
+ unsigned long bau_data:1; /* RW */
+ unsigned long proc_general:1; /* RW */
+ unsigned long xh_tlb_int0:1; /* RW */
+ unsigned long xh_tlb_int1:1; /* RW */
+ unsigned long xh_tlb_int2:1; /* RW */
+ unsigned long xh_tlb_int3:1; /* RW */
+ unsigned long xh_tlb_int4:1; /* RW */
+ unsigned long xh_tlb_int5:1; /* RW */
+ unsigned long rdm_tlb_int0:1; /* RW */
+ unsigned long rdm_tlb_int1:1; /* RW */
+ unsigned long rdm_tlb_int2:1; /* RW */
+ unsigned long rdm_tlb_int3:1; /* RW */
+ unsigned long rdm_tlb_int4:1; /* RW */
+ unsigned long rdm_tlb_int5:1; /* RW */
+ unsigned long rdm_tlb_int6:1; /* RW */
+ unsigned long rdm_tlb_int7:1; /* RW */
+ unsigned long rdm_tlb_int8:1; /* RW */
+ unsigned long rdm_tlb_int9:1; /* RW */
+ unsigned long rdm_tlb_int10:1; /* RW */
+ unsigned long rdm_tlb_int11:1; /* RW */
+ unsigned long rdm_tlb_int12:1; /* RW */
+ unsigned long rdm_tlb_int13:1; /* RW */
+ unsigned long rdm_tlb_int14:1; /* RW */
+ unsigned long rdm_tlb_int15:1; /* RW */
+ unsigned long rdm_tlb_int16:1; /* RW */
+ unsigned long rdm_tlb_int17:1; /* RW */
+ unsigned long rdm_tlb_int18:1; /* RW */
+ unsigned long rdm_tlb_int19:1; /* RW */
+ unsigned long rdm_tlb_int20:1; /* RW */
+ unsigned long rdm_tlb_int21:1; /* RW */
+ unsigned long rdm_tlb_int22:1; /* RW */
+ unsigned long rdm_tlb_int23:1; /* RW */
+ unsigned long rsvd_38_63:26;
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_event_occurred1_s {
+ unsigned long ipi_int:1; /* RW */
+ unsigned long extio_int0:1; /* RW */
+ unsigned long extio_int1:1; /* RW */
+ unsigned long extio_int2:1; /* RW */
+ unsigned long extio_int3:1; /* RW */
+ unsigned long profile_int:1; /* RW */
+ unsigned long bau_data:1; /* RW */
+ unsigned long proc_general:1; /* RW */
+ unsigned long xh_tlb_int0:1; /* RW */
+ unsigned long xh_tlb_int1:1; /* RW */
+ unsigned long xh_tlb_int2:1; /* RW */
+ unsigned long xh_tlb_int3:1; /* RW */
+ unsigned long xh_tlb_int4:1; /* RW */
+ unsigned long xh_tlb_int5:1; /* RW */
+ unsigned long rdm_tlb_int0:1; /* RW */
+ unsigned long rdm_tlb_int1:1; /* RW */
+ unsigned long rdm_tlb_int2:1; /* RW */
+ unsigned long rdm_tlb_int3:1; /* RW */
+ unsigned long rdm_tlb_int4:1; /* RW */
+ unsigned long rdm_tlb_int5:1; /* RW */
+ unsigned long rdm_tlb_int6:1; /* RW */
+ unsigned long rdm_tlb_int7:1; /* RW */
+ unsigned long rdm_tlb_int8:1; /* RW */
+ unsigned long rdm_tlb_int9:1; /* RW */
+ unsigned long rdm_tlb_int10:1; /* RW */
+ unsigned long rdm_tlb_int11:1; /* RW */
+ unsigned long rdm_tlb_int12:1; /* RW */
+ unsigned long rdm_tlb_int13:1; /* RW */
+ unsigned long rdm_tlb_int14:1; /* RW */
+ unsigned long rdm_tlb_int15:1; /* RW */
+ unsigned long rdm_tlb_int16:1; /* RW */
+ unsigned long rdm_tlb_int17:1; /* RW */
+ unsigned long rdm_tlb_int18:1; /* RW */
+ unsigned long rdm_tlb_int19:1; /* RW */
+ unsigned long rdm_tlb_int20:1; /* RW */
+ unsigned long rdm_tlb_int21:1; /* RW */
+ unsigned long rdm_tlb_int22:1; /* RW */
+ unsigned long rdm_tlb_int23:1; /* RW */
+ unsigned long rsvd_38_63:26;
+ } s5;
+
+ /* UV4 unique struct */
+ struct uv4h_event_occurred1_s {
+ unsigned long profile_int:1; /* RW */
+ unsigned long bau_data:1; /* RW */
+ unsigned long proc_general:1; /* RW */
+ unsigned long gr0_tlb_int0:1; /* RW */
+ unsigned long gr0_tlb_int1:1; /* RW */
+ unsigned long gr0_tlb_int2:1; /* RW */
+ unsigned long gr0_tlb_int3:1; /* RW */
+ unsigned long gr0_tlb_int4:1; /* RW */
+ unsigned long gr0_tlb_int5:1; /* RW */
+ unsigned long gr0_tlb_int6:1; /* RW */
+ unsigned long gr0_tlb_int7:1; /* RW */
+ unsigned long gr0_tlb_int8:1; /* RW */
+ unsigned long gr0_tlb_int9:1; /* RW */
+ unsigned long gr0_tlb_int10:1; /* RW */
+ unsigned long gr0_tlb_int11:1; /* RW */
+ unsigned long gr0_tlb_int12:1; /* RW */
+ unsigned long gr0_tlb_int13:1; /* RW */
+ unsigned long gr0_tlb_int14:1; /* RW */
+ unsigned long gr0_tlb_int15:1; /* RW */
+ unsigned long gr0_tlb_int16:1; /* RW */
+ unsigned long gr0_tlb_int17:1; /* RW */
+ unsigned long gr0_tlb_int18:1; /* RW */
+ unsigned long gr0_tlb_int19:1; /* RW */
+ unsigned long gr0_tlb_int20:1; /* RW */
+ unsigned long gr0_tlb_int21:1; /* RW */
+ unsigned long gr0_tlb_int22:1; /* RW */
+ unsigned long gr0_tlb_int23:1; /* RW */
+ unsigned long gr1_tlb_int0:1; /* RW */
+ unsigned long gr1_tlb_int1:1; /* RW */
+ unsigned long gr1_tlb_int2:1; /* RW */
+ unsigned long gr1_tlb_int3:1; /* RW */
+ unsigned long gr1_tlb_int4:1; /* RW */
+ unsigned long gr1_tlb_int5:1; /* RW */
+ unsigned long gr1_tlb_int6:1; /* RW */
+ unsigned long gr1_tlb_int7:1; /* RW */
+ unsigned long gr1_tlb_int8:1; /* RW */
+ unsigned long gr1_tlb_int9:1; /* RW */
+ unsigned long gr1_tlb_int10:1; /* RW */
+ unsigned long gr1_tlb_int11:1; /* RW */
+ unsigned long gr1_tlb_int12:1; /* RW */
+ unsigned long gr1_tlb_int13:1; /* RW */
+ unsigned long gr1_tlb_int14:1; /* RW */
+ unsigned long gr1_tlb_int15:1; /* RW */
+ unsigned long gr1_tlb_int16:1; /* RW */
+ unsigned long gr1_tlb_int17:1; /* RW */
+ unsigned long gr1_tlb_int18:1; /* RW */
+ unsigned long gr1_tlb_int19:1; /* RW */
+ unsigned long gr1_tlb_int20:1; /* RW */
+ unsigned long gr1_tlb_int21:1; /* RW */
+ unsigned long gr1_tlb_int22:1; /* RW */
+ unsigned long gr1_tlb_int23:1; /* RW */
+ unsigned long rsvd_51_63:13;
} s4;
-};
-/* ========================================================================= */
-/* UVH_GR0_TLB_MMR_READ_DATA_HI */
-/* ========================================================================= */
-#define UV2H_GR0_TLB_MMR_READ_DATA_HI 0xc010a0UL
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI 0xc010a0UL
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI 0x6010a0UL
-#define UVH_GR0_TLB_MMR_READ_DATA_HI ( \
- is_uv2_hub() ? UV2H_GR0_TLB_MMR_READ_DATA_HI : \
- is_uv3_hub() ? UV3H_GR0_TLB_MMR_READ_DATA_HI : \
- /*is_uv4_hub*/ UV4H_GR0_TLB_MMR_READ_DATA_HI)
-
-#define UVH_GR0_TLB_MMR_READ_DATA_HI_PFN_SHFT 0
-
-#define UVXH_GR0_TLB_MMR_READ_DATA_HI_PFN_SHFT 0
-
-#define UV2H_GR0_TLB_MMR_READ_DATA_HI_PFN_SHFT 0
-#define UV2H_GR0_TLB_MMR_READ_DATA_HI_GAA_SHFT 41
-#define UV2H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_SHFT 43
-#define UV2H_GR0_TLB_MMR_READ_DATA_HI_LARGER_SHFT 44
-#define UV2H_GR0_TLB_MMR_READ_DATA_HI_PFN_MASK 0x000001ffffffffffUL
-#define UV2H_GR0_TLB_MMR_READ_DATA_HI_GAA_MASK 0x0000060000000000UL
-#define UV2H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_MASK 0x0000080000000000UL
-#define UV2H_GR0_TLB_MMR_READ_DATA_HI_LARGER_MASK 0x0000100000000000UL
-
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_PFN_SHFT 0
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_GAA_SHFT 41
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_SHFT 43
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_LARGER_SHFT 44
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_AA_EXT_SHFT 45
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_WAY_ECC_SHFT 55
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_PFN_MASK 0x000001ffffffffffUL
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_GAA_MASK 0x0000060000000000UL
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_MASK 0x0000080000000000UL
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_LARGER_MASK 0x0000100000000000UL
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_AA_EXT_MASK 0x0000200000000000UL
-#define UV3H_GR0_TLB_MMR_READ_DATA_HI_WAY_ECC_MASK 0xff80000000000000UL
-
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_PFN_SHFT 0
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_PNID_SHFT 34
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_GAA_SHFT 49
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_SHFT 51
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_LARGER_SHFT 52
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_AA_EXT_SHFT 53
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_WAY_ECC_SHFT 55
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_PFN_MASK 0x00000003ffffffffUL
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_PNID_MASK 0x0001fffc00000000UL
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_GAA_MASK 0x0006000000000000UL
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_DIRTY_MASK 0x0008000000000000UL
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_LARGER_MASK 0x0010000000000000UL
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_AA_EXT_MASK 0x0020000000000000UL
-#define UV4H_GR0_TLB_MMR_READ_DATA_HI_WAY_ECC_MASK 0xff80000000000000UL
-
-
-union uvh_gr0_tlb_mmr_read_data_hi_u {
- unsigned long v;
- struct uv2h_gr0_tlb_mmr_read_data_hi_s {
- unsigned long pfn:41; /* RO */
- unsigned long gaa:2; /* RO */
- unsigned long dirty:1; /* RO */
- unsigned long larger:1; /* RO */
- unsigned long rsvd_45_63:19;
- } s2;
- struct uv3h_gr0_tlb_mmr_read_data_hi_s {
- unsigned long pfn:41; /* RO */
- unsigned long gaa:2; /* RO */
- unsigned long dirty:1; /* RO */
- unsigned long larger:1; /* RO */
- unsigned long aa_ext:1; /* RO */
- unsigned long undef_46_54:9; /* Undefined */
- unsigned long way_ecc:9; /* RO */
+ /* UV3 unique struct */
+ struct uv3h_event_occurred1_s {
+ unsigned long bau_data:1; /* RW */
+ unsigned long power_management_req:1; /* RW */
+ unsigned long message_accelerator_int0:1; /* RW */
+ unsigned long message_accelerator_int1:1; /* RW */
+ unsigned long message_accelerator_int2:1; /* RW */
+ unsigned long message_accelerator_int3:1; /* RW */
+ unsigned long message_accelerator_int4:1; /* RW */
+ unsigned long message_accelerator_int5:1; /* RW */
+ unsigned long message_accelerator_int6:1; /* RW */
+ unsigned long message_accelerator_int7:1; /* RW */
+ unsigned long message_accelerator_int8:1; /* RW */
+ unsigned long message_accelerator_int9:1; /* RW */
+ unsigned long message_accelerator_int10:1; /* RW */
+ unsigned long message_accelerator_int11:1; /* RW */
+ unsigned long message_accelerator_int12:1; /* RW */
+ unsigned long message_accelerator_int13:1; /* RW */
+ unsigned long message_accelerator_int14:1; /* RW */
+ unsigned long message_accelerator_int15:1; /* RW */
+ unsigned long gr0_tlb_int0:1; /* RW */
+ unsigned long gr0_tlb_int1:1; /* RW */
+ unsigned long gr0_tlb_int2:1; /* RW */
+ unsigned long gr0_tlb_int3:1; /* RW */
+ unsigned long gr0_tlb_int4:1; /* RW */
+ unsigned long gr0_tlb_int5:1; /* RW */
+ unsigned long gr0_tlb_int6:1; /* RW */
+ unsigned long gr0_tlb_int7:1; /* RW */
+ unsigned long gr0_tlb_int8:1; /* RW */
+ unsigned long gr0_tlb_int9:1; /* RW */
+ unsigned long gr0_tlb_int10:1; /* RW */
+ unsigned long gr0_tlb_int11:1; /* RW */
+ unsigned long gr0_tlb_int12:1; /* RW */
+ unsigned long gr0_tlb_int13:1; /* RW */
+ unsigned long gr0_tlb_int14:1; /* RW */
+ unsigned long gr0_tlb_int15:1; /* RW */
+ unsigned long gr1_tlb_int0:1; /* RW */
+ unsigned long gr1_tlb_int1:1; /* RW */
+ unsigned long gr1_tlb_int2:1; /* RW */
+ unsigned long gr1_tlb_int3:1; /* RW */
+ unsigned long gr1_tlb_int4:1; /* RW */
+ unsigned long gr1_tlb_int5:1; /* RW */
+ unsigned long gr1_tlb_int6:1; /* RW */
+ unsigned long gr1_tlb_int7:1; /* RW */
+ unsigned long gr1_tlb_int8:1; /* RW */
+ unsigned long gr1_tlb_int9:1; /* RW */
+ unsigned long gr1_tlb_int10:1; /* RW */
+ unsigned long gr1_tlb_int11:1; /* RW */
+ unsigned long gr1_tlb_int12:1; /* RW */
+ unsigned long gr1_tlb_int13:1; /* RW */
+ unsigned long gr1_tlb_int14:1; /* RW */
+ unsigned long gr1_tlb_int15:1; /* RW */
+ unsigned long rtc_interval_int:1; /* RW */
+ unsigned long bau_dashboard_int:1; /* RW */
+ unsigned long rsvd_52_63:12;
} s3;
- struct uv4h_gr0_tlb_mmr_read_data_hi_s {
- unsigned long pfn:34; /* RO */
- unsigned long pnid:15; /* RO */
- unsigned long gaa:2; /* RO */
- unsigned long dirty:1; /* RO */
- unsigned long larger:1; /* RO */
- unsigned long aa_ext:1; /* RO */
- unsigned long undef_54:1; /* Undefined */
- unsigned long way_ecc:9; /* RO */
- } s4;
-};
-/* ========================================================================= */
-/* UVH_GR0_TLB_MMR_READ_DATA_LO */
-/* ========================================================================= */
-#define UV2H_GR0_TLB_MMR_READ_DATA_LO 0xc010a8UL
-#define UV3H_GR0_TLB_MMR_READ_DATA_LO 0xc010a8UL
-#define UV4H_GR0_TLB_MMR_READ_DATA_LO 0x6010a8UL
-#define UVH_GR0_TLB_MMR_READ_DATA_LO ( \
- is_uv2_hub() ? UV2H_GR0_TLB_MMR_READ_DATA_LO : \
- is_uv3_hub() ? UV3H_GR0_TLB_MMR_READ_DATA_LO : \
- /*is_uv4_hub*/ UV4H_GR0_TLB_MMR_READ_DATA_LO)
-
-#define UVH_GR0_TLB_MMR_READ_DATA_LO_VPN_SHFT 0
-#define UVH_GR0_TLB_MMR_READ_DATA_LO_ASID_SHFT 39
-#define UVH_GR0_TLB_MMR_READ_DATA_LO_VALID_SHFT 63
-#define UVH_GR0_TLB_MMR_READ_DATA_LO_VPN_MASK 0x0000007fffffffffUL
-#define UVH_GR0_TLB_MMR_READ_DATA_LO_ASID_MASK 0x7fffff8000000000UL
-#define UVH_GR0_TLB_MMR_READ_DATA_LO_VALID_MASK 0x8000000000000000UL
-
-#define UVXH_GR0_TLB_MMR_READ_DATA_LO_VPN_SHFT 0
-#define UVXH_GR0_TLB_MMR_READ_DATA_LO_ASID_SHFT 39
-#define UVXH_GR0_TLB_MMR_READ_DATA_LO_VALID_SHFT 63
-#define UVXH_GR0_TLB_MMR_READ_DATA_LO_VPN_MASK 0x0000007fffffffffUL
-#define UVXH_GR0_TLB_MMR_READ_DATA_LO_ASID_MASK 0x7fffff8000000000UL
-#define UVXH_GR0_TLB_MMR_READ_DATA_LO_VALID_MASK 0x8000000000000000UL
-
-#define UV2H_GR0_TLB_MMR_READ_DATA_LO_VPN_SHFT 0
-#define UV2H_GR0_TLB_MMR_READ_DATA_LO_ASID_SHFT 39
-#define UV2H_GR0_TLB_MMR_READ_DATA_LO_VALID_SHFT 63
-#define UV2H_GR0_TLB_MMR_READ_DATA_LO_VPN_MASK 0x0000007fffffffffUL
-#define UV2H_GR0_TLB_MMR_READ_DATA_LO_ASID_MASK 0x7fffff8000000000UL
-#define UV2H_GR0_TLB_MMR_READ_DATA_LO_VALID_MASK 0x8000000000000000UL
-
-#define UV3H_GR0_TLB_MMR_READ_DATA_LO_VPN_SHFT 0
-#define UV3H_GR0_TLB_MMR_READ_DATA_LO_ASID_SHFT 39
-#define UV3H_GR0_TLB_MMR_READ_DATA_LO_VALID_SHFT 63
-#define UV3H_GR0_TLB_MMR_READ_DATA_LO_VPN_MASK 0x0000007fffffffffUL
-#define UV3H_GR0_TLB_MMR_READ_DATA_LO_ASID_MASK 0x7fffff8000000000UL
-#define UV3H_GR0_TLB_MMR_READ_DATA_LO_VALID_MASK 0x8000000000000000UL
-
-#define UV4H_GR0_TLB_MMR_READ_DATA_LO_VPN_SHFT 0
-#define UV4H_GR0_TLB_MMR_READ_DATA_LO_ASID_SHFT 39
-#define UV4H_GR0_TLB_MMR_READ_DATA_LO_VALID_SHFT 63
-#define UV4H_GR0_TLB_MMR_READ_DATA_LO_VPN_MASK 0x0000007fffffffffUL
-#define UV4H_GR0_TLB_MMR_READ_DATA_LO_ASID_MASK 0x7fffff8000000000UL
-#define UV4H_GR0_TLB_MMR_READ_DATA_LO_VALID_MASK 0x8000000000000000UL
-
-
-union uvh_gr0_tlb_mmr_read_data_lo_u {
- unsigned long v;
- struct uvh_gr0_tlb_mmr_read_data_lo_s {
- unsigned long vpn:39; /* RO */
- unsigned long asid:24; /* RO */
- unsigned long valid:1; /* RO */
- } s;
- struct uvxh_gr0_tlb_mmr_read_data_lo_s {
- unsigned long vpn:39; /* RO */
- unsigned long asid:24; /* RO */
- unsigned long valid:1; /* RO */
- } sx;
- struct uv2h_gr0_tlb_mmr_read_data_lo_s {
- unsigned long vpn:39; /* RO */
- unsigned long asid:24; /* RO */
- unsigned long valid:1; /* RO */
+ /* UV2 unique struct */
+ struct uv2h_event_occurred1_s {
+ unsigned long bau_data:1; /* RW */
+ unsigned long power_management_req:1; /* RW */
+ unsigned long message_accelerator_int0:1; /* RW */
+ unsigned long message_accelerator_int1:1; /* RW */
+ unsigned long message_accelerator_int2:1; /* RW */
+ unsigned long message_accelerator_int3:1; /* RW */
+ unsigned long message_accelerator_int4:1; /* RW */
+ unsigned long message_accelerator_int5:1; /* RW */
+ unsigned long message_accelerator_int6:1; /* RW */
+ unsigned long message_accelerator_int7:1; /* RW */
+ unsigned long message_accelerator_int8:1; /* RW */
+ unsigned long message_accelerator_int9:1; /* RW */
+ unsigned long message_accelerator_int10:1; /* RW */
+ unsigned long message_accelerator_int11:1; /* RW */
+ unsigned long message_accelerator_int12:1; /* RW */
+ unsigned long message_accelerator_int13:1; /* RW */
+ unsigned long message_accelerator_int14:1; /* RW */
+ unsigned long message_accelerator_int15:1; /* RW */
+ unsigned long gr0_tlb_int0:1; /* RW */
+ unsigned long gr0_tlb_int1:1; /* RW */
+ unsigned long gr0_tlb_int2:1; /* RW */
+ unsigned long gr0_tlb_int3:1; /* RW */
+ unsigned long gr0_tlb_int4:1; /* RW */
+ unsigned long gr0_tlb_int5:1; /* RW */
+ unsigned long gr0_tlb_int6:1; /* RW */
+ unsigned long gr0_tlb_int7:1; /* RW */
+ unsigned long gr0_tlb_int8:1; /* RW */
+ unsigned long gr0_tlb_int9:1; /* RW */
+ unsigned long gr0_tlb_int10:1; /* RW */
+ unsigned long gr0_tlb_int11:1; /* RW */
+ unsigned long gr0_tlb_int12:1; /* RW */
+ unsigned long gr0_tlb_int13:1; /* RW */
+ unsigned long gr0_tlb_int14:1; /* RW */
+ unsigned long gr0_tlb_int15:1; /* RW */
+ unsigned long gr1_tlb_int0:1; /* RW */
+ unsigned long gr1_tlb_int1:1; /* RW */
+ unsigned long gr1_tlb_int2:1; /* RW */
+ unsigned long gr1_tlb_int3:1; /* RW */
+ unsigned long gr1_tlb_int4:1; /* RW */
+ unsigned long gr1_tlb_int5:1; /* RW */
+ unsigned long gr1_tlb_int6:1; /* RW */
+ unsigned long gr1_tlb_int7:1; /* RW */
+ unsigned long gr1_tlb_int8:1; /* RW */
+ unsigned long gr1_tlb_int9:1; /* RW */
+ unsigned long gr1_tlb_int10:1; /* RW */
+ unsigned long gr1_tlb_int11:1; /* RW */
+ unsigned long gr1_tlb_int12:1; /* RW */
+ unsigned long gr1_tlb_int13:1; /* RW */
+ unsigned long gr1_tlb_int14:1; /* RW */
+ unsigned long gr1_tlb_int15:1; /* RW */
+ unsigned long rtc_interval_int:1; /* RW */
+ unsigned long bau_dashboard_int:1; /* RW */
+ unsigned long rsvd_52_63:12;
} s2;
- struct uv3h_gr0_tlb_mmr_read_data_lo_s {
- unsigned long vpn:39; /* RO */
- unsigned long asid:24; /* RO */
- unsigned long valid:1; /* RO */
- } s3;
- struct uv4h_gr0_tlb_mmr_read_data_lo_s {
- unsigned long vpn:39; /* RO */
- unsigned long asid:24; /* RO */
- unsigned long valid:1; /* RO */
- } s4;
};
/* ========================================================================= */
-/* UVH_GR1_TLB_INT0_CONFIG */
+/* UVH_EVENT_OCCURRED1_ALIAS */
/* ========================================================================= */
-#define UV2H_GR1_TLB_INT0_CONFIG 0x61f00UL
-#define UV3H_GR1_TLB_INT0_CONFIG 0x61f00UL
-#define UV4H_GR1_TLB_INT0_CONFIG 0x62100UL
-#define UVH_GR1_TLB_INT0_CONFIG ( \
- is_uv2_hub() ? UV2H_GR1_TLB_INT0_CONFIG : \
- is_uv3_hub() ? UV3H_GR1_TLB_INT0_CONFIG : \
- /*is_uv4_hub*/ UV4H_GR1_TLB_INT0_CONFIG)
-
-#define UVH_GR1_TLB_INT0_CONFIG_VECTOR_SHFT 0
-#define UVH_GR1_TLB_INT0_CONFIG_DM_SHFT 8
-#define UVH_GR1_TLB_INT0_CONFIG_DESTMODE_SHFT 11
-#define UVH_GR1_TLB_INT0_CONFIG_STATUS_SHFT 12
-#define UVH_GR1_TLB_INT0_CONFIG_P_SHFT 13
-#define UVH_GR1_TLB_INT0_CONFIG_T_SHFT 15
-#define UVH_GR1_TLB_INT0_CONFIG_M_SHFT 16
-#define UVH_GR1_TLB_INT0_CONFIG_APIC_ID_SHFT 32
-#define UVH_GR1_TLB_INT0_CONFIG_VECTOR_MASK 0x00000000000000ffUL
-#define UVH_GR1_TLB_INT0_CONFIG_DM_MASK 0x0000000000000700UL
-#define UVH_GR1_TLB_INT0_CONFIG_DESTMODE_MASK 0x0000000000000800UL
-#define UVH_GR1_TLB_INT0_CONFIG_STATUS_MASK 0x0000000000001000UL
-#define UVH_GR1_TLB_INT0_CONFIG_P_MASK 0x0000000000002000UL
-#define UVH_GR1_TLB_INT0_CONFIG_T_MASK 0x0000000000008000UL
-#define UVH_GR1_TLB_INT0_CONFIG_M_MASK 0x0000000000010000UL
-#define UVH_GR1_TLB_INT0_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+#define UVH_EVENT_OCCURRED1_ALIAS 0x70088UL
-union uvh_gr1_tlb_int0_config_u {
- unsigned long v;
- struct uvh_gr1_tlb_int0_config_s {
- unsigned long vector_:8; /* RW */
- unsigned long dm:3; /* RW */
- unsigned long destmode:1; /* RW */
- unsigned long status:1; /* RO */
- unsigned long p:1; /* RO */
- unsigned long rsvd_14:1;
- unsigned long t:1; /* RO */
- unsigned long m:1; /* RW */
- unsigned long rsvd_17_31:15;
- unsigned long apic_id:32; /* RW */
- } s;
-};
-
/* ========================================================================= */
-/* UVH_GR1_TLB_INT1_CONFIG */
+/* UVH_EVENT_OCCURRED2 */
/* ========================================================================= */
-#define UV2H_GR1_TLB_INT1_CONFIG 0x61f40UL
-#define UV3H_GR1_TLB_INT1_CONFIG 0x61f40UL
-#define UV4H_GR1_TLB_INT1_CONFIG 0x62140UL
-#define UVH_GR1_TLB_INT1_CONFIG ( \
- is_uv2_hub() ? UV2H_GR1_TLB_INT1_CONFIG : \
- is_uv3_hub() ? UV3H_GR1_TLB_INT1_CONFIG : \
- /*is_uv4_hub*/ UV4H_GR1_TLB_INT1_CONFIG)
-
-#define UVH_GR1_TLB_INT1_CONFIG_VECTOR_SHFT 0
-#define UVH_GR1_TLB_INT1_CONFIG_DM_SHFT 8
-#define UVH_GR1_TLB_INT1_CONFIG_DESTMODE_SHFT 11
-#define UVH_GR1_TLB_INT1_CONFIG_STATUS_SHFT 12
-#define UVH_GR1_TLB_INT1_CONFIG_P_SHFT 13
-#define UVH_GR1_TLB_INT1_CONFIG_T_SHFT 15
-#define UVH_GR1_TLB_INT1_CONFIG_M_SHFT 16
-#define UVH_GR1_TLB_INT1_CONFIG_APIC_ID_SHFT 32
-#define UVH_GR1_TLB_INT1_CONFIG_VECTOR_MASK 0x00000000000000ffUL
-#define UVH_GR1_TLB_INT1_CONFIG_DM_MASK 0x0000000000000700UL
-#define UVH_GR1_TLB_INT1_CONFIG_DESTMODE_MASK 0x0000000000000800UL
-#define UVH_GR1_TLB_INT1_CONFIG_STATUS_MASK 0x0000000000001000UL
-#define UVH_GR1_TLB_INT1_CONFIG_P_MASK 0x0000000000002000UL
-#define UVH_GR1_TLB_INT1_CONFIG_T_MASK 0x0000000000008000UL
-#define UVH_GR1_TLB_INT1_CONFIG_M_MASK 0x0000000000010000UL
-#define UVH_GR1_TLB_INT1_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+#define UVH_EVENT_OCCURRED2 0x70100UL
+
+
+
+/* UVYH common defines */
+#define UVYH_EVENT_OCCURRED2_RTC_INTERVAL_INT_SHFT 0
+#define UVYH_EVENT_OCCURRED2_RTC_INTERVAL_INT_MASK 0x0000000000000001UL
+#define UVYH_EVENT_OCCURRED2_BAU_DASHBOARD_INT_SHFT 1
+#define UVYH_EVENT_OCCURRED2_BAU_DASHBOARD_INT_MASK 0x0000000000000002UL
+#define UVYH_EVENT_OCCURRED2_RTC_0_SHFT 2
+#define UVYH_EVENT_OCCURRED2_RTC_0_MASK 0x0000000000000004UL
+#define UVYH_EVENT_OCCURRED2_RTC_1_SHFT 3
+#define UVYH_EVENT_OCCURRED2_RTC_1_MASK 0x0000000000000008UL
+#define UVYH_EVENT_OCCURRED2_RTC_2_SHFT 4
+#define UVYH_EVENT_OCCURRED2_RTC_2_MASK 0x0000000000000010UL
+#define UVYH_EVENT_OCCURRED2_RTC_3_SHFT 5
+#define UVYH_EVENT_OCCURRED2_RTC_3_MASK 0x0000000000000020UL
+#define UVYH_EVENT_OCCURRED2_RTC_4_SHFT 6
+#define UVYH_EVENT_OCCURRED2_RTC_4_MASK 0x0000000000000040UL
+#define UVYH_EVENT_OCCURRED2_RTC_5_SHFT 7
+#define UVYH_EVENT_OCCURRED2_RTC_5_MASK 0x0000000000000080UL
+#define UVYH_EVENT_OCCURRED2_RTC_6_SHFT 8
+#define UVYH_EVENT_OCCURRED2_RTC_6_MASK 0x0000000000000100UL
+#define UVYH_EVENT_OCCURRED2_RTC_7_SHFT 9
+#define UVYH_EVENT_OCCURRED2_RTC_7_MASK 0x0000000000000200UL
+#define UVYH_EVENT_OCCURRED2_RTC_8_SHFT 10
+#define UVYH_EVENT_OCCURRED2_RTC_8_MASK 0x0000000000000400UL
+#define UVYH_EVENT_OCCURRED2_RTC_9_SHFT 11
+#define UVYH_EVENT_OCCURRED2_RTC_9_MASK 0x0000000000000800UL
+#define UVYH_EVENT_OCCURRED2_RTC_10_SHFT 12
+#define UVYH_EVENT_OCCURRED2_RTC_10_MASK 0x0000000000001000UL
+#define UVYH_EVENT_OCCURRED2_RTC_11_SHFT 13
+#define UVYH_EVENT_OCCURRED2_RTC_11_MASK 0x0000000000002000UL
+#define UVYH_EVENT_OCCURRED2_RTC_12_SHFT 14
+#define UVYH_EVENT_OCCURRED2_RTC_12_MASK 0x0000000000004000UL
+#define UVYH_EVENT_OCCURRED2_RTC_13_SHFT 15
+#define UVYH_EVENT_OCCURRED2_RTC_13_MASK 0x0000000000008000UL
+#define UVYH_EVENT_OCCURRED2_RTC_14_SHFT 16
+#define UVYH_EVENT_OCCURRED2_RTC_14_MASK 0x0000000000010000UL
+#define UVYH_EVENT_OCCURRED2_RTC_15_SHFT 17
+#define UVYH_EVENT_OCCURRED2_RTC_15_MASK 0x0000000000020000UL
+#define UVYH_EVENT_OCCURRED2_RTC_16_SHFT 18
+#define UVYH_EVENT_OCCURRED2_RTC_16_MASK 0x0000000000040000UL
+#define UVYH_EVENT_OCCURRED2_RTC_17_SHFT 19
+#define UVYH_EVENT_OCCURRED2_RTC_17_MASK 0x0000000000080000UL
+#define UVYH_EVENT_OCCURRED2_RTC_18_SHFT 20
+#define UVYH_EVENT_OCCURRED2_RTC_18_MASK 0x0000000000100000UL
+#define UVYH_EVENT_OCCURRED2_RTC_19_SHFT 21
+#define UVYH_EVENT_OCCURRED2_RTC_19_MASK 0x0000000000200000UL
+#define UVYH_EVENT_OCCURRED2_RTC_20_SHFT 22
+#define UVYH_EVENT_OCCURRED2_RTC_20_MASK 0x0000000000400000UL
+#define UVYH_EVENT_OCCURRED2_RTC_21_SHFT 23
+#define UVYH_EVENT_OCCURRED2_RTC_21_MASK 0x0000000000800000UL
+#define UVYH_EVENT_OCCURRED2_RTC_22_SHFT 24
+#define UVYH_EVENT_OCCURRED2_RTC_22_MASK 0x0000000001000000UL
+#define UVYH_EVENT_OCCURRED2_RTC_23_SHFT 25
+#define UVYH_EVENT_OCCURRED2_RTC_23_MASK 0x0000000002000000UL
+#define UVYH_EVENT_OCCURRED2_RTC_24_SHFT 26
+#define UVYH_EVENT_OCCURRED2_RTC_24_MASK 0x0000000004000000UL
+#define UVYH_EVENT_OCCURRED2_RTC_25_SHFT 27
+#define UVYH_EVENT_OCCURRED2_RTC_25_MASK 0x0000000008000000UL
+#define UVYH_EVENT_OCCURRED2_RTC_26_SHFT 28
+#define UVYH_EVENT_OCCURRED2_RTC_26_MASK 0x0000000010000000UL
+#define UVYH_EVENT_OCCURRED2_RTC_27_SHFT 29
+#define UVYH_EVENT_OCCURRED2_RTC_27_MASK 0x0000000020000000UL
+#define UVYH_EVENT_OCCURRED2_RTC_28_SHFT 30
+#define UVYH_EVENT_OCCURRED2_RTC_28_MASK 0x0000000040000000UL
+#define UVYH_EVENT_OCCURRED2_RTC_29_SHFT 31
+#define UVYH_EVENT_OCCURRED2_RTC_29_MASK 0x0000000080000000UL
+#define UVYH_EVENT_OCCURRED2_RTC_30_SHFT 32
+#define UVYH_EVENT_OCCURRED2_RTC_30_MASK 0x0000000100000000UL
+#define UVYH_EVENT_OCCURRED2_RTC_31_SHFT 33
+#define UVYH_EVENT_OCCURRED2_RTC_31_MASK 0x0000000200000000UL
+
+/* UV4 unique defines */
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT0_SHFT 0
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT0_MASK 0x0000000000000001UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT1_SHFT 1
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT1_MASK 0x0000000000000002UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT2_SHFT 2
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT2_MASK 0x0000000000000004UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT3_SHFT 3
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT3_MASK 0x0000000000000008UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT4_SHFT 4
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT4_MASK 0x0000000000000010UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT5_SHFT 5
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT5_MASK 0x0000000000000020UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT6_SHFT 6
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT6_MASK 0x0000000000000040UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT7_SHFT 7
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT7_MASK 0x0000000000000080UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT8_SHFT 8
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT8_MASK 0x0000000000000100UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT9_SHFT 9
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT9_MASK 0x0000000000000200UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT10_SHFT 10
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT10_MASK 0x0000000000000400UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT11_SHFT 11
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT11_MASK 0x0000000000000800UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT12_SHFT 12
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT12_MASK 0x0000000000001000UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT13_SHFT 13
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT13_MASK 0x0000000000002000UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT14_SHFT 14
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT14_MASK 0x0000000000004000UL
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT15_SHFT 15
+#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT15_MASK 0x0000000000008000UL
+#define UV4H_EVENT_OCCURRED2_RTC_INTERVAL_INT_SHFT 16
+#define UV4H_EVENT_OCCURRED2_RTC_INTERVAL_INT_MASK 0x0000000000010000UL
+#define UV4H_EVENT_OCCURRED2_BAU_DASHBOARD_INT_SHFT 17
+#define UV4H_EVENT_OCCURRED2_BAU_DASHBOARD_INT_MASK 0x0000000000020000UL
+#define UV4H_EVENT_OCCURRED2_RTC_0_SHFT 18
+#define UV4H_EVENT_OCCURRED2_RTC_0_MASK 0x0000000000040000UL
+#define UV4H_EVENT_OCCURRED2_RTC_1_SHFT 19
+#define UV4H_EVENT_OCCURRED2_RTC_1_MASK 0x0000000000080000UL
+#define UV4H_EVENT_OCCURRED2_RTC_2_SHFT 20
+#define UV4H_EVENT_OCCURRED2_RTC_2_MASK 0x0000000000100000UL
+#define UV4H_EVENT_OCCURRED2_RTC_3_SHFT 21
+#define UV4H_EVENT_OCCURRED2_RTC_3_MASK 0x0000000000200000UL
+#define UV4H_EVENT_OCCURRED2_RTC_4_SHFT 22
+#define UV4H_EVENT_OCCURRED2_RTC_4_MASK 0x0000000000400000UL
+#define UV4H_EVENT_OCCURRED2_RTC_5_SHFT 23
+#define UV4H_EVENT_OCCURRED2_RTC_5_MASK 0x0000000000800000UL
+#define UV4H_EVENT_OCCURRED2_RTC_6_SHFT 24
+#define UV4H_EVENT_OCCURRED2_RTC_6_MASK 0x0000000001000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_7_SHFT 25
+#define UV4H_EVENT_OCCURRED2_RTC_7_MASK 0x0000000002000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_8_SHFT 26
+#define UV4H_EVENT_OCCURRED2_RTC_8_MASK 0x0000000004000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_9_SHFT 27
+#define UV4H_EVENT_OCCURRED2_RTC_9_MASK 0x0000000008000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_10_SHFT 28
+#define UV4H_EVENT_OCCURRED2_RTC_10_MASK 0x0000000010000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_11_SHFT 29
+#define UV4H_EVENT_OCCURRED2_RTC_11_MASK 0x0000000020000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_12_SHFT 30
+#define UV4H_EVENT_OCCURRED2_RTC_12_MASK 0x0000000040000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_13_SHFT 31
+#define UV4H_EVENT_OCCURRED2_RTC_13_MASK 0x0000000080000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_14_SHFT 32
+#define UV4H_EVENT_OCCURRED2_RTC_14_MASK 0x0000000100000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_15_SHFT 33
+#define UV4H_EVENT_OCCURRED2_RTC_15_MASK 0x0000000200000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_16_SHFT 34
+#define UV4H_EVENT_OCCURRED2_RTC_16_MASK 0x0000000400000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_17_SHFT 35
+#define UV4H_EVENT_OCCURRED2_RTC_17_MASK 0x0000000800000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_18_SHFT 36
+#define UV4H_EVENT_OCCURRED2_RTC_18_MASK 0x0000001000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_19_SHFT 37
+#define UV4H_EVENT_OCCURRED2_RTC_19_MASK 0x0000002000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_20_SHFT 38
+#define UV4H_EVENT_OCCURRED2_RTC_20_MASK 0x0000004000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_21_SHFT 39
+#define UV4H_EVENT_OCCURRED2_RTC_21_MASK 0x0000008000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_22_SHFT 40
+#define UV4H_EVENT_OCCURRED2_RTC_22_MASK 0x0000010000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_23_SHFT 41
+#define UV4H_EVENT_OCCURRED2_RTC_23_MASK 0x0000020000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_24_SHFT 42
+#define UV4H_EVENT_OCCURRED2_RTC_24_MASK 0x0000040000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_25_SHFT 43
+#define UV4H_EVENT_OCCURRED2_RTC_25_MASK 0x0000080000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_26_SHFT 44
+#define UV4H_EVENT_OCCURRED2_RTC_26_MASK 0x0000100000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_27_SHFT 45
+#define UV4H_EVENT_OCCURRED2_RTC_27_MASK 0x0000200000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_28_SHFT 46
+#define UV4H_EVENT_OCCURRED2_RTC_28_MASK 0x0000400000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_29_SHFT 47
+#define UV4H_EVENT_OCCURRED2_RTC_29_MASK 0x0000800000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_30_SHFT 48
+#define UV4H_EVENT_OCCURRED2_RTC_30_MASK 0x0001000000000000UL
+#define UV4H_EVENT_OCCURRED2_RTC_31_SHFT 49
+#define UV4H_EVENT_OCCURRED2_RTC_31_MASK 0x0002000000000000UL
+
+/* UV3 unique defines */
+#define UV3H_EVENT_OCCURRED2_RTC_0_SHFT 0
+#define UV3H_EVENT_OCCURRED2_RTC_0_MASK 0x0000000000000001UL
+#define UV3H_EVENT_OCCURRED2_RTC_1_SHFT 1
+#define UV3H_EVENT_OCCURRED2_RTC_1_MASK 0x0000000000000002UL
+#define UV3H_EVENT_OCCURRED2_RTC_2_SHFT 2
+#define UV3H_EVENT_OCCURRED2_RTC_2_MASK 0x0000000000000004UL
+#define UV3H_EVENT_OCCURRED2_RTC_3_SHFT 3
+#define UV3H_EVENT_OCCURRED2_RTC_3_MASK 0x0000000000000008UL
+#define UV3H_EVENT_OCCURRED2_RTC_4_SHFT 4
+#define UV3H_EVENT_OCCURRED2_RTC_4_MASK 0x0000000000000010UL
+#define UV3H_EVENT_OCCURRED2_RTC_5_SHFT 5
+#define UV3H_EVENT_OCCURRED2_RTC_5_MASK 0x0000000000000020UL
+#define UV3H_EVENT_OCCURRED2_RTC_6_SHFT 6
+#define UV3H_EVENT_OCCURRED2_RTC_6_MASK 0x0000000000000040UL
+#define UV3H_EVENT_OCCURRED2_RTC_7_SHFT 7
+#define UV3H_EVENT_OCCURRED2_RTC_7_MASK 0x0000000000000080UL
+#define UV3H_EVENT_OCCURRED2_RTC_8_SHFT 8
+#define UV3H_EVENT_OCCURRED2_RTC_8_MASK 0x0000000000000100UL
+#define UV3H_EVENT_OCCURRED2_RTC_9_SHFT 9
+#define UV3H_EVENT_OCCURRED2_RTC_9_MASK 0x0000000000000200UL
+#define UV3H_EVENT_OCCURRED2_RTC_10_SHFT 10
+#define UV3H_EVENT_OCCURRED2_RTC_10_MASK 0x0000000000000400UL
+#define UV3H_EVENT_OCCURRED2_RTC_11_SHFT 11
+#define UV3H_EVENT_OCCURRED2_RTC_11_MASK 0x0000000000000800UL
+#define UV3H_EVENT_OCCURRED2_RTC_12_SHFT 12
+#define UV3H_EVENT_OCCURRED2_RTC_12_MASK 0x0000000000001000UL
+#define UV3H_EVENT_OCCURRED2_RTC_13_SHFT 13
+#define UV3H_EVENT_OCCURRED2_RTC_13_MASK 0x0000000000002000UL
+#define UV3H_EVENT_OCCURRED2_RTC_14_SHFT 14
+#define UV3H_EVENT_OCCURRED2_RTC_14_MASK 0x0000000000004000UL
+#define UV3H_EVENT_OCCURRED2_RTC_15_SHFT 15
+#define UV3H_EVENT_OCCURRED2_RTC_15_MASK 0x0000000000008000UL
+#define UV3H_EVENT_OCCURRED2_RTC_16_SHFT 16
+#define UV3H_EVENT_OCCURRED2_RTC_16_MASK 0x0000000000010000UL
+#define UV3H_EVENT_OCCURRED2_RTC_17_SHFT 17
+#define UV3H_EVENT_OCCURRED2_RTC_17_MASK 0x0000000000020000UL
+#define UV3H_EVENT_OCCURRED2_RTC_18_SHFT 18
+#define UV3H_EVENT_OCCURRED2_RTC_18_MASK 0x0000000000040000UL
+#define UV3H_EVENT_OCCURRED2_RTC_19_SHFT 19
+#define UV3H_EVENT_OCCURRED2_RTC_19_MASK 0x0000000000080000UL
+#define UV3H_EVENT_OCCURRED2_RTC_20_SHFT 20
+#define UV3H_EVENT_OCCURRED2_RTC_20_MASK 0x0000000000100000UL
+#define UV3H_EVENT_OCCURRED2_RTC_21_SHFT 21
+#define UV3H_EVENT_OCCURRED2_RTC_21_MASK 0x0000000000200000UL
+#define UV3H_EVENT_OCCURRED2_RTC_22_SHFT 22
+#define UV3H_EVENT_OCCURRED2_RTC_22_MASK 0x0000000000400000UL
+#define UV3H_EVENT_OCCURRED2_RTC_23_SHFT 23
+#define UV3H_EVENT_OCCURRED2_RTC_23_MASK 0x0000000000800000UL
+#define UV3H_EVENT_OCCURRED2_RTC_24_SHFT 24
+#define UV3H_EVENT_OCCURRED2_RTC_24_MASK 0x0000000001000000UL
+#define UV3H_EVENT_OCCURRED2_RTC_25_SHFT 25
+#define UV3H_EVENT_OCCURRED2_RTC_25_MASK 0x0000000002000000UL
+#define UV3H_EVENT_OCCURRED2_RTC_26_SHFT 26
+#define UV3H_EVENT_OCCURRED2_RTC_26_MASK 0x0000000004000000UL
+#define UV3H_EVENT_OCCURRED2_RTC_27_SHFT 27
+#define UV3H_EVENT_OCCURRED2_RTC_27_MASK 0x0000000008000000UL
+#define UV3H_EVENT_OCCURRED2_RTC_28_SHFT 28
+#define UV3H_EVENT_OCCURRED2_RTC_28_MASK 0x0000000010000000UL
+#define UV3H_EVENT_OCCURRED2_RTC_29_SHFT 29
+#define UV3H_EVENT_OCCURRED2_RTC_29_MASK 0x0000000020000000UL
+#define UV3H_EVENT_OCCURRED2_RTC_30_SHFT 30
+#define UV3H_EVENT_OCCURRED2_RTC_30_MASK 0x0000000040000000UL
+#define UV3H_EVENT_OCCURRED2_RTC_31_SHFT 31
+#define UV3H_EVENT_OCCURRED2_RTC_31_MASK 0x0000000080000000UL
+
+/* UV2 unique defines */
+#define UV2H_EVENT_OCCURRED2_RTC_0_SHFT 0
+#define UV2H_EVENT_OCCURRED2_RTC_0_MASK 0x0000000000000001UL
+#define UV2H_EVENT_OCCURRED2_RTC_1_SHFT 1
+#define UV2H_EVENT_OCCURRED2_RTC_1_MASK 0x0000000000000002UL
+#define UV2H_EVENT_OCCURRED2_RTC_2_SHFT 2
+#define UV2H_EVENT_OCCURRED2_RTC_2_MASK 0x0000000000000004UL
+#define UV2H_EVENT_OCCURRED2_RTC_3_SHFT 3
+#define UV2H_EVENT_OCCURRED2_RTC_3_MASK 0x0000000000000008UL
+#define UV2H_EVENT_OCCURRED2_RTC_4_SHFT 4
+#define UV2H_EVENT_OCCURRED2_RTC_4_MASK 0x0000000000000010UL
+#define UV2H_EVENT_OCCURRED2_RTC_5_SHFT 5
+#define UV2H_EVENT_OCCURRED2_RTC_5_MASK 0x0000000000000020UL
+#define UV2H_EVENT_OCCURRED2_RTC_6_SHFT 6
+#define UV2H_EVENT_OCCURRED2_RTC_6_MASK 0x0000000000000040UL
+#define UV2H_EVENT_OCCURRED2_RTC_7_SHFT 7
+#define UV2H_EVENT_OCCURRED2_RTC_7_MASK 0x0000000000000080UL
+#define UV2H_EVENT_OCCURRED2_RTC_8_SHFT 8
+#define UV2H_EVENT_OCCURRED2_RTC_8_MASK 0x0000000000000100UL
+#define UV2H_EVENT_OCCURRED2_RTC_9_SHFT 9
+#define UV2H_EVENT_OCCURRED2_RTC_9_MASK 0x0000000000000200UL
+#define UV2H_EVENT_OCCURRED2_RTC_10_SHFT 10
+#define UV2H_EVENT_OCCURRED2_RTC_10_MASK 0x0000000000000400UL
+#define UV2H_EVENT_OCCURRED2_RTC_11_SHFT 11
+#define UV2H_EVENT_OCCURRED2_RTC_11_MASK 0x0000000000000800UL
+#define UV2H_EVENT_OCCURRED2_RTC_12_SHFT 12
+#define UV2H_EVENT_OCCURRED2_RTC_12_MASK 0x0000000000001000UL
+#define UV2H_EVENT_OCCURRED2_RTC_13_SHFT 13
+#define UV2H_EVENT_OCCURRED2_RTC_13_MASK 0x0000000000002000UL
+#define UV2H_EVENT_OCCURRED2_RTC_14_SHFT 14
+#define UV2H_EVENT_OCCURRED2_RTC_14_MASK 0x0000000000004000UL
+#define UV2H_EVENT_OCCURRED2_RTC_15_SHFT 15
+#define UV2H_EVENT_OCCURRED2_RTC_15_MASK 0x0000000000008000UL
+#define UV2H_EVENT_OCCURRED2_RTC_16_SHFT 16
+#define UV2H_EVENT_OCCURRED2_RTC_16_MASK 0x0000000000010000UL
+#define UV2H_EVENT_OCCURRED2_RTC_17_SHFT 17
+#define UV2H_EVENT_OCCURRED2_RTC_17_MASK 0x0000000000020000UL
+#define UV2H_EVENT_OCCURRED2_RTC_18_SHFT 18
+#define UV2H_EVENT_OCCURRED2_RTC_18_MASK 0x0000000000040000UL
+#define UV2H_EVENT_OCCURRED2_RTC_19_SHFT 19
+#define UV2H_EVENT_OCCURRED2_RTC_19_MASK 0x0000000000080000UL
+#define UV2H_EVENT_OCCURRED2_RTC_20_SHFT 20
+#define UV2H_EVENT_OCCURRED2_RTC_20_MASK 0x0000000000100000UL
+#define UV2H_EVENT_OCCURRED2_RTC_21_SHFT 21
+#define UV2H_EVENT_OCCURRED2_RTC_21_MASK 0x0000000000200000UL
+#define UV2H_EVENT_OCCURRED2_RTC_22_SHFT 22
+#define UV2H_EVENT_OCCURRED2_RTC_22_MASK 0x0000000000400000UL
+#define UV2H_EVENT_OCCURRED2_RTC_23_SHFT 23
+#define UV2H_EVENT_OCCURRED2_RTC_23_MASK 0x0000000000800000UL
+#define UV2H_EVENT_OCCURRED2_RTC_24_SHFT 24
+#define UV2H_EVENT_OCCURRED2_RTC_24_MASK 0x0000000001000000UL
+#define UV2H_EVENT_OCCURRED2_RTC_25_SHFT 25
+#define UV2H_EVENT_OCCURRED2_RTC_25_MASK 0x0000000002000000UL
+#define UV2H_EVENT_OCCURRED2_RTC_26_SHFT 26
+#define UV2H_EVENT_OCCURRED2_RTC_26_MASK 0x0000000004000000UL
+#define UV2H_EVENT_OCCURRED2_RTC_27_SHFT 27
+#define UV2H_EVENT_OCCURRED2_RTC_27_MASK 0x0000000008000000UL
+#define UV2H_EVENT_OCCURRED2_RTC_28_SHFT 28
+#define UV2H_EVENT_OCCURRED2_RTC_28_MASK 0x0000000010000000UL
+#define UV2H_EVENT_OCCURRED2_RTC_29_SHFT 29
+#define UV2H_EVENT_OCCURRED2_RTC_29_MASK 0x0000000020000000UL
+#define UV2H_EVENT_OCCURRED2_RTC_30_SHFT 30
+#define UV2H_EVENT_OCCURRED2_RTC_30_MASK 0x0000000040000000UL
+#define UV2H_EVENT_OCCURRED2_RTC_31_SHFT 31
+#define UV2H_EVENT_OCCURRED2_RTC_31_MASK 0x0000000080000000UL
+
+#define UVH_EVENT_OCCURRED2_RTC_1_MASK ( \
+ is_uv(UV5) ? 0x0000000000000008UL : \
+ is_uv(UV4) ? 0x0000000000080000UL : \
+ is_uv(UV3) ? 0x0000000000000002UL : \
+ is_uv(UV2) ? 0x0000000000000002UL : \
+ 0)
+#define UVH_EVENT_OCCURRED2_RTC_1_SHFT ( \
+ is_uv(UV5) ? 3 : \
+ is_uv(UV4) ? 19 : \
+ is_uv(UV3) ? 1 : \
+ is_uv(UV2) ? 1 : \
+ -1)
+
+union uvyh_event_occurred2_u {
+ unsigned long v;
+
+ /* UVYH common struct */
+ struct uvyh_event_occurred2_s {
+ unsigned long rtc_interval_int:1; /* RW */
+ unsigned long bau_dashboard_int:1; /* RW */
+ unsigned long rtc_0:1; /* RW */
+ unsigned long rtc_1:1; /* RW */
+ unsigned long rtc_2:1; /* RW */
+ unsigned long rtc_3:1; /* RW */
+ unsigned long rtc_4:1; /* RW */
+ unsigned long rtc_5:1; /* RW */
+ unsigned long rtc_6:1; /* RW */
+ unsigned long rtc_7:1; /* RW */
+ unsigned long rtc_8:1; /* RW */
+ unsigned long rtc_9:1; /* RW */
+ unsigned long rtc_10:1; /* RW */
+ unsigned long rtc_11:1; /* RW */
+ unsigned long rtc_12:1; /* RW */
+ unsigned long rtc_13:1; /* RW */
+ unsigned long rtc_14:1; /* RW */
+ unsigned long rtc_15:1; /* RW */
+ unsigned long rtc_16:1; /* RW */
+ unsigned long rtc_17:1; /* RW */
+ unsigned long rtc_18:1; /* RW */
+ unsigned long rtc_19:1; /* RW */
+ unsigned long rtc_20:1; /* RW */
+ unsigned long rtc_21:1; /* RW */
+ unsigned long rtc_22:1; /* RW */
+ unsigned long rtc_23:1; /* RW */
+ unsigned long rtc_24:1; /* RW */
+ unsigned long rtc_25:1; /* RW */
+ unsigned long rtc_26:1; /* RW */
+ unsigned long rtc_27:1; /* RW */
+ unsigned long rtc_28:1; /* RW */
+ unsigned long rtc_29:1; /* RW */
+ unsigned long rtc_30:1; /* RW */
+ unsigned long rtc_31:1; /* RW */
+ unsigned long rsvd_34_63:30;
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_event_occurred2_s {
+ unsigned long rtc_interval_int:1; /* RW */
+ unsigned long bau_dashboard_int:1; /* RW */
+ unsigned long rtc_0:1; /* RW */
+ unsigned long rtc_1:1; /* RW */
+ unsigned long rtc_2:1; /* RW */
+ unsigned long rtc_3:1; /* RW */
+ unsigned long rtc_4:1; /* RW */
+ unsigned long rtc_5:1; /* RW */
+ unsigned long rtc_6:1; /* RW */
+ unsigned long rtc_7:1; /* RW */
+ unsigned long rtc_8:1; /* RW */
+ unsigned long rtc_9:1; /* RW */
+ unsigned long rtc_10:1; /* RW */
+ unsigned long rtc_11:1; /* RW */
+ unsigned long rtc_12:1; /* RW */
+ unsigned long rtc_13:1; /* RW */
+ unsigned long rtc_14:1; /* RW */
+ unsigned long rtc_15:1; /* RW */
+ unsigned long rtc_16:1; /* RW */
+ unsigned long rtc_17:1; /* RW */
+ unsigned long rtc_18:1; /* RW */
+ unsigned long rtc_19:1; /* RW */
+ unsigned long rtc_20:1; /* RW */
+ unsigned long rtc_21:1; /* RW */
+ unsigned long rtc_22:1; /* RW */
+ unsigned long rtc_23:1; /* RW */
+ unsigned long rtc_24:1; /* RW */
+ unsigned long rtc_25:1; /* RW */
+ unsigned long rtc_26:1; /* RW */
+ unsigned long rtc_27:1; /* RW */
+ unsigned long rtc_28:1; /* RW */
+ unsigned long rtc_29:1; /* RW */
+ unsigned long rtc_30:1; /* RW */
+ unsigned long rtc_31:1; /* RW */
+ unsigned long rsvd_34_63:30;
+ } s5;
+
+ /* UV4 unique struct */
+ struct uv4h_event_occurred2_s {
+ unsigned long message_accelerator_int0:1; /* RW */
+ unsigned long message_accelerator_int1:1; /* RW */
+ unsigned long message_accelerator_int2:1; /* RW */
+ unsigned long message_accelerator_int3:1; /* RW */
+ unsigned long message_accelerator_int4:1; /* RW */
+ unsigned long message_accelerator_int5:1; /* RW */
+ unsigned long message_accelerator_int6:1; /* RW */
+ unsigned long message_accelerator_int7:1; /* RW */
+ unsigned long message_accelerator_int8:1; /* RW */
+ unsigned long message_accelerator_int9:1; /* RW */
+ unsigned long message_accelerator_int10:1; /* RW */
+ unsigned long message_accelerator_int11:1; /* RW */
+ unsigned long message_accelerator_int12:1; /* RW */
+ unsigned long message_accelerator_int13:1; /* RW */
+ unsigned long message_accelerator_int14:1; /* RW */
+ unsigned long message_accelerator_int15:1; /* RW */
+ unsigned long rtc_interval_int:1; /* RW */
+ unsigned long bau_dashboard_int:1; /* RW */
+ unsigned long rtc_0:1; /* RW */
+ unsigned long rtc_1:1; /* RW */
+ unsigned long rtc_2:1; /* RW */
+ unsigned long rtc_3:1; /* RW */
+ unsigned long rtc_4:1; /* RW */
+ unsigned long rtc_5:1; /* RW */
+ unsigned long rtc_6:1; /* RW */
+ unsigned long rtc_7:1; /* RW */
+ unsigned long rtc_8:1; /* RW */
+ unsigned long rtc_9:1; /* RW */
+ unsigned long rtc_10:1; /* RW */
+ unsigned long rtc_11:1; /* RW */
+ unsigned long rtc_12:1; /* RW */
+ unsigned long rtc_13:1; /* RW */
+ unsigned long rtc_14:1; /* RW */
+ unsigned long rtc_15:1; /* RW */
+ unsigned long rtc_16:1; /* RW */
+ unsigned long rtc_17:1; /* RW */
+ unsigned long rtc_18:1; /* RW */
+ unsigned long rtc_19:1; /* RW */
+ unsigned long rtc_20:1; /* RW */
+ unsigned long rtc_21:1; /* RW */
+ unsigned long rtc_22:1; /* RW */
+ unsigned long rtc_23:1; /* RW */
+ unsigned long rtc_24:1; /* RW */
+ unsigned long rtc_25:1; /* RW */
+ unsigned long rtc_26:1; /* RW */
+ unsigned long rtc_27:1; /* RW */
+ unsigned long rtc_28:1; /* RW */
+ unsigned long rtc_29:1; /* RW */
+ unsigned long rtc_30:1; /* RW */
+ unsigned long rtc_31:1; /* RW */
+ unsigned long rsvd_50_63:14;
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_event_occurred2_s {
+ unsigned long rtc_0:1; /* RW */
+ unsigned long rtc_1:1; /* RW */
+ unsigned long rtc_2:1; /* RW */
+ unsigned long rtc_3:1; /* RW */
+ unsigned long rtc_4:1; /* RW */
+ unsigned long rtc_5:1; /* RW */
+ unsigned long rtc_6:1; /* RW */
+ unsigned long rtc_7:1; /* RW */
+ unsigned long rtc_8:1; /* RW */
+ unsigned long rtc_9:1; /* RW */
+ unsigned long rtc_10:1; /* RW */
+ unsigned long rtc_11:1; /* RW */
+ unsigned long rtc_12:1; /* RW */
+ unsigned long rtc_13:1; /* RW */
+ unsigned long rtc_14:1; /* RW */
+ unsigned long rtc_15:1; /* RW */
+ unsigned long rtc_16:1; /* RW */
+ unsigned long rtc_17:1; /* RW */
+ unsigned long rtc_18:1; /* RW */
+ unsigned long rtc_19:1; /* RW */
+ unsigned long rtc_20:1; /* RW */
+ unsigned long rtc_21:1; /* RW */
+ unsigned long rtc_22:1; /* RW */
+ unsigned long rtc_23:1; /* RW */
+ unsigned long rtc_24:1; /* RW */
+ unsigned long rtc_25:1; /* RW */
+ unsigned long rtc_26:1; /* RW */
+ unsigned long rtc_27:1; /* RW */
+ unsigned long rtc_28:1; /* RW */
+ unsigned long rtc_29:1; /* RW */
+ unsigned long rtc_30:1; /* RW */
+ unsigned long rtc_31:1; /* RW */
+ unsigned long rsvd_32_63:32;
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_event_occurred2_s {
+ unsigned long rtc_0:1; /* RW */
+ unsigned long rtc_1:1; /* RW */
+ unsigned long rtc_2:1; /* RW */
+ unsigned long rtc_3:1; /* RW */
+ unsigned long rtc_4:1; /* RW */
+ unsigned long rtc_5:1; /* RW */
+ unsigned long rtc_6:1; /* RW */
+ unsigned long rtc_7:1; /* RW */
+ unsigned long rtc_8:1; /* RW */
+ unsigned long rtc_9:1; /* RW */
+ unsigned long rtc_10:1; /* RW */
+ unsigned long rtc_11:1; /* RW */
+ unsigned long rtc_12:1; /* RW */
+ unsigned long rtc_13:1; /* RW */
+ unsigned long rtc_14:1; /* RW */
+ unsigned long rtc_15:1; /* RW */
+ unsigned long rtc_16:1; /* RW */
+ unsigned long rtc_17:1; /* RW */
+ unsigned long rtc_18:1; /* RW */
+ unsigned long rtc_19:1; /* RW */
+ unsigned long rtc_20:1; /* RW */
+ unsigned long rtc_21:1; /* RW */
+ unsigned long rtc_22:1; /* RW */
+ unsigned long rtc_23:1; /* RW */
+ unsigned long rtc_24:1; /* RW */
+ unsigned long rtc_25:1; /* RW */
+ unsigned long rtc_26:1; /* RW */
+ unsigned long rtc_27:1; /* RW */
+ unsigned long rtc_28:1; /* RW */
+ unsigned long rtc_29:1; /* RW */
+ unsigned long rtc_30:1; /* RW */
+ unsigned long rtc_31:1; /* RW */
+ unsigned long rsvd_32_63:32;
+ } s2;
+};
+
+/* ========================================================================= */
+/* UVH_EVENT_OCCURRED2_ALIAS */
+/* ========================================================================= */
+#define UVH_EVENT_OCCURRED2_ALIAS 0x70108UL
+
+
+/* ========================================================================= */
+/* UVH_EXTIO_INT0_BROADCAST */
+/* ========================================================================= */
+#define UVH_EXTIO_INT0_BROADCAST 0x61448UL
+
+/* UVH common defines*/
+#define UVH_EXTIO_INT0_BROADCAST_ENABLE_SHFT 0
+#define UVH_EXTIO_INT0_BROADCAST_ENABLE_MASK 0x0000000000000001UL
+
+
+union uvh_extio_int0_broadcast_u {
+ unsigned long v;
+
+ /* UVH common struct */
+ struct uvh_extio_int0_broadcast_s {
+ unsigned long enable:1; /* RW */
+ unsigned long rsvd_1_63:63;
+ } s;
+
+ /* UV5 unique struct */
+ struct uv5h_extio_int0_broadcast_s {
+ unsigned long enable:1; /* RW */
+ unsigned long rsvd_1_63:63;
+ } s5;
+
+ /* UV4 unique struct */
+ struct uv4h_extio_int0_broadcast_s {
+ unsigned long enable:1; /* RW */
+ unsigned long rsvd_1_63:63;
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_extio_int0_broadcast_s {
+ unsigned long enable:1; /* RW */
+ unsigned long rsvd_1_63:63;
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_extio_int0_broadcast_s {
+ unsigned long enable:1; /* RW */
+ unsigned long rsvd_1_63:63;
+ } s2;
+};
+
+/* ========================================================================= */
+/* UVH_GR0_GAM_GR_CONFIG */
+/* ========================================================================= */
+#define UVH_GR0_GAM_GR_CONFIG ( \
+ is_uv(UV5) ? 0x600028UL : \
+ is_uv(UV4) ? 0x600028UL : \
+ is_uv(UV3) ? 0xc00028UL : \
+ is_uv(UV2) ? 0xc00028UL : \
+ 0)
+
+
+
+/* UVYH common defines */
+#define UVYH_GR0_GAM_GR_CONFIG_SUBSPACE_SHFT 10
+#define UVYH_GR0_GAM_GR_CONFIG_SUBSPACE_MASK 0x0000000000000400UL
+
+/* UV4 unique defines */
+#define UV4H_GR0_GAM_GR_CONFIG_SUBSPACE_SHFT 10
+#define UV4H_GR0_GAM_GR_CONFIG_SUBSPACE_MASK 0x0000000000000400UL
+
+/* UV3 unique defines */
+#define UV3H_GR0_GAM_GR_CONFIG_M_SKT_SHFT 0
+#define UV3H_GR0_GAM_GR_CONFIG_M_SKT_MASK 0x000000000000003fUL
+#define UV3H_GR0_GAM_GR_CONFIG_SUBSPACE_SHFT 10
+#define UV3H_GR0_GAM_GR_CONFIG_SUBSPACE_MASK 0x0000000000000400UL
+
+/* UV2 unique defines */
+#define UV2H_GR0_GAM_GR_CONFIG_N_GR_SHFT 0
+#define UV2H_GR0_GAM_GR_CONFIG_N_GR_MASK 0x000000000000000fUL
+
+
+union uvyh_gr0_gam_gr_config_u {
+ unsigned long v;
+ /* UVYH common struct */
+ struct uvyh_gr0_gam_gr_config_s {
+ unsigned long rsvd_0_9:10;
+ unsigned long subspace:1; /* RW */
+ unsigned long rsvd_11_63:53;
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_gr0_gam_gr_config_s {
+ unsigned long rsvd_0_9:10;
+ unsigned long subspace:1; /* RW */
+ unsigned long rsvd_11_63:53;
+ } s5;
+
+ /* UV4 unique struct */
+ struct uv4h_gr0_gam_gr_config_s {
+ unsigned long rsvd_0_9:10;
+ unsigned long subspace:1; /* RW */
+ unsigned long rsvd_11_63:53;
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_gr0_gam_gr_config_s {
+ unsigned long m_skt:6; /* RW */
+ unsigned long undef_6_9:4; /* Undefined */
+ unsigned long subspace:1; /* RW */
+ unsigned long reserved:53;
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_gr0_gam_gr_config_s {
+ unsigned long n_gr:4; /* RW */
+ unsigned long reserved:60;
+ } s2;
+};
+
+/* ========================================================================= */
+/* UVH_GR0_TLB_INT0_CONFIG */
+/* ========================================================================= */
+#define UVH_GR0_TLB_INT0_CONFIG ( \
+ is_uv(UV4) ? 0x61b00UL : \
+ is_uv(UV3) ? 0x61b00UL : \
+ is_uv(UV2) ? 0x61b00UL : \
+ uv_undefined("UVH_GR0_TLB_INT0_CONFIG"))
+
+
+/* UVXH common defines */
+#define UVXH_GR0_TLB_INT0_CONFIG_VECTOR_SHFT 0
+#define UVXH_GR0_TLB_INT0_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVXH_GR0_TLB_INT0_CONFIG_DM_SHFT 8
+#define UVXH_GR0_TLB_INT0_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVXH_GR0_TLB_INT0_CONFIG_DESTMODE_SHFT 11
+#define UVXH_GR0_TLB_INT0_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVXH_GR0_TLB_INT0_CONFIG_STATUS_SHFT 12
+#define UVXH_GR0_TLB_INT0_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVXH_GR0_TLB_INT0_CONFIG_P_SHFT 13
+#define UVXH_GR0_TLB_INT0_CONFIG_P_MASK 0x0000000000002000UL
+#define UVXH_GR0_TLB_INT0_CONFIG_T_SHFT 15
+#define UVXH_GR0_TLB_INT0_CONFIG_T_MASK 0x0000000000008000UL
+#define UVXH_GR0_TLB_INT0_CONFIG_M_SHFT 16
+#define UVXH_GR0_TLB_INT0_CONFIG_M_MASK 0x0000000000010000UL
+#define UVXH_GR0_TLB_INT0_CONFIG_APIC_ID_SHFT 32
+#define UVXH_GR0_TLB_INT0_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+
+
+union uvh_gr0_tlb_int0_config_u {
+ unsigned long v;
+
+ /* UVH common struct */
+ struct uvh_gr0_tlb_int0_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s;
+
+ /* UVXH common struct */
+ struct uvxh_gr0_tlb_int0_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } sx;
+
+ /* UV4 unique struct */
+ struct uv4h_gr0_tlb_int0_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_gr0_tlb_int0_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_gr0_tlb_int0_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s2;
+};
+
+/* ========================================================================= */
+/* UVH_GR0_TLB_INT1_CONFIG */
+/* ========================================================================= */
+#define UVH_GR0_TLB_INT1_CONFIG ( \
+ is_uv(UV4) ? 0x61b40UL : \
+ is_uv(UV3) ? 0x61b40UL : \
+ is_uv(UV2) ? 0x61b40UL : \
+ uv_undefined("UVH_GR0_TLB_INT1_CONFIG"))
+
+
+/* UVXH common defines */
+#define UVXH_GR0_TLB_INT1_CONFIG_VECTOR_SHFT 0
+#define UVXH_GR0_TLB_INT1_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVXH_GR0_TLB_INT1_CONFIG_DM_SHFT 8
+#define UVXH_GR0_TLB_INT1_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVXH_GR0_TLB_INT1_CONFIG_DESTMODE_SHFT 11
+#define UVXH_GR0_TLB_INT1_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVXH_GR0_TLB_INT1_CONFIG_STATUS_SHFT 12
+#define UVXH_GR0_TLB_INT1_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVXH_GR0_TLB_INT1_CONFIG_P_SHFT 13
+#define UVXH_GR0_TLB_INT1_CONFIG_P_MASK 0x0000000000002000UL
+#define UVXH_GR0_TLB_INT1_CONFIG_T_SHFT 15
+#define UVXH_GR0_TLB_INT1_CONFIG_T_MASK 0x0000000000008000UL
+#define UVXH_GR0_TLB_INT1_CONFIG_M_SHFT 16
+#define UVXH_GR0_TLB_INT1_CONFIG_M_MASK 0x0000000000010000UL
+#define UVXH_GR0_TLB_INT1_CONFIG_APIC_ID_SHFT 32
+#define UVXH_GR0_TLB_INT1_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+
+
+union uvh_gr0_tlb_int1_config_u {
+ unsigned long v;
+
+ /* UVH common struct */
+ struct uvh_gr0_tlb_int1_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s;
+
+ /* UVXH common struct */
+ struct uvxh_gr0_tlb_int1_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } sx;
+
+ /* UV4 unique struct */
+ struct uv4h_gr0_tlb_int1_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s4;
-union uvh_gr1_tlb_int1_config_u {
- unsigned long v;
- struct uvh_gr1_tlb_int1_config_s {
+ /* UV3 unique struct */
+ struct uv3h_gr0_tlb_int1_config_s {
unsigned long vector_:8; /* RW */
unsigned long dm:3; /* RW */
unsigned long destmode:1; /* RW */
unsigned long m:1; /* RW */
unsigned long rsvd_17_31:15;
unsigned long apic_id:32; /* RW */
- } s;
-};
-
-/* ========================================================================= */
-/* UVH_GR1_TLB_MMR_CONTROL */
-/* ========================================================================= */
-#define UV2H_GR1_TLB_MMR_CONTROL 0x1001080UL
-#define UV3H_GR1_TLB_MMR_CONTROL 0x1001080UL
-#define UV4H_GR1_TLB_MMR_CONTROL 0x701080UL
-#define UVH_GR1_TLB_MMR_CONTROL ( \
- is_uv2_hub() ? UV2H_GR1_TLB_MMR_CONTROL : \
- is_uv3_hub() ? UV3H_GR1_TLB_MMR_CONTROL : \
- /*is_uv4_hub*/ UV4H_GR1_TLB_MMR_CONTROL)
-
-#define UVH_GR1_TLB_MMR_CONTROL_INDEX_SHFT 0
-#define UVH_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT 16
-#define UVH_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
-#define UVH_GR1_TLB_MMR_CONTROL_MMR_WRITE_SHFT 30
-#define UVH_GR1_TLB_MMR_CONTROL_MMR_READ_SHFT 31
-#define UVH_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK 0x0000000000010000UL
-#define UVH_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
-#define UVH_GR1_TLB_MMR_CONTROL_MMR_WRITE_MASK 0x0000000040000000UL
-#define UVH_GR1_TLB_MMR_CONTROL_MMR_READ_MASK 0x0000000080000000UL
-
-#define UVXH_GR1_TLB_MMR_CONTROL_INDEX_SHFT 0
-#define UVXH_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT 16
-#define UVXH_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
-#define UVXH_GR1_TLB_MMR_CONTROL_MMR_WRITE_SHFT 30
-#define UVXH_GR1_TLB_MMR_CONTROL_MMR_READ_SHFT 31
-#define UVXH_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT 32
-#define UVXH_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK 0x0000000000010000UL
-#define UVXH_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
-#define UVXH_GR1_TLB_MMR_CONTROL_MMR_WRITE_MASK 0x0000000040000000UL
-#define UVXH_GR1_TLB_MMR_CONTROL_MMR_READ_MASK 0x0000000080000000UL
-#define UVXH_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_MASK 0x0000000100000000UL
-
-#define UV2H_GR1_TLB_MMR_CONTROL_INDEX_SHFT 0
-#define UV2H_GR1_TLB_MMR_CONTROL_MEM_SEL_SHFT 12
-#define UV2H_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT 16
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_WRITE_SHFT 30
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_READ_SHFT 31
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT 32
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_INJ_CON_SHFT 48
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_INJ_TLBRAM_SHFT 52
-#define UV2H_GR1_TLB_MMR_CONTROL_INDEX_MASK 0x0000000000000fffUL
-#define UV2H_GR1_TLB_MMR_CONTROL_MEM_SEL_MASK 0x0000000000003000UL
-#define UV2H_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK 0x0000000000010000UL
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_WRITE_MASK 0x0000000040000000UL
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_READ_MASK 0x0000000080000000UL
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_MASK 0x0000000100000000UL
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_INJ_CON_MASK 0x0001000000000000UL
-#define UV2H_GR1_TLB_MMR_CONTROL_MMR_INJ_TLBRAM_MASK 0x0010000000000000UL
-
-#define UV3H_GR1_TLB_MMR_CONTROL_INDEX_SHFT 0
-#define UV3H_GR1_TLB_MMR_CONTROL_MEM_SEL_SHFT 12
-#define UV3H_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT 16
-#define UV3H_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
-#define UV3H_GR1_TLB_MMR_CONTROL_ECC_SEL_SHFT 21
-#define UV3H_GR1_TLB_MMR_CONTROL_MMR_WRITE_SHFT 30
-#define UV3H_GR1_TLB_MMR_CONTROL_MMR_READ_SHFT 31
-#define UV3H_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT 32
-#define UV3H_GR1_TLB_MMR_CONTROL_INDEX_MASK 0x0000000000000fffUL
-#define UV3H_GR1_TLB_MMR_CONTROL_MEM_SEL_MASK 0x0000000000003000UL
-#define UV3H_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK 0x0000000000010000UL
-#define UV3H_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
-#define UV3H_GR1_TLB_MMR_CONTROL_ECC_SEL_MASK 0x0000000000200000UL
-#define UV3H_GR1_TLB_MMR_CONTROL_MMR_WRITE_MASK 0x0000000040000000UL
-#define UV3H_GR1_TLB_MMR_CONTROL_MMR_READ_MASK 0x0000000080000000UL
-#define UV3H_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_MASK 0x0000000100000000UL
-
-#define UV4H_GR1_TLB_MMR_CONTROL_INDEX_SHFT 0
-#define UV4H_GR1_TLB_MMR_CONTROL_MEM_SEL_SHFT 13
-#define UV4H_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_SHFT 16
-#define UV4H_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_SHFT 20
-#define UV4H_GR1_TLB_MMR_CONTROL_ECC_SEL_SHFT 21
-#define UV4H_GR1_TLB_MMR_CONTROL_MMR_WRITE_SHFT 30
-#define UV4H_GR1_TLB_MMR_CONTROL_MMR_READ_SHFT 31
-#define UV4H_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_SHFT 32
-#define UV4H_GR1_TLB_MMR_CONTROL_PAGE_SIZE_SHFT 59
-#define UV4H_GR1_TLB_MMR_CONTROL_INDEX_MASK 0x0000000000001fffUL
-#define UV4H_GR1_TLB_MMR_CONTROL_MEM_SEL_MASK 0x0000000000006000UL
-#define UV4H_GR1_TLB_MMR_CONTROL_AUTO_VALID_EN_MASK 0x0000000000010000UL
-#define UV4H_GR1_TLB_MMR_CONTROL_MMR_HASH_INDEX_EN_MASK 0x0000000000100000UL
-#define UV4H_GR1_TLB_MMR_CONTROL_ECC_SEL_MASK 0x0000000000200000UL
-#define UV4H_GR1_TLB_MMR_CONTROL_MMR_WRITE_MASK 0x0000000040000000UL
-#define UV4H_GR1_TLB_MMR_CONTROL_MMR_READ_MASK 0x0000000080000000UL
-#define UV4H_GR1_TLB_MMR_CONTROL_MMR_OP_DONE_MASK 0x0000000100000000UL
-#define UV4H_GR1_TLB_MMR_CONTROL_PAGE_SIZE_MASK 0xf800000000000000UL
-
-
-union uvh_gr1_tlb_mmr_control_u {
- unsigned long v;
- struct uvh_gr1_tlb_mmr_control_s {
- unsigned long rsvd_0_15:16;
- unsigned long auto_valid_en:1; /* RW */
- unsigned long rsvd_17_19:3;
- unsigned long mmr_hash_index_en:1; /* RW */
- unsigned long rsvd_21_29:9;
- unsigned long mmr_write:1; /* WP */
- unsigned long mmr_read:1; /* WP */
- unsigned long rsvd_32_48:17;
- unsigned long rsvd_49_51:3;
- unsigned long rsvd_52_63:12;
- } s;
- struct uvxh_gr1_tlb_mmr_control_s {
- unsigned long rsvd_0_15:16;
- unsigned long auto_valid_en:1; /* RW */
- unsigned long rsvd_17_19:3;
- unsigned long mmr_hash_index_en:1; /* RW */
- unsigned long rsvd_21_29:9;
- unsigned long mmr_write:1; /* WP */
- unsigned long mmr_read:1; /* WP */
- unsigned long mmr_op_done:1; /* RW */
- unsigned long rsvd_33_47:15;
- unsigned long rsvd_48:1;
- unsigned long rsvd_49_51:3;
- unsigned long rsvd_52_63:12;
- } sx;
- struct uv2h_gr1_tlb_mmr_control_s {
- unsigned long index:12; /* RW */
- unsigned long mem_sel:2; /* RW */
- unsigned long rsvd_14_15:2;
- unsigned long auto_valid_en:1; /* RW */
- unsigned long rsvd_17_19:3;
- unsigned long mmr_hash_index_en:1; /* RW */
- unsigned long rsvd_21_29:9;
- unsigned long mmr_write:1; /* WP */
- unsigned long mmr_read:1; /* WP */
- unsigned long mmr_op_done:1; /* RW */
- unsigned long rsvd_33_47:15;
- unsigned long mmr_inj_con:1; /* RW */
- unsigned long rsvd_49_51:3;
- unsigned long mmr_inj_tlbram:1; /* RW */
- unsigned long rsvd_53_63:11;
- } s2;
- struct uv3h_gr1_tlb_mmr_control_s {
- unsigned long index:12; /* RW */
- unsigned long mem_sel:2; /* RW */
- unsigned long rsvd_14_15:2;
- unsigned long auto_valid_en:1; /* RW */
- unsigned long rsvd_17_19:3;
- unsigned long mmr_hash_index_en:1; /* RW */
- unsigned long ecc_sel:1; /* RW */
- unsigned long rsvd_22_29:8;
- unsigned long mmr_write:1; /* WP */
- unsigned long mmr_read:1; /* WP */
- unsigned long mmr_op_done:1; /* RW */
- unsigned long rsvd_33_47:15;
- unsigned long undef_48:1; /* Undefined */
- unsigned long rsvd_49_51:3;
- unsigned long undef_52:1; /* Undefined */
- unsigned long rsvd_53_63:11;
- } s3;
- struct uv4h_gr1_tlb_mmr_control_s {
- unsigned long index:13; /* RW */
- unsigned long mem_sel:2; /* RW */
- unsigned long rsvd_15:1;
- unsigned long auto_valid_en:1; /* RW */
- unsigned long rsvd_17_19:3;
- unsigned long mmr_hash_index_en:1; /* RW */
- unsigned long ecc_sel:1; /* RW */
- unsigned long rsvd_22_29:8;
- unsigned long mmr_write:1; /* WP */
- unsigned long mmr_read:1; /* WP */
- unsigned long mmr_op_done:1; /* RW */
- unsigned long rsvd_33_47:15;
- unsigned long undef_48:1; /* Undefined */
- unsigned long rsvd_49_51:3;
- unsigned long rsvd_52_58:7;
- unsigned long page_size:5; /* RW */
- } s4;
-};
-
-/* ========================================================================= */
-/* UVH_GR1_TLB_MMR_READ_DATA_HI */
-/* ========================================================================= */
-#define UV2H_GR1_TLB_MMR_READ_DATA_HI 0x10010a0UL
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI 0x10010a0UL
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI 0x7010a0UL
-#define UVH_GR1_TLB_MMR_READ_DATA_HI ( \
- is_uv2_hub() ? UV2H_GR1_TLB_MMR_READ_DATA_HI : \
- is_uv3_hub() ? UV3H_GR1_TLB_MMR_READ_DATA_HI : \
- /*is_uv4_hub*/ UV4H_GR1_TLB_MMR_READ_DATA_HI)
-
-#define UVH_GR1_TLB_MMR_READ_DATA_HI_PFN_SHFT 0
-
-#define UVXH_GR1_TLB_MMR_READ_DATA_HI_PFN_SHFT 0
-
-#define UV2H_GR1_TLB_MMR_READ_DATA_HI_PFN_SHFT 0
-#define UV2H_GR1_TLB_MMR_READ_DATA_HI_GAA_SHFT 41
-#define UV2H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_SHFT 43
-#define UV2H_GR1_TLB_MMR_READ_DATA_HI_LARGER_SHFT 44
-#define UV2H_GR1_TLB_MMR_READ_DATA_HI_PFN_MASK 0x000001ffffffffffUL
-#define UV2H_GR1_TLB_MMR_READ_DATA_HI_GAA_MASK 0x0000060000000000UL
-#define UV2H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_MASK 0x0000080000000000UL
-#define UV2H_GR1_TLB_MMR_READ_DATA_HI_LARGER_MASK 0x0000100000000000UL
-
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_PFN_SHFT 0
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_GAA_SHFT 41
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_SHFT 43
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_LARGER_SHFT 44
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_AA_EXT_SHFT 45
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_WAY_ECC_SHFT 55
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_PFN_MASK 0x000001ffffffffffUL
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_GAA_MASK 0x0000060000000000UL
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_MASK 0x0000080000000000UL
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_LARGER_MASK 0x0000100000000000UL
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_AA_EXT_MASK 0x0000200000000000UL
-#define UV3H_GR1_TLB_MMR_READ_DATA_HI_WAY_ECC_MASK 0xff80000000000000UL
-
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_PFN_SHFT 0
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_PNID_SHFT 34
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_GAA_SHFT 49
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_SHFT 51
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_LARGER_SHFT 52
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_AA_EXT_SHFT 53
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_WAY_ECC_SHFT 55
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_PFN_MASK 0x00000003ffffffffUL
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_PNID_MASK 0x0001fffc00000000UL
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_GAA_MASK 0x0006000000000000UL
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_DIRTY_MASK 0x0008000000000000UL
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_LARGER_MASK 0x0010000000000000UL
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_AA_EXT_MASK 0x0020000000000000UL
-#define UV4H_GR1_TLB_MMR_READ_DATA_HI_WAY_ECC_MASK 0xff80000000000000UL
-
-
-union uvh_gr1_tlb_mmr_read_data_hi_u {
- unsigned long v;
- struct uv2h_gr1_tlb_mmr_read_data_hi_s {
- unsigned long pfn:41; /* RO */
- unsigned long gaa:2; /* RO */
- unsigned long dirty:1; /* RO */
- unsigned long larger:1; /* RO */
- unsigned long rsvd_45_63:19;
- } s2;
- struct uv3h_gr1_tlb_mmr_read_data_hi_s {
- unsigned long pfn:41; /* RO */
- unsigned long gaa:2; /* RO */
- unsigned long dirty:1; /* RO */
- unsigned long larger:1; /* RO */
- unsigned long aa_ext:1; /* RO */
- unsigned long undef_46_54:9; /* Undefined */
- unsigned long way_ecc:9; /* RO */
} s3;
- struct uv4h_gr1_tlb_mmr_read_data_hi_s {
- unsigned long pfn:34; /* RO */
- unsigned long pnid:15; /* RO */
- unsigned long gaa:2; /* RO */
- unsigned long dirty:1; /* RO */
- unsigned long larger:1; /* RO */
- unsigned long aa_ext:1; /* RO */
- unsigned long undef_54:1; /* Undefined */
- unsigned long way_ecc:9; /* RO */
- } s4;
-};
-/* ========================================================================= */
-/* UVH_GR1_TLB_MMR_READ_DATA_LO */
-/* ========================================================================= */
-#define UV2H_GR1_TLB_MMR_READ_DATA_LO 0x10010a8UL
-#define UV3H_GR1_TLB_MMR_READ_DATA_LO 0x10010a8UL
-#define UV4H_GR1_TLB_MMR_READ_DATA_LO 0x7010a8UL
-#define UVH_GR1_TLB_MMR_READ_DATA_LO ( \
- is_uv2_hub() ? UV2H_GR1_TLB_MMR_READ_DATA_LO : \
- is_uv3_hub() ? UV3H_GR1_TLB_MMR_READ_DATA_LO : \
- /*is_uv4_hub*/ UV4H_GR1_TLB_MMR_READ_DATA_LO)
-
-#define UVH_GR1_TLB_MMR_READ_DATA_LO_VPN_SHFT 0
-#define UVH_GR1_TLB_MMR_READ_DATA_LO_ASID_SHFT 39
-#define UVH_GR1_TLB_MMR_READ_DATA_LO_VALID_SHFT 63
-#define UVH_GR1_TLB_MMR_READ_DATA_LO_VPN_MASK 0x0000007fffffffffUL
-#define UVH_GR1_TLB_MMR_READ_DATA_LO_ASID_MASK 0x7fffff8000000000UL
-#define UVH_GR1_TLB_MMR_READ_DATA_LO_VALID_MASK 0x8000000000000000UL
-
-#define UVXH_GR1_TLB_MMR_READ_DATA_LO_VPN_SHFT 0
-#define UVXH_GR1_TLB_MMR_READ_DATA_LO_ASID_SHFT 39
-#define UVXH_GR1_TLB_MMR_READ_DATA_LO_VALID_SHFT 63
-#define UVXH_GR1_TLB_MMR_READ_DATA_LO_VPN_MASK 0x0000007fffffffffUL
-#define UVXH_GR1_TLB_MMR_READ_DATA_LO_ASID_MASK 0x7fffff8000000000UL
-#define UVXH_GR1_TLB_MMR_READ_DATA_LO_VALID_MASK 0x8000000000000000UL
-
-#define UV2H_GR1_TLB_MMR_READ_DATA_LO_VPN_SHFT 0
-#define UV2H_GR1_TLB_MMR_READ_DATA_LO_ASID_SHFT 39
-#define UV2H_GR1_TLB_MMR_READ_DATA_LO_VALID_SHFT 63
-#define UV2H_GR1_TLB_MMR_READ_DATA_LO_VPN_MASK 0x0000007fffffffffUL
-#define UV2H_GR1_TLB_MMR_READ_DATA_LO_ASID_MASK 0x7fffff8000000000UL
-#define UV2H_GR1_TLB_MMR_READ_DATA_LO_VALID_MASK 0x8000000000000000UL
-
-#define UV3H_GR1_TLB_MMR_READ_DATA_LO_VPN_SHFT 0
-#define UV3H_GR1_TLB_MMR_READ_DATA_LO_ASID_SHFT 39
-#define UV3H_GR1_TLB_MMR_READ_DATA_LO_VALID_SHFT 63
-#define UV3H_GR1_TLB_MMR_READ_DATA_LO_VPN_MASK 0x0000007fffffffffUL
-#define UV3H_GR1_TLB_MMR_READ_DATA_LO_ASID_MASK 0x7fffff8000000000UL
-#define UV3H_GR1_TLB_MMR_READ_DATA_LO_VALID_MASK 0x8000000000000000UL
-
-#define UV4H_GR1_TLB_MMR_READ_DATA_LO_VPN_SHFT 0
-#define UV4H_GR1_TLB_MMR_READ_DATA_LO_ASID_SHFT 39
-#define UV4H_GR1_TLB_MMR_READ_DATA_LO_VALID_SHFT 63
-#define UV4H_GR1_TLB_MMR_READ_DATA_LO_VPN_MASK 0x0000007fffffffffUL
-#define UV4H_GR1_TLB_MMR_READ_DATA_LO_ASID_MASK 0x7fffff8000000000UL
-#define UV4H_GR1_TLB_MMR_READ_DATA_LO_VALID_MASK 0x8000000000000000UL
-
-
-union uvh_gr1_tlb_mmr_read_data_lo_u {
- unsigned long v;
- struct uvh_gr1_tlb_mmr_read_data_lo_s {
- unsigned long vpn:39; /* RO */
- unsigned long asid:24; /* RO */
- unsigned long valid:1; /* RO */
- } s;
- struct uvxh_gr1_tlb_mmr_read_data_lo_s {
- unsigned long vpn:39; /* RO */
- unsigned long asid:24; /* RO */
- unsigned long valid:1; /* RO */
- } sx;
- struct uv2h_gr1_tlb_mmr_read_data_lo_s {
- unsigned long vpn:39; /* RO */
- unsigned long asid:24; /* RO */
- unsigned long valid:1; /* RO */
+ /* UV2 unique struct */
+ struct uv2h_gr0_tlb_int1_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
} s2;
- struct uv3h_gr1_tlb_mmr_read_data_lo_s {
- unsigned long vpn:39; /* RO */
- unsigned long asid:24; /* RO */
- unsigned long valid:1; /* RO */
- } s3;
- struct uv4h_gr1_tlb_mmr_read_data_lo_s {
- unsigned long vpn:39; /* RO */
- unsigned long asid:24; /* RO */
- unsigned long valid:1; /* RO */
- } s4;
};
/* ========================================================================= */
-/* UVH_INT_CMPB */
+/* UVH_GR1_TLB_INT0_CONFIG */
/* ========================================================================= */
-#define UVH_INT_CMPB 0x22080UL
-
-#define UVH_INT_CMPB_REAL_TIME_CMPB_SHFT 0
-#define UVH_INT_CMPB_REAL_TIME_CMPB_MASK 0x00ffffffffffffffUL
+#define UVH_GR1_TLB_INT0_CONFIG ( \
+ is_uv(UV4) ? 0x62100UL : \
+ is_uv(UV3) ? 0x61f00UL : \
+ is_uv(UV2) ? 0x61f00UL : \
+ uv_undefined("UVH_GR1_TLB_INT0_CONFIG"))
+
+
+/* UVXH common defines */
+#define UVXH_GR1_TLB_INT0_CONFIG_VECTOR_SHFT 0
+#define UVXH_GR1_TLB_INT0_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVXH_GR1_TLB_INT0_CONFIG_DM_SHFT 8
+#define UVXH_GR1_TLB_INT0_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVXH_GR1_TLB_INT0_CONFIG_DESTMODE_SHFT 11
+#define UVXH_GR1_TLB_INT0_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVXH_GR1_TLB_INT0_CONFIG_STATUS_SHFT 12
+#define UVXH_GR1_TLB_INT0_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVXH_GR1_TLB_INT0_CONFIG_P_SHFT 13
+#define UVXH_GR1_TLB_INT0_CONFIG_P_MASK 0x0000000000002000UL
+#define UVXH_GR1_TLB_INT0_CONFIG_T_SHFT 15
+#define UVXH_GR1_TLB_INT0_CONFIG_T_MASK 0x0000000000008000UL
+#define UVXH_GR1_TLB_INT0_CONFIG_M_SHFT 16
+#define UVXH_GR1_TLB_INT0_CONFIG_M_MASK 0x0000000000010000UL
+#define UVXH_GR1_TLB_INT0_CONFIG_APIC_ID_SHFT 32
+#define UVXH_GR1_TLB_INT0_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
-union uvh_int_cmpb_u {
+union uvh_gr1_tlb_int0_config_u {
unsigned long v;
- struct uvh_int_cmpb_s {
- unsigned long real_time_cmpb:56; /* RW */
- unsigned long rsvd_56_63:8;
- } s;
-};
-
-/* ========================================================================= */
-/* UVH_INT_CMPC */
-/* ========================================================================= */
-#define UVH_INT_CMPC 0x22100UL
-
-
-#define UVXH_INT_CMPC_REAL_TIME_CMP_2_SHFT 0
-#define UVXH_INT_CMPC_REAL_TIME_CMP_2_MASK 0x00ffffffffffffffUL
-
-union uvh_int_cmpc_u {
- unsigned long v;
- struct uvh_int_cmpc_s {
- unsigned long real_time_cmpc:56; /* RW */
- unsigned long rsvd_56_63:8;
+ /* UVH common struct */
+ struct uvh_gr1_tlb_int0_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
} s;
-};
-
-/* ========================================================================= */
-/* UVH_INT_CMPD */
-/* ========================================================================= */
-#define UVH_INT_CMPD 0x22180UL
+ /* UVXH common struct */
+ struct uvxh_gr1_tlb_int0_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } sx;
-#define UVXH_INT_CMPD_REAL_TIME_CMP_3_SHFT 0
-#define UVXH_INT_CMPD_REAL_TIME_CMP_3_MASK 0x00ffffffffffffffUL
+ /* UV4 unique struct */
+ struct uv4h_gr1_tlb_int0_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s4;
+ /* UV3 unique struct */
+ struct uv3h_gr1_tlb_int0_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s3;
-union uvh_int_cmpd_u {
- unsigned long v;
- struct uvh_int_cmpd_s {
- unsigned long real_time_cmpd:56; /* RW */
- unsigned long rsvd_56_63:8;
- } s;
+ /* UV2 unique struct */
+ struct uv2h_gr1_tlb_int0_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s2;
};
/* ========================================================================= */
-/* UVH_IPI_INT */
+/* UVH_GR1_TLB_INT1_CONFIG */
/* ========================================================================= */
-#define UVH_IPI_INT 0x60500UL
-
-#define UV2H_IPI_INT_32 0x348
-#define UV3H_IPI_INT_32 0x348
-#define UV4H_IPI_INT_32 0x268
-#define UVH_IPI_INT_32 ( \
- is_uv2_hub() ? UV2H_IPI_INT_32 : \
- is_uv3_hub() ? UV3H_IPI_INT_32 : \
- /*is_uv4_hub*/ UV4H_IPI_INT_32)
-
-#define UVH_IPI_INT_VECTOR_SHFT 0
-#define UVH_IPI_INT_DELIVERY_MODE_SHFT 8
-#define UVH_IPI_INT_DESTMODE_SHFT 11
-#define UVH_IPI_INT_APIC_ID_SHFT 16
-#define UVH_IPI_INT_SEND_SHFT 63
-#define UVH_IPI_INT_VECTOR_MASK 0x00000000000000ffUL
-#define UVH_IPI_INT_DELIVERY_MODE_MASK 0x0000000000000700UL
-#define UVH_IPI_INT_DESTMODE_MASK 0x0000000000000800UL
-#define UVH_IPI_INT_APIC_ID_MASK 0x0000ffffffff0000UL
-#define UVH_IPI_INT_SEND_MASK 0x8000000000000000UL
+#define UVH_GR1_TLB_INT1_CONFIG ( \
+ is_uv(UV4) ? 0x62140UL : \
+ is_uv(UV3) ? 0x61f40UL : \
+ is_uv(UV2) ? 0x61f40UL : \
+ uv_undefined("UVH_GR1_TLB_INT1_CONFIG"))
+
+
+/* UVXH common defines */
+#define UVXH_GR1_TLB_INT1_CONFIG_VECTOR_SHFT 0
+#define UVXH_GR1_TLB_INT1_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVXH_GR1_TLB_INT1_CONFIG_DM_SHFT 8
+#define UVXH_GR1_TLB_INT1_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVXH_GR1_TLB_INT1_CONFIG_DESTMODE_SHFT 11
+#define UVXH_GR1_TLB_INT1_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVXH_GR1_TLB_INT1_CONFIG_STATUS_SHFT 12
+#define UVXH_GR1_TLB_INT1_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVXH_GR1_TLB_INT1_CONFIG_P_SHFT 13
+#define UVXH_GR1_TLB_INT1_CONFIG_P_MASK 0x0000000000002000UL
+#define UVXH_GR1_TLB_INT1_CONFIG_T_SHFT 15
+#define UVXH_GR1_TLB_INT1_CONFIG_T_MASK 0x0000000000008000UL
+#define UVXH_GR1_TLB_INT1_CONFIG_M_SHFT 16
+#define UVXH_GR1_TLB_INT1_CONFIG_M_MASK 0x0000000000010000UL
+#define UVXH_GR1_TLB_INT1_CONFIG_APIC_ID_SHFT 32
+#define UVXH_GR1_TLB_INT1_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
-union uvh_ipi_int_u {
+union uvh_gr1_tlb_int1_config_u {
unsigned long v;
- struct uvh_ipi_int_s {
+
+ /* UVH common struct */
+ struct uvh_gr1_tlb_int1_config_s {
unsigned long vector_:8; /* RW */
- unsigned long delivery_mode:3; /* RW */
+ unsigned long dm:3; /* RW */
unsigned long destmode:1; /* RW */
- unsigned long rsvd_12_15:4;
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
unsigned long apic_id:32; /* RW */
- unsigned long rsvd_48_62:15;
- unsigned long send:1; /* WP */
} s;
-};
-
-/* ========================================================================= */
-/* UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST */
-/* ========================================================================= */
-#define UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST 0x320050UL
-#define UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST 0x320050UL
-#define UV4H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST uv_undefined("UV4H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST")
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST ( \
- is_uv2_hub() ? UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST : \
- is_uv3_hub() ? UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST : \
- /*is_uv4_hub*/ UV4H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST)
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_32 0x9c0
+ /* UVXH common struct */
+ struct uvxh_gr1_tlb_int1_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } sx;
-#define UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_ADDRESS_SHFT 4
-#define UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_NODE_ID_SHFT 49
-#define UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_ADDRESS_MASK 0x000007fffffffff0UL
-#define UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_NODE_ID_MASK 0x7ffe000000000000UL
-
-#define UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_ADDRESS_SHFT 4
-#define UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_NODE_ID_SHFT 49
-#define UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_ADDRESS_MASK 0x000007fffffffff0UL
-#define UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST_NODE_ID_MASK 0x7ffe000000000000UL
+ /* UV4 unique struct */
+ struct uv4h_gr1_tlb_int1_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s4;
+ /* UV3 unique struct */
+ struct uv3h_gr1_tlb_int1_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s3;
-union uvh_lb_bau_intd_payload_queue_first_u {
- unsigned long v;
- struct uv2h_lb_bau_intd_payload_queue_first_s {
- unsigned long rsvd_0_3:4;
- unsigned long address:39; /* RW */
- unsigned long rsvd_43_48:6;
- unsigned long node_id:14; /* RW */
- unsigned long rsvd_63:1;
+ /* UV2 unique struct */
+ struct uv2h_gr1_tlb_int1_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
} s2;
- struct uv3h_lb_bau_intd_payload_queue_first_s {
- unsigned long rsvd_0_3:4;
- unsigned long address:39; /* RW */
- unsigned long rsvd_43_48:6;
- unsigned long node_id:14; /* RW */
- unsigned long rsvd_63:1;
- } s3;
};
/* ========================================================================= */
-/* UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST */
+/* UVH_INT_CMPB */
/* ========================================================================= */
-#define UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_LAST 0x320060UL
-#define UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_LAST 0x320060UL
-#define UV4H_LB_BAU_INTD_PAYLOAD_QUEUE_LAST uv_undefined("UV4H_LB_BAU_INTD_PAYLOAD_QUEUE_LAST")
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST ( \
- is_uv2_hub() ? UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_LAST : \
- is_uv3_hub() ? UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_LAST : \
- /*is_uv4_hub*/ UV4H_LB_BAU_INTD_PAYLOAD_QUEUE_LAST)
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_32 0x9c8
-
-
-#define UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_ADDRESS_SHFT 4
-#define UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_ADDRESS_MASK 0x000007fffffffff0UL
+#define UVH_INT_CMPB 0x22080UL
-#define UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_ADDRESS_SHFT 4
-#define UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_LAST_ADDRESS_MASK 0x000007fffffffff0UL
+/* UVH common defines*/
+#define UVH_INT_CMPB_REAL_TIME_CMPB_SHFT 0
+#define UVH_INT_CMPB_REAL_TIME_CMPB_MASK 0x00ffffffffffffffUL
-union uvh_lb_bau_intd_payload_queue_last_u {
+union uvh_int_cmpb_u {
unsigned long v;
- struct uv2h_lb_bau_intd_payload_queue_last_s {
- unsigned long rsvd_0_3:4;
- unsigned long address:39; /* RW */
- unsigned long rsvd_43_63:21;
- } s2;
- struct uv3h_lb_bau_intd_payload_queue_last_s {
- unsigned long rsvd_0_3:4;
- unsigned long address:39; /* RW */
- unsigned long rsvd_43_63:21;
- } s3;
-};
-
-/* ========================================================================= */
-/* UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL */
-/* ========================================================================= */
-#define UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL 0x320070UL
-#define UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL 0x320070UL
-#define UV4H_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL uv_undefined("UV4H_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL")
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL ( \
- is_uv2_hub() ? UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL : \
- is_uv3_hub() ? UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL : \
- /*is_uv4_hub*/ UV4H_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL)
-#define UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_32 0x9d0
-
-#define UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_ADDRESS_SHFT 4
-#define UV2H_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_ADDRESS_MASK 0x000007fffffffff0UL
-
-#define UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_ADDRESS_SHFT 4
-#define UV3H_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL_ADDRESS_MASK 0x000007fffffffff0UL
+ /* UVH common struct */
+ struct uvh_int_cmpb_s {
+ unsigned long real_time_cmpb:56; /* RW */
+ unsigned long rsvd_56_63:8;
+ } s;
+ /* UV5 unique struct */
+ struct uv5h_int_cmpb_s {
+ unsigned long real_time_cmpb:56; /* RW */
+ unsigned long rsvd_56_63:8;
+ } s5;
-union uvh_lb_bau_intd_payload_queue_tail_u {
- unsigned long v;
- struct uv2h_lb_bau_intd_payload_queue_tail_s {
- unsigned long rsvd_0_3:4;
- unsigned long address:39; /* RW */
- unsigned long rsvd_43_63:21;
- } s2;
- struct uv3h_lb_bau_intd_payload_queue_tail_s {
- unsigned long rsvd_0_3:4;
- unsigned long address:39; /* RW */
- unsigned long rsvd_43_63:21;
- } s3;
-};
+ /* UV4 unique struct */
+ struct uv4h_int_cmpb_s {
+ unsigned long real_time_cmpb:56; /* RW */
+ unsigned long rsvd_56_63:8;
+ } s4;
-/* ========================================================================= */
-/* UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE */
-/* ========================================================================= */
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE 0x320080UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE 0x320080UL
-#define UV4H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE uv_undefined("UV4H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE")
-#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE ( \
- is_uv2_hub() ? UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE : \
- is_uv3_hub() ? UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE : \
- /*is_uv4_hub*/ UV4H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE)
-#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_32 0xa68
-
-
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_0_SHFT 0
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_1_SHFT 1
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_2_SHFT 2
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_3_SHFT 3
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_4_SHFT 4
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_5_SHFT 5
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_6_SHFT 6
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_7_SHFT 7
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_0_SHFT 8
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_1_SHFT 9
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_2_SHFT 10
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_3_SHFT 11
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_4_SHFT 12
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_5_SHFT 13
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_6_SHFT 14
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_7_SHFT 15
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_0_MASK 0x0000000000000001UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_1_MASK 0x0000000000000002UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_2_MASK 0x0000000000000004UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_3_MASK 0x0000000000000008UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_4_MASK 0x0000000000000010UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_5_MASK 0x0000000000000020UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_6_MASK 0x0000000000000040UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_7_MASK 0x0000000000000080UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_0_MASK 0x0000000000000100UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_1_MASK 0x0000000000000200UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_2_MASK 0x0000000000000400UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_3_MASK 0x0000000000000800UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_4_MASK 0x0000000000001000UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_5_MASK 0x0000000000002000UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_6_MASK 0x0000000000004000UL
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_7_MASK 0x0000000000008000UL
-
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_0_SHFT 0
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_1_SHFT 1
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_2_SHFT 2
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_3_SHFT 3
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_4_SHFT 4
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_5_SHFT 5
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_6_SHFT 6
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_7_SHFT 7
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_0_SHFT 8
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_1_SHFT 9
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_2_SHFT 10
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_3_SHFT 11
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_4_SHFT 12
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_5_SHFT 13
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_6_SHFT 14
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_7_SHFT 15
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_0_MASK 0x0000000000000001UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_1_MASK 0x0000000000000002UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_2_MASK 0x0000000000000004UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_3_MASK 0x0000000000000008UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_4_MASK 0x0000000000000010UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_5_MASK 0x0000000000000020UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_6_MASK 0x0000000000000040UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_PENDING_7_MASK 0x0000000000000080UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_0_MASK 0x0000000000000100UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_1_MASK 0x0000000000000200UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_2_MASK 0x0000000000000400UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_3_MASK 0x0000000000000800UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_4_MASK 0x0000000000001000UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_5_MASK 0x0000000000002000UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_6_MASK 0x0000000000004000UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_TIMEOUT_7_MASK 0x0000000000008000UL
-
-
-union uvh_lb_bau_intd_software_acknowledge_u {
- unsigned long v;
- struct uv2h_lb_bau_intd_software_acknowledge_s {
- unsigned long pending_0:1; /* RW */
- unsigned long pending_1:1; /* RW */
- unsigned long pending_2:1; /* RW */
- unsigned long pending_3:1; /* RW */
- unsigned long pending_4:1; /* RW */
- unsigned long pending_5:1; /* RW */
- unsigned long pending_6:1; /* RW */
- unsigned long pending_7:1; /* RW */
- unsigned long timeout_0:1; /* RW */
- unsigned long timeout_1:1; /* RW */
- unsigned long timeout_2:1; /* RW */
- unsigned long timeout_3:1; /* RW */
- unsigned long timeout_4:1; /* RW */
- unsigned long timeout_5:1; /* RW */
- unsigned long timeout_6:1; /* RW */
- unsigned long timeout_7:1; /* RW */
- unsigned long rsvd_16_63:48;
- } s2;
- struct uv3h_lb_bau_intd_software_acknowledge_s {
- unsigned long pending_0:1; /* RW */
- unsigned long pending_1:1; /* RW */
- unsigned long pending_2:1; /* RW */
- unsigned long pending_3:1; /* RW */
- unsigned long pending_4:1; /* RW */
- unsigned long pending_5:1; /* RW */
- unsigned long pending_6:1; /* RW */
- unsigned long pending_7:1; /* RW */
- unsigned long timeout_0:1; /* RW */
- unsigned long timeout_1:1; /* RW */
- unsigned long timeout_2:1; /* RW */
- unsigned long timeout_3:1; /* RW */
- unsigned long timeout_4:1; /* RW */
- unsigned long timeout_5:1; /* RW */
- unsigned long timeout_6:1; /* RW */
- unsigned long timeout_7:1; /* RW */
- unsigned long rsvd_16_63:48;
+ /* UV3 unique struct */
+ struct uv3h_int_cmpb_s {
+ unsigned long real_time_cmpb:56; /* RW */
+ unsigned long rsvd_56_63:8;
} s3;
-};
-
-/* ========================================================================= */
-/* UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS */
-/* ========================================================================= */
-#define UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS 0x320088UL
-#define UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS 0x320088UL
-#define UV4H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS uv_undefined("UV4H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS")
-#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS ( \
- is_uv2_hub() ? UV2H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS : \
- is_uv3_hub() ? UV3H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS : \
- /*is_uv4_hub*/ UV4H_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS)
-#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS_32 0xa70
-
-/* ========================================================================= */
-/* UVH_LB_BAU_MISC_CONTROL */
-/* ========================================================================= */
-#define UV2H_LB_BAU_MISC_CONTROL 0x320170UL
-#define UV3H_LB_BAU_MISC_CONTROL 0x320170UL
-#define UV4H_LB_BAU_MISC_CONTROL 0xc8170UL
-#define UVH_LB_BAU_MISC_CONTROL ( \
- is_uv2_hub() ? UV2H_LB_BAU_MISC_CONTROL : \
- is_uv3_hub() ? UV3H_LB_BAU_MISC_CONTROL : \
- /*is_uv4_hub*/ UV4H_LB_BAU_MISC_CONTROL)
-
-#define UV2H_LB_BAU_MISC_CONTROL_32 0xa10
-#define UV3H_LB_BAU_MISC_CONTROL_32 0xa10
-#define UV4H_LB_BAU_MISC_CONTROL_32 0xa18
-#define UVH_LB_BAU_MISC_CONTROL_32 ( \
- is_uv2_hub() ? UV2H_LB_BAU_MISC_CONTROL_32 : \
- is_uv3_hub() ? UV3H_LB_BAU_MISC_CONTROL_32 : \
- /*is_uv4_hub*/ UV4H_LB_BAU_MISC_CONTROL_32)
-
-#define UVH_LB_BAU_MISC_CONTROL_REJECTION_DELAY_SHFT 0
-#define UVH_LB_BAU_MISC_CONTROL_APIC_MODE_SHFT 8
-#define UVH_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_SHFT 9
-#define UVH_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_SHFT 10
-#define UVH_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_SHFT 11
-#define UVH_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_SHFT 14
-#define UVH_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_SHFT 20
-#define UVH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_SHFT 21
-#define UVH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_SHFT 22
-#define UVH_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_SHFT 23
-#define UVH_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_SHFT 24
-#define UVH_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_SHFT 27
-#define UVH_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_SHFT 28
-#define UVH_LB_BAU_MISC_CONTROL_FUN_SHFT 48
-#define UVH_LB_BAU_MISC_CONTROL_REJECTION_DELAY_MASK 0x00000000000000ffUL
-#define UVH_LB_BAU_MISC_CONTROL_APIC_MODE_MASK 0x0000000000000100UL
-#define UVH_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_MASK 0x0000000000000200UL
-#define UVH_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_MASK 0x0000000000000400UL
-#define UVH_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_MASK 0x0000000000003800UL
-#define UVH_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_MASK 0x0000000000004000UL
-#define UVH_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_MASK 0x0000000000100000UL
-#define UVH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_MASK 0x0000000000200000UL
-#define UVH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_MASK 0x0000000000400000UL
-#define UVH_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_MASK 0x0000000000800000UL
-#define UVH_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000007000000UL
-#define UVH_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_MASK 0x0000000008000000UL
-#define UVH_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000010000000UL
-#define UVH_LB_BAU_MISC_CONTROL_FUN_MASK 0xffff000000000000UL
-
-#define UVXH_LB_BAU_MISC_CONTROL_REJECTION_DELAY_SHFT 0
-#define UVXH_LB_BAU_MISC_CONTROL_APIC_MODE_SHFT 8
-#define UVXH_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_SHFT 9
-#define UVXH_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_SHFT 10
-#define UVXH_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_SHFT 11
-#define UVXH_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_SHFT 14
-#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_SHFT 20
-#define UVXH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_SHFT 21
-#define UVXH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_SHFT 22
-#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_SHFT 23
-#define UVXH_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_SHFT 24
-#define UVXH_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_SHFT 27
-#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_SHFT 28
-#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_SHFT 29
-#define UVXH_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_SHFT 30
-#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_SHFT 31
-#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_SHFT 32
-#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT 33
-#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_SHFT 34
-#define UVXH_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_SHFT 35
-#define UVXH_LB_BAU_MISC_CONTROL_FUN_SHFT 48
-#define UVXH_LB_BAU_MISC_CONTROL_REJECTION_DELAY_MASK 0x00000000000000ffUL
-#define UVXH_LB_BAU_MISC_CONTROL_APIC_MODE_MASK 0x0000000000000100UL
-#define UVXH_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_MASK 0x0000000000000200UL
-#define UVXH_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_MASK 0x0000000000000400UL
-#define UVXH_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_MASK 0x0000000000003800UL
-#define UVXH_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_MASK 0x0000000000004000UL
-#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_MASK 0x0000000000100000UL
-#define UVXH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_MASK 0x0000000000200000UL
-#define UVXH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_MASK 0x0000000000400000UL
-#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_MASK 0x0000000000800000UL
-#define UVXH_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000007000000UL
-#define UVXH_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_MASK 0x0000000008000000UL
-#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000010000000UL
-#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_MASK 0x0000000020000000UL
-#define UVXH_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_MASK 0x0000000040000000UL
-#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_MASK 0x0000000080000000UL
-#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_MASK 0x0000000100000000UL
-#define UVXH_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_MASK 0x0000000200000000UL
-#define UVXH_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_MASK 0x0000000400000000UL
-#define UVXH_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_MASK 0x0000000800000000UL
-#define UVXH_LB_BAU_MISC_CONTROL_FUN_MASK 0xffff000000000000UL
-
-#define UV2H_LB_BAU_MISC_CONTROL_REJECTION_DELAY_SHFT 0
-#define UV2H_LB_BAU_MISC_CONTROL_APIC_MODE_SHFT 8
-#define UV2H_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_SHFT 9
-#define UV2H_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_SHFT 10
-#define UV2H_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_SHFT 11
-#define UV2H_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_SHFT 14
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT 15
-#define UV2H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT 16
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_SHFT 20
-#define UV2H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_SHFT 21
-#define UV2H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_SHFT 22
-#define UV2H_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_SHFT 23
-#define UV2H_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_SHFT 24
-#define UV2H_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_SHFT 27
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_SHFT 28
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_SHFT 29
-#define UV2H_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_SHFT 30
-#define UV2H_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_SHFT 31
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_SHFT 32
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT 33
-#define UV2H_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_SHFT 34
-#define UV2H_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_SHFT 35
-#define UV2H_LB_BAU_MISC_CONTROL_FUN_SHFT 48
-#define UV2H_LB_BAU_MISC_CONTROL_REJECTION_DELAY_MASK 0x00000000000000ffUL
-#define UV2H_LB_BAU_MISC_CONTROL_APIC_MODE_MASK 0x0000000000000100UL
-#define UV2H_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_MASK 0x0000000000000200UL
-#define UV2H_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_MASK 0x0000000000000400UL
-#define UV2H_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_MASK 0x0000000000003800UL
-#define UV2H_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_MASK 0x0000000000004000UL
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_MASK 0x0000000000008000UL
-#define UV2H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK 0x00000000000f0000UL
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_MASK 0x0000000000100000UL
-#define UV2H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_MASK 0x0000000000200000UL
-#define UV2H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_MASK 0x0000000000400000UL
-#define UV2H_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_MASK 0x0000000000800000UL
-#define UV2H_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000007000000UL
-#define UV2H_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_MASK 0x0000000008000000UL
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000010000000UL
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_MASK 0x0000000020000000UL
-#define UV2H_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_MASK 0x0000000040000000UL
-#define UV2H_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_MASK 0x0000000080000000UL
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_MASK 0x0000000100000000UL
-#define UV2H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_MASK 0x0000000200000000UL
-#define UV2H_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_MASK 0x0000000400000000UL
-#define UV2H_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_MASK 0x0000000800000000UL
-#define UV2H_LB_BAU_MISC_CONTROL_FUN_MASK 0xffff000000000000UL
-
-#define UV3H_LB_BAU_MISC_CONTROL_REJECTION_DELAY_SHFT 0
-#define UV3H_LB_BAU_MISC_CONTROL_APIC_MODE_SHFT 8
-#define UV3H_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_SHFT 9
-#define UV3H_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_SHFT 10
-#define UV3H_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_SHFT 11
-#define UV3H_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_SHFT 14
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT 15
-#define UV3H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT 16
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_SHFT 20
-#define UV3H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_SHFT 21
-#define UV3H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_SHFT 22
-#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_SHFT 23
-#define UV3H_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_SHFT 24
-#define UV3H_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_SHFT 27
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_SHFT 28
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_SHFT 29
-#define UV3H_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_SHFT 30
-#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_SHFT 31
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_SHFT 32
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT 33
-#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_SHFT 34
-#define UV3H_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_SHFT 35
-#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_QUIESCE_MSGS_TO_QPI_SHFT 36
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_PREFETCH_HINT_SHFT 37
-#define UV3H_LB_BAU_MISC_CONTROL_THREAD_KILL_TIMEBASE_SHFT 38
-#define UV3H_LB_BAU_MISC_CONTROL_FUN_SHFT 48
-#define UV3H_LB_BAU_MISC_CONTROL_REJECTION_DELAY_MASK 0x00000000000000ffUL
-#define UV3H_LB_BAU_MISC_CONTROL_APIC_MODE_MASK 0x0000000000000100UL
-#define UV3H_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_MASK 0x0000000000000200UL
-#define UV3H_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_MASK 0x0000000000000400UL
-#define UV3H_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_MASK 0x0000000000003800UL
-#define UV3H_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_MASK 0x0000000000004000UL
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_MASK 0x0000000000008000UL
-#define UV3H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK 0x00000000000f0000UL
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_MASK 0x0000000000100000UL
-#define UV3H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_MASK 0x0000000000200000UL
-#define UV3H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_MASK 0x0000000000400000UL
-#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_MASK 0x0000000000800000UL
-#define UV3H_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000007000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_MASK 0x0000000008000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000010000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_MASK 0x0000000020000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_MASK 0x0000000040000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_MASK 0x0000000080000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_MASK 0x0000000100000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_MASK 0x0000000200000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_MASK 0x0000000400000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_MASK 0x0000000800000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_SUPPRESS_QUIESCE_MSGS_TO_QPI_MASK 0x0000001000000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_PREFETCH_HINT_MASK 0x0000002000000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_THREAD_KILL_TIMEBASE_MASK 0x00003fc000000000UL
-#define UV3H_LB_BAU_MISC_CONTROL_FUN_MASK 0xffff000000000000UL
-
-#define UV4H_LB_BAU_MISC_CONTROL_REJECTION_DELAY_SHFT 0
-#define UV4H_LB_BAU_MISC_CONTROL_APIC_MODE_SHFT 8
-#define UV4H_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_SHFT 9
-#define UV4H_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_SHFT 10
-#define UV4H_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_SHFT 11
-#define UV4H_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_SHFT 14
-#define UV4H_LB_BAU_MISC_CONTROL_RESERVED_15_19_SHFT 15
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_SHFT 20
-#define UV4H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_SHFT 21
-#define UV4H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_SHFT 22
-#define UV4H_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_SHFT 23
-#define UV4H_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_SHFT 24
-#define UV4H_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_SHFT 27
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_SHFT 28
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_SHFT 29
-#define UV4H_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_SHFT 30
-#define UV4H_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_SHFT 31
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_SHFT 32
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT 33
-#define UV4H_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_SHFT 34
-#define UV4H_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_SHFT 35
-#define UV4H_LB_BAU_MISC_CONTROL_SUPPRESS_QUIESCE_MSGS_TO_QPI_SHFT 36
-#define UV4H_LB_BAU_MISC_CONTROL_RESERVED_37_SHFT 37
-#define UV4H_LB_BAU_MISC_CONTROL_THREAD_KILL_TIMEBASE_SHFT 38
-#define UV4H_LB_BAU_MISC_CONTROL_ADDRESS_INTERLEAVE_SELECT_SHFT 46
-#define UV4H_LB_BAU_MISC_CONTROL_FUN_SHFT 48
-#define UV4H_LB_BAU_MISC_CONTROL_REJECTION_DELAY_MASK 0x00000000000000ffUL
-#define UV4H_LB_BAU_MISC_CONTROL_APIC_MODE_MASK 0x0000000000000100UL
-#define UV4H_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_MASK 0x0000000000000200UL
-#define UV4H_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_MASK 0x0000000000000400UL
-#define UV4H_LB_BAU_MISC_CONTROL_QPI_AGENT_PRESENCE_VECTOR_MASK 0x0000000000003800UL
-#define UV4H_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_MASK 0x0000000000004000UL
-#define UV4H_LB_BAU_MISC_CONTROL_RESERVED_15_19_MASK 0x00000000000f8000UL
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_MASK 0x0000000000100000UL
-#define UV4H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_MASK 0x0000000000200000UL
-#define UV4H_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_MASK 0x0000000000400000UL
-#define UV4H_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_MASK 0x0000000000800000UL
-#define UV4H_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000007000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_MASK 0x0000000008000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000010000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_AUTOMATIC_APIC_MODE_SELECTION_MASK 0x0000000020000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_APIC_MODE_STATUS_MASK 0x0000000040000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_SUPPRESS_INTERRUPTS_TO_SELF_MASK 0x0000000080000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_LOCK_BASED_SYSTEM_FLUSH_MASK 0x0000000100000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_MASK 0x0000000200000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_SUPPRESS_INT_PRIO_UDT_TO_SELF_MASK 0x0000000400000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_USE_LEGACY_DESCRIPTOR_FORMATS_MASK 0x0000000800000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_SUPPRESS_QUIESCE_MSGS_TO_QPI_MASK 0x0000001000000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_RESERVED_37_MASK 0x0000002000000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_THREAD_KILL_TIMEBASE_MASK 0x00003fc000000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_ADDRESS_INTERLEAVE_SELECT_MASK 0x0000400000000000UL
-#define UV4H_LB_BAU_MISC_CONTROL_FUN_MASK 0xffff000000000000UL
-
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_MASK \
- uv_undefined("UV4H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_MASK")
-#define UVH_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_MASK ( \
- is_uv2_hub() ? UV2H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_MASK : \
- is_uv3_hub() ? UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_MASK : \
- /*is_uv4_hub*/ UV4H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_MASK)
-#define UV4H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT \
- uv_undefined("UV4H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT")
-#define UVH_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT ( \
- is_uv2_hub() ? UV2H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT : \
- is_uv3_hub() ? UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT : \
- /*is_uv4_hub*/ UV4H_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT)
-#define UV4H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK \
- uv_undefined("UV4H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK")
-#define UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK ( \
- is_uv2_hub() ? UV2H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK : \
- is_uv3_hub() ? UV3H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK : \
- /*is_uv4_hub*/ UV4H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK)
-#define UV4H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT \
- uv_undefined("UV4H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT")
-#define UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT ( \
- is_uv2_hub() ? UV2H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT : \
- is_uv3_hub() ? UV3H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT : \
- /*is_uv4_hub*/ UV4H_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT)
-
-union uvh_lb_bau_misc_control_u {
- unsigned long v;
- struct uvh_lb_bau_misc_control_s {
- unsigned long rejection_delay:8; /* RW */
- unsigned long apic_mode:1; /* RW */
- unsigned long force_broadcast:1; /* RW */
- unsigned long force_lock_nop:1; /* RW */
- unsigned long qpi_agent_presence_vector:3; /* RW */
- unsigned long descriptor_fetch_mode:1; /* RW */
- unsigned long rsvd_15_19:5;
- unsigned long enable_dual_mapping_mode:1; /* RW */
- unsigned long vga_io_port_decode_enable:1; /* RW */
- unsigned long vga_io_port_16_bit_decode:1; /* RW */
- unsigned long suppress_dest_registration:1; /* RW */
- unsigned long programmed_initial_priority:3; /* RW */
- unsigned long use_incoming_priority:1; /* RW */
- unsigned long enable_programmed_initial_priority:1;/* RW */
- unsigned long rsvd_29_47:19;
- unsigned long fun:16; /* RW */
- } s;
- struct uvxh_lb_bau_misc_control_s {
- unsigned long rejection_delay:8; /* RW */
- unsigned long apic_mode:1; /* RW */
- unsigned long force_broadcast:1; /* RW */
- unsigned long force_lock_nop:1; /* RW */
- unsigned long qpi_agent_presence_vector:3; /* RW */
- unsigned long descriptor_fetch_mode:1; /* RW */
- unsigned long rsvd_15_19:5;
- unsigned long enable_dual_mapping_mode:1; /* RW */
- unsigned long vga_io_port_decode_enable:1; /* RW */
- unsigned long vga_io_port_16_bit_decode:1; /* RW */
- unsigned long suppress_dest_registration:1; /* RW */
- unsigned long programmed_initial_priority:3; /* RW */
- unsigned long use_incoming_priority:1; /* RW */
- unsigned long enable_programmed_initial_priority:1;/* RW */
- unsigned long enable_automatic_apic_mode_selection:1;/* RW */
- unsigned long apic_mode_status:1; /* RO */
- unsigned long suppress_interrupts_to_self:1; /* RW */
- unsigned long enable_lock_based_system_flush:1;/* RW */
- unsigned long enable_extended_sb_status:1; /* RW */
- unsigned long suppress_int_prio_udt_to_self:1;/* RW */
- unsigned long use_legacy_descriptor_formats:1;/* RW */
- unsigned long rsvd_36_47:12;
- unsigned long fun:16; /* RW */
- } sx;
- struct uv2h_lb_bau_misc_control_s {
- unsigned long rejection_delay:8; /* RW */
- unsigned long apic_mode:1; /* RW */
- unsigned long force_broadcast:1; /* RW */
- unsigned long force_lock_nop:1; /* RW */
- unsigned long qpi_agent_presence_vector:3; /* RW */
- unsigned long descriptor_fetch_mode:1; /* RW */
- unsigned long enable_intd_soft_ack_mode:1; /* RW */
- unsigned long intd_soft_ack_timeout_period:4; /* RW */
- unsigned long enable_dual_mapping_mode:1; /* RW */
- unsigned long vga_io_port_decode_enable:1; /* RW */
- unsigned long vga_io_port_16_bit_decode:1; /* RW */
- unsigned long suppress_dest_registration:1; /* RW */
- unsigned long programmed_initial_priority:3; /* RW */
- unsigned long use_incoming_priority:1; /* RW */
- unsigned long enable_programmed_initial_priority:1;/* RW */
- unsigned long enable_automatic_apic_mode_selection:1;/* RW */
- unsigned long apic_mode_status:1; /* RO */
- unsigned long suppress_interrupts_to_self:1; /* RW */
- unsigned long enable_lock_based_system_flush:1;/* RW */
- unsigned long enable_extended_sb_status:1; /* RW */
- unsigned long suppress_int_prio_udt_to_self:1;/* RW */
- unsigned long use_legacy_descriptor_formats:1;/* RW */
- unsigned long rsvd_36_47:12;
- unsigned long fun:16; /* RW */
+ /* UV2 unique struct */
+ struct uv2h_int_cmpb_s {
+ unsigned long real_time_cmpb:56; /* RW */
+ unsigned long rsvd_56_63:8;
} s2;
- struct uv3h_lb_bau_misc_control_s {
- unsigned long rejection_delay:8; /* RW */
- unsigned long apic_mode:1; /* RW */
- unsigned long force_broadcast:1; /* RW */
- unsigned long force_lock_nop:1; /* RW */
- unsigned long qpi_agent_presence_vector:3; /* RW */
- unsigned long descriptor_fetch_mode:1; /* RW */
- unsigned long enable_intd_soft_ack_mode:1; /* RW */
- unsigned long intd_soft_ack_timeout_period:4; /* RW */
- unsigned long enable_dual_mapping_mode:1; /* RW */
- unsigned long vga_io_port_decode_enable:1; /* RW */
- unsigned long vga_io_port_16_bit_decode:1; /* RW */
- unsigned long suppress_dest_registration:1; /* RW */
- unsigned long programmed_initial_priority:3; /* RW */
- unsigned long use_incoming_priority:1; /* RW */
- unsigned long enable_programmed_initial_priority:1;/* RW */
- unsigned long enable_automatic_apic_mode_selection:1;/* RW */
- unsigned long apic_mode_status:1; /* RO */
- unsigned long suppress_interrupts_to_self:1; /* RW */
- unsigned long enable_lock_based_system_flush:1;/* RW */
- unsigned long enable_extended_sb_status:1; /* RW */
- unsigned long suppress_int_prio_udt_to_self:1;/* RW */
- unsigned long use_legacy_descriptor_formats:1;/* RW */
- unsigned long suppress_quiesce_msgs_to_qpi:1; /* RW */
- unsigned long enable_intd_prefetch_hint:1; /* RW */
- unsigned long thread_kill_timebase:8; /* RW */
- unsigned long rsvd_46_47:2;
- unsigned long fun:16; /* RW */
- } s3;
- struct uv4h_lb_bau_misc_control_s {
- unsigned long rejection_delay:8; /* RW */
- unsigned long apic_mode:1; /* RW */
- unsigned long force_broadcast:1; /* RW */
- unsigned long force_lock_nop:1; /* RW */
- unsigned long qpi_agent_presence_vector:3; /* RW */
- unsigned long descriptor_fetch_mode:1; /* RW */
- unsigned long rsvd_15_19:5;
- unsigned long enable_dual_mapping_mode:1; /* RW */
- unsigned long vga_io_port_decode_enable:1; /* RW */
- unsigned long vga_io_port_16_bit_decode:1; /* RW */
- unsigned long suppress_dest_registration:1; /* RW */
- unsigned long programmed_initial_priority:3; /* RW */
- unsigned long use_incoming_priority:1; /* RW */
- unsigned long enable_programmed_initial_priority:1;/* RW */
- unsigned long enable_automatic_apic_mode_selection:1;/* RW */
- unsigned long apic_mode_status:1; /* RO */
- unsigned long suppress_interrupts_to_self:1; /* RW */
- unsigned long enable_lock_based_system_flush:1;/* RW */
- unsigned long enable_extended_sb_status:1; /* RW */
- unsigned long suppress_int_prio_udt_to_self:1;/* RW */
- unsigned long use_legacy_descriptor_formats:1;/* RW */
- unsigned long suppress_quiesce_msgs_to_qpi:1; /* RW */
- unsigned long rsvd_37:1;
- unsigned long thread_kill_timebase:8; /* RW */
- unsigned long address_interleave_select:1; /* RW */
- unsigned long rsvd_47:1;
- unsigned long fun:16; /* RW */
- } s4;
};
/* ========================================================================= */
-/* UVH_LB_BAU_SB_ACTIVATION_CONTROL */
+/* UVH_IPI_INT */
/* ========================================================================= */
-#define UV2H_LB_BAU_SB_ACTIVATION_CONTROL 0x320020UL
-#define UV3H_LB_BAU_SB_ACTIVATION_CONTROL 0x320020UL
-#define UV4H_LB_BAU_SB_ACTIVATION_CONTROL 0xc8020UL
-#define UVH_LB_BAU_SB_ACTIVATION_CONTROL ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_ACTIVATION_CONTROL : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_ACTIVATION_CONTROL : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_ACTIVATION_CONTROL)
-
-#define UV2H_LB_BAU_SB_ACTIVATION_CONTROL_32 0x9a8
-#define UV3H_LB_BAU_SB_ACTIVATION_CONTROL_32 0x9a8
-#define UV4H_LB_BAU_SB_ACTIVATION_CONTROL_32 0x9c8
-#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_32 ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_ACTIVATION_CONTROL_32 : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_ACTIVATION_CONTROL_32 : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_ACTIVATION_CONTROL_32)
-
-#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_INDEX_SHFT 0
-#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT 62
-#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_INIT_SHFT 63
-#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_INDEX_MASK 0x000000000000003fUL
-#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_MASK 0x4000000000000000UL
-#define UVH_LB_BAU_SB_ACTIVATION_CONTROL_INIT_MASK 0x8000000000000000UL
-
-
-union uvh_lb_bau_sb_activation_control_u {
- unsigned long v;
- struct uvh_lb_bau_sb_activation_control_s {
- unsigned long index:6; /* RW */
- unsigned long rsvd_6_61:56;
- unsigned long push:1; /* WP */
- unsigned long init:1; /* WP */
- } s;
-};
+#define UVH_IPI_INT 0x60500UL
-/* ========================================================================= */
-/* UVH_LB_BAU_SB_ACTIVATION_STATUS_0 */
-/* ========================================================================= */
-#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_0 0x320030UL
-#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_0 0x320030UL
-#define UV4H_LB_BAU_SB_ACTIVATION_STATUS_0 0xc8030UL
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0 ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_ACTIVATION_STATUS_0 : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_ACTIVATION_STATUS_0 : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_ACTIVATION_STATUS_0)
-
-#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_0_32 0x9b0
-#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_0_32 0x9b0
-#define UV4H_LB_BAU_SB_ACTIVATION_STATUS_0_32 0x9d0
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0_32 ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_ACTIVATION_STATUS_0_32 : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_ACTIVATION_STATUS_0_32 : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_ACTIVATION_STATUS_0_32)
-
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0_STATUS_SHFT 0
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_0_STATUS_MASK 0xffffffffffffffffUL
-
-
-union uvh_lb_bau_sb_activation_status_0_u {
- unsigned long v;
- struct uvh_lb_bau_sb_activation_status_0_s {
- unsigned long status:64; /* RW */
- } s;
-};
+/* UVH common defines*/
+#define UVH_IPI_INT_VECTOR_SHFT 0
+#define UVH_IPI_INT_VECTOR_MASK 0x00000000000000ffUL
+#define UVH_IPI_INT_DELIVERY_MODE_SHFT 8
+#define UVH_IPI_INT_DELIVERY_MODE_MASK 0x0000000000000700UL
+#define UVH_IPI_INT_DESTMODE_SHFT 11
+#define UVH_IPI_INT_DESTMODE_MASK 0x0000000000000800UL
+#define UVH_IPI_INT_APIC_ID_SHFT 16
+#define UVH_IPI_INT_APIC_ID_MASK 0x0000ffffffff0000UL
+#define UVH_IPI_INT_SEND_SHFT 63
+#define UVH_IPI_INT_SEND_MASK 0x8000000000000000UL
-/* ========================================================================= */
-/* UVH_LB_BAU_SB_ACTIVATION_STATUS_1 */
-/* ========================================================================= */
-#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_1 0x320040UL
-#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_1 0x320040UL
-#define UV4H_LB_BAU_SB_ACTIVATION_STATUS_1 0xc8040UL
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1 ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_ACTIVATION_STATUS_1 : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_ACTIVATION_STATUS_1 : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_ACTIVATION_STATUS_1)
-
-#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_1_32 0x9b8
-#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_1_32 0x9b8
-#define UV4H_LB_BAU_SB_ACTIVATION_STATUS_1_32 0x9d8
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1_32 ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_ACTIVATION_STATUS_1_32 : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_ACTIVATION_STATUS_1_32 : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_ACTIVATION_STATUS_1_32)
-
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1_STATUS_SHFT 0
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_1_STATUS_MASK 0xffffffffffffffffUL
-
-
-union uvh_lb_bau_sb_activation_status_1_u {
+
+union uvh_ipi_int_u {
unsigned long v;
- struct uvh_lb_bau_sb_activation_status_1_s {
- unsigned long status:64; /* RW */
+
+ /* UVH common struct */
+ struct uvh_ipi_int_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long delivery_mode:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long rsvd_12_15:4;
+ unsigned long apic_id:32; /* RW */
+ unsigned long rsvd_48_62:15;
+ unsigned long send:1; /* WP */
} s;
-};
-/* ========================================================================= */
-/* UVH_LB_BAU_SB_DESCRIPTOR_BASE */
-/* ========================================================================= */
-#define UV2H_LB_BAU_SB_DESCRIPTOR_BASE 0x320010UL
-#define UV3H_LB_BAU_SB_DESCRIPTOR_BASE 0x320010UL
-#define UV4H_LB_BAU_SB_DESCRIPTOR_BASE 0xc8010UL
-#define UVH_LB_BAU_SB_DESCRIPTOR_BASE ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_DESCRIPTOR_BASE : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_DESCRIPTOR_BASE : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_DESCRIPTOR_BASE)
-
-#define UV2H_LB_BAU_SB_DESCRIPTOR_BASE_32 0x9a0
-#define UV3H_LB_BAU_SB_DESCRIPTOR_BASE_32 0x9a0
-#define UV4H_LB_BAU_SB_DESCRIPTOR_BASE_32 0x9c0
-#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_32 ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_DESCRIPTOR_BASE_32 : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_DESCRIPTOR_BASE_32 : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_DESCRIPTOR_BASE_32)
-
-#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_SHFT 12
-
-#define UV2H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT 49
-#define UV2H_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_MASK 0x000007fffffff000UL
-#define UV2H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK 0x7ffe000000000000UL
-
-#define UV3H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT 49
-#define UV3H_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_MASK 0x000007fffffff000UL
-#define UV3H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK 0x7ffe000000000000UL
-
-#define UV4H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT 49
-#define UV4H_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_MASK 0x00003ffffffff000UL
-#define UV4H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK 0x7ffe000000000000UL
-
-#define UV4AH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT 53
-#define UV4AH_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_MASK 0x000ffffffffff000UL
-#define UV4AH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK 0xffe0000000000000UL
-
-#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT : \
- is_uv4a_hub() ? UV4AH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT)
-
-#define UVH_LB_BAU_SB_DESCRIPTOR_PAGE_ADDRESS_MASK ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_DESCRIPTOR_PAGE_ADDRESS_MASK : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_DESCRIPTOR_PAGE_ADDRESS_MASK : \
- is_uv4a_hub() ? UV4AH_LB_BAU_SB_DESCRIPTOR_PAGE_ADDRESS_MASK : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_DESCRIPTOR_PAGE_ADDRESS_MASK)
-
-#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK : \
- is_uv4a_hub() ? UV4AH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK)
+ /* UV5 unique struct */
+ struct uv5h_ipi_int_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long delivery_mode:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long rsvd_12_15:4;
+ unsigned long apic_id:32; /* RW */
+ unsigned long rsvd_48_62:15;
+ unsigned long send:1; /* WP */
+ } s5;
+
+ /* UV4 unique struct */
+ struct uv4h_ipi_int_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long delivery_mode:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long rsvd_12_15:4;
+ unsigned long apic_id:32; /* RW */
+ unsigned long rsvd_48_62:15;
+ unsigned long send:1; /* WP */
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_ipi_int_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long delivery_mode:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long rsvd_12_15:4;
+ unsigned long apic_id:32; /* RW */
+ unsigned long rsvd_48_62:15;
+ unsigned long send:1; /* WP */
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_ipi_int_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long delivery_mode:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long rsvd_12_15:4;
+ unsigned long apic_id:32; /* RW */
+ unsigned long rsvd_48_62:15;
+ unsigned long send:1; /* WP */
+ } s2;
+};
/* ========================================================================= */
/* UVH_NODE_ID */
/* ========================================================================= */
#define UVH_NODE_ID 0x0UL
-#define UV2H_NODE_ID 0x0UL
-#define UV3H_NODE_ID 0x0UL
-#define UV4H_NODE_ID 0x0UL
+/* UVH common defines*/
#define UVH_NODE_ID_FORCE1_SHFT 0
-#define UVH_NODE_ID_MANUFACTURER_SHFT 1
-#define UVH_NODE_ID_PART_NUMBER_SHFT 12
-#define UVH_NODE_ID_REVISION_SHFT 28
-#define UVH_NODE_ID_NODE_ID_SHFT 32
#define UVH_NODE_ID_FORCE1_MASK 0x0000000000000001UL
+#define UVH_NODE_ID_MANUFACTURER_SHFT 1
#define UVH_NODE_ID_MANUFACTURER_MASK 0x0000000000000ffeUL
+#define UVH_NODE_ID_PART_NUMBER_SHFT 12
#define UVH_NODE_ID_PART_NUMBER_MASK 0x000000000ffff000UL
+#define UVH_NODE_ID_REVISION_SHFT 28
#define UVH_NODE_ID_REVISION_MASK 0x00000000f0000000UL
-#define UVH_NODE_ID_NODE_ID_MASK 0x00007fff00000000UL
+#define UVH_NODE_ID_NODE_ID_SHFT 32
+#define UVH_NODE_ID_NI_PORT_SHFT 57
-#define UVXH_NODE_ID_FORCE1_SHFT 0
-#define UVXH_NODE_ID_MANUFACTURER_SHFT 1
-#define UVXH_NODE_ID_PART_NUMBER_SHFT 12
-#define UVXH_NODE_ID_REVISION_SHFT 28
-#define UVXH_NODE_ID_NODE_ID_SHFT 32
-#define UVXH_NODE_ID_NODES_PER_BIT_SHFT 50
-#define UVXH_NODE_ID_NI_PORT_SHFT 57
-#define UVXH_NODE_ID_FORCE1_MASK 0x0000000000000001UL
-#define UVXH_NODE_ID_MANUFACTURER_MASK 0x0000000000000ffeUL
-#define UVXH_NODE_ID_PART_NUMBER_MASK 0x000000000ffff000UL
-#define UVXH_NODE_ID_REVISION_MASK 0x00000000f0000000UL
+/* UVXH common defines */
#define UVXH_NODE_ID_NODE_ID_MASK 0x00007fff00000000UL
+#define UVXH_NODE_ID_NODES_PER_BIT_SHFT 50
#define UVXH_NODE_ID_NODES_PER_BIT_MASK 0x01fc000000000000UL
#define UVXH_NODE_ID_NI_PORT_MASK 0x3e00000000000000UL
-#define UV2H_NODE_ID_FORCE1_SHFT 0
-#define UV2H_NODE_ID_MANUFACTURER_SHFT 1
-#define UV2H_NODE_ID_PART_NUMBER_SHFT 12
-#define UV2H_NODE_ID_REVISION_SHFT 28
-#define UV2H_NODE_ID_NODE_ID_SHFT 32
-#define UV2H_NODE_ID_NODES_PER_BIT_SHFT 50
-#define UV2H_NODE_ID_NI_PORT_SHFT 57
-#define UV2H_NODE_ID_FORCE1_MASK 0x0000000000000001UL
-#define UV2H_NODE_ID_MANUFACTURER_MASK 0x0000000000000ffeUL
-#define UV2H_NODE_ID_PART_NUMBER_MASK 0x000000000ffff000UL
-#define UV2H_NODE_ID_REVISION_MASK 0x00000000f0000000UL
-#define UV2H_NODE_ID_NODE_ID_MASK 0x00007fff00000000UL
-#define UV2H_NODE_ID_NODES_PER_BIT_MASK 0x01fc000000000000UL
-#define UV2H_NODE_ID_NI_PORT_MASK 0x3e00000000000000UL
-
-#define UV3H_NODE_ID_FORCE1_SHFT 0
-#define UV3H_NODE_ID_MANUFACTURER_SHFT 1
-#define UV3H_NODE_ID_PART_NUMBER_SHFT 12
-#define UV3H_NODE_ID_REVISION_SHFT 28
-#define UV3H_NODE_ID_NODE_ID_SHFT 32
-#define UV3H_NODE_ID_ROUTER_SELECT_SHFT 48
-#define UV3H_NODE_ID_RESERVED_2_SHFT 49
-#define UV3H_NODE_ID_NODES_PER_BIT_SHFT 50
-#define UV3H_NODE_ID_NI_PORT_SHFT 57
-#define UV3H_NODE_ID_FORCE1_MASK 0x0000000000000001UL
-#define UV3H_NODE_ID_MANUFACTURER_MASK 0x0000000000000ffeUL
-#define UV3H_NODE_ID_PART_NUMBER_MASK 0x000000000ffff000UL
-#define UV3H_NODE_ID_REVISION_MASK 0x00000000f0000000UL
-#define UV3H_NODE_ID_NODE_ID_MASK 0x00007fff00000000UL
-#define UV3H_NODE_ID_ROUTER_SELECT_MASK 0x0001000000000000UL
-#define UV3H_NODE_ID_RESERVED_2_MASK 0x0002000000000000UL
-#define UV3H_NODE_ID_NODES_PER_BIT_MASK 0x01fc000000000000UL
-#define UV3H_NODE_ID_NI_PORT_MASK 0x3e00000000000000UL
-
-#define UV4H_NODE_ID_FORCE1_SHFT 0
-#define UV4H_NODE_ID_MANUFACTURER_SHFT 1
-#define UV4H_NODE_ID_PART_NUMBER_SHFT 12
-#define UV4H_NODE_ID_REVISION_SHFT 28
-#define UV4H_NODE_ID_NODE_ID_SHFT 32
+/* UVYH common defines */
+#define UVYH_NODE_ID_NODE_ID_MASK 0x0000007f00000000UL
+#define UVYH_NODE_ID_NI_PORT_MASK 0x7e00000000000000UL
+
+/* UV4 unique defines */
#define UV4H_NODE_ID_ROUTER_SELECT_SHFT 48
-#define UV4H_NODE_ID_RESERVED_2_SHFT 49
-#define UV4H_NODE_ID_NODES_PER_BIT_SHFT 50
-#define UV4H_NODE_ID_NI_PORT_SHFT 57
-#define UV4H_NODE_ID_FORCE1_MASK 0x0000000000000001UL
-#define UV4H_NODE_ID_MANUFACTURER_MASK 0x0000000000000ffeUL
-#define UV4H_NODE_ID_PART_NUMBER_MASK 0x000000000ffff000UL
-#define UV4H_NODE_ID_REVISION_MASK 0x00000000f0000000UL
-#define UV4H_NODE_ID_NODE_ID_MASK 0x00007fff00000000UL
#define UV4H_NODE_ID_ROUTER_SELECT_MASK 0x0001000000000000UL
+#define UV4H_NODE_ID_RESERVED_2_SHFT 49
#define UV4H_NODE_ID_RESERVED_2_MASK 0x0002000000000000UL
-#define UV4H_NODE_ID_NODES_PER_BIT_MASK 0x01fc000000000000UL
-#define UV4H_NODE_ID_NI_PORT_MASK 0x3e00000000000000UL
+
+/* UV3 unique defines */
+#define UV3H_NODE_ID_ROUTER_SELECT_SHFT 48
+#define UV3H_NODE_ID_ROUTER_SELECT_MASK 0x0001000000000000UL
+#define UV3H_NODE_ID_RESERVED_2_SHFT 49
+#define UV3H_NODE_ID_RESERVED_2_MASK 0x0002000000000000UL
union uvh_node_id_u {
unsigned long v;
+
+ /* UVH common struct */
struct uvh_node_id_s {
unsigned long force1:1; /* RO */
unsigned long manufacturer:11; /* RO */
unsigned long part_number:16; /* RO */
unsigned long revision:4; /* RO */
- unsigned long node_id:15; /* RW */
- unsigned long rsvd_47_63:17;
+ unsigned long rsvd_32_63:32;
} s;
+
+ /* UVXH common struct */
struct uvxh_node_id_s {
unsigned long force1:1; /* RO */
unsigned long manufacturer:11; /* RO */
unsigned long ni_port:5; /* RO */
unsigned long rsvd_62_63:2;
} sx;
- struct uv2h_node_id_s {
+
+ /* UVYH common struct */
+ struct uvyh_node_id_s {
+ unsigned long force1:1; /* RO */
+ unsigned long manufacturer:11; /* RO */
+ unsigned long part_number:16; /* RO */
+ unsigned long revision:4; /* RO */
+ unsigned long node_id:7; /* RW */
+ unsigned long rsvd_39_56:18;
+ unsigned long ni_port:6; /* RO */
+ unsigned long rsvd_63:1;
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_node_id_s {
+ unsigned long force1:1; /* RO */
+ unsigned long manufacturer:11; /* RO */
+ unsigned long part_number:16; /* RO */
+ unsigned long revision:4; /* RO */
+ unsigned long node_id:7; /* RW */
+ unsigned long rsvd_39_56:18;
+ unsigned long ni_port:6; /* RO */
+ unsigned long rsvd_63:1;
+ } s5;
+
+ /* UV4 unique struct */
+ struct uv4h_node_id_s {
unsigned long force1:1; /* RO */
unsigned long manufacturer:11; /* RO */
unsigned long part_number:16; /* RO */
unsigned long revision:4; /* RO */
unsigned long node_id:15; /* RW */
- unsigned long rsvd_47_49:3;
+ unsigned long rsvd_47:1;
+ unsigned long router_select:1; /* RO */
+ unsigned long rsvd_49:1;
unsigned long nodes_per_bit:7; /* RO */
unsigned long ni_port:5; /* RO */
unsigned long rsvd_62_63:2;
- } s2;
+ } s4;
+
+ /* UV3 unique struct */
struct uv3h_node_id_s {
unsigned long force1:1; /* RO */
unsigned long manufacturer:11; /* RO */
unsigned long ni_port:5; /* RO */
unsigned long rsvd_62_63:2;
} s3;
- struct uv4h_node_id_s {
+
+ /* UV2 unique struct */
+ struct uv2h_node_id_s {
unsigned long force1:1; /* RO */
unsigned long manufacturer:11; /* RO */
unsigned long part_number:16; /* RO */
unsigned long revision:4; /* RO */
unsigned long node_id:15; /* RW */
- unsigned long rsvd_47:1;
- unsigned long router_select:1; /* RO */
- unsigned long rsvd_49:1;
+ unsigned long rsvd_47_49:3;
unsigned long nodes_per_bit:7; /* RO */
unsigned long ni_port:5; /* RO */
unsigned long rsvd_62_63:2;
- } s4;
+ } s2;
+};
+
+/* ========================================================================= */
+/* UVH_NODE_PRESENT_0 */
+/* ========================================================================= */
+#define UVH_NODE_PRESENT_0 ( \
+ is_uv(UV5) ? 0x1400UL : \
+ 0)
+
+
+/* UVYH common defines */
+#define UVYH_NODE_PRESENT_0_NODES_SHFT 0
+#define UVYH_NODE_PRESENT_0_NODES_MASK 0xffffffffffffffffUL
+
+
+union uvh_node_present_0_u {
+ unsigned long v;
+
+ /* UVH common struct */
+ struct uvh_node_present_0_s {
+ unsigned long nodes:64; /* RW */
+ } s;
+
+ /* UVYH common struct */
+ struct uvyh_node_present_0_s {
+ unsigned long nodes:64; /* RW */
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_node_present_0_s {
+ unsigned long nodes:64; /* RW */
+ } s5;
+};
+
+/* ========================================================================= */
+/* UVH_NODE_PRESENT_1 */
+/* ========================================================================= */
+#define UVH_NODE_PRESENT_1 ( \
+ is_uv(UV5) ? 0x1408UL : \
+ 0)
+
+
+/* UVYH common defines */
+#define UVYH_NODE_PRESENT_1_NODES_SHFT 0
+#define UVYH_NODE_PRESENT_1_NODES_MASK 0xffffffffffffffffUL
+
+
+union uvh_node_present_1_u {
+ unsigned long v;
+
+ /* UVH common struct */
+ struct uvh_node_present_1_s {
+ unsigned long nodes:64; /* RW */
+ } s;
+
+ /* UVYH common struct */
+ struct uvyh_node_present_1_s {
+ unsigned long nodes:64; /* RW */
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_node_present_1_s {
+ unsigned long nodes:64; /* RW */
+ } s5;
};
/* ========================================================================= */
/* UVH_NODE_PRESENT_TABLE */
/* ========================================================================= */
-#define UVH_NODE_PRESENT_TABLE 0x1400UL
+#define UVH_NODE_PRESENT_TABLE ( \
+ is_uv(UV4) ? 0x1400UL : \
+ is_uv(UV3) ? 0x1400UL : \
+ is_uv(UV2) ? 0x1400UL : \
+ 0)
-#define UV2H_NODE_PRESENT_TABLE_DEPTH 16
-#define UV3H_NODE_PRESENT_TABLE_DEPTH 16
-#define UV4H_NODE_PRESENT_TABLE_DEPTH 4
#define UVH_NODE_PRESENT_TABLE_DEPTH ( \
- is_uv2_hub() ? UV2H_NODE_PRESENT_TABLE_DEPTH : \
- is_uv3_hub() ? UV3H_NODE_PRESENT_TABLE_DEPTH : \
- /*is_uv4_hub*/ UV4H_NODE_PRESENT_TABLE_DEPTH)
+ is_uv(UV4) ? 4 : \
+ is_uv(UV3) ? 16 : \
+ is_uv(UV2) ? 16 : \
+ 0)
+
-#define UVH_NODE_PRESENT_TABLE_NODES_SHFT 0
-#define UVH_NODE_PRESENT_TABLE_NODES_MASK 0xffffffffffffffffUL
+/* UVXH common defines */
+#define UVXH_NODE_PRESENT_TABLE_NODES_SHFT 0
+#define UVXH_NODE_PRESENT_TABLE_NODES_MASK 0xffffffffffffffffUL
union uvh_node_present_table_u {
unsigned long v;
+
+ /* UVH common struct */
struct uvh_node_present_table_s {
unsigned long nodes:64; /* RW */
} s;
+
+ /* UVXH common struct */
+ struct uvxh_node_present_table_s {
+ unsigned long nodes:64; /* RW */
+ } sx;
+
+ /* UV4 unique struct */
+ struct uv4h_node_present_table_s {
+ unsigned long nodes:64; /* RW */
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_node_present_table_s {
+ unsigned long nodes:64; /* RW */
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_node_present_table_s {
+ unsigned long nodes:64; /* RW */
+ } s2;
};
/* ========================================================================= */
-/* UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR */
+/* UVH_RH10_GAM_ADDR_MAP_CONFIG */
/* ========================================================================= */
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR 0x16000c8UL
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR 0x16000c8UL
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR 0x4800c8UL
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR)
-
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_SHFT 24
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_SHFT 48
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_SHFT 63
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_MASK 0x00000000ff000000UL
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_SHFT 24
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_SHFT 48
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_SHFT 63
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_MASK 0x00000000ff000000UL
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_SHFT 24
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_SHFT 48
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_SHFT 63
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_MASK 0x00000000ff000000UL
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_SHFT 24
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_SHFT 48
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_SHFT 63
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_MASK 0x00000000ff000000UL
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_SHFT 24
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_SHFT 48
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_SHFT 63
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_MASK 0x00000000ff000000UL
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_MASK 0x8000000000000000UL
-
-
-union uvh_rh_gam_alias210_overlay_config_0_mmr_u {
+#define UVH_RH10_GAM_ADDR_MAP_CONFIG ( \
+ is_uv(UV5) ? 0x470000UL : \
+ 0)
+
+
+/* UVYH common defines */
+#define UVYH_RH10_GAM_ADDR_MAP_CONFIG_N_SKT_SHFT 6
+#define UVYH_RH10_GAM_ADDR_MAP_CONFIG_N_SKT_MASK 0x00000000000001c0UL
+#define UVYH_RH10_GAM_ADDR_MAP_CONFIG_LS_ENABLE_SHFT 12
+#define UVYH_RH10_GAM_ADDR_MAP_CONFIG_LS_ENABLE_MASK 0x0000000000001000UL
+#define UVYH_RH10_GAM_ADDR_MAP_CONFIG_MK_TME_KEYID_BITS_SHFT 16
+#define UVYH_RH10_GAM_ADDR_MAP_CONFIG_MK_TME_KEYID_BITS_MASK 0x00000000000f0000UL
+
+
+union uvh_rh10_gam_addr_map_config_u {
unsigned long v;
- struct uvh_rh_gam_alias210_overlay_config_0_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
- unsigned long enable:1; /* RW */
+
+ /* UVH common struct */
+ struct uvh_rh10_gam_addr_map_config_s {
+ unsigned long undef_0_5:6; /* Undefined */
+ unsigned long n_skt:3; /* RW */
+ unsigned long undef_9_11:3; /* Undefined */
+ unsigned long ls_enable:1; /* RW */
+ unsigned long undef_13_15:3; /* Undefined */
+ unsigned long mk_tme_keyid_bits:4; /* RW */
+ unsigned long rsvd_20_63:44;
} s;
- struct uvxh_rh_gam_alias210_overlay_config_0_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
- unsigned long enable:1; /* RW */
- } sx;
- struct uv2h_rh_gam_alias210_overlay_config_0_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
- unsigned long enable:1; /* RW */
- } s2;
- struct uv3h_rh_gam_alias210_overlay_config_0_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
- unsigned long enable:1; /* RW */
- } s3;
- struct uv4h_rh_gam_alias210_overlay_config_0_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
- unsigned long enable:1; /* RW */
- } s4;
+
+ /* UVYH common struct */
+ struct uvyh_rh10_gam_addr_map_config_s {
+ unsigned long undef_0_5:6; /* Undefined */
+ unsigned long n_skt:3; /* RW */
+ unsigned long undef_9_11:3; /* Undefined */
+ unsigned long ls_enable:1; /* RW */
+ unsigned long undef_13_15:3; /* Undefined */
+ unsigned long mk_tme_keyid_bits:4; /* RW */
+ unsigned long rsvd_20_63:44;
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_rh10_gam_addr_map_config_s {
+ unsigned long undef_0_5:6; /* Undefined */
+ unsigned long n_skt:3; /* RW */
+ unsigned long undef_9_11:3; /* Undefined */
+ unsigned long ls_enable:1; /* RW */
+ unsigned long undef_13_15:3; /* Undefined */
+ unsigned long mk_tme_keyid_bits:4; /* RW */
+ } s5;
};
/* ========================================================================= */
-/* UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR */
+/* UVH_RH10_GAM_GRU_OVERLAY_CONFIG */
/* ========================================================================= */
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR 0x16000d8UL
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR 0x16000d8UL
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR 0x4800d8UL
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR)
-
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_SHFT 24
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_SHFT 48
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_SHFT 63
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_MASK 0x00000000ff000000UL
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_SHFT 24
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_SHFT 48
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_SHFT 63
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_MASK 0x00000000ff000000UL
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_SHFT 24
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_SHFT 48
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_SHFT 63
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_MASK 0x00000000ff000000UL
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_SHFT 24
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_SHFT 48
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_SHFT 63
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_MASK 0x00000000ff000000UL
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_SHFT 24
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_SHFT 48
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_SHFT 63
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_MASK 0x00000000ff000000UL
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_MASK 0x8000000000000000UL
-
-
-union uvh_rh_gam_alias210_overlay_config_1_mmr_u {
+#define UVH_RH10_GAM_GRU_OVERLAY_CONFIG ( \
+ is_uv(UV5) ? 0x4700b0UL : \
+ 0)
+
+
+/* UVYH common defines */
+#define UVYH_RH10_GAM_GRU_OVERLAY_CONFIG_BASE_SHFT 25
+#define UVYH_RH10_GAM_GRU_OVERLAY_CONFIG_BASE_MASK 0x000ffffffe000000UL
+#define UVYH_RH10_GAM_GRU_OVERLAY_CONFIG_N_GRU_SHFT 52
+#define UVYH_RH10_GAM_GRU_OVERLAY_CONFIG_N_GRU_MASK 0x0070000000000000UL
+#define UVYH_RH10_GAM_GRU_OVERLAY_CONFIG_ENABLE_SHFT 63
+#define UVYH_RH10_GAM_GRU_OVERLAY_CONFIG_ENABLE_MASK 0x8000000000000000UL
+
+#define UVH_RH10_GAM_GRU_OVERLAY_CONFIG_BASE_MASK ( \
+ is_uv(UV5) ? 0x000ffffffe000000UL : \
+ 0)
+#define UVH_RH10_GAM_GRU_OVERLAY_CONFIG_BASE_SHFT ( \
+ is_uv(UV5) ? 25 : \
+ -1)
+
+union uvh_rh10_gam_gru_overlay_config_u {
unsigned long v;
- struct uvh_rh_gam_alias210_overlay_config_1_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
- unsigned long enable:1; /* RW */
- } s;
- struct uvxh_rh_gam_alias210_overlay_config_1_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
- unsigned long enable:1; /* RW */
- } sx;
- struct uv2h_rh_gam_alias210_overlay_config_1_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
- unsigned long enable:1; /* RW */
- } s2;
- struct uv3h_rh_gam_alias210_overlay_config_1_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
- unsigned long enable:1; /* RW */
- } s3;
- struct uv4h_rh_gam_alias210_overlay_config_1_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
+
+ /* UVH common struct */
+ struct uvh_rh10_gam_gru_overlay_config_s {
+ unsigned long undef_0_24:25; /* Undefined */
+ unsigned long base:27; /* RW */
+ unsigned long n_gru:3; /* RW */
+ unsigned long undef_55_62:8; /* Undefined */
unsigned long enable:1; /* RW */
- } s4;
+ } s;
+
+ /* UVYH common struct */
+ struct uvyh_rh10_gam_gru_overlay_config_s {
+ unsigned long undef_0_24:25; /* Undefined */
+ unsigned long base:27; /* RW */
+ unsigned long n_gru:3; /* RW */
+ unsigned long undef_55_62:8; /* Undefined */
+ unsigned long enable:1; /* RW */
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_rh10_gam_gru_overlay_config_s {
+ unsigned long undef_0_24:25; /* Undefined */
+ unsigned long base:27; /* RW */
+ unsigned long n_gru:3; /* RW */
+ unsigned long undef_55_62:8; /* Undefined */
+ unsigned long enable:1; /* RW */
+ } s5;
};
/* ========================================================================= */
-/* UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR */
+/* UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG0 */
/* ========================================================================= */
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR 0x16000e8UL
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR 0x16000e8UL
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR 0x4800e8UL
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR)
-
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_SHFT 24
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_SHFT 48
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_SHFT 63
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_MASK 0x00000000ff000000UL
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_SHFT 24
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_SHFT 48
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_SHFT 63
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_MASK 0x00000000ff000000UL
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_SHFT 24
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_SHFT 48
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_SHFT 63
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_MASK 0x00000000ff000000UL
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_SHFT 24
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_SHFT 48
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_SHFT 63
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_MASK 0x00000000ff000000UL
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_SHFT 24
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_SHFT 48
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_SHFT 63
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_MASK 0x00000000ff000000UL
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_MASK 0x001f000000000000UL
-#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_MASK 0x8000000000000000UL
-
-
-union uvh_rh_gam_alias210_overlay_config_2_mmr_u {
+#define UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG0 ( \
+ is_uv(UV5) ? 0x473000UL : \
+ 0)
+
+
+/* UVYH common defines */
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG0_BASE_SHFT 26
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG0_BASE_MASK 0x000ffffffc000000UL
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG0_M_IO_SHFT 52
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG0_M_IO_MASK 0x03f0000000000000UL
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG0_ENABLE_SHFT 63
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG0_ENABLE_MASK 0x8000000000000000UL
+
+#define UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG0_BASE_MASK ( \
+ is_uv(UV5) ? 0x000ffffffc000000UL : \
+ 0)
+#define UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG0_BASE_SHFT ( \
+ is_uv(UV5) ? 26 : \
+ -1)
+
+union uvh_rh10_gam_mmioh_overlay_config0_u {
unsigned long v;
- struct uvh_rh_gam_alias210_overlay_config_2_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
+
+ /* UVH common struct */
+ struct uvh_rh10_gam_mmioh_overlay_config0_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:26; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long undef_62:1; /* Undefined */
unsigned long enable:1; /* RW */
} s;
- struct uvxh_rh_gam_alias210_overlay_config_2_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
- unsigned long enable:1; /* RW */
- } sx;
- struct uv2h_rh_gam_alias210_overlay_config_2_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
- unsigned long enable:1; /* RW */
- } s2;
- struct uv3h_rh_gam_alias210_overlay_config_2_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
+
+ /* UVYH common struct */
+ struct uvyh_rh10_gam_mmioh_overlay_config0_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:26; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long undef_62:1; /* Undefined */
unsigned long enable:1; /* RW */
- } s3;
- struct uv4h_rh_gam_alias210_overlay_config_2_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long base:8; /* RW */
- unsigned long rsvd_32_47:16;
- unsigned long m_alias:5; /* RW */
- unsigned long rsvd_53_62:10;
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_rh10_gam_mmioh_overlay_config0_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:26; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long undef_62:1; /* Undefined */
unsigned long enable:1; /* RW */
- } s4;
+ } s5;
};
/* ========================================================================= */
-/* UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR */
+/* UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG1 */
/* ========================================================================= */
-#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR 0x16000d0UL
-#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR 0x16000d0UL
-#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR 0x4800d0UL
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR)
+#define UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG1 ( \
+ is_uv(UV5) ? 0x474000UL : \
+ 0)
+
+
+/* UVYH common defines */
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG1_BASE_SHFT 26
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG1_BASE_MASK 0x000ffffffc000000UL
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG1_M_IO_SHFT 52
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG1_M_IO_MASK 0x03f0000000000000UL
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG1_ENABLE_SHFT 63
+#define UVYH_RH10_GAM_MMIOH_OVERLAY_CONFIG1_ENABLE_MASK 0x8000000000000000UL
+
+#define UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG1_BASE_MASK ( \
+ is_uv(UV5) ? 0x000ffffffc000000UL : \
+ 0)
+#define UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG1_BASE_SHFT ( \
+ is_uv(UV5) ? 26 : \
+ -1)
+
+union uvh_rh10_gam_mmioh_overlay_config1_u {
+ unsigned long v;
+
+ /* UVH common struct */
+ struct uvh_rh10_gam_mmioh_overlay_config1_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:26; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long undef_62:1; /* Undefined */
+ unsigned long enable:1; /* RW */
+ } s;
+
+ /* UVYH common struct */
+ struct uvyh_rh10_gam_mmioh_overlay_config1_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:26; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long undef_62:1; /* Undefined */
+ unsigned long enable:1; /* RW */
+ } sy;
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+ /* UV5 unique struct */
+ struct uv5h_rh10_gam_mmioh_overlay_config1_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:26; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long undef_62:1; /* Undefined */
+ unsigned long enable:1; /* RW */
+ } s5;
+};
-#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
-#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+/* ========================================================================= */
+/* UVH_RH10_GAM_MMIOH_REDIRECT_CONFIG0 */
+/* ========================================================================= */
+#define UVH_RH10_GAM_MMIOH_REDIRECT_CONFIG0 ( \
+ is_uv(UV5) ? 0x473800UL : \
+ 0)
-#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
-#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+#define UVH_RH10_GAM_MMIOH_REDIRECT_CONFIG0_DEPTH ( \
+ is_uv(UV5) ? 128 : \
+ 0)
-#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
-#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
-#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
-#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+/* UVYH common defines */
+#define UVYH_RH10_GAM_MMIOH_REDIRECT_CONFIG0_NASID_SHFT 0
+#define UVYH_RH10_GAM_MMIOH_REDIRECT_CONFIG0_NASID_MASK 0x000000000000007fUL
-union uvh_rh_gam_alias210_redirect_config_0_mmr_u {
+union uvh_rh10_gam_mmioh_redirect_config0_u {
unsigned long v;
- struct uvh_rh_gam_alias210_redirect_config_0_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
+
+ /* UVH common struct */
+ struct uvh_rh10_gam_mmioh_redirect_config0_s {
+ unsigned long nasid:7; /* RW */
+ unsigned long rsvd_7_63:57;
} s;
- struct uvxh_rh_gam_alias210_redirect_config_0_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } sx;
- struct uv2h_rh_gam_alias210_redirect_config_0_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } s2;
- struct uv3h_rh_gam_alias210_redirect_config_0_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } s3;
- struct uv4h_rh_gam_alias210_redirect_config_0_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } s4;
+
+ /* UVYH common struct */
+ struct uvyh_rh10_gam_mmioh_redirect_config0_s {
+ unsigned long nasid:7; /* RW */
+ unsigned long rsvd_7_63:57;
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_rh10_gam_mmioh_redirect_config0_s {
+ unsigned long nasid:7; /* RW */
+ unsigned long rsvd_7_63:57;
+ } s5;
};
/* ========================================================================= */
-/* UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR */
+/* UVH_RH10_GAM_MMIOH_REDIRECT_CONFIG1 */
/* ========================================================================= */
-#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR 0x16000e0UL
-#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR 0x16000e0UL
-#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR 0x4800e0UL
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR)
+#define UVH_RH10_GAM_MMIOH_REDIRECT_CONFIG1 ( \
+ is_uv(UV5) ? 0x474800UL : \
+ 0)
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+#define UVH_RH10_GAM_MMIOH_REDIRECT_CONFIG1_DEPTH ( \
+ is_uv(UV5) ? 128 : \
+ 0)
-#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
-#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
-#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
-#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+/* UVYH common defines */
+#define UVYH_RH10_GAM_MMIOH_REDIRECT_CONFIG1_NASID_SHFT 0
+#define UVYH_RH10_GAM_MMIOH_REDIRECT_CONFIG1_NASID_MASK 0x000000000000007fUL
-#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
-#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
-#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
-#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
-
-
-union uvh_rh_gam_alias210_redirect_config_1_mmr_u {
+union uvh_rh10_gam_mmioh_redirect_config1_u {
unsigned long v;
- struct uvh_rh_gam_alias210_redirect_config_1_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
+
+ /* UVH common struct */
+ struct uvh_rh10_gam_mmioh_redirect_config1_s {
+ unsigned long nasid:7; /* RW */
+ unsigned long rsvd_7_63:57;
} s;
- struct uvxh_rh_gam_alias210_redirect_config_1_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } sx;
- struct uv2h_rh_gam_alias210_redirect_config_1_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } s2;
- struct uv3h_rh_gam_alias210_redirect_config_1_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } s3;
- struct uv4h_rh_gam_alias210_redirect_config_1_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } s4;
+
+ /* UVYH common struct */
+ struct uvyh_rh10_gam_mmioh_redirect_config1_s {
+ unsigned long nasid:7; /* RW */
+ unsigned long rsvd_7_63:57;
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_rh10_gam_mmioh_redirect_config1_s {
+ unsigned long nasid:7; /* RW */
+ unsigned long rsvd_7_63:57;
+ } s5;
};
/* ========================================================================= */
-/* UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR */
+/* UVH_RH10_GAM_MMR_OVERLAY_CONFIG */
/* ========================================================================= */
-#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR 0x16000f0UL
-#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR 0x16000f0UL
-#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR 0x4800f0UL
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR)
+#define UVH_RH10_GAM_MMR_OVERLAY_CONFIG ( \
+ is_uv(UV5) ? 0x470090UL : \
+ 0)
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
-#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
-#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
-#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+/* UVYH common defines */
+#define UVYH_RH10_GAM_MMR_OVERLAY_CONFIG_BASE_SHFT 25
+#define UVYH_RH10_GAM_MMR_OVERLAY_CONFIG_BASE_MASK 0x000ffffffe000000UL
+#define UVYH_RH10_GAM_MMR_OVERLAY_CONFIG_ENABLE_SHFT 63
+#define UVYH_RH10_GAM_MMR_OVERLAY_CONFIG_ENABLE_MASK 0x8000000000000000UL
-#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
-#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+#define UVH_RH10_GAM_MMR_OVERLAY_CONFIG_BASE_MASK ( \
+ is_uv(UV5) ? 0x000ffffffe000000UL : \
+ 0)
+#define UVH_RH10_GAM_MMR_OVERLAY_CONFIG_BASE_SHFT ( \
+ is_uv(UV5) ? 25 : \
+ -1)
-#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
-#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+union uvh_rh10_gam_mmr_overlay_config_u {
+ unsigned long v;
-#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
-#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+ /* UVH common struct */
+ struct uvh_rh10_gam_mmr_overlay_config_s {
+ unsigned long undef_0_24:25; /* Undefined */
+ unsigned long base:27; /* RW */
+ unsigned long undef_52_62:11; /* Undefined */
+ unsigned long enable:1; /* RW */
+ } s;
+ /* UVYH common struct */
+ struct uvyh_rh10_gam_mmr_overlay_config_s {
+ unsigned long undef_0_24:25; /* Undefined */
+ unsigned long base:27; /* RW */
+ unsigned long undef_52_62:11; /* Undefined */
+ unsigned long enable:1; /* RW */
+ } sy;
-union uvh_rh_gam_alias210_redirect_config_2_mmr_u {
- unsigned long v;
- struct uvh_rh_gam_alias210_redirect_config_2_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } s;
- struct uvxh_rh_gam_alias210_redirect_config_2_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } sx;
- struct uv2h_rh_gam_alias210_redirect_config_2_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } s2;
- struct uv3h_rh_gam_alias210_redirect_config_2_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } s3;
- struct uv4h_rh_gam_alias210_redirect_config_2_mmr_s {
- unsigned long rsvd_0_23:24;
- unsigned long dest_base:22; /* RW */
- unsigned long rsvd_46_63:18;
- } s4;
+ /* UV5 unique struct */
+ struct uv5h_rh10_gam_mmr_overlay_config_s {
+ unsigned long undef_0_24:25; /* Undefined */
+ unsigned long base:27; /* RW */
+ unsigned long undef_52_62:11; /* Undefined */
+ unsigned long enable:1; /* RW */
+ } s5;
};
/* ========================================================================= */
-/* UVH_RH_GAM_CONFIG_MMR */
+/* UVH_RH_GAM_ADDR_MAP_CONFIG */
/* ========================================================================= */
-#define UV2H_RH_GAM_CONFIG_MMR 0x1600000UL
-#define UV3H_RH_GAM_CONFIG_MMR 0x1600000UL
-#define UV4H_RH_GAM_CONFIG_MMR 0x480000UL
-#define UVH_RH_GAM_CONFIG_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_CONFIG_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_CONFIG_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_CONFIG_MMR)
-
-#define UVH_RH_GAM_CONFIG_MMR_N_SKT_SHFT 6
-#define UVH_RH_GAM_CONFIG_MMR_N_SKT_MASK 0x00000000000003c0UL
+#define UVH_RH_GAM_ADDR_MAP_CONFIG ( \
+ is_uv(UV4) ? 0x480000UL : \
+ is_uv(UV3) ? 0x1600000UL : \
+ is_uv(UV2) ? 0x1600000UL : \
+ 0)
-#define UVXH_RH_GAM_CONFIG_MMR_N_SKT_SHFT 6
-#define UVXH_RH_GAM_CONFIG_MMR_N_SKT_MASK 0x00000000000003c0UL
-#define UV2H_RH_GAM_CONFIG_MMR_M_SKT_SHFT 0
-#define UV2H_RH_GAM_CONFIG_MMR_N_SKT_SHFT 6
-#define UV2H_RH_GAM_CONFIG_MMR_M_SKT_MASK 0x000000000000003fUL
-#define UV2H_RH_GAM_CONFIG_MMR_N_SKT_MASK 0x00000000000003c0UL
+/* UVXH common defines */
+#define UVXH_RH_GAM_ADDR_MAP_CONFIG_N_SKT_SHFT 6
+#define UVXH_RH_GAM_ADDR_MAP_CONFIG_N_SKT_MASK 0x00000000000003c0UL
-#define UV3H_RH_GAM_CONFIG_MMR_M_SKT_SHFT 0
-#define UV3H_RH_GAM_CONFIG_MMR_N_SKT_SHFT 6
-#define UV3H_RH_GAM_CONFIG_MMR_M_SKT_MASK 0x000000000000003fUL
-#define UV3H_RH_GAM_CONFIG_MMR_N_SKT_MASK 0x00000000000003c0UL
+/* UV3 unique defines */
+#define UV3H_RH_GAM_ADDR_MAP_CONFIG_M_SKT_SHFT 0
+#define UV3H_RH_GAM_ADDR_MAP_CONFIG_M_SKT_MASK 0x000000000000003fUL
-#define UV4H_RH_GAM_CONFIG_MMR_N_SKT_SHFT 6
-#define UV4H_RH_GAM_CONFIG_MMR_N_SKT_MASK 0x00000000000003c0UL
+/* UV2 unique defines */
+#define UV2H_RH_GAM_ADDR_MAP_CONFIG_M_SKT_SHFT 0
+#define UV2H_RH_GAM_ADDR_MAP_CONFIG_M_SKT_MASK 0x000000000000003fUL
-union uvh_rh_gam_config_mmr_u {
+union uvh_rh_gam_addr_map_config_u {
unsigned long v;
- struct uvh_rh_gam_config_mmr_s {
+
+ /* UVH common struct */
+ struct uvh_rh_gam_addr_map_config_s {
unsigned long rsvd_0_5:6;
unsigned long n_skt:4; /* RW */
unsigned long rsvd_10_63:54;
} s;
- struct uvxh_rh_gam_config_mmr_s {
+
+ /* UVXH common struct */
+ struct uvxh_rh_gam_addr_map_config_s {
unsigned long rsvd_0_5:6;
unsigned long n_skt:4; /* RW */
unsigned long rsvd_10_63:54;
} sx;
- struct uv2h_rh_gam_config_mmr_s {
- unsigned long m_skt:6; /* RW */
+
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_addr_map_config_s {
+ unsigned long rsvd_0_5:6;
unsigned long n_skt:4; /* RW */
unsigned long rsvd_10_63:54;
- } s2;
- struct uv3h_rh_gam_config_mmr_s {
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_addr_map_config_s {
unsigned long m_skt:6; /* RW */
unsigned long n_skt:4; /* RW */
unsigned long rsvd_10_63:54;
} s3;
- struct uv4h_rh_gam_config_mmr_s {
- unsigned long rsvd_0_5:6;
+
+ /* UV2 unique struct */
+ struct uv2h_rh_gam_addr_map_config_s {
+ unsigned long m_skt:6; /* RW */
unsigned long n_skt:4; /* RW */
unsigned long rsvd_10_63:54;
- } s4;
+ } s2;
};
/* ========================================================================= */
-/* UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR */
+/* UVH_RH_GAM_ALIAS_0_OVERLAY_CONFIG */
/* ========================================================================= */
-#define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR 0x1600010UL
-#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR 0x1600010UL
-#define UV4H_RH_GAM_GRU_OVERLAY_CONFIG_MMR 0x480010UL
-#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_GRU_OVERLAY_CONFIG_MMR)
-
-#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_SHFT 52
-#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
-#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_MASK 0x00f0000000000000UL
-#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_SHFT 52
-#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
-#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_MASK 0x00f0000000000000UL
-#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT 28
-#define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_SHFT 52
-#define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
-#define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffff0000000UL
-#define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_MASK 0x00f0000000000000UL
-#define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT 28
-#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_SHFT 52
-#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_MODE_SHFT 62
-#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
-#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffff0000000UL
-#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_MASK 0x00f0000000000000UL
-#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_MODE_MASK 0x4000000000000000UL
-#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV4H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT 26
-#define UV4H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_SHFT 52
-#define UV4H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
-#define UV4H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffffc000000UL
-#define UV4H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_N_GRU_MASK 0x00f0000000000000UL
-#define UV4H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK ( \
- is_uv2_hub() ? UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK : \
- is_uv3_hub() ? UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK : \
- /*is_uv4_hub*/ UV4H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK)
-#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT ( \
- is_uv2_hub() ? UV2H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT : \
- is_uv3_hub() ? UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT : \
- /*is_uv4_hub*/ UV4H_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT)
-
-union uvh_rh_gam_gru_overlay_config_mmr_u {
+#define UVH_RH_GAM_ALIAS_0_OVERLAY_CONFIG ( \
+ is_uv(UV4) ? 0x4800c8UL : \
+ is_uv(UV3) ? 0x16000c8UL : \
+ is_uv(UV2) ? 0x16000c8UL : \
+ 0)
+
+
+/* UVXH common defines */
+#define UVXH_RH_GAM_ALIAS_0_OVERLAY_CONFIG_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS_0_OVERLAY_CONFIG_BASE_MASK 0x00000000ff000000UL
+#define UVXH_RH_GAM_ALIAS_0_OVERLAY_CONFIG_M_ALIAS_SHFT 48
+#define UVXH_RH_GAM_ALIAS_0_OVERLAY_CONFIG_M_ALIAS_MASK 0x001f000000000000UL
+#define UVXH_RH_GAM_ALIAS_0_OVERLAY_CONFIG_ENABLE_SHFT 63
+#define UVXH_RH_GAM_ALIAS_0_OVERLAY_CONFIG_ENABLE_MASK 0x8000000000000000UL
+
+
+union uvh_rh_gam_alias_0_overlay_config_u {
unsigned long v;
- struct uvh_rh_gam_gru_overlay_config_mmr_s {
- unsigned long rsvd_0_51:52;
- unsigned long n_gru:4; /* RW */
- unsigned long rsvd_56_62:7;
+
+ /* UVH common struct */
+ struct uvh_rh_gam_alias_0_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
unsigned long enable:1; /* RW */
} s;
- struct uvxh_rh_gam_gru_overlay_config_mmr_s {
- unsigned long rsvd_0_45:46;
- unsigned long rsvd_46_51:6;
- unsigned long n_gru:4; /* RW */
- unsigned long rsvd_56_62:7;
- unsigned long enable:1; /* RW */
- } sx;
- struct uv2h_rh_gam_gru_overlay_config_mmr_s {
- unsigned long rsvd_0_27:28;
- unsigned long base:18; /* RW */
- unsigned long rsvd_46_51:6;
- unsigned long n_gru:4; /* RW */
- unsigned long rsvd_56_62:7;
- unsigned long enable:1; /* RW */
- } s2;
- struct uv3h_rh_gam_gru_overlay_config_mmr_s {
- unsigned long rsvd_0_27:28;
- unsigned long base:18; /* RW */
- unsigned long rsvd_46_51:6;
- unsigned long n_gru:4; /* RW */
- unsigned long rsvd_56_61:6;
- unsigned long mode:1; /* RW */
- unsigned long enable:1; /* RW */
- } s3;
- struct uv4h_rh_gam_gru_overlay_config_mmr_s {
- unsigned long rsvd_0_24:25;
- unsigned long undef_25:1; /* Undefined */
- unsigned long base:20; /* RW */
- unsigned long rsvd_46_51:6;
- unsigned long n_gru:4; /* RW */
- unsigned long rsvd_56_62:7;
- unsigned long enable:1; /* RW */
- } s4;
-};
-/* ========================================================================= */
-/* UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR */
-/* ========================================================================= */
-#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR uv_undefined("UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR")
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR 0x1603000UL
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR 0x483000UL
-#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR)
-
-
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_SHFT 26
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT 46
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_SHFT 63
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK 0x00003ffffc000000UL
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK 0x000fc00000000000UL
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_SHFT 26
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT 46
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_SHFT 63
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK 0x00003ffffc000000UL
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK 0x000fc00000000000UL
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT 52
-#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK 0x000ffffffc000000UL
-#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK 0x03f0000000000000UL
-#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT ( \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT : \
- is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT)
-
-#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK ( \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK : \
- is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK)
-
-#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK ( \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK : \
- is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK)
-
-#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK ( \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK : \
- is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK)
-
-union uvh_rh_gam_mmioh_overlay_config0_mmr_u {
- unsigned long v;
- struct uv3h_rh_gam_mmioh_overlay_config0_mmr_s {
- unsigned long rsvd_0_25:26;
- unsigned long base:20; /* RW */
- unsigned long m_io:6; /* RW */
- unsigned long n_io:4;
- unsigned long rsvd_56_62:7;
- unsigned long enable:1; /* RW */
- } s3;
- struct uv4h_rh_gam_mmioh_overlay_config0_mmr_s {
- unsigned long rsvd_0_25:26;
- unsigned long base:20; /* RW */
- unsigned long m_io:6; /* RW */
- unsigned long n_io:4;
- unsigned long rsvd_56_62:7;
- unsigned long enable:1; /* RW */
- } s4;
- struct uv4ah_rh_gam_mmioh_overlay_config0_mmr_s {
- unsigned long rsvd_0_25:26;
- unsigned long base:26; /* RW */
- unsigned long m_io:6; /* RW */
- unsigned long n_io:4;
- unsigned long undef_62:1; /* Undefined */
+ /* UVXH common struct */
+ struct uvxh_rh_gam_alias_0_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
unsigned long enable:1; /* RW */
- } s4a;
-};
+ } sx;
-/* ========================================================================= */
-/* UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR */
-/* ========================================================================= */
-#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR uv_undefined("UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR")
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR 0x1603000UL
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR 0x484000UL
-#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR)
-
-
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_SHFT 26
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT 46
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_SHFT 63
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK 0x00003ffffc000000UL
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK 0x000fc00000000000UL
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_SHFT 26
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT 46
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_SHFT 63
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK 0x00003ffffc000000UL
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK 0x000fc00000000000UL
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT 52
-#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK 0x000ffffffc000000UL
-#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK 0x03f0000000000000UL
-
-#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT ( \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT : \
- is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT)
-
-#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK ( \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK : \
- is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK)
-
-#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK ( \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK : \
- is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK)
-
-union uvh_rh_gam_mmioh_overlay_config1_mmr_u {
- unsigned long v;
- struct uv3h_rh_gam_mmioh_overlay_config1_mmr_s {
- unsigned long rsvd_0_25:26;
- unsigned long base:20; /* RW */
- unsigned long m_io:6; /* RW */
- unsigned long n_io:4;
- unsigned long rsvd_56_62:7;
- unsigned long enable:1; /* RW */
- } s3;
- struct uv4h_rh_gam_mmioh_overlay_config1_mmr_s {
- unsigned long rsvd_0_25:26;
- unsigned long base:20; /* RW */
- unsigned long m_io:6; /* RW */
- unsigned long n_io:4;
- unsigned long rsvd_56_62:7;
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_alias_0_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
unsigned long enable:1; /* RW */
} s4;
- struct uv4ah_rh_gam_mmioh_overlay_config1_mmr_s {
- unsigned long rsvd_0_25:26;
- unsigned long base:26; /* RW */
- unsigned long m_io:6; /* RW */
- unsigned long n_io:4;
- unsigned long undef_62:1; /* Undefined */
+
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_alias_0_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
unsigned long enable:1; /* RW */
- } s4a;
-};
+ } s3;
-/* ========================================================================= */
-/* UVH_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR */
-/* ========================================================================= */
-#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR 0x1600030UL
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR uv_undefined("UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR")
-#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR uv_undefined("UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR")
-#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR)
-
-
-#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_BASE_SHFT 27
-#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_M_IO_SHFT 46
-#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_N_IO_SHFT 52
-#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
-#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffff8000000UL
-#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_M_IO_MASK 0x000fc00000000000UL
-#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_N_IO_MASK 0x00f0000000000000UL
-#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
-
-
-union uvh_rh_gam_mmioh_overlay_config_mmr_u {
- unsigned long v;
- struct uv2h_rh_gam_mmioh_overlay_config_mmr_s {
- unsigned long rsvd_0_26:27;
- unsigned long base:19; /* RW */
- unsigned long m_io:6; /* RW */
- unsigned long n_io:4; /* RW */
- unsigned long rsvd_56_62:7;
+ /* UV2 unique struct */
+ struct uv2h_rh_gam_alias_0_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
unsigned long enable:1; /* RW */
} s2;
};
/* ========================================================================= */
-/* UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR */
+/* UVH_RH_GAM_ALIAS_0_REDIRECT_CONFIG */
/* ========================================================================= */
-#define UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR uv_undefined("UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR")
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR 0x1603800UL
-#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR 0x483800UL
-#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR)
+#define UVH_RH_GAM_ALIAS_0_REDIRECT_CONFIG ( \
+ is_uv(UV4) ? 0x4800d0UL : \
+ is_uv(UV3) ? 0x16000d0UL : \
+ is_uv(UV2) ? 0x16000d0UL : \
+ 0)
-#define UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH uv_undefined("UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH")
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH 128
-#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH 128
-#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH ( \
- is_uv2_hub() ? UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH : \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH)
+/* UVXH common defines */
+#define UVXH_RH_GAM_ALIAS_0_REDIRECT_CONFIG_DEST_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS_0_REDIRECT_CONFIG_DEST_BASE_MASK 0x00003fffff000000UL
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_SHFT 0
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK 0x0000000000007fffUL
-#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_SHFT 0
-#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK 0x0000000000007fffUL
+union uvh_rh_gam_alias_0_redirect_config_u {
+ unsigned long v;
+
+ /* UVH common struct */
+ struct uvh_rh_gam_alias_0_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } s;
-#define UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK 0x0000000000000fffUL
+ /* UVXH common struct */
+ struct uvxh_rh_gam_alias_0_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } sx;
-#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK ( \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK : \
- is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK)
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_alias_0_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } s4;
-union uvh_rh_gam_mmioh_redirect_config0_mmr_u {
- unsigned long v;
- struct uv3h_rh_gam_mmioh_redirect_config0_mmr_s {
- unsigned long nasid:15; /* RW */
- unsigned long rsvd_15_63:49;
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_alias_0_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
} s3;
- struct uv4h_rh_gam_mmioh_redirect_config0_mmr_s {
- unsigned long nasid:15; /* RW */
- unsigned long rsvd_15_63:49;
- } s4;
- struct uv4ah_rh_gam_mmioh_redirect_config0_mmr_s {
- unsigned long nasid:12; /* RW */
- unsigned long rsvd_12_63:52;
- } s4a;
+
+ /* UV2 unique struct */
+ struct uv2h_rh_gam_alias_0_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } s2;
};
/* ========================================================================= */
-/* UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR */
+/* UVH_RH_GAM_ALIAS_1_OVERLAY_CONFIG */
/* ========================================================================= */
-#define UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR uv_undefined("UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR")
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR 0x1604800UL
-#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR 0x484800UL
-#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR)
-
-#define UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH uv_undefined("UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH")
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH 128
-#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH 128
-#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH ( \
- is_uv2_hub() ? UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH : \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH)
+#define UVH_RH_GAM_ALIAS_1_OVERLAY_CONFIG ( \
+ is_uv(UV4) ? 0x4800d8UL : \
+ is_uv(UV3) ? 0x16000d8UL : \
+ is_uv(UV2) ? 0x16000d8UL : \
+ 0)
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_SHFT 0
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK 0x0000000000007fffUL
+/* UVXH common defines */
+#define UVXH_RH_GAM_ALIAS_1_OVERLAY_CONFIG_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS_1_OVERLAY_CONFIG_BASE_MASK 0x00000000ff000000UL
+#define UVXH_RH_GAM_ALIAS_1_OVERLAY_CONFIG_M_ALIAS_SHFT 48
+#define UVXH_RH_GAM_ALIAS_1_OVERLAY_CONFIG_M_ALIAS_MASK 0x001f000000000000UL
+#define UVXH_RH_GAM_ALIAS_1_OVERLAY_CONFIG_ENABLE_SHFT 63
+#define UVXH_RH_GAM_ALIAS_1_OVERLAY_CONFIG_ENABLE_MASK 0x8000000000000000UL
-#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_SHFT 0
-#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK 0x0000000000007fffUL
-#define UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK 0x0000000000000fffUL
-
-#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK ( \
- is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK : \
- is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK)
-
-union uvh_rh_gam_mmioh_redirect_config1_mmr_u {
+union uvh_rh_gam_alias_1_overlay_config_u {
unsigned long v;
- struct uv3h_rh_gam_mmioh_redirect_config1_mmr_s {
- unsigned long nasid:15; /* RW */
- unsigned long rsvd_15_63:49;
- } s3;
- struct uv4h_rh_gam_mmioh_redirect_config1_mmr_s {
- unsigned long nasid:15; /* RW */
- unsigned long rsvd_15_63:49;
- } s4;
- struct uv4ah_rh_gam_mmioh_redirect_config1_mmr_s {
- unsigned long nasid:12; /* RW */
- unsigned long rsvd_12_63:52;
- } s4a;
-};
-/* ========================================================================= */
-/* UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR */
-/* ========================================================================= */
-#define UV2H_RH_GAM_MMR_OVERLAY_CONFIG_MMR 0x1600028UL
-#define UV3H_RH_GAM_MMR_OVERLAY_CONFIG_MMR 0x1600028UL
-#define UV4H_RH_GAM_MMR_OVERLAY_CONFIG_MMR 0x480028UL
-#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR ( \
- is_uv2_hub() ? UV2H_RH_GAM_MMR_OVERLAY_CONFIG_MMR : \
- is_uv3_hub() ? UV3H_RH_GAM_MMR_OVERLAY_CONFIG_MMR : \
- /*is_uv4_hub*/ UV4H_RH_GAM_MMR_OVERLAY_CONFIG_MMR)
-
-#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT 26
-#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
-#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffffc000000UL
-#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT 26
-#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
-#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffffc000000UL
-#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV2H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT 26
-#define UV2H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
-#define UV2H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffffc000000UL
-#define UV2H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV3H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT 26
-#define UV3H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
-#define UV3H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffffc000000UL
-#define UV3H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
-
-#define UV4H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT 26
-#define UV4H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
-#define UV4H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffffc000000UL
-#define UV4H_RH_GAM_MMR_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
-
-
-union uvh_rh_gam_mmr_overlay_config_mmr_u {
- unsigned long v;
- struct uvh_rh_gam_mmr_overlay_config_mmr_s {
- unsigned long rsvd_0_25:26;
- unsigned long base:20; /* RW */
- unsigned long rsvd_46_62:17;
+ /* UVH common struct */
+ struct uvh_rh_gam_alias_1_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
unsigned long enable:1; /* RW */
} s;
- struct uvxh_rh_gam_mmr_overlay_config_mmr_s {
- unsigned long rsvd_0_25:26;
- unsigned long base:20; /* RW */
- unsigned long rsvd_46_62:17;
+
+ /* UVXH common struct */
+ struct uvxh_rh_gam_alias_1_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
unsigned long enable:1; /* RW */
} sx;
- struct uv2h_rh_gam_mmr_overlay_config_mmr_s {
- unsigned long rsvd_0_25:26;
- unsigned long base:20; /* RW */
- unsigned long rsvd_46_62:17;
+
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_alias_1_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
unsigned long enable:1; /* RW */
- } s2;
- struct uv3h_rh_gam_mmr_overlay_config_mmr_s {
- unsigned long rsvd_0_25:26;
- unsigned long base:20; /* RW */
- unsigned long rsvd_46_62:17;
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_alias_1_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
unsigned long enable:1; /* RW */
} s3;
- struct uv4h_rh_gam_mmr_overlay_config_mmr_s {
- unsigned long rsvd_0_25:26;
- unsigned long base:20; /* RW */
- unsigned long rsvd_46_62:17;
+
+ /* UV2 unique struct */
+ struct uv2h_rh_gam_alias_1_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
unsigned long enable:1; /* RW */
- } s4;
+ } s2;
};
/* ========================================================================= */
-/* UVH_RTC */
+/* UVH_RH_GAM_ALIAS_1_REDIRECT_CONFIG */
/* ========================================================================= */
-#define UV2H_RTC 0x340000UL
-#define UV3H_RTC 0x340000UL
-#define UV4H_RTC 0xe0000UL
-#define UVH_RTC ( \
- is_uv2_hub() ? UV2H_RTC : \
- is_uv3_hub() ? UV3H_RTC : \
- /*is_uv4_hub*/ UV4H_RTC)
+#define UVH_RH_GAM_ALIAS_1_REDIRECT_CONFIG ( \
+ is_uv(UV4) ? 0x4800e0UL : \
+ is_uv(UV3) ? 0x16000e0UL : \
+ is_uv(UV2) ? 0x16000e0UL : \
+ 0)
-#define UVH_RTC_REAL_TIME_CLOCK_SHFT 0
-#define UVH_RTC_REAL_TIME_CLOCK_MASK 0x00ffffffffffffffUL
+/* UVXH common defines */
+#define UVXH_RH_GAM_ALIAS_1_REDIRECT_CONFIG_DEST_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS_1_REDIRECT_CONFIG_DEST_BASE_MASK 0x00003fffff000000UL
-union uvh_rtc_u {
+
+union uvh_rh_gam_alias_1_redirect_config_u {
unsigned long v;
- struct uvh_rtc_s {
- unsigned long real_time_clock:56; /* RW */
- unsigned long rsvd_56_63:8;
+
+ /* UVH common struct */
+ struct uvh_rh_gam_alias_1_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
} s;
+
+ /* UVXH common struct */
+ struct uvxh_rh_gam_alias_1_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } sx;
+
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_alias_1_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_alias_1_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_rh_gam_alias_1_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } s2;
};
/* ========================================================================= */
-/* UVH_RTC1_INT_CONFIG */
+/* UVH_RH_GAM_ALIAS_2_OVERLAY_CONFIG */
/* ========================================================================= */
-#define UVH_RTC1_INT_CONFIG 0x615c0UL
+#define UVH_RH_GAM_ALIAS_2_OVERLAY_CONFIG ( \
+ is_uv(UV4) ? 0x4800e8UL : \
+ is_uv(UV3) ? 0x16000e8UL : \
+ is_uv(UV2) ? 0x16000e8UL : \
+ 0)
-#define UVH_RTC1_INT_CONFIG_VECTOR_SHFT 0
-#define UVH_RTC1_INT_CONFIG_DM_SHFT 8
-#define UVH_RTC1_INT_CONFIG_DESTMODE_SHFT 11
-#define UVH_RTC1_INT_CONFIG_STATUS_SHFT 12
-#define UVH_RTC1_INT_CONFIG_P_SHFT 13
-#define UVH_RTC1_INT_CONFIG_T_SHFT 15
-#define UVH_RTC1_INT_CONFIG_M_SHFT 16
-#define UVH_RTC1_INT_CONFIG_APIC_ID_SHFT 32
-#define UVH_RTC1_INT_CONFIG_VECTOR_MASK 0x00000000000000ffUL
-#define UVH_RTC1_INT_CONFIG_DM_MASK 0x0000000000000700UL
-#define UVH_RTC1_INT_CONFIG_DESTMODE_MASK 0x0000000000000800UL
-#define UVH_RTC1_INT_CONFIG_STATUS_MASK 0x0000000000001000UL
-#define UVH_RTC1_INT_CONFIG_P_MASK 0x0000000000002000UL
-#define UVH_RTC1_INT_CONFIG_T_MASK 0x0000000000008000UL
-#define UVH_RTC1_INT_CONFIG_M_MASK 0x0000000000010000UL
-#define UVH_RTC1_INT_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
+/* UVXH common defines */
+#define UVXH_RH_GAM_ALIAS_2_OVERLAY_CONFIG_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS_2_OVERLAY_CONFIG_BASE_MASK 0x00000000ff000000UL
+#define UVXH_RH_GAM_ALIAS_2_OVERLAY_CONFIG_M_ALIAS_SHFT 48
+#define UVXH_RH_GAM_ALIAS_2_OVERLAY_CONFIG_M_ALIAS_MASK 0x001f000000000000UL
+#define UVXH_RH_GAM_ALIAS_2_OVERLAY_CONFIG_ENABLE_SHFT 63
+#define UVXH_RH_GAM_ALIAS_2_OVERLAY_CONFIG_ENABLE_MASK 0x8000000000000000UL
-union uvh_rtc1_int_config_u {
+
+union uvh_rh_gam_alias_2_overlay_config_u {
unsigned long v;
- struct uvh_rtc1_int_config_s {
- unsigned long vector_:8; /* RW */
- unsigned long dm:3; /* RW */
- unsigned long destmode:1; /* RW */
- unsigned long status:1; /* RO */
- unsigned long p:1; /* RO */
- unsigned long rsvd_14:1;
- unsigned long t:1; /* RO */
- unsigned long m:1; /* RW */
- unsigned long rsvd_17_31:15;
- unsigned long apic_id:32; /* RW */
+
+ /* UVH common struct */
+ struct uvh_rh_gam_alias_2_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
+ unsigned long enable:1; /* RW */
} s;
+
+ /* UVXH common struct */
+ struct uvxh_rh_gam_alias_2_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
+ unsigned long enable:1; /* RW */
+ } sx;
+
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_alias_2_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
+ unsigned long enable:1; /* RW */
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_alias_2_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
+ unsigned long enable:1; /* RW */
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_rh_gam_alias_2_overlay_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long base:8; /* RW */
+ unsigned long rsvd_32_47:16;
+ unsigned long m_alias:5; /* RW */
+ unsigned long rsvd_53_62:10;
+ unsigned long enable:1; /* RW */
+ } s2;
};
/* ========================================================================= */
-/* UVH_SCRATCH5 */
+/* UVH_RH_GAM_ALIAS_2_REDIRECT_CONFIG */
/* ========================================================================= */
-#define UV2H_SCRATCH5 0x2d0200UL
-#define UV3H_SCRATCH5 0x2d0200UL
-#define UV4H_SCRATCH5 0xb0200UL
-#define UVH_SCRATCH5 ( \
- is_uv2_hub() ? UV2H_SCRATCH5 : \
- is_uv3_hub() ? UV3H_SCRATCH5 : \
- /*is_uv4_hub*/ UV4H_SCRATCH5)
-
-#define UV2H_SCRATCH5_32 0x778
-#define UV3H_SCRATCH5_32 0x778
-#define UV4H_SCRATCH5_32 0x798
-#define UVH_SCRATCH5_32 ( \
- is_uv2_hub() ? UV2H_SCRATCH5_32 : \
- is_uv3_hub() ? UV3H_SCRATCH5_32 : \
- /*is_uv4_hub*/ UV4H_SCRATCH5_32)
+#define UVH_RH_GAM_ALIAS_2_REDIRECT_CONFIG ( \
+ is_uv(UV4) ? 0x4800f0UL : \
+ is_uv(UV3) ? 0x16000f0UL : \
+ is_uv(UV2) ? 0x16000f0UL : \
+ 0)
-#define UVH_SCRATCH5_SCRATCH5_SHFT 0
-#define UVH_SCRATCH5_SCRATCH5_MASK 0xffffffffffffffffUL
+/* UVXH common defines */
+#define UVXH_RH_GAM_ALIAS_2_REDIRECT_CONFIG_DEST_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS_2_REDIRECT_CONFIG_DEST_BASE_MASK 0x00003fffff000000UL
-union uvh_scratch5_u {
+
+union uvh_rh_gam_alias_2_redirect_config_u {
unsigned long v;
- struct uvh_scratch5_s {
- unsigned long scratch5:64; /* RW, W1CS */
- } s;
-};
-/* ========================================================================= */
-/* UVH_SCRATCH5_ALIAS */
-/* ========================================================================= */
-#define UV2H_SCRATCH5_ALIAS 0x2d0208UL
-#define UV3H_SCRATCH5_ALIAS 0x2d0208UL
-#define UV4H_SCRATCH5_ALIAS 0xb0208UL
-#define UVH_SCRATCH5_ALIAS ( \
- is_uv2_hub() ? UV2H_SCRATCH5_ALIAS : \
- is_uv3_hub() ? UV3H_SCRATCH5_ALIAS : \
- /*is_uv4_hub*/ UV4H_SCRATCH5_ALIAS)
+ /* UVH common struct */
+ struct uvh_rh_gam_alias_2_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } s;
-#define UV2H_SCRATCH5_ALIAS_32 0x780
-#define UV3H_SCRATCH5_ALIAS_32 0x780
-#define UV4H_SCRATCH5_ALIAS_32 0x7a0
-#define UVH_SCRATCH5_ALIAS_32 ( \
- is_uv2_hub() ? UV2H_SCRATCH5_ALIAS_32 : \
- is_uv3_hub() ? UV3H_SCRATCH5_ALIAS_32 : \
- /*is_uv4_hub*/ UV4H_SCRATCH5_ALIAS_32)
+ /* UVXH common struct */
+ struct uvxh_rh_gam_alias_2_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } sx;
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_alias_2_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } s4;
-/* ========================================================================= */
-/* UVH_SCRATCH5_ALIAS_2 */
-/* ========================================================================= */
-#define UV2H_SCRATCH5_ALIAS_2 0x2d0210UL
-#define UV3H_SCRATCH5_ALIAS_2 0x2d0210UL
-#define UV4H_SCRATCH5_ALIAS_2 0xb0210UL
-#define UVH_SCRATCH5_ALIAS_2 ( \
- is_uv2_hub() ? UV2H_SCRATCH5_ALIAS_2 : \
- is_uv3_hub() ? UV3H_SCRATCH5_ALIAS_2 : \
- /*is_uv4_hub*/ UV4H_SCRATCH5_ALIAS_2)
-#define UVH_SCRATCH5_ALIAS_2_32 0x788
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_alias_2_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } s3;
+ /* UV2 unique struct */
+ struct uv2h_rh_gam_alias_2_redirect_config_s {
+ unsigned long rsvd_0_23:24;
+ unsigned long dest_base:22; /* RW */
+ unsigned long rsvd_46_63:18;
+ } s2;
+};
/* ========================================================================= */
-/* UVXH_EVENT_OCCURRED2 */
+/* UVH_RH_GAM_GRU_OVERLAY_CONFIG */
/* ========================================================================= */
-#define UVXH_EVENT_OCCURRED2 0x70100UL
-
-#define UV2H_EVENT_OCCURRED2_32 0xb68
-#define UV3H_EVENT_OCCURRED2_32 0xb68
-#define UV4H_EVENT_OCCURRED2_32 0x608
-#define UVH_EVENT_OCCURRED2_32 ( \
- is_uv2_hub() ? UV2H_EVENT_OCCURRED2_32 : \
- is_uv3_hub() ? UV3H_EVENT_OCCURRED2_32 : \
- /*is_uv4_hub*/ UV4H_EVENT_OCCURRED2_32)
-
-
-#define UV2H_EVENT_OCCURRED2_RTC_0_SHFT 0
-#define UV2H_EVENT_OCCURRED2_RTC_1_SHFT 1
-#define UV2H_EVENT_OCCURRED2_RTC_2_SHFT 2
-#define UV2H_EVENT_OCCURRED2_RTC_3_SHFT 3
-#define UV2H_EVENT_OCCURRED2_RTC_4_SHFT 4
-#define UV2H_EVENT_OCCURRED2_RTC_5_SHFT 5
-#define UV2H_EVENT_OCCURRED2_RTC_6_SHFT 6
-#define UV2H_EVENT_OCCURRED2_RTC_7_SHFT 7
-#define UV2H_EVENT_OCCURRED2_RTC_8_SHFT 8
-#define UV2H_EVENT_OCCURRED2_RTC_9_SHFT 9
-#define UV2H_EVENT_OCCURRED2_RTC_10_SHFT 10
-#define UV2H_EVENT_OCCURRED2_RTC_11_SHFT 11
-#define UV2H_EVENT_OCCURRED2_RTC_12_SHFT 12
-#define UV2H_EVENT_OCCURRED2_RTC_13_SHFT 13
-#define UV2H_EVENT_OCCURRED2_RTC_14_SHFT 14
-#define UV2H_EVENT_OCCURRED2_RTC_15_SHFT 15
-#define UV2H_EVENT_OCCURRED2_RTC_16_SHFT 16
-#define UV2H_EVENT_OCCURRED2_RTC_17_SHFT 17
-#define UV2H_EVENT_OCCURRED2_RTC_18_SHFT 18
-#define UV2H_EVENT_OCCURRED2_RTC_19_SHFT 19
-#define UV2H_EVENT_OCCURRED2_RTC_20_SHFT 20
-#define UV2H_EVENT_OCCURRED2_RTC_21_SHFT 21
-#define UV2H_EVENT_OCCURRED2_RTC_22_SHFT 22
-#define UV2H_EVENT_OCCURRED2_RTC_23_SHFT 23
-#define UV2H_EVENT_OCCURRED2_RTC_24_SHFT 24
-#define UV2H_EVENT_OCCURRED2_RTC_25_SHFT 25
-#define UV2H_EVENT_OCCURRED2_RTC_26_SHFT 26
-#define UV2H_EVENT_OCCURRED2_RTC_27_SHFT 27
-#define UV2H_EVENT_OCCURRED2_RTC_28_SHFT 28
-#define UV2H_EVENT_OCCURRED2_RTC_29_SHFT 29
-#define UV2H_EVENT_OCCURRED2_RTC_30_SHFT 30
-#define UV2H_EVENT_OCCURRED2_RTC_31_SHFT 31
-#define UV2H_EVENT_OCCURRED2_RTC_0_MASK 0x0000000000000001UL
-#define UV2H_EVENT_OCCURRED2_RTC_1_MASK 0x0000000000000002UL
-#define UV2H_EVENT_OCCURRED2_RTC_2_MASK 0x0000000000000004UL
-#define UV2H_EVENT_OCCURRED2_RTC_3_MASK 0x0000000000000008UL
-#define UV2H_EVENT_OCCURRED2_RTC_4_MASK 0x0000000000000010UL
-#define UV2H_EVENT_OCCURRED2_RTC_5_MASK 0x0000000000000020UL
-#define UV2H_EVENT_OCCURRED2_RTC_6_MASK 0x0000000000000040UL
-#define UV2H_EVENT_OCCURRED2_RTC_7_MASK 0x0000000000000080UL
-#define UV2H_EVENT_OCCURRED2_RTC_8_MASK 0x0000000000000100UL
-#define UV2H_EVENT_OCCURRED2_RTC_9_MASK 0x0000000000000200UL
-#define UV2H_EVENT_OCCURRED2_RTC_10_MASK 0x0000000000000400UL
-#define UV2H_EVENT_OCCURRED2_RTC_11_MASK 0x0000000000000800UL
-#define UV2H_EVENT_OCCURRED2_RTC_12_MASK 0x0000000000001000UL
-#define UV2H_EVENT_OCCURRED2_RTC_13_MASK 0x0000000000002000UL
-#define UV2H_EVENT_OCCURRED2_RTC_14_MASK 0x0000000000004000UL
-#define UV2H_EVENT_OCCURRED2_RTC_15_MASK 0x0000000000008000UL
-#define UV2H_EVENT_OCCURRED2_RTC_16_MASK 0x0000000000010000UL
-#define UV2H_EVENT_OCCURRED2_RTC_17_MASK 0x0000000000020000UL
-#define UV2H_EVENT_OCCURRED2_RTC_18_MASK 0x0000000000040000UL
-#define UV2H_EVENT_OCCURRED2_RTC_19_MASK 0x0000000000080000UL
-#define UV2H_EVENT_OCCURRED2_RTC_20_MASK 0x0000000000100000UL
-#define UV2H_EVENT_OCCURRED2_RTC_21_MASK 0x0000000000200000UL
-#define UV2H_EVENT_OCCURRED2_RTC_22_MASK 0x0000000000400000UL
-#define UV2H_EVENT_OCCURRED2_RTC_23_MASK 0x0000000000800000UL
-#define UV2H_EVENT_OCCURRED2_RTC_24_MASK 0x0000000001000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_25_MASK 0x0000000002000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_26_MASK 0x0000000004000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_27_MASK 0x0000000008000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_28_MASK 0x0000000010000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_29_MASK 0x0000000020000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_30_MASK 0x0000000040000000UL
-#define UV2H_EVENT_OCCURRED2_RTC_31_MASK 0x0000000080000000UL
+#define UVH_RH_GAM_GRU_OVERLAY_CONFIG ( \
+ is_uv(UV4) ? 0x480010UL : \
+ is_uv(UV3) ? 0x1600010UL : \
+ is_uv(UV2) ? 0x1600010UL : \
+ 0)
+
+
+/* UVXH common defines */
+#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_N_GRU_SHFT 52
+#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_N_GRU_MASK 0x00f0000000000000UL
+#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_ENABLE_SHFT 63
+#define UVXH_RH_GAM_GRU_OVERLAY_CONFIG_ENABLE_MASK 0x8000000000000000UL
+
+/* UV4A unique defines */
+#define UV4AH_RH_GAM_GRU_OVERLAY_CONFIG_BASE_SHFT 26
+#define UV4AH_RH_GAM_GRU_OVERLAY_CONFIG_BASE_MASK 0x000ffffffc000000UL
+
+/* UV4 unique defines */
+#define UV4H_RH_GAM_GRU_OVERLAY_CONFIG_BASE_SHFT 26
+#define UV4H_RH_GAM_GRU_OVERLAY_CONFIG_BASE_MASK 0x00003ffffc000000UL
+
+/* UV3 unique defines */
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_BASE_SHFT 28
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_BASE_MASK 0x00003ffff0000000UL
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MODE_SHFT 62
+#define UV3H_RH_GAM_GRU_OVERLAY_CONFIG_MODE_MASK 0x4000000000000000UL
+
+/* UV2 unique defines */
+#define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_BASE_SHFT 28
+#define UV2H_RH_GAM_GRU_OVERLAY_CONFIG_BASE_MASK 0x00003ffff0000000UL
+
+#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_BASE_MASK ( \
+ is_uv(UV4A) ? 0x000ffffffc000000UL : \
+ is_uv(UV4) ? 0x00003ffffc000000UL : \
+ is_uv(UV3) ? 0x00003ffff0000000UL : \
+ is_uv(UV2) ? 0x00003ffff0000000UL : \
+ 0)
+#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_BASE_SHFT ( \
+ is_uv(UV4) ? 26 : \
+ is_uv(UV3) ? 28 : \
+ is_uv(UV2) ? 28 : \
+ -1)
+
+union uvh_rh_gam_gru_overlay_config_u {
+ unsigned long v;
-#define UV3H_EVENT_OCCURRED2_RTC_0_SHFT 0
-#define UV3H_EVENT_OCCURRED2_RTC_1_SHFT 1
-#define UV3H_EVENT_OCCURRED2_RTC_2_SHFT 2
-#define UV3H_EVENT_OCCURRED2_RTC_3_SHFT 3
-#define UV3H_EVENT_OCCURRED2_RTC_4_SHFT 4
-#define UV3H_EVENT_OCCURRED2_RTC_5_SHFT 5
-#define UV3H_EVENT_OCCURRED2_RTC_6_SHFT 6
-#define UV3H_EVENT_OCCURRED2_RTC_7_SHFT 7
-#define UV3H_EVENT_OCCURRED2_RTC_8_SHFT 8
-#define UV3H_EVENT_OCCURRED2_RTC_9_SHFT 9
-#define UV3H_EVENT_OCCURRED2_RTC_10_SHFT 10
-#define UV3H_EVENT_OCCURRED2_RTC_11_SHFT 11
-#define UV3H_EVENT_OCCURRED2_RTC_12_SHFT 12
-#define UV3H_EVENT_OCCURRED2_RTC_13_SHFT 13
-#define UV3H_EVENT_OCCURRED2_RTC_14_SHFT 14
-#define UV3H_EVENT_OCCURRED2_RTC_15_SHFT 15
-#define UV3H_EVENT_OCCURRED2_RTC_16_SHFT 16
-#define UV3H_EVENT_OCCURRED2_RTC_17_SHFT 17
-#define UV3H_EVENT_OCCURRED2_RTC_18_SHFT 18
-#define UV3H_EVENT_OCCURRED2_RTC_19_SHFT 19
-#define UV3H_EVENT_OCCURRED2_RTC_20_SHFT 20
-#define UV3H_EVENT_OCCURRED2_RTC_21_SHFT 21
-#define UV3H_EVENT_OCCURRED2_RTC_22_SHFT 22
-#define UV3H_EVENT_OCCURRED2_RTC_23_SHFT 23
-#define UV3H_EVENT_OCCURRED2_RTC_24_SHFT 24
-#define UV3H_EVENT_OCCURRED2_RTC_25_SHFT 25
-#define UV3H_EVENT_OCCURRED2_RTC_26_SHFT 26
-#define UV3H_EVENT_OCCURRED2_RTC_27_SHFT 27
-#define UV3H_EVENT_OCCURRED2_RTC_28_SHFT 28
-#define UV3H_EVENT_OCCURRED2_RTC_29_SHFT 29
-#define UV3H_EVENT_OCCURRED2_RTC_30_SHFT 30
-#define UV3H_EVENT_OCCURRED2_RTC_31_SHFT 31
-#define UV3H_EVENT_OCCURRED2_RTC_0_MASK 0x0000000000000001UL
-#define UV3H_EVENT_OCCURRED2_RTC_1_MASK 0x0000000000000002UL
-#define UV3H_EVENT_OCCURRED2_RTC_2_MASK 0x0000000000000004UL
-#define UV3H_EVENT_OCCURRED2_RTC_3_MASK 0x0000000000000008UL
-#define UV3H_EVENT_OCCURRED2_RTC_4_MASK 0x0000000000000010UL
-#define UV3H_EVENT_OCCURRED2_RTC_5_MASK 0x0000000000000020UL
-#define UV3H_EVENT_OCCURRED2_RTC_6_MASK 0x0000000000000040UL
-#define UV3H_EVENT_OCCURRED2_RTC_7_MASK 0x0000000000000080UL
-#define UV3H_EVENT_OCCURRED2_RTC_8_MASK 0x0000000000000100UL
-#define UV3H_EVENT_OCCURRED2_RTC_9_MASK 0x0000000000000200UL
-#define UV3H_EVENT_OCCURRED2_RTC_10_MASK 0x0000000000000400UL
-#define UV3H_EVENT_OCCURRED2_RTC_11_MASK 0x0000000000000800UL
-#define UV3H_EVENT_OCCURRED2_RTC_12_MASK 0x0000000000001000UL
-#define UV3H_EVENT_OCCURRED2_RTC_13_MASK 0x0000000000002000UL
-#define UV3H_EVENT_OCCURRED2_RTC_14_MASK 0x0000000000004000UL
-#define UV3H_EVENT_OCCURRED2_RTC_15_MASK 0x0000000000008000UL
-#define UV3H_EVENT_OCCURRED2_RTC_16_MASK 0x0000000000010000UL
-#define UV3H_EVENT_OCCURRED2_RTC_17_MASK 0x0000000000020000UL
-#define UV3H_EVENT_OCCURRED2_RTC_18_MASK 0x0000000000040000UL
-#define UV3H_EVENT_OCCURRED2_RTC_19_MASK 0x0000000000080000UL
-#define UV3H_EVENT_OCCURRED2_RTC_20_MASK 0x0000000000100000UL
-#define UV3H_EVENT_OCCURRED2_RTC_21_MASK 0x0000000000200000UL
-#define UV3H_EVENT_OCCURRED2_RTC_22_MASK 0x0000000000400000UL
-#define UV3H_EVENT_OCCURRED2_RTC_23_MASK 0x0000000000800000UL
-#define UV3H_EVENT_OCCURRED2_RTC_24_MASK 0x0000000001000000UL
-#define UV3H_EVENT_OCCURRED2_RTC_25_MASK 0x0000000002000000UL
-#define UV3H_EVENT_OCCURRED2_RTC_26_MASK 0x0000000004000000UL
-#define UV3H_EVENT_OCCURRED2_RTC_27_MASK 0x0000000008000000UL
-#define UV3H_EVENT_OCCURRED2_RTC_28_MASK 0x0000000010000000UL
-#define UV3H_EVENT_OCCURRED2_RTC_29_MASK 0x0000000020000000UL
-#define UV3H_EVENT_OCCURRED2_RTC_30_MASK 0x0000000040000000UL
-#define UV3H_EVENT_OCCURRED2_RTC_31_MASK 0x0000000080000000UL
+ /* UVH common struct */
+ struct uvh_rh_gam_gru_overlay_config_s {
+ unsigned long rsvd_0_45:46;
+ unsigned long rsvd_46_51:6;
+ unsigned long n_gru:4; /* RW */
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } s;
+
+ /* UVXH common struct */
+ struct uvxh_rh_gam_gru_overlay_config_s {
+ unsigned long rsvd_0_45:46;
+ unsigned long rsvd_46_51:6;
+ unsigned long n_gru:4; /* RW */
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } sx;
+
+ /* UV4A unique struct */
+ struct uv4ah_rh_gam_gru_overlay_config_s {
+ unsigned long rsvd_0_24:25;
+ unsigned long undef_25:1; /* Undefined */
+ unsigned long base:26; /* RW */
+ unsigned long n_gru:4; /* RW */
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } s4a;
+
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_gru_overlay_config_s {
+ unsigned long rsvd_0_24:25;
+ unsigned long undef_25:1; /* Undefined */
+ unsigned long base:20; /* RW */
+ unsigned long rsvd_46_51:6;
+ unsigned long n_gru:4; /* RW */
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_gru_overlay_config_s {
+ unsigned long rsvd_0_27:28;
+ unsigned long base:18; /* RW */
+ unsigned long rsvd_46_51:6;
+ unsigned long n_gru:4; /* RW */
+ unsigned long rsvd_56_61:6;
+ unsigned long mode:1; /* RW */
+ unsigned long enable:1; /* RW */
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_rh_gam_gru_overlay_config_s {
+ unsigned long rsvd_0_27:28;
+ unsigned long base:18; /* RW */
+ unsigned long rsvd_46_51:6;
+ unsigned long n_gru:4; /* RW */
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } s2;
+};
+
+/* ========================================================================= */
+/* UVH_RH_GAM_MMIOH_OVERLAY_CONFIG */
+/* ========================================================================= */
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG ( \
+ is_uv(UV2) ? 0x1600030UL : \
+ 0)
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT0_SHFT 0
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT1_SHFT 1
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT2_SHFT 2
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT3_SHFT 3
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT4_SHFT 4
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT5_SHFT 5
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT6_SHFT 6
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT7_SHFT 7
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT8_SHFT 8
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT9_SHFT 9
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT10_SHFT 10
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT11_SHFT 11
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT12_SHFT 12
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT13_SHFT 13
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT14_SHFT 14
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT15_SHFT 15
-#define UV4H_EVENT_OCCURRED2_RTC_INTERVAL_INT_SHFT 16
-#define UV4H_EVENT_OCCURRED2_BAU_DASHBOARD_INT_SHFT 17
-#define UV4H_EVENT_OCCURRED2_RTC_0_SHFT 18
-#define UV4H_EVENT_OCCURRED2_RTC_1_SHFT 19
-#define UV4H_EVENT_OCCURRED2_RTC_2_SHFT 20
-#define UV4H_EVENT_OCCURRED2_RTC_3_SHFT 21
-#define UV4H_EVENT_OCCURRED2_RTC_4_SHFT 22
-#define UV4H_EVENT_OCCURRED2_RTC_5_SHFT 23
-#define UV4H_EVENT_OCCURRED2_RTC_6_SHFT 24
-#define UV4H_EVENT_OCCURRED2_RTC_7_SHFT 25
-#define UV4H_EVENT_OCCURRED2_RTC_8_SHFT 26
-#define UV4H_EVENT_OCCURRED2_RTC_9_SHFT 27
-#define UV4H_EVENT_OCCURRED2_RTC_10_SHFT 28
-#define UV4H_EVENT_OCCURRED2_RTC_11_SHFT 29
-#define UV4H_EVENT_OCCURRED2_RTC_12_SHFT 30
-#define UV4H_EVENT_OCCURRED2_RTC_13_SHFT 31
-#define UV4H_EVENT_OCCURRED2_RTC_14_SHFT 32
-#define UV4H_EVENT_OCCURRED2_RTC_15_SHFT 33
-#define UV4H_EVENT_OCCURRED2_RTC_16_SHFT 34
-#define UV4H_EVENT_OCCURRED2_RTC_17_SHFT 35
-#define UV4H_EVENT_OCCURRED2_RTC_18_SHFT 36
-#define UV4H_EVENT_OCCURRED2_RTC_19_SHFT 37
-#define UV4H_EVENT_OCCURRED2_RTC_20_SHFT 38
-#define UV4H_EVENT_OCCURRED2_RTC_21_SHFT 39
-#define UV4H_EVENT_OCCURRED2_RTC_22_SHFT 40
-#define UV4H_EVENT_OCCURRED2_RTC_23_SHFT 41
-#define UV4H_EVENT_OCCURRED2_RTC_24_SHFT 42
-#define UV4H_EVENT_OCCURRED2_RTC_25_SHFT 43
-#define UV4H_EVENT_OCCURRED2_RTC_26_SHFT 44
-#define UV4H_EVENT_OCCURRED2_RTC_27_SHFT 45
-#define UV4H_EVENT_OCCURRED2_RTC_28_SHFT 46
-#define UV4H_EVENT_OCCURRED2_RTC_29_SHFT 47
-#define UV4H_EVENT_OCCURRED2_RTC_30_SHFT 48
-#define UV4H_EVENT_OCCURRED2_RTC_31_SHFT 49
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT0_MASK 0x0000000000000001UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT1_MASK 0x0000000000000002UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT2_MASK 0x0000000000000004UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT3_MASK 0x0000000000000008UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT4_MASK 0x0000000000000010UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT5_MASK 0x0000000000000020UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT6_MASK 0x0000000000000040UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT7_MASK 0x0000000000000080UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT8_MASK 0x0000000000000100UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT9_MASK 0x0000000000000200UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT10_MASK 0x0000000000000400UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT11_MASK 0x0000000000000800UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT12_MASK 0x0000000000001000UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT13_MASK 0x0000000000002000UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT14_MASK 0x0000000000004000UL
-#define UV4H_EVENT_OCCURRED2_MESSAGE_ACCELERATOR_INT15_MASK 0x0000000000008000UL
-#define UV4H_EVENT_OCCURRED2_RTC_INTERVAL_INT_MASK 0x0000000000010000UL
-#define UV4H_EVENT_OCCURRED2_BAU_DASHBOARD_INT_MASK 0x0000000000020000UL
-#define UV4H_EVENT_OCCURRED2_RTC_0_MASK 0x0000000000040000UL
-#define UV4H_EVENT_OCCURRED2_RTC_1_MASK 0x0000000000080000UL
-#define UV4H_EVENT_OCCURRED2_RTC_2_MASK 0x0000000000100000UL
-#define UV4H_EVENT_OCCURRED2_RTC_3_MASK 0x0000000000200000UL
-#define UV4H_EVENT_OCCURRED2_RTC_4_MASK 0x0000000000400000UL
-#define UV4H_EVENT_OCCURRED2_RTC_5_MASK 0x0000000000800000UL
-#define UV4H_EVENT_OCCURRED2_RTC_6_MASK 0x0000000001000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_7_MASK 0x0000000002000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_8_MASK 0x0000000004000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_9_MASK 0x0000000008000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_10_MASK 0x0000000010000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_11_MASK 0x0000000020000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_12_MASK 0x0000000040000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_13_MASK 0x0000000080000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_14_MASK 0x0000000100000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_15_MASK 0x0000000200000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_16_MASK 0x0000000400000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_17_MASK 0x0000000800000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_18_MASK 0x0000001000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_19_MASK 0x0000002000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_20_MASK 0x0000004000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_21_MASK 0x0000008000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_22_MASK 0x0000010000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_23_MASK 0x0000020000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_24_MASK 0x0000040000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_25_MASK 0x0000080000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_26_MASK 0x0000100000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_27_MASK 0x0000200000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_28_MASK 0x0000400000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_29_MASK 0x0000800000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_30_MASK 0x0001000000000000UL
-#define UV4H_EVENT_OCCURRED2_RTC_31_MASK 0x0002000000000000UL
-#define UVXH_EVENT_OCCURRED2_RTC_1_MASK ( \
- is_uv2_hub() ? UV2H_EVENT_OCCURRED2_RTC_1_MASK : \
- is_uv3_hub() ? UV3H_EVENT_OCCURRED2_RTC_1_MASK : \
- /*is_uv4_hub*/ UV4H_EVENT_OCCURRED2_RTC_1_MASK)
-union uvh_event_occurred2_u {
+/* UV2 unique defines */
+#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_BASE_SHFT 27
+#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_BASE_MASK 0x00003ffff8000000UL
+#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_M_IO_SHFT 46
+#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_M_IO_MASK 0x000fc00000000000UL
+#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_N_IO_SHFT 52
+#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_N_IO_MASK 0x00f0000000000000UL
+#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_ENABLE_SHFT 63
+#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_ENABLE_MASK 0x8000000000000000UL
+
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG_BASE_SHFT ( \
+ is_uv(UV2) ? 27 : \
+ uv_undefined("UVH_RH_GAM_MMIOH_OVERLAY_CONFIG_BASE_SHFT"))
+
+union uvh_rh_gam_mmioh_overlay_config_u {
unsigned long v;
- struct uv2h_event_occurred2_s {
- unsigned long rtc_0:1; /* RW */
- unsigned long rtc_1:1; /* RW */
- unsigned long rtc_2:1; /* RW */
- unsigned long rtc_3:1; /* RW */
- unsigned long rtc_4:1; /* RW */
- unsigned long rtc_5:1; /* RW */
- unsigned long rtc_6:1; /* RW */
- unsigned long rtc_7:1; /* RW */
- unsigned long rtc_8:1; /* RW */
- unsigned long rtc_9:1; /* RW */
- unsigned long rtc_10:1; /* RW */
- unsigned long rtc_11:1; /* RW */
- unsigned long rtc_12:1; /* RW */
- unsigned long rtc_13:1; /* RW */
- unsigned long rtc_14:1; /* RW */
- unsigned long rtc_15:1; /* RW */
- unsigned long rtc_16:1; /* RW */
- unsigned long rtc_17:1; /* RW */
- unsigned long rtc_18:1; /* RW */
- unsigned long rtc_19:1; /* RW */
- unsigned long rtc_20:1; /* RW */
- unsigned long rtc_21:1; /* RW */
- unsigned long rtc_22:1; /* RW */
- unsigned long rtc_23:1; /* RW */
- unsigned long rtc_24:1; /* RW */
- unsigned long rtc_25:1; /* RW */
- unsigned long rtc_26:1; /* RW */
- unsigned long rtc_27:1; /* RW */
- unsigned long rtc_28:1; /* RW */
- unsigned long rtc_29:1; /* RW */
- unsigned long rtc_30:1; /* RW */
- unsigned long rtc_31:1; /* RW */
- unsigned long rsvd_32_63:32;
+
+ /* UVH common struct */
+ struct uvh_rh_gam_mmioh_overlay_config_s {
+ unsigned long rsvd_0_26:27;
+ unsigned long base:19; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4; /* RW */
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } s;
+
+ /* UVXH common struct */
+ struct uvxh_rh_gam_mmioh_overlay_config_s {
+ unsigned long rsvd_0_26:27;
+ unsigned long base:19; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4; /* RW */
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } sx;
+
+ /* UV2 unique struct */
+ struct uv2h_rh_gam_mmioh_overlay_config_s {
+ unsigned long rsvd_0_26:27;
+ unsigned long base:19; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4; /* RW */
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
} s2;
- struct uv3h_event_occurred2_s {
- unsigned long rtc_0:1; /* RW */
- unsigned long rtc_1:1; /* RW */
- unsigned long rtc_2:1; /* RW */
- unsigned long rtc_3:1; /* RW */
- unsigned long rtc_4:1; /* RW */
- unsigned long rtc_5:1; /* RW */
- unsigned long rtc_6:1; /* RW */
- unsigned long rtc_7:1; /* RW */
- unsigned long rtc_8:1; /* RW */
- unsigned long rtc_9:1; /* RW */
- unsigned long rtc_10:1; /* RW */
- unsigned long rtc_11:1; /* RW */
- unsigned long rtc_12:1; /* RW */
- unsigned long rtc_13:1; /* RW */
- unsigned long rtc_14:1; /* RW */
- unsigned long rtc_15:1; /* RW */
- unsigned long rtc_16:1; /* RW */
- unsigned long rtc_17:1; /* RW */
- unsigned long rtc_18:1; /* RW */
- unsigned long rtc_19:1; /* RW */
- unsigned long rtc_20:1; /* RW */
- unsigned long rtc_21:1; /* RW */
- unsigned long rtc_22:1; /* RW */
- unsigned long rtc_23:1; /* RW */
- unsigned long rtc_24:1; /* RW */
- unsigned long rtc_25:1; /* RW */
- unsigned long rtc_26:1; /* RW */
- unsigned long rtc_27:1; /* RW */
- unsigned long rtc_28:1; /* RW */
- unsigned long rtc_29:1; /* RW */
- unsigned long rtc_30:1; /* RW */
- unsigned long rtc_31:1; /* RW */
- unsigned long rsvd_32_63:32;
+};
+
+/* ========================================================================= */
+/* UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0 */
+/* ========================================================================= */
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0 ( \
+ is_uv(UV4) ? 0x483000UL : \
+ is_uv(UV3) ? 0x1603000UL : \
+ 0)
+
+/* UV4A unique defines */
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_BASE_SHFT 26
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_BASE_MASK 0x000ffffffc000000UL
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_M_IO_SHFT 52
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_M_IO_MASK 0x03f0000000000000UL
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_ENABLE_SHFT 63
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_ENABLE_MASK 0x8000000000000000UL
+
+/* UV4 unique defines */
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_BASE_SHFT 26
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_BASE_MASK 0x00003ffffc000000UL
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_M_IO_SHFT 46
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_M_IO_MASK 0x000fc00000000000UL
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_ENABLE_SHFT 63
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_ENABLE_MASK 0x8000000000000000UL
+
+/* UV3 unique defines */
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_BASE_SHFT 26
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_BASE_MASK 0x00003ffffc000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_M_IO_SHFT 46
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_M_IO_MASK 0x000fc00000000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_ENABLE_SHFT 63
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_ENABLE_MASK 0x8000000000000000UL
+
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_BASE_MASK ( \
+ is_uv(UV4A) ? 0x000ffffffc000000UL : \
+ is_uv(UV4) ? 0x00003ffffc000000UL : \
+ is_uv(UV3) ? 0x00003ffffc000000UL : \
+ 0)
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_BASE_SHFT ( \
+ is_uv(UV4) ? 26 : \
+ is_uv(UV3) ? 26 : \
+ -1)
+
+union uvh_rh_gam_mmioh_overlay_config0_u {
+ unsigned long v;
+
+ /* UVH common struct */
+ struct uvh_rh_gam_mmioh_overlay_config0_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } s;
+
+ /* UVXH common struct */
+ struct uvxh_rh_gam_mmioh_overlay_config0_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } sx;
+
+ /* UV4A unique struct */
+ struct uv4ah_rh_gam_mmioh_overlay_config0_mmr_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:26; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long undef_62:1; /* Undefined */
+ unsigned long enable:1; /* RW */
+ } s4a;
+
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_mmioh_overlay_config0_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_mmioh_overlay_config0_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
} s3;
- struct uv4h_event_occurred2_s {
- unsigned long message_accelerator_int0:1; /* RW */
- unsigned long message_accelerator_int1:1; /* RW */
- unsigned long message_accelerator_int2:1; /* RW */
- unsigned long message_accelerator_int3:1; /* RW */
- unsigned long message_accelerator_int4:1; /* RW */
- unsigned long message_accelerator_int5:1; /* RW */
- unsigned long message_accelerator_int6:1; /* RW */
- unsigned long message_accelerator_int7:1; /* RW */
- unsigned long message_accelerator_int8:1; /* RW */
- unsigned long message_accelerator_int9:1; /* RW */
- unsigned long message_accelerator_int10:1; /* RW */
- unsigned long message_accelerator_int11:1; /* RW */
- unsigned long message_accelerator_int12:1; /* RW */
- unsigned long message_accelerator_int13:1; /* RW */
- unsigned long message_accelerator_int14:1; /* RW */
- unsigned long message_accelerator_int15:1; /* RW */
- unsigned long rtc_interval_int:1; /* RW */
- unsigned long bau_dashboard_int:1; /* RW */
- unsigned long rtc_0:1; /* RW */
- unsigned long rtc_1:1; /* RW */
- unsigned long rtc_2:1; /* RW */
- unsigned long rtc_3:1; /* RW */
- unsigned long rtc_4:1; /* RW */
- unsigned long rtc_5:1; /* RW */
- unsigned long rtc_6:1; /* RW */
- unsigned long rtc_7:1; /* RW */
- unsigned long rtc_8:1; /* RW */
- unsigned long rtc_9:1; /* RW */
- unsigned long rtc_10:1; /* RW */
- unsigned long rtc_11:1; /* RW */
- unsigned long rtc_12:1; /* RW */
- unsigned long rtc_13:1; /* RW */
- unsigned long rtc_14:1; /* RW */
- unsigned long rtc_15:1; /* RW */
- unsigned long rtc_16:1; /* RW */
- unsigned long rtc_17:1; /* RW */
- unsigned long rtc_18:1; /* RW */
- unsigned long rtc_19:1; /* RW */
- unsigned long rtc_20:1; /* RW */
- unsigned long rtc_21:1; /* RW */
- unsigned long rtc_22:1; /* RW */
- unsigned long rtc_23:1; /* RW */
- unsigned long rtc_24:1; /* RW */
- unsigned long rtc_25:1; /* RW */
- unsigned long rtc_26:1; /* RW */
- unsigned long rtc_27:1; /* RW */
- unsigned long rtc_28:1; /* RW */
- unsigned long rtc_29:1; /* RW */
- unsigned long rtc_30:1; /* RW */
- unsigned long rtc_31:1; /* RW */
- unsigned long rsvd_50_63:14;
+};
+
+/* ========================================================================= */
+/* UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1 */
+/* ========================================================================= */
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1 ( \
+ is_uv(UV4) ? 0x484000UL : \
+ is_uv(UV3) ? 0x1604000UL : \
+ 0)
+
+/* UV4A unique defines */
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_BASE_SHFT 26
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_BASE_MASK 0x000ffffffc000000UL
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_M_IO_SHFT 52
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_M_IO_MASK 0x03f0000000000000UL
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_ENABLE_SHFT 63
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_ENABLE_MASK 0x8000000000000000UL
+
+/* UV4 unique defines */
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_BASE_SHFT 26
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_BASE_MASK 0x00003ffffc000000UL
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_M_IO_SHFT 46
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_M_IO_MASK 0x000fc00000000000UL
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_ENABLE_SHFT 63
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_ENABLE_MASK 0x8000000000000000UL
+
+/* UV3 unique defines */
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_BASE_SHFT 26
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_BASE_MASK 0x00003ffffc000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_M_IO_SHFT 46
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_M_IO_MASK 0x000fc00000000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_ENABLE_SHFT 63
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_ENABLE_MASK 0x8000000000000000UL
+
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_BASE_MASK ( \
+ is_uv(UV4A) ? 0x000ffffffc000000UL : \
+ is_uv(UV4) ? 0x00003ffffc000000UL : \
+ is_uv(UV3) ? 0x00003ffffc000000UL : \
+ 0)
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_BASE_SHFT ( \
+ is_uv(UV4) ? 26 : \
+ is_uv(UV3) ? 26 : \
+ -1)
+
+union uvh_rh_gam_mmioh_overlay_config1_u {
+ unsigned long v;
+
+ /* UVH common struct */
+ struct uvh_rh_gam_mmioh_overlay_config1_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } s;
+
+ /* UVXH common struct */
+ struct uvxh_rh_gam_mmioh_overlay_config1_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } sx;
+
+ /* UV4A unique struct */
+ struct uv4ah_rh_gam_mmioh_overlay_config1_mmr_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:26; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long undef_62:1; /* Undefined */
+ unsigned long enable:1; /* RW */
+ } s4a;
+
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_mmioh_overlay_config1_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
} s4;
+
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_mmioh_overlay_config1_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long m_io:6; /* RW */
+ unsigned long n_io:4;
+ unsigned long rsvd_56_62:7;
+ unsigned long enable:1; /* RW */
+ } s3;
};
/* ========================================================================= */
-/* UVXH_EVENT_OCCURRED2_ALIAS */
+/* UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0 */
/* ========================================================================= */
-#define UVXH_EVENT_OCCURRED2_ALIAS 0x70108UL
+#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0 ( \
+ is_uv(UV4) ? 0x483800UL : \
+ is_uv(UV3) ? 0x1603800UL : \
+ 0)
-#define UV2H_EVENT_OCCURRED2_ALIAS_32 0xb70
-#define UV3H_EVENT_OCCURRED2_ALIAS_32 0xb70
-#define UV4H_EVENT_OCCURRED2_ALIAS_32 0x610
-#define UVH_EVENT_OCCURRED2_ALIAS_32 ( \
- is_uv2_hub() ? UV2H_EVENT_OCCURRED2_ALIAS_32 : \
- is_uv3_hub() ? UV3H_EVENT_OCCURRED2_ALIAS_32 : \
- /*is_uv4_hub*/ UV4H_EVENT_OCCURRED2_ALIAS_32)
+#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_DEPTH ( \
+ is_uv(UV4) ? 128 : \
+ is_uv(UV3) ? 128 : \
+ 0)
+/* UV4A unique defines */
+#define UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG0_NASID_SHFT 0
+#define UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG0_NASID_MASK 0x0000000000000fffUL
-/* ========================================================================= */
-/* UVXH_LB_BAU_SB_ACTIVATION_STATUS_2 */
-/* ========================================================================= */
-#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_2 0x320130UL
-#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_2 0x320130UL
-#define UV4H_LB_BAU_SB_ACTIVATION_STATUS_2 0xc8130UL
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_2 ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_ACTIVATION_STATUS_2 : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_ACTIVATION_STATUS_2 : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_ACTIVATION_STATUS_2)
+/* UV4 unique defines */
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_NASID_SHFT 0
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_NASID_MASK 0x0000000000007fffUL
+
+/* UV3 unique defines */
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_NASID_SHFT 0
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_NASID_MASK 0x0000000000007fffUL
-#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_2_32 0x9f0
-#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_2_32 0x9f0
-#define UV4H_LB_BAU_SB_ACTIVATION_STATUS_2_32 0xa10
-#define UVH_LB_BAU_SB_ACTIVATION_STATUS_2_32 ( \
- is_uv2_hub() ? UV2H_LB_BAU_SB_ACTIVATION_STATUS_2_32 : \
- is_uv3_hub() ? UV3H_LB_BAU_SB_ACTIVATION_STATUS_2_32 : \
- /*is_uv4_hub*/ UV4H_LB_BAU_SB_ACTIVATION_STATUS_2_32)
-#define UVXH_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_SHFT 0
-#define UVXH_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_MASK 0xffffffffffffffffUL
+union uvh_rh_gam_mmioh_redirect_config0_u {
+ unsigned long v;
-#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_SHFT 0
-#define UV2H_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_MASK 0xffffffffffffffffUL
+ /* UVH common struct */
+ struct uvh_rh_gam_mmioh_redirect_config0_s {
+ unsigned long nasid:15; /* RW */
+ unsigned long rsvd_15_63:49;
+ } s;
-#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_SHFT 0
-#define UV3H_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_MASK 0xffffffffffffffffUL
+ /* UVXH common struct */
+ struct uvxh_rh_gam_mmioh_redirect_config0_s {
+ unsigned long nasid:15; /* RW */
+ unsigned long rsvd_15_63:49;
+ } sx;
-#define UV4H_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_SHFT 0
-#define UV4H_LB_BAU_SB_ACTIVATION_STATUS_2_AUX_ERROR_MASK 0xffffffffffffffffUL
+ struct uv4ah_rh_gam_mmioh_redirect_config0_s {
+ unsigned long nasid:12; /* RW */
+ unsigned long rsvd_12_63:52;
+ } s4a;
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_mmioh_redirect_config0_s {
+ unsigned long nasid:15; /* RW */
+ unsigned long rsvd_15_63:49;
+ } s4;
-union uvxh_lb_bau_sb_activation_status_2_u {
- unsigned long v;
- struct uvxh_lb_bau_sb_activation_status_2_s {
- unsigned long aux_error:64; /* RW */
- } sx;
- struct uv2h_lb_bau_sb_activation_status_2_s {
- unsigned long aux_error:64; /* RW */
- } s2;
- struct uv3h_lb_bau_sb_activation_status_2_s {
- unsigned long aux_error:64; /* RW */
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_mmioh_redirect_config0_s {
+ unsigned long nasid:15; /* RW */
+ unsigned long rsvd_15_63:49;
} s3;
- struct uv4h_lb_bau_sb_activation_status_2_s {
- unsigned long aux_error:64; /* RW */
- } s4;
};
/* ========================================================================= */
-/* UV3H_GR0_GAM_GR_CONFIG */
+/* UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1 */
/* ========================================================================= */
-#define UV3H_GR0_GAM_GR_CONFIG 0xc00028UL
+#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1 ( \
+ is_uv(UV4) ? 0x484800UL : \
+ is_uv(UV3) ? 0x1604800UL : \
+ 0)
-#define UV3H_GR0_GAM_GR_CONFIG_M_SKT_SHFT 0
-#define UV3H_GR0_GAM_GR_CONFIG_SUBSPACE_SHFT 10
-#define UV3H_GR0_GAM_GR_CONFIG_M_SKT_MASK 0x000000000000003fUL
-#define UV3H_GR0_GAM_GR_CONFIG_SUBSPACE_MASK 0x0000000000000400UL
+#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_DEPTH ( \
+ is_uv(UV4) ? 128 : \
+ is_uv(UV3) ? 128 : \
+ 0)
+
+/* UV4A unique defines */
+#define UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG0_NASID_SHFT 0
+#define UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG0_NASID_MASK 0x0000000000000fffUL
+
+/* UV4 unique defines */
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_NASID_SHFT 0
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_NASID_MASK 0x0000000000007fffUL
+
+/* UV3 unique defines */
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_NASID_SHFT 0
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_NASID_MASK 0x0000000000007fffUL
-union uv3h_gr0_gam_gr_config_u {
+
+union uvh_rh_gam_mmioh_redirect_config1_u {
unsigned long v;
- struct uv3h_gr0_gam_gr_config_s {
- unsigned long m_skt:6; /* RW */
- unsigned long undef_6_9:4; /* Undefined */
- unsigned long subspace:1; /* RW */
- unsigned long reserved:53;
+
+ /* UVH common struct */
+ struct uvh_rh_gam_mmioh_redirect_config1_s {
+ unsigned long nasid:15; /* RW */
+ unsigned long rsvd_15_63:49;
+ } s;
+
+ /* UVXH common struct */
+ struct uvxh_rh_gam_mmioh_redirect_config1_s {
+ unsigned long nasid:15; /* RW */
+ unsigned long rsvd_15_63:49;
+ } sx;
+
+ struct uv4ah_rh_gam_mmioh_redirect_config1_s {
+ unsigned long nasid:12; /* RW */
+ unsigned long rsvd_12_63:52;
+ } s4a;
+
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_mmioh_redirect_config1_s {
+ unsigned long nasid:15; /* RW */
+ unsigned long rsvd_15_63:49;
+ } s4;
+
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_mmioh_redirect_config1_s {
+ unsigned long nasid:15; /* RW */
+ unsigned long rsvd_15_63:49;
} s3;
};
/* ========================================================================= */
-/* UV4H_LB_PROC_INTD_QUEUE_FIRST */
+/* UVH_RH_GAM_MMR_OVERLAY_CONFIG */
/* ========================================================================= */
-#define UV4H_LB_PROC_INTD_QUEUE_FIRST 0xa4100UL
+#define UVH_RH_GAM_MMR_OVERLAY_CONFIG ( \
+ is_uv(UV4) ? 0x480028UL : \
+ is_uv(UV3) ? 0x1600028UL : \
+ is_uv(UV2) ? 0x1600028UL : \
+ 0)
+
+
+/* UVXH common defines */
+#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_BASE_SHFT 26
+#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_BASE_MASK ( \
+ is_uv(UV4A) ? 0x000ffffffc000000UL : \
+ is_uv(UV4) ? 0x00003ffffc000000UL : \
+ is_uv(UV3) ? 0x00003ffffc000000UL : \
+ is_uv(UV2) ? 0x00003ffffc000000UL : \
+ 0)
+#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_ENABLE_SHFT 63
+#define UVXH_RH_GAM_MMR_OVERLAY_CONFIG_ENABLE_MASK 0x8000000000000000UL
+
+/* UV4A unique defines */
+#define UV4AH_RH_GAM_GRU_OVERLAY_CONFIG_BASE_SHFT 26
+#define UV4AH_RH_GAM_GRU_OVERLAY_CONFIG_BASE_MASK 0x000ffffffc000000UL
+
+#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_BASE_MASK ( \
+ is_uv(UV4A) ? 0x000ffffffc000000UL : \
+ is_uv(UV4) ? 0x00003ffffc000000UL : \
+ is_uv(UV3) ? 0x00003ffffc000000UL : \
+ is_uv(UV2) ? 0x00003ffffc000000UL : \
+ 0)
+
+#define UVH_RH_GAM_MMR_OVERLAY_CONFIG_BASE_SHFT ( \
+ is_uv(UV4) ? 26 : \
+ is_uv(UV3) ? 26 : \
+ is_uv(UV2) ? 26 : \
+ -1)
+
+union uvh_rh_gam_mmr_overlay_config_u {
+ unsigned long v;
+
+ /* UVH common struct */
+ struct uvh_rh_gam_mmr_overlay_config_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long rsvd_46_62:17;
+ unsigned long enable:1; /* RW */
+ } s;
-#define UV4H_LB_PROC_INTD_QUEUE_FIRST_FIRST_PAYLOAD_ADDRESS_SHFT 6
-#define UV4H_LB_PROC_INTD_QUEUE_FIRST_FIRST_PAYLOAD_ADDRESS_MASK 0x00003fffffffffc0UL
+ /* UVXH common struct */
+ struct uvxh_rh_gam_mmr_overlay_config_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long rsvd_46_62:17;
+ unsigned long enable:1; /* RW */
+ } sx;
-union uv4h_lb_proc_intd_queue_first_u {
- unsigned long v;
- struct uv4h_lb_proc_intd_queue_first_s {
- unsigned long undef_0_5:6; /* Undefined */
- unsigned long first_payload_address:40; /* RW */
+ /* UV4 unique struct */
+ struct uv4h_rh_gam_mmr_overlay_config_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long rsvd_46_62:17;
+ unsigned long enable:1; /* RW */
} s4;
+
+ /* UV3 unique struct */
+ struct uv3h_rh_gam_mmr_overlay_config_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long rsvd_46_62:17;
+ unsigned long enable:1; /* RW */
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_rh_gam_mmr_overlay_config_s {
+ unsigned long rsvd_0_25:26;
+ unsigned long base:20; /* RW */
+ unsigned long rsvd_46_62:17;
+ unsigned long enable:1; /* RW */
+ } s2;
};
/* ========================================================================= */
-/* UV4H_LB_PROC_INTD_QUEUE_LAST */
+/* UVH_RTC */
/* ========================================================================= */
-#define UV4H_LB_PROC_INTD_QUEUE_LAST 0xa4108UL
+#define UVH_RTC ( \
+ is_uv(UV5) ? 0xe0000UL : \
+ is_uv(UV4) ? 0xe0000UL : \
+ is_uv(UV3) ? 0x340000UL : \
+ is_uv(UV2) ? 0x340000UL : \
+ 0)
+
+/* UVH common defines*/
+#define UVH_RTC_REAL_TIME_CLOCK_SHFT 0
+#define UVH_RTC_REAL_TIME_CLOCK_MASK 0x00ffffffffffffffUL
-#define UV4H_LB_PROC_INTD_QUEUE_LAST_LAST_PAYLOAD_ADDRESS_SHFT 5
-#define UV4H_LB_PROC_INTD_QUEUE_LAST_LAST_PAYLOAD_ADDRESS_MASK 0x00003fffffffffe0UL
-union uv4h_lb_proc_intd_queue_last_u {
+union uvh_rtc_u {
unsigned long v;
- struct uv4h_lb_proc_intd_queue_last_s {
- unsigned long undef_0_4:5; /* Undefined */
- unsigned long last_payload_address:41; /* RW */
+
+ /* UVH common struct */
+ struct uvh_rtc_s {
+ unsigned long real_time_clock:56; /* RW */
+ unsigned long rsvd_56_63:8;
+ } s;
+
+ /* UV5 unique struct */
+ struct uv5h_rtc_s {
+ unsigned long real_time_clock:56; /* RW */
+ unsigned long rsvd_56_63:8;
+ } s5;
+
+ /* UV4 unique struct */
+ struct uv4h_rtc_s {
+ unsigned long real_time_clock:56; /* RW */
+ unsigned long rsvd_56_63:8;
} s4;
+
+ /* UV3 unique struct */
+ struct uv3h_rtc_s {
+ unsigned long real_time_clock:56; /* RW */
+ unsigned long rsvd_56_63:8;
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_rtc_s {
+ unsigned long real_time_clock:56; /* RW */
+ unsigned long rsvd_56_63:8;
+ } s2;
};
/* ========================================================================= */
-/* UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR */
+/* UVH_RTC1_INT_CONFIG */
/* ========================================================================= */
-#define UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR 0xa4118UL
+#define UVH_RTC1_INT_CONFIG 0x615c0UL
+
+/* UVH common defines*/
+#define UVH_RTC1_INT_CONFIG_VECTOR_SHFT 0
+#define UVH_RTC1_INT_CONFIG_VECTOR_MASK 0x00000000000000ffUL
+#define UVH_RTC1_INT_CONFIG_DM_SHFT 8
+#define UVH_RTC1_INT_CONFIG_DM_MASK 0x0000000000000700UL
+#define UVH_RTC1_INT_CONFIG_DESTMODE_SHFT 11
+#define UVH_RTC1_INT_CONFIG_DESTMODE_MASK 0x0000000000000800UL
+#define UVH_RTC1_INT_CONFIG_STATUS_SHFT 12
+#define UVH_RTC1_INT_CONFIG_STATUS_MASK 0x0000000000001000UL
+#define UVH_RTC1_INT_CONFIG_P_SHFT 13
+#define UVH_RTC1_INT_CONFIG_P_MASK 0x0000000000002000UL
+#define UVH_RTC1_INT_CONFIG_T_SHFT 15
+#define UVH_RTC1_INT_CONFIG_T_MASK 0x0000000000008000UL
+#define UVH_RTC1_INT_CONFIG_M_SHFT 16
+#define UVH_RTC1_INT_CONFIG_M_MASK 0x0000000000010000UL
+#define UVH_RTC1_INT_CONFIG_APIC_ID_SHFT 32
+#define UVH_RTC1_INT_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
-#define UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR_SOFT_ACK_PENDING_FLAGS_SHFT 0
-#define UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR_SOFT_ACK_PENDING_FLAGS_MASK 0x00000000000000ffUL
-union uv4h_lb_proc_intd_soft_ack_clear_u {
+union uvh_rtc1_int_config_u {
unsigned long v;
- struct uv4h_lb_proc_intd_soft_ack_clear_s {
- unsigned long soft_ack_pending_flags:8; /* WP */
+
+ /* UVH common struct */
+ struct uvh_rtc1_int_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s;
+
+ /* UV5 unique struct */
+ struct uv5h_rtc1_int_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s5;
+
+ /* UV4 unique struct */
+ struct uv4h_rtc1_int_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
} s4;
+
+ /* UV3 unique struct */
+ struct uv3h_rtc1_int_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_rtc1_int_config_s {
+ unsigned long vector_:8; /* RW */
+ unsigned long dm:3; /* RW */
+ unsigned long destmode:1; /* RW */
+ unsigned long status:1; /* RO */
+ unsigned long p:1; /* RO */
+ unsigned long rsvd_14:1;
+ unsigned long t:1; /* RO */
+ unsigned long m:1; /* RW */
+ unsigned long rsvd_17_31:15;
+ unsigned long apic_id:32; /* RW */
+ } s2;
};
/* ========================================================================= */
-/* UV4H_LB_PROC_INTD_SOFT_ACK_PENDING */
+/* UVH_SCRATCH5 */
/* ========================================================================= */
-#define UV4H_LB_PROC_INTD_SOFT_ACK_PENDING 0xa4110UL
+#define UVH_SCRATCH5 ( \
+ is_uv(UV5) ? 0xb0200UL : \
+ is_uv(UV4) ? 0xb0200UL : \
+ is_uv(UV3) ? 0x2d0200UL : \
+ is_uv(UV2) ? 0x2d0200UL : \
+ 0)
+#define UV5H_SCRATCH5 0xb0200UL
+#define UV4H_SCRATCH5 0xb0200UL
+#define UV3H_SCRATCH5 0x2d0200UL
+#define UV2H_SCRATCH5 0x2d0200UL
+
+/* UVH common defines*/
+#define UVH_SCRATCH5_SCRATCH5_SHFT 0
+#define UVH_SCRATCH5_SCRATCH5_MASK 0xffffffffffffffffUL
+
+/* UVXH common defines */
+#define UVXH_SCRATCH5_SCRATCH5_SHFT 0
+#define UVXH_SCRATCH5_SCRATCH5_MASK 0xffffffffffffffffUL
+
+/* UVYH common defines */
+#define UVYH_SCRATCH5_SCRATCH5_SHFT 0
+#define UVYH_SCRATCH5_SCRATCH5_MASK 0xffffffffffffffffUL
+
+/* UV5 unique defines */
+#define UV5H_SCRATCH5_SCRATCH5_SHFT 0
+#define UV5H_SCRATCH5_SCRATCH5_MASK 0xffffffffffffffffUL
+
+/* UV4 unique defines */
+#define UV4H_SCRATCH5_SCRATCH5_SHFT 0
+#define UV4H_SCRATCH5_SCRATCH5_MASK 0xffffffffffffffffUL
+
+/* UV3 unique defines */
+#define UV3H_SCRATCH5_SCRATCH5_SHFT 0
+#define UV3H_SCRATCH5_SCRATCH5_MASK 0xffffffffffffffffUL
-#define UV4H_LB_PROC_INTD_SOFT_ACK_PENDING_SOFT_ACK_FLAGS_SHFT 0
-#define UV4H_LB_PROC_INTD_SOFT_ACK_PENDING_SOFT_ACK_FLAGS_MASK 0x00000000000000ffUL
+/* UV2 unique defines */
+#define UV2H_SCRATCH5_SCRATCH5_SHFT 0
+#define UV2H_SCRATCH5_SCRATCH5_MASK 0xffffffffffffffffUL
-union uv4h_lb_proc_intd_soft_ack_pending_u {
+
+union uvh_scratch5_u {
unsigned long v;
- struct uv4h_lb_proc_intd_soft_ack_pending_s {
- unsigned long soft_ack_flags:8; /* RW */
+
+ /* UVH common struct */
+ struct uvh_scratch5_s {
+ unsigned long scratch5:64; /* RW */
+ } s;
+
+ /* UVXH common struct */
+ struct uvxh_scratch5_s {
+ unsigned long scratch5:64; /* RW */
+ } sx;
+
+ /* UVYH common struct */
+ struct uvyh_scratch5_s {
+ unsigned long scratch5:64; /* RW */
+ } sy;
+
+ /* UV5 unique struct */
+ struct uv5h_scratch5_s {
+ unsigned long scratch5:64; /* RW */
+ } s5;
+
+ /* UV4 unique struct */
+ struct uv4h_scratch5_s {
+ unsigned long scratch5:64; /* RW */
} s4;
+
+ /* UV3 unique struct */
+ struct uv3h_scratch5_s {
+ unsigned long scratch5:64; /* RW */
+ } s3;
+
+ /* UV2 unique struct */
+ struct uv2h_scratch5_s {
+ unsigned long scratch5:64; /* RW */
+ } s2;
};
+/* ========================================================================= */
+/* UVH_SCRATCH5_ALIAS */
+/* ========================================================================= */
+#define UVH_SCRATCH5_ALIAS ( \
+ is_uv(UV5) ? 0xb0208UL : \
+ is_uv(UV4) ? 0xb0208UL : \
+ is_uv(UV3) ? 0x2d0208UL : \
+ is_uv(UV2) ? 0x2d0208UL : \
+ 0)
+#define UV5H_SCRATCH5_ALIAS 0xb0208UL
+#define UV4H_SCRATCH5_ALIAS 0xb0208UL
+#define UV3H_SCRATCH5_ALIAS 0x2d0208UL
+#define UV2H_SCRATCH5_ALIAS 0x2d0208UL
+
+
+/* ========================================================================= */
+/* UVH_SCRATCH5_ALIAS_2 */
+/* ========================================================================= */
+#define UVH_SCRATCH5_ALIAS_2 ( \
+ is_uv(UV5) ? 0xb0210UL : \
+ is_uv(UV4) ? 0xb0210UL : \
+ is_uv(UV3) ? 0x2d0210UL : \
+ is_uv(UV2) ? 0x2d0210UL : \
+ 0)
+#define UV5H_SCRATCH5_ALIAS_2 0xb0210UL
+#define UV4H_SCRATCH5_ALIAS_2 0xb0210UL
+#define UV3H_SCRATCH5_ALIAS_2 0x2d0210UL
+#define UV2H_SCRATCH5_ALIAS_2 0x2d0210UL
+
+
#endif /* _ASM_X86_UV_UV_MMRS_H */
struct mpc_cpu;
struct mpc_table;
struct cpuinfo_x86;
+struct irq_domain;
/**
* struct x86_init_mpparse - platform specific mpparse ops
- * @mpc_record: platform specific mpc record accounting
* @setup_ioapic_ids: platform specific ioapic id override
- * @mpc_apic_id: platform specific mpc apic id assignment
- * @smp_read_mpc_oem: platform specific oem mpc table setup
- * @mpc_oem_pci_bus: platform specific pci bus setup (default NULL)
- * @mpc_oem_bus_info: platform specific mpc bus info
* @find_smp_config: find the smp configuration
* @get_smp_config: get the smp configuration
*/
struct x86_init_mpparse {
- void (*mpc_record)(unsigned int mode);
void (*setup_ioapic_ids)(void);
- int (*mpc_apic_id)(struct mpc_cpu *m);
- void (*smp_read_mpc_oem)(struct mpc_table *mpc);
- void (*mpc_oem_pci_bus)(struct mpc_bus *m);
- void (*mpc_oem_bus_info)(struct mpc_bus *m, char *name);
void (*find_smp_config)(void);
void (*get_smp_config)(unsigned int early);
};
* @intr_init: interrupt init code
* @intr_mode_select: interrupt delivery mode selection
* @intr_mode_init: interrupt delivery mode setup
+ * @create_pci_msi_domain: Create the PCI/MSI interrupt domain
*/
struct x86_init_irqs {
void (*pre_vector_init)(void);
void (*intr_init)(void);
void (*intr_mode_select)(void);
void (*intr_mode_init)(void);
+ struct irq_domain *(*create_pci_msi_domain)(void);
};
/**
struct pci_dev;
struct x86_msi_ops {
- int (*setup_msi_irqs)(struct pci_dev *dev, int nvec, int type);
- void (*teardown_msi_irq)(unsigned int irq);
- void (*teardown_msi_irqs)(struct pci_dev *dev);
void (*restore_msi_irqs)(struct pci_dev *dev);
};
obj-y += pci-iommu_table.o
obj-y += resource.o
obj-y += irqflags.o
+obj-y += static_call.o
obj-y += process.o
obj-y += fpu/
*/
goto out_put;
+ case RET_INSN_OPCODE:
+ int3_emulate_ret(regs);
+ break;
+
case CALL_INSN_OPCODE:
int3_emulate_call(regs, (long)ip + tp->rel32);
break;
switch (tp->opcode) {
case INT3_INSN_OPCODE:
+ case RET_INSN_OPCODE:
break;
case CALL_INSN_OPCODE:
break;
}
+ if (x86_platform.apic_post_init)
+ x86_platform.apic_post_init();
+
apic_bsp_setup(upmode);
}
{
init_irq_alloc_info(info, NULL);
info->type = X86_IRQ_ALLOC_TYPE_IOAPIC;
- info->ioapic_node = node;
- info->ioapic_trigger = trigger;
- info->ioapic_polarity = polarity;
- info->ioapic_valid = 1;
+ info->ioapic.node = node;
+ info->ioapic.trigger = trigger;
+ info->ioapic.polarity = polarity;
+ info->ioapic.valid = 1;
}
#ifndef CONFIG_ACPI
copy_irq_alloc_info(dst, src);
dst->type = X86_IRQ_ALLOC_TYPE_IOAPIC;
- dst->ioapic_id = mpc_ioapic_id(ioapic_idx);
- dst->ioapic_pin = pin;
- dst->ioapic_valid = 1;
- if (src && src->ioapic_valid) {
- dst->ioapic_node = src->ioapic_node;
- dst->ioapic_trigger = src->ioapic_trigger;
- dst->ioapic_polarity = src->ioapic_polarity;
+ dst->devid = mpc_ioapic_id(ioapic_idx);
+ dst->ioapic.pin = pin;
+ dst->ioapic.valid = 1;
+ if (src && src->ioapic.valid) {
+ dst->ioapic.node = src->ioapic.node;
+ dst->ioapic.trigger = src->ioapic.trigger;
+ dst->ioapic.polarity = src->ioapic.polarity;
} else {
- dst->ioapic_node = NUMA_NO_NODE;
+ dst->ioapic.node = NUMA_NO_NODE;
if (acpi_get_override_irq(gsi, &trigger, &polarity) >= 0) {
- dst->ioapic_trigger = trigger;
- dst->ioapic_polarity = polarity;
+ dst->ioapic.trigger = trigger;
+ dst->ioapic.polarity = polarity;
} else {
/*
* PCI interrupts are always active low level
* triggered.
*/
- dst->ioapic_trigger = IOAPIC_LEVEL;
- dst->ioapic_polarity = IOAPIC_POL_LOW;
+ dst->ioapic.trigger = IOAPIC_LEVEL;
+ dst->ioapic.polarity = IOAPIC_POL_LOW;
}
}
}
static int ioapic_alloc_attr_node(struct irq_alloc_info *info)
{
- return (info && info->ioapic_valid) ? info->ioapic_node : NUMA_NO_NODE;
+ return (info && info->ioapic.valid) ? info->ioapic.node : NUMA_NO_NODE;
}
static void mp_register_handler(unsigned int irq, unsigned long trigger)
* pin with real trigger and polarity attributes.
*/
if (irq < nr_legacy_irqs() && data->count == 1) {
- if (info->ioapic_trigger != data->trigger)
- mp_register_handler(irq, info->ioapic_trigger);
- data->entry.trigger = data->trigger = info->ioapic_trigger;
- data->entry.polarity = data->polarity = info->ioapic_polarity;
+ if (info->ioapic.trigger != data->trigger)
+ mp_register_handler(irq, info->ioapic.trigger);
+ data->entry.trigger = data->trigger = info->ioapic.trigger;
+ data->entry.polarity = data->polarity = info->ioapic.polarity;
}
- return data->trigger == info->ioapic_trigger &&
- data->polarity == info->ioapic_polarity;
+ return data->trigger == info->ioapic.trigger &&
+ data->polarity == info->ioapic.polarity;
}
static int alloc_irq_from_domain(struct irq_domain *domain, int ioapic, u32 gsi,
if (!mp_check_pin_attr(irq, info))
return -EBUSY;
if (__add_pin_to_irq_node(irq_data->chip_data, node, ioapic,
- info->ioapic_pin))
+ info->ioapic.pin))
return -ENOMEM;
} else {
info->flags |= X86_IRQ_ALLOC_LEGACY;
struct irq_alloc_info info;
ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 0, 0);
- info.ioapic_id = mpc_ioapic_id(ioapic);
- info.ioapic_pin = pin;
+ info.devid = mpc_ioapic_id(ioapic);
+ info.ioapic.pin = pin;
mutex_lock(&ioapic_mutex);
irq = alloc_isa_irq_from_domain(domain, 0, ioapic, pin, &info);
mutex_unlock(&ioapic_mutex);
legacy_pic->init(0);
legacy_pic->make_irq(0);
apic_write(APIC_LVT0, APIC_DM_EXTINT);
+ legacy_pic->unmask(0);
unlock_ExtINT_logic();
return 0;
init_irq_alloc_info(&info, NULL);
- info.type = X86_IRQ_ALLOC_TYPE_IOAPIC;
- info.ioapic_id = mpc_ioapic_id(ioapic);
- parent = irq_remapping_get_ir_irq_domain(&info);
+ info.type = X86_IRQ_ALLOC_TYPE_IOAPIC_GET_PARENT;
+ info.devid = mpc_ioapic_id(ioapic);
+ parent = irq_remapping_get_irq_domain(&info);
if (!parent)
parent = x86_vector_domain;
else
static void mp_irqdomain_get_attr(u32 gsi, struct mp_chip_data *data,
struct irq_alloc_info *info)
{
- if (info && info->ioapic_valid) {
- data->trigger = info->ioapic_trigger;
- data->polarity = info->ioapic_polarity;
+ if (info && info->ioapic.valid) {
+ data->trigger = info->ioapic.trigger;
+ data->polarity = info->ioapic.polarity;
} else if (acpi_get_override_irq(gsi, &data->trigger,
&data->polarity) < 0) {
/* PCI interrupts are always active low level triggered. */
return -EINVAL;
ioapic = mp_irqdomain_ioapic_idx(domain);
- pin = info->ioapic_pin;
+ pin = info->ioapic.pin;
if (irq_find_mapping(domain, (irq_hw_number_t)pin) > 0)
return -EEXIST;
if (!data)
return -ENOMEM;
- info->ioapic_entry = &data->entry;
+ info->ioapic.entry = &data->entry;
ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, info);
if (ret < 0) {
kfree(data);
}
INIT_LIST_HEAD(&data->irq_2_pin);
- irq_data->hwirq = info->ioapic_pin;
+ irq_data->hwirq = info->ioapic.pin;
irq_data->chip = (domain->parent == x86_vector_domain) ?
&ioapic_chip : &ioapic_ir_chip;
irq_data->chip_data = data;
add_pin_to_irq_node(data, ioapic_alloc_attr_node(info), ioapic, pin);
local_irq_save(flags);
- if (info->ioapic_entry)
- mp_setup_entry(cfg, data, info->ioapic_entry);
+ if (info->ioapic.entry)
+ mp_setup_entry(cfg, data, info->ioapic.entry);
mp_register_handler(virq, data->trigger);
if (virq < nr_legacy_irqs())
legacy_pic->mask(virq);
#include <asm/apic.h>
#include <asm/irq_remapping.h>
-static struct irq_domain *msi_default_domain;
+struct irq_domain *x86_pci_msi_default_domain __ro_after_init;
static void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg)
{
MSI_DATA_VECTOR(cfg->vector);
}
-static void irq_msi_compose_msg(struct irq_data *data, struct msi_msg *msg)
+void x86_vector_msi_compose_msg(struct irq_data *data, struct msi_msg *msg)
{
__irq_msi_compose_msg(irqd_cfg(data), msg);
}
.irq_mask = pci_msi_mask_irq,
.irq_ack = irq_chip_ack_parent,
.irq_retrigger = irq_chip_retrigger_hierarchy,
- .irq_compose_msi_msg = irq_msi_compose_msg,
.irq_set_affinity = msi_set_affinity,
.flags = IRQCHIP_SKIP_SET_WAKE,
};
-int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
-{
- struct irq_domain *domain;
- struct irq_alloc_info info;
-
- init_irq_alloc_info(&info, NULL);
- info.type = X86_IRQ_ALLOC_TYPE_MSI;
- info.msi_dev = dev;
-
- domain = irq_remapping_get_irq_domain(&info);
- if (domain == NULL)
- domain = msi_default_domain;
- if (domain == NULL)
- return -ENOSYS;
-
- return msi_domain_alloc_irqs(domain, &dev->dev, nvec);
-}
-
-void native_teardown_msi_irq(unsigned int irq)
-{
- irq_domain_free_irqs(irq, 1);
-}
-
-static irq_hw_number_t pci_msi_get_hwirq(struct msi_domain_info *info,
- msi_alloc_info_t *arg)
-{
- return arg->msi_hwirq;
-}
-
int pci_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec,
msi_alloc_info_t *arg)
{
struct msi_desc *desc = first_pci_msi_entry(pdev);
init_irq_alloc_info(arg, NULL);
- arg->msi_dev = pdev;
if (desc->msi_attrib.is_msix) {
- arg->type = X86_IRQ_ALLOC_TYPE_MSIX;
+ arg->type = X86_IRQ_ALLOC_TYPE_PCI_MSIX;
} else {
- arg->type = X86_IRQ_ALLOC_TYPE_MSI;
+ arg->type = X86_IRQ_ALLOC_TYPE_PCI_MSI;
arg->flags |= X86_IRQ_ALLOC_CONTIGUOUS_VECTORS;
}
}
EXPORT_SYMBOL_GPL(pci_msi_prepare);
-void pci_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
-{
- arg->msi_hwirq = pci_msi_domain_calc_hwirq(arg->msi_dev, desc);
-}
-EXPORT_SYMBOL_GPL(pci_msi_set_desc);
-
static struct msi_domain_ops pci_msi_domain_ops = {
- .get_hwirq = pci_msi_get_hwirq,
.msi_prepare = pci_msi_prepare,
- .set_desc = pci_msi_set_desc,
};
static struct msi_domain_info pci_msi_domain_info = {
.handler_name = "edge",
};
-void __init arch_init_msi_domain(struct irq_domain *parent)
+struct irq_domain * __init native_create_pci_msi_domain(void)
{
struct fwnode_handle *fn;
+ struct irq_domain *d;
if (disable_apic)
- return;
+ return NULL;
fn = irq_domain_alloc_named_fwnode("PCI-MSI");
- if (fn) {
- msi_default_domain =
- pci_msi_create_irq_domain(fn, &pci_msi_domain_info,
- parent);
- }
- if (!msi_default_domain) {
+ if (!fn)
+ return NULL;
+
+ d = pci_msi_create_irq_domain(fn, &pci_msi_domain_info,
+ x86_vector_domain);
+ if (!d) {
irq_domain_free_fwnode(fn);
- pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n");
+ pr_warn("Failed to initialize PCI-MSI irqdomain.\n");
} else {
- msi_default_domain->flags |= IRQ_DOMAIN_MSI_NOMASK_QUIRK;
+ d->flags |= IRQ_DOMAIN_MSI_NOMASK_QUIRK;
}
+ return d;
+}
+
+void __init x86_create_pci_msi_domain(void)
+{
+ x86_pci_msi_default_domain = x86_init.irqs.create_pci_msi_domain();
}
#ifdef CONFIG_IRQ_REMAP
.irq_mask = pci_msi_mask_irq,
.irq_ack = irq_chip_ack_parent,
.irq_retrigger = irq_chip_retrigger_hierarchy,
- .irq_set_vcpu_affinity = irq_chip_set_vcpu_affinity_parent,
.flags = IRQCHIP_SKIP_SET_WAKE,
};
.irq_ack = irq_chip_ack_parent,
.irq_set_affinity = msi_domain_set_affinity,
.irq_retrigger = irq_chip_retrigger_hierarchy,
- .irq_compose_msi_msg = irq_msi_compose_msg,
.irq_write_msi_msg = dmar_msi_write_msg,
.flags = IRQCHIP_SKIP_SET_WAKE,
};
-static irq_hw_number_t dmar_msi_get_hwirq(struct msi_domain_info *info,
- msi_alloc_info_t *arg)
-{
- return arg->dmar_id;
-}
-
static int dmar_msi_init(struct irq_domain *domain,
struct msi_domain_info *info, unsigned int virq,
irq_hw_number_t hwirq, msi_alloc_info_t *arg)
{
- irq_domain_set_info(domain, virq, arg->dmar_id, info->chip, NULL,
- handle_edge_irq, arg->dmar_data, "edge");
+ irq_domain_set_info(domain, virq, arg->devid, info->chip, NULL,
+ handle_edge_irq, arg->data, "edge");
return 0;
}
static struct msi_domain_ops dmar_msi_domain_ops = {
- .get_hwirq = dmar_msi_get_hwirq,
.msi_init = dmar_msi_init,
};
static struct msi_domain_info dmar_msi_domain_info = {
.ops = &dmar_msi_domain_ops,
.chip = &dmar_msi_controller,
+ .flags = MSI_FLAG_USE_DEF_DOM_OPS,
};
static struct irq_domain *dmar_get_irq_domain(void)
init_irq_alloc_info(&info, NULL);
info.type = X86_IRQ_ALLOC_TYPE_DMAR;
- info.dmar_id = id;
- info.dmar_data = arg;
+ info.devid = id;
+ info.hwirq = id;
+ info.data = arg;
return irq_domain_alloc_irqs(domain, 1, node, &info);
}
.irq_ack = irq_chip_ack_parent,
.irq_set_affinity = msi_domain_set_affinity,
.irq_retrigger = irq_chip_retrigger_hierarchy,
- .irq_compose_msi_msg = irq_msi_compose_msg,
.irq_write_msi_msg = hpet_msi_write_msg,
.flags = IRQCHIP_SKIP_SET_WAKE,
};
-static irq_hw_number_t hpet_msi_get_hwirq(struct msi_domain_info *info,
- msi_alloc_info_t *arg)
-{
- return arg->hpet_index;
-}
-
static int hpet_msi_init(struct irq_domain *domain,
struct msi_domain_info *info, unsigned int virq,
irq_hw_number_t hwirq, msi_alloc_info_t *arg)
{
irq_set_status_flags(virq, IRQ_MOVE_PCNTXT);
- irq_domain_set_info(domain, virq, arg->hpet_index, info->chip, NULL,
- handle_edge_irq, arg->hpet_data, "edge");
+ irq_domain_set_info(domain, virq, arg->hwirq, info->chip, NULL,
+ handle_edge_irq, arg->data, "edge");
return 0;
}
}
static struct msi_domain_ops hpet_msi_domain_ops = {
- .get_hwirq = hpet_msi_get_hwirq,
.msi_init = hpet_msi_init,
.msi_free = hpet_msi_free,
};
static struct msi_domain_info hpet_msi_domain_info = {
.ops = &hpet_msi_domain_ops,
.chip = &hpet_msi_controller,
+ .flags = MSI_FLAG_USE_DEF_DOM_OPS,
};
struct irq_domain *hpet_create_irq_domain(int hpet_id)
domain_info->data = (void *)(long)hpet_id;
init_irq_alloc_info(&info, NULL);
- info.type = X86_IRQ_ALLOC_TYPE_HPET;
- info.hpet_id = hpet_id;
- parent = irq_remapping_get_ir_irq_domain(&info);
+ info.type = X86_IRQ_ALLOC_TYPE_HPET_GET_PARENT;
+ info.devid = hpet_id;
+ parent = irq_remapping_get_irq_domain(&info);
if (parent == NULL)
parent = x86_vector_domain;
else
init_irq_alloc_info(&info, NULL);
info.type = X86_IRQ_ALLOC_TYPE_HPET;
- info.hpet_data = hc;
- info.hpet_id = hpet_dev_id(domain);
- info.hpet_index = dev_num;
+ info.data = hc;
+ info.devid = hpet_dev_id(domain);
+ info.hwirq = dev_num;
return irq_domain_alloc_irqs(domain, 1, NUMA_NO_NODE, &info);
}
if (apic->setup_apic_routing)
apic->setup_apic_routing();
-
- if (x86_platform.apic_post_init)
- x86_platform.apic_post_init();
}
void __init generic_apic_probe(void)
break;
}
}
-
- if (x86_platform.apic_post_init)
- x86_platform.apic_post_init();
}
int __init default_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
BUG_ON(x86_vector_domain == NULL);
irq_set_default_host(x86_vector_domain);
- arch_init_msi_domain(x86_vector_domain);
-
BUG_ON(!alloc_cpumask_var(&vector_searchmask, GFP_KERNEL));
/*
.name = "APIC",
.irq_ack = apic_ack_edge,
.irq_set_affinity = apic_set_affinity,
+ .irq_compose_msi_msg = x86_vector_msi_compose_msg,
.irq_retrigger = apic_retrigger_irq,
};
*
* SGI UV APIC functions (note: not an Intel compatible APIC)
*
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
* Copyright (C) 2007-2014 Silicon Graphics, Inc. All rights reserved.
*/
#include <linux/crash_dump.h>
static int uv_hubless_system;
static u64 gru_start_paddr, gru_end_paddr;
static union uvh_apicid uvh_apicid;
+static int uv_node_id;
-/* Unpack OEM/TABLE ID's to be NULL terminated strings */
+/* Unpack AT/OEM/TABLE ID's to be NULL terminated strings */
+static u8 uv_archtype[UV_AT_SIZE];
static u8 oem_id[ACPI_OEM_ID_SIZE + 1];
static u8 oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
-/* Information derived from CPUID: */
+/* Information derived from CPUID and some UV MMRs */
static struct {
unsigned int apicid_shift;
unsigned int apicid_mask;
unsigned int socketid_shift; /* aka pnode_shift for UV2/3 */
unsigned int pnode_mask;
+ unsigned int nasid_shift;
unsigned int gpa_shift;
unsigned int gnode_shift;
+ unsigned int m_skt;
+ unsigned int n_skt;
} uv_cpuid;
static int uv_min_hub_revision_id;
static inline bool is_GRU_range(u64 start, u64 end)
{
+ if (!gru_start_paddr)
+ return false;
+
return start >= gru_start_paddr && end <= gru_end_paddr;
}
return is_ISA_range(start, end) || is_GRU_range(start, end);
}
-static int __init early_get_pnodeid(void)
+static void __init early_get_pnodeid(void)
{
- union uvh_node_id_u node_id;
- union uvh_rh_gam_config_mmr_u m_n_config;
int pnode;
- /* Currently, all blades have same revision number */
+ uv_cpuid.m_skt = 0;
+ if (UVH_RH10_GAM_ADDR_MAP_CONFIG) {
+ union uvh_rh10_gam_addr_map_config_u m_n_config;
+
+ m_n_config.v = uv_early_read_mmr(UVH_RH10_GAM_ADDR_MAP_CONFIG);
+ uv_cpuid.n_skt = m_n_config.s.n_skt;
+ uv_cpuid.nasid_shift = 0;
+ } else if (UVH_RH_GAM_ADDR_MAP_CONFIG) {
+ union uvh_rh_gam_addr_map_config_u m_n_config;
+
+ m_n_config.v = uv_early_read_mmr(UVH_RH_GAM_ADDR_MAP_CONFIG);
+ uv_cpuid.n_skt = m_n_config.s.n_skt;
+ if (is_uv(UV3))
+ uv_cpuid.m_skt = m_n_config.s3.m_skt;
+ if (is_uv(UV2))
+ uv_cpuid.m_skt = m_n_config.s2.m_skt;
+ uv_cpuid.nasid_shift = 1;
+ } else {
+ unsigned long GAM_ADDR_MAP_CONFIG = 0;
+
+ WARN(GAM_ADDR_MAP_CONFIG == 0,
+ "UV: WARN: GAM_ADDR_MAP_CONFIG is not available\n");
+ uv_cpuid.n_skt = 0;
+ uv_cpuid.nasid_shift = 0;
+ }
+
+ if (is_uv(UV4|UVY))
+ uv_cpuid.gnode_shift = 2; /* min partition is 4 sockets */
+
+ uv_cpuid.pnode_mask = (1 << uv_cpuid.n_skt) - 1;
+ pnode = (uv_node_id >> uv_cpuid.nasid_shift) & uv_cpuid.pnode_mask;
+ uv_cpuid.gpa_shift = 46; /* Default unless changed */
+
+ pr_info("UV: n_skt:%d pnmsk:%x pn:%x\n",
+ uv_cpuid.n_skt, uv_cpuid.pnode_mask, pnode);
+}
+
+/* Running on a UV Hubbed system, determine which UV Hub Type it is */
+static int __init early_set_hub_type(void)
+{
+ union uvh_node_id_u node_id;
+
+ /*
+ * The NODE_ID MMR is always at offset 0.
+ * Contains the chip part # + revision.
+ * Node_id field started with 15 bits,
+ * ... now 7 but upper 8 are masked to 0.
+ * All blades/nodes have the same part # and hub revision.
+ */
node_id.v = uv_early_read_mmr(UVH_NODE_ID);
- m_n_config.v = uv_early_read_mmr(UVH_RH_GAM_CONFIG_MMR);
- uv_min_hub_revision_id = node_id.s.revision;
+ uv_node_id = node_id.sx.node_id;
switch (node_id.s.part_number) {
- case UV2_HUB_PART_NUMBER:
- case UV2_HUB_PART_NUMBER_X:
- uv_min_hub_revision_id += UV2_HUB_REVISION_BASE - 1;
+
+ case UV5_HUB_PART_NUMBER:
+ uv_min_hub_revision_id = node_id.s.revision
+ + UV5_HUB_REVISION_BASE;
+ uv_hub_type_set(UV5);
break;
+
+ /* UV4/4A only have a revision difference */
+ case UV4_HUB_PART_NUMBER:
+ uv_min_hub_revision_id = node_id.s.revision
+ + UV4_HUB_REVISION_BASE;
+ uv_hub_type_set(UV4);
+ if (uv_min_hub_revision_id == UV4A_HUB_REVISION_BASE)
+ uv_hub_type_set(UV4|UV4A);
+ break;
+
case UV3_HUB_PART_NUMBER:
case UV3_HUB_PART_NUMBER_X:
- uv_min_hub_revision_id += UV3_HUB_REVISION_BASE;
+ uv_min_hub_revision_id = node_id.s.revision
+ + UV3_HUB_REVISION_BASE;
+ uv_hub_type_set(UV3);
break;
- /* Update: UV4A has only a modified revision to indicate HUB fixes */
- case UV4_HUB_PART_NUMBER:
- uv_min_hub_revision_id += UV4_HUB_REVISION_BASE - 1;
- uv_cpuid.gnode_shift = 2; /* min partition is 4 sockets */
+ case UV2_HUB_PART_NUMBER:
+ case UV2_HUB_PART_NUMBER_X:
+ uv_min_hub_revision_id = node_id.s.revision
+ + UV2_HUB_REVISION_BASE - 1;
+ uv_hub_type_set(UV2);
break;
+
+ default:
+ return 0;
}
- uv_hub_info->hub_revision = uv_min_hub_revision_id;
- uv_cpuid.pnode_mask = (1 << m_n_config.s.n_skt) - 1;
- pnode = (node_id.s.node_id >> 1) & uv_cpuid.pnode_mask;
- uv_cpuid.gpa_shift = 46; /* Default unless changed */
+ pr_info("UV: part#:%x rev:%d rev_id:%d UVtype:0x%x\n",
+ node_id.s.part_number, node_id.s.revision,
+ uv_min_hub_revision_id, is_uv(~0));
- pr_info("UV: rev:%d part#:%x nodeid:%04x n_skt:%d pnmsk:%x pn:%x\n",
- node_id.s.revision, node_id.s.part_number, node_id.s.node_id,
- m_n_config.s.n_skt, uv_cpuid.pnode_mask, pnode);
- return pnode;
+ return 1;
}
static void __init uv_tsc_check_sync(void)
int sync_state;
int mmr_shift;
char *state;
- bool valid;
- /* Accommodate different UV arch BIOSes */
+ /* Different returns from different UV BIOS versions */
mmr = uv_early_read_mmr(UVH_TSC_SYNC_MMR);
mmr_shift =
is_uv2_hub() ? UVH_TSC_SYNC_SHIFT_UV2K : UVH_TSC_SYNC_SHIFT;
sync_state = (mmr >> mmr_shift) & UVH_TSC_SYNC_MASK;
+ /* Check if TSC is valid for all sockets */
switch (sync_state) {
case UVH_TSC_SYNC_VALID:
state = "in sync";
- valid = true;
+ mark_tsc_async_resets("UV BIOS");
break;
- case UVH_TSC_SYNC_INVALID:
- state = "unstable";
- valid = false;
+ /* If BIOS state unknown, don't do anything */
+ case UVH_TSC_SYNC_UNKNOWN:
+ state = "unknown";
break;
+
+ /* Otherwise, BIOS indicates problem with TSC */
default:
- state = "unknown: assuming valid";
- valid = true;
+ state = "unstable";
+ mark_tsc_unstable("UV BIOS");
break;
}
pr_info("UV: TSC sync state from BIOS:0%d(%s)\n", sync_state, state);
-
- /* Mark flag that says TSC != 0 is valid for socket 0 */
- if (valid)
- mark_tsc_async_resets("UV BIOS");
- else
- mark_tsc_unstable("UV BIOS");
}
+/* Selector for (4|4A|5) structs */
+#define uvxy_field(sname, field, undef) ( \
+ is_uv(UV4A) ? sname.s4a.field : \
+ is_uv(UV4) ? sname.s4.field : \
+ is_uv(UV3) ? sname.s3.field : \
+ undef)
+
/* [Copied from arch/x86/kernel/cpu/topology.c:detect_extended_topology()] */
#define SMT_LEVEL 0 /* Leaf 0xb SMT level */
strncpy(to, from, len-1);
}
-static int __init uv_acpi_madt_oem_check(char *_oem_id, char *_oem_table_id)
+/* Find UV arch type entry in UVsystab */
+static unsigned long __init early_find_archtype(struct uv_systab *st)
+{
+ int i;
+
+ for (i = 0; st->entry[i].type != UV_SYSTAB_TYPE_UNUSED; i++) {
+ unsigned long ptr = st->entry[i].offset;
+
+ if (!ptr)
+ continue;
+ ptr += (unsigned long)st;
+ if (st->entry[i].type == UV_SYSTAB_TYPE_ARCH_TYPE)
+ return ptr;
+ }
+ return 0;
+}
+
+/* Validate UV arch type field in UVsystab */
+static int __init decode_arch_type(unsigned long ptr)
+{
+ struct uv_arch_type_entry *uv_ate = (struct uv_arch_type_entry *)ptr;
+ int n = strlen(uv_ate->archtype);
+
+ if (n > 0 && n < sizeof(uv_ate->archtype)) {
+ pr_info("UV: UVarchtype received from BIOS\n");
+ uv_stringify(UV_AT_SIZE, uv_archtype, uv_ate->archtype);
+ return 1;
+ }
+ return 0;
+}
+
+/* Determine if UV arch type entry might exist in UVsystab */
+static int __init early_get_arch_type(void)
{
- int pnodeid;
- int uv_apic;
+ unsigned long uvst_physaddr, uvst_size, ptr;
+ struct uv_systab *st;
+ u32 rev;
+ int ret;
+
+ uvst_physaddr = get_uv_systab_phys(0);
+ if (!uvst_physaddr)
+ return 0;
+
+ st = early_memremap_ro(uvst_physaddr, sizeof(struct uv_systab));
+ if (!st) {
+ pr_err("UV: Cannot access UVsystab, remap failed\n");
+ return 0;
+ }
+ rev = st->revision;
+ if (rev < UV_SYSTAB_VERSION_UV5) {
+ early_memunmap(st, sizeof(struct uv_systab));
+ return 0;
+ }
+
+ uvst_size = st->size;
+ early_memunmap(st, sizeof(struct uv_systab));
+ st = early_memremap_ro(uvst_physaddr, uvst_size);
+ if (!st) {
+ pr_err("UV: Cannot access UVarchtype, remap failed\n");
+ return 0;
+ }
+
+ ptr = early_find_archtype(st);
+ if (!ptr) {
+ early_memunmap(st, uvst_size);
+ return 0;
+ }
+
+ ret = decode_arch_type(ptr);
+ early_memunmap(st, uvst_size);
+ return ret;
+}
+
+static int __init uv_set_system_type(char *_oem_id)
+{
+ /* Save OEM_ID passed from ACPI MADT */
uv_stringify(sizeof(oem_id), oem_id, _oem_id);
- uv_stringify(sizeof(oem_table_id), oem_table_id, _oem_table_id);
- if (strncmp(oem_id, "SGI", 3) != 0) {
- if (strncmp(oem_id, "NSGI", 4) != 0)
+ /* Check if BIOS sent us a UVarchtype */
+ if (!early_get_arch_type())
+
+ /* If not use OEM ID for UVarchtype */
+ uv_stringify(UV_AT_SIZE, uv_archtype, _oem_id);
+
+ /* Check if not hubbed */
+ if (strncmp(uv_archtype, "SGI", 3) != 0) {
+
+ /* (Not hubbed), check if not hubless */
+ if (strncmp(uv_archtype, "NSGI", 4) != 0)
+
+ /* (Not hubless), not a UV */
return 0;
- /* UV4 Hubless, CH, (0x11:UV4+Any) */
- if (strncmp(oem_id, "NSGI4", 5) == 0)
+ /* UV4 Hubless: CH */
+ if (strncmp(uv_archtype, "NSGI4", 5) == 0)
uv_hubless_system = 0x11;
- /* UV3 Hubless, UV300/MC990X w/o hub (0x9:UV3+Any) */
+ /* UV3 Hubless: UV300/MC990X w/o hub */
else
uv_hubless_system = 0x9;
- pr_info("UV: OEM IDs %s/%s, HUBLESS(0x%x)\n",
- oem_id, oem_table_id, uv_hubless_system);
-
+ pr_info("UV: OEM IDs %s/%s, SystemType %d, HUBLESS ID %x\n",
+ oem_id, oem_table_id, uv_system_type, uv_hubless_system);
return 0;
}
return 0;
}
- /* Set up early hub type field in uv_hub_info for Node 0 */
- uv_cpu_info->p_uv_hub_info = &uv_hub_info_node0;
+ /* Set hubbed type if true */
+ uv_hub_info->hub_revision =
+ !strncmp(uv_archtype, "SGI5", 4) ? UV5_HUB_REVISION_BASE :
+ !strncmp(uv_archtype, "SGI4", 4) ? UV4_HUB_REVISION_BASE :
+ !strncmp(uv_archtype, "SGI3", 4) ? UV3_HUB_REVISION_BASE :
+ !strcmp(uv_archtype, "SGI2") ? UV2_HUB_REVISION_BASE : 0;
+
+ switch (uv_hub_info->hub_revision) {
+ case UV5_HUB_REVISION_BASE:
+ uv_hubbed_system = 0x21;
+ uv_hub_type_set(UV5);
+ break;
- /*
- * Determine UV arch type.
- * SGI2: UV2000/3000
- * SGI3: UV300 (truncated to 4 chars because of different varieties)
- * SGI4: UV400 (truncated to 4 chars because of different varieties)
- */
- if (!strncmp(oem_id, "SGI4", 4)) {
- uv_hub_info->hub_revision = UV4_HUB_REVISION_BASE;
+ case UV4_HUB_REVISION_BASE:
uv_hubbed_system = 0x11;
+ uv_hub_type_set(UV4);
+ break;
- } else if (!strncmp(oem_id, "SGI3", 4)) {
- uv_hub_info->hub_revision = UV3_HUB_REVISION_BASE;
+ case UV3_HUB_REVISION_BASE:
uv_hubbed_system = 0x9;
+ uv_hub_type_set(UV3);
+ break;
- } else if (!strcmp(oem_id, "SGI2")) {
- uv_hub_info->hub_revision = UV2_HUB_REVISION_BASE;
+ case UV2_HUB_REVISION_BASE:
uv_hubbed_system = 0x5;
+ uv_hub_type_set(UV2);
+ break;
- } else {
- uv_hub_info->hub_revision = 0;
- goto badbios;
+ default:
+ return 0;
}
- pnodeid = early_get_pnodeid();
- early_get_apic_socketid_shift();
+ /* Get UV hub chip part number & revision */
+ early_set_hub_type();
+ /* Other UV setup functions */
+ early_get_pnodeid();
+ early_get_apic_socketid_shift();
x86_platform.is_untracked_pat_range = uv_is_untracked_pat_range;
x86_platform.nmi_init = uv_nmi_init;
+ uv_tsc_check_sync();
+
+ return 1;
+}
+
+/* Called early to probe for the correct APIC driver */
+static int __init uv_acpi_madt_oem_check(char *_oem_id, char *_oem_table_id)
+{
+ /* Set up early hub info fields for Node 0 */
+ uv_cpu_info->p_uv_hub_info = &uv_hub_info_node0;
+
+ /* If not UV, return. */
+ if (likely(uv_set_system_type(_oem_id) == 0))
+ return 0;
+
+ /* Save and Decode OEM Table ID */
+ uv_stringify(sizeof(oem_table_id), oem_table_id, _oem_table_id);
- if (!strcmp(oem_table_id, "UVX")) {
- /* This is the most common hardware variant: */
+ /* This is the most common hardware variant, x2apic mode */
+ if (!strcmp(oem_table_id, "UVX"))
uv_system_type = UV_X2APIC;
- uv_apic = 0;
- } else if (!strcmp(oem_table_id, "UVL")) {
- /* Only used for very small systems: */
+ /* Only used for very small systems, usually 1 chassis, legacy mode */
+ else if (!strcmp(oem_table_id, "UVL"))
uv_system_type = UV_LEGACY_APIC;
- uv_apic = 0;
- } else {
+ else
goto badbios;
- }
- pr_info("UV: OEM IDs %s/%s, System/HUB Types %d/%d, uv_apic %d\n", oem_id, oem_table_id, uv_system_type, uv_min_hub_revision_id, uv_apic);
- uv_tsc_check_sync();
+ pr_info("UV: OEM IDs %s/%s, System/UVType %d/0x%x, HUB RevID %d\n",
+ oem_id, oem_table_id, uv_system_type, is_uv(UV_ANY),
+ uv_min_hub_revision_id);
- return uv_apic;
+ return 0;
badbios:
- pr_err("UV: OEM_ID:%s OEM_TABLE_ID:%s\n", oem_id, oem_table_id);
- pr_err("Current UV Type or BIOS not supported\n");
+ pr_err("UV: UVarchtype:%s not supported\n", uv_archtype);
BUG();
}
};
#define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_LENGTH 3
-#define DEST_SHIFT UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT
+#define DEST_SHIFT UVXH_RH_GAM_ALIAS_0_REDIRECT_CONFIG_DEST_BASE_SHFT
static __init void get_lowmem_redirect(unsigned long *base, unsigned long *size)
{
- union uvh_rh_gam_alias210_overlay_config_2_mmr_u alias;
- union uvh_rh_gam_alias210_redirect_config_2_mmr_u redirect;
+ union uvh_rh_gam_alias_2_overlay_config_u alias;
+ union uvh_rh_gam_alias_2_redirect_config_u redirect;
unsigned long m_redirect;
unsigned long m_overlay;
int i;
for (i = 0; i < UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_LENGTH; i++) {
switch (i) {
case 0:
- m_redirect = UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR;
- m_overlay = UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR;
+ m_redirect = UVH_RH_GAM_ALIAS_0_REDIRECT_CONFIG;
+ m_overlay = UVH_RH_GAM_ALIAS_0_OVERLAY_CONFIG;
break;
case 1:
- m_redirect = UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR;
- m_overlay = UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR;
+ m_redirect = UVH_RH_GAM_ALIAS_1_REDIRECT_CONFIG;
+ m_overlay = UVH_RH_GAM_ALIAS_1_OVERLAY_CONFIG;
break;
case 2:
- m_redirect = UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR;
- m_overlay = UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR;
+ m_redirect = UVH_RH_GAM_ALIAS_2_REDIRECT_CONFIG;
+ m_overlay = UVH_RH_GAM_ALIAS_2_OVERLAY_CONFIG;
break;
}
alias.v = uv_read_local_mmr(m_overlay);
}
enum map_type {map_wb, map_uc};
+static const char * const mt[] = { "WB", "UC" };
static __init void map_high(char *id, unsigned long base, int pshift, int bshift, int max_pnode, enum map_type map_type)
{
pr_info("UV: Map %s_HI base address NULL\n", id);
return;
}
- pr_debug("UV: Map %s_HI 0x%lx - 0x%lx\n", id, paddr, paddr + bytes);
if (map_type == map_uc)
init_extra_mapping_uc(paddr, bytes);
else
init_extra_mapping_wb(paddr, bytes);
+
+ pr_info("UV: Map %s_HI 0x%lx - 0x%lx %s (%d segments)\n",
+ id, paddr, paddr + bytes, mt[map_type], max_pnode + 1);
}
static __init void map_gru_high(int max_pnode)
{
- union uvh_rh_gam_gru_overlay_config_mmr_u gru;
- int shift = UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_SHFT;
- unsigned long mask = UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR_BASE_MASK;
- unsigned long base;
+ union uvh_rh_gam_gru_overlay_config_u gru;
+ unsigned long mask, base;
+ int shift;
+
+ if (UVH_RH_GAM_GRU_OVERLAY_CONFIG) {
+ gru.v = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG);
+ shift = UVH_RH_GAM_GRU_OVERLAY_CONFIG_BASE_SHFT;
+ mask = UVH_RH_GAM_GRU_OVERLAY_CONFIG_BASE_MASK;
+ } else if (UVH_RH10_GAM_GRU_OVERLAY_CONFIG) {
+ gru.v = uv_read_local_mmr(UVH_RH10_GAM_GRU_OVERLAY_CONFIG);
+ shift = UVH_RH10_GAM_GRU_OVERLAY_CONFIG_BASE_SHFT;
+ mask = UVH_RH10_GAM_GRU_OVERLAY_CONFIG_BASE_MASK;
+ } else {
+ pr_err("UV: GRU unavailable (no MMR)\n");
+ return;
+ }
- gru.v = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR);
if (!gru.s.enable) {
- pr_info("UV: GRU disabled\n");
+ pr_info("UV: GRU disabled (by BIOS)\n");
return;
}
static __init void map_mmr_high(int max_pnode)
{
- union uvh_rh_gam_mmr_overlay_config_mmr_u mmr;
- int shift = UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR_BASE_SHFT;
+ unsigned long base;
+ int shift;
+ bool enable;
+
+ if (UVH_RH10_GAM_MMR_OVERLAY_CONFIG) {
+ union uvh_rh10_gam_mmr_overlay_config_u mmr;
+
+ mmr.v = uv_read_local_mmr(UVH_RH10_GAM_MMR_OVERLAY_CONFIG);
+ enable = mmr.s.enable;
+ base = mmr.s.base;
+ shift = UVH_RH10_GAM_MMR_OVERLAY_CONFIG_BASE_SHFT;
+ } else if (UVH_RH_GAM_MMR_OVERLAY_CONFIG) {
+ union uvh_rh_gam_mmr_overlay_config_u mmr;
+
+ mmr.v = uv_read_local_mmr(UVH_RH_GAM_MMR_OVERLAY_CONFIG);
+ enable = mmr.s.enable;
+ base = mmr.s.base;
+ shift = UVH_RH_GAM_MMR_OVERLAY_CONFIG_BASE_SHFT;
+ } else {
+ pr_err("UV:%s:RH_GAM_MMR_OVERLAY_CONFIG MMR undefined?\n",
+ __func__);
+ return;
+ }
- mmr.v = uv_read_local_mmr(UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR);
- if (mmr.s.enable)
- map_high("MMR", mmr.s.base, shift, shift, max_pnode, map_uc);
+ if (enable)
+ map_high("MMR", base, shift, shift, max_pnode, map_uc);
else
pr_info("UV: MMR disabled\n");
}
-/* UV3/4 have identical MMIOH overlay configs, UV4A is slightly different */
-static __init void map_mmioh_high_uv34(int index, int min_pnode, int max_pnode)
-{
- unsigned long overlay;
- unsigned long mmr;
- unsigned long base;
- unsigned long nasid_mask;
- unsigned long m_overlay;
- int i, n, shift, m_io, max_io;
- int nasid, lnasid, fi, li;
- char *id;
-
- if (index == 0) {
- id = "MMIOH0";
- m_overlay = UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR;
- overlay = uv_read_local_mmr(m_overlay);
- base = overlay & UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK;
- mmr = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR;
- m_io = (overlay & UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK)
- >> UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT;
- shift = UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT;
- n = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH;
- nasid_mask = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK;
- } else {
- id = "MMIOH1";
- m_overlay = UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR;
- overlay = uv_read_local_mmr(m_overlay);
- base = overlay & UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK;
- mmr = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR;
- m_io = (overlay & UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK)
- >> UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT;
- shift = UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT;
- n = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH;
- nasid_mask = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK;
+/* Arch specific ENUM cases */
+enum mmioh_arch {
+ UV2_MMIOH = -1,
+ UVY_MMIOH0, UVY_MMIOH1,
+ UVX_MMIOH0, UVX_MMIOH1,
+};
+
+/* Calculate and Map MMIOH Regions */
+static void __init calc_mmioh_map(enum mmioh_arch index,
+ int min_pnode, int max_pnode,
+ int shift, unsigned long base, int m_io, int n_io)
+{
+ unsigned long mmr, nasid_mask;
+ int nasid, min_nasid, max_nasid, lnasid, mapped;
+ int i, fi, li, n, max_io;
+ char id[8];
+
+ /* One (UV2) mapping */
+ if (index == UV2_MMIOH) {
+ strncpy(id, "MMIOH", sizeof(id));
+ max_io = max_pnode;
+ mapped = 0;
+ goto map_exit;
}
- pr_info("UV: %s overlay 0x%lx base:0x%lx m_io:%d\n", id, overlay, base, m_io);
- if (!(overlay & UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK)) {
- pr_info("UV: %s disabled\n", id);
+
+ /* small and large MMIOH mappings */
+ switch (index) {
+ case UVY_MMIOH0:
+ mmr = UVH_RH10_GAM_MMIOH_REDIRECT_CONFIG0;
+ nasid_mask = UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG0_BASE_MASK;
+ n = UVH_RH10_GAM_MMIOH_REDIRECT_CONFIG0_DEPTH;
+ min_nasid = min_pnode;
+ max_nasid = max_pnode;
+ mapped = 1;
+ break;
+ case UVY_MMIOH1:
+ mmr = UVH_RH10_GAM_MMIOH_REDIRECT_CONFIG1;
+ nasid_mask = UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG1_BASE_MASK;
+ n = UVH_RH10_GAM_MMIOH_REDIRECT_CONFIG1_DEPTH;
+ min_nasid = min_pnode;
+ max_nasid = max_pnode;
+ mapped = 1;
+ break;
+ case UVX_MMIOH0:
+ mmr = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0;
+ nasid_mask = UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_BASE_MASK;
+ n = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_DEPTH;
+ min_nasid = min_pnode * 2;
+ max_nasid = max_pnode * 2;
+ mapped = 1;
+ break;
+ case UVX_MMIOH1:
+ mmr = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1;
+ nasid_mask = UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_BASE_MASK;
+ n = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_DEPTH;
+ min_nasid = min_pnode * 2;
+ max_nasid = max_pnode * 2;
+ mapped = 1;
+ break;
+ default:
+ pr_err("UV:%s:Invalid mapping type:%d\n", __func__, index);
return;
}
- /* Convert to NASID: */
- min_pnode *= 2;
- max_pnode *= 2;
- max_io = lnasid = fi = li = -1;
+ /* enum values chosen so (index mod 2) is MMIOH 0/1 (low/high) */
+ snprintf(id, sizeof(id), "MMIOH%d", index%2);
+ max_io = lnasid = fi = li = -1;
for (i = 0; i < n; i++) {
unsigned long m_redirect = mmr + i * 8;
unsigned long redirect = uv_read_local_mmr(m_redirect);
pr_info("UV: %s redirect base 0x%lx(@0x%lx) 0x%04x\n",
id, redirect, m_redirect, nasid);
- /* Invalid NASID: */
- if (nasid < min_pnode || max_pnode < nasid)
+ /* Invalid NASID check */
+ if (nasid < min_nasid || max_nasid < nasid) {
+ pr_err("UV:%s:Invalid NASID:%x (range:%x..%x)\n",
+ __func__, index, min_nasid, max_nasid);
nasid = -1;
+ }
if (nasid == lnasid) {
li = i;
}
addr1 = (base << shift) + f * (1ULL << m_io);
addr2 = (base << shift) + (l + 1) * (1ULL << m_io);
- pr_info("UV: %s[%03d..%03d] NASID 0x%04x ADDR 0x%016lx - 0x%016lx\n", id, fi, li, lnasid, addr1, addr2);
+ pr_info("UV: %s[%03d..%03d] NASID 0x%04x ADDR 0x%016lx - 0x%016lx\n",
+ id, fi, li, lnasid, addr1, addr2);
if (max_io < l)
max_io = l;
}
lnasid = nasid;
}
- pr_info("UV: %s base:0x%lx shift:%d M_IO:%d MAX_IO:%d\n", id, base, shift, m_io, max_io);
+map_exit:
+ pr_info("UV: %s base:0x%lx shift:%d m_io:%d max_io:%d max_pnode:0x%x\n",
+ id, base, shift, m_io, max_io, max_pnode);
- if (max_io >= 0)
+ if (max_io >= 0 && !mapped)
map_high(id, base, shift, m_io, max_io, map_uc);
}
static __init void map_mmioh_high(int min_pnode, int max_pnode)
{
- union uvh_rh_gam_mmioh_overlay_config_mmr_u mmioh;
- unsigned long mmr, base;
- int shift, enable, m_io, n_io;
+ /* UVY flavor */
+ if (UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG0) {
+ union uvh_rh10_gam_mmioh_overlay_config0_u mmioh0;
+ union uvh_rh10_gam_mmioh_overlay_config1_u mmioh1;
+
+ mmioh0.v = uv_read_local_mmr(UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG0);
+ if (unlikely(mmioh0.s.enable == 0))
+ pr_info("UV: MMIOH0 disabled\n");
+ else
+ calc_mmioh_map(UVY_MMIOH0, min_pnode, max_pnode,
+ UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG0_BASE_SHFT,
+ mmioh0.s.base, mmioh0.s.m_io, mmioh0.s.n_io);
- if (is_uv3_hub() || is_uv4_hub()) {
- /* Map both MMIOH regions: */
- map_mmioh_high_uv34(0, min_pnode, max_pnode);
- map_mmioh_high_uv34(1, min_pnode, max_pnode);
+ mmioh1.v = uv_read_local_mmr(UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG1);
+ if (unlikely(mmioh1.s.enable == 0))
+ pr_info("UV: MMIOH1 disabled\n");
+ else
+ calc_mmioh_map(UVY_MMIOH1, min_pnode, max_pnode,
+ UVH_RH10_GAM_MMIOH_OVERLAY_CONFIG1_BASE_SHFT,
+ mmioh1.s.base, mmioh1.s.m_io, mmioh1.s.n_io);
return;
}
+ /* UVX flavor */
+ if (UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0) {
+ union uvh_rh_gam_mmioh_overlay_config0_u mmioh0;
+ union uvh_rh_gam_mmioh_overlay_config1_u mmioh1;
+
+ mmioh0.v = uv_read_local_mmr(UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0);
+ if (unlikely(mmioh0.s.enable == 0))
+ pr_info("UV: MMIOH0 disabled\n");
+ else {
+ unsigned long base = uvxy_field(mmioh0, base, 0);
+ int m_io = uvxy_field(mmioh0, m_io, 0);
+ int n_io = uvxy_field(mmioh0, n_io, 0);
+
+ calc_mmioh_map(UVX_MMIOH0, min_pnode, max_pnode,
+ UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_BASE_SHFT,
+ base, m_io, n_io);
+ }
- if (is_uv2_hub()) {
- mmr = UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR;
- shift = UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR_BASE_SHFT;
- mmioh.v = uv_read_local_mmr(mmr);
- enable = !!mmioh.s2.enable;
- base = mmioh.s2.base;
- m_io = mmioh.s2.m_io;
- n_io = mmioh.s2.n_io;
-
- if (enable) {
- max_pnode &= (1 << n_io) - 1;
- pr_info("UV: base:0x%lx shift:%d N_IO:%d M_IO:%d max_pnode:0x%x\n",
- base, shift, m_io, n_io, max_pnode);
- map_high("MMIOH", base, shift, m_io, max_pnode, map_uc);
- } else {
- pr_info("UV: MMIOH disabled\n");
+ mmioh1.v = uv_read_local_mmr(UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1);
+ if (unlikely(mmioh1.s.enable == 0))
+ pr_info("UV: MMIOH1 disabled\n");
+ else {
+ unsigned long base = uvxy_field(mmioh1, base, 0);
+ int m_io = uvxy_field(mmioh1, m_io, 0);
+ int n_io = uvxy_field(mmioh1, n_io, 0);
+
+ calc_mmioh_map(UVX_MMIOH1, min_pnode, max_pnode,
+ UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_BASE_SHFT,
+ base, m_io, n_io);
}
+ return;
+ }
+
+ /* UV2 flavor */
+ if (UVH_RH_GAM_MMIOH_OVERLAY_CONFIG) {
+ union uvh_rh_gam_mmioh_overlay_config_u mmioh;
+
+ mmioh.v = uv_read_local_mmr(UVH_RH_GAM_MMIOH_OVERLAY_CONFIG);
+ if (unlikely(mmioh.s2.enable == 0))
+ pr_info("UV: MMIOH disabled\n");
+ else
+ calc_mmioh_map(UV2_MMIOH, min_pnode, max_pnode,
+ UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG_BASE_SHFT,
+ mmioh.s2.base, mmioh.s2.m_io, mmioh.s2.n_io);
+ return;
}
}
static __init void map_low_mmrs(void)
{
- init_extra_mapping_uc(UV_GLOBAL_MMR32_BASE, UV_GLOBAL_MMR32_SIZE);
- init_extra_mapping_uc(UV_LOCAL_MMR_BASE, UV_LOCAL_MMR_SIZE);
+ if (UV_GLOBAL_MMR32_BASE)
+ init_extra_mapping_uc(UV_GLOBAL_MMR32_BASE, UV_GLOBAL_MMR32_SIZE);
+
+ if (UV_LOCAL_MMR_BASE)
+ init_extra_mapping_uc(UV_LOCAL_MMR_BASE, UV_LOCAL_MMR_SIZE);
}
static __init void uv_rtc_init(void)
}
}
-/*
- * percpu heartbeat timer
- */
-static void uv_heartbeat(struct timer_list *timer)
-{
- unsigned char bits = uv_scir_info->state;
-
- /* Flip heartbeat bit: */
- bits ^= SCIR_CPU_HEARTBEAT;
-
- /* Is this CPU idle? */
- if (idle_cpu(raw_smp_processor_id()))
- bits &= ~SCIR_CPU_ACTIVITY;
- else
- bits |= SCIR_CPU_ACTIVITY;
-
- /* Update system controller interface reg: */
- uv_set_scir_bits(bits);
-
- /* Enable next timer period: */
- mod_timer(timer, jiffies + SCIR_CPU_HB_INTERVAL);
-}
-
-static int uv_heartbeat_enable(unsigned int cpu)
-{
- while (!uv_cpu_scir_info(cpu)->enabled) {
- struct timer_list *timer = &uv_cpu_scir_info(cpu)->timer;
-
- uv_set_cpu_scir_bits(cpu, SCIR_CPU_HEARTBEAT|SCIR_CPU_ACTIVITY);
- timer_setup(timer, uv_heartbeat, TIMER_PINNED);
- timer->expires = jiffies + SCIR_CPU_HB_INTERVAL;
- add_timer_on(timer, cpu);
- uv_cpu_scir_info(cpu)->enabled = 1;
-
- /* Also ensure that boot CPU is enabled: */
- cpu = 0;
- }
- return 0;
-}
-
-#ifdef CONFIG_HOTPLUG_CPU
-static int uv_heartbeat_disable(unsigned int cpu)
-{
- if (uv_cpu_scir_info(cpu)->enabled) {
- uv_cpu_scir_info(cpu)->enabled = 0;
- del_timer(&uv_cpu_scir_info(cpu)->timer);
- }
- uv_set_cpu_scir_bits(cpu, 0xff);
- return 0;
-}
-
-static __init void uv_scir_register_cpu_notifier(void)
-{
- cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "x86/x2apic-uvx:online",
- uv_heartbeat_enable, uv_heartbeat_disable);
-}
-
-#else /* !CONFIG_HOTPLUG_CPU */
-
-static __init void uv_scir_register_cpu_notifier(void)
-{
-}
-
-static __init int uv_init_heartbeat(void)
-{
- int cpu;
-
- if (is_uv_system()) {
- for_each_online_cpu(cpu)
- uv_heartbeat_enable(cpu);
- }
-
- return 0;
-}
-
-late_initcall(uv_init_heartbeat);
-
-#endif /* !CONFIG_HOTPLUG_CPU */
-
/* Direct Legacy VGA I/O traffic to designated IOH */
static int uv_set_vga_state(struct pci_dev *pdev, bool decode, unsigned int command_bits, u32 flags)
{
unsigned char n_lshift;
};
+/* Initialize caller's MN struct and fill in values */
static void get_mn(struct mn *mnp)
{
- union uvh_rh_gam_config_mmr_u m_n_config;
- union uv3h_gr0_gam_gr_config_u m_gr_config;
-
- /* Make sure the whole structure is well initialized: */
memset(mnp, 0, sizeof(*mnp));
-
- m_n_config.v = uv_read_local_mmr(UVH_RH_GAM_CONFIG_MMR);
- mnp->n_val = m_n_config.s.n_skt;
-
- if (is_uv4_hub()) {
+ mnp->n_val = uv_cpuid.n_skt;
+ if (is_uv(UV4|UVY)) {
mnp->m_val = 0;
mnp->n_lshift = 0;
} else if (is_uv3_hub()) {
- mnp->m_val = m_n_config.s3.m_skt;
- m_gr_config.v = uv_read_local_mmr(UV3H_GR0_GAM_GR_CONFIG);
+ union uvyh_gr0_gam_gr_config_u m_gr_config;
+
+ mnp->m_val = uv_cpuid.m_skt;
+ m_gr_config.v = uv_read_local_mmr(UVH_GR0_GAM_GR_CONFIG);
mnp->n_lshift = m_gr_config.s3.m_skt;
} else if (is_uv2_hub()) {
- mnp->m_val = m_n_config.s2.m_skt;
+ mnp->m_val = uv_cpuid.m_skt;
mnp->n_lshift = mnp->m_val == 40 ? 40 : 39;
}
mnp->m_shift = mnp->m_val ? 64 - mnp->m_val : 0;
static void __init uv_init_hub_info(struct uv_hub_info_s *hi)
{
- union uvh_node_id_u node_id;
struct mn mn;
get_mn(&mn);
hi->m_shift = mn.m_shift;
hi->n_lshift = mn.n_lshift ? mn.n_lshift : 0;
hi->hub_revision = uv_hub_info->hub_revision;
+ hi->hub_type = uv_hub_info->hub_type;
hi->pnode_mask = uv_cpuid.pnode_mask;
+ hi->nasid_shift = uv_cpuid.nasid_shift;
hi->min_pnode = _min_pnode;
hi->min_socket = _min_socket;
hi->pnode_to_socket = _pnode_to_socket;
hi->gr_table_len = _gr_table_len;
hi->gr_table = _gr_table;
- node_id.v = uv_read_local_mmr(UVH_NODE_ID);
uv_cpuid.gnode_shift = max_t(unsigned int, uv_cpuid.gnode_shift, mn.n_val);
- hi->gnode_extra = (node_id.s.node_id & ~((1 << uv_cpuid.gnode_shift) - 1)) >> 1;
+ hi->gnode_extra = (uv_node_id & ~((1 << uv_cpuid.gnode_shift) - 1)) >> 1;
if (mn.m_val)
hi->gnode_upper = (u64)hi->gnode_extra << mn.m_val;
hi->gpa_shift = uv_gp_table->gpa_shift;
hi->gpa_mask = (1UL << hi->gpa_shift) - 1;
} else {
- hi->global_mmr_base = uv_read_local_mmr(UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR) & ~UV_MMR_ENABLE;
+ hi->global_mmr_base =
+ uv_read_local_mmr(UVH_RH_GAM_MMR_OVERLAY_CONFIG) &
+ ~UV_MMR_ENABLE;
hi->global_mmr_shift = _UV_GLOBAL_MMR64_PNODE_SHIFT;
}
/* Show system specific info: */
pr_info("UV: N:%d M:%d m_shift:%d n_lshift:%d\n", hi->n_val, hi->m_val, hi->m_shift, hi->n_lshift);
pr_info("UV: gpa_mask/shift:0x%lx/%d pnode_mask:0x%x apic_pns:%d\n", hi->gpa_mask, hi->gpa_shift, hi->pnode_mask, hi->apic_pnode_shift);
- pr_info("UV: mmr_base/shift:0x%lx/%ld gru_base/shift:0x%lx/%ld\n", hi->global_mmr_base, hi->global_mmr_shift, hi->global_gru_base, hi->global_gru_shift);
+ pr_info("UV: mmr_base/shift:0x%lx/%ld\n", hi->global_mmr_base, hi->global_mmr_shift);
+ if (hi->global_gru_base)
+ pr_info("UV: gru_base/shift:0x%lx/%ld\n",
+ hi->global_gru_base, hi->global_gru_shift);
+
pr_info("UV: gnode_upper:0x%lx gnode_extra:0x%x\n", hi->gnode_upper, hi->gnode_extra);
}
pr_info("UV: GRT: %d entries, sockets(min:%x,max:%x) pnodes(min:%x,max:%x)\n", index, _min_socket, _max_socket, _min_pnode, _max_pnode);
}
+/* Walk through UVsystab decoding the fields */
static int __init decode_uv_systab(void)
{
struct uv_systab *st;
int i;
- /* If system is uv3 or lower, there is no extended UVsystab */
- if (is_uv_hubbed(0xfffffe) < uv(4) && is_uv_hubless(0xfffffe) < uv(4))
- return 0; /* No extended UVsystab required */
-
+ /* Get mapped UVsystab pointer */
st = uv_systab;
+
+ /* If UVsystab is version 1, there is no extended UVsystab */
+ if (st && st->revision == UV_SYSTAB_VERSION_1)
+ return 0;
+
if ((!st) || (st->revision < UV_SYSTAB_VERSION_UV4_LATEST)) {
int rev = st ? st->revision : 0;
- pr_err("UV: BIOS UVsystab version(%x) mismatch, expecting(%x)\n", rev, UV_SYSTAB_VERSION_UV4_LATEST);
- pr_err("UV: Cannot support UV operations, switching to generic PC\n");
+ pr_err("UV: BIOS UVsystab mismatch, (%x < %x)\n",
+ rev, UV_SYSTAB_VERSION_UV4_LATEST);
+ pr_err("UV: Does not support UV, switch to non-UV x86_64\n");
uv_system_type = UV_NONE;
return -EINVAL;
if (!ptr)
continue;
- ptr = ptr + (unsigned long)st;
+ /* point to payload */
+ ptr += (unsigned long)st;
switch (st->entry[i].type) {
case UV_SYSTAB_TYPE_GAM_PARAMS:
case UV_SYSTAB_TYPE_GAM_RNG_TBL:
decode_gam_rng_tbl(ptr);
break;
+
+ case UV_SYSTAB_TYPE_ARCH_TYPE:
+ /* already processed in early startup */
+ break;
+
+ default:
+ pr_err("UV:%s:Unrecognized UV_SYSTAB_TYPE:%d, skipped\n",
+ __func__, st->entry[i].type);
+ break;
}
}
return 0;
}
-/*
- * Set up physical blade translations from UVH_NODE_PRESENT_TABLE
- * .. NB: UVH_NODE_PRESENT_TABLE is going away,
- * .. being replaced by GAM Range Table
- */
+/* Set up physical blade translations from UVH_NODE_PRESENT_TABLE */
static __init void boot_init_possible_blades(struct uv_hub_info_s *hub_info)
{
+ unsigned long np;
int i, uv_pb = 0;
- pr_info("UV: NODE_PRESENT_DEPTH = %d\n", UVH_NODE_PRESENT_TABLE_DEPTH);
- for (i = 0; i < UVH_NODE_PRESENT_TABLE_DEPTH; i++) {
- unsigned long np;
-
- np = uv_read_local_mmr(UVH_NODE_PRESENT_TABLE + i * 8);
- if (np)
+ if (UVH_NODE_PRESENT_TABLE) {
+ pr_info("UV: NODE_PRESENT_DEPTH = %d\n",
+ UVH_NODE_PRESENT_TABLE_DEPTH);
+ for (i = 0; i < UVH_NODE_PRESENT_TABLE_DEPTH; i++) {
+ np = uv_read_local_mmr(UVH_NODE_PRESENT_TABLE + i * 8);
pr_info("UV: NODE_PRESENT(%d) = 0x%016lx\n", i, np);
-
+ uv_pb += hweight64(np);
+ }
+ }
+ if (UVH_NODE_PRESENT_0) {
+ np = uv_read_local_mmr(UVH_NODE_PRESENT_0);
+ pr_info("UV: NODE_PRESENT_0 = 0x%016lx\n", np);
+ uv_pb += hweight64(np);
+ }
+ if (UVH_NODE_PRESENT_1) {
+ np = uv_read_local_mmr(UVH_NODE_PRESENT_1);
+ pr_info("UV: NODE_PRESENT_1 = 0x%016lx\n", np);
uv_pb += hweight64(np);
}
if (uv_possible_blades != uv_pb)
uv_possible_blades = uv_pb;
+
+ pr_info("UV: number nodes/possible blades %d\n", uv_pb);
}
static void __init build_socket_tables(void)
pr_info("UV: No UVsystab socket table, ignoring\n");
return;
}
- pr_crit("UV: Error: UVsystab address translations not available!\n");
+ pr_err("UV: Error: UVsystab address translations not available!\n");
BUG();
}
return 0;
}
-static int __maybe_unused proc_oemid_show(struct seq_file *file, void *data)
+static int __maybe_unused proc_archtype_show(struct seq_file *file, void *data)
{
- seq_printf(file, "%s/%s\n", oem_id, oem_table_id);
+ seq_printf(file, "%s/%s\n", uv_archtype, oem_table_id);
return 0;
}
struct proc_dir_entry *pde;
pde = proc_mkdir(UV_PROC_NODE, NULL);
- proc_create_single("oemid", 0, pde, proc_oemid_show);
+ proc_create_single("archtype", 0, pde, proc_archtype_show);
if (hubless)
proc_create_single("hubless", 0, pde, proc_hubless_show);
else
struct uv_hub_info_s hub_info = {0};
int bytes, cpu, nodeid;
unsigned short min_pnode = 9999, max_pnode = 0;
- char *hub = is_uv4_hub() ? "UV400" :
+ char *hub = is_uv5_hub() ? "UV500" :
+ is_uv4_hub() ? "UV400" :
is_uv3_hub() ? "UV300" :
is_uv2_hub() ? "UV2000/3000" : NULL;
map_low_mmrs();
- /* Get uv_systab for decoding: */
+ /* Get uv_systab for decoding, setup UV BIOS calls */
uv_bios_init();
/* If there's an UVsystab problem then abort UV init: */
- if (decode_uv_systab() < 0)
+ if (decode_uv_systab() < 0) {
+ pr_err("UV: Mangled UVsystab format\n");
return;
+ }
build_socket_tables();
build_uv_gr_table();
uv_hub_info_list(numa_node_id)->pnode = pnode;
else if (uv_cpu_hub_info(cpu)->pnode == 0xffff)
uv_cpu_hub_info(cpu)->pnode = pnode;
-
- uv_cpu_scir_info(cpu)->offset = uv_scir_offset(apicid);
}
for_each_node(nodeid) {
uv_nmi_setup();
uv_cpu_init();
- uv_scir_register_cpu_notifier();
uv_setup_proc_files(0);
/* Register Legacy VGA I/O redirection handler: */
c->x86_cache_alignment = c->x86_clflush_size * 2;
set_cpu_cap(c, X86_FEATURE_REP_GOOD);
}
+
+ if (c->x86 >= 7)
+ set_cpu_cap(c, X86_FEATURE_REP_GOOD);
}
enum {
static void early_init_centaur(struct cpuinfo_x86 *c)
{
- switch (c->x86) {
#ifdef CONFIG_X86_32
- case 5:
- /* Emulate MTRRs using Centaur's MCR. */
+ /* Emulate MTRRs using Centaur's MCR. */
+ if (c->x86 == 5)
set_cpu_cap(c, X86_FEATURE_CENTAUR_MCR);
- break;
#endif
- case 6:
- if (c->x86_model >= 0xf)
- set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
- break;
- }
+ if ((c->x86 == 6 && c->x86_model >= 0xf) ||
+ (c->x86 >= 7))
+ set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
+
#ifdef CONFIG_X86_64
set_cpu_cap(c, X86_FEATURE_SYSENTER32);
#endif
set_cpu_cap(c, X86_FEATURE_ARCH_PERFMON);
}
- switch (c->x86) {
#ifdef CONFIG_X86_32
- case 5:
+ if (c->x86 == 5) {
switch (c->x86_model) {
case 4:
name = "C6";
c->x86_cache_size = (cc>>24)+(dd>>24);
}
sprintf(c->x86_model_id, "WinChip %s", name);
- break;
+ }
#endif
- case 6:
+ if (c->x86 == 6 || c->x86 >= 7)
init_c3(c);
- break;
- }
#ifdef CONFIG_X86_64
set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
#endif
#include <linux/syscore_ops.h>
#include <linux/pgtable.h>
+#include <asm/cmdline.h>
#include <asm/stackprotector.h>
#include <asm/perf_event.h>
#include <asm/mmu_context.h>
#endif
}
+/*
+ * We parse cpu parameters early because fpu__init_system() is executed
+ * before parse_early_param().
+ */
+static void __init cpu_parse_early_param(void)
+{
+ char arg[128];
+ char *argptr = arg;
+ int arglen, res, bit;
+
+#ifdef CONFIG_X86_32
+ if (cmdline_find_option_bool(boot_command_line, "no387"))
+#ifdef CONFIG_MATH_EMULATION
+ setup_clear_cpu_cap(X86_FEATURE_FPU);
+#else
+ pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
+#endif
+
+ if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
+ setup_clear_cpu_cap(X86_FEATURE_FXSR);
+#endif
+
+ if (cmdline_find_option_bool(boot_command_line, "noxsave"))
+ setup_clear_cpu_cap(X86_FEATURE_XSAVE);
+
+ if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
+ setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
+
+ if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
+ setup_clear_cpu_cap(X86_FEATURE_XSAVES);
+
+ arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
+ if (arglen <= 0)
+ return;
+
+ pr_info("Clearing CPUID bits:");
+ do {
+ res = get_option(&argptr, &bit);
+ if (res == 0 || res == 3)
+ break;
+
+ /* If the argument was too long, the last bit may be cut off */
+ if (res == 1 && arglen >= sizeof(arg))
+ break;
+
+ if (bit >= 0 && bit < NCAPINTS * 32) {
+ pr_cont(" " X86_CAP_FMT, x86_cap_flag(bit));
+ setup_clear_cpu_cap(bit);
+ }
+ } while (res == 2);
+ pr_cont("\n");
+}
+
/*
* Do minimum CPU detection early.
* Fields really needed: vendor, cpuid_level, family, model, mask,
get_cpu_cap(c);
get_cpu_address_sizes(c);
setup_force_cpu_cap(X86_FEATURE_CPUID);
+ cpu_parse_early_param();
if (this_cpu->c_early_init)
this_cpu->c_early_init(c);
* ESPFIX issue, we can change this.
*/
#ifdef CONFIG_X86_32
-# ifdef CONFIG_PARAVIRT_XXL
- do {
- extern void native_iret(void);
- if (pv_ops.cpu.iret == native_iret)
- set_cpu_bug(c, X86_BUG_ESPFIX);
- } while (0);
-# else
set_cpu_bug(c, X86_BUG_ESPFIX);
-# endif
#endif
}
{ X86_FEATURE_CQM_MBM_TOTAL, X86_FEATURE_CQM_LLC },
{ X86_FEATURE_CQM_MBM_LOCAL, X86_FEATURE_CQM_LLC },
{ X86_FEATURE_AVX512_BF16, X86_FEATURE_AVX512VL },
+ { X86_FEATURE_ENQCMD, X86_FEATURE_XSAVES },
+ { X86_FEATURE_PER_THREAD_MBA, X86_FEATURE_MBA },
{}
};
}
static struct smca_hwid smca_hwid_mcatypes[] = {
- /* { bank_type, hwid_mcatype, xec_bitmap } */
+ /* { bank_type, hwid_mcatype } */
/* Reserved type */
- { SMCA_RESERVED, HWID_MCATYPE(0x00, 0x0), 0x0 },
+ { SMCA_RESERVED, HWID_MCATYPE(0x00, 0x0) },
/* ZN Core (HWID=0xB0) MCA types */
- { SMCA_LS, HWID_MCATYPE(0xB0, 0x0), 0x1FFFFF },
- { SMCA_LS_V2, HWID_MCATYPE(0xB0, 0x10), 0xFFFFFF },
- { SMCA_IF, HWID_MCATYPE(0xB0, 0x1), 0x3FFF },
- { SMCA_L2_CACHE, HWID_MCATYPE(0xB0, 0x2), 0xF },
- { SMCA_DE, HWID_MCATYPE(0xB0, 0x3), 0x1FF },
+ { SMCA_LS, HWID_MCATYPE(0xB0, 0x0) },
+ { SMCA_LS_V2, HWID_MCATYPE(0xB0, 0x10) },
+ { SMCA_IF, HWID_MCATYPE(0xB0, 0x1) },
+ { SMCA_L2_CACHE, HWID_MCATYPE(0xB0, 0x2) },
+ { SMCA_DE, HWID_MCATYPE(0xB0, 0x3) },
/* HWID 0xB0 MCATYPE 0x4 is Reserved */
- { SMCA_EX, HWID_MCATYPE(0xB0, 0x5), 0xFFF },
- { SMCA_FP, HWID_MCATYPE(0xB0, 0x6), 0x7F },
- { SMCA_L3_CACHE, HWID_MCATYPE(0xB0, 0x7), 0xFF },
+ { SMCA_EX, HWID_MCATYPE(0xB0, 0x5) },
+ { SMCA_FP, HWID_MCATYPE(0xB0, 0x6) },
+ { SMCA_L3_CACHE, HWID_MCATYPE(0xB0, 0x7) },
/* Data Fabric MCA types */
- { SMCA_CS, HWID_MCATYPE(0x2E, 0x0), 0x1FF },
- { SMCA_PIE, HWID_MCATYPE(0x2E, 0x1), 0x1F },
- { SMCA_CS_V2, HWID_MCATYPE(0x2E, 0x2), 0x3FFF },
+ { SMCA_CS, HWID_MCATYPE(0x2E, 0x0) },
+ { SMCA_PIE, HWID_MCATYPE(0x2E, 0x1) },
+ { SMCA_CS_V2, HWID_MCATYPE(0x2E, 0x2) },
/* Unified Memory Controller MCA type */
- { SMCA_UMC, HWID_MCATYPE(0x96, 0x0), 0xFF },
+ { SMCA_UMC, HWID_MCATYPE(0x96, 0x0) },
/* Parameter Block MCA type */
- { SMCA_PB, HWID_MCATYPE(0x05, 0x0), 0x1 },
+ { SMCA_PB, HWID_MCATYPE(0x05, 0x0) },
/* Platform Security Processor MCA type */
- { SMCA_PSP, HWID_MCATYPE(0xFF, 0x0), 0x1 },
- { SMCA_PSP_V2, HWID_MCATYPE(0xFF, 0x1), 0x3FFFF },
+ { SMCA_PSP, HWID_MCATYPE(0xFF, 0x0) },
+ { SMCA_PSP_V2, HWID_MCATYPE(0xFF, 0x1) },
/* System Management Unit MCA type */
- { SMCA_SMU, HWID_MCATYPE(0x01, 0x0), 0x1 },
- { SMCA_SMU_V2, HWID_MCATYPE(0x01, 0x1), 0x7FF },
+ { SMCA_SMU, HWID_MCATYPE(0x01, 0x0) },
+ { SMCA_SMU_V2, HWID_MCATYPE(0x01, 0x1) },
/* Microprocessor 5 Unit MCA type */
- { SMCA_MP5, HWID_MCATYPE(0x01, 0x2), 0x3FF },
+ { SMCA_MP5, HWID_MCATYPE(0x01, 0x2) },
/* Northbridge IO Unit MCA type */
- { SMCA_NBIO, HWID_MCATYPE(0x18, 0x0), 0x1F },
+ { SMCA_NBIO, HWID_MCATYPE(0x18, 0x0) },
/* PCI Express Unit MCA type */
- { SMCA_PCIE, HWID_MCATYPE(0x46, 0x0), 0x1F },
+ { SMCA_PCIE, HWID_MCATYPE(0x46, 0x0) },
};
struct smca_bank smca_banks[MAX_NR_BANKS];
#include <linux/debugfs.h>
#include <linux/irq_work.h>
#include <linux/export.h>
-#include <linux/jump_label.h>
#include <linux/set_memory.h>
#include <linux/sync_core.h>
#include <linux/task_work.h>
return -1;
}
+__visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup,
+ struct pt_regs *regs, int trapnr,
+ unsigned long error_code,
+ unsigned long fault_addr)
+{
+ pr_emerg("MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n",
+ (unsigned int)regs->cx, regs->ip, (void *)regs->ip);
+
+ show_stack_regs(regs);
+
+ panic("MCA architectural violation!\n");
+
+ while (true)
+ cpu_relax();
+
+ return true;
+}
+
/* MSR access wrappers used for error injection */
-static u64 mce_rdmsrl(u32 msr)
+static noinstr u64 mce_rdmsrl(u32 msr)
{
- u64 v;
+ DECLARE_ARGS(val, low, high);
if (__this_cpu_read(injectm.finished)) {
- int offset = msr_to_offset(msr);
+ int offset;
+ u64 ret;
+
+ instrumentation_begin();
+ offset = msr_to_offset(msr);
if (offset < 0)
- return 0;
- return *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
- }
+ ret = 0;
+ else
+ ret = *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
- if (rdmsrl_safe(msr, &v)) {
- WARN_ONCE(1, "mce: Unable to read MSR 0x%x!\n", msr);
- /*
- * Return zero in case the access faulted. This should
- * not happen normally but can happen if the CPU does
- * something weird, or if the code is buggy.
- */
- v = 0;
+ instrumentation_end();
+
+ return ret;
}
- return v;
+ /*
+ * RDMSR on MCA MSRs should not fault. If they do, this is very much an
+ * architectural violation and needs to be reported to hw vendor. Panic
+ * the box to not allow any further progress.
+ */
+ asm volatile("1: rdmsr\n"
+ "2:\n"
+ _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_rdmsr_fault)
+ : EAX_EDX_RET(val, low, high) : "c" (msr));
+
+
+ return EAX_EDX_VAL(val, low, high);
+}
+
+__visible bool ex_handler_wrmsr_fault(const struct exception_table_entry *fixup,
+ struct pt_regs *regs, int trapnr,
+ unsigned long error_code,
+ unsigned long fault_addr)
+{
+ pr_emerg("MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n",
+ (unsigned int)regs->cx, (unsigned int)regs->dx, (unsigned int)regs->ax,
+ regs->ip, (void *)regs->ip);
+
+ show_stack_regs(regs);
+
+ panic("MCA architectural violation!\n");
+
+ while (true)
+ cpu_relax();
+
+ return true;
}
-static void mce_wrmsrl(u32 msr, u64 v)
+static noinstr void mce_wrmsrl(u32 msr, u64 v)
{
+ u32 low, high;
+
if (__this_cpu_read(injectm.finished)) {
- int offset = msr_to_offset(msr);
+ int offset;
+
+ instrumentation_begin();
+ offset = msr_to_offset(msr);
if (offset >= 0)
*(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
+
+ instrumentation_end();
+
return;
}
- wrmsrl(msr, v);
+
+ low = (u32)v;
+ high = (u32)(v >> 32);
+
+ /* See comment in mce_rdmsrl() */
+ asm volatile("1: wrmsr\n"
+ "2:\n"
+ _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_wrmsr_fault)
+ : : "c" (msr), "a"(low), "d" (high) : "memory");
}
/*
goto clear_it;
mce_read_aux(&m, i);
- m.severity = mce_severity(&m, mca_cfg.tolerant, NULL, false);
+ m.severity = mce_severity(&m, NULL, mca_cfg.tolerant, NULL, false);
/*
* Don't get the IP here because it's unlikely to
* have anything to do with the actual error location.
quirk_no_way_out(i, m, regs);
m->bank = i;
- if (mce_severity(m, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) {
+ if (mce_severity(m, regs, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) {
mce_read_aux(m, i);
*msg = tmp;
return 1;
struct mce *m = NULL;
int global_worst = 0;
char *msg = NULL;
- char *nmsg = NULL;
/*
* This CPU is the Monarch and the other CPUs have run
* Grade the severity of the errors of all the CPUs.
*/
for_each_possible_cpu(cpu) {
- int severity = mce_severity(&per_cpu(mces_seen, cpu),
- mca_cfg.tolerant,
- &nmsg, true);
- if (severity > global_worst) {
- msg = nmsg;
- global_worst = severity;
+ struct mce *mtmp = &per_cpu(mces_seen, cpu);
+
+ if (mtmp->severity > global_worst) {
+ global_worst = mtmp->severity;
m = &per_cpu(mces_seen, cpu);
}
}
* This dumps all the mces in the log buffer and stops the
* other CPUs.
*/
- if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
+ if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) {
+ /* call mce_severity() to get "msg" for panic */
+ mce_severity(m, NULL, mca_cfg.tolerant, &msg, true);
mce_panic("Fatal machine check", m, msg);
+ }
/*
* For UC somewhere we let the CPU who detects it handle it.
return false;
}
-static void __mc_scan_banks(struct mce *m, struct mce *final,
+static void __mc_scan_banks(struct mce *m, struct pt_regs *regs, struct mce *final,
unsigned long *toclear, unsigned long *valid_banks,
int no_way_out, int *worst)
{
/* Set taint even when machine check was not enabled. */
add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
- severity = mce_severity(m, cfg->tolerant, NULL, true);
+ severity = mce_severity(m, regs, cfg->tolerant, NULL, true);
/*
* When machine check was for corrected/deferred handler don't
if (!p->mce_ripv)
flags |= MF_MUST_KILL;
- if (!memory_failure(p->mce_addr >> PAGE_SHIFT, flags)) {
+ if (!memory_failure(p->mce_addr >> PAGE_SHIFT, flags) &&
+ !(p->mce_kflags & MCE_IN_KERNEL_COPYIN)) {
set_mce_nospec(p->mce_addr >> PAGE_SHIFT, p->mce_whole_page);
+ sync_core();
return;
}
- pr_err("Memory error not recovered");
- kill_me_now(cb);
+ if (p->mce_vaddr != (void __user *)-1l) {
+ force_sig_mceerr(BUS_MCEERR_AR, p->mce_vaddr, PAGE_SHIFT);
+ } else {
+ pr_err("Memory error not recovered");
+ kill_me_now(cb);
+ }
+}
+
+static void queue_task_work(struct mce *m, int kill_it)
+{
+ current->mce_addr = m->addr;
+ current->mce_kflags = m->kflags;
+ current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV);
+ current->mce_whole_page = whole_page(m);
+
+ if (kill_it)
+ current->mce_kill_me.func = kill_me_now;
+ else
+ current->mce_kill_me.func = kill_me_maybe;
+
+ task_work_add(current, ¤t->mce_kill_me, true);
}
/*
order = mce_start(&no_way_out);
}
- __mc_scan_banks(&m, final, toclear, valid_banks, no_way_out, &worst);
+ __mc_scan_banks(&m, regs, final, toclear, valid_banks, no_way_out, &worst);
if (!no_way_out)
mce_clear_state(toclear);
* make sure we have the right "msg".
*/
if (worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) {
- mce_severity(&m, cfg->tolerant, &msg, true);
+ mce_severity(&m, regs, cfg->tolerant, &msg, true);
mce_panic("Local fatal machine check!", &m, msg);
}
}
if (worst > 0)
irq_work_queue(&mce_irq_work);
- mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
-
- sync_core();
-
if (worst != MCE_AR_SEVERITY && !kill_it)
- return;
+ goto out;
/* Fault was in user mode and we need to take some action */
if ((m.cs & 3) == 3) {
/* If this triggers there is no way to recover. Die hard. */
BUG_ON(!on_thread_stack() || !user_mode(regs));
- current->mce_addr = m.addr;
- current->mce_ripv = !!(m.mcgstatus & MCG_STATUS_RIPV);
- current->mce_whole_page = whole_page(&m);
- current->mce_kill_me.func = kill_me_maybe;
- if (kill_it)
- current->mce_kill_me.func = kill_me_now;
- task_work_add(current, ¤t->mce_kill_me, true);
+ queue_task_work(&m, kill_it);
+
} else {
/*
* Handle an MCE which has happened in kernel space but from
if (!fixup_exception(regs, X86_TRAP_MC, 0, 0))
mce_panic("Failed kernel mode recovery", &m, msg);
}
+
+ if (m.kflags & MCE_IN_KERNEL_COPYIN)
+ queue_task_work(&m, kill_it);
}
+out:
+ mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
}
EXPORT_SYMBOL_GPL(do_machine_check);
static __always_inline void exc_machine_check_kernel(struct pt_regs *regs)
{
+ bool irq_state;
+
WARN_ON_ONCE(user_mode(regs));
/*
mce_check_crashing_cpu())
return;
- nmi_enter();
+ irq_state = idtentry_enter_nmi(regs);
/*
* The call targets are marked noinstr, but objtool can't figure
* that out because it's an indirect call. Annotate it.
if (regs->flags & X86_EFLAGS_IF)
trace_hardirqs_on_prepare();
instrumentation_end();
- nmi_exit();
+ idtentry_exit_nmi(regs, irq_state);
}
static __always_inline void exc_machine_check_user(struct pt_regs *regs)
and older.
* mce=nobootlog Don't log MCEs from before booting.
* mce=bios_cmci_threshold Don't program the CMCI threshold
- * mce=recovery force enable memcpy_mcsafe()
+ * mce=recovery force enable copy_mc_fragile()
*/
static int __init mcheck_enable(char *str)
{
static void __init mcheck_debugfs_init(void) { }
#endif
-DEFINE_STATIC_KEY_FALSE(mcsafe_key);
-EXPORT_SYMBOL_GPL(mcsafe_key);
-
static int __init mcheck_late_init(void)
{
if (mca_cfg.recovery)
- static_branch_inc(&mcsafe_key);
+ enable_copy_mc_fragile();
mcheck_debugfs_init();
unlock:
mutex_unlock(&mce_chrdev_read_mutex);
- mce->kflags |= MCE_HANDLED_MCELOG;
+ if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
+ mce->kflags |= MCE_HANDLED_MCELOG;
+
return NOTIFY_OK;
}
int mce_gen_pool_init(void);
struct llist_node *mce_gen_pool_prepare_records(void);
-extern int (*mce_severity)(struct mce *a, int tolerant, char **msg, bool is_excp);
+extern int (*mce_severity)(struct mce *a, struct pt_regs *regs,
+ int tolerant, char **msg, bool is_excp);
struct dentry *mce_get_debugfs_dir(void);
extern mce_banks_t mce_banks_ce_disabled;
static inline bool amd_filter_mce(struct mce *m) { return false; };
#endif
+__visible bool ex_handler_rdmsr_fault(const struct exception_table_entry *fixup,
+ struct pt_regs *regs, int trapnr,
+ unsigned long error_code,
+ unsigned long fault_addr);
+
+__visible bool ex_handler_wrmsr_fault(const struct exception_table_entry *fixup,
+ struct pt_regs *regs, int trapnr,
+ unsigned long error_code,
+ unsigned long fault_addr);
+
#endif /* __X86_MCE_INTERNAL_H__ */
#include <linux/seq_file.h>
#include <linux/init.h>
#include <linux/debugfs.h>
-#include <asm/mce.h>
#include <linux/uaccess.h>
+#include <asm/mce.h>
+#include <asm/intel-family.h>
+#include <asm/traps.h>
+#include <asm/insn.h>
+#include <asm/insn-eval.h>
+
#include "internal.h"
/*
unsigned char context;
unsigned char excp;
unsigned char covered;
+ unsigned char cpu_model;
+ unsigned char cpu_minstepping;
+ unsigned char bank_lo, bank_hi;
char *msg;
} severities[] = {
#define MCESEV(s, m, c...) { .sev = MCE_ ## s ## _SEVERITY, .msg = m, ## c }
+#define BANK_RANGE(l, h) .bank_lo = l, .bank_hi = h
+#define MODEL_STEPPING(m, s) .cpu_model = m, .cpu_minstepping = s
#define KERNEL .context = IN_KERNEL
#define USER .context = IN_USER
#define KERNEL_RECOV .context = IN_KERNEL_RECOV
PANIC, "In kernel and no restart IP",
EXCP, KERNEL_RECOV, MCGMASK(MCG_STATUS_RIPV, 0)
),
- MCESEV(
- DEFERRED, "Deferred error",
- NOSER, MASK(MCI_STATUS_UC|MCI_STATUS_DEFERRED|MCI_STATUS_POISON, MCI_STATUS_DEFERRED)
- ),
MCESEV(
KEEP, "Corrected error",
NOSER, BITCLR(MCI_STATUS_UC)
),
-
/*
* known AO MCACODs reported via MCE or CMC:
*
AO, "Action optional: last level cache writeback error",
SER, MASK(MCI_UC_AR|MCACOD, MCI_STATUS_UC|MCACOD_L3WB)
),
+ /*
+ * Quirk for Skylake/Cascade Lake. Patrol scrubber may be configured
+ * to report uncorrected errors using CMCI with a special signature.
+ * UC=0, MSCOD=0x0010, MCACOD=binary(000X 0000 1100 XXXX) reported
+ * in one of the memory controller banks.
+ * Set severity to "AO" for same action as normal patrol scrub error.
+ */
+ MCESEV(
+ AO, "Uncorrected Patrol Scrub Error",
+ SER, MASK(MCI_STATUS_UC|MCI_ADDR|0xffffeff0, MCI_ADDR|0x001000c0),
+ MODEL_STEPPING(INTEL_FAM6_SKYLAKE_X, 4), BANK_RANGE(13, 18)
+ ),
/* ignore OVER for UCNA */
MCESEV(
#define mc_recoverable(mcg) (((mcg) & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) == \
(MCG_STATUS_RIPV|MCG_STATUS_EIPV))
+static bool is_copy_from_user(struct pt_regs *regs)
+{
+ u8 insn_buf[MAX_INSN_SIZE];
+ struct insn insn;
+ unsigned long addr;
+
+ if (copy_from_kernel_nofault(insn_buf, (void *)regs->ip, MAX_INSN_SIZE))
+ return false;
+
+ kernel_insn_init(&insn, insn_buf, MAX_INSN_SIZE);
+ insn_get_opcode(&insn);
+ if (!insn.opcode.got)
+ return false;
+
+ switch (insn.opcode.value) {
+ /* MOV mem,reg */
+ case 0x8A: case 0x8B:
+ /* MOVZ mem,reg */
+ case 0xB60F: case 0xB70F:
+ insn_get_modrm(&insn);
+ insn_get_sib(&insn);
+ if (!insn.modrm.got || !insn.sib.got)
+ return false;
+ addr = (unsigned long)insn_get_addr_ref(&insn, regs);
+ break;
+ /* REP MOVS */
+ case 0xA4: case 0xA5:
+ addr = regs->si;
+ break;
+ default:
+ return false;
+ }
+
+ if (fault_in_kernel_space(addr))
+ return false;
+
+ current->mce_vaddr = (void __user *)addr;
+
+ return true;
+}
+
/*
* If mcgstatus indicated that ip/cs on the stack were
* no good, then "m->cs" will be zero and we will have
* distinguish an exception taken in user from from one
* taken in the kernel.
*/
-static int error_context(struct mce *m)
+static int error_context(struct mce *m, struct pt_regs *regs)
{
+ enum handler_type t;
+
if ((m->cs & 3) == 3)
return IN_USER;
+ if (!mc_recoverable(m->mcgstatus))
+ return IN_KERNEL;
- if (mc_recoverable(m->mcgstatus) && ex_has_fault_handler(m->ip)) {
+ t = ex_get_fault_handler_type(m->ip);
+ if (t == EX_HANDLER_FAULT) {
m->kflags |= MCE_IN_KERNEL_RECOV;
return IN_KERNEL_RECOV;
}
+ if (t == EX_HANDLER_UACCESS && regs && is_copy_from_user(regs)) {
+ m->kflags |= MCE_IN_KERNEL_RECOV;
+ m->kflags |= MCE_IN_KERNEL_COPYIN;
+ return IN_KERNEL_RECOV;
+ }
return IN_KERNEL;
}
* See AMD Error Scope Hierarchy table in a newer BKDG. For example
* 49125_15h_Models_30h-3Fh_BKDG.pdf, section "RAS Features"
*/
-static int mce_severity_amd(struct mce *m, int tolerant, char **msg, bool is_excp)
+static int mce_severity_amd(struct mce *m, struct pt_regs *regs, int tolerant,
+ char **msg, bool is_excp)
{
- enum context ctx = error_context(m);
+ enum context ctx = error_context(m, regs);
/* Processor Context Corrupt, no need to fumble too much, die! */
if (m->status & MCI_STATUS_PCC)
return MCE_KEEP_SEVERITY;
}
-static int mce_severity_intel(struct mce *m, int tolerant, char **msg, bool is_excp)
+static int mce_severity_intel(struct mce *m, struct pt_regs *regs,
+ int tolerant, char **msg, bool is_excp)
{
enum exception excp = (is_excp ? EXCP_CONTEXT : NO_EXCP);
- enum context ctx = error_context(m);
+ enum context ctx = error_context(m, regs);
struct severity *s;
for (s = severities;; s++) {
continue;
if (s->excp && excp != s->excp)
continue;
+ if (s->cpu_model && boot_cpu_data.x86_model != s->cpu_model)
+ continue;
+ if (s->cpu_minstepping && boot_cpu_data.x86_stepping < s->cpu_minstepping)
+ continue;
+ if (s->bank_lo && (m->bank < s->bank_lo || m->bank > s->bank_hi))
+ continue;
if (msg)
*msg = s->msg;
s->covered = 1;
}
/* Default to mce_severity_intel */
-int (*mce_severity)(struct mce *m, int tolerant, char **msg, bool is_excp) =
+int (*mce_severity)(struct mce *m, struct pt_regs *regs, int tolerant, char **msg, bool is_excp) =
mce_severity_intel;
void __init mcheck_vendor_init_severity(void)
hv_host_info_edx >> 24, hv_host_info_edx & 0xFFFFFF);
}
- if (ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS &&
+ if (ms_hyperv.features & HV_ACCESS_FREQUENCY_MSRS &&
ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE) {
x86_platform.calibrate_tsc = hv_get_tsc_khz;
x86_platform.calibrate_cpu = hv_get_tsc_khz;
crash_kexec_post_notifiers = true;
#ifdef CONFIG_X86_LOCAL_APIC
- if (ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS &&
+ if (ms_hyperv.features & HV_ACCESS_FREQUENCY_MSRS &&
ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE) {
/*
* Get the APIC frequency.
machine_ops.shutdown = hv_machine_shutdown;
machine_ops.crash_shutdown = hv_machine_crash_shutdown;
#endif
- if (ms_hyperv.features & HV_X64_ACCESS_TSC_INVARIANT) {
+ if (ms_hyperv.features & HV_ACCESS_TSC_INVARIANT) {
wrmsrl(HV_X64_MSR_TSC_INVARIANT_CONTROL, 0x1);
setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE);
} else {
alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_hyperv_callback);
/* Setup the IDT for reenlightenment notifications */
- if (ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT) {
+ if (ms_hyperv.features & HV_ACCESS_REENLIGHTENMENT) {
alloc_intr_gate(HYPERV_REENLIGHTENMENT_VECTOR,
asm_sysvec_hyperv_reenlightenment);
}
.name = "MB",
.domains = domain_init(RDT_RESOURCE_MBA),
.cache_level = 3,
+ .parse_ctrlval = parse_bw,
.format_str = "%d=%*u",
.fflags = RFTYPE_RES_MB,
},
{
union cpuid_0x10_3_eax eax;
union cpuid_0x10_x_edx edx;
- u32 ebx, ecx;
+ u32 ebx, ecx, max_delay;
cpuid_count(0x00000010, 3, &eax.full, &ebx, &ecx, &edx.full);
r->num_closid = edx.split.cos_max + 1;
- r->membw.max_delay = eax.split.max_delay + 1;
+ max_delay = eax.split.max_delay + 1;
r->default_ctrl = MAX_MBA_BW;
+ r->membw.arch_needs_linear = true;
if (ecx & MBA_IS_LINEAR) {
r->membw.delay_linear = true;
- r->membw.min_bw = MAX_MBA_BW - r->membw.max_delay;
- r->membw.bw_gran = MAX_MBA_BW - r->membw.max_delay;
+ r->membw.min_bw = MAX_MBA_BW - max_delay;
+ r->membw.bw_gran = MAX_MBA_BW - max_delay;
} else {
if (!rdt_get_mb_table(r))
return false;
+ r->membw.arch_needs_linear = false;
}
r->data_width = 3;
+ if (boot_cpu_has(X86_FEATURE_PER_THREAD_MBA))
+ r->membw.throttle_mode = THREAD_THROTTLE_PER_THREAD;
+ else
+ r->membw.throttle_mode = THREAD_THROTTLE_MAX;
+ thread_throttle_mode_init();
+
r->alloc_capable = true;
r->alloc_enabled = true;
/* AMD does not use delay */
r->membw.delay_linear = false;
+ r->membw.arch_needs_linear = false;
+ /*
+ * AMD does not use memory delay throttle model to control
+ * the allocation like Intel does.
+ */
+ r->membw.throttle_mode = THREAD_THROTTLE_UNDEFINED;
r->membw.min_bw = 0;
r->membw.bw_gran = 1;
/* Max value is 2048, Data width should be 4 in decimal */
rdt_get_cdp_config(RDT_RESOURCE_L2, RDT_RESOURCE_L2CODE);
}
-static int get_cache_id(int cpu, int level)
-{
- struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu);
- int i;
-
- for (i = 0; i < ci->num_leaves; i++) {
- if (ci->info_list[i].level == level)
- return ci->info_list[i].id;
- }
-
- return -1;
-}
-
static void
mba_wrmsr_amd(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r)
{
*/
static void domain_add_cpu(int cpu, struct rdt_resource *r)
{
- int id = get_cache_id(cpu, r->cache_level);
+ int id = get_cpu_cacheinfo_id(cpu, r->cache_level);
struct list_head *add_pos = NULL;
struct rdt_domain *d;
d = rdt_find_domain(r, id, &add_pos);
if (IS_ERR(d)) {
- pr_warn("Could't find cache id for cpu %d\n", cpu);
+ pr_warn("Couldn't find cache id for CPU %d\n", cpu);
return;
}
static void domain_remove_cpu(int cpu, struct rdt_resource *r)
{
- int id = get_cache_id(cpu, r->cache_level);
+ int id = get_cpu_cacheinfo_id(cpu, r->cache_level);
struct rdt_domain *d;
d = rdt_find_domain(r, id, NULL);
if (IS_ERR_OR_NULL(d)) {
- pr_warn("Could't find cache id for cpu %d\n", cpu);
+ pr_warn("Couldn't find cache id for CPU %d\n", cpu);
return;
}
r->rid == RDT_RESOURCE_L3CODE ||
r->rid == RDT_RESOURCE_L2 ||
r->rid == RDT_RESOURCE_L2DATA ||
- r->rid == RDT_RESOURCE_L2CODE)
- r->cbm_validate = cbm_validate_intel;
- else if (r->rid == RDT_RESOURCE_MBA) {
+ r->rid == RDT_RESOURCE_L2CODE) {
+ r->cache.arch_has_sparse_bitmaps = false;
+ r->cache.arch_has_empty_bitmaps = false;
+ } else if (r->rid == RDT_RESOURCE_MBA) {
r->msr_base = MSR_IA32_MBA_THRTL_BASE;
r->msr_update = mba_wrmsr_intel;
- r->parse_ctrlval = parse_bw_intel;
}
}
}
r->rid == RDT_RESOURCE_L3CODE ||
r->rid == RDT_RESOURCE_L2 ||
r->rid == RDT_RESOURCE_L2DATA ||
- r->rid == RDT_RESOURCE_L2CODE)
- r->cbm_validate = cbm_validate_amd;
- else if (r->rid == RDT_RESOURCE_MBA) {
+ r->rid == RDT_RESOURCE_L2CODE) {
+ r->cache.arch_has_sparse_bitmaps = true;
+ r->cache.arch_has_empty_bitmaps = true;
+ } else if (r->rid == RDT_RESOURCE_MBA) {
r->msr_base = MSR_IA32_MBA_BW_BASE;
r->msr_update = mba_wrmsr_amd;
- r->parse_ctrlval = parse_bw_amd;
}
}
}
#include <linux/slab.h>
#include "internal.h"
-/*
- * Check whether MBA bandwidth percentage value is correct. The value is
- * checked against the minimum and maximum bandwidth values specified by
- * the hardware. The allocated bandwidth percentage is rounded to the next
- * control step available on the hardware.
- */
-static bool bw_validate_amd(char *buf, unsigned long *data,
- struct rdt_resource *r)
-{
- unsigned long bw;
- int ret;
-
- ret = kstrtoul(buf, 10, &bw);
- if (ret) {
- rdt_last_cmd_printf("Non-decimal digit in MB value %s\n", buf);
- return false;
- }
-
- if (bw < r->membw.min_bw || bw > r->default_ctrl) {
- rdt_last_cmd_printf("MB value %ld out of range [%d,%d]\n", bw,
- r->membw.min_bw, r->default_ctrl);
- return false;
- }
-
- *data = roundup(bw, (unsigned long)r->membw.bw_gran);
- return true;
-}
-
-int parse_bw_amd(struct rdt_parse_data *data, struct rdt_resource *r,
- struct rdt_domain *d)
-{
- unsigned long bw_val;
-
- if (d->have_new_ctrl) {
- rdt_last_cmd_printf("Duplicate domain %d\n", d->id);
- return -EINVAL;
- }
-
- if (!bw_validate_amd(data->buf, &bw_val, r))
- return -EINVAL;
-
- d->new_ctrl = bw_val;
- d->have_new_ctrl = true;
-
- return 0;
-}
-
/*
* Check whether MBA bandwidth percentage value is correct. The value is
* checked against the minimum and max bandwidth values specified by the
/*
* Only linear delay values is supported for current Intel SKUs.
*/
- if (!r->membw.delay_linear) {
+ if (!r->membw.delay_linear && r->membw.arch_needs_linear) {
rdt_last_cmd_puts("No support for non-linear MB domains\n");
return false;
}
return true;
}
-int parse_bw_intel(struct rdt_parse_data *data, struct rdt_resource *r,
- struct rdt_domain *d)
+int parse_bw(struct rdt_parse_data *data, struct rdt_resource *r,
+ struct rdt_domain *d)
{
unsigned long bw_val;
}
/*
- * Check whether a cache bit mask is valid. The SDM says:
+ * Check whether a cache bit mask is valid.
+ * For Intel the SDM says:
* Please note that all (and only) contiguous '1' combinations
* are allowed (e.g. FFFFH, 0FF0H, 003CH, etc.).
* Additionally Haswell requires at least two bits set.
+ * AMD allows non-contiguous bitmasks.
*/
-bool cbm_validate_intel(char *buf, u32 *data, struct rdt_resource *r)
+static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
{
unsigned long first_bit, zero_bit, val;
unsigned int cbm_len = r->cache.cbm_len;
return false;
}
- if (val == 0 || val > r->default_ctrl) {
+ if ((!r->cache.arch_has_empty_bitmaps && val == 0) ||
+ val > r->default_ctrl) {
rdt_last_cmd_puts("Mask out of range\n");
return false;
}
first_bit = find_first_bit(&val, cbm_len);
zero_bit = find_next_zero_bit(&val, cbm_len, first_bit);
- if (find_next_bit(&val, cbm_len, zero_bit) < cbm_len) {
+ /* Are non-contiguous bitmaps allowed? */
+ if (!r->cache.arch_has_sparse_bitmaps &&
+ (find_next_bit(&val, cbm_len, zero_bit) < cbm_len)) {
rdt_last_cmd_printf("The mask %lx has non-consecutive 1-bits\n", val);
return false;
}
return true;
}
-/*
- * Check whether a cache bit mask is valid. AMD allows non-contiguous
- * bitmasks
- */
-bool cbm_validate_amd(char *buf, u32 *data, struct rdt_resource *r)
-{
- unsigned long val;
- int ret;
-
- ret = kstrtoul(buf, 16, &val);
- if (ret) {
- rdt_last_cmd_printf("Non-hex character in the mask %s\n", buf);
- return false;
- }
-
- if (val > r->default_ctrl) {
- rdt_last_cmd_puts("Mask out of range\n");
- return false;
- }
-
- *data = val;
- return true;
-}
-
/*
* Read one cache bit mask (hex). Check that it is valid for the current
* resource type.
return -EINVAL;
}
- if (!r->cbm_validate(data->buf, &cbm_val, r))
+ if (!cbm_validate(data->buf, &cbm_val, r))
return -EINVAL;
if ((rdtgrp->mode == RDT_MODE_EXCLUSIVE ||
* struct mbm_state - status for each MBM counter in each domain
* @chunks: Total data moved (multiply by rdt_group.mon_scale to get bytes)
* @prev_msr Value of IA32_QM_CTR for this RMID last time we read it
- * @chunks_bw Total local data moved. Used for bandwidth calculation
* @prev_bw_msr:Value of previous IA32_QM_CTR for bandwidth counting
* @prev_bw The most recent bandwidth in MBps
* @delta_bw Difference between the current and previous bandwidth
struct mbm_state {
u64 chunks;
u64 prev_msr;
- u64 chunks_bw;
u64 prev_bw_msr;
u32 prev_bw;
u32 delta_bw;
* in a cache bit mask
* @shareable_bits: Bitmask of shareable resource with other
* executing entities
+ * @arch_has_sparse_bitmaps: True if a bitmap like f00f is valid.
+ * @arch_has_empty_bitmaps: True if the '0' bitmap is valid.
*/
struct rdt_cache {
unsigned int cbm_len;
unsigned int cbm_idx_mult;
unsigned int cbm_idx_offset;
unsigned int shareable_bits;
+ bool arch_has_sparse_bitmaps;
+ bool arch_has_empty_bitmaps;
+};
+
+/**
+ * enum membw_throttle_mode - System's memory bandwidth throttling mode
+ * @THREAD_THROTTLE_UNDEFINED: Not relevant to the system
+ * @THREAD_THROTTLE_MAX: Memory bandwidth is throttled at the core
+ * always using smallest bandwidth percentage
+ * assigned to threads, aka "max throttling"
+ * @THREAD_THROTTLE_PER_THREAD: Memory bandwidth is throttled at the thread
+ */
+enum membw_throttle_mode {
+ THREAD_THROTTLE_UNDEFINED = 0,
+ THREAD_THROTTLE_MAX,
+ THREAD_THROTTLE_PER_THREAD,
};
/**
* struct rdt_membw - Memory bandwidth allocation related data
- * @max_delay: Max throttle delay. Delay is the hardware
- * representation for memory bandwidth.
* @min_bw: Minimum memory bandwidth percentage user can request
* @bw_gran: Granularity at which the memory bandwidth is allocated
* @delay_linear: True if memory B/W delay is in linear scale
+ * @arch_needs_linear: True if we can't configure non-linear resources
+ * @throttle_mode: Bandwidth throttling mode when threads request
+ * different memory bandwidths
* @mba_sc: True if MBA software controller(mba_sc) is enabled
* @mb_map: Mapping of memory B/W percentage to memory B/W delay
*/
struct rdt_membw {
- u32 max_delay;
- u32 min_bw;
- u32 bw_gran;
- u32 delay_linear;
- bool mba_sc;
- u32 *mb_map;
+ u32 min_bw;
+ u32 bw_gran;
+ u32 delay_linear;
+ bool arch_needs_linear;
+ enum membw_throttle_mode throttle_mode;
+ bool mba_sc;
+ u32 *mb_map;
};
static inline bool is_llc_occupancy_enabled(void)
* @cache: Cache allocation related data
* @format_str: Per resource format string to show domain value
* @parse_ctrlval: Per resource function pointer to parse control values
- * @cbm_validate Cache bitmask validate function
* @evt_list: List of monitoring events
* @num_rmid: Number of RMIDs available
* @mon_scale: cqm counter * mon_scale = occupancy in bytes
int (*parse_ctrlval)(struct rdt_parse_data *data,
struct rdt_resource *r,
struct rdt_domain *d);
- bool (*cbm_validate)(char *buf, u32 *data, struct rdt_resource *r);
struct list_head evt_list;
int num_rmid;
unsigned int mon_scale;
int parse_cbm(struct rdt_parse_data *data, struct rdt_resource *r,
struct rdt_domain *d);
-int parse_bw_intel(struct rdt_parse_data *data, struct rdt_resource *r,
- struct rdt_domain *d);
-int parse_bw_amd(struct rdt_parse_data *data, struct rdt_resource *r,
- struct rdt_domain *d);
+int parse_bw(struct rdt_parse_data *data, struct rdt_resource *r,
+ struct rdt_domain *d);
extern struct mutex rdtgroup_mutex;
void cqm_handle_limbo(struct work_struct *work);
bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d);
void __check_limbo(struct rdt_domain *d, bool force_free);
-bool cbm_validate_intel(char *buf, u32 *data, struct rdt_resource *r);
-bool cbm_validate_amd(char *buf, u32 *data, struct rdt_resource *r);
void rdt_domain_reconfigure_cdp(struct rdt_resource *r);
+void __init thread_throttle_mode_init(void);
#endif /* _ASM_X86_RESCTRL_INTERNAL_H */
return;
chunks = mbm_overflow_count(m->prev_bw_msr, tval, rr->r->mbm_width);
- m->chunks_bw += chunks;
- m->chunks = m->chunks_bw;
+ m->chunks += chunks;
cur_bw = (chunks * r->mon_scale) >> 20;
if (m->delta_comp)
mutex_lock(&rdtgroup_mutex);
r = &rdt_resources_all[RDT_RESOURCE_L3];
- d = get_domain_from_cpu(cpu, r);
-
- if (!d) {
- pr_warn_once("Failure to get domain for limbo worker\n");
- goto out_unlock;
- }
+ d = container_of(work, struct rdt_domain, cqm_limbo.work);
__check_limbo(d, false);
if (has_busy_rmid(r, d))
schedule_delayed_work_on(cpu, &d->cqm_limbo, delay);
-out_unlock:
mutex_unlock(&rdtgroup_mutex);
}
goto out_unlock;
r = &rdt_resources_all[RDT_RESOURCE_L3];
-
- d = get_domain_from_cpu(cpu, r);
- if (!d)
- goto out_unlock;
+ d = container_of(work, struct rdt_domain, mbm_over.work);
list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) {
mbm_update(r, d, prgrp->mon.rmid);
return ret;
}
+static bool is_closid_match(struct task_struct *t, struct rdtgroup *r)
+{
+ return (rdt_alloc_capable &&
+ (r->type == RDTCTRL_GROUP) && (t->closid == r->closid));
+}
+
+static bool is_rmid_match(struct task_struct *t, struct rdtgroup *r)
+{
+ return (rdt_mon_capable &&
+ (r->type == RDTMON_GROUP) && (t->rmid == r->mon.rmid));
+}
+
/**
* rdtgroup_tasks_assigned - Test if tasks have been assigned to resource group
* @r: Resource group
rcu_read_lock();
for_each_process_thread(p, t) {
- if ((r->type == RDTCTRL_GROUP && t->closid == r->closid) ||
- (r->type == RDTMON_GROUP && t->rmid == r->mon.rmid)) {
+ if (is_closid_match(t, r) || is_rmid_match(t, r)) {
ret = 1;
break;
}
rcu_read_lock();
for_each_process_thread(p, t) {
- if ((r->type == RDTCTRL_GROUP && t->closid == r->closid) ||
- (r->type == RDTMON_GROUP && t->rmid == r->mon.rmid))
+ if (is_closid_match(t, r) || is_rmid_match(t, r))
seq_printf(s, "%d\n", t->pid);
}
rcu_read_unlock();
return 0;
}
+static int rdt_thread_throttle_mode_show(struct kernfs_open_file *of,
+ struct seq_file *seq, void *v)
+{
+ struct rdt_resource *r = of->kn->parent->priv;
+
+ if (r->membw.throttle_mode == THREAD_THROTTLE_PER_THREAD)
+ seq_puts(seq, "per-thread\n");
+ else
+ seq_puts(seq, "max\n");
+
+ return 0;
+}
+
static ssize_t max_threshold_occ_write(struct kernfs_open_file *of,
char *buf, size_t nbytes, loff_t off)
{
.seq_show = rdt_delay_linear_show,
.fflags = RF_CTRL_INFO | RFTYPE_RES_MB,
},
+ /*
+ * Platform specific which (if any) capabilities are provided by
+ * thread_throttle_mode. Defer "fflags" initialization to platform
+ * discovery.
+ */
+ {
+ .name = "thread_throttle_mode",
+ .mode = 0444,
+ .kf_ops = &rdtgroup_kf_single_ops,
+ .seq_show = rdt_thread_throttle_mode_show,
+ },
{
.name = "max_threshold_occupancy",
.mode = 0644,
lockdep_assert_held(&rdtgroup_mutex);
for (rft = rfts; rft < rfts + len; rft++) {
- if ((fflags & rft->fflags) == rft->fflags) {
+ if (rft->fflags && ((fflags & rft->fflags) == rft->fflags)) {
ret = rdtgroup_add_file(kn, rft);
if (ret)
goto error;
return ret;
}
+static struct rftype *rdtgroup_get_rftype_by_name(const char *name)
+{
+ struct rftype *rfts, *rft;
+ int len;
+
+ rfts = res_common_files;
+ len = ARRAY_SIZE(res_common_files);
+
+ for (rft = rfts; rft < rfts + len; rft++) {
+ if (!strcmp(rft->name, name))
+ return rft;
+ }
+
+ return NULL;
+}
+
+void __init thread_throttle_mode_init(void)
+{
+ struct rftype *rft;
+
+ rft = rdtgroup_get_rftype_by_name("thread_throttle_mode");
+ if (!rft)
+ return;
+
+ rft->fflags = RF_CTRL_INFO | RFTYPE_RES_MB;
+}
+
/**
* rdtgroup_kn_mode_restrict - Restrict user access to named resctrl file
* @r: The resource group with which the file is associated.
return 0;
}
-static bool is_closid_match(struct task_struct *t, struct rdtgroup *r)
-{
- return (rdt_alloc_capable &&
- (r->type == RDTCTRL_GROUP) && (t->closid == r->closid));
-}
-
-static bool is_rmid_match(struct task_struct *t, struct rdtgroup *r)
-{
- return (rdt_mon_capable &&
- (r->type == RDTMON_GROUP) && (t->rmid == r->mon.rmid));
-}
-
/*
* Move tasks from one to the other group. If @from is NULL, then all tasks
* in the systems are moved unconditionally (used for teardown).
* It may also be ok since that would enable debugging of RDT before
* resctrl is mounted.
* The reason why the debugfs directory is created here and not in
- * rdt_mount() is because rdt_mount() takes rdtgroup_mutex and
+ * rdt_get_tree() is because rdt_get_tree() takes rdtgroup_mutex and
* during the debugfs directory creation also &sb->s_type->i_mutex_key
* (the lockdep class of inode->i_rwsem). Other filesystem
* interactions (eg. SyS_getdents) have the lock ordering:
{ X86_FEATURE_CDP_L3, CPUID_ECX, 2, 0x00000010, 1 },
{ X86_FEATURE_CDP_L2, CPUID_ECX, 2, 0x00000010, 2 },
{ X86_FEATURE_MBA, CPUID_EBX, 3, 0x00000010, 0 },
+ { X86_FEATURE_PER_THREAD_MBA, CPUID_ECX, 0, 0x00000010, 3 },
{ X86_FEATURE_HW_PSTATE, CPUID_EDX, 7, 0x80000007, 0 },
{ X86_FEATURE_CPB, CPUID_EDX, 9, 0x80000007, 0 },
{ X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 },
{ X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 },
{ X86_FEATURE_SME, CPUID_EAX, 0, 0x8000001f, 0 },
{ X86_FEATURE_SEV, CPUID_EAX, 1, 0x8000001f, 0 },
+ { X86_FEATURE_SME_COHERENT, CPUID_EAX, 10, 0x8000001f, 0 },
{ 0, 0, 0, 0, 0 }
};
it = &of_ioapic_type[type_index];
ioapic_set_alloc_attr(&tmp, NUMA_NO_NODE, it->trigger, it->polarity);
- tmp.ioapic_id = mpc_ioapic_id(mp_irqdomain_ioapic_idx(domain));
- tmp.ioapic_pin = fwspec->param[0];
+ tmp.devid = mpc_ioapic_id(mp_irqdomain_ioapic_idx(domain));
+ tmp.ioapic.pin = fwspec->param[0];
return mp_irqdomain_alloc(domain, virq, nr_irqs, &tmp);
}
unsigned long prologue = regs->ip - PROLOGUE_SIZE;
if (copy_code(regs, opcodes, prologue, sizeof(opcodes))) {
- printk("%sCode: Bad RIP value.\n", loglvl);
+ printk("%sCode: Unable to access opcode bytes at RIP 0x%lx.\n",
+ loglvl, prologue);
} else {
printk("%sCode: %" __stringify(PROLOGUE_SIZE) "ph <%02x> %"
__stringify(EPILOGUE_SIZE) "ph\n", loglvl, opcodes,
#include <asm/fpu/internal.h>
#include <asm/tlbflush.h>
#include <asm/setup.h>
-#include <asm/cmdline.h>
#include <linux/sched.h>
#include <linux/sched/task.h>
on_boot_cpu = 0;
}
-/*
- * We parse fpu parameters early because fpu__init_system() is executed
- * before parse_early_param().
- */
-static void __init fpu__init_parse_early_param(void)
-{
- char arg[32];
- char *argptr = arg;
- int bit;
-
-#ifdef CONFIG_X86_32
- if (cmdline_find_option_bool(boot_command_line, "no387"))
-#ifdef CONFIG_MATH_EMULATION
- setup_clear_cpu_cap(X86_FEATURE_FPU);
-#else
- pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
-#endif
-
- if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
- setup_clear_cpu_cap(X86_FEATURE_FXSR);
-#endif
-
- if (cmdline_find_option_bool(boot_command_line, "noxsave"))
- setup_clear_cpu_cap(X86_FEATURE_XSAVE);
-
- if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
- setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
-
- if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
- setup_clear_cpu_cap(X86_FEATURE_XSAVES);
-
- if (cmdline_find_option(boot_command_line, "clearcpuid", arg,
- sizeof(arg)) &&
- get_option(&argptr, &bit) &&
- bit >= 0 &&
- bit < NCAPINTS * 32)
- setup_clear_cpu_cap(bit);
-}
-
/*
* Called on the boot CPU once per system bootup, to set up the initial
* FPU state that is later cloned into all processes:
*/
void __init fpu__init_system(struct cpuinfo_x86 *c)
{
- fpu__init_parse_early_param();
fpu__init_system_early_generic(c);
/*
"AVX-512 ZMM_Hi256" ,
"Processor Trace (unused)" ,
"Protection Keys User registers",
+ "PASID state",
"unknown xstate feature" ,
};
X86_FEATURE_AVX512F,
X86_FEATURE_INTEL_PT,
X86_FEATURE_PKU,
+ X86_FEATURE_ENQCMD,
};
/*
print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
print_xstate_feature(XFEATURE_MASK_PKRU);
+ print_xstate_feature(XFEATURE_MASK_PASID);
}
/*
XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state);
XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state);
XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state);
+ XCHECK_SZ(sz, nr, XFEATURE_PASID, struct ia32_pasid_state);
/*
* Make *SURE* to add any feature numbers in below if
if ((nr < XFEATURE_YMM) ||
(nr >= XFEATURE_MAX) ||
(nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) ||
- ((nr >= XFEATURE_RSRVD_COMP_10) && (nr <= XFEATURE_LBR))) {
+ ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_LBR))) {
WARN_ONCE(1, "no structure for xstate: %d\n", nr);
XSTATE_WARN_ON(1);
}
return 0;
}
#endif /* CONFIG_PROC_PID_ARCH_STATUS */
+
+#ifdef CONFIG_IOMMU_SUPPORT
+void update_pasid(void)
+{
+ u64 pasid_state;
+ u32 pasid;
+
+ if (!cpu_feature_enabled(X86_FEATURE_ENQCMD))
+ return;
+
+ if (!current->mm)
+ return;
+
+ pasid = READ_ONCE(current->mm->pasid);
+ /* Set the valid bit in the PASID MSR/state only for valid pasid. */
+ pasid_state = pasid == PASID_DISABLED ?
+ pasid : pasid | MSR_IA32_PASID_VALID;
+
+ /*
+ * No need to hold fregs_lock() since the task's fpstate won't
+ * be changed by others (e.g. ptrace) while the task is being
+ * switched to or is in IPI.
+ */
+ if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {
+ /* The MSR is active and can be directly updated. */
+ wrmsrl(MSR_IA32_PASID, pasid_state);
+ } else {
+ struct fpu *fpu = ¤t->thread.fpu;
+ struct ia32_pasid_state *ppasid_state;
+ struct xregs_state *xsave;
+
+ /*
+ * The CPU's xstate registers are not currently active. Just
+ * update the PASID state in the memory buffer here. The
+ * PASID MSR will be loaded when returning to user mode.
+ */
+ xsave = &fpu->state.xsave;
+ xsave->header.xfeatures |= XFEATURE_MASK_PASID;
+ ppasid_state = get_xsave_addr(xsave, XFEATURE_PASID);
+ /*
+ * Since XFEATURE_MASK_PASID is set in xfeatures, ppasid_state
+ * won't be NULL and no need to check its value.
+ *
+ * Only update the task's PASID state when it's different
+ * from the mm's pasid.
+ */
+ if (ppasid_state->pasid != pasid_state) {
+ /*
+ * Invalid fpregs so that state restoring will pick up
+ * the PASID state.
+ */
+ __fpu_invalidate_fpregs_state(fpu);
+ ppasid_state->pasid = pasid_state;
+ }
+ }
+}
+#endif /* CONFIG_IOMMU_SUPPORT */
return 0;
}
-/*
- * Dump the debug register contents to the user.
- * We can't dump our per cpu values because it
- * may contain cpu wide breakpoint, something that
- * doesn't belong to the current task.
- *
- * TODO: include non-ptrace user breakpoints (perf)
- */
-void aout_dump_debugregs(struct user *dump)
-{
- int i;
- int dr7 = 0;
- struct perf_event *bp;
- struct arch_hw_breakpoint *info;
- struct thread_struct *thread = ¤t->thread;
-
- for (i = 0; i < HBP_NUM; i++) {
- bp = thread->ptrace_bps[i];
-
- if (bp && !bp->attr.disabled) {
- dump->u_debugreg[i] = bp->attr.bp_addr;
- info = counter_arch_bp(bp);
- dr7 |= encode_dr7(i, info->len, info->type);
- } else {
- dump->u_debugreg[i] = 0;
- }
- }
-
- dump->u_debugreg[4] = 0;
- dump->u_debugreg[5] = 0;
- dump->u_debugreg[6] = current->thread.debugreg6;
-
- dump->u_debugreg[7] = dr7;
-}
-EXPORT_SYMBOL_GPL(aout_dump_debugregs);
-
/*
* Release the user breakpoints used by ptrace
*/
t->ptrace_bps[i] = NULL;
}
- t->debugreg6 = 0;
+ t->virtual_dr6 = 0;
t->ptrace_dr7 = 0;
}
set_debugreg(__this_cpu_read(cpu_debugreg[1]), 1);
set_debugreg(__this_cpu_read(cpu_debugreg[2]), 2);
set_debugreg(__this_cpu_read(cpu_debugreg[3]), 3);
- set_debugreg(current->thread.debugreg6, 6);
+ set_debugreg(DR6_RESERVED, 6);
set_debugreg(__this_cpu_read(cpu_dr7), 7);
}
EXPORT_SYMBOL_GPL(hw_breakpoint_restore);
*/
static int hw_breakpoint_handler(struct die_args *args)
{
- int i, cpu, rc = NOTIFY_STOP;
+ int i, rc = NOTIFY_STOP;
struct perf_event *bp;
- unsigned long dr6;
unsigned long *dr6_p;
+ unsigned long dr6;
/* The DR6 value is pointed by args->err */
dr6_p = (unsigned long *)ERR_PTR(args->err);
if ((dr6 & DR_TRAP_BITS) == 0)
return NOTIFY_DONE;
- /*
- * Assert that local interrupts are disabled
- * Reset the DRn bits in the virtualized register value.
- * The ptrace trigger routine will add in whatever is needed.
- */
- current->thread.debugreg6 &= ~DR_TRAP_BITS;
- cpu = get_cpu();
-
/* Handle all the breakpoints that were triggered */
for (i = 0; i < HBP_NUM; ++i) {
if (likely(!(dr6 & (DR_TRAP0 << i))))
*/
rcu_read_lock();
- bp = per_cpu(bp_per_reg[i], cpu);
+ bp = this_cpu_read(bp_per_reg[i]);
/*
* Reset the 'i'th TRAP bit in dr6 to denote completion of
* exception handling
* breakpoints (to generate signals) and b) when the system has
* taken exception due to multiple causes
*/
- if ((current->thread.debugreg6 & DR_TRAP_BITS) ||
+ if ((current->thread.virtual_dr6 & DR_TRAP_BITS) ||
(dr6 & (~DR_TRAP_BITS)))
rc = NOTIFY_DONE;
- put_cpu();
-
return rc;
}
# endif
# ifdef CONFIG_IRQ_WORK
INTG(IRQ_WORK_VECTOR, asm_sysvec_irq_work),
-# endif
-# ifdef CONFIG_X86_UV
- INTG(UV_BAU_MESSAGE, asm_sysvec_uv_bau_message),
# endif
INTG(SPURIOUS_APIC_VECTOR, asm_sysvec_spurious_apic_interrupt),
INTG(ERROR_APIC_VECTOR, asm_sysvec_error_interrupt),
struct pt_regs *regs)
{
if (IS_ENABLED(CONFIG_X86_64))
- run_on_irqstack_cond(desc->handle_irq, desc, regs);
+ run_irq_on_irqstack_cond(desc->handle_irq, desc, regs);
else
__handle_irq(desc, regs);
}
void do_softirq_own_stack(void)
{
- run_on_irqstack_cond(__do_softirq, NULL, NULL);
+ run_on_irqstack_cond(__do_softirq, NULL);
}
struct task_struct *tsk = current;
int i;
- for (i = 0; i < 4; i++)
+ for (i = 0; i < 4; i++) {
if (breakinfo[i].enabled)
- tsk->thread.debugreg6 |= (DR_TRAP0 << i);
+ tsk->thread.virtual_dr6 |= (DR_TRAP0 << i);
+ }
}
void kgdb_arch_late(void)
NOKPROBE_SYMBOL(kretprobe_trampoline);
STACK_FRAME_NON_STANDARD(kretprobe_trampoline);
+
/*
* Called from kretprobe_trampoline
*/
__used __visible void *trampoline_handler(struct pt_regs *regs)
{
- struct kretprobe_instance *ri = NULL;
- struct hlist_head *head, empty_rp;
- struct hlist_node *tmp;
- unsigned long flags, orig_ret_address = 0;
- unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
- kprobe_opcode_t *correct_ret_addr = NULL;
- void *frame_pointer;
- bool skipped = false;
-
- /*
- * Set a dummy kprobe for avoiding kretprobe recursion.
- * Since kretprobe never run in kprobe handler, kprobe must not
- * be running at this point.
- */
- kprobe_busy_begin();
-
- INIT_HLIST_HEAD(&empty_rp);
- kretprobe_hash_lock(current, &head, &flags);
/* fixup registers */
regs->cs = __KERNEL_CS;
#ifdef CONFIG_X86_32
- regs->cs |= get_kernel_rpl();
regs->gs = 0;
#endif
- /* We use pt_regs->sp for return address holder. */
- frame_pointer = ®s->sp;
- regs->ip = trampoline_address;
+ regs->ip = (unsigned long)&kretprobe_trampoline;
regs->orig_ax = ~0UL;
- /*
- * It is possible to have multiple instances associated with a given
- * task either because multiple functions in the call path have
- * return probes installed on them, and/or more than one
- * return probe was registered for a target function.
- *
- * We can handle this because:
- * - instances are always pushed into the head of the list
- * - when multiple return probes are registered for the same
- * function, the (chronologically) first instance's ret_addr
- * will be the real return address, and all the rest will
- * point to kretprobe_trampoline.
- */
- hlist_for_each_entry(ri, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
- /*
- * Return probes must be pushed on this hash list correct
- * order (same as return order) so that it can be popped
- * correctly. However, if we find it is pushed it incorrect
- * order, this means we find a function which should not be
- * probed, because the wrong order entry is pushed on the
- * path of processing other kretprobe itself.
- */
- if (ri->fp != frame_pointer) {
- if (!skipped)
- pr_warn("kretprobe is stacked incorrectly. Trying to fixup.\n");
- skipped = true;
- continue;
- }
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- if (skipped)
- pr_warn("%ps must be blacklisted because of incorrect kretprobe order\n",
- ri->rp->kp.addr);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_assert(ri, orig_ret_address, trampoline_address);
-
- correct_ret_addr = ri->ret_addr;
- hlist_for_each_entry_safe(ri, tmp, head, hlist) {
- if (ri->task != current)
- /* another task is sharing our hash bucket */
- continue;
- if (ri->fp != frame_pointer)
- continue;
-
- orig_ret_address = (unsigned long)ri->ret_addr;
- if (ri->rp && ri->rp->handler) {
- __this_cpu_write(current_kprobe, &ri->rp->kp);
- ri->ret_addr = correct_ret_addr;
- ri->rp->handler(ri, regs);
- __this_cpu_write(current_kprobe, &kprobe_busy);
- }
-
- recycle_rp_inst(ri, &empty_rp);
-
- if (orig_ret_address != trampoline_address)
- /*
- * This is the real return address. Any other
- * instances associated with this task are for
- * other calls deeper on the call stack
- */
- break;
- }
-
- kretprobe_hash_unlock(current, &flags);
-
- kprobe_busy_end();
-
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
- return (void *)orig_ret_address;
+ return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, ®s->sp);
}
NOKPROBE_SYMBOL(trampoline_handler);
#include <linux/ftrace.h>
#include <linux/frame.h>
#include <linux/pgtable.h>
+#include <linux/static_call.h>
#include <asm/text-patching.h>
#include <asm/cacheflush.h>
/* Save skipped registers */
regs->cs = __KERNEL_CS;
#ifdef CONFIG_X86_32
- regs->cs |= get_kernel_rpl();
regs->gs = 0;
#endif
regs->ip = (unsigned long)op->kp.addr + INT3_INSN_SIZE;
/* Check whether the address range is reserved */
if (ftrace_text_reserved(src, src + len - 1) ||
alternatives_text_reserved(src, src + len - 1) ||
- jump_label_text_reserved(src, src + len - 1))
+ jump_label_text_reserved(src, src + len - 1) ||
+ static_call_text_reserved(src, src + len - 1))
return -EBUSY;
return len;
#include <asm/irqdomain.h>
#include <asm/mtrr.h>
#include <asm/mpspec.h>
-#include <asm/io_apic.h>
#include <asm/proto.h>
#include <asm/bios_ebda.h>
#include <asm/e820/api.h>
return sum & 0xFF;
}
-int __init default_mpc_apic_id(struct mpc_cpu *m)
-{
- return m->apicid;
-}
-
static void __init MP_processor_info(struct mpc_cpu *m)
{
int apicid;
return;
}
- apicid = x86_init.mpparse.mpc_apic_id(m);
+ apicid = m->apicid;
if (m->cpuflag & CPU_BOOTPROCESSOR) {
bootup_cpu = " (Bootup-CPU)";
}
#ifdef CONFIG_X86_IO_APIC
-void __init default_mpc_oem_bus_info(struct mpc_bus *m, char *str)
+static void __init mpc_oem_bus_info(struct mpc_bus *m, char *str)
{
memcpy(str, m->bustype, 6);
str[6] = 0;
{
char str[7];
- x86_init.mpparse.mpc_oem_bus_info(m, str);
+ mpc_oem_bus_info(m, str);
#if MAX_MP_BUSSES < 256
if (m->busid >= MAX_MP_BUSSES) {
mp_bus_id_to_type[m->busid] = MP_BUS_ISA;
#endif
} else if (strncmp(str, BUSTYPE_PCI, sizeof(BUSTYPE_PCI) - 1) == 0) {
- if (x86_init.mpparse.mpc_oem_pci_bus)
- x86_init.mpparse.mpc_oem_pci_bus(m);
-
clear_bit(m->busid, mp_bus_not_pci);
#ifdef CONFIG_EISA
mp_bus_id_to_type[m->busid] = MP_BUS_PCI;
1, mpc, mpc->length, 1);
}
-void __init default_smp_read_mpc_oem(struct mpc_table *mpc) { }
-
static int __init smp_read_mpc(struct mpc_table *mpc, unsigned early)
{
char str[16];
if (early)
return 1;
- if (mpc->oemptr)
- x86_init.mpparse.smp_read_mpc_oem(mpc);
-
- /*
- * Now process the configuration blocks.
- */
- x86_init.mpparse.mpc_record(0);
-
+ /* Now process the configuration blocks. */
while (count < mpc->length) {
switch (*mpt) {
case MP_PROCESSOR:
count = mpc->length;
break;
}
- x86_init.mpparse.mpc_record(1);
}
if (!num_processors)
static int filter_write(u32 reg)
{
+ /*
+ * MSRs writes usually happen all at once, and can easily saturate kmsg.
+ * Only allow one message every 30 seconds.
+ *
+ * It's possible to be smarter here and do it (for example) per-MSR, but
+ * it would certainly be more complex, and this is enough at least to
+ * avoid saturating the ring buffer.
+ */
+ static DEFINE_RATELIMIT_STATE(fw_rs, 30 * HZ, 1);
+
switch (allow_writes) {
case MSR_WRITES_ON: return 0;
case MSR_WRITES_OFF: return -EPERM;
default: break;
}
+ if (!__ratelimit(&fw_rs))
+ return 0;
+
if (reg == MSR_IA32_ENERGY_PERF_BIAS)
return 0;
- pr_err_ratelimited("Write to unrecognized MSR 0x%x by %s\n"
- "Please report to x86@kernel.org\n",
- reg, current->comm);
+ pr_err("Write to unrecognized MSR 0x%x by %s (pid: %d). Please report to x86@kernel.org.\n",
+ reg, current->comm, current->pid);
return 0;
}
static void nmi_check_duration(struct nmiaction *action, u64 duration)
{
- u64 whole_msecs = READ_ONCE(action->max_duration);
int remainder_ns, decimal_msecs;
if (duration < nmi_longest_ns || duration < action->max_duration)
action->max_duration = duration;
- remainder_ns = do_div(whole_msecs, (1000 * 1000));
+ remainder_ns = do_div(duration, (1000 * 1000));
decimal_msecs = remainder_ns / 1000;
printk_ratelimited(KERN_INFO
"INFO: NMI handler (%ps) took too long to run: %lld.%03d msecs\n",
- action->handler, whole_msecs, decimal_msecs);
+ action->handler, duration, decimal_msecs);
}
static int nmi_handle(unsigned int type, struct pt_regs *regs)
struct pv_info pv_info = {
.name = "bare hardware",
#ifdef CONFIG_PARAVIRT_XXL
- .kernel_rpl = 0,
- .shared_kernel_pmd = 1, /* Only used when CONFIG_X86_PAE is set */
-
-#ifdef CONFIG_X86_64
.extra_user_64bit_cs = __USER_CS,
#endif
-#endif
};
/* 64-bit pagetable entries */
.cpu.load_idt = native_load_idt,
.cpu.store_tr = native_store_tr,
.cpu.load_tls = native_load_tls,
-#ifdef CONFIG_X86_64
.cpu.load_gs_index = native_load_gs_index,
-#endif
.cpu.write_ldt_entry = native_write_ldt_entry,
.cpu.write_gdt_entry = native_write_gdt_entry,
.cpu.write_idt_entry = native_write_idt_entry,
.cpu.load_sp0 = native_load_sp0,
-#ifdef CONFIG_X86_64
.cpu.usergs_sysret64 = native_usergs_sysret64,
-#endif
.cpu.iret = native_iret,
.cpu.swapgs = native_swapgs,
.mmu.release_p4d = paravirt_nop,
.mmu.set_pte = native_set_pte,
- .mmu.set_pte_at = native_set_pte_at,
.mmu.set_pmd = native_set_pmd,
.mmu.ptep_modify_prot_start = __ptep_modify_prot_start,
.mmu.ptep_modify_prot_commit = __ptep_modify_prot_commit,
-#if CONFIG_PGTABLE_LEVELS >= 3
-#ifdef CONFIG_X86_PAE
- .mmu.set_pte_atomic = native_set_pte_atomic,
- .mmu.pte_clear = native_pte_clear,
- .mmu.pmd_clear = native_pmd_clear,
-#endif
.mmu.set_pud = native_set_pud,
.mmu.pmd_val = PTE_IDENT,
.mmu.make_pmd = PTE_IDENT,
-#if CONFIG_PGTABLE_LEVELS >= 4
.mmu.pud_val = PTE_IDENT,
.mmu.make_pud = PTE_IDENT,
.mmu.set_pgd = native_set_pgd,
#endif /* CONFIG_PGTABLE_LEVELS >= 5 */
-#endif /* CONFIG_PGTABLE_LEVELS >= 4 */
-#endif /* CONFIG_PGTABLE_LEVELS >= 3 */
.mmu.pte_val = PTE_IDENT,
.mmu.pgd_val = PTE_IDENT,
const unsigned char mmu_read_cr3[3];
const unsigned char mmu_write_cr3[3];
const unsigned char irq_restore_fl[2];
-# ifdef CONFIG_X86_64
const unsigned char cpu_wbinvd[2];
const unsigned char cpu_usergs_sysret64[6];
const unsigned char cpu_swapgs[3];
const unsigned char mov64[3];
-# else
- const unsigned char cpu_iret[1];
-# endif
};
static const struct patch_xxl patch_data_xxl = {
.irq_save_fl = { 0x9c, 0x58 }, // pushf; pop %[re]ax
.mmu_read_cr2 = { 0x0f, 0x20, 0xd0 }, // mov %cr2, %[re]ax
.mmu_read_cr3 = { 0x0f, 0x20, 0xd8 }, // mov %cr3, %[re]ax
-# ifdef CONFIG_X86_64
.mmu_write_cr3 = { 0x0f, 0x22, 0xdf }, // mov %rdi, %cr3
.irq_restore_fl = { 0x57, 0x9d }, // push %rdi; popfq
.cpu_wbinvd = { 0x0f, 0x09 }, // wbinvd
0x48, 0x0f, 0x07 }, // swapgs; sysretq
.cpu_swapgs = { 0x0f, 0x01, 0xf8 }, // swapgs
.mov64 = { 0x48, 0x89, 0xf8 }, // mov %rdi, %rax
-# else
- .mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3
- .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf
- .cpu_iret = { 0xcf }, // iret
-# endif
};
unsigned int paravirt_patch_ident_64(void *insn_buff, unsigned int len)
{
-#ifdef CONFIG_X86_64
return PATCH(xxl, mov64, insn_buff, len);
-#endif
- return 0;
}
# endif /* CONFIG_PARAVIRT_XXL */
PATCH_CASE(mmu, read_cr3, xxl, insn_buff, len);
PATCH_CASE(mmu, write_cr3, xxl, insn_buff, len);
-# ifdef CONFIG_X86_64
PATCH_CASE(cpu, usergs_sysret64, xxl, insn_buff, len);
PATCH_CASE(cpu, swapgs, xxl, insn_buff, len);
PATCH_CASE(cpu, wbinvd, xxl, insn_buff, len);
-# else
- PATCH_CASE(cpu, iret, xxl, insn_buff, len);
-# endif
#endif
#ifdef CONFIG_PARAVIRT_SPINLOCKS
{
unsigned long gsbase;
- if (static_cpu_has(X86_FEATURE_FSGSBASE)) {
+ if (boot_cpu_has(X86_FEATURE_FSGSBASE)) {
unsigned long flags;
local_irq_save(flags);
void x86_gsbase_write_cpu_inactive(unsigned long gsbase)
{
- if (static_cpu_has(X86_FEATURE_FSGSBASE)) {
+ if (boot_cpu_has(X86_FEATURE_FSGSBASE)) {
unsigned long flags;
local_irq_save(flags);
if (task == current)
fsbase = x86_fsbase_read_cpu();
- else if (static_cpu_has(X86_FEATURE_FSGSBASE) ||
+ else if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||
(task->thread.fsindex == 0))
fsbase = task->thread.fsbase;
else
if (task == current)
gsbase = x86_gsbase_read_cpu_inactive();
- else if (static_cpu_has(X86_FEATURE_FSGSBASE) ||
+ else if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||
(task->thread.gsindex == 0))
gsbase = task->thread.gsbase;
else
break;
}
- thread->debugreg6 |= (DR_TRAP0 << i);
+ thread->virtual_dr6 |= (DR_TRAP0 << i);
}
/*
if (bp)
val = bp->hw.info.address;
} else if (n == 6) {
- val = thread->debugreg6;
+ val = thread->virtual_dr6 ^ DR6_RESERVED; /* Flip back to arch polarity */
} else if (n == 7) {
val = thread->ptrace_dr7;
}
if (n < HBP_NUM) {
rc = ptrace_set_breakpoint_addr(tsk, n, val);
} else if (n == 6) {
- thread->debugreg6 = val;
+ thread->virtual_dr6 = val ^ DR6_RESERVED; /* Flip to positive polarity */
rc = 0;
} else if (n == 7) {
rc = ptrace_write_dr7(tsk, val);
#include <asm/hpet.h>
#include <asm/setup.h>
+#include <asm/mce.h>
#if defined(CONFIG_X86_IO_APIC) && defined(CONFIG_SMP) && defined(CONFIG_PCI)
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_NB_F3,
amd_disable_seq_and_redirect_scrub);
-#if defined(CONFIG_X86_64) && defined(CONFIG_X86_MCE)
-#include <linux/jump_label.h>
-#include <asm/string_64.h>
-
/* Ivy Bridge, Haswell, Broadwell */
static void quirk_intel_brickland_xeon_ras_cap(struct pci_dev *pdev)
{
pci_read_config_dword(pdev, 0x84, &capid0);
if (capid0 & 0x10)
- static_branch_inc(&mcsafe_key);
+ enable_copy_mc_fragile();
}
/* Skylake */
* enabled, so memory machine check recovery is also enabled.
*/
if ((capid0 & 0xc0) == 0xc0 || (capid5 & 0x1e0))
- static_branch_inc(&mcsafe_key);
+ enable_copy_mc_fragile();
}
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x0ec3, quirk_intel_brickland_xeon_ras_cap);
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x6fc0, quirk_intel_brickland_xeon_ras_cap);
DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2083, quirk_intel_purley_xeon_ras_cap);
#endif
-#endif
bool x86_apple_machine;
EXPORT_SYMBOL(x86_apple_machine);
#include <linux/hugetlb.h>
#include <linux/tboot.h>
#include <linux/usb/xhci-dbgp.h>
+#include <linux/static_call.h>
#include <uapi/linux/mount.h>
early_cpu_init();
arch_init_ideal_nops();
jump_label_init();
+ static_call_init();
early_ioremap_init();
setup_olpc_ofw_pgd();
efi_fake_memmap();
efi_find_mirror();
efi_esrt_init();
+ efi_mokvar_table_init();
/*
* The EFI specification says that boot service code won't be
*/
BUILD_BUG_ON(NSIGILL != 11);
BUILD_BUG_ON(NSIGFPE != 15);
- BUILD_BUG_ON(NSIGSEGV != 7);
+ BUILD_BUG_ON(NSIGSEGV != 9);
BUILD_BUG_ON(NSIGBUS != 5);
BUILD_BUG_ON(NSIGTRAP != 5);
BUILD_BUG_ON(NSIGCHLD != 6);
struct unwind_state state;
unsigned long addr;
- if (regs && !consume_entry(cookie, regs->ip, false))
+ if (regs && !consume_entry(cookie, regs->ip))
return;
for (unwind_start(&state, task, regs, NULL); !unwind_done(&state);
unwind_next_frame(&state)) {
addr = unwind_get_return_address(&state);
- if (!addr || !consume_entry(cookie, addr, false))
+ if (!addr || !consume_entry(cookie, addr))
break;
}
}
if (!addr)
return -EINVAL;
- if (!consume_entry(cookie, addr, false))
+ if (!consume_entry(cookie, addr))
return -EINVAL;
}
{
const void __user *fp = (const void __user *)regs->bp;
- if (!consume_entry(cookie, regs->ip, false))
+ if (!consume_entry(cookie, regs->ip))
return;
while (1) {
break;
if (!frame.ret_addr)
break;
- if (!consume_entry(cookie, frame.ret_addr, false))
+ if (!consume_entry(cookie, frame.ret_addr))
break;
fp = frame.next_fp;
}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/static_call.h>
+#include <linux/memory.h>
+#include <linux/bug.h>
+#include <asm/text-patching.h>
+
+enum insn_type {
+ CALL = 0, /* site call */
+ NOP = 1, /* site cond-call */
+ JMP = 2, /* tramp / site tail-call */
+ RET = 3, /* tramp / site cond-tail-call */
+};
+
+static void __ref __static_call_transform(void *insn, enum insn_type type, void *func)
+{
+ int size = CALL_INSN_SIZE;
+ const void *code;
+
+ switch (type) {
+ case CALL:
+ code = text_gen_insn(CALL_INSN_OPCODE, insn, func);
+ break;
+
+ case NOP:
+ code = ideal_nops[NOP_ATOMIC5];
+ break;
+
+ case JMP:
+ code = text_gen_insn(JMP32_INSN_OPCODE, insn, func);
+ break;
+
+ case RET:
+ code = text_gen_insn(RET_INSN_OPCODE, insn, func);
+ size = RET_INSN_SIZE;
+ break;
+ }
+
+ if (memcmp(insn, code, size) == 0)
+ return;
+
+ if (unlikely(system_state == SYSTEM_BOOTING))
+ return text_poke_early(insn, code, size);
+
+ text_poke_bp(insn, code, size, NULL);
+}
+
+static void __static_call_validate(void *insn, bool tail)
+{
+ u8 opcode = *(u8 *)insn;
+
+ if (tail) {
+ if (opcode == JMP32_INSN_OPCODE ||
+ opcode == RET_INSN_OPCODE)
+ return;
+ } else {
+ if (opcode == CALL_INSN_OPCODE ||
+ !memcmp(insn, ideal_nops[NOP_ATOMIC5], 5))
+ return;
+ }
+
+ /*
+ * If we ever trigger this, our text is corrupt, we'll probably not live long.
+ */
+ WARN_ONCE(1, "unexpected static_call insn opcode 0x%x at %pS\n", opcode, insn);
+}
+
+static inline enum insn_type __sc_insn(bool null, bool tail)
+{
+ /*
+ * Encode the following table without branches:
+ *
+ * tail null insn
+ * -----+-------+------
+ * 0 | 0 | CALL
+ * 0 | 1 | NOP
+ * 1 | 0 | JMP
+ * 1 | 1 | RET
+ */
+ return 2*tail + null;
+}
+
+void arch_static_call_transform(void *site, void *tramp, void *func, bool tail)
+{
+ mutex_lock(&text_mutex);
+
+ if (tramp) {
+ __static_call_validate(tramp, true);
+ __static_call_transform(tramp, __sc_insn(!func, true), func);
+ }
+
+ if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site) {
+ __static_call_validate(site, tail);
+ __static_call_transform(site, __sc_insn(!func, tail), func);
+ }
+
+ mutex_unlock(&text_mutex);
+}
+EXPORT_SYMBOL_GPL(arch_static_call_transform);
DEFINE_IDTENTRY(exc_divide_error)
{
- do_error_trap(regs, 0, "divide_error", X86_TRAP_DE, SIGFPE,
+ do_error_trap(regs, 0, "divide error", X86_TRAP_DE, SIGFPE,
FPE_INTDIV, error_get_trap_addr(regs));
}
* Keep it simple: clear DR6 immediately.
*/
get_debugreg(dr6, 6);
- set_debugreg(0, 6);
- /* Filter out all the reserved bits which are preset to 1 */
- dr6 &= ~DR6_RESERVED;
+ set_debugreg(DR6_RESERVED, 6);
+ dr6 ^= DR6_RESERVED; /* Flip to positive polarity */
+
+ /*
+ * Clear the virtual DR6 value, ptrace routines will set bits here for
+ * things we want signals for.
+ */
+ current->thread.virtual_dr6 = 0;
+
+ /*
+ * The SDM says "The processor clears the BTF flag when it
+ * generates a debug exception." Clear TIF_BLOCKSTEP to keep
+ * TIF_BLOCKSTEP in sync with the hardware BTF flag.
+ */
+ clear_thread_flag(TIF_BLOCKSTEP);
return dr6;
}
*
* May run on IST stack.
*/
-static void handle_debug(struct pt_regs *regs, unsigned long dr6, bool user)
-{
- struct task_struct *tsk = current;
- bool user_icebp;
- int si_code;
-
- /*
- * The SDM says "The processor clears the BTF flag when it
- * generates a debug exception." Clear TIF_BLOCKSTEP to keep
- * TIF_BLOCKSTEP in sync with the hardware BTF flag.
- */
- clear_thread_flag(TIF_BLOCKSTEP);
-
- /*
- * If DR6 is zero, no point in trying to handle it. The kernel is
- * not using INT1.
- */
- if (!user && !dr6)
- return;
+static bool notify_debug(struct pt_regs *regs, unsigned long *dr6)
+{
/*
- * If dr6 has no reason to give us about the origin of this trap,
- * then it's very likely the result of an icebp/int01 trap.
- * User wants a sigtrap for that.
+ * Notifiers will clear bits in @dr6 to indicate the event has been
+ * consumed - hw_breakpoint_handler(), single_stop_cont().
+ *
+ * Notifiers will set bits in @virtual_dr6 to indicate the desire
+ * for signals - ptrace_triggered(), kgdb_hw_overflow_handler().
*/
- user_icebp = user && !dr6;
-
- /* Store the virtualized DR6 value */
- tsk->thread.debugreg6 = dr6;
-
-#ifdef CONFIG_KPROBES
- if (kprobe_debug_handler(regs)) {
- return;
- }
-#endif
-
- if (notify_die(DIE_DEBUG, "debug", regs, (long)&dr6, 0,
- SIGTRAP) == NOTIFY_STOP) {
- return;
- }
-
- /* It's safe to allow irq's after DR6 has been saved */
- cond_local_irq_enable(regs);
-
- if (v8086_mode(regs)) {
- handle_vm86_trap((struct kernel_vm86_regs *) regs, 0,
- X86_TRAP_DB);
- goto out;
- }
-
- if (WARN_ON_ONCE((dr6 & DR_STEP) && !user_mode(regs))) {
- /*
- * Historical junk that used to handle SYSENTER single-stepping.
- * This should be unreachable now. If we survive for a while
- * without anyone hitting this warning, we'll turn this into
- * an oops.
- */
- tsk->thread.debugreg6 &= ~DR_STEP;
- set_tsk_thread_flag(tsk, TIF_SINGLESTEP);
- regs->flags &= ~X86_EFLAGS_TF;
- }
-
- si_code = get_si_code(tsk->thread.debugreg6);
- if (tsk->thread.debugreg6 & (DR_STEP | DR_TRAP_BITS) || user_icebp)
- send_sigtrap(regs, 0, si_code);
+ if (notify_die(DIE_DEBUG, "debug", regs, (long)dr6, 0, SIGTRAP) == NOTIFY_STOP)
+ return true;
-out:
- cond_local_irq_disable(regs);
+ return false;
}
static __always_inline void exc_debug_kernel(struct pt_regs *regs,
if ((dr6 & DR_STEP) && is_sysenter_singlestep(regs))
dr6 &= ~DR_STEP;
- handle_debug(regs, dr6, false);
+ if (kprobe_debug_handler(regs))
+ goto out;
+
+ /*
+ * The kernel doesn't use INT1
+ */
+ if (!dr6)
+ goto out;
+ if (notify_debug(regs, &dr6))
+ goto out;
+
+ /*
+ * The kernel doesn't use TF single-step outside of:
+ *
+ * - Kprobes, consumed through kprobe_debug_handler()
+ * - KGDB, consumed through notify_debug()
+ *
+ * So if we get here with DR_STEP set, something is wonky.
+ *
+ * A known way to trigger this is through QEMU's GDB stub,
+ * which leaks #DB into the guest and causes IST recursion.
+ */
+ if (WARN_ON_ONCE(dr6 & DR_STEP))
+ regs->flags &= ~X86_EFLAGS_TF;
+out:
instrumentation_end();
idtentry_exit_nmi(regs, irq_state);
static __always_inline void exc_debug_user(struct pt_regs *regs,
unsigned long dr6)
{
+ bool icebp;
+
/*
* If something gets miswired and we end up here for a kernel mode
* #DB, we will malfunction.
irqentry_enter_from_user_mode(regs);
instrumentation_begin();
- handle_debug(regs, dr6, true);
+ /*
+ * If dr6 has no reason to give us about the origin of this trap,
+ * then it's very likely the result of an icebp/int01 trap.
+ * User wants a sigtrap for that.
+ */
+ icebp = !dr6;
+ if (notify_debug(regs, &dr6))
+ goto out;
+
+ /* It's safe to allow irq's after DR6 has been saved */
+ local_irq_enable();
+
+ if (v8086_mode(regs)) {
+ handle_vm86_trap((struct kernel_vm86_regs *)regs, 0, X86_TRAP_DB);
+ goto out_irq;
+ }
+
+ /* Add the virtual_dr6 bits for signals. */
+ dr6 |= current->thread.virtual_dr6;
+ if (dr6 & (DR_STEP | DR_TRAP_BITS) || icebp)
+ send_sigtrap(regs, 0, get_si_code(dr6));
+
+out_irq:
+ local_irq_disable();
+out:
instrumentation_end();
irqentry_exit_to_user_mode(regs);
}
struct cyc2ns {
struct cyc2ns_data data[2]; /* 0 + 2*16 = 32 */
- seqcount_t seq; /* 32 + 4 = 36 */
+ seqcount_latch_t seq; /* 32 + 4 = 36 */
}; /* fits one cacheline */
preempt_disable_notrace();
do {
- seq = this_cpu_read(cyc2ns.seq.sequence);
+ seq = this_cpu_read(cyc2ns.seq.seqcount.sequence);
idx = seq & 1;
data->cyc2ns_offset = this_cpu_read(cyc2ns.data[idx].cyc2ns_offset);
data->cyc2ns_mul = this_cpu_read(cyc2ns.data[idx].cyc2ns_mul);
data->cyc2ns_shift = this_cpu_read(cyc2ns.data[idx].cyc2ns_shift);
- } while (unlikely(seq != this_cpu_read(cyc2ns.seq.sequence)));
+ } while (unlikely(seq != this_cpu_read(cyc2ns.seq.seqcount.sequence)));
}
__always_inline void cyc2ns_read_end(void)
{
struct cyc2ns *c2n = this_cpu_ptr(&cyc2ns);
- seqcount_init(&c2n->seq);
+ seqcount_latch_init(&c2n->seq);
__set_cyc2ns_scale(tsc_khz, smp_processor_id(), rdtsc());
}
for_each_possible_cpu(cpu) {
if (cpu != this_cpu) {
- seqcount_init(&c2n->seq);
+ seqcount_latch_init(&c2n->seq);
c2n = per_cpu_ptr(&cyc2ns, cpu);
c2n->data[0] = data[0];
c2n->data[1] = data[1];
* value that, lies close to the top of the kernel memory. The limit for the GDT
* and the IDT are set to zero.
*
- * Given that SLDT and STR are not commonly used in programs that run on WineHQ
- * or DOSEMU2, they are not emulated.
- *
- * The instruction smsw is emulated to return the value that the register CR0
+ * The instruction SMSW is emulated to return the value that the register CR0
* has at boot time as set in the head_32.
+ * SLDT and STR are emulated to return the values that the kernel programmatically
+ * assigns:
+ * - SLDT returns (GDT_ENTRY_LDT * 8) if an LDT has been set, 0 if not.
+ * - STR returns (GDT_ENTRY_TSS * 8).
*
* Emulation is provided for both 32-bit and 64-bit processes.
*
*data_size += UMIP_GDT_IDT_LIMIT_SIZE;
memcpy(data, &dummy_limit, UMIP_GDT_IDT_LIMIT_SIZE);
- } else if (umip_inst == UMIP_INST_SMSW) {
- unsigned long dummy_value = CR0_STATE;
+ } else if (umip_inst == UMIP_INST_SMSW || umip_inst == UMIP_INST_SLDT ||
+ umip_inst == UMIP_INST_STR) {
+ unsigned long dummy_value;
+
+ if (umip_inst == UMIP_INST_SMSW) {
+ dummy_value = CR0_STATE;
+ } else if (umip_inst == UMIP_INST_STR) {
+ dummy_value = GDT_ENTRY_TSS * 8;
+ } else if (umip_inst == UMIP_INST_SLDT) {
+#ifdef CONFIG_MODIFY_LDT_SYSCALL
+ down_read(¤t->mm->context.ldt_usr_sem);
+ if (current->mm->context.ldt)
+ dummy_value = GDT_ENTRY_LDT * 8;
+ else
+ dummy_value = 0;
+ up_read(¤t->mm->context.ldt_usr_sem);
+#else
+ dummy_value = 0;
+#endif
+ }
/*
- * Even though the CR0 register has 4 bytes, the number
+ * For these 3 instructions, the number
* of bytes to be copied in the result buffer is determined
* by whether the operand is a register or a memory location.
* If operand is a register, return as many bytes as the operand
* size. If operand is memory, return only the two least
- * siginificant bytes of CR0.
+ * siginificant bytes.
*/
if (X86_MODRM_MOD(insn->modrm.value) == 3)
*data_size = insn->opnd_bytes;
*data_size = 2;
memcpy(data, &dummy_value, *data_size);
- /* STR and SLDT are not emulated */
} else {
return -EINVAL;
}
umip_pr_warn(regs, "%s instruction cannot be used by applications.\n",
umip_insns[umip_inst]);
- /* Do not emulate (spoof) SLDT or STR. */
- if (umip_inst == UMIP_INST_STR || umip_inst == UMIP_INST_SLDT)
- return false;
-
umip_pr_warn(regs, "For now, expensive software emulation returns the result.\n");
if (emulate_umip_insn(&insn, umip_inst, dummy_data, &dummy_data_size,
ENTRY_TEXT
ALIGN_ENTRY_TEXT_END
SOFTIRQENTRY_TEXT
+ STATIC_CALL_TEXT
*(.fixup)
*(.gnu.warning)
STABS_DEBUG
DWARF_DEBUG
+ ELF_DETAILS
DISCARDS
-}
+ /*
+ * Make sure that the .got.plt is either completely empty or it
+ * contains only the lazy dispatch entries.
+ */
+ .got.plt (INFO) : { *(.got.plt) }
+ ASSERT(SIZEOF(.got.plt) == 0 ||
+#ifdef CONFIG_X86_64
+ SIZEOF(.got.plt) == 0x18,
+#else
+ SIZEOF(.got.plt) == 0xc,
+#endif
+ "Unexpected GOT/PLT entries detected!")
+
+ /*
+ * Sections that should stay zero sized, which is safer to
+ * explicitly check instead of blindly discarding.
+ */
+ .got : {
+ *(.got) *(.igot.*)
+ }
+ ASSERT(SIZEOF(.got) == 0, "Unexpected GOT entries detected!")
+
+ .plt : {
+ *(.plt) *(.plt.*) *(.iplt)
+ }
+ ASSERT(SIZEOF(.plt) == 0, "Unexpected run-time procedure linkages detected!")
+
+ .rel.dyn : {
+ *(.rel.*) *(.rel_*)
+ }
+ ASSERT(SIZEOF(.rel.dyn) == 0, "Unexpected run-time relocations (.rel) detected!")
+
+ .rela.dyn : {
+ *(.rela.*) *(.rela_*)
+ }
+ ASSERT(SIZEOF(.rela.dyn) == 0, "Unexpected run-time relocations (.rela) detected!")
+}
#ifdef CONFIG_X86_32
/*
#include <asm/tsc.h>
#include <asm/iommu.h>
#include <asm/mach_traps.h>
+#include <asm/irqdomain.h>
void x86_init_noop(void) { }
void __init x86_init_uint_noop(unsigned int unused) { }
},
.mpparse = {
- .mpc_record = x86_init_uint_noop,
.setup_ioapic_ids = x86_init_noop,
- .mpc_apic_id = default_mpc_apic_id,
- .smp_read_mpc_oem = default_smp_read_mpc_oem,
- .mpc_oem_bus_info = default_mpc_oem_bus_info,
.find_smp_config = default_find_smp_config,
.get_smp_config = default_get_smp_config,
},
.pre_vector_init = init_ISA_irqs,
.intr_init = native_init_IRQ,
.intr_mode_select = apic_intr_mode_select,
- .intr_mode_init = apic_intr_mode_init
+ .intr_mode_init = apic_intr_mode_init,
+ .create_pci_msi_domain = native_create_pci_msi_domain,
},
.oem = {
#if defined(CONFIG_PCI_MSI)
struct x86_msi_ops x86_msi __ro_after_init = {
- .setup_msi_irqs = native_setup_msi_irqs,
- .teardown_msi_irq = native_teardown_msi_irq,
- .teardown_msi_irqs = default_teardown_msi_irqs,
.restore_msi_irqs = default_restore_msi_irqs,
};
/* MSI arch specific hooks */
-int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
-{
- return x86_msi.setup_msi_irqs(dev, nvec, type);
-}
-
-void arch_teardown_msi_irqs(struct pci_dev *dev)
-{
- x86_msi.teardown_msi_irqs(dev);
-}
-
-void arch_teardown_msi_irq(unsigned int irq)
-{
- x86_msi.teardown_msi_irq(irq);
-}
-
void arch_restore_msi_irqs(struct pci_dev *dev)
{
x86_msi.restore_msi_irqs(dev);
F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) |
F(SPEC_CTRL_SSBD) | F(ARCH_CAPABILITIES) | F(INTEL_STIBP) |
F(MD_CLEAR) | F(AVX512_VP2INTERSECT) | F(FSRM) |
- F(SERIALIZE)
+ F(SERIALIZE) | F(TSXLDTRK)
);
/* TSC_ADJUST and ARCH_CAPABILITIES are emulated in software. */
break;
case HYPERV_CPUID_FEATURES:
- ent->eax |= HV_X64_MSR_VP_RUNTIME_AVAILABLE;
+ ent->eax |= HV_MSR_VP_RUNTIME_AVAILABLE;
ent->eax |= HV_MSR_TIME_REF_COUNT_AVAILABLE;
- ent->eax |= HV_X64_MSR_SYNIC_AVAILABLE;
+ ent->eax |= HV_MSR_SYNIC_AVAILABLE;
ent->eax |= HV_MSR_SYNTIMER_AVAILABLE;
- ent->eax |= HV_X64_MSR_APIC_ACCESS_AVAILABLE;
- ent->eax |= HV_X64_MSR_HYPERCALL_AVAILABLE;
- ent->eax |= HV_X64_MSR_VP_INDEX_AVAILABLE;
- ent->eax |= HV_X64_MSR_RESET_AVAILABLE;
+ ent->eax |= HV_MSR_APIC_ACCESS_AVAILABLE;
+ ent->eax |= HV_MSR_HYPERCALL_AVAILABLE;
+ ent->eax |= HV_MSR_VP_INDEX_AVAILABLE;
+ ent->eax |= HV_MSR_RESET_AVAILABLE;
ent->eax |= HV_MSR_REFERENCE_TSC_AVAILABLE;
- ent->eax |= HV_X64_ACCESS_FREQUENCY_MSRS;
- ent->eax |= HV_X64_ACCESS_REENLIGHTENMENT;
+ ent->eax |= HV_ACCESS_FREQUENCY_MSRS;
+ ent->eax |= HV_ACCESS_REENLIGHTENMENT;
- ent->ebx |= HV_X64_POST_MESSAGES;
- ent->ebx |= HV_X64_SIGNAL_EVENTS;
+ ent->ebx |= HV_POST_MESSAGES;
+ ent->ebx |= HV_SIGNAL_EVENTS;
ent->edx |= HV_FEATURE_FREQUENCY_MSRS_AVAILABLE;
ent->edx |= HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE;
uint8_t *page_virtual;
unsigned long i;
- if (npages == 0 || pages == NULL)
+ if (this_cpu_has(X86_FEATURE_SME_COHERENT) || npages == 0 ||
+ pages == NULL)
return;
for (i = 0; i < npages; i++) {
return 1;
}
+static int invd_interception(struct vcpu_svm *svm)
+{
+ /* Treat an INVD instruction as a NOP and just skip it. */
+ return kvm_skip_emulated_instruction(&svm->vcpu);
+}
+
static int invlpg_interception(struct vcpu_svm *svm)
{
if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
[SVM_EXIT_RDPMC] = rdpmc_interception,
[SVM_EXIT_CPUID] = cpuid_interception,
[SVM_EXIT_IRET] = iret_interception,
- [SVM_EXIT_INVD] = emulate_on_interception,
+ [SVM_EXIT_INVD] = invd_interception,
[SVM_EXIT_PAUSE] = pause_interception,
[SVM_EXIT_HLT] = halt_interception,
[SVM_EXIT_INVLPG] = invlpg_interception,
module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
#endif
+extern bool __read_mostly allow_smaller_maxphyaddr;
+module_param(allow_smaller_maxphyaddr, bool, S_IRUGO);
+
#define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD)
#define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE
#define KVM_VM_CR0_ALWAYS_ON \
*/
if (is_guest_mode(vcpu))
eb |= get_vmcs12(vcpu)->exception_bitmap;
+ else {
+ /*
+ * If EPT is enabled, #PF is only trapped if MAXPHYADDR is mismatched
+ * between guest and host. In that case we only care about present
+ * faults. For vmcs02, however, PFEC_MASK and PFEC_MATCH are set in
+ * prepare_vmcs02_rare.
+ */
+ bool selective_pf_trap = enable_ept && (eb & (1u << PF_VECTOR));
+ int mask = selective_pf_trap ? PFERR_PRESENT_MASK : 0;
+ vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask);
+ vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, mask);
+ }
vmcs_write32(EXCEPTION_BITMAP, eb);
}
vmx->pt_desc.guest.output_mask = 0x7F;
vmcs_write64(GUEST_IA32_RTIT_CTL, 0);
}
-
- /*
- * If EPT is enabled, #PF is only trapped if MAXPHYADDR is mismatched
- * between guest and host. In that case we only care about present
- * faults.
- */
- if (enable_ept) {
- vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, PFERR_PRESENT_MASK);
- vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, PFERR_PRESENT_MASK);
- }
}
static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
* EPT will cause page fault only if we need to
* detect illegal GPAs.
*/
+ WARN_ON_ONCE(!allow_smaller_maxphyaddr);
kvm_fixup_and_inject_pf_error(vcpu, cr2, error_code);
return 1;
} else
* would also use advanced VM-exit information for EPT violations to
* reconstruct the page fault error code.
*/
- if (unlikely(kvm_mmu_is_illegal_gpa(vcpu, gpa)))
+ if (unlikely(allow_smaller_maxphyaddr && kvm_mmu_is_illegal_gpa(vcpu, gpa)))
return kvm_emulate_instruction(vcpu, 0);
return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
vmx_check_vmcs12_offsets();
/*
- * Intel processors don't have problems with
- * GUEST_MAXPHYADDR < HOST_MAXPHYADDR so enable
- * it for VMX by default
+ * Shadow paging doesn't have a (further) performance penalty
+ * from GUEST_MAXPHYADDR < HOST_MAXPHYADDR so enable it
+ * by default
*/
- allow_smaller_maxphyaddr = true;
+ if (!enable_ept)
+ allow_smaller_maxphyaddr = true;
return 0;
}
static inline bool vmx_need_pf_intercept(struct kvm_vcpu *vcpu)
{
- return !enable_ept || cpuid_maxphyaddr(vcpu) < boot_cpu_data.x86_phys_bits;
+ if (!enable_ept)
+ return true;
+
+ return allow_smaller_maxphyaddr && cpuid_maxphyaddr(vcpu) < boot_cpu_data.x86_phys_bits;
}
void dump_vmcs(void);
u64 __read_mostly host_efer;
EXPORT_SYMBOL_GPL(host_efer);
-bool __read_mostly allow_smaller_maxphyaddr;
+bool __read_mostly allow_smaller_maxphyaddr = 0;
EXPORT_SYMBOL_GPL(allow_smaller_maxphyaddr);
static u64 __read_mostly host_xss;
unsigned long old_cr4 = kvm_read_cr4(vcpu);
unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE |
X86_CR4_SMEP;
+ unsigned long mmu_role_bits = pdptr_bits | X86_CR4_SMAP | X86_CR4_PKE;
if (kvm_valid_cr4(vcpu, cr4))
return 1;
if (kvm_x86_ops.set_cr4(vcpu, cr4))
return 1;
- if (((cr4 ^ old_cr4) & pdptr_bits) ||
+ if (((cr4 ^ old_cr4) & mmu_role_bits) ||
(!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
kvm_mmu_reset_context(vcpu);
case MSR_IA32_POWER_CTL:
msr_info->data = vcpu->arch.msr_ia32_power_ctl;
break;
- case MSR_IA32_TSC:
- msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + vcpu->arch.tsc_offset;
+ case MSR_IA32_TSC: {
+ /*
+ * Intel SDM states that MSR_IA32_TSC read adds the TSC offset
+ * even when not intercepted. AMD manual doesn't explicitly
+ * state this but appears to behave the same.
+ *
+ * On userspace reads and writes, however, we unconditionally
+ * operate L1's TSC value to ensure backwards-compatible
+ * behavior for migration.
+ */
+ u64 tsc_offset = msr_info->host_initiated ? vcpu->arch.l1_tsc_offset :
+ vcpu->arch.tsc_offset;
+
+ msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + tsc_offset;
break;
+ }
case MSR_MTRRcap:
case 0x200 ... 0x2ff:
return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
lib-y := delay.o misc.o cmdline.o cpu.o
lib-y += usercopy_$(BITS).o usercopy.o getuser.o putuser.o
lib-y += memcpy_$(BITS).o
+lib-$(CONFIG_ARCH_HAS_COPY_MC) += copy_mc.o copy_mc_64.o
lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o
lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
/*
unsigned int csum_partial_copy_generic (const char *src, char *dst,
- int len, int sum, int *src_err_ptr, int *dst_err_ptr)
+ int len)
*/
/*
* Copy from ds while checksumming, otherwise like csum_partial
- *
- * The macros SRC and DST specify the type of access for the instruction.
- * thus we can call a custom exception handler for all access types.
- *
- * FIXME: could someone double-check whether I haven't mixed up some SRC and
- * DST definitions? It's damn hard to trigger all cases. I hope I got
- * them all but there's no guarantee.
*/
-#define SRC(y...) \
+#define EXC(y...) \
9999: y; \
_ASM_EXTABLE_UA(9999b, 6001f)
-#define DST(y...) \
- 9999: y; \
- _ASM_EXTABLE_UA(9999b, 6002f)
-
#ifndef CONFIG_X86_USE_PPRO_CHECKSUM
#define ARGBASE 16
pushl %edi
pushl %esi
pushl %ebx
- movl ARGBASE+16(%esp),%eax # sum
movl ARGBASE+12(%esp),%ecx # len
movl ARGBASE+4(%esp),%esi # src
movl ARGBASE+8(%esp),%edi # dst
+ movl $-1, %eax # sum
testl $2, %edi # Check alignment.
jz 2f # Jump if alignment is ok.
subl $2, %ecx # Alignment uses up two bytes.
jae 1f # Jump if we had at least two bytes.
addl $2, %ecx # ecx was < 2. Deal with it.
jmp 4f
-SRC(1: movw (%esi), %bx )
+EXC(1: movw (%esi), %bx )
addl $2, %esi
-DST( movw %bx, (%edi) )
+EXC( movw %bx, (%edi) )
addl $2, %edi
addw %bx, %ax
adcl $0, %eax
movl %ecx, FP(%esp)
shrl $5, %ecx
jz 2f
- testl %esi, %esi
-SRC(1: movl (%esi), %ebx )
-SRC( movl 4(%esi), %edx )
+ testl %esi, %esi # what's wrong with clc?
+EXC(1: movl (%esi), %ebx )
+EXC( movl 4(%esi), %edx )
adcl %ebx, %eax
-DST( movl %ebx, (%edi) )
+EXC( movl %ebx, (%edi) )
adcl %edx, %eax
-DST( movl %edx, 4(%edi) )
+EXC( movl %edx, 4(%edi) )
-SRC( movl 8(%esi), %ebx )
-SRC( movl 12(%esi), %edx )
+EXC( movl 8(%esi), %ebx )
+EXC( movl 12(%esi), %edx )
adcl %ebx, %eax
-DST( movl %ebx, 8(%edi) )
+EXC( movl %ebx, 8(%edi) )
adcl %edx, %eax
-DST( movl %edx, 12(%edi) )
+EXC( movl %edx, 12(%edi) )
-SRC( movl 16(%esi), %ebx )
-SRC( movl 20(%esi), %edx )
+EXC( movl 16(%esi), %ebx )
+EXC( movl 20(%esi), %edx )
adcl %ebx, %eax
-DST( movl %ebx, 16(%edi) )
+EXC( movl %ebx, 16(%edi) )
adcl %edx, %eax
-DST( movl %edx, 20(%edi) )
+EXC( movl %edx, 20(%edi) )
-SRC( movl 24(%esi), %ebx )
-SRC( movl 28(%esi), %edx )
+EXC( movl 24(%esi), %ebx )
+EXC( movl 28(%esi), %edx )
adcl %ebx, %eax
-DST( movl %ebx, 24(%edi) )
+EXC( movl %ebx, 24(%edi) )
adcl %edx, %eax
-DST( movl %edx, 28(%edi) )
+EXC( movl %edx, 28(%edi) )
lea 32(%esi), %esi
lea 32(%edi), %edi
andl $0x1c, %edx
je 4f
shrl $2, %edx # This clears CF
-SRC(3: movl (%esi), %ebx )
+EXC(3: movl (%esi), %ebx )
adcl %ebx, %eax
-DST( movl %ebx, (%edi) )
+EXC( movl %ebx, (%edi) )
lea 4(%esi), %esi
lea 4(%edi), %edi
dec %edx
jz 7f
cmpl $2, %ecx
jb 5f
-SRC( movw (%esi), %cx )
+EXC( movw (%esi), %cx )
leal 2(%esi), %esi
-DST( movw %cx, (%edi) )
+EXC( movw %cx, (%edi) )
leal 2(%edi), %edi
je 6f
shll $16,%ecx
-SRC(5: movb (%esi), %cl )
-DST( movb %cl, (%edi) )
+EXC(5: movb (%esi), %cl )
+EXC( movb %cl, (%edi) )
6: addl %ecx, %eax
adcl $0, %eax
7:
-5000:
# Exception handler:
.section .fixup, "ax"
6001:
- movl ARGBASE+20(%esp), %ebx # src_err_ptr
- movl $-EFAULT, (%ebx)
-
- # zero the complete destination - computing the rest
- # is too much work
- movl ARGBASE+8(%esp), %edi # dst
- movl ARGBASE+12(%esp), %ecx # len
- xorl %eax,%eax
- rep ; stosb
-
- jmp 5000b
-
-6002:
- movl ARGBASE+24(%esp), %ebx # dst_err_ptr
- movl $-EFAULT,(%ebx)
- jmp 5000b
+ xorl %eax, %eax
+ jmp 7b
.previous
/* Version for PentiumII/PPro */
#define ROUND1(x) \
- SRC(movl x(%esi), %ebx ) ; \
+ EXC(movl x(%esi), %ebx ) ; \
addl %ebx, %eax ; \
- DST(movl %ebx, x(%edi) ) ;
+ EXC(movl %ebx, x(%edi) ) ;
#define ROUND(x) \
- SRC(movl x(%esi), %ebx ) ; \
+ EXC(movl x(%esi), %ebx ) ; \
adcl %ebx, %eax ; \
- DST(movl %ebx, x(%edi) ) ;
+ EXC(movl %ebx, x(%edi) ) ;
#define ARGBASE 12
movl ARGBASE+4(%esp),%esi #src
movl ARGBASE+8(%esp),%edi #dst
movl ARGBASE+12(%esp),%ecx #len
- movl ARGBASE+16(%esp),%eax #sum
+ movl $-1, %eax #sum
# movl %ecx, %edx
movl %ecx, %ebx
movl %esi, %edx
JMP_NOSPEC ebx
1: addl $64,%esi
addl $64,%edi
- SRC(movb -32(%edx),%bl) ; SRC(movb (%edx),%bl)
+ EXC(movb -32(%edx),%bl) ; EXC(movb (%edx),%bl)
ROUND1(-64) ROUND(-60) ROUND(-56) ROUND(-52)
ROUND (-48) ROUND(-44) ROUND(-40) ROUND(-36)
ROUND (-32) ROUND(-28) ROUND(-24) ROUND(-20)
jz 7f
cmpl $2, %edx
jb 5f
-SRC( movw (%esi), %dx )
+EXC( movw (%esi), %dx )
leal 2(%esi), %esi
-DST( movw %dx, (%edi) )
+EXC( movw %dx, (%edi) )
leal 2(%edi), %edi
je 6f
shll $16,%edx
5:
-SRC( movb (%esi), %dl )
-DST( movb %dl, (%edi) )
+EXC( movb (%esi), %dl )
+EXC( movb %dl, (%edi) )
6: addl %edx, %eax
adcl $0, %eax
7:
.section .fixup, "ax"
-6001: movl ARGBASE+20(%esp), %ebx # src_err_ptr
- movl $-EFAULT, (%ebx)
- # zero the complete destination (computing the rest is too much work)
- movl ARGBASE+8(%esp),%edi # dst
- movl ARGBASE+12(%esp),%ecx # len
- xorl %eax,%eax
- rep; stosb
- jmp 7b
-6002: movl ARGBASE+24(%esp), %ebx # dst_err_ptr
- movl $-EFAULT, (%ebx)
+6001: xorl %eax, %eax
jmp 7b
.previous
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
+
+#include <linux/jump_label.h>
+#include <linux/uaccess.h>
+#include <linux/export.h>
+#include <linux/string.h>
+#include <linux/types.h>
+
+#include <asm/mce.h>
+
+#ifdef CONFIG_X86_MCE
+/*
+ * See COPY_MC_TEST for self-test of the copy_mc_fragile()
+ * implementation.
+ */
+static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key);
+
+void enable_copy_mc_fragile(void)
+{
+ static_branch_inc(©_mc_fragile_key);
+}
+#define copy_mc_fragile_enabled (static_branch_unlikely(©_mc_fragile_key))
+
+/*
+ * Similar to copy_user_handle_tail, probe for the write fault point, or
+ * source exception point.
+ */
+__visible notrace unsigned long
+copy_mc_fragile_handle_tail(char *to, char *from, unsigned len)
+{
+ for (; len; --len, to++, from++)
+ if (copy_mc_fragile(to, from, 1))
+ break;
+ return len;
+}
+#else
+/*
+ * No point in doing careful copying, or consulting a static key when
+ * there is no #MC handler in the CONFIG_X86_MCE=n case.
+ */
+void enable_copy_mc_fragile(void)
+{
+}
+#define copy_mc_fragile_enabled (0)
+#endif
+
+unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len);
+
+/**
+ * copy_mc_to_kernel - memory copy that handles source exceptions
+ *
+ * @dst: destination address
+ * @src: source address
+ * @len: number of bytes to copy
+ *
+ * Call into the 'fragile' version on systems that benefit from avoiding
+ * corner case poison consumption scenarios, For example, accessing
+ * poison across 2 cachelines with a single instruction. Almost all
+ * other uses case can use copy_mc_enhanced_fast_string() for a fast
+ * recoverable copy, or fallback to plain memcpy.
+ *
+ * Return 0 for success, or number of bytes not copied if there was an
+ * exception.
+ */
+unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
+{
+ if (copy_mc_fragile_enabled)
+ return copy_mc_fragile(dst, src, len);
+ if (static_cpu_has(X86_FEATURE_ERMS))
+ return copy_mc_enhanced_fast_string(dst, src, len);
+ memcpy(dst, src, len);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(copy_mc_to_kernel);
+
+unsigned long __must_check copy_mc_to_user(void *dst, const void *src, unsigned len)
+{
+ unsigned long ret;
+
+ if (copy_mc_fragile_enabled) {
+ __uaccess_begin();
+ ret = copy_mc_fragile(dst, src, len);
+ __uaccess_end();
+ return ret;
+ }
+
+ if (static_cpu_has(X86_FEATURE_ERMS)) {
+ __uaccess_begin();
+ ret = copy_mc_enhanced_fast_string(dst, src, len);
+ __uaccess_end();
+ return ret;
+ }
+
+ return copy_user_generic(dst, src, len);
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
+
+#include <linux/linkage.h>
+#include <asm/copy_mc_test.h>
+#include <asm/export.h>
+#include <asm/asm.h>
+
+#ifndef CONFIG_UML
+
+#ifdef CONFIG_X86_MCE
+COPY_MC_TEST_CTL
+
+/*
+ * copy_mc_fragile - copy memory with indication if an exception / fault happened
+ *
+ * The 'fragile' version is opted into by platform quirks and takes
+ * pains to avoid unrecoverable corner cases like 'fast-string'
+ * instruction sequences, and consuming poison across a cacheline
+ * boundary. The non-fragile version is equivalent to memcpy()
+ * regardless of CPU machine-check-recovery capability.
+ */
+SYM_FUNC_START(copy_mc_fragile)
+ cmpl $8, %edx
+ /* Less than 8 bytes? Go to byte copy loop */
+ jb .L_no_whole_words
+
+ /* Check for bad alignment of source */
+ testl $7, %esi
+ /* Already aligned */
+ jz .L_8byte_aligned
+
+ /* Copy one byte at a time until source is 8-byte aligned */
+ movl %esi, %ecx
+ andl $7, %ecx
+ subl $8, %ecx
+ negl %ecx
+ subl %ecx, %edx
+.L_read_leading_bytes:
+ movb (%rsi), %al
+ COPY_MC_TEST_SRC %rsi 1 .E_leading_bytes
+ COPY_MC_TEST_DST %rdi 1 .E_leading_bytes
+.L_write_leading_bytes:
+ movb %al, (%rdi)
+ incq %rsi
+ incq %rdi
+ decl %ecx
+ jnz .L_read_leading_bytes
+
+.L_8byte_aligned:
+ movl %edx, %ecx
+ andl $7, %edx
+ shrl $3, %ecx
+ jz .L_no_whole_words
+
+.L_read_words:
+ movq (%rsi), %r8
+ COPY_MC_TEST_SRC %rsi 8 .E_read_words
+ COPY_MC_TEST_DST %rdi 8 .E_write_words
+.L_write_words:
+ movq %r8, (%rdi)
+ addq $8, %rsi
+ addq $8, %rdi
+ decl %ecx
+ jnz .L_read_words
+
+ /* Any trailing bytes? */
+.L_no_whole_words:
+ andl %edx, %edx
+ jz .L_done_memcpy_trap
+
+ /* Copy trailing bytes */
+ movl %edx, %ecx
+.L_read_trailing_bytes:
+ movb (%rsi), %al
+ COPY_MC_TEST_SRC %rsi 1 .E_trailing_bytes
+ COPY_MC_TEST_DST %rdi 1 .E_trailing_bytes
+.L_write_trailing_bytes:
+ movb %al, (%rdi)
+ incq %rsi
+ incq %rdi
+ decl %ecx
+ jnz .L_read_trailing_bytes
+
+ /* Copy successful. Return zero */
+.L_done_memcpy_trap:
+ xorl %eax, %eax
+.L_done:
+ ret
+SYM_FUNC_END(copy_mc_fragile)
+EXPORT_SYMBOL_GPL(copy_mc_fragile)
+
+ .section .fixup, "ax"
+ /*
+ * Return number of bytes not copied for any failure. Note that
+ * there is no "tail" handling since the source buffer is 8-byte
+ * aligned and poison is cacheline aligned.
+ */
+.E_read_words:
+ shll $3, %ecx
+.E_leading_bytes:
+ addl %edx, %ecx
+.E_trailing_bytes:
+ mov %ecx, %eax
+ jmp .L_done
+
+ /*
+ * For write fault handling, given the destination is unaligned,
+ * we handle faults on multi-byte writes with a byte-by-byte
+ * copy up to the write-protected page.
+ */
+.E_write_words:
+ shll $3, %ecx
+ addl %edx, %ecx
+ movl %ecx, %edx
+ jmp copy_mc_fragile_handle_tail
+
+ .previous
+
+ _ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes)
+ _ASM_EXTABLE_FAULT(.L_read_words, .E_read_words)
+ _ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes)
+ _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)
+ _ASM_EXTABLE(.L_write_words, .E_write_words)
+ _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)
+#endif /* CONFIG_X86_MCE */
+
+/*
+ * copy_mc_enhanced_fast_string - memory copy with exception handling
+ *
+ * Fast string copy + fault / exception handling. If the CPU does
+ * support machine check exception recovery, but does not support
+ * recovering from fast-string exceptions then this CPU needs to be
+ * added to the copy_mc_fragile_key set of quirks. Otherwise, absent any
+ * machine check recovery support this version should be no slower than
+ * standard memcpy.
+ */
+SYM_FUNC_START(copy_mc_enhanced_fast_string)
+ movq %rdi, %rax
+ movq %rdx, %rcx
+.L_copy:
+ rep movsb
+ /* Copy successful. Return zero */
+ xorl %eax, %eax
+ ret
+SYM_FUNC_END(copy_mc_enhanced_fast_string)
+
+ .section .fixup, "ax"
+.E_copy:
+ /*
+ * On fault %rcx is updated such that the copy instruction could
+ * optionally be restarted at the fault position, i.e. it
+ * contains 'bytes remaining'. A non-zero return indicates error
+ * to copy_mc_generic() users, or indicate short transfers to
+ * user-copy routines.
+ */
+ movq %rcx, %rax
+ ret
+
+ .previous
+
+ _ASM_EXTABLE_FAULT(.L_copy, .E_copy)
+#endif /* !CONFIG_UML */
#include <asm/asm.h>
#include <asm/smap.h>
#include <asm/export.h>
+#include <asm/trapnr.h>
.macro ALIGN_DESTINATION
/* check for bad alignment of destination */
jmp .Lcopy_user_handle_tail
.previous
- _ASM_EXTABLE_UA(100b, 103b)
- _ASM_EXTABLE_UA(101b, 103b)
+ _ASM_EXTABLE_CPY(100b, 103b)
+ _ASM_EXTABLE_CPY(101b, 103b)
.endm
/*
60: jmp .Lcopy_user_handle_tail /* ecx is zerorest also */
.previous
- _ASM_EXTABLE_UA(1b, 30b)
- _ASM_EXTABLE_UA(2b, 30b)
- _ASM_EXTABLE_UA(3b, 30b)
- _ASM_EXTABLE_UA(4b, 30b)
- _ASM_EXTABLE_UA(5b, 30b)
- _ASM_EXTABLE_UA(6b, 30b)
- _ASM_EXTABLE_UA(7b, 30b)
- _ASM_EXTABLE_UA(8b, 30b)
- _ASM_EXTABLE_UA(9b, 30b)
- _ASM_EXTABLE_UA(10b, 30b)
- _ASM_EXTABLE_UA(11b, 30b)
- _ASM_EXTABLE_UA(12b, 30b)
- _ASM_EXTABLE_UA(13b, 30b)
- _ASM_EXTABLE_UA(14b, 30b)
- _ASM_EXTABLE_UA(15b, 30b)
- _ASM_EXTABLE_UA(16b, 30b)
- _ASM_EXTABLE_UA(18b, 40b)
- _ASM_EXTABLE_UA(19b, 40b)
- _ASM_EXTABLE_UA(21b, 50b)
- _ASM_EXTABLE_UA(22b, 50b)
+ _ASM_EXTABLE_CPY(1b, 30b)
+ _ASM_EXTABLE_CPY(2b, 30b)
+ _ASM_EXTABLE_CPY(3b, 30b)
+ _ASM_EXTABLE_CPY(4b, 30b)
+ _ASM_EXTABLE_CPY(5b, 30b)
+ _ASM_EXTABLE_CPY(6b, 30b)
+ _ASM_EXTABLE_CPY(7b, 30b)
+ _ASM_EXTABLE_CPY(8b, 30b)
+ _ASM_EXTABLE_CPY(9b, 30b)
+ _ASM_EXTABLE_CPY(10b, 30b)
+ _ASM_EXTABLE_CPY(11b, 30b)
+ _ASM_EXTABLE_CPY(12b, 30b)
+ _ASM_EXTABLE_CPY(13b, 30b)
+ _ASM_EXTABLE_CPY(14b, 30b)
+ _ASM_EXTABLE_CPY(15b, 30b)
+ _ASM_EXTABLE_CPY(16b, 30b)
+ _ASM_EXTABLE_CPY(18b, 40b)
+ _ASM_EXTABLE_CPY(19b, 40b)
+ _ASM_EXTABLE_CPY(21b, 50b)
+ _ASM_EXTABLE_CPY(22b, 50b)
SYM_FUNC_END(copy_user_generic_unrolled)
EXPORT_SYMBOL(copy_user_generic_unrolled)
jmp .Lcopy_user_handle_tail
.previous
- _ASM_EXTABLE_UA(1b, 11b)
- _ASM_EXTABLE_UA(3b, 12b)
+ _ASM_EXTABLE_CPY(1b, 11b)
+ _ASM_EXTABLE_CPY(3b, 12b)
SYM_FUNC_END(copy_user_generic_string)
EXPORT_SYMBOL(copy_user_generic_string)
jmp .Lcopy_user_handle_tail
.previous
- _ASM_EXTABLE_UA(1b, 12b)
+ _ASM_EXTABLE_CPY(1b, 12b)
SYM_FUNC_END(copy_user_enhanced_fast_string)
EXPORT_SYMBOL(copy_user_enhanced_fast_string)
* Try to copy last bytes and clear the rest if needed.
* Since protection fault in copy_from/to_user is not a normal situation,
* it is not necessary to optimize tail handling.
+ * Don't try to copy the tail if machine check happened
*
* Input:
* rdi destination
*/
SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail)
movl %edx,%ecx
+ cmp $X86_TRAP_MC,%eax /* check if X86_TRAP_MC */
+ je 3f
1: rep movsb
2: mov %ecx,%eax
ASM_CLAC
ret
- _ASM_EXTABLE_UA(1b, 2b)
+ /*
+ * Return zero to pretend that this copy succeeded. This
+ * is counter-intuitive, but needed to prevent the code
+ * in lib/iov_iter.c from retrying and running back into
+ * the poison cache line again. The machine check handler
+ * will ensure that a SIGBUS is sent to the task.
+ */
+3: xorl %eax,%eax
+ ASM_CLAC
+ ret
+
+ _ASM_EXTABLE_CPY(1b, 2b)
SYM_CODE_END(.Lcopy_user_handle_tail)
/*
jmp .Lcopy_user_handle_tail
.previous
- _ASM_EXTABLE_UA(1b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(2b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(3b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(4b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(5b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(6b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(7b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(8b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(9b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(10b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(11b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(12b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(13b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(14b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(15b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(16b, .L_fixup_4x8b_copy)
- _ASM_EXTABLE_UA(20b, .L_fixup_8b_copy)
- _ASM_EXTABLE_UA(21b, .L_fixup_8b_copy)
- _ASM_EXTABLE_UA(30b, .L_fixup_4b_copy)
- _ASM_EXTABLE_UA(31b, .L_fixup_4b_copy)
- _ASM_EXTABLE_UA(40b, .L_fixup_1b_copy)
- _ASM_EXTABLE_UA(41b, .L_fixup_1b_copy)
+ _ASM_EXTABLE_CPY(1b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(2b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(3b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(4b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(5b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(6b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(7b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(8b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(9b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(10b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(11b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(12b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(13b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(14b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(15b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(16b, .L_fixup_4x8b_copy)
+ _ASM_EXTABLE_CPY(20b, .L_fixup_8b_copy)
+ _ASM_EXTABLE_CPY(21b, .L_fixup_8b_copy)
+ _ASM_EXTABLE_CPY(30b, .L_fixup_4b_copy)
+ _ASM_EXTABLE_CPY(31b, .L_fixup_4b_copy)
+ _ASM_EXTABLE_CPY(40b, .L_fixup_1b_copy)
+ _ASM_EXTABLE_CPY(41b, .L_fixup_1b_copy)
SYM_FUNC_END(__copy_user_nocache)
EXPORT_SYMBOL(__copy_user_nocache)
* rdi source
* rsi destination
* edx len (32bit)
- * ecx sum (32bit)
- * r8 src_err_ptr (int)
- * r9 dst_err_ptr (int)
*
* Output
* eax 64bit sum. undefined in case of exception.
.macro source
10:
- _ASM_EXTABLE_UA(10b, .Lbad_source)
+ _ASM_EXTABLE_UA(10b, .Lfault)
.endm
.macro dest
20:
- _ASM_EXTABLE_UA(20b, .Lbad_dest)
+ _ASM_EXTABLE_UA(20b, .Lfault)
.endm
- /*
- * No _ASM_EXTABLE_UA; this is used for intentional prefetch on a
- * potentially unmapped kernel address.
- */
- .macro ignore L=.Lignore
-30:
- _ASM_EXTABLE(30b, \L)
- .endm
-
-
SYM_FUNC_START(csum_partial_copy_generic)
- cmpl $3*64, %edx
- jle .Lignore
-
-.Lignore:
- subq $7*8, %rsp
- movq %rbx, 2*8(%rsp)
- movq %r12, 3*8(%rsp)
- movq %r14, 4*8(%rsp)
- movq %r13, 5*8(%rsp)
- movq %r15, 6*8(%rsp)
+ subq $5*8, %rsp
+ movq %rbx, 0*8(%rsp)
+ movq %r12, 1*8(%rsp)
+ movq %r14, 2*8(%rsp)
+ movq %r13, 3*8(%rsp)
+ movq %r15, 4*8(%rsp)
- movq %r8, (%rsp)
- movq %r9, 1*8(%rsp)
-
- movl %ecx, %eax
+ movl $-1, %eax
+ xorl %r9d, %r9d
movl %edx, %ecx
+ cmpl $8, %ecx
+ jb .Lshort
- xorl %r9d, %r9d
- movq %rcx, %r12
+ testb $7, %sil
+ jne .Lunaligned
+.Laligned:
+ movl %ecx, %r12d
shrq $6, %r12
jz .Lhandle_tail /* < 64 */
source
movq 56(%rdi), %r13
- ignore 2f
+30:
+ /*
+ * No _ASM_EXTABLE_UA; this is used for intentional prefetch on a
+ * potentially unmapped kernel address.
+ */
+ _ASM_EXTABLE(30b, 2f)
prefetcht0 5*64(%rdi)
2:
adcq %rbx, %rax
dest
movq %r13, 56(%rsi)
-3:
-
leaq 64(%rdi), %rdi
leaq 64(%rsi), %rsi
/* do last up to 56 bytes */
.Lhandle_tail:
- /* ecx: count */
- movl %ecx, %r10d
+ /* ecx: count, rcx.63: the end result needs to be rol8 */
+ movq %rcx, %r10
andl $63, %ecx
shrl $3, %ecx
jz .Lfold
.Lhandle_7:
movl %r10d, %ecx
andl $7, %ecx
+.L1: /* .Lshort rejoins the common path here */
shrl $1, %ecx
jz .Lhandle_1
movl $2, %edx
adcl %r9d, %eax /* carry */
.Lende:
- movq 2*8(%rsp), %rbx
- movq 3*8(%rsp), %r12
- movq 4*8(%rsp), %r14
- movq 5*8(%rsp), %r13
- movq 6*8(%rsp), %r15
- addq $7*8, %rsp
+ testq %r10, %r10
+ js .Lwas_odd
+.Lout:
+ movq 0*8(%rsp), %rbx
+ movq 1*8(%rsp), %r12
+ movq 2*8(%rsp), %r14
+ movq 3*8(%rsp), %r13
+ movq 4*8(%rsp), %r15
+ addq $5*8, %rsp
ret
+.Lshort:
+ movl %ecx, %r10d
+ jmp .L1
+.Lunaligned:
+ xorl %ebx, %ebx
+ testb $1, %sil
+ jne .Lodd
+1: testb $2, %sil
+ je 2f
+ source
+ movw (%rdi), %bx
+ dest
+ movw %bx, (%rsi)
+ leaq 2(%rdi), %rdi
+ subq $2, %rcx
+ leaq 2(%rsi), %rsi
+ addq %rbx, %rax
+2: testb $4, %sil
+ je .Laligned
+ source
+ movl (%rdi), %ebx
+ dest
+ movl %ebx, (%rsi)
+ leaq 4(%rdi), %rdi
+ subq $4, %rcx
+ leaq 4(%rsi), %rsi
+ addq %rbx, %rax
+ jmp .Laligned
+
+.Lodd:
+ source
+ movb (%rdi), %bl
+ dest
+ movb %bl, (%rsi)
+ leaq 1(%rdi), %rdi
+ leaq 1(%rsi), %rsi
+ /* decrement, set MSB */
+ leaq -1(%rcx, %rcx), %rcx
+ rorq $1, %rcx
+ shll $8, %ebx
+ addq %rbx, %rax
+ jmp 1b
+
+.Lwas_odd:
+ roll $8, %eax
+ jmp .Lout
- /* Exception handlers. Very simple, zeroing is done in the wrappers */
-.Lbad_source:
- movq (%rsp), %rax
- testq %rax, %rax
- jz .Lende
- movl $-EFAULT, (%rax)
- jmp .Lende
-
-.Lbad_dest:
- movq 8(%rsp), %rax
- testq %rax, %rax
- jz .Lende
- movl $-EFAULT, (%rax)
- jmp .Lende
+ /* Exception: just return 0 */
+.Lfault:
+ xorl %eax, %eax
+ jmp .Lout
SYM_FUNC_END(csum_partial_copy_generic)
* src and dst are best aligned to 64bits.
*/
__wsum
-csum_and_copy_from_user(const void __user *src, void *dst,
- int len, __wsum isum, int *errp)
+csum_and_copy_from_user(const void __user *src, void *dst, int len)
{
- might_sleep();
- *errp = 0;
+ __wsum sum;
+ might_sleep();
if (!user_access_begin(src, len))
- goto out_err;
-
- /*
- * Why 6, not 7? To handle odd addresses aligned we
- * would need to do considerable complications to fix the
- * checksum which is defined as an 16bit accumulator. The
- * fix alignment code is primarily for performance
- * compatibility with 32bit and that will handle odd
- * addresses slowly too.
- */
- if (unlikely((unsigned long)src & 6)) {
- while (((unsigned long)src & 6) && len >= 2) {
- __u16 val16;
-
- unsafe_get_user(val16, (const __u16 __user *)src, out);
-
- *(__u16 *)dst = val16;
- isum = (__force __wsum)add32_with_carry(
- (__force unsigned)isum, val16);
- src += 2;
- dst += 2;
- len -= 2;
- }
- }
- isum = csum_partial_copy_generic((__force const void *)src,
- dst, len, isum, errp, NULL);
- user_access_end();
- if (unlikely(*errp))
- goto out_err;
-
- return isum;
-
-out:
+ return 0;
+ sum = csum_partial_copy_generic((__force const void *)src, dst, len);
user_access_end();
-out_err:
- *errp = -EFAULT;
- memset(dst, 0, len);
-
- return isum;
+ return sum;
}
EXPORT_SYMBOL(csum_and_copy_from_user);
* src and dst are best aligned to 64bits.
*/
__wsum
-csum_and_copy_to_user(const void *src, void __user *dst,
- int len, __wsum isum, int *errp)
+csum_and_copy_to_user(const void *src, void __user *dst, int len)
{
- __wsum ret;
+ __wsum sum;
might_sleep();
-
- if (!user_access_begin(dst, len)) {
- *errp = -EFAULT;
+ if (!user_access_begin(dst, len))
return 0;
- }
-
- if (unlikely((unsigned long)dst & 6)) {
- while (((unsigned long)dst & 6) && len >= 2) {
- __u16 val16 = *(__u16 *)src;
-
- isum = (__force __wsum)add32_with_carry(
- (__force unsigned)isum, val16);
- unsafe_put_user(val16, (__u16 __user *)dst, out);
- src += 2;
- dst += 2;
- len -= 2;
- }
- }
-
- *errp = 0;
- ret = csum_partial_copy_generic(src, (void __force *)dst,
- len, isum, NULL, errp);
- user_access_end();
- return ret;
-out:
+ sum = csum_partial_copy_generic(src, (void __force *)dst, len);
user_access_end();
- *errp = -EFAULT;
- return isum;
+ return sum;
}
EXPORT_SYMBOL(csum_and_copy_to_user);
* Returns an 32bit unfolded checksum of the buffer.
*/
__wsum
-csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
+csum_partial_copy_nocheck(const void *src, void *dst, int len)
{
- return csum_partial_copy_generic(src, dst, len, sum, NULL, NULL);
+ return csum_partial_copy_generic(src, dst, len);
}
EXPORT_SYMBOL(csum_partial_copy_nocheck);
#include <asm/smap.h>
#include <asm/export.h>
+#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
+
.text
SYM_FUNC_START(__get_user_1)
mov PER_CPU_VAR(current_task), %_ASM_DX
SYM_FUNC_END(__get_user_8)
EXPORT_SYMBOL(__get_user_8)
+/* .. and the same for __get_user, just without the range checks */
+SYM_FUNC_START(__get_user_nocheck_1)
+ ASM_STAC
+ ASM_BARRIER_NOSPEC
+6: movzbl (%_ASM_AX),%edx
+ xor %eax,%eax
+ ASM_CLAC
+ ret
+SYM_FUNC_END(__get_user_nocheck_1)
+EXPORT_SYMBOL(__get_user_nocheck_1)
+
+SYM_FUNC_START(__get_user_nocheck_2)
+ ASM_STAC
+ ASM_BARRIER_NOSPEC
+7: movzwl (%_ASM_AX),%edx
+ xor %eax,%eax
+ ASM_CLAC
+ ret
+SYM_FUNC_END(__get_user_nocheck_2)
+EXPORT_SYMBOL(__get_user_nocheck_2)
+
+SYM_FUNC_START(__get_user_nocheck_4)
+ ASM_STAC
+ ASM_BARRIER_NOSPEC
+8: movl (%_ASM_AX),%edx
+ xor %eax,%eax
+ ASM_CLAC
+ ret
+SYM_FUNC_END(__get_user_nocheck_4)
+EXPORT_SYMBOL(__get_user_nocheck_4)
+
+SYM_FUNC_START(__get_user_nocheck_8)
+ ASM_STAC
+ ASM_BARRIER_NOSPEC
+#ifdef CONFIG_X86_64
+9: movq (%_ASM_AX),%rdx
+#else
+9: movl (%_ASM_AX),%edx
+10: movl 4(%_ASM_AX),%ecx
+#endif
+ xor %eax,%eax
+ ASM_CLAC
+ ret
+SYM_FUNC_END(__get_user_nocheck_8)
+EXPORT_SYMBOL(__get_user_nocheck_8)
+
SYM_CODE_START_LOCAL(.Lbad_get_user_clac)
ASM_CLAC
SYM_CODE_END(.Lbad_get_user_8_clac)
#endif
+/* get_user */
_ASM_EXTABLE_UA(1b, .Lbad_get_user_clac)
_ASM_EXTABLE_UA(2b, .Lbad_get_user_clac)
_ASM_EXTABLE_UA(3b, .Lbad_get_user_clac)
_ASM_EXTABLE_UA(4b, .Lbad_get_user_8_clac)
_ASM_EXTABLE_UA(5b, .Lbad_get_user_8_clac)
#endif
+
+/* __get_user */
+ _ASM_EXTABLE_UA(6b, .Lbad_get_user_clac)
+ _ASM_EXTABLE_UA(7b, .Lbad_get_user_clac)
+ _ASM_EXTABLE_UA(8b, .Lbad_get_user_clac)
+#ifdef CONFIG_X86_64
+ _ASM_EXTABLE_UA(9b, .Lbad_get_user_clac)
+#else
+ _ASM_EXTABLE_UA(9b, .Lbad_get_user_8_clac)
+ _ASM_EXTABLE_UA(10b, .Lbad_get_user_8_clac)
+#endif
#include <linux/linkage.h>
#include <asm/errno.h>
#include <asm/cpufeatures.h>
-#include <asm/mcsafe_test.h>
#include <asm/alternative-asm.h>
#include <asm/export.h>
SYM_FUNC_END(memcpy_orig)
.popsection
-
-#ifndef CONFIG_UML
-
-MCSAFE_TEST_CTL
-
-/*
- * __memcpy_mcsafe - memory copy with machine check exception handling
- * Note that we only catch machine checks when reading the source addresses.
- * Writes to target are posted and don't generate machine checks.
- */
-SYM_FUNC_START(__memcpy_mcsafe)
- cmpl $8, %edx
- /* Less than 8 bytes? Go to byte copy loop */
- jb .L_no_whole_words
-
- /* Check for bad alignment of source */
- testl $7, %esi
- /* Already aligned */
- jz .L_8byte_aligned
-
- /* Copy one byte at a time until source is 8-byte aligned */
- movl %esi, %ecx
- andl $7, %ecx
- subl $8, %ecx
- negl %ecx
- subl %ecx, %edx
-.L_read_leading_bytes:
- movb (%rsi), %al
- MCSAFE_TEST_SRC %rsi 1 .E_leading_bytes
- MCSAFE_TEST_DST %rdi 1 .E_leading_bytes
-.L_write_leading_bytes:
- movb %al, (%rdi)
- incq %rsi
- incq %rdi
- decl %ecx
- jnz .L_read_leading_bytes
-
-.L_8byte_aligned:
- movl %edx, %ecx
- andl $7, %edx
- shrl $3, %ecx
- jz .L_no_whole_words
-
-.L_read_words:
- movq (%rsi), %r8
- MCSAFE_TEST_SRC %rsi 8 .E_read_words
- MCSAFE_TEST_DST %rdi 8 .E_write_words
-.L_write_words:
- movq %r8, (%rdi)
- addq $8, %rsi
- addq $8, %rdi
- decl %ecx
- jnz .L_read_words
-
- /* Any trailing bytes? */
-.L_no_whole_words:
- andl %edx, %edx
- jz .L_done_memcpy_trap
-
- /* Copy trailing bytes */
- movl %edx, %ecx
-.L_read_trailing_bytes:
- movb (%rsi), %al
- MCSAFE_TEST_SRC %rsi 1 .E_trailing_bytes
- MCSAFE_TEST_DST %rdi 1 .E_trailing_bytes
-.L_write_trailing_bytes:
- movb %al, (%rdi)
- incq %rsi
- incq %rdi
- decl %ecx
- jnz .L_read_trailing_bytes
-
- /* Copy successful. Return zero */
-.L_done_memcpy_trap:
- xorl %eax, %eax
-.L_done:
- ret
-SYM_FUNC_END(__memcpy_mcsafe)
-EXPORT_SYMBOL_GPL(__memcpy_mcsafe)
-
- .section .fixup, "ax"
- /*
- * Return number of bytes not copied for any failure. Note that
- * there is no "tail" handling since the source buffer is 8-byte
- * aligned and poison is cacheline aligned.
- */
-.E_read_words:
- shll $3, %ecx
-.E_leading_bytes:
- addl %edx, %ecx
-.E_trailing_bytes:
- mov %ecx, %eax
- jmp .L_done
-
- /*
- * For write fault handling, given the destination is unaligned,
- * we handle faults on multi-byte writes with a byte-by-byte
- * copy up to the write-protected page.
- */
-.E_write_words:
- shll $3, %ecx
- addl %edx, %ecx
- movl %ecx, %edx
- jmp mcsafe_handle_tail
-
- .previous
-
- _ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes)
- _ASM_EXTABLE_FAULT(.L_read_words, .E_read_words)
- _ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes)
- _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)
- _ASM_EXTABLE(.L_write_words, .E_write_words)
- _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)
-#endif
* Inputs: %eax[:%edx] contains the data
* %ecx contains the address
*
- * Outputs: %eax is error code (0 or -EFAULT)
+ * Outputs: %ecx is error code (0 or -EFAULT)
+ *
+ * Clobbers: %ebx needed for task pointer
*
* These functions should not modify any other registers,
* as they get called from within inline assembly.
ENTER
cmp TASK_addr_limit(%_ASM_BX),%_ASM_CX
jae .Lbad_put_user
+SYM_INNER_LABEL(__put_user_nocheck_1, SYM_L_GLOBAL)
ASM_STAC
1: movb %al,(%_ASM_CX)
- xor %eax,%eax
+ xor %ecx,%ecx
ASM_CLAC
ret
SYM_FUNC_END(__put_user_1)
EXPORT_SYMBOL(__put_user_1)
+EXPORT_SYMBOL(__put_user_nocheck_1)
SYM_FUNC_START(__put_user_2)
ENTER
sub $1,%_ASM_BX
cmp %_ASM_BX,%_ASM_CX
jae .Lbad_put_user
+SYM_INNER_LABEL(__put_user_nocheck_2, SYM_L_GLOBAL)
ASM_STAC
2: movw %ax,(%_ASM_CX)
- xor %eax,%eax
+ xor %ecx,%ecx
ASM_CLAC
ret
SYM_FUNC_END(__put_user_2)
EXPORT_SYMBOL(__put_user_2)
+EXPORT_SYMBOL(__put_user_nocheck_2)
SYM_FUNC_START(__put_user_4)
ENTER
sub $3,%_ASM_BX
cmp %_ASM_BX,%_ASM_CX
jae .Lbad_put_user
+SYM_INNER_LABEL(__put_user_nocheck_4, SYM_L_GLOBAL)
ASM_STAC
3: movl %eax,(%_ASM_CX)
- xor %eax,%eax
+ xor %ecx,%ecx
ASM_CLAC
ret
SYM_FUNC_END(__put_user_4)
EXPORT_SYMBOL(__put_user_4)
+EXPORT_SYMBOL(__put_user_nocheck_4)
SYM_FUNC_START(__put_user_8)
ENTER
sub $7,%_ASM_BX
cmp %_ASM_BX,%_ASM_CX
jae .Lbad_put_user
+SYM_INNER_LABEL(__put_user_nocheck_8, SYM_L_GLOBAL)
ASM_STAC
4: mov %_ASM_AX,(%_ASM_CX)
#ifdef CONFIG_X86_32
5: movl %edx,4(%_ASM_CX)
#endif
- xor %eax,%eax
+ xor %ecx,%ecx
ASM_CLAC
RET
SYM_FUNC_END(__put_user_8)
EXPORT_SYMBOL(__put_user_8)
+EXPORT_SYMBOL(__put_user_nocheck_8)
SYM_CODE_START_LOCAL(.Lbad_put_user_clac)
ASM_CLAC
.Lbad_put_user:
- movl $-EFAULT,%eax
+ movl $-EFAULT,%ecx
RET
SYM_CODE_END(.Lbad_put_user_clac)
}
EXPORT_SYMBOL(clear_user);
-/*
- * Similar to copy_user_handle_tail, probe for the write fault point,
- * but reuse __memcpy_mcsafe in case a new read error is encountered.
- * clac() is handled in _copy_to_iter_mcsafe().
- */
-__visible notrace unsigned long
-mcsafe_handle_tail(char *to, char *from, unsigned len)
-{
- for (; len; --len, to++, from++) {
- /*
- * Call the assembly routine back directly since
- * memcpy_mcsafe() may silently fallback to memcpy.
- */
- unsigned long rem = __memcpy_mcsafe(to, from, 1);
-
- if (rem)
- break;
- }
- return len;
-}
-
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
/**
* clean_cache_range - write back a cache range with CLWB
*/
if (size < 8) {
if (!IS_ALIGNED(dest, 4) || size != 4)
- clean_cache_range(dst, 1);
+ clean_cache_range(dst, size);
} else {
if (!IS_ALIGNED(dest, 8)) {
dest = ALIGN(dest, boot_cpu_data.x86_clflush_size);
}
EXPORT_SYMBOL(ex_handler_uaccess);
+__visible bool ex_handler_copy(const struct exception_table_entry *fixup,
+ struct pt_regs *regs, int trapnr,
+ unsigned long error_code,
+ unsigned long fault_addr)
+{
+ WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?");
+ regs->ip = ex_fixup_addr(fixup);
+ regs->ax = trapnr;
+ return true;
+}
+EXPORT_SYMBOL(ex_handler_copy);
+
__visible bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup,
struct pt_regs *regs, int trapnr,
unsigned long error_code,
}
EXPORT_SYMBOL(ex_handler_clear_fs);
-__visible bool ex_has_fault_handler(unsigned long ip)
+enum handler_type ex_get_fault_handler_type(unsigned long ip)
{
const struct exception_table_entry *e;
ex_handler_t handler;
e = search_exception_tables(ip);
if (!e)
- return false;
+ return EX_HANDLER_NONE;
handler = ex_fixup_handler(e);
-
- return handler == ex_handler_fault;
+ if (handler == ex_handler_fault)
+ return EX_HANDLER_FAULT;
+ else if (handler == ex_handler_uaccess || handler == ex_handler_copy)
+ return EX_HANDLER_UACCESS;
+ else
+ return EX_HANDLER_OTHER;
}
int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code,
return 0;
}
-static int fault_in_kernel_space(unsigned long address)
+bool fault_in_kernel_space(unsigned long address)
{
/*
* On 64-bit systems, the vsyscall page is at an address above
sync_global_pgds_l4(start, end);
}
-void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
-{
- sync_global_pgds(start, end);
-}
-
/*
* NOTE: This function is marked __ref because it calls __init function
* (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0.
if (!p4d)
goto failed;
- /*
- * With 5-level paging the P4D level is not folded. So the PGDs
- * are now populated and there is no need to walk down to the
- * PUD level.
- */
if (pgtable_l5_enabled())
continue;
+ /*
+ * The goal here is to allocate all possibly required
+ * hardware page tables pointed to by the top hardware
+ * level.
+ *
+ * On 4-level systems, the P4D layer is folded away and
+ * the above code does no preallocation. Below, go down
+ * to the pud _software_ level to ensure the second
+ * hardware level is allocated on 4-level systems too.
+ */
lvl = "pud";
pud = pud_alloc(&init_mm, p4d, addr);
if (!pud)
/*
* Before changing the encryption attribute, we need to flush caches.
*/
- cpa_flush(&cpa, 1);
+ cpa_flush(&cpa, !this_cpu_has(X86_FEATURE_SME_COHERENT));
ret = __change_page_attr_set_clr(&cpa, 1);
#include <asm/nospec-branch.h>
#include <asm/cache.h>
#include <asm/apic.h>
-#include <asm/uv/uv.h>
#include "mm_internal.h"
trace_tlb_flush(TLB_REMOTE_SEND_IPI,
(info->end - info->start) >> PAGE_SHIFT);
- if (is_uv_system()) {
- /*
- * This whole special case is confused. UV has a "Broadcast
- * Assist Unit", which seems to be a fancy way to send IPIs.
- * Back when x86 used an explicit TLB flush IPI, UV was
- * optimized to use its own mechanism. These days, x86 uses
- * smp_call_function_many(), but UV still uses a manual IPI,
- * and that IPI's action is out of date -- it does a manual
- * flush instead of calling flush_tlb_func_remote(). This
- * means that the percpu tlb_gen variables won't be updated
- * and we'll do pointless flushes on future context switches.
- *
- * Rather than hooking native_flush_tlb_others() here, I think
- * that UV should be updated so that smp_call_function_many(),
- * etc, are optimal on UV.
- */
- cpumask = uv_flush_tlb_others(cpumask, info);
- if (cpumask)
- smp_call_function_many(cpumask, flush_tlb_func_remote,
- (void *)info, 1);
- return;
- }
-
/*
* If no page tables were freed, we can skip sending IPIs to
* CPUs in lazy TLB mode. They will flush the CPU themselves
#include <asm/smp.h>
#include <asm/pci_x86.h>
#include <asm/setup.h>
+#include <asm/irqdomain.h>
unsigned int pci_probe = PCI_PROBE_BIOS | PCI_PROBE_CONF1 | PCI_PROBE_CONF2 |
PCI_PROBE_MMCONF;
int pcibios_add_device(struct pci_dev *dev)
{
- struct setup_data *data;
struct pci_setup_rom *rom;
+ struct irq_domain *msidom;
+ struct setup_data *data;
u64 pa_data;
pa_data = boot_params.hdr.setup_data;
memunmap(data);
}
set_dev_domain_options(dev);
+
+ /*
+ * Setup the initial MSI domain of the device. If the underlying
+ * bus has a PCI/MSI irqdomain associated use the bus domain,
+ * otherwise set the default domain. This ensures that special irq
+ * domains e.g. VMD are preserved. The default ensures initial
+ * operation if irq remapping is not active. If irq remapping is
+ * active it will overwrite the domain pointer when the device is
+ * associated to a remapping domain.
+ */
+ msidom = dev_get_msi_domain(&dev->bus->dev);
+ if (!msidom)
+ msidom = x86_pci_msi_default_domain;
+ dev_set_msi_domain(&dev->dev, msidom);
return 0;
}
#include <linux/init.h>
#include <asm/pci_x86.h>
#include <asm/x86_init.h>
+#include <asm/irqdomain.h>
/* arch_initcall has too random ordering, so call the initializers
in the right sequence from here. */
static __init int pci_arch_init(void)
{
-#ifdef CONFIG_PCI_DIRECT
- int type = 0;
+ int type;
+
+ x86_create_pci_msi_domain();
type = pci_direct_probe();
-#endif
if (!(pci_probe & PCI_PROBE_NOEARLY))
pci_mmcfg_early_init();
if (x86_init.pci.arch_init && !x86_init.pci.arch_init())
return 0;
-#ifdef CONFIG_PCI_BIOS
pci_pcbios_init();
-#endif
+
/*
* don't check for raw_pci_ops here because we want pcbios as last
* fallback, yet it's needed to run first to set pcibios_last_bus
* in case legacy PCI probing is used. otherwise detecting peer busses
* fails.
*/
-#ifdef CONFIG_PCI_DIRECT
pci_direct_init(type);
-#endif
+
if (!raw_pci_ops && !raw_pci_ext_ops)
printk(KERN_ERR
"PCI: Fatal: No config space access function found\n");
struct xen_pci_frontend_ops *xen_pci_frontend;
EXPORT_SYMBOL_GPL(xen_pci_frontend);
+struct xen_msi_ops {
+ int (*setup_msi_irqs)(struct pci_dev *dev, int nvec, int type);
+ void (*teardown_msi_irqs)(struct pci_dev *dev);
+};
+
+static struct xen_msi_ops xen_msi_ops __ro_after_init;
+
static int xen_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
{
int irq, ret, i;
WARN(ret && ret != -ENOSYS, "restore_msi -> %d\n", ret);
}
}
-#endif
+#else /* CONFIG_XEN_DOM0 */
+#define xen_initdom_setup_msi_irqs NULL
+#define xen_initdom_restore_msi_irqs NULL
+#endif /* !CONFIG_XEN_DOM0 */
static void xen_teardown_msi_irqs(struct pci_dev *dev)
{
struct msi_desc *msidesc;
+ int i;
+
+ for_each_pci_msi_entry(msidesc, dev) {
+ if (msidesc->irq) {
+ for (i = 0; i < msidesc->nvec_used; i++)
+ xen_destroy_irq(msidesc->irq + i);
+ }
+ }
+}
+
+static void xen_pv_teardown_msi_irqs(struct pci_dev *dev)
+{
+ struct msi_desc *msidesc = first_pci_msi_entry(dev);
- msidesc = first_pci_msi_entry(dev);
if (msidesc->msi_attrib.is_msix)
xen_pci_frontend_disable_msix(dev);
else
xen_pci_frontend_disable_msi(dev);
- /* Free the IRQ's and the msidesc using the generic code. */
- default_teardown_msi_irqs(dev);
+ xen_teardown_msi_irqs(dev);
}
-static void xen_teardown_msi_irq(unsigned int irq)
+static int xen_msi_domain_alloc_irqs(struct irq_domain *domain,
+ struct device *dev, int nvec)
{
- xen_destroy_irq(irq);
+ int type;
+
+ if (WARN_ON_ONCE(!dev_is_pci(dev)))
+ return -EINVAL;
+
+ if (first_msi_entry(dev)->msi_attrib.is_msix)
+ type = PCI_CAP_ID_MSIX;
+ else
+ type = PCI_CAP_ID_MSI;
+
+ return xen_msi_ops.setup_msi_irqs(to_pci_dev(dev), nvec, type);
}
-#endif
+static void xen_msi_domain_free_irqs(struct irq_domain *domain,
+ struct device *dev)
+{
+ if (WARN_ON_ONCE(!dev_is_pci(dev)))
+ return;
+
+ xen_msi_ops.teardown_msi_irqs(to_pci_dev(dev));
+}
+
+static struct msi_domain_ops xen_pci_msi_domain_ops = {
+ .domain_alloc_irqs = xen_msi_domain_alloc_irqs,
+ .domain_free_irqs = xen_msi_domain_free_irqs,
+};
+
+static struct msi_domain_info xen_pci_msi_domain_info = {
+ .ops = &xen_pci_msi_domain_ops,
+};
+
+/*
+ * This irq domain is a blatant violation of the irq domain design, but
+ * distangling XEN into real irq domains is not a job for mere mortals with
+ * limited XENology. But it's the least dangerous way for a mere mortal to
+ * get rid of the arch_*_msi_irqs() hackery in order to store the irq
+ * domain pointer in struct device. This irq domain wrappery allows to do
+ * that without breaking XEN terminally.
+ */
+static __init struct irq_domain *xen_create_pci_msi_domain(void)
+{
+ struct irq_domain *d = NULL;
+ struct fwnode_handle *fn;
+
+ fn = irq_domain_alloc_named_fwnode("XEN-MSI");
+ if (fn)
+ d = msi_create_irq_domain(fn, &xen_pci_msi_domain_info, NULL);
+
+ /* FIXME: No idea how to survive if this fails */
+ BUG_ON(!d);
+
+ return d;
+}
+
+static __init void xen_setup_pci_msi(void)
+{
+ if (xen_pv_domain()) {
+ if (xen_initial_domain()) {
+ xen_msi_ops.setup_msi_irqs = xen_initdom_setup_msi_irqs;
+ x86_msi.restore_msi_irqs = xen_initdom_restore_msi_irqs;
+ } else {
+ xen_msi_ops.setup_msi_irqs = xen_setup_msi_irqs;
+ }
+ xen_msi_ops.teardown_msi_irqs = xen_pv_teardown_msi_irqs;
+ pci_msi_ignore_mask = 1;
+ } else if (xen_hvm_domain()) {
+ xen_msi_ops.setup_msi_irqs = xen_hvm_setup_msi_irqs;
+ xen_msi_ops.teardown_msi_irqs = xen_teardown_msi_irqs;
+ } else {
+ WARN_ON_ONCE(1);
+ return;
+ }
+
+ /*
+ * Override the PCI/MSI irq domain init function. No point
+ * in allocating the native domain and never use it.
+ */
+ x86_init.irqs.create_pci_msi_domain = xen_create_pci_msi_domain;
+}
+
+#else /* CONFIG_PCI_MSI */
+static inline void xen_setup_pci_msi(void) { }
+#endif /* CONFIG_PCI_MSI */
int __init pci_xen_init(void)
{
/* Keep ACPI out of the picture */
acpi_noirq_set();
-#ifdef CONFIG_PCI_MSI
- x86_msi.setup_msi_irqs = xen_setup_msi_irqs;
- x86_msi.teardown_msi_irq = xen_teardown_msi_irq;
- x86_msi.teardown_msi_irqs = xen_teardown_msi_irqs;
- pci_msi_ignore_mask = 1;
-#endif
+ xen_setup_pci_msi();
return 0;
}
#ifdef CONFIG_PCI_MSI
-void __init xen_msi_init(void)
+static void __init xen_hvm_msi_init(void)
{
if (!disable_apic) {
/*
((eax & XEN_HVM_CPUID_APIC_ACCESS_VIRT) && boot_cpu_has(X86_FEATURE_APIC)))
return;
}
-
- x86_msi.setup_msi_irqs = xen_hvm_setup_msi_irqs;
- x86_msi.teardown_msi_irq = xen_teardown_msi_irq;
+ xen_setup_pci_msi();
}
#endif
* We need to wait until after x2apic is initialized
* before we can set MSI IRQ ops.
*/
- x86_platform.apic_post_init = xen_msi_init;
+ x86_platform.apic_post_init = xen_hvm_msi_init;
#endif
return 0;
}
{
int irq;
-#ifdef CONFIG_PCI_MSI
- x86_msi.setup_msi_irqs = xen_initdom_setup_msi_irqs;
- x86_msi.teardown_msi_irq = xen_teardown_msi_irq;
- x86_msi.restore_msi_irqs = xen_initdom_restore_msi_irqs;
- pci_msi_ignore_mask = 1;
-#endif
+ xen_setup_pci_msi();
__acpi_register_gsi = acpi_register_gsi_xen;
__acpi_unregister_gsi = NULL;
/*
&efi.tpm_log,
&efi.tpm_final_log,
&efi_rng_seed,
+#ifdef CONFIG_LOAD_UEFI_KEYS
+ &efi.mokvar_table,
+#endif
};
u64 efi_setup; /* efi setup_data physical address */
# SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_X86_UV) += tlb_uv.o bios_uv.o uv_irq.o uv_sysfs.o uv_time.o uv_nmi.o
+obj-$(CONFIG_X86_UV) += bios_uv.o uv_irq.o uv_sysfs.o uv_time.o uv_nmi.o
/*
* BIOS run time interface routines.
*
- * Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved.
- * Copyright (c) Russ Anderson <rja@sgi.com>
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
+ * Copyright (C) 2007-2017 Silicon Graphics, Inc. All rights reserved.
+ * Copyright (c) Russ Anderson <rja@sgi.com>
*/
#include <linux/efi.h>
(u64)decode, (u64)domain, (u64)bus, 0, 0);
}
-int uv_bios_init(void)
+unsigned long get_uv_systab_phys(bool msg)
{
- uv_systab = NULL;
if ((uv_systab_phys == EFI_INVALID_TABLE_ADDR) ||
!uv_systab_phys || efi_runtime_disabled()) {
- pr_crit("UV: UVsystab: missing\n");
- return -EEXIST;
+ if (msg)
+ pr_crit("UV: UVsystab: missing\n");
+ return 0;
}
+ return uv_systab_phys;
+}
+
+int uv_bios_init(void)
+{
+ unsigned long uv_systab_phys_addr;
+
+ uv_systab = NULL;
+ uv_systab_phys_addr = get_uv_systab_phys(1);
+ if (!uv_systab_phys_addr)
+ return -EEXIST;
- uv_systab = ioremap(uv_systab_phys, sizeof(struct uv_systab));
+ uv_systab = ioremap(uv_systab_phys_addr, sizeof(struct uv_systab));
if (!uv_systab || strncmp(uv_systab->signature, UV_SYSTAB_SIG, 4)) {
pr_err("UV: UVsystab: bad signature!\n");
iounmap(uv_systab);
int size = uv_systab->size;
iounmap(uv_systab);
- uv_systab = ioremap(uv_systab_phys, size);
+ uv_systab = ioremap(uv_systab_phys_addr, size);
if (!uv_systab) {
pr_err("UV: UVsystab: ioremap(%d) failed!\n", size);
return -EFAULT;
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * SGI UltraViolet TLB flush routines.
- *
- * (c) 2008-2014 Cliff Wickman <cpw@sgi.com>, SGI.
- */
-#include <linux/seq_file.h>
-#include <linux/proc_fs.h>
-#include <linux/debugfs.h>
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-
-#include <asm/mmu_context.h>
-#include <asm/uv/uv.h>
-#include <asm/uv/uv_mmrs.h>
-#include <asm/uv/uv_hub.h>
-#include <asm/uv/uv_bau.h>
-#include <asm/apic.h>
-#include <asm/tsc.h>
-#include <asm/irq_vectors.h>
-#include <asm/timer.h>
-
-static struct bau_operations ops __ro_after_init;
-
-static int timeout_us;
-static bool nobau = true;
-static int nobau_perm;
-
-/* tunables: */
-static int max_concurr = MAX_BAU_CONCURRENT;
-static int max_concurr_const = MAX_BAU_CONCURRENT;
-static int plugged_delay = PLUGGED_DELAY;
-static int plugsb4reset = PLUGSB4RESET;
-static int giveup_limit = GIVEUP_LIMIT;
-static int timeoutsb4reset = TIMEOUTSB4RESET;
-static int ipi_reset_limit = IPI_RESET_LIMIT;
-static int complete_threshold = COMPLETE_THRESHOLD;
-static int congested_respns_us = CONGESTED_RESPONSE_US;
-static int congested_reps = CONGESTED_REPS;
-static int disabled_period = DISABLED_PERIOD;
-
-static struct tunables tunables[] = {
- {&max_concurr, MAX_BAU_CONCURRENT}, /* must be [0] */
- {&plugged_delay, PLUGGED_DELAY},
- {&plugsb4reset, PLUGSB4RESET},
- {&timeoutsb4reset, TIMEOUTSB4RESET},
- {&ipi_reset_limit, IPI_RESET_LIMIT},
- {&complete_threshold, COMPLETE_THRESHOLD},
- {&congested_respns_us, CONGESTED_RESPONSE_US},
- {&congested_reps, CONGESTED_REPS},
- {&disabled_period, DISABLED_PERIOD},
- {&giveup_limit, GIVEUP_LIMIT}
-};
-
-static struct dentry *tunables_dir;
-
-/* these correspond to the statistics printed by ptc_seq_show() */
-static char *stat_description[] = {
- "sent: number of shootdown messages sent",
- "stime: time spent sending messages",
- "numuvhubs: number of hubs targeted with shootdown",
- "numuvhubs16: number times 16 or more hubs targeted",
- "numuvhubs8: number times 8 or more hubs targeted",
- "numuvhubs4: number times 4 or more hubs targeted",
- "numuvhubs2: number times 2 or more hubs targeted",
- "numuvhubs1: number times 1 hub targeted",
- "numcpus: number of cpus targeted with shootdown",
- "dto: number of destination timeouts",
- "retries: destination timeout retries sent",
- "rok: : destination timeouts successfully retried",
- "resetp: ipi-style resource resets for plugs",
- "resett: ipi-style resource resets for timeouts",
- "giveup: fall-backs to ipi-style shootdowns",
- "sto: number of source timeouts",
- "bz: number of stay-busy's",
- "throt: number times spun in throttle",
- "swack: image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE",
- "recv: shootdown messages received",
- "rtime: time spent processing messages",
- "all: shootdown all-tlb messages",
- "one: shootdown one-tlb messages",
- "mult: interrupts that found multiple messages",
- "none: interrupts that found no messages",
- "retry: number of retry messages processed",
- "canc: number messages canceled by retries",
- "nocan: number retries that found nothing to cancel",
- "reset: number of ipi-style reset requests processed",
- "rcan: number messages canceled by reset requests",
- "disable: number times use of the BAU was disabled",
- "enable: number times use of the BAU was re-enabled"
-};
-
-static int __init setup_bau(char *arg)
-{
- int result;
-
- if (!arg)
- return -EINVAL;
-
- result = strtobool(arg, &nobau);
- if (result)
- return result;
-
- /* we need to flip the logic here, so that bau=y sets nobau to false */
- nobau = !nobau;
-
- if (!nobau)
- pr_info("UV BAU Enabled\n");
- else
- pr_info("UV BAU Disabled\n");
-
- return 0;
-}
-early_param("bau", setup_bau);
-
-/* base pnode in this partition */
-static int uv_base_pnode __read_mostly;
-
-static DEFINE_PER_CPU(struct ptc_stats, ptcstats);
-static DEFINE_PER_CPU(struct bau_control, bau_control);
-static DEFINE_PER_CPU(cpumask_var_t, uv_flush_tlb_mask);
-
-static void
-set_bau_on(void)
-{
- int cpu;
- struct bau_control *bcp;
-
- if (nobau_perm) {
- pr_info("BAU not initialized; cannot be turned on\n");
- return;
- }
- nobau = false;
- for_each_present_cpu(cpu) {
- bcp = &per_cpu(bau_control, cpu);
- bcp->nobau = false;
- }
- pr_info("BAU turned on\n");
- return;
-}
-
-static void
-set_bau_off(void)
-{
- int cpu;
- struct bau_control *bcp;
-
- nobau = true;
- for_each_present_cpu(cpu) {
- bcp = &per_cpu(bau_control, cpu);
- bcp->nobau = true;
- }
- pr_info("BAU turned off\n");
- return;
-}
-
-/*
- * Determine the first node on a uvhub. 'Nodes' are used for kernel
- * memory allocation.
- */
-static int __init uvhub_to_first_node(int uvhub)
-{
- int node, b;
-
- for_each_online_node(node) {
- b = uv_node_to_blade_id(node);
- if (uvhub == b)
- return node;
- }
- return -1;
-}
-
-/*
- * Determine the apicid of the first cpu on a uvhub.
- */
-static int __init uvhub_to_first_apicid(int uvhub)
-{
- int cpu;
-
- for_each_present_cpu(cpu)
- if (uvhub == uv_cpu_to_blade_id(cpu))
- return per_cpu(x86_cpu_to_apicid, cpu);
- return -1;
-}
-
-/*
- * Free a software acknowledge hardware resource by clearing its Pending
- * bit. This will return a reply to the sender.
- * If the message has timed out, a reply has already been sent by the
- * hardware but the resource has not been released. In that case our
- * clear of the Timeout bit (as well) will free the resource. No reply will
- * be sent (the hardware will only do one reply per message).
- */
-static void reply_to_message(struct msg_desc *mdp, struct bau_control *bcp,
- int do_acknowledge)
-{
- unsigned long dw;
- struct bau_pq_entry *msg;
-
- msg = mdp->msg;
- if (!msg->canceled && do_acknowledge) {
- dw = (msg->swack_vec << UV_SW_ACK_NPENDING) | msg->swack_vec;
- ops.write_l_sw_ack(dw);
- }
- msg->replied_to = 1;
- msg->swack_vec = 0;
-}
-
-/*
- * Process the receipt of a RETRY message
- */
-static void bau_process_retry_msg(struct msg_desc *mdp,
- struct bau_control *bcp)
-{
- int i;
- int cancel_count = 0;
- unsigned long msg_res;
- unsigned long mmr = 0;
- struct bau_pq_entry *msg = mdp->msg;
- struct bau_pq_entry *msg2;
- struct ptc_stats *stat = bcp->statp;
-
- stat->d_retries++;
- /*
- * cancel any message from msg+1 to the retry itself
- */
- for (msg2 = msg+1, i = 0; i < DEST_Q_SIZE; msg2++, i++) {
- if (msg2 > mdp->queue_last)
- msg2 = mdp->queue_first;
- if (msg2 == msg)
- break;
-
- /* same conditions for cancellation as do_reset */
- if ((msg2->replied_to == 0) && (msg2->canceled == 0) &&
- (msg2->swack_vec) && ((msg2->swack_vec &
- msg->swack_vec) == 0) &&
- (msg2->sending_cpu == msg->sending_cpu) &&
- (msg2->msg_type != MSG_NOOP)) {
- mmr = ops.read_l_sw_ack();
- msg_res = msg2->swack_vec;
- /*
- * This is a message retry; clear the resources held
- * by the previous message only if they timed out.
- * If it has not timed out we have an unexpected
- * situation to report.
- */
- if (mmr & (msg_res << UV_SW_ACK_NPENDING)) {
- unsigned long mr;
- /*
- * Is the resource timed out?
- * Make everyone ignore the cancelled message.
- */
- msg2->canceled = 1;
- stat->d_canceled++;
- cancel_count++;
- mr = (msg_res << UV_SW_ACK_NPENDING) | msg_res;
- ops.write_l_sw_ack(mr);
- }
- }
- }
- if (!cancel_count)
- stat->d_nocanceled++;
-}
-
-/*
- * Do all the things a cpu should do for a TLB shootdown message.
- * Other cpu's may come here at the same time for this message.
- */
-static void bau_process_message(struct msg_desc *mdp, struct bau_control *bcp,
- int do_acknowledge)
-{
- short socket_ack_count = 0;
- short *sp;
- struct atomic_short *asp;
- struct ptc_stats *stat = bcp->statp;
- struct bau_pq_entry *msg = mdp->msg;
- struct bau_control *smaster = bcp->socket_master;
-
- /*
- * This must be a normal message, or retry of a normal message
- */
- if (msg->address == TLB_FLUSH_ALL) {
- flush_tlb_local();
- stat->d_alltlb++;
- } else {
- flush_tlb_one_user(msg->address);
- stat->d_onetlb++;
- }
- stat->d_requestee++;
-
- /*
- * One cpu on each uvhub has the additional job on a RETRY
- * of releasing the resource held by the message that is
- * being retried. That message is identified by sending
- * cpu number.
- */
- if (msg->msg_type == MSG_RETRY && bcp == bcp->uvhub_master)
- bau_process_retry_msg(mdp, bcp);
-
- /*
- * This is a swack message, so we have to reply to it.
- * Count each responding cpu on the socket. This avoids
- * pinging the count's cache line back and forth between
- * the sockets.
- */
- sp = &smaster->socket_acknowledge_count[mdp->msg_slot];
- asp = (struct atomic_short *)sp;
- socket_ack_count = atom_asr(1, asp);
- if (socket_ack_count == bcp->cpus_in_socket) {
- int msg_ack_count;
- /*
- * Both sockets dump their completed count total into
- * the message's count.
- */
- *sp = 0;
- asp = (struct atomic_short *)&msg->acknowledge_count;
- msg_ack_count = atom_asr(socket_ack_count, asp);
-
- if (msg_ack_count == bcp->cpus_in_uvhub) {
- /*
- * All cpus in uvhub saw it; reply
- * (unless we are in the UV2 workaround)
- */
- reply_to_message(mdp, bcp, do_acknowledge);
- }
- }
-
- return;
-}
-
-/*
- * Determine the first cpu on a pnode.
- */
-static int pnode_to_first_cpu(int pnode, struct bau_control *smaster)
-{
- int cpu;
- struct hub_and_pnode *hpp;
-
- for_each_present_cpu(cpu) {
- hpp = &smaster->thp[cpu];
- if (pnode == hpp->pnode)
- return cpu;
- }
- return -1;
-}
-
-/*
- * Last resort when we get a large number of destination timeouts is
- * to clear resources held by a given cpu.
- * Do this with IPI so that all messages in the BAU message queue
- * can be identified by their nonzero swack_vec field.
- *
- * This is entered for a single cpu on the uvhub.
- * The sender want's this uvhub to free a specific message's
- * swack resources.
- */
-static void do_reset(void *ptr)
-{
- int i;
- struct bau_control *bcp = &per_cpu(bau_control, smp_processor_id());
- struct reset_args *rap = (struct reset_args *)ptr;
- struct bau_pq_entry *msg;
- struct ptc_stats *stat = bcp->statp;
-
- stat->d_resets++;
- /*
- * We're looking for the given sender, and
- * will free its swack resource.
- * If all cpu's finally responded after the timeout, its
- * message 'replied_to' was set.
- */
- for (msg = bcp->queue_first, i = 0; i < DEST_Q_SIZE; msg++, i++) {
- unsigned long msg_res;
- /* do_reset: same conditions for cancellation as
- bau_process_retry_msg() */
- if ((msg->replied_to == 0) &&
- (msg->canceled == 0) &&
- (msg->sending_cpu == rap->sender) &&
- (msg->swack_vec) &&
- (msg->msg_type != MSG_NOOP)) {
- unsigned long mmr;
- unsigned long mr;
- /*
- * make everyone else ignore this message
- */
- msg->canceled = 1;
- /*
- * only reset the resource if it is still pending
- */
- mmr = ops.read_l_sw_ack();
- msg_res = msg->swack_vec;
- mr = (msg_res << UV_SW_ACK_NPENDING) | msg_res;
- if (mmr & msg_res) {
- stat->d_rcanceled++;
- ops.write_l_sw_ack(mr);
- }
- }
- }
- return;
-}
-
-/*
- * Use IPI to get all target uvhubs to release resources held by
- * a given sending cpu number.
- */
-static void reset_with_ipi(struct pnmask *distribution, struct bau_control *bcp)
-{
- int pnode;
- int apnode;
- int maskbits;
- int sender = bcp->cpu;
- cpumask_t *mask = bcp->uvhub_master->cpumask;
- struct bau_control *smaster = bcp->socket_master;
- struct reset_args reset_args;
-
- reset_args.sender = sender;
- cpumask_clear(mask);
- /* find a single cpu for each uvhub in this distribution mask */
- maskbits = sizeof(struct pnmask) * BITSPERBYTE;
- /* each bit is a pnode relative to the partition base pnode */
- for (pnode = 0; pnode < maskbits; pnode++) {
- int cpu;
- if (!bau_uvhub_isset(pnode, distribution))
- continue;
- apnode = pnode + bcp->partition_base_pnode;
- cpu = pnode_to_first_cpu(apnode, smaster);
- cpumask_set_cpu(cpu, mask);
- }
-
- /* IPI all cpus; preemption is already disabled */
- smp_call_function_many(mask, do_reset, (void *)&reset_args, 1);
- return;
-}
-
-/*
- * Not to be confused with cycles_2_ns() from tsc.c; this gives a relative
- * number, not an absolute. It converts a duration in cycles to a duration in
- * ns.
- */
-static inline unsigned long long cycles_2_ns(unsigned long long cyc)
-{
- struct cyc2ns_data data;
- unsigned long long ns;
-
- cyc2ns_read_begin(&data);
- ns = mul_u64_u32_shr(cyc, data.cyc2ns_mul, data.cyc2ns_shift);
- cyc2ns_read_end();
-
- return ns;
-}
-
-/*
- * The reverse of the above; converts a duration in ns to a duration in cycles.
- */
-static inline unsigned long long ns_2_cycles(unsigned long long ns)
-{
- struct cyc2ns_data data;
- unsigned long long cyc;
-
- cyc2ns_read_begin(&data);
- cyc = (ns << data.cyc2ns_shift) / data.cyc2ns_mul;
- cyc2ns_read_end();
-
- return cyc;
-}
-
-static inline unsigned long cycles_2_us(unsigned long long cyc)
-{
- return cycles_2_ns(cyc) / NSEC_PER_USEC;
-}
-
-static inline cycles_t sec_2_cycles(unsigned long sec)
-{
- return ns_2_cycles(sec * NSEC_PER_SEC);
-}
-
-static inline unsigned long long usec_2_cycles(unsigned long usec)
-{
- return ns_2_cycles(usec * NSEC_PER_USEC);
-}
-
-/*
- * wait for all cpus on this hub to finish their sends and go quiet
- * leaves uvhub_quiesce set so that no new broadcasts are started by
- * bau_flush_send_and_wait()
- */
-static inline void quiesce_local_uvhub(struct bau_control *hmaster)
-{
- atom_asr(1, (struct atomic_short *)&hmaster->uvhub_quiesce);
-}
-
-/*
- * mark this quiet-requestor as done
- */
-static inline void end_uvhub_quiesce(struct bau_control *hmaster)
-{
- atom_asr(-1, (struct atomic_short *)&hmaster->uvhub_quiesce);
-}
-
-/*
- * UV2 could have an extra bit of status in the ACTIVATION_STATUS_2 register.
- * But not currently used.
- */
-static unsigned long uv2_3_read_status(unsigned long offset, int rshft, int desc)
-{
- return ((read_lmmr(offset) >> rshft) & UV_ACT_STATUS_MASK) << 1;
-}
-
-/*
- * Entered when a bau descriptor has gone into a permanent busy wait because
- * of a hardware bug.
- * Workaround the bug.
- */
-static int handle_uv2_busy(struct bau_control *bcp)
-{
- struct ptc_stats *stat = bcp->statp;
-
- stat->s_uv2_wars++;
- bcp->busy = 1;
- return FLUSH_GIVEUP;
-}
-
-static int uv2_3_wait_completion(struct bau_desc *bau_desc,
- struct bau_control *bcp, long try)
-{
- unsigned long descriptor_stat;
- cycles_t ttm;
- u64 mmr_offset = bcp->status_mmr;
- int right_shift = bcp->status_index;
- int desc = bcp->uvhub_cpu;
- long busy_reps = 0;
- struct ptc_stats *stat = bcp->statp;
-
- descriptor_stat = uv2_3_read_status(mmr_offset, right_shift, desc);
-
- /* spin on the status MMR, waiting for it to go idle */
- while (descriptor_stat != UV2H_DESC_IDLE) {
- if (descriptor_stat == UV2H_DESC_SOURCE_TIMEOUT) {
- /*
- * A h/w bug on the destination side may
- * have prevented the message being marked
- * pending, thus it doesn't get replied to
- * and gets continually nacked until it times
- * out with a SOURCE_TIMEOUT.
- */
- stat->s_stimeout++;
- return FLUSH_GIVEUP;
- } else if (descriptor_stat == UV2H_DESC_DEST_TIMEOUT) {
- ttm = get_cycles();
-
- /*
- * Our retries may be blocked by all destination
- * swack resources being consumed, and a timeout
- * pending. In that case hardware returns the
- * ERROR that looks like a destination timeout.
- * Without using the extended status we have to
- * deduce from the short time that this was a
- * strong nack.
- */
- if (cycles_2_us(ttm - bcp->send_message) < timeout_us) {
- bcp->conseccompletes = 0;
- stat->s_plugged++;
- /* FLUSH_RETRY_PLUGGED causes hang on boot */
- return FLUSH_GIVEUP;
- }
- stat->s_dtimeout++;
- bcp->conseccompletes = 0;
- /* FLUSH_RETRY_TIMEOUT causes hang on boot */
- return FLUSH_GIVEUP;
- } else {
- busy_reps++;
- if (busy_reps > 1000000) {
- /* not to hammer on the clock */
- busy_reps = 0;
- ttm = get_cycles();
- if ((ttm - bcp->send_message) > bcp->timeout_interval)
- return handle_uv2_busy(bcp);
- }
- /*
- * descriptor_stat is still BUSY
- */
- cpu_relax();
- }
- descriptor_stat = uv2_3_read_status(mmr_offset, right_shift, desc);
- }
- bcp->conseccompletes++;
- return FLUSH_COMPLETE;
-}
-
-/*
- * Returns the status of current BAU message for cpu desc as a bit field
- * [Error][Busy][Aux]
- */
-static u64 read_status(u64 status_mmr, int index, int desc)
-{
- u64 stat;
-
- stat = ((read_lmmr(status_mmr) >> index) & UV_ACT_STATUS_MASK) << 1;
- stat |= (read_lmmr(UVH_LB_BAU_SB_ACTIVATION_STATUS_2) >> desc) & 0x1;
-
- return stat;
-}
-
-static int uv4_wait_completion(struct bau_desc *bau_desc,
- struct bau_control *bcp, long try)
-{
- struct ptc_stats *stat = bcp->statp;
- u64 descriptor_stat;
- u64 mmr = bcp->status_mmr;
- int index = bcp->status_index;
- int desc = bcp->uvhub_cpu;
-
- descriptor_stat = read_status(mmr, index, desc);
-
- /* spin on the status MMR, waiting for it to go idle */
- while (descriptor_stat != UV2H_DESC_IDLE) {
- switch (descriptor_stat) {
- case UV2H_DESC_SOURCE_TIMEOUT:
- stat->s_stimeout++;
- return FLUSH_GIVEUP;
-
- case UV2H_DESC_DEST_TIMEOUT:
- stat->s_dtimeout++;
- bcp->conseccompletes = 0;
- return FLUSH_RETRY_TIMEOUT;
-
- case UV2H_DESC_DEST_STRONG_NACK:
- stat->s_plugged++;
- bcp->conseccompletes = 0;
- return FLUSH_RETRY_PLUGGED;
-
- case UV2H_DESC_DEST_PUT_ERR:
- bcp->conseccompletes = 0;
- return FLUSH_GIVEUP;
-
- default:
- /* descriptor_stat is still BUSY */
- cpu_relax();
- }
- descriptor_stat = read_status(mmr, index, desc);
- }
- bcp->conseccompletes++;
- return FLUSH_COMPLETE;
-}
-
-/*
- * Our retries are blocked by all destination sw ack resources being
- * in use, and a timeout is pending. In that case hardware immediately
- * returns the ERROR that looks like a destination timeout.
- */
-static void destination_plugged(struct bau_desc *bau_desc,
- struct bau_control *bcp,
- struct bau_control *hmaster, struct ptc_stats *stat)
-{
- udelay(bcp->plugged_delay);
- bcp->plugged_tries++;
-
- if (bcp->plugged_tries >= bcp->plugsb4reset) {
- bcp->plugged_tries = 0;
-
- quiesce_local_uvhub(hmaster);
-
- spin_lock(&hmaster->queue_lock);
- reset_with_ipi(&bau_desc->distribution, bcp);
- spin_unlock(&hmaster->queue_lock);
-
- end_uvhub_quiesce(hmaster);
-
- bcp->ipi_attempts++;
- stat->s_resets_plug++;
- }
-}
-
-static void destination_timeout(struct bau_desc *bau_desc,
- struct bau_control *bcp, struct bau_control *hmaster,
- struct ptc_stats *stat)
-{
- hmaster->max_concurr = 1;
- bcp->timeout_tries++;
- if (bcp->timeout_tries >= bcp->timeoutsb4reset) {
- bcp->timeout_tries = 0;
-
- quiesce_local_uvhub(hmaster);
-
- spin_lock(&hmaster->queue_lock);
- reset_with_ipi(&bau_desc->distribution, bcp);
- spin_unlock(&hmaster->queue_lock);
-
- end_uvhub_quiesce(hmaster);
-
- bcp->ipi_attempts++;
- stat->s_resets_timeout++;
- }
-}
-
-/*
- * Stop all cpus on a uvhub from using the BAU for a period of time.
- * This is reversed by check_enable.
- */
-static void disable_for_period(struct bau_control *bcp, struct ptc_stats *stat)
-{
- int tcpu;
- struct bau_control *tbcp;
- struct bau_control *hmaster;
- cycles_t tm1;
-
- hmaster = bcp->uvhub_master;
- spin_lock(&hmaster->disable_lock);
- if (!bcp->baudisabled) {
- stat->s_bau_disabled++;
- tm1 = get_cycles();
- for_each_present_cpu(tcpu) {
- tbcp = &per_cpu(bau_control, tcpu);
- if (tbcp->uvhub_master == hmaster) {
- tbcp->baudisabled = 1;
- tbcp->set_bau_on_time =
- tm1 + bcp->disabled_period;
- }
- }
- }
- spin_unlock(&hmaster->disable_lock);
-}
-
-static void count_max_concurr(int stat, struct bau_control *bcp,
- struct bau_control *hmaster)
-{
- bcp->plugged_tries = 0;
- bcp->timeout_tries = 0;
- if (stat != FLUSH_COMPLETE)
- return;
- if (bcp->conseccompletes <= bcp->complete_threshold)
- return;
- if (hmaster->max_concurr >= hmaster->max_concurr_const)
- return;
- hmaster->max_concurr++;
-}
-
-static void record_send_stats(cycles_t time1, cycles_t time2,
- struct bau_control *bcp, struct ptc_stats *stat,
- int completion_status, int try)
-{
- cycles_t elapsed;
-
- if (time2 > time1) {
- elapsed = time2 - time1;
- stat->s_time += elapsed;
-
- if ((completion_status == FLUSH_COMPLETE) && (try == 1)) {
- bcp->period_requests++;
- bcp->period_time += elapsed;
- if ((elapsed > usec_2_cycles(bcp->cong_response_us)) &&
- (bcp->period_requests > bcp->cong_reps) &&
- ((bcp->period_time / bcp->period_requests) >
- usec_2_cycles(bcp->cong_response_us))) {
- stat->s_congested++;
- disable_for_period(bcp, stat);
- }
- }
- } else
- stat->s_requestor--;
-
- if (completion_status == FLUSH_COMPLETE && try > 1)
- stat->s_retriesok++;
- else if (completion_status == FLUSH_GIVEUP) {
- stat->s_giveup++;
- if (get_cycles() > bcp->period_end)
- bcp->period_giveups = 0;
- bcp->period_giveups++;
- if (bcp->period_giveups == 1)
- bcp->period_end = get_cycles() + bcp->disabled_period;
- if (bcp->period_giveups > bcp->giveup_limit) {
- disable_for_period(bcp, stat);
- stat->s_giveuplimit++;
- }
- }
-}
-
-/*
- * Handle the completion status of a message send.
- */
-static void handle_cmplt(int completion_status, struct bau_desc *bau_desc,
- struct bau_control *bcp, struct bau_control *hmaster,
- struct ptc_stats *stat)
-{
- if (completion_status == FLUSH_RETRY_PLUGGED)
- destination_plugged(bau_desc, bcp, hmaster, stat);
- else if (completion_status == FLUSH_RETRY_TIMEOUT)
- destination_timeout(bau_desc, bcp, hmaster, stat);
-}
-
-/*
- * Send a broadcast and wait for it to complete.
- *
- * The flush_mask contains the cpus the broadcast is to be sent to including
- * cpus that are on the local uvhub.
- *
- * Returns 0 if all flushing represented in the mask was done.
- * Returns 1 if it gives up entirely and the original cpu mask is to be
- * returned to the kernel.
- */
-static int uv_flush_send_and_wait(struct cpumask *flush_mask,
- struct bau_control *bcp,
- struct bau_desc *bau_desc)
-{
- int seq_number = 0;
- int completion_stat = 0;
- long try = 0;
- unsigned long index;
- cycles_t time1;
- cycles_t time2;
- struct ptc_stats *stat = bcp->statp;
- struct bau_control *hmaster = bcp->uvhub_master;
- struct uv2_3_bau_msg_header *uv2_3_hdr = NULL;
-
- while (hmaster->uvhub_quiesce)
- cpu_relax();
-
- time1 = get_cycles();
- uv2_3_hdr = &bau_desc->header.uv2_3_hdr;
-
- do {
- if (try == 0) {
- uv2_3_hdr->msg_type = MSG_REGULAR;
- seq_number = bcp->message_number++;
- } else {
- uv2_3_hdr->msg_type = MSG_RETRY;
- stat->s_retry_messages++;
- }
-
- uv2_3_hdr->sequence = seq_number;
- index = (1UL << AS_PUSH_SHIFT) | bcp->uvhub_cpu;
- bcp->send_message = get_cycles();
-
- write_mmr_activation(index);
-
- try++;
- completion_stat = ops.wait_completion(bau_desc, bcp, try);
-
- handle_cmplt(completion_stat, bau_desc, bcp, hmaster, stat);
-
- if (bcp->ipi_attempts >= bcp->ipi_reset_limit) {
- bcp->ipi_attempts = 0;
- stat->s_overipilimit++;
- completion_stat = FLUSH_GIVEUP;
- break;
- }
- cpu_relax();
- } while ((completion_stat == FLUSH_RETRY_PLUGGED) ||
- (completion_stat == FLUSH_RETRY_TIMEOUT));
-
- time2 = get_cycles();
-
- count_max_concurr(completion_stat, bcp, hmaster);
-
- while (hmaster->uvhub_quiesce)
- cpu_relax();
-
- atomic_dec(&hmaster->active_descriptor_count);
-
- record_send_stats(time1, time2, bcp, stat, completion_stat, try);
-
- if (completion_stat == FLUSH_GIVEUP)
- /* FLUSH_GIVEUP will fall back to using IPI's for tlb flush */
- return 1;
- return 0;
-}
-
-/*
- * The BAU is disabled for this uvhub. When the disabled time period has
- * expired re-enable it.
- * Return 0 if it is re-enabled for all cpus on this uvhub.
- */
-static int check_enable(struct bau_control *bcp, struct ptc_stats *stat)
-{
- int tcpu;
- struct bau_control *tbcp;
- struct bau_control *hmaster;
-
- hmaster = bcp->uvhub_master;
- spin_lock(&hmaster->disable_lock);
- if (bcp->baudisabled && (get_cycles() >= bcp->set_bau_on_time)) {
- stat->s_bau_reenabled++;
- for_each_present_cpu(tcpu) {
- tbcp = &per_cpu(bau_control, tcpu);
- if (tbcp->uvhub_master == hmaster) {
- tbcp->baudisabled = 0;
- tbcp->period_requests = 0;
- tbcp->period_time = 0;
- tbcp->period_giveups = 0;
- }
- }
- spin_unlock(&hmaster->disable_lock);
- return 0;
- }
- spin_unlock(&hmaster->disable_lock);
- return -1;
-}
-
-static void record_send_statistics(struct ptc_stats *stat, int locals, int hubs,
- int remotes, struct bau_desc *bau_desc)
-{
- stat->s_requestor++;
- stat->s_ntargcpu += remotes + locals;
- stat->s_ntargremotes += remotes;
- stat->s_ntarglocals += locals;
-
- /* uvhub statistics */
- hubs = bau_uvhub_weight(&bau_desc->distribution);
- if (locals) {
- stat->s_ntarglocaluvhub++;
- stat->s_ntargremoteuvhub += (hubs - 1);
- } else
- stat->s_ntargremoteuvhub += hubs;
-
- stat->s_ntarguvhub += hubs;
-
- if (hubs >= 16)
- stat->s_ntarguvhub16++;
- else if (hubs >= 8)
- stat->s_ntarguvhub8++;
- else if (hubs >= 4)
- stat->s_ntarguvhub4++;
- else if (hubs >= 2)
- stat->s_ntarguvhub2++;
- else
- stat->s_ntarguvhub1++;
-}
-
-/*
- * Translate a cpu mask to the uvhub distribution mask in the BAU
- * activation descriptor.
- */
-static int set_distrib_bits(struct cpumask *flush_mask, struct bau_control *bcp,
- struct bau_desc *bau_desc, int *localsp, int *remotesp)
-{
- int cpu;
- int pnode;
- int cnt = 0;
- struct hub_and_pnode *hpp;
-
- for_each_cpu(cpu, flush_mask) {
- /*
- * The distribution vector is a bit map of pnodes, relative
- * to the partition base pnode (and the partition base nasid
- * in the header).
- * Translate cpu to pnode and hub using a local memory array.
- */
- hpp = &bcp->socket_master->thp[cpu];
- pnode = hpp->pnode - bcp->partition_base_pnode;
- bau_uvhub_set(pnode, &bau_desc->distribution);
- cnt++;
- if (hpp->uvhub == bcp->uvhub)
- (*localsp)++;
- else
- (*remotesp)++;
- }
- if (!cnt)
- return 1;
- return 0;
-}
-
-/*
- * globally purge translation cache of a virtual address or all TLB's
- * @cpumask: mask of all cpu's in which the address is to be removed
- * @mm: mm_struct containing virtual address range
- * @start: start virtual address to be removed from TLB
- * @end: end virtual address to be remove from TLB
- * @cpu: the current cpu
- *
- * This is the entry point for initiating any UV global TLB shootdown.
- *
- * Purges the translation caches of all specified processors of the given
- * virtual address, or purges all TLB's on specified processors.
- *
- * The caller has derived the cpumask from the mm_struct. This function
- * is called only if there are bits set in the mask. (e.g. flush_tlb_page())
- *
- * The cpumask is converted into a uvhubmask of the uvhubs containing
- * those cpus.
- *
- * Note that this function should be called with preemption disabled.
- *
- * Returns NULL if all remote flushing was done.
- * Returns pointer to cpumask if some remote flushing remains to be
- * done. The returned pointer is valid till preemption is re-enabled.
- */
-const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
- const struct flush_tlb_info *info)
-{
- unsigned int cpu = smp_processor_id();
- int locals = 0, remotes = 0, hubs = 0;
- struct bau_desc *bau_desc;
- struct cpumask *flush_mask;
- struct ptc_stats *stat;
- struct bau_control *bcp;
- unsigned long descriptor_status, status, address;
-
- bcp = &per_cpu(bau_control, cpu);
-
- if (bcp->nobau)
- return cpumask;
-
- stat = bcp->statp;
- stat->s_enters++;
-
- if (bcp->busy) {
- descriptor_status =
- read_lmmr(UVH_LB_BAU_SB_ACTIVATION_STATUS_0);
- status = ((descriptor_status >> (bcp->uvhub_cpu *
- UV_ACT_STATUS_SIZE)) & UV_ACT_STATUS_MASK) << 1;
- if (status == UV2H_DESC_BUSY)
- return cpumask;
- bcp->busy = 0;
- }
-
- /* bau was disabled due to slow response */
- if (bcp->baudisabled) {
- if (check_enable(bcp, stat)) {
- stat->s_ipifordisabled++;
- return cpumask;
- }
- }
-
- /*
- * Each sending cpu has a per-cpu mask which it fills from the caller's
- * cpu mask. All cpus are converted to uvhubs and copied to the
- * activation descriptor.
- */
- flush_mask = (struct cpumask *)per_cpu(uv_flush_tlb_mask, cpu);
- /* don't actually do a shootdown of the local cpu */
- cpumask_andnot(flush_mask, cpumask, cpumask_of(cpu));
-
- if (cpumask_test_cpu(cpu, cpumask))
- stat->s_ntargself++;
-
- bau_desc = bcp->descriptor_base;
- bau_desc += (ITEMS_PER_DESC * bcp->uvhub_cpu);
- bau_uvhubs_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE);
- if (set_distrib_bits(flush_mask, bcp, bau_desc, &locals, &remotes))
- return NULL;
-
- record_send_statistics(stat, locals, hubs, remotes, bau_desc);
-
- if (!info->end || (info->end - info->start) <= PAGE_SIZE)
- address = info->start;
- else
- address = TLB_FLUSH_ALL;
-
- switch (bcp->uvhub_version) {
- case UV_BAU_V2:
- case UV_BAU_V3:
- bau_desc->payload.uv2_3.address = address;
- bau_desc->payload.uv2_3.sending_cpu = cpu;
- break;
- case UV_BAU_V4:
- bau_desc->payload.uv4.address = address;
- bau_desc->payload.uv4.sending_cpu = cpu;
- bau_desc->payload.uv4.qualifier = BAU_DESC_QUALIFIER;
- break;
- }
-
- /*
- * uv_flush_send_and_wait returns 0 if all cpu's were messaged,
- * or 1 if it gave up and the original cpumask should be returned.
- */
- if (!uv_flush_send_and_wait(flush_mask, bcp, bau_desc))
- return NULL;
- else
- return cpumask;
-}
-
-/*
- * Search the message queue for any 'other' unprocessed message with the
- * same software acknowledge resource bit vector as the 'msg' message.
- */
-static struct bau_pq_entry *find_another_by_swack(struct bau_pq_entry *msg,
- struct bau_control *bcp)
-{
- struct bau_pq_entry *msg_next = msg + 1;
- unsigned char swack_vec = msg->swack_vec;
-
- if (msg_next > bcp->queue_last)
- msg_next = bcp->queue_first;
- while (msg_next != msg) {
- if ((msg_next->canceled == 0) && (msg_next->replied_to == 0) &&
- (msg_next->swack_vec == swack_vec))
- return msg_next;
- msg_next++;
- if (msg_next > bcp->queue_last)
- msg_next = bcp->queue_first;
- }
- return NULL;
-}
-
-/*
- * UV2 needs to work around a bug in which an arriving message has not
- * set a bit in the UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE register.
- * Such a message must be ignored.
- */
-static void process_uv2_message(struct msg_desc *mdp, struct bau_control *bcp)
-{
- unsigned long mmr_image;
- unsigned char swack_vec;
- struct bau_pq_entry *msg = mdp->msg;
- struct bau_pq_entry *other_msg;
-
- mmr_image = ops.read_l_sw_ack();
- swack_vec = msg->swack_vec;
-
- if ((swack_vec & mmr_image) == 0) {
- /*
- * This message was assigned a swack resource, but no
- * reserved acknowlegment is pending.
- * The bug has prevented this message from setting the MMR.
- */
- /*
- * Some message has set the MMR 'pending' bit; it might have
- * been another message. Look for that message.
- */
- other_msg = find_another_by_swack(msg, bcp);
- if (other_msg) {
- /*
- * There is another. Process this one but do not
- * ack it.
- */
- bau_process_message(mdp, bcp, 0);
- /*
- * Let the natural processing of that other message
- * acknowledge it. Don't get the processing of sw_ack's
- * out of order.
- */
- return;
- }
- }
-
- /*
- * Either the MMR shows this one pending a reply or there is no
- * other message using this sw_ack, so it is safe to acknowledge it.
- */
- bau_process_message(mdp, bcp, 1);
-
- return;
-}
-
-/*
- * The BAU message interrupt comes here. (registered by set_intr_gate)
- * See entry_64.S
- *
- * We received a broadcast assist message.
- *
- * Interrupts are disabled; this interrupt could represent
- * the receipt of several messages.
- *
- * All cores/threads on this hub get this interrupt.
- * The last one to see it does the software ack.
- * (the resource will not be freed until noninterruptable cpus see this
- * interrupt; hardware may timeout the s/w ack and reply ERROR)
- */
-DEFINE_IDTENTRY_SYSVEC(sysvec_uv_bau_message)
-{
- int count = 0;
- cycles_t time_start;
- struct bau_pq_entry *msg;
- struct bau_control *bcp;
- struct ptc_stats *stat;
- struct msg_desc msgdesc;
-
- ack_APIC_irq();
- kvm_set_cpu_l1tf_flush_l1d();
- time_start = get_cycles();
-
- bcp = &per_cpu(bau_control, smp_processor_id());
- stat = bcp->statp;
-
- msgdesc.queue_first = bcp->queue_first;
- msgdesc.queue_last = bcp->queue_last;
-
- msg = bcp->bau_msg_head;
- while (msg->swack_vec) {
- count++;
-
- msgdesc.msg_slot = msg - msgdesc.queue_first;
- msgdesc.msg = msg;
- if (bcp->uvhub_version == UV_BAU_V2)
- process_uv2_message(&msgdesc, bcp);
- else
- /* no error workaround for uv3 */
- bau_process_message(&msgdesc, bcp, 1);
-
- msg++;
- if (msg > msgdesc.queue_last)
- msg = msgdesc.queue_first;
- bcp->bau_msg_head = msg;
- }
- stat->d_time += (get_cycles() - time_start);
- if (!count)
- stat->d_nomsg++;
- else if (count > 1)
- stat->d_multmsg++;
-}
-
-/*
- * Each target uvhub (i.e. a uvhub that has cpu's) needs to have
- * shootdown message timeouts enabled. The timeout does not cause
- * an interrupt, but causes an error message to be returned to
- * the sender.
- */
-static void __init enable_timeouts(void)
-{
- int uvhub;
- int nuvhubs;
- int pnode;
- unsigned long mmr_image;
-
- nuvhubs = uv_num_possible_blades();
-
- for (uvhub = 0; uvhub < nuvhubs; uvhub++) {
- if (!uv_blade_nr_possible_cpus(uvhub))
- continue;
-
- pnode = uv_blade_to_pnode(uvhub);
- mmr_image = read_mmr_misc_control(pnode);
- /*
- * Set the timeout period and then lock it in, in three
- * steps; captures and locks in the period.
- *
- * To program the period, the SOFT_ACK_MODE must be off.
- */
- mmr_image &= ~(1L << SOFTACK_MSHIFT);
- write_mmr_misc_control(pnode, mmr_image);
- /*
- * Set the 4-bit period.
- */
- mmr_image &= ~((unsigned long)0xf << SOFTACK_PSHIFT);
- mmr_image |= (SOFTACK_TIMEOUT_PERIOD << SOFTACK_PSHIFT);
- write_mmr_misc_control(pnode, mmr_image);
-
- mmr_image |= (1L << SOFTACK_MSHIFT);
- if (is_uv2_hub()) {
- /* do not touch the legacy mode bit */
- /* hw bug workaround; do not use extended status */
- mmr_image &= ~(1L << UV2_EXT_SHFT);
- } else if (is_uv3_hub()) {
- mmr_image &= ~(1L << PREFETCH_HINT_SHFT);
- mmr_image |= (1L << SB_STATUS_SHFT);
- }
- write_mmr_misc_control(pnode, mmr_image);
- }
-}
-
-static void *ptc_seq_start(struct seq_file *file, loff_t *offset)
-{
- if (*offset < num_possible_cpus())
- return offset;
- return NULL;
-}
-
-static void *ptc_seq_next(struct seq_file *file, void *data, loff_t *offset)
-{
- (*offset)++;
- if (*offset < num_possible_cpus())
- return offset;
- return NULL;
-}
-
-static void ptc_seq_stop(struct seq_file *file, void *data)
-{
-}
-
-/*
- * Display the statistics thru /proc/sgi_uv/ptc_statistics
- * 'data' points to the cpu number
- * Note: see the descriptions in stat_description[].
- */
-static int ptc_seq_show(struct seq_file *file, void *data)
-{
- struct ptc_stats *stat;
- struct bau_control *bcp;
- int cpu;
-
- cpu = *(loff_t *)data;
- if (!cpu) {
- seq_puts(file,
- "# cpu bauoff sent stime self locals remotes ncpus localhub ");
- seq_puts(file, "remotehub numuvhubs numuvhubs16 numuvhubs8 ");
- seq_puts(file,
- "numuvhubs4 numuvhubs2 numuvhubs1 dto snacks retries ");
- seq_puts(file,
- "rok resetp resett giveup sto bz throt disable ");
- seq_puts(file,
- "enable wars warshw warwaits enters ipidis plugged ");
- seq_puts(file,
- "ipiover glim cong swack recv rtime all one mult ");
- seq_puts(file, "none retry canc nocan reset rcan\n");
- }
- if (cpu < num_possible_cpus() && cpu_online(cpu)) {
- bcp = &per_cpu(bau_control, cpu);
- if (bcp->nobau) {
- seq_printf(file, "cpu %d bau disabled\n", cpu);
- return 0;
- }
- stat = bcp->statp;
- /* source side statistics */
- seq_printf(file,
- "cpu %d %d %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld ",
- cpu, bcp->nobau, stat->s_requestor,
- cycles_2_us(stat->s_time),
- stat->s_ntargself, stat->s_ntarglocals,
- stat->s_ntargremotes, stat->s_ntargcpu,
- stat->s_ntarglocaluvhub, stat->s_ntargremoteuvhub,
- stat->s_ntarguvhub, stat->s_ntarguvhub16);
- seq_printf(file, "%ld %ld %ld %ld %ld %ld ",
- stat->s_ntarguvhub8, stat->s_ntarguvhub4,
- stat->s_ntarguvhub2, stat->s_ntarguvhub1,
- stat->s_dtimeout, stat->s_strongnacks);
- seq_printf(file, "%ld %ld %ld %ld %ld %ld %ld %ld ",
- stat->s_retry_messages, stat->s_retriesok,
- stat->s_resets_plug, stat->s_resets_timeout,
- stat->s_giveup, stat->s_stimeout,
- stat->s_busy, stat->s_throttles);
- seq_printf(file, "%ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld ",
- stat->s_bau_disabled, stat->s_bau_reenabled,
- stat->s_uv2_wars, stat->s_uv2_wars_hw,
- stat->s_uv2_war_waits, stat->s_enters,
- stat->s_ipifordisabled, stat->s_plugged,
- stat->s_overipilimit, stat->s_giveuplimit,
- stat->s_congested);
-
- /* destination side statistics */
- seq_printf(file,
- "%lx %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld\n",
- ops.read_g_sw_ack(uv_cpu_to_pnode(cpu)),
- stat->d_requestee, cycles_2_us(stat->d_time),
- stat->d_alltlb, stat->d_onetlb, stat->d_multmsg,
- stat->d_nomsg, stat->d_retries, stat->d_canceled,
- stat->d_nocanceled, stat->d_resets,
- stat->d_rcanceled);
- }
- return 0;
-}
-
-/*
- * Display the tunables thru debugfs
- */
-static ssize_t tunables_read(struct file *file, char __user *userbuf,
- size_t count, loff_t *ppos)
-{
- char *buf;
- int ret;
-
- buf = kasprintf(GFP_KERNEL, "%s %s %s\n%d %d %d %d %d %d %d %d %d %d\n",
- "max_concur plugged_delay plugsb4reset timeoutsb4reset",
- "ipi_reset_limit complete_threshold congested_response_us",
- "congested_reps disabled_period giveup_limit",
- max_concurr, plugged_delay, plugsb4reset,
- timeoutsb4reset, ipi_reset_limit, complete_threshold,
- congested_respns_us, congested_reps, disabled_period,
- giveup_limit);
-
- if (!buf)
- return -ENOMEM;
-
- ret = simple_read_from_buffer(userbuf, count, ppos, buf, strlen(buf));
- kfree(buf);
- return ret;
-}
-
-/*
- * handle a write to /proc/sgi_uv/ptc_statistics
- * -1: reset the statistics
- * 0: display meaning of the statistics
- */
-static ssize_t ptc_proc_write(struct file *file, const char __user *user,
- size_t count, loff_t *data)
-{
- int cpu;
- int i;
- int elements;
- long input_arg;
- char optstr[64];
- struct ptc_stats *stat;
-
- if (count == 0 || count > sizeof(optstr))
- return -EINVAL;
- if (copy_from_user(optstr, user, count))
- return -EFAULT;
- optstr[count - 1] = '\0';
-
- if (!strcmp(optstr, "on")) {
- set_bau_on();
- return count;
- } else if (!strcmp(optstr, "off")) {
- set_bau_off();
- return count;
- }
-
- if (kstrtol(optstr, 10, &input_arg) < 0) {
- pr_debug("%s is invalid\n", optstr);
- return -EINVAL;
- }
-
- if (input_arg == 0) {
- elements = ARRAY_SIZE(stat_description);
- pr_debug("# cpu: cpu number\n");
- pr_debug("Sender statistics:\n");
- for (i = 0; i < elements; i++)
- pr_debug("%s\n", stat_description[i]);
- } else if (input_arg == -1) {
- for_each_present_cpu(cpu) {
- stat = &per_cpu(ptcstats, cpu);
- memset(stat, 0, sizeof(struct ptc_stats));
- }
- }
-
- return count;
-}
-
-static int local_atoi(const char *name)
-{
- int val = 0;
-
- for (;; name++) {
- switch (*name) {
- case '0' ... '9':
- val = 10*val+(*name-'0');
- break;
- default:
- return val;
- }
- }
-}
-
-/*
- * Parse the values written to /sys/kernel/debug/sgi_uv/bau_tunables.
- * Zero values reset them to defaults.
- */
-static int parse_tunables_write(struct bau_control *bcp, char *instr,
- int count)
-{
- char *p;
- char *q;
- int cnt = 0;
- int val;
- int e = ARRAY_SIZE(tunables);
-
- p = instr + strspn(instr, WHITESPACE);
- q = p;
- for (; *p; p = q + strspn(q, WHITESPACE)) {
- q = p + strcspn(p, WHITESPACE);
- cnt++;
- if (q == p)
- break;
- }
- if (cnt != e) {
- pr_info("bau tunable error: should be %d values\n", e);
- return -EINVAL;
- }
-
- p = instr + strspn(instr, WHITESPACE);
- q = p;
- for (cnt = 0; *p; p = q + strspn(q, WHITESPACE), cnt++) {
- q = p + strcspn(p, WHITESPACE);
- val = local_atoi(p);
- switch (cnt) {
- case 0:
- if (val == 0) {
- max_concurr = MAX_BAU_CONCURRENT;
- max_concurr_const = MAX_BAU_CONCURRENT;
- continue;
- }
- if (val < 1 || val > bcp->cpus_in_uvhub) {
- pr_debug(
- "Error: BAU max concurrent %d is invalid\n",
- val);
- return -EINVAL;
- }
- max_concurr = val;
- max_concurr_const = val;
- continue;
- default:
- if (val == 0)
- *tunables[cnt].tunp = tunables[cnt].deflt;
- else
- *tunables[cnt].tunp = val;
- continue;
- }
- }
- return 0;
-}
-
-/*
- * Handle a write to debugfs. (/sys/kernel/debug/sgi_uv/bau_tunables)
- */
-static ssize_t tunables_write(struct file *file, const char __user *user,
- size_t count, loff_t *data)
-{
- int cpu;
- int ret;
- char instr[100];
- struct bau_control *bcp;
-
- if (count == 0 || count > sizeof(instr)-1)
- return -EINVAL;
- if (copy_from_user(instr, user, count))
- return -EFAULT;
-
- instr[count] = '\0';
-
- cpu = get_cpu();
- bcp = &per_cpu(bau_control, cpu);
- ret = parse_tunables_write(bcp, instr, count);
- put_cpu();
- if (ret)
- return ret;
-
- for_each_present_cpu(cpu) {
- bcp = &per_cpu(bau_control, cpu);
- bcp->max_concurr = max_concurr;
- bcp->max_concurr_const = max_concurr;
- bcp->plugged_delay = plugged_delay;
- bcp->plugsb4reset = plugsb4reset;
- bcp->timeoutsb4reset = timeoutsb4reset;
- bcp->ipi_reset_limit = ipi_reset_limit;
- bcp->complete_threshold = complete_threshold;
- bcp->cong_response_us = congested_respns_us;
- bcp->cong_reps = congested_reps;
- bcp->disabled_period = sec_2_cycles(disabled_period);
- bcp->giveup_limit = giveup_limit;
- }
- return count;
-}
-
-static const struct seq_operations uv_ptc_seq_ops = {
- .start = ptc_seq_start,
- .next = ptc_seq_next,
- .stop = ptc_seq_stop,
- .show = ptc_seq_show
-};
-
-static int ptc_proc_open(struct inode *inode, struct file *file)
-{
- return seq_open(file, &uv_ptc_seq_ops);
-}
-
-static int tunables_open(struct inode *inode, struct file *file)
-{
- return 0;
-}
-
-static const struct proc_ops uv_ptc_proc_ops = {
- .proc_open = ptc_proc_open,
- .proc_read = seq_read,
- .proc_write = ptc_proc_write,
- .proc_lseek = seq_lseek,
- .proc_release = seq_release,
-};
-
-static const struct file_operations tunables_fops = {
- .open = tunables_open,
- .read = tunables_read,
- .write = tunables_write,
- .llseek = default_llseek,
-};
-
-static int __init uv_ptc_init(void)
-{
- struct proc_dir_entry *proc_uv_ptc;
-
- if (!is_uv_system())
- return 0;
-
- proc_uv_ptc = proc_create(UV_PTC_BASENAME, 0444, NULL,
- &uv_ptc_proc_ops);
- if (!proc_uv_ptc) {
- pr_err("unable to create %s proc entry\n",
- UV_PTC_BASENAME);
- return -EINVAL;
- }
-
- tunables_dir = debugfs_create_dir(UV_BAU_TUNABLES_DIR, NULL);
- debugfs_create_file(UV_BAU_TUNABLES_FILE, 0600, tunables_dir, NULL,
- &tunables_fops);
- return 0;
-}
-
-/*
- * Initialize the sending side's sending buffers.
- */
-static void activation_descriptor_init(int node, int pnode, int base_pnode)
-{
- int i;
- int cpu;
- unsigned long gpa;
- unsigned long m;
- unsigned long n;
- size_t dsize;
- struct bau_desc *bau_desc;
- struct bau_desc *bd2;
- struct uv2_3_bau_msg_header *uv2_3_hdr;
- struct bau_control *bcp;
-
- /*
- * each bau_desc is 64 bytes; there are 8 (ITEMS_PER_DESC)
- * per cpu; and one per cpu on the uvhub (ADP_SZ)
- */
- dsize = sizeof(struct bau_desc) * ADP_SZ * ITEMS_PER_DESC;
- bau_desc = kmalloc_node(dsize, GFP_KERNEL, node);
- BUG_ON(!bau_desc);
-
- gpa = uv_gpa(bau_desc);
- n = uv_gpa_to_gnode(gpa);
- m = ops.bau_gpa_to_offset(gpa);
-
- /* the 14-bit pnode */
- write_mmr_descriptor_base(pnode,
- (n << UVH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT | m));
- /*
- * Initializing all 8 (ITEMS_PER_DESC) descriptors for each
- * cpu even though we only use the first one; one descriptor can
- * describe a broadcast to 256 uv hubs.
- */
- for (i = 0, bd2 = bau_desc; i < (ADP_SZ * ITEMS_PER_DESC); i++, bd2++) {
- memset(bd2, 0, sizeof(struct bau_desc));
- /*
- * BIOS uses legacy mode, but uv2 and uv3 hardware always
- * uses native mode for selective broadcasts.
- */
- uv2_3_hdr = &bd2->header.uv2_3_hdr;
- uv2_3_hdr->swack_flag = 1;
- uv2_3_hdr->base_dest_nasid = UV_PNODE_TO_NASID(base_pnode);
- uv2_3_hdr->dest_subnodeid = UV_LB_SUBNODEID;
- uv2_3_hdr->command = UV_NET_ENDPOINT_INTD;
- }
- for_each_present_cpu(cpu) {
- if (pnode != uv_blade_to_pnode(uv_cpu_to_blade_id(cpu)))
- continue;
- bcp = &per_cpu(bau_control, cpu);
- bcp->descriptor_base = bau_desc;
- }
-}
-
-/*
- * initialize the destination side's receiving buffers
- * entered for each uvhub in the partition
- * - node is first node (kernel memory notion) on the uvhub
- * - pnode is the uvhub's physical identifier
- */
-static void pq_init(int node, int pnode)
-{
- int cpu;
- size_t plsize;
- char *cp;
- void *vp;
- unsigned long gnode, first, last, tail;
- struct bau_pq_entry *pqp;
- struct bau_control *bcp;
-
- plsize = (DEST_Q_SIZE + 1) * sizeof(struct bau_pq_entry);
- vp = kmalloc_node(plsize, GFP_KERNEL, node);
- BUG_ON(!vp);
-
- pqp = (struct bau_pq_entry *)vp;
- cp = (char *)pqp + 31;
- pqp = (struct bau_pq_entry *)(((unsigned long)cp >> 5) << 5);
-
- for_each_present_cpu(cpu) {
- if (pnode != uv_cpu_to_pnode(cpu))
- continue;
- /* for every cpu on this pnode: */
- bcp = &per_cpu(bau_control, cpu);
- bcp->queue_first = pqp;
- bcp->bau_msg_head = pqp;
- bcp->queue_last = pqp + (DEST_Q_SIZE - 1);
- }
-
- first = ops.bau_gpa_to_offset(uv_gpa(pqp));
- last = ops.bau_gpa_to_offset(uv_gpa(pqp + (DEST_Q_SIZE - 1)));
-
- /*
- * Pre UV4, the gnode is required to locate the payload queue
- * and the payload queue tail must be maintained by the kernel.
- */
- bcp = &per_cpu(bau_control, smp_processor_id());
- if (bcp->uvhub_version <= UV_BAU_V3) {
- tail = first;
- gnode = uv_gpa_to_gnode(uv_gpa(pqp));
- first = (gnode << UV_PAYLOADQ_GNODE_SHIFT) | tail;
- write_mmr_payload_tail(pnode, tail);
- }
-
- ops.write_payload_first(pnode, first);
- ops.write_payload_last(pnode, last);
-
- /* in effect, all msg_type's are set to MSG_NOOP */
- memset(pqp, 0, sizeof(struct bau_pq_entry) * DEST_Q_SIZE);
-}
-
-/*
- * Initialization of each UV hub's structures
- */
-static void __init init_uvhub(int uvhub, int vector, int base_pnode)
-{
- int node;
- int pnode;
- unsigned long apicid;
-
- node = uvhub_to_first_node(uvhub);
- pnode = uv_blade_to_pnode(uvhub);
-
- activation_descriptor_init(node, pnode, base_pnode);
-
- pq_init(node, pnode);
- /*
- * The below initialization can't be in firmware because the
- * messaging IRQ will be determined by the OS.
- */
- apicid = uvhub_to_first_apicid(uvhub);
- write_mmr_data_config(pnode, ((apicid << 32) | vector));
-}
-
-/*
- * We will set BAU_MISC_CONTROL with a timeout period.
- * But the BIOS has set UVH_AGING_PRESCALE_SEL and UVH_TRANSACTION_TIMEOUT.
- * So the destination timeout period has to be calculated from them.
- */
-static int calculate_destination_timeout(void)
-{
- unsigned long mmr_image;
- int mult1;
- int base;
- int ret;
-
- /* same destination timeout for uv2 and uv3 */
- /* 4 bits 0/1 for 10/80us base, 3 bits of multiplier */
- mmr_image = uv_read_local_mmr(UVH_LB_BAU_MISC_CONTROL);
- mmr_image = (mmr_image & UV_SA_MASK) >> UV_SA_SHFT;
- if (mmr_image & (1L << UV2_ACK_UNITS_SHFT))
- base = 80;
- else
- base = 10;
- mult1 = mmr_image & UV2_ACK_MASK;
- ret = mult1 * base;
-
- return ret;
-}
-
-static void __init init_per_cpu_tunables(void)
-{
- int cpu;
- struct bau_control *bcp;
-
- for_each_present_cpu(cpu) {
- bcp = &per_cpu(bau_control, cpu);
- bcp->baudisabled = 0;
- if (nobau)
- bcp->nobau = true;
- bcp->statp = &per_cpu(ptcstats, cpu);
- /* time interval to catch a hardware stay-busy bug */
- bcp->timeout_interval = usec_2_cycles(2*timeout_us);
- bcp->max_concurr = max_concurr;
- bcp->max_concurr_const = max_concurr;
- bcp->plugged_delay = plugged_delay;
- bcp->plugsb4reset = plugsb4reset;
- bcp->timeoutsb4reset = timeoutsb4reset;
- bcp->ipi_reset_limit = ipi_reset_limit;
- bcp->complete_threshold = complete_threshold;
- bcp->cong_response_us = congested_respns_us;
- bcp->cong_reps = congested_reps;
- bcp->disabled_period = sec_2_cycles(disabled_period);
- bcp->giveup_limit = giveup_limit;
- spin_lock_init(&bcp->queue_lock);
- spin_lock_init(&bcp->uvhub_lock);
- spin_lock_init(&bcp->disable_lock);
- }
-}
-
-/*
- * Scan all cpus to collect blade and socket summaries.
- */
-static int __init get_cpu_topology(int base_pnode,
- struct uvhub_desc *uvhub_descs,
- unsigned char *uvhub_mask)
-{
- int cpu;
- int pnode;
- int uvhub;
- int socket;
- struct bau_control *bcp;
- struct uvhub_desc *bdp;
- struct socket_desc *sdp;
-
- for_each_present_cpu(cpu) {
- bcp = &per_cpu(bau_control, cpu);
-
- memset(bcp, 0, sizeof(struct bau_control));
-
- pnode = uv_cpu_hub_info(cpu)->pnode;
- if ((pnode - base_pnode) >= UV_DISTRIBUTION_SIZE) {
- pr_emerg(
- "cpu %d pnode %d-%d beyond %d; BAU disabled\n",
- cpu, pnode, base_pnode, UV_DISTRIBUTION_SIZE);
- return 1;
- }
-
- bcp->osnode = cpu_to_node(cpu);
- bcp->partition_base_pnode = base_pnode;
-
- uvhub = uv_cpu_hub_info(cpu)->numa_blade_id;
- *(uvhub_mask + (uvhub/8)) |= (1 << (uvhub%8));
- bdp = &uvhub_descs[uvhub];
-
- bdp->num_cpus++;
- bdp->uvhub = uvhub;
- bdp->pnode = pnode;
-
- /* kludge: 'assuming' one node per socket, and assuming that
- disabling a socket just leaves a gap in node numbers */
- socket = bcp->osnode & 1;
- bdp->socket_mask |= (1 << socket);
- sdp = &bdp->socket[socket];
- sdp->cpu_number[sdp->num_cpus] = cpu;
- sdp->num_cpus++;
- if (sdp->num_cpus > MAX_CPUS_PER_SOCKET) {
- pr_emerg("%d cpus per socket invalid\n",
- sdp->num_cpus);
- return 1;
- }
- }
- return 0;
-}
-
-/*
- * Each socket is to get a local array of pnodes/hubs.
- */
-static void make_per_cpu_thp(struct bau_control *smaster)
-{
- int cpu;
- size_t hpsz = sizeof(struct hub_and_pnode) * num_possible_cpus();
-
- smaster->thp = kzalloc_node(hpsz, GFP_KERNEL, smaster->osnode);
- for_each_present_cpu(cpu) {
- smaster->thp[cpu].pnode = uv_cpu_hub_info(cpu)->pnode;
- smaster->thp[cpu].uvhub = uv_cpu_hub_info(cpu)->numa_blade_id;
- }
-}
-
-/*
- * Each uvhub is to get a local cpumask.
- */
-static void make_per_hub_cpumask(struct bau_control *hmaster)
-{
- int sz = sizeof(cpumask_t);
-
- hmaster->cpumask = kzalloc_node(sz, GFP_KERNEL, hmaster->osnode);
-}
-
-/*
- * Initialize all the per_cpu information for the cpu's on a given socket,
- * given what has been gathered into the socket_desc struct.
- * And reports the chosen hub and socket masters back to the caller.
- */
-static int scan_sock(struct socket_desc *sdp, struct uvhub_desc *bdp,
- struct bau_control **smasterp,
- struct bau_control **hmasterp)
-{
- int i, cpu, uvhub_cpu;
- struct bau_control *bcp;
-
- for (i = 0; i < sdp->num_cpus; i++) {
- cpu = sdp->cpu_number[i];
- bcp = &per_cpu(bau_control, cpu);
- bcp->cpu = cpu;
- if (i == 0) {
- *smasterp = bcp;
- if (!(*hmasterp))
- *hmasterp = bcp;
- }
- bcp->cpus_in_uvhub = bdp->num_cpus;
- bcp->cpus_in_socket = sdp->num_cpus;
- bcp->socket_master = *smasterp;
- bcp->uvhub = bdp->uvhub;
- if (is_uv2_hub())
- bcp->uvhub_version = UV_BAU_V2;
- else if (is_uv3_hub())
- bcp->uvhub_version = UV_BAU_V3;
- else if (is_uv4_hub())
- bcp->uvhub_version = UV_BAU_V4;
- else {
- pr_emerg("uvhub version not 1, 2, 3, or 4\n");
- return 1;
- }
- bcp->uvhub_master = *hmasterp;
- uvhub_cpu = uv_cpu_blade_processor_id(cpu);
- bcp->uvhub_cpu = uvhub_cpu;
-
- /*
- * The ERROR and BUSY status registers are located pairwise over
- * the STATUS_0 and STATUS_1 mmrs; each an array[32] of 2 bits.
- */
- if (uvhub_cpu < UV_CPUS_PER_AS) {
- bcp->status_mmr = UVH_LB_BAU_SB_ACTIVATION_STATUS_0;
- bcp->status_index = uvhub_cpu * UV_ACT_STATUS_SIZE;
- } else {
- bcp->status_mmr = UVH_LB_BAU_SB_ACTIVATION_STATUS_1;
- bcp->status_index = (uvhub_cpu - UV_CPUS_PER_AS)
- * UV_ACT_STATUS_SIZE;
- }
-
- if (bcp->uvhub_cpu >= MAX_CPUS_PER_UVHUB) {
- pr_emerg("%d cpus per uvhub invalid\n",
- bcp->uvhub_cpu);
- return 1;
- }
- }
- return 0;
-}
-
-/*
- * Summarize the blade and socket topology into the per_cpu structures.
- */
-static int __init summarize_uvhub_sockets(int nuvhubs,
- struct uvhub_desc *uvhub_descs,
- unsigned char *uvhub_mask)
-{
- int socket;
- int uvhub;
- unsigned short socket_mask;
-
- for (uvhub = 0; uvhub < nuvhubs; uvhub++) {
- struct uvhub_desc *bdp;
- struct bau_control *smaster = NULL;
- struct bau_control *hmaster = NULL;
-
- if (!(*(uvhub_mask + (uvhub/8)) & (1 << (uvhub%8))))
- continue;
-
- bdp = &uvhub_descs[uvhub];
- socket_mask = bdp->socket_mask;
- socket = 0;
- while (socket_mask) {
- struct socket_desc *sdp;
- if ((socket_mask & 1)) {
- sdp = &bdp->socket[socket];
- if (scan_sock(sdp, bdp, &smaster, &hmaster))
- return 1;
- make_per_cpu_thp(smaster);
- }
- socket++;
- socket_mask = (socket_mask >> 1);
- }
- make_per_hub_cpumask(hmaster);
- }
- return 0;
-}
-
-/*
- * initialize the bau_control structure for each cpu
- */
-static int __init init_per_cpu(int nuvhubs, int base_part_pnode)
-{
- struct uvhub_desc *uvhub_descs;
- unsigned char *uvhub_mask = NULL;
-
- if (is_uv3_hub() || is_uv2_hub())
- timeout_us = calculate_destination_timeout();
-
- uvhub_descs = kcalloc(nuvhubs, sizeof(struct uvhub_desc), GFP_KERNEL);
- if (!uvhub_descs)
- goto fail;
-
- uvhub_mask = kzalloc((nuvhubs+7)/8, GFP_KERNEL);
- if (!uvhub_mask)
- goto fail;
-
- if (get_cpu_topology(base_part_pnode, uvhub_descs, uvhub_mask))
- goto fail;
-
- if (summarize_uvhub_sockets(nuvhubs, uvhub_descs, uvhub_mask))
- goto fail;
-
- kfree(uvhub_descs);
- kfree(uvhub_mask);
- init_per_cpu_tunables();
- return 0;
-
-fail:
- kfree(uvhub_descs);
- kfree(uvhub_mask);
- return 1;
-}
-
-static const struct bau_operations uv2_3_bau_ops __initconst = {
- .bau_gpa_to_offset = uv_gpa_to_offset,
- .read_l_sw_ack = read_mmr_sw_ack,
- .read_g_sw_ack = read_gmmr_sw_ack,
- .write_l_sw_ack = write_mmr_sw_ack,
- .write_g_sw_ack = write_gmmr_sw_ack,
- .write_payload_first = write_mmr_payload_first,
- .write_payload_last = write_mmr_payload_last,
- .wait_completion = uv2_3_wait_completion,
-};
-
-static const struct bau_operations uv4_bau_ops __initconst = {
- .bau_gpa_to_offset = uv_gpa_to_soc_phys_ram,
- .read_l_sw_ack = read_mmr_proc_sw_ack,
- .read_g_sw_ack = read_gmmr_proc_sw_ack,
- .write_l_sw_ack = write_mmr_proc_sw_ack,
- .write_g_sw_ack = write_gmmr_proc_sw_ack,
- .write_payload_first = write_mmr_proc_payload_first,
- .write_payload_last = write_mmr_proc_payload_last,
- .wait_completion = uv4_wait_completion,
-};
-
-/*
- * Initialization of BAU-related structures
- */
-static int __init uv_bau_init(void)
-{
- int uvhub;
- int pnode;
- int nuvhubs;
- int cur_cpu;
- int cpus;
- int vector;
- cpumask_var_t *mask;
-
- if (!is_uv_system())
- return 0;
-
- if (is_uv4_hub())
- ops = uv4_bau_ops;
- else if (is_uv3_hub())
- ops = uv2_3_bau_ops;
- else if (is_uv2_hub())
- ops = uv2_3_bau_ops;
-
- nuvhubs = uv_num_possible_blades();
- if (nuvhubs < 2) {
- pr_crit("UV: BAU disabled - insufficient hub count\n");
- goto err_bau_disable;
- }
-
- for_each_possible_cpu(cur_cpu) {
- mask = &per_cpu(uv_flush_tlb_mask, cur_cpu);
- zalloc_cpumask_var_node(mask, GFP_KERNEL, cpu_to_node(cur_cpu));
- }
-
- uv_base_pnode = 0x7fffffff;
- for (uvhub = 0; uvhub < nuvhubs; uvhub++) {
- cpus = uv_blade_nr_possible_cpus(uvhub);
- if (cpus && (uv_blade_to_pnode(uvhub) < uv_base_pnode))
- uv_base_pnode = uv_blade_to_pnode(uvhub);
- }
-
- /* software timeouts are not supported on UV4 */
- if (is_uv3_hub() || is_uv2_hub())
- enable_timeouts();
-
- if (init_per_cpu(nuvhubs, uv_base_pnode)) {
- pr_crit("UV: BAU disabled - per CPU init failed\n");
- goto err_bau_disable;
- }
-
- vector = UV_BAU_MESSAGE;
- for_each_possible_blade(uvhub) {
- if (uv_blade_nr_possible_cpus(uvhub))
- init_uvhub(uvhub, vector, uv_base_pnode);
- }
-
- for_each_possible_blade(uvhub) {
- if (uv_blade_nr_possible_cpus(uvhub)) {
- unsigned long val;
- unsigned long mmr;
- pnode = uv_blade_to_pnode(uvhub);
- /* INIT the bau */
- val = 1L << 63;
- write_gmmr_activation(pnode, val);
- mmr = 1; /* should be 1 to broadcast to both sockets */
- write_mmr_data_broadcast(pnode, mmr);
- }
- }
-
- return 0;
-
-err_bau_disable:
-
- for_each_possible_cpu(cur_cpu)
- free_cpumask_var(per_cpu(uv_flush_tlb_mask, cur_cpu));
-
- set_bau_off();
- nobau_perm = 1;
-
- return -EINVAL;
-}
-core_initcall(uv_bau_init);
-fs_initcall(uv_ptc_init);
ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
if (ret >= 0) {
- if (info->uv_limit == UV_AFFINITY_CPU)
+ if (info->uv.limit == UV_AFFINITY_CPU)
irq_set_status_flags(virq, IRQ_NO_BALANCING);
else
irq_set_status_flags(virq, IRQ_MOVE_PCNTXT);
- chip_data->pnode = uv_blade_to_pnode(info->uv_blade);
- chip_data->offset = info->uv_offset;
+ chip_data->pnode = uv_blade_to_pnode(info->uv.blade);
+ chip_data->offset = info->uv.offset;
irq_domain_set_info(domain, virq, virq, &uv_irq_chip, chip_data,
- handle_percpu_irq, NULL, info->uv_name);
+ handle_percpu_irq, NULL, info->uv.name);
} else {
kfree(chip_data);
}
init_irq_alloc_info(&info, cpumask_of(cpu));
info.type = X86_IRQ_ALLOC_TYPE_UV;
- info.uv_limit = limit;
- info.uv_blade = mmr_blade;
- info.uv_offset = mmr_offset;
- info.uv_name = irq_name;
+ info.uv.limit = limit;
+ info.uv.blade = mmr_blade;
+ info.uv.offset = mmr_offset;
+ info.uv.name = irq_name;
return irq_domain_alloc_irqs(domain, 1,
uv_blade_to_memory_nid(mmr_blade), &info);
/*
* SGI NMI support routines
*
- * Copyright (c) 2009-2013 Silicon Graphics, Inc. All Rights Reserved.
- * Copyright (c) Mike Travis
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
+ * Copyright (C) 2007-2017 Silicon Graphics, Inc. All rights reserved.
+ * Copyright (c) Mike Travis
*/
#include <linux/cpu.h>
DEFINE_PER_CPU(struct uv_cpu_nmi_s, uv_cpu_nmi);
+/* Newer SMM NMI handler, not present in all systems */
+static unsigned long uvh_nmi_mmrx; /* UVH_EVENT_OCCURRED0/1 */
+static unsigned long uvh_nmi_mmrx_clear; /* UVH_EVENT_OCCURRED0/1_ALIAS */
+static int uvh_nmi_mmrx_shift; /* UVH_EVENT_OCCURRED0/1_EXTIO_INT0_SHFT */
+static char *uvh_nmi_mmrx_type; /* "EXTIO_INT0" */
+
+/* Non-zero indicates newer SMM NMI handler present */
+static unsigned long uvh_nmi_mmrx_supported; /* UVH_EXTIO_INT0_BROADCAST */
+
+/* Indicates to BIOS that we want to use the newer SMM NMI handler */
+static unsigned long uvh_nmi_mmrx_req; /* UVH_BIOS_KERNEL_MMR_ALIAS_2 */
+static int uvh_nmi_mmrx_req_shift; /* 62 */
+
/* UV hubless values */
#define NMI_CONTROL_PORT 0x70
#define NMI_DUMMY_PORT 0x71
/* Setup which NMI support is present in system */
static void uv_nmi_setup_mmrs(void)
{
- if (uv_read_local_mmr(UVH_NMI_MMRX_SUPPORTED)) {
- uv_write_local_mmr(UVH_NMI_MMRX_REQ,
- 1UL << UVH_NMI_MMRX_REQ_SHIFT);
- nmi_mmr = UVH_NMI_MMRX;
- nmi_mmr_clear = UVH_NMI_MMRX_CLEAR;
- nmi_mmr_pending = 1UL << UVH_NMI_MMRX_SHIFT;
- pr_info("UV: SMI NMI support: %s\n", UVH_NMI_MMRX_TYPE);
+ /* First determine arch specific MMRs to handshake with BIOS */
+ if (UVH_EVENT_OCCURRED0_EXTIO_INT0_MASK) {
+ uvh_nmi_mmrx = UVH_EVENT_OCCURRED0;
+ uvh_nmi_mmrx_clear = UVH_EVENT_OCCURRED0_ALIAS;
+ uvh_nmi_mmrx_shift = UVH_EVENT_OCCURRED0_EXTIO_INT0_SHFT;
+ uvh_nmi_mmrx_type = "OCRD0-EXTIO_INT0";
+
+ uvh_nmi_mmrx_supported = UVH_EXTIO_INT0_BROADCAST;
+ uvh_nmi_mmrx_req = UVH_BIOS_KERNEL_MMR_ALIAS_2;
+ uvh_nmi_mmrx_req_shift = 62;
+
+ } else if (UVH_EVENT_OCCURRED1_EXTIO_INT0_MASK) {
+ uvh_nmi_mmrx = UVH_EVENT_OCCURRED1;
+ uvh_nmi_mmrx_clear = UVH_EVENT_OCCURRED1_ALIAS;
+ uvh_nmi_mmrx_shift = UVH_EVENT_OCCURRED1_EXTIO_INT0_SHFT;
+ uvh_nmi_mmrx_type = "OCRD1-EXTIO_INT0";
+
+ uvh_nmi_mmrx_supported = UVH_EXTIO_INT0_BROADCAST;
+ uvh_nmi_mmrx_req = UVH_BIOS_KERNEL_MMR_ALIAS_2;
+ uvh_nmi_mmrx_req_shift = 62;
+
+ } else {
+ pr_err("UV:%s:cannot find EVENT_OCCURRED*_EXTIO_INT0\n",
+ __func__);
+ return;
+ }
+
+ /* Then find out if new NMI is supported */
+ if (likely(uv_read_local_mmr(uvh_nmi_mmrx_supported))) {
+ uv_write_local_mmr(uvh_nmi_mmrx_req,
+ 1UL << uvh_nmi_mmrx_req_shift);
+ nmi_mmr = uvh_nmi_mmrx;
+ nmi_mmr_clear = uvh_nmi_mmrx_clear;
+ nmi_mmr_pending = 1UL << uvh_nmi_mmrx_shift;
+ pr_info("UV: SMI NMI support: %s\n", uvh_nmi_mmrx_type);
} else {
nmi_mmr = UVH_NMI_MMR;
nmi_mmr_clear = UVH_NMI_MMR_CLEAR;
/* Ensure NMI enabled in Processor Interface Reg: */
uv_reassert_nmi();
uv_register_nmi_notifier();
- pr_info("UV: Hubless NMI enabled\n");
+ pr_info("UV: PCH NMI enabled\n");
}
/*
* SGI RTC clock/timer routines.
*
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
* Copyright (c) 2009-2013 Silicon Graphics, Inc. All Rights Reserved.
* Copyright (c) Dimitri Sivanich
*/
struct {
int lcpu; /* systemwide logical cpu number */
u64 expires; /* next timer expiration for this cpu */
- } cpu[1];
+ } cpu[];
};
/*
/* Check for an RTC interrupt pending */
static int uv_intr_pending(int pnode)
{
- if (is_uvx_hub())
- return uv_read_global_mmr64(pnode, UVXH_EVENT_OCCURRED2) &
- UVXH_EVENT_OCCURRED2_RTC_1_MASK;
- return 0;
+ return uv_read_global_mmr64(pnode, UVH_EVENT_OCCURRED2) &
+ UVH_EVENT_OCCURRED2_RTC_1_MASK;
}
/* Setup interrupt and return non-zero if early expiration occurred. */
UVH_RTC1_INT_CONFIG_M_MASK);
uv_write_global_mmr64(pnode, UVH_INT_CMPB, -1L);
- uv_write_global_mmr64(pnode, UVXH_EVENT_OCCURRED2_ALIAS,
- UVXH_EVENT_OCCURRED2_RTC_1_MASK);
+ uv_write_global_mmr64(pnode, UVH_EVENT_OCCURRED2_ALIAS,
+ UVH_EVENT_OCCURRED2_RTC_1_MASK);
val = (X86_PLATFORM_IPI_VECTOR << UVH_RTC1_INT_CONFIG_VECTOR_SHFT) |
((u64)apicid << UVH_RTC1_INT_CONFIG_APIC_ID_SHFT);
struct uv_rtc_timer_head *head = blade_info[bid];
if (!head) {
- head = kmalloc_node(sizeof(struct uv_rtc_timer_head) +
- (uv_blade_nr_possible_cpus(bid) *
- 2 * sizeof(u64)),
+ head = kmalloc_node(struct_size(head, cpu,
+ uv_blade_nr_possible_cpus(bid)),
GFP_KERNEL, nid);
if (!head) {
uv_rtc_deallocate_timers();
*/
extern __wsum csum_partial(const void *buff, int len, __wsum sum);
-/*
- * Note: when you get a NULL pointer exception here this means someone
- * passed in an incorrect kernel address to one of these functions.
- *
- * If you use these functions directly please don't forget the
- * access_ok().
- */
-
-static __inline__
-__wsum csum_partial_copy_nocheck(const void *src, void *dst,
- int len, __wsum sum)
-{
- memcpy(dst, src, len);
- return csum_partial(dst, len, sum);
-}
-
/**
* csum_fold - Fold and invert a 32bit checksum.
* sum: 32bit unfolded sum
return csum_fold(sum);
}
-/*
- * Copy and checksum to user
- */
-#define HAVE_CSUM_COPY_USER
-static __inline__ __wsum csum_and_copy_to_user(const void *src,
- void __user *dst,
- int len, __wsum sum, int *err_ptr)
-{
- if (access_ok(dst, len)) {
- if (copy_to_user(dst, src, len)) {
- *err_ptr = -EFAULT;
- return (__force __wsum)-1;
- }
-
- return csum_partial(src, len, sum);
- }
-
- if (len)
- *err_ptr = -EFAULT;
-
- return (__force __wsum)-1; /* invalid checksum */
-}
-
#endif
}
static const struct pv_info xen_info __initconst = {
- .shared_kernel_pmd = 0,
-
.extra_user_64bit_cs = FLAT_USER_CS64,
.name = "Xen",
};
xen_start_info->nr_pages);
xen_reserve_special_pages();
- /* keep using Xen gdt for now; no urgent need to change it */
-
- pv_info.kernel_rpl = 0;
-
/*
* We used to do this in xen_arch_setup, but that is too late
* on AMD were early_cpu_init (run before ->arch_setup()) calls
__xen_set_pte(ptep, pteval);
}
-static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep, pte_t pteval)
-{
- trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval);
- __xen_set_pte(ptep, pteval);
-}
-
pte_t xen_ptep_modify_prot_start(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep)
{
.release_pmd = xen_release_pmd_init,
.set_pte = xen_set_pte_init,
- .set_pte_at = xen_set_pte_at,
.set_pmd = xen_set_pmd_hyper,
.ptep_modify_prot_start = __ptep_modify_prot_start,
* better 64-bit) boundary
*/
-asmlinkage __wsum csum_partial_copy_generic(const void *src, void *dst,
- int len, __wsum sum,
- int *src_err_ptr, int *dst_err_ptr);
+asmlinkage __wsum csum_partial_copy_generic(const void *src, void *dst, int len);
+#define _HAVE_ARCH_CSUM_AND_COPY
/*
* Note: when you get a NULL pointer exception here this means someone
* passed in an incorrect kernel address to one of these functions.
*/
static inline
-__wsum csum_partial_copy_nocheck(const void *src, void *dst,
- int len, __wsum sum)
+__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len)
{
- return csum_partial_copy_generic(src, dst, len, sum, NULL, NULL);
+ return csum_partial_copy_generic(src, dst, len);
}
#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER
static inline
__wsum csum_and_copy_from_user(const void __user *src, void *dst,
- int len, __wsum sum, int *err_ptr)
+ int len)
{
- if (access_ok(src, len))
- return csum_partial_copy_generic((__force const void *)src, dst,
- len, sum, err_ptr, NULL);
- if (len)
- *err_ptr = -EFAULT;
- return sum;
+ if (!access_ok(src, len))
+ return 0;
+ return csum_partial_copy_generic((__force const void *)src, dst, len);
}
/*
*/
#define HAVE_CSUM_COPY_USER
static __inline__ __wsum csum_and_copy_to_user(const void *src,
- void __user *dst, int len,
- __wsum sum, int *err_ptr)
+ void __user *dst, int len)
{
- if (access_ok(dst, len))
- return csum_partial_copy_generic(src,dst,len,sum,NULL,err_ptr);
-
- if (len)
- *err_ptr = -EFAULT;
-
- return (__force __wsum)-1; /* invalid checksum */
+ if (!access_ok(dst, len))
+ return 0;
+ return csum_partial_copy_generic(src, (__force void *)dst, len);
}
#endif
*/
/*
-unsigned int csum_partial_copy_generic (const char *src, char *dst, int len,
- int sum, int *src_err_ptr, int *dst_err_ptr)
+unsigned int csum_partial_copy_generic (const char *src, char *dst, int len)
a2 = src
a3 = dst
a4 = len
a5 = sum
- a6 = src_err_ptr
- a7 = dst_err_ptr
a8 = temp
a9 = temp
a10 = temp
- a11 = original len for exception handling
- a12 = original dst for exception handling
This function is optimized for 4-byte aligned addresses. Other
alignments work, but not nearly as efficiently.
ENTRY(csum_partial_copy_generic)
abi_entry_default
- mov a12, a3
- mov a11, a4
+ movi a5, -1
or a10, a2, a3
/* We optimize the following alignment tests for the 4-byte
#endif
EX(10f) l32i a9, a2, 0
EX(10f) l32i a8, a2, 4
-EX(11f) s32i a9, a3, 0
-EX(11f) s32i a8, a3, 4
+EX(10f) s32i a9, a3, 0
+EX(10f) s32i a8, a3, 4
ONES_ADD(a5, a9)
ONES_ADD(a5, a8)
EX(10f) l32i a9, a2, 8
EX(10f) l32i a8, a2, 12
-EX(11f) s32i a9, a3, 8
-EX(11f) s32i a8, a3, 12
+EX(10f) s32i a9, a3, 8
+EX(10f) s32i a8, a3, 12
ONES_ADD(a5, a9)
ONES_ADD(a5, a8)
EX(10f) l32i a9, a2, 16
EX(10f) l32i a8, a2, 20
-EX(11f) s32i a9, a3, 16
-EX(11f) s32i a8, a3, 20
+EX(10f) s32i a9, a3, 16
+EX(10f) s32i a8, a3, 20
ONES_ADD(a5, a9)
ONES_ADD(a5, a8)
EX(10f) l32i a9, a2, 24
EX(10f) l32i a8, a2, 28
-EX(11f) s32i a9, a3, 24
-EX(11f) s32i a8, a3, 28
+EX(10f) s32i a9, a3, 24
+EX(10f) s32i a8, a3, 28
ONES_ADD(a5, a9)
ONES_ADD(a5, a8)
addi a2, a2, 32
.Loop6:
#endif
EX(10f) l32i a9, a2, 0
-EX(11f) s32i a9, a3, 0
+EX(10f) s32i a9, a3, 0
ONES_ADD(a5, a9)
addi a2, a2, 4
addi a3, a3, 4
.Loop7:
#endif
EX(10f) l16ui a9, a2, 0
-EX(11f) s16i a9, a3, 0
+EX(10f) s16i a9, a3, 0
ONES_ADD(a5, a9)
addi a2, a2, 2
addi a3, a3, 2
/* This section processes a possible trailing odd byte. */
_bbci.l a4, 0, 8f /* 1-byte chunk */
EX(10f) l8ui a9, a2, 0
-EX(11f) s8i a9, a3, 0
+EX(10f) s8i a9, a3, 0
#ifdef __XTENSA_EB__
slli a9, a9, 8 /* shift byte to bits 8..15 */
#endif
#endif
EX(10f) l8ui a9, a2, 0
EX(10f) l8ui a8, a2, 1
-EX(11f) s8i a9, a3, 0
-EX(11f) s8i a8, a3, 1
+EX(10f) s8i a9, a3, 0
+EX(10f) s8i a8, a3, 1
#ifdef __XTENSA_EB__
slli a9, a9, 8 /* combine into a single 16-bit value */
#else /* for checksum computation */
# Exception handler:
.section .fixup, "ax"
-/*
- a6 = src_err_ptr
- a7 = dst_err_ptr
- a11 = original len for exception handling
- a12 = original dst for exception handling
-*/
-
10:
- _movi a2, -EFAULT
- s32i a2, a6, 0 /* src_err_ptr */
-
- # clear the complete destination - computing the rest
- # is too much work
- movi a2, 0
-#if XCHAL_HAVE_LOOPS
- loopgtz a11, 2f
-#else
- beqz a11, 2f
- add a11, a11, a12 /* a11 = ending address */
-.Leloop:
-#endif
- s8i a2, a12, 0
- addi a12, a12, 1
-#if !XCHAL_HAVE_LOOPS
- blt a12, a11, .Leloop
-#endif
-2:
- abi_ret_default
-
-11:
- movi a2, -EFAULT
- s32i a2, a7, 0 /* dst_err_ptr */
movi a2, 0
abi_ret_default
__bio_clone_fast(b, bio);
- bio_crypt_clone(b, bio, gfp_mask);
+ if (bio_crypt_clone(b, bio, gfp_mask) < 0)
+ goto err_put;
- if (bio_integrity(bio)) {
- int ret;
-
- ret = bio_integrity_clone(b, bio, gfp_mask);
-
- if (ret < 0) {
- bio_put(b);
- return NULL;
- }
- }
+ if (bio_integrity(bio) &&
+ bio_integrity_clone(b, bio, gfp_mask) < 0)
+ goto err_put;
return b;
+
+err_put:
+ bio_put(b);
+ return NULL;
}
EXPORT_SYMBOL(bio_clone_fast);
{
char b[BDEVNAME_SIZE];
- printk(KERN_INFO "attempt to access beyond end of device\n");
- printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n",
- bio_devname(bio, b), bio->bi_opf,
- (unsigned long long)bio_end_sector(bio),
- (long long)maxsector);
+ pr_info_ratelimited("attempt to access beyond end of device\n"
+ "%s: rw=%d, want=%llu, limit=%llu\n",
+ bio_devname(bio, b), bio->bi_opf,
+ bio_end_sector(bio), maxsector);
}
#ifdef CONFIG_FAIL_MAKE_REQUEST
/*
* For a REQ_NOWAIT based request, return -EOPNOTSUPP
- * if queue is not a request based queue.
+ * if queue does not support NOWAIT.
*/
- if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_mq(q))
+ if ((bio->bi_opf & REQ_NOWAIT) && !blk_queue_nowait(q))
goto not_supported;
if (should_fail_bio(bio))
if (rq->bio) {
rq->biotail->bi_next = bio;
rq->biotail = bio;
- } else
+ } else {
rq->bio = rq->biotail = bio;
+ }
+ bio = NULL;
}
/* Copy attributes of the original request to the clone request. */
rq->nr_phys_segments = rq_src->nr_phys_segments;
rq->ioprio = rq_src->ioprio;
- if (rq->bio)
- blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask);
+ if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
+ goto free_and_out;
return 0;
__blk_crypto_free_request(rq);
}
-void __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
- gfp_t gfp_mask);
-static inline void blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
- gfp_t gfp_mask)
+int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
+ gfp_t gfp_mask);
+/**
+ * blk_crypto_rq_bio_prep - Prepare a request's crypt_ctx when its first bio
+ * is inserted
+ * @rq: The request to prepare
+ * @bio: The first bio being inserted into the request
+ * @gfp_mask: Memory allocation flags
+ *
+ * Return: 0 on success, -ENOMEM if out of memory. -ENOMEM is only possible if
+ * @gfp_mask doesn't include %__GFP_DIRECT_RECLAIM.
+ */
+static inline int blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
+ gfp_t gfp_mask)
{
if (bio_has_crypt_ctx(bio))
- __blk_crypto_rq_bio_prep(rq, bio, gfp_mask);
+ return __blk_crypto_rq_bio_prep(rq, bio, gfp_mask);
+ return 0;
}
/**
void bio_crypt_set_ctx(struct bio *bio, const struct blk_crypto_key *key,
const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], gfp_t gfp_mask)
{
- struct bio_crypt_ctx *bc = mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
+ struct bio_crypt_ctx *bc;
+
+ /*
+ * The caller must use a gfp_mask that contains __GFP_DIRECT_RECLAIM so
+ * that the mempool_alloc() can't fail.
+ */
+ WARN_ON_ONCE(!(gfp_mask & __GFP_DIRECT_RECLAIM));
+
+ bc = mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
bc->bc_key = key;
memcpy(bc->bc_dun, dun, sizeof(bc->bc_dun));
bio->bi_crypt_context = NULL;
}
-void __bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask)
+int __bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask)
{
dst->bi_crypt_context = mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
+ if (!dst->bi_crypt_context)
+ return -ENOMEM;
*dst->bi_crypt_context = *src->bi_crypt_context;
+ return 0;
}
EXPORT_SYMBOL_GPL(__bio_crypt_clone);
return false;
}
-/**
- * __blk_crypto_rq_bio_prep - Prepare a request's crypt_ctx when its first bio
- * is inserted
- *
- * @rq: The request to prepare
- * @bio: The first bio being inserted into the request
- * @gfp_mask: gfp mask
- */
-void __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
- gfp_t gfp_mask)
+int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
+ gfp_t gfp_mask)
{
- if (!rq->crypt_ctx)
+ if (!rq->crypt_ctx) {
rq->crypt_ctx = mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
+ if (!rq->crypt_ctx)
+ return -ENOMEM;
+ }
*rq->crypt_ctx = *bio->bi_crypt_context;
+ return 0;
}
/**
return true;
}
-EXPORT_SYMBOL(blk_integrity_merge_rq);
bool blk_integrity_merge_bio(struct request_queue *q, struct request *req,
struct bio *bio)
return true;
}
-EXPORT_SYMBOL(blk_integrity_merge_bio);
struct integrity_sysfs_entry {
struct attribute attr;
MIN_DELAY = 250,
MAX_DELAY = 250 * USEC_PER_MSEC,
- /*
- * Halve debts if total usage keeps staying under 25% w/o any shortages
- * for over 100ms.
- */
- DEBT_BUSY_USAGE_PCT = 25,
- DEBT_REDUCTION_IDLE_DUR = 100 * USEC_PER_MSEC,
+ /* halve debts if avg usage over 100ms is under 50% */
+ DFGV_USAGE_PCT = 50,
+ DFGV_PERIOD = 100 * USEC_PER_MSEC,
/* don't let cmds which take a very long time pin lagging for too long */
MAX_LAGGING_PERIODS = 10,
bool weights_updated;
atomic_t hweight_gen; /* for lazy hweights */
- /* the last time debt cancel condition wasn't met */
- u64 debt_busy_at;
+ /* debt forgivness */
+ u64 dfgv_period_at;
+ u64 dfgv_period_rem;
+ u64 dfgv_usage_us_sum;
u64 autop_too_fast_at;
u64 autop_too_slow_at;
static const char *q_name(struct request_queue *q)
{
- if (test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
+ if (blk_queue_registered(q))
return kobject_name(q->kobj.parent);
else
return "<unknown>";
if (ioc->running == IOC_IDLE) {
ioc->running = IOC_RUNNING;
- ioc->debt_busy_at = now->now;
+ ioc->dfgv_period_at = now->now;
+ ioc->dfgv_period_rem = 0;
ioc_start_period(ioc, now);
}
list_del_init(&iocg->walk_list);
}
+/*
+ * A low weight iocg can amass a large amount of debt, for example, when
+ * anonymous memory gets reclaimed aggressively. If the system has a lot of
+ * memory paired with a slow IO device, the debt can span multiple seconds or
+ * more. If there are no other subsequent IO issuers, the in-debt iocg may end
+ * up blocked paying its debt while the IO device is idle.
+ *
+ * The following protects against such cases. If the device has been
+ * sufficiently idle for a while, the debts are halved and delays are
+ * recalculated.
+ */
+static void ioc_forgive_debts(struct ioc *ioc, u64 usage_us_sum, int nr_debtors,
+ struct ioc_now *now)
+{
+ struct ioc_gq *iocg;
+ u64 dur, usage_pct, nr_cycles;
+
+ /* if no debtor, reset the cycle */
+ if (!nr_debtors) {
+ ioc->dfgv_period_at = now->now;
+ ioc->dfgv_period_rem = 0;
+ ioc->dfgv_usage_us_sum = 0;
+ return;
+ }
+
+ /*
+ * Debtors can pass through a lot of writes choking the device and we
+ * don't want to be forgiving debts while the device is struggling from
+ * write bursts. If we're missing latency targets, consider the device
+ * fully utilized.
+ */
+ if (ioc->busy_level > 0)
+ usage_us_sum = max_t(u64, usage_us_sum, ioc->period_us);
+
+ ioc->dfgv_usage_us_sum += usage_us_sum;
+ if (time_before64(now->now, ioc->dfgv_period_at + DFGV_PERIOD))
+ return;
+
+ /*
+ * At least DFGV_PERIOD has passed since the last period. Calculate the
+ * average usage and reset the period counters.
+ */
+ dur = now->now - ioc->dfgv_period_at;
+ usage_pct = div64_u64(100 * ioc->dfgv_usage_us_sum, dur);
+
+ ioc->dfgv_period_at = now->now;
+ ioc->dfgv_usage_us_sum = 0;
+
+ /* if was too busy, reset everything */
+ if (usage_pct > DFGV_USAGE_PCT) {
+ ioc->dfgv_period_rem = 0;
+ return;
+ }
+
+ /*
+ * Usage is lower than threshold. Let's forgive some debts. Debt
+ * forgiveness runs off of the usual ioc timer but its period usually
+ * doesn't match ioc's. Compensate the difference by performing the
+ * reduction as many times as would fit in the duration since the last
+ * run and carrying over the left-over duration in @ioc->dfgv_period_rem
+ * - if ioc period is 75% of DFGV_PERIOD, one out of three consecutive
+ * reductions is doubled.
+ */
+ nr_cycles = dur + ioc->dfgv_period_rem;
+ ioc->dfgv_period_rem = do_div(nr_cycles, DFGV_PERIOD);
+
+ list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
+ u64 __maybe_unused old_debt, __maybe_unused old_delay;
+
+ if (!iocg->abs_vdebt && !iocg->delay)
+ continue;
+
+ spin_lock(&iocg->waitq.lock);
+
+ old_debt = iocg->abs_vdebt;
+ old_delay = iocg->delay;
+
+ if (iocg->abs_vdebt)
+ iocg->abs_vdebt = iocg->abs_vdebt >> nr_cycles ?: 1;
+ if (iocg->delay)
+ iocg->delay = iocg->delay >> nr_cycles ?: 1;
+
+ iocg_kick_waitq(iocg, true, now);
+
+ TRACE_IOCG_PATH(iocg_forgive_debt, iocg, now, usage_pct,
+ old_debt, iocg->abs_vdebt,
+ old_delay, iocg->delay);
+
+ spin_unlock(&iocg->waitq.lock);
+ }
+}
+
static void ioc_timer_fn(struct timer_list *timer)
{
struct ioc *ioc = container_of(timer, struct ioc, timer);
iocg->delay) {
/* might be oversleeping vtime / hweight changes, kick */
iocg_kick_waitq(iocg, true, &now);
- if (iocg->abs_vdebt)
+ if (iocg->abs_vdebt || iocg->delay)
nr_debtors++;
} else if (iocg_is_idle(iocg)) {
/* no waiter and idle, deactivate */
list_for_each_entry_safe(iocg, tiocg, &surpluses, surplus_list)
list_del_init(&iocg->surplus_list);
- /*
- * A low weight iocg can amass a large amount of debt, for example, when
- * anonymous memory gets reclaimed aggressively. If the system has a lot
- * of memory paired with a slow IO device, the debt can span multiple
- * seconds or more. If there are no other subsequent IO issuers, the
- * in-debt iocg may end up blocked paying its debt while the IO device
- * is idle.
- *
- * The following protects against such pathological cases. If the device
- * has been sufficiently idle for a substantial amount of time, the
- * debts are halved. The criteria are on the conservative side as we
- * want to resolve the rare extreme cases without impacting regular
- * operation by forgiving debts too readily.
- */
- if (nr_shortages ||
- div64_u64(100 * usage_us_sum, now.now - ioc->period_at) >=
- DEBT_BUSY_USAGE_PCT)
- ioc->debt_busy_at = now.now;
-
- if (nr_debtors &&
- now.now - ioc->debt_busy_at >= DEBT_REDUCTION_IDLE_DUR) {
- list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
- if (iocg->abs_vdebt) {
- spin_lock(&iocg->waitq.lock);
- iocg->abs_vdebt /= 2;
- iocg_kick_waitq(iocg, true, &now);
- spin_unlock(&iocg->waitq.lock);
- }
- }
- ioc->debt_busy_at = now.now;
- }
-
/*
* If q is getting clogged or we're missing too much, we're issuing
* too much IO and should lower vtime rate. If we're not missing
ioc_refresh_params(ioc, false);
+ ioc_forgive_debts(ioc, usage_us_sum, nr_debtors, &now);
+
/*
* This period is done. Move onto the next one. If nothing's
* going on with the device, stop the timer.
static void __exit ioc_exit(void)
{
- return blkcg_policy_unregister(&blkcg_policy_iocost);
+ blkcg_policy_unregister(&blkcg_policy_iocost);
}
module_init(ioc_init);
static void __exit iolatency_exit(void)
{
- return blkcg_policy_unregister(&blkcg_policy_iolatency);
+ blkcg_policy_unregister(&blkcg_policy_iolatency);
}
module_init(iolatency_init);
return -EINVAL;
/* In case the discard request is in a partition */
- if (bdev->bd_partno)
+ if (bdev_is_partition(bdev))
part_offset = bdev->bd_part->start_sect;
while (nr_sects) {
return ll_new_hw_segment(req, bio, nr_segs);
}
-int ll_front_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
+static int ll_front_merge_fn(struct request *req, struct bio *bio,
+ unsigned int nr_segs)
{
if (req_gap_front_merge(req, bio))
return 0;
return next;
}
-struct request *attempt_back_merge(struct request_queue *q, struct request *rq)
+static struct request *attempt_back_merge(struct request_queue *q,
+ struct request *rq)
{
struct request *next = elv_latter_request(q, rq);
return NULL;
}
-struct request *attempt_front_merge(struct request_queue *q, struct request *rq)
+static struct request *attempt_front_merge(struct request_queue *q,
+ struct request *rq)
{
struct request *prev = elv_former_request(q, rq);
part_stat_unlock();
}
-enum bio_merge_status bio_attempt_back_merge(struct request *req,
- struct bio *bio,
- unsigned int nr_segs)
+enum bio_merge_status {
+ BIO_MERGE_OK,
+ BIO_MERGE_NONE,
+ BIO_MERGE_FAILED,
+};
+
+static enum bio_merge_status bio_attempt_back_merge(struct request *req,
+ struct bio *bio, unsigned int nr_segs)
{
const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
return BIO_MERGE_OK;
}
-enum bio_merge_status bio_attempt_front_merge(struct request *req,
- struct bio *bio,
- unsigned int nr_segs)
+static enum bio_merge_status bio_attempt_front_merge(struct request *req,
+ struct bio *bio, unsigned int nr_segs)
{
const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
return BIO_MERGE_OK;
}
-enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
- struct request *req,
- struct bio *bio)
+static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
+ struct request *req, struct bio *bio)
{
unsigned short segments = blk_rq_nr_discard_segments(req);
return false;
}
EXPORT_SYMBOL_GPL(blk_bio_list_merge);
+
+bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
+ unsigned int nr_segs, struct request **merged_request)
+{
+ struct request *rq;
+
+ switch (elv_merge(q, &rq, bio)) {
+ case ELEVATOR_BACK_MERGE:
+ if (!blk_mq_sched_allow_merge(q, rq, bio))
+ return false;
+ if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
+ return false;
+ *merged_request = attempt_back_merge(q, rq);
+ if (!*merged_request)
+ elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
+ return true;
+ case ELEVATOR_FRONT_MERGE:
+ if (!blk_mq_sched_allow_merge(q, rq, bio))
+ return false;
+ if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
+ return false;
+ *merged_request = attempt_front_merge(q, rq);
+ if (!*merged_request)
+ elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
+ return true;
+ case ELEVATOR_DISCARD_MERGE:
+ return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;
+ default:
+ return false;
+ }
+}
+EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
}
}
-bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
- unsigned int nr_segs, struct request **merged_request)
-{
- struct request *rq;
-
- switch (elv_merge(q, &rq, bio)) {
- case ELEVATOR_BACK_MERGE:
- if (!blk_mq_sched_allow_merge(q, rq, bio))
- return false;
- if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
- return false;
- *merged_request = attempt_back_merge(q, rq);
- if (!*merged_request)
- elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
- return true;
- case ELEVATOR_FRONT_MERGE:
- if (!blk_mq_sched_allow_merge(q, rq, bio))
- return false;
- if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
- return false;
- *merged_request = attempt_front_merge(q, rq);
- if (!*merged_request)
- elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
- return true;
- case ELEVATOR_DISCARD_MERGE:
- return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;
- default:
- return false;
- }
-}
-EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
-
bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio,
unsigned int nr_segs)
{
struct blk_mq_ctx *ctx = rq->mq_ctx;
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
- /* flush rq in flush machinery need to be dispatched directly */
- if (!(rq->rq_flags & RQF_FLUSH_SEQ) && op_is_flush(rq->cmd_flags)) {
- blk_insert_flush(rq);
- goto run;
- }
-
WARN_ON(e && (rq->tag != BLK_MQ_NO_TAG));
if (blk_mq_sched_bypass_insert(hctx, !!e, rq)) {
struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
kobj);
- cancel_delayed_work_sync(&hctx->run_work);
-
if (hctx->flags & BLK_MQ_F_BLOCKING)
cleanup_srcu_struct(hctx->srcu);
blk_free_flush_queue(hctx->fq);
/*
* __blk_mq_update_nr_hw_queues() updates nr_hw_queues and queue_hw_ctx
* while the queue is frozen. So we can use q_usage_counter to avoid
- * racing with it. __blk_mq_update_nr_hw_queues() uses
- * synchronize_rcu() to ensure this function left the critical section
- * below.
+ * racing with it.
*/
if (!percpu_ref_tryget(&q->q_usage_counter))
return;
{
struct mq_inflight *mi = priv;
- if (rq->part == mi->part)
+ if (rq->part == mi->part && blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
mi->inflight[rq_data_dir(rq)]++;
return true;
hctx->dispatched[queued_to_index(queued)]++;
+ /* If we didn't flush the entire list, we could have told the driver
+ * there was more coming, but that turned out to be a lie.
+ */
+ if ((!list_empty(list) || errors) && q->mq_ops->commit_rqs && queued)
+ q->mq_ops->commit_rqs(hctx);
/*
* Any items that need requeuing? Stuff them into hctx->dispatch,
* that is where we will continue on next queue run.
blk_mq_release_budgets(q, nr_budgets);
- /*
- * If we didn't flush the entire list, we could have told
- * the driver there was more coming, but that turned out to
- * be a lie.
- */
- if (q->mq_ops->commit_rqs && queued)
- q->mq_ops->commit_rqs(hctx);
-
spin_lock(&hctx->lock);
list_splice_tail_init(list, &hctx->dispatch);
spin_unlock(&hctx->lock);
/*
* If we are stopped, don't run the queue.
*/
- if (test_bit(BLK_MQ_S_STOPPED, &hctx->state))
+ if (blk_mq_hctx_stopped(hctx))
return;
__blk_mq_run_hw_queue(hctx);
static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
unsigned int nr_segs)
{
+ int err;
+
if (bio->bi_opf & REQ_RAHEAD)
rq->cmd_flags |= REQ_FAILFAST_MASK;
rq->__sector = bio->bi_iter.bi_sector;
rq->write_hint = bio->bi_write_hint;
blk_rq_bio_prep(rq, bio, nr_segs);
- blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
+
+ /* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
+ err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
+ WARN_ON_ONCE(err);
blk_account_io_start(rq);
}
struct list_head *list)
{
int queued = 0;
+ int errors = 0;
while (!list_empty(list)) {
blk_status_t ret;
break;
}
blk_mq_end_request(rq, ret);
+ errors++;
} else
queued++;
}
* the driver there was more coming, but that turned out to
* be a lie.
*/
- if (!list_empty(list) && hctx->queue->mq_ops->commit_rqs && queued)
+ if ((!list_empty(list) || errors) &&
+ hctx->queue->mq_ops->commit_rqs && queued)
hctx->queue->mq_ops->commit_rqs(hctx);
}
{
int i;
- for (i = 0; i < set->nr_hw_queues; i++)
+ for (i = 0; i < set->nr_hw_queues; i++) {
if (!__blk_mq_alloc_map_and_request(set, i))
goto out_unwind;
+ cond_resched();
+ }
return 0;
}
EXPORT_SYMBOL_GPL(blk_queue_can_use_dma_map_merging);
+/**
+ * blk_queue_set_zoned - configure a disk queue zoned model.
+ * @disk: the gendisk of the queue to configure
+ * @model: the zoned model to set
+ *
+ * Set the zoned model of the request queue of @disk according to @model.
+ * When @model is BLK_ZONED_HM (host managed), this should be called only
+ * if zoned block device support is enabled (CONFIG_BLK_DEV_ZONED option).
+ * If @model specifies BLK_ZONED_HA (host aware), the effective model used
+ * depends on CONFIG_BLK_DEV_ZONED settings and on the existence of partitions
+ * on the disk.
+ */
+void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model)
+{
+ switch (model) {
+ case BLK_ZONED_HM:
+ /*
+ * Host managed devices are supported only if
+ * CONFIG_BLK_DEV_ZONED is enabled.
+ */
+ WARN_ON_ONCE(!IS_ENABLED(CONFIG_BLK_DEV_ZONED));
+ break;
+ case BLK_ZONED_HA:
+ /*
+ * Host aware devices can be treated either as regular block
+ * devices (similar to drive managed devices) or as zoned block
+ * devices to take advantage of the zone command set, similarly
+ * to host managed devices. We try the latter if there are no
+ * partitions and zoned block device support is enabled, else
+ * we do nothing special as far as the block layer is concerned.
+ */
+ if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) ||
+ disk_has_partitions(disk))
+ model = BLK_ZONED_NONE;
+ break;
+ case BLK_ZONED_NONE:
+ default:
+ if (WARN_ON_ONCE(model != BLK_ZONED_NONE))
+ model = BLK_ZONED_NONE;
+ break;
+ }
+
+ disk->queue->limits.zoned = model;
+}
+EXPORT_SYMBOL_GPL(blk_queue_set_zoned);
+
static int __init blk_settings_init(void)
{
blk_max_low_pfn = max_low_pfn - 1;
if (q->elevator) {
ioc_clear_queue(q);
__elevator_exit(q, q->elevator);
- q->elevator = NULL;
}
/*
blk_free_queue_stats(q->stats);
- if (queue_is_mq(q))
+ if (queue_is_mq(q)) {
+ struct blk_mq_hw_ctx *hctx;
+ int i;
+
cancel_delayed_work_sync(&q->requeue_work);
+ queue_for_each_hw_ctx(q, hctx, i)
+ cancel_delayed_work_sync(&hctx->run_work);
+ }
+
blk_exit_queue(q);
blk_queue_free_zone_bitmaps(q);
int ret;
struct device *dev = disk_to_dev(disk);
struct request_queue *q = disk->queue;
- bool has_elevator = false;
if (WARN_ON(!q))
return -ENXIO;
kobject_put(&dev->kobj);
return ret;
}
- has_elevator = true;
}
blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
/* Now everything is ready and send out KOBJ_ADD uevent */
kobject_uevent(&q->kobj, KOBJ_ADD);
- if (has_elevator)
+ if (q->elevator)
kobject_uevent(&q->elevator->kobj, KOBJ_ADD);
mutex_unlock(&q->sysfs_lock);
*/
static struct bio *throtl_peek_queued(struct list_head *queued)
{
- struct throtl_qnode *qn = list_first_entry(queued, struct throtl_qnode, node);
+ struct throtl_qnode *qn;
struct bio *bio;
if (list_empty(queued))
return NULL;
+ qn = list_first_entry(queued, struct throtl_qnode, node);
bio = bio_list_peek(&qn->bios);
WARN_ON_ONCE(!bio);
return bio;
static struct bio *throtl_pop_queued(struct list_head *queued,
struct throtl_grp **tg_to_put)
{
- struct throtl_qnode *qn = list_first_entry(queued, struct throtl_qnode, node);
+ struct throtl_qnode *qn;
struct bio *bio;
if (list_empty(queued))
return NULL;
+ qn = list_first_entry(queued, struct throtl_qnode, node);
bio = bio_list_pop(&qn->bios);
WARN_ON_ONCE(!bio);
throtl_rb_first(struct throtl_service_queue *parent_sq)
{
struct rb_node *n;
- /* Service tree is empty */
- if (!parent_sq->nr_pending)
- return NULL;
n = rb_first_cached(&parent_sq->pending_tree);
WARN_ON_ONCE(!n);
leftmost);
}
-static void __throtl_enqueue_tg(struct throtl_grp *tg)
-{
- tg_service_queue_add(tg);
- tg->flags |= THROTL_TG_PENDING;
- tg->service_queue.parent_sq->nr_pending++;
-}
-
static void throtl_enqueue_tg(struct throtl_grp *tg)
{
- if (!(tg->flags & THROTL_TG_PENDING))
- __throtl_enqueue_tg(tg);
-}
-
-static void __throtl_dequeue_tg(struct throtl_grp *tg)
-{
- throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq);
- tg->flags &= ~THROTL_TG_PENDING;
+ if (!(tg->flags & THROTL_TG_PENDING)) {
+ tg_service_queue_add(tg);
+ tg->flags |= THROTL_TG_PENDING;
+ tg->service_queue.parent_sq->nr_pending++;
+ }
}
static void throtl_dequeue_tg(struct throtl_grp *tg)
{
- if (tg->flags & THROTL_TG_PENDING)
- __throtl_dequeue_tg(tg);
+ if (tg->flags & THROTL_TG_PENDING) {
+ throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq);
+ tg->flags &= ~THROTL_TG_PENDING;
+ }
}
/* Call with queue lock held */
static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw,
unsigned long jiffy_end)
{
- tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice);
+ throtl_set_slice_end(tg, rw, jiffy_end);
throtl_log(&tg->service_queue,
"[%c] extend slice start=%lu end=%lu jiffies=%lu",
rw == READ ? 'R' : 'W', tg->slice_start[rw],
unsigned int nr_disp = 0;
while (1) {
- struct throtl_grp *tg = throtl_rb_first(parent_sq);
+ struct throtl_grp *tg;
struct throtl_service_queue *sq;
+ if (!parent_sq->nr_pending)
+ break;
+
+ tg = throtl_rb_first(parent_sq);
if (!tg)
break;
goto out_finish;
ret = -EINVAL;
- if (!strcmp(tok, "rbps"))
+ if (!strcmp(tok, "rbps") && val > 1)
v[0] = val;
- else if (!strcmp(tok, "wbps"))
+ else if (!strcmp(tok, "wbps") && val > 1)
v[1] = val;
- else if (!strcmp(tok, "riops"))
+ else if (!strcmp(tok, "riops") && val > 1)
v[2] = min_t(u64, val, UINT_MAX);
- else if (!strcmp(tok, "wiops"))
+ else if (!strcmp(tok, "wiops") && val > 1)
v[3] = min_t(u64, val, UINT_MAX);
else if (off == LIMIT_LOW && !strcmp(tok, "idle"))
idle_time = val;
queue_work(kthrotld_workqueue, &td->dispatch_work);
}
-static void throtl_downgrade_state(struct throtl_data *td, int new)
+static void throtl_downgrade_state(struct throtl_data *td)
{
td->scale /= 2;
return;
}
- td->limit_index = new;
+ td->limit_index = LIMIT_LOW;
td->low_downgrade_time = jiffies;
}
* cgroups
*/
if (throtl_hierarchy_can_downgrade(tg))
- throtl_downgrade_state(tg->td, LIMIT_LOW);
+ throtl_downgrade_state(tg->td);
tg->last_bytes_disp[READ] = 0;
tg->last_bytes_disp[WRITE] = 0;
static void blk_throtl_update_idletime(struct throtl_grp *tg)
{
- unsigned long now = ktime_get_ns() >> 10;
+ unsigned long now;
unsigned long last_finish_time = tg->last_finish_time;
- if (now <= last_finish_time || last_finish_time == 0 ||
+ if (last_finish_time == 0)
+ return;
+
+ now = ktime_get_ns() >> 10;
+ if (now <= last_finish_time ||
last_finish_time == tg->checked_last_finish_time)
return;
unsigned long last_latency[2] = { 0 };
unsigned long latency[2];
- if (!blk_queue_nonrot(td->queue))
+ if (!blk_queue_nonrot(td->queue) || !td->limit_valid[LIMIT_LOW])
return;
if (time_before(jiffies, td->last_calculate_time + HZ))
return;
if (!blkg)
return;
tg = blkg_to_tg(blkg);
+ if (!tg->td->limit_valid[LIMIT_LOW])
+ return;
finish_time_ns = ktime_get_ns();
tg->last_finish_time = finish_time_ns >> 10;
spinlock_t mq_flush_lock;
};
-enum bio_merge_status {
- BIO_MERGE_OK,
- BIO_MERGE_NONE,
- BIO_MERGE_FAILED,
-};
-
extern struct kmem_cache *blk_requestq_cachep;
extern struct kobj_type blk_queue_ktype;
extern struct ida blk_queue_ida;
return true;
}
+bool blk_integrity_merge_rq(struct request_queue *, struct request *,
+ struct request *);
+bool blk_integrity_merge_bio(struct request_queue *, struct request *,
+ struct bio *);
+
static inline bool integrity_req_gap_back_merge(struct request *req,
struct bio *next)
{
void blk_integrity_add(struct gendisk *);
void blk_integrity_del(struct gendisk *);
#else /* CONFIG_BLK_DEV_INTEGRITY */
+static inline bool blk_integrity_merge_rq(struct request_queue *rq,
+ struct request *r1, struct request *r2)
+{
+ return true;
+}
+static inline bool blk_integrity_merge_bio(struct request_queue *rq,
+ struct request *r, struct bio *b)
+{
+ return true;
+}
static inline bool integrity_req_gap_back_merge(struct request *req,
struct bio *next)
{
unsigned long blk_rq_timeout(unsigned long timeout);
void blk_add_timer(struct request *req);
-enum bio_merge_status bio_attempt_front_merge(struct request *req,
- struct bio *bio,
- unsigned int nr_segs);
-enum bio_merge_status bio_attempt_back_merge(struct request *req,
- struct bio *bio,
- unsigned int nr_segs);
-enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
- struct request *req,
- struct bio *bio);
bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
unsigned int nr_segs, struct request **same_queue_rq);
bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
void __blk_queue_split(struct bio **bio, unsigned int *nr_segs);
int ll_back_merge_fn(struct request *req, struct bio *bio,
unsigned int nr_segs);
-int ll_front_merge_fn(struct request *req, struct bio *bio,
- unsigned int nr_segs);
-struct request *attempt_back_merge(struct request_queue *q, struct request *rq);
-struct request *attempt_front_merge(struct request_queue *q, struct request *rq);
int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
struct request *next);
unsigned int blk_recalc_rq_segments(struct request *rq);
break;
}
- bio_crypt_clone(bio, bio_src, gfp_mask);
+ if (bio_crypt_clone(bio, bio_src, gfp_mask) < 0)
+ goto err_put;
- if (bio_integrity(bio_src)) {
- int ret;
-
- ret = bio_integrity_clone(bio, bio_src, gfp_mask);
- if (ret < 0) {
- bio_put(bio);
- return NULL;
- }
- }
+ if (bio_integrity(bio_src) &&
+ bio_integrity_clone(bio, bio_src, gfp_mask) < 0)
+ goto err_put;
bio_clone_blkg_association(bio, bio_src);
blkcg_bio_issue_init(bio);
return bio;
+
+err_put:
+ bio_put(bio);
+ return NULL;
}
static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
void __elevator_exit(struct request_queue *q, struct elevator_queue *e)
{
mutex_lock(&e->sysfs_lock);
- if (e->type->ops.exit_sched)
- blk_mq_exit_sched(q, e);
+ blk_mq_exit_sched(q, e);
mutex_unlock(&e->sysfs_lock);
kobject_put(&e->kobj);
.release = elevator_release,
};
-/*
- * elv_register_queue is called from either blk_register_queue or
- * elevator_switch, elevator switch is prevented from being happen
- * in the two paths, so it is safe to not hold q->sysfs_lock.
- */
int elv_register_queue(struct request_queue *q, bool uevent)
{
struct elevator_queue *e = q->elevator;
int error;
+ lockdep_assert_held(&q->sysfs_lock);
+
error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
if (!error) {
struct elv_fs_entry *attr = e->type->elevator_attrs;
return error;
}
-/*
- * elv_unregister_queue is called from either blk_unregister_queue or
- * elevator_switch, elevator switch is prevented from being happen
- * in the two paths, so it is safe to not hold q->sysfs_lock.
- */
void elv_unregister_queue(struct request_queue *q)
{
+ lockdep_assert_held(&q->sysfs_lock);
+
if (q) {
struct elevator_queue *e = q->elevator;
static inline bool elv_support_iosched(struct request_queue *q)
{
- if (!q->mq_ops ||
+ if (!queue_is_mq(q) ||
(q->tag_set && (q->tag_set->flags & BLK_MQ_F_NO_SCHED)))
return false;
return true;
if (!elv_support_iosched(q))
return;
- WARN_ON_ONCE(test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags));
+ WARN_ON_ONCE(blk_queue_registered(q));
if (unlikely(q->elevator))
return;
{
int ret;
- if (!queue_is_mq(q) || !elv_support_iosched(q))
+ if (!elv_support_iosched(q))
return count;
ret = __elevator_change(q, name);
const char *bdevname(struct block_device *bdev, char *buf)
{
- return disk_name(bdev->bd_disk, bdev->bd_part->partno, buf);
+ return disk_name(bdev->bd_disk, bdev->bd_partno, buf);
}
EXPORT_SYMBOL(bdevname);
part = disk_get_part(disk, partno);
if (part)
- bdev = bdget(part_devt(part));
+ bdev = bdget_part(part);
disk_put_part(part);
return bdev;
return -EACCES;
if (copy_from_user(&p, upart, sizeof(struct blkpg_partition)))
return -EFAULT;
- if (bdev != bdev->bd_contains)
+ if (bdev_is_partition(bdev))
return -EINVAL;
if (p.pno <= 0)
{
int ret;
- if (!disk_part_scan_enabled(bdev->bd_disk) || bdev != bdev->bd_contains)
+ if (!disk_part_scan_enabled(bdev->bd_disk) || bdev_is_partition(bdev))
return -EINVAL;
if (!capable(CAP_SYS_ADMIN))
return -EACCES;
return -ENXIO;
ret = -ENOMEM;
- bdevp = bdget(part_devt(part));
+ bdevp = bdget_part(part);
if (!bdevp)
goto out_put_part;
if (!disk->fops->getgeo)
goto out_exit;
fn = symbol_get(dasd_biodasdinfo);
- if (!fn)
- goto out_exit;
blocksize = bdev_logical_block_size(bdev);
if (blocksize <= 0)
goto out_symbol;
geo->start = get_start_sect(bdev);
if (disk->fops->getgeo(bdev, geo))
goto out_freeall;
- if (fn(disk, info)) {
+ if (!fn || fn(disk, info)) {
kfree(info);
info = NULL;
}
out_nogeo:
kfree(info);
out_symbol:
- symbol_put(dasd_biodasdinfo);
+ if (fn)
+ symbol_put(dasd_biodasdinfo);
out_exit:
return res;
}
struct iov_iter i;
struct iovec *iov = NULL;
-#ifdef CONFIG_COMPAT
- if (in_compat_syscall())
- ret = compat_import_iovec(rq_data_dir(rq),
- hdr->dxferp, hdr->iovec_count,
- 0, &iov, &i);
- else
-#endif
- ret = import_iovec(rq_data_dir(rq),
- hdr->dxferp, hdr->iovec_count,
- 0, &iov, &i);
+ ret = import_iovec(rq_data_dir(rq), hdr->dxferp,
+ hdr->iovec_count, 0, &iov, &i);
if (ret < 0)
goto out_free_cdb;
compat_int_t stat;
compat_caddr_t sense;
unsigned char data_direction;
+ unsigned char pad[3];
compat_int_t quiet;
compat_int_t timeout;
compat_caddr_t unused;
int scsi_verify_blk_ioctl(struct block_device *bd, unsigned int cmd)
{
- if (bd && bd == bd->bd_contains)
+ if (bd && !bdev_is_partition(bd))
return 0;
if (capable(CAP_SYS_RAWIO))
standard algorithms (called GOST algorithms). Only signature verification
is implemented.
+config CRYPTO_SM2
+ tristate "SM2 algorithm"
+ select CRYPTO_SM3
+ select CRYPTO_AKCIPHER
+ select CRYPTO_MANAGER
+ select MPILIB
+ select ASN1
+ help
+ Generic implementation of the SM2 public key algorithm. It was
+ published by State Encryption Management Bureau, China.
+ as specified by OSCCA GM/T 0003.1-2012 -- 0003.5-2012.
+
+ References:
+ https://tools.ietf.org/html/draft-shen-sm2-ecdsa-02
+ http://www.oscca.gov.cn/sca/xxgk/2010-12/17/content_1002386.shtml
+ http://www.gmbz.org.cn/main/bzlb.html
+
config CRYPTO_CURVE25519
tristate "Curve25519 algorithm"
select CRYPTO_KPP
config CRYPTO_ANUBIS
tristate "Anubis cipher algorithm"
+ depends on CRYPTO_USER_API_ENABLE_OBSOLETE
select CRYPTO_ALGAPI
help
Anubis cipher algorithm.
config CRYPTO_ARC4
tristate "ARC4 cipher algorithm"
+ depends on CRYPTO_USER_API_ENABLE_OBSOLETE
select CRYPTO_SKCIPHER
select CRYPTO_LIB_ARC4
help
config CRYPTO_KHAZAD
tristate "Khazad cipher algorithm"
+ depends on CRYPTO_USER_API_ENABLE_OBSOLETE
select CRYPTO_ALGAPI
help
Khazad cipher algorithm.
config CRYPTO_SEED
tristate "SEED cipher algorithm"
+ depends on CRYPTO_USER_API_ENABLE_OBSOLETE
select CRYPTO_ALGAPI
help
SEED cipher algorithm (RFC4269).
config CRYPTO_TEA
tristate "TEA, XTEA and XETA cipher algorithms"
+ depends on CRYPTO_USER_API_ENABLE_OBSOLETE
select CRYPTO_ALGAPI
help
TEA cipher algorithm.
This option enables the user-spaces interface for random
number generator algorithms.
+config CRYPTO_USER_API_RNG_CAVP
+ bool "Enable CAVP testing of DRBG"
+ depends on CRYPTO_USER_API_RNG && CRYPTO_DRBG
+ help
+ This option enables extra API for CAVP testing via the user-space
+ interface: resetting of DRBG entropy, and providing Additional Data.
+ This should only be enabled for CAVP testing. You should say
+ no unless you know what this is.
+
config CRYPTO_USER_API_AEAD
tristate "User-space interface for AEAD cipher algorithms"
depends on NET
This option enables the user-spaces interface for AEAD
cipher algorithms.
+config CRYPTO_USER_API_ENABLE_OBSOLETE
+ bool "Enable obsolete cryptographic algorithms for userspace"
+ depends on CRYPTO_USER_API
+ default y
+ help
+ Allow obsolete cryptographic algorithms to be selected that have
+ already been phased out from internal use by the kernel, and are
+ only useful for userspace clients that still rely on them.
+
config CRYPTO_STATS
bool "Crypto usage statistics for User-space"
depends on CRYPTO_USER
rsa_generic-y += rsa-pkcs1pad.o
obj-$(CONFIG_CRYPTO_RSA) += rsa_generic.o
+$(obj)/sm2signature.asn1.o: $(obj)/sm2signature.asn1.c $(obj)/sm2signature.asn1.h
+$(obj)/sm2.o: $(obj)/sm2signature.asn1.h
+
+sm2_generic-y += sm2signature.asn1.o
+sm2_generic-y += sm2.o
+
+obj-$(CONFIG_CRYPTO_SM2) += sm2_generic.o
+
crypto_acompress-y := acompress.o
crypto_acompress-y += scompress.o
obj-$(CONFIG_CRYPTO_ACOMP2) += crypto_acompress.o
if (!type->setauthsize)
goto unlock;
err = type->setauthsize(ask->private, optlen);
+ break;
+ case ALG_SET_DRBG_ENTROPY:
+ if (sock->state == SS_CONNECTED)
+ goto unlock;
+ if (!type->setentropy)
+ goto unlock;
+
+ err = type->setentropy(ask->private, optval, optlen);
}
unlock:
security_sock_graft(sk2, newsock);
security_sk_clone(sk, sk2);
+ /*
+ * newsock->ops assigned here to allow type->accept call to override
+ * them when required.
+ */
+ newsock->ops = type->ops;
err = type->accept(ask->private, sk2);
nokey = err == -ENOKEY;
alg_sk(sk2)->parent = sk;
alg_sk(sk2)->type = type;
- newsock->ops = type->ops;
newsock->state = SS_CONNECTED;
if (nokey)
#include <crypto/internal/hash.h>
#include <crypto/scatterwalk.h>
-#include <linux/bug.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
unsigned int nbytes = min(walk->entrylen,
((unsigned int)(PAGE_SIZE)) - offset);
- if (walk->flags & CRYPTO_ALG_ASYNC)
- walk->data = kmap(walk->pg);
- else
- walk->data = kmap_atomic(walk->pg);
+ walk->data = kmap_atomic(walk->pg);
walk->data += offset;
if (offset & alignmask) {
}
}
- if (walk->flags & CRYPTO_ALG_ASYNC)
- kunmap(walk->pg);
- else {
- kunmap_atomic(walk->data);
- /*
- * The may sleep test only makes sense for sync users.
- * Async users don't need to sleep here anyway.
- */
- crypto_yield(walk->flags);
- }
+ kunmap_atomic(walk->data);
+ crypto_yield(walk->flags);
if (err)
return err;
walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req));
walk->sg = req->src;
- walk->flags = req->base.flags & CRYPTO_TFM_REQ_MASK;
+ walk->flags = req->base.flags;
return hash_walk_new_entry(walk);
}
EXPORT_SYMBOL_GPL(crypto_hash_walk_first);
-int crypto_ahash_walk_first(struct ahash_request *req,
- struct crypto_hash_walk *walk)
-{
- walk->total = req->nbytes;
-
- if (!walk->total) {
- walk->entrylen = 0;
- return 0;
- }
-
- walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req));
- walk->sg = req->src;
- walk->flags = req->base.flags & CRYPTO_TFM_REQ_MASK;
- walk->flags |= CRYPTO_ALG_ASYNC;
-
- BUILD_BUG_ON(CRYPTO_TFM_REQ_MASK & CRYPTO_ALG_ASYNC);
-
- return hash_walk_new_entry(walk);
-}
-EXPORT_SYMBOL_GPL(crypto_ahash_walk_first);
-
static int ahash_setkey_unaligned(struct crypto_ahash *tfm, const u8 *key,
unsigned int keylen)
{
return ahash_def_finup_finish1(req, err);
}
+static void crypto_ahash_exit_tfm(struct crypto_tfm *tfm)
+{
+ struct crypto_ahash *hash = __crypto_ahash_cast(tfm);
+ struct ahash_alg *alg = crypto_ahash_alg(hash);
+
+ alg->exit_tfm(hash);
+}
+
static int crypto_ahash_init_tfm(struct crypto_tfm *tfm)
{
struct crypto_ahash *hash = __crypto_ahash_cast(tfm);
ahash_set_needkey(hash);
}
- return 0;
+ if (alg->exit_tfm)
+ tfm->exit = crypto_ahash_exit_tfm;
+
+ return alg->init_tfm ? alg->init_tfm(hash) : 0;
}
static unsigned int crypto_ahash_extsize(struct crypto_alg *alg)
SYNC_SKCIPHER_REQUEST_ON_STACK(skreq, null_tfm);
skcipher_request_set_sync_tfm(skreq, null_tfm);
- skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ skcipher_request_set_callback(skreq, CRYPTO_TFM_REQ_MAY_SLEEP,
NULL, NULL);
skcipher_request_set_crypt(skreq, src, dst, len, NULL);
/*
* Make sure sufficient data is present -- note, the same check is
- * is also present in sendmsg/sendpage. The checks in sendpage/sendmsg
+ * also present in sendmsg/sendpage. The checks in sendpage/sendmsg
* shall provide an information to the data sender that something is
* wrong, but they are irrelevant to maintain the kernel integrity.
* We need this check here too in case user space decides to not honor
areq->outlen = outlen;
aead_request_set_callback(&areq->cra_u.aead_req,
- CRYPTO_TFM_REQ_MAY_BACKLOG,
+ CRYPTO_TFM_REQ_MAY_SLEEP,
af_alg_async_cb, areq);
err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) :
crypto_aead_decrypt(&areq->cra_u.aead_req);
/* AIO operation in progress */
- if (err == -EINPROGRESS || err == -EBUSY)
+ if (err == -EINPROGRESS)
return -EIOCBQUEUED;
sock_put(sk);
} else {
/* Synchronous operation */
aead_request_set_callback(&areq->cra_u.aead_req,
+ CRYPTO_TFM_REQ_MAY_SLEEP |
CRYPTO_TFM_REQ_MAY_BACKLOG,
crypto_req_done, &ctx->wait);
err = crypto_wait_req(ctx->enc ?
* DAMAGE.
*/
+#include <linux/capability.h>
#include <linux/module.h>
#include <crypto/rng.h>
#include <linux/random.h>
#define MAXSIZE 128
unsigned int len;
struct crypto_rng *drng;
+ u8 *addtl;
+ size_t addtl_len;
};
-static int rng_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
- int flags)
+struct rng_parent_ctx {
+ struct crypto_rng *drng;
+ u8 *entropy;
+};
+
+static void rng_reset_addtl(struct rng_ctx *ctx)
{
- struct sock *sk = sock->sk;
- struct alg_sock *ask = alg_sk(sk);
- struct rng_ctx *ctx = ask->private;
- int err;
+ kfree_sensitive(ctx->addtl);
+ ctx->addtl = NULL;
+ ctx->addtl_len = 0;
+}
+
+static int _rng_recvmsg(struct crypto_rng *drng, struct msghdr *msg, size_t len,
+ u8 *addtl, size_t addtl_len)
+{
+ int err = 0;
int genlen = 0;
u8 result[MAXSIZE];
* seeding as they automatically seed. The X9.31 DRNG will return
* an error if it was not seeded properly.
*/
- genlen = crypto_rng_get_bytes(ctx->drng, result, len);
+ genlen = crypto_rng_generate(drng, addtl, addtl_len, result, len);
if (genlen < 0)
return genlen;
return err ? err : len;
}
+static int rng_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
+ int flags)
+{
+ struct sock *sk = sock->sk;
+ struct alg_sock *ask = alg_sk(sk);
+ struct rng_ctx *ctx = ask->private;
+
+ return _rng_recvmsg(ctx->drng, msg, len, NULL, 0);
+}
+
+static int rng_test_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
+ int flags)
+{
+ struct sock *sk = sock->sk;
+ struct alg_sock *ask = alg_sk(sk);
+ struct rng_ctx *ctx = ask->private;
+ int ret;
+
+ lock_sock(sock->sk);
+ ret = _rng_recvmsg(ctx->drng, msg, len, ctx->addtl, ctx->addtl_len);
+ rng_reset_addtl(ctx);
+ release_sock(sock->sk);
+
+ return ret;
+}
+
+static int rng_test_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
+{
+ int err;
+ struct alg_sock *ask = alg_sk(sock->sk);
+ struct rng_ctx *ctx = ask->private;
+
+ lock_sock(sock->sk);
+ if (len > MAXSIZE) {
+ err = -EMSGSIZE;
+ goto unlock;
+ }
+
+ rng_reset_addtl(ctx);
+ ctx->addtl = kmalloc(len, GFP_KERNEL);
+ if (!ctx->addtl) {
+ err = -ENOMEM;
+ goto unlock;
+ }
+
+ err = memcpy_from_msg(ctx->addtl, msg, len);
+ if (err) {
+ rng_reset_addtl(ctx);
+ goto unlock;
+ }
+ ctx->addtl_len = len;
+
+unlock:
+ release_sock(sock->sk);
+ return err ? err : len;
+}
+
static struct proto_ops algif_rng_ops = {
.family = PF_ALG,
.recvmsg = rng_recvmsg,
};
+static struct proto_ops __maybe_unused algif_rng_test_ops = {
+ .family = PF_ALG,
+
+ .connect = sock_no_connect,
+ .socketpair = sock_no_socketpair,
+ .getname = sock_no_getname,
+ .ioctl = sock_no_ioctl,
+ .listen = sock_no_listen,
+ .shutdown = sock_no_shutdown,
+ .mmap = sock_no_mmap,
+ .bind = sock_no_bind,
+ .accept = sock_no_accept,
+ .sendpage = sock_no_sendpage,
+
+ .release = af_alg_release,
+ .recvmsg = rng_test_recvmsg,
+ .sendmsg = rng_test_sendmsg,
+};
+
static void *rng_bind(const char *name, u32 type, u32 mask)
{
- return crypto_alloc_rng(name, type, mask);
+ struct rng_parent_ctx *pctx;
+ struct crypto_rng *rng;
+
+ pctx = kzalloc(sizeof(*pctx), GFP_KERNEL);
+ if (!pctx)
+ return ERR_PTR(-ENOMEM);
+
+ rng = crypto_alloc_rng(name, type, mask);
+ if (IS_ERR(rng)) {
+ kfree(pctx);
+ return ERR_CAST(rng);
+ }
+
+ pctx->drng = rng;
+ return pctx;
}
static void rng_release(void *private)
{
- crypto_free_rng(private);
+ struct rng_parent_ctx *pctx = private;
+
+ if (unlikely(!pctx))
+ return;
+ crypto_free_rng(pctx->drng);
+ kfree_sensitive(pctx->entropy);
+ kfree_sensitive(pctx);
}
static void rng_sock_destruct(struct sock *sk)
struct alg_sock *ask = alg_sk(sk);
struct rng_ctx *ctx = ask->private;
+ rng_reset_addtl(ctx);
sock_kfree_s(sk, ctx, ctx->len);
af_alg_release_parent(sk);
}
static int rng_accept_parent(void *private, struct sock *sk)
{
struct rng_ctx *ctx;
+ struct rng_parent_ctx *pctx = private;
struct alg_sock *ask = alg_sk(sk);
unsigned int len = sizeof(*ctx);
return -ENOMEM;
ctx->len = len;
+ ctx->addtl = NULL;
+ ctx->addtl_len = 0;
/*
* No seeding done at that point -- if multiple accepts are
* state of the RNG.
*/
- ctx->drng = private;
+ ctx->drng = pctx->drng;
ask->private = ctx;
sk->sk_destruct = rng_sock_destruct;
+ /*
+ * Non NULL pctx->entropy means that CAVP test has been initiated on
+ * this socket, replace proto_ops algif_rng_ops with algif_rng_test_ops.
+ */
+ if (IS_ENABLED(CONFIG_CRYPTO_USER_API_RNG_CAVP) && pctx->entropy)
+ sk->sk_socket->ops = &algif_rng_test_ops;
+
return 0;
}
static int rng_setkey(void *private, const u8 *seed, unsigned int seedlen)
{
+ struct rng_parent_ctx *pctx = private;
/*
* Check whether seedlen is of sufficient size is done in RNG
* implementations.
*/
- return crypto_rng_reset(private, seed, seedlen);
+ return crypto_rng_reset(pctx->drng, seed, seedlen);
+}
+
+static int __maybe_unused rng_setentropy(void *private, sockptr_t entropy,
+ unsigned int len)
+{
+ struct rng_parent_ctx *pctx = private;
+ u8 *kentropy = NULL;
+
+ if (!capable(CAP_SYS_ADMIN))
+ return -EACCES;
+
+ if (pctx->entropy)
+ return -EINVAL;
+
+ if (len > MAXSIZE)
+ return -EMSGSIZE;
+
+ if (len) {
+ kentropy = memdup_sockptr(entropy, len);
+ if (IS_ERR(kentropy))
+ return PTR_ERR(kentropy);
+ }
+
+ crypto_rng_alg(pctx->drng)->set_ent(pctx->drng, kentropy, len);
+ /*
+ * Since rng doesn't perform any memory management for the entropy
+ * buffer, save kentropy pointer to pctx now to free it after use.
+ */
+ pctx->entropy = kentropy;
+ return 0;
}
static const struct af_alg_type algif_type_rng = {
.release = rng_release,
.accept = rng_accept_parent,
.setkey = rng_setkey,
+#ifdef CONFIG_CRYPTO_USER_API_RNG_CAVP
+ .setentropy = rng_setentropy,
+#endif
.ops = &algif_rng_ops,
.name = "rng",
.owner = THIS_MODULE
crypto_skcipher_decrypt(&areq->cra_u.skcipher_req);
/* AIO operation in progress */
- if (err == -EINPROGRESS || err == -EBUSY)
+ if (err == -EINPROGRESS)
return -EIOCBQUEUED;
sock_put(sk);
#include <crypto/arc4.h>
#include <crypto/internal/skcipher.h>
#include <linux/init.h>
+#include <linux/kernel.h>
#include <linux/module.h>
+#include <linux/sched.h>
static int crypto_arc4_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
unsigned int key_len)
return err;
}
+static int crypto_arc4_init(struct crypto_skcipher *tfm)
+{
+ pr_warn_ratelimited("\"%s\" (%ld) uses obsolete ecb(arc4) skcipher\n",
+ current->comm, (unsigned long)current->pid);
+
+ return 0;
+}
+
static struct skcipher_alg arc4_alg = {
/*
* For legacy reasons, this is named "ecb(arc4)", not "arc4".
.setkey = crypto_arc4_setkey,
.encrypt = crypto_arc4_crypt,
.decrypt = crypto_arc4_crypt,
+ .init = crypto_arc4_init,
};
static int __init arc4_init(void)
#include <keys/asymmetric-subtype.h>
#include <crypto/public_key.h>
#include <crypto/akcipher.h>
+#include <crypto/sm2.h>
+#include <crypto/sm3_base.h>
MODULE_DESCRIPTION("In-software asymmetric public-key subtype");
MODULE_AUTHOR("Red Hat, Inc.");
return ret;
}
+#if IS_REACHABLE(CONFIG_CRYPTO_SM2)
+static int cert_sig_digest_update(const struct public_key_signature *sig,
+ struct crypto_akcipher *tfm_pkey)
+{
+ struct crypto_shash *tfm;
+ struct shash_desc *desc;
+ size_t desc_size;
+ unsigned char dgst[SM3_DIGEST_SIZE];
+ int ret;
+
+ BUG_ON(!sig->data);
+
+ ret = sm2_compute_z_digest(tfm_pkey, SM2_DEFAULT_USERID,
+ SM2_DEFAULT_USERID_LEN, dgst);
+ if (ret)
+ return ret;
+
+ tfm = crypto_alloc_shash(sig->hash_algo, 0, 0);
+ if (IS_ERR(tfm))
+ return PTR_ERR(tfm);
+
+ desc_size = crypto_shash_descsize(tfm) + sizeof(*desc);
+ desc = kzalloc(desc_size, GFP_KERNEL);
+ if (!desc) {
+ ret = -ENOMEM;
+ goto error_free_tfm;
+ }
+
+ desc->tfm = tfm;
+
+ ret = crypto_shash_init(desc);
+ if (ret < 0)
+ goto error_free_desc;
+
+ ret = crypto_shash_update(desc, dgst, SM3_DIGEST_SIZE);
+ if (ret < 0)
+ goto error_free_desc;
+
+ ret = crypto_shash_finup(desc, sig->data, sig->data_size, sig->digest);
+
+error_free_desc:
+ kfree(desc);
+error_free_tfm:
+ crypto_free_shash(tfm);
+ return ret;
+}
+#else
+static inline int cert_sig_digest_update(
+ const struct public_key_signature *sig,
+ struct crypto_akcipher *tfm_pkey)
+{
+ return -ENOTSUPP;
+}
+#endif /* ! IS_REACHABLE(CONFIG_CRYPTO_SM2) */
+
/*
* Verify a signature using a public key.
*/
if (ret)
goto error_free_key;
+ if (strcmp(sig->pkey_algo, "sm2") == 0 && sig->data_size) {
+ ret = cert_sig_digest_update(sig, tfm);
+ if (ret)
+ goto error_free_key;
+ }
+
sg_init_table(src_sg, 2);
sg_set_buf(&src_sg[0], sig->s, sig->s_size);
sg_set_buf(&src_sg[1], sig->digest, sig->digest_size);
case OID_gost2012Signature512:
ctx->cert->sig->hash_algo = "streebog512";
goto ecrdsa;
+
+ case OID_SM2_with_SM3:
+ ctx->cert->sig->hash_algo = "sm3";
+ goto sm2;
}
rsa_pkcs1:
ctx->cert->sig->encoding = "raw";
ctx->algo_oid = ctx->last_oid;
return 0;
+sm2:
+ ctx->cert->sig->pkey_algo = "sm2";
+ ctx->cert->sig->encoding = "raw";
+ ctx->algo_oid = ctx->last_oid;
+ return 0;
}
/*
}
if (strcmp(ctx->cert->sig->pkey_algo, "rsa") == 0 ||
- strcmp(ctx->cert->sig->pkey_algo, "ecrdsa") == 0) {
+ strcmp(ctx->cert->sig->pkey_algo, "ecrdsa") == 0 ||
+ strcmp(ctx->cert->sig->pkey_algo, "sm2") == 0) {
/* Discard the BIT STRING metadata */
if (vlen < 1 || *(const u8 *)value != 0)
return -EBADMSG;
struct x509_parse_context *ctx = context;
ctx->key_algo = ctx->last_oid;
- if (ctx->last_oid == OID_rsaEncryption)
+ switch (ctx->last_oid) {
+ case OID_rsaEncryption:
ctx->cert->pub->pkey_algo = "rsa";
- else if (ctx->last_oid == OID_gost2012PKey256 ||
- ctx->last_oid == OID_gost2012PKey512)
+ break;
+ case OID_gost2012PKey256:
+ case OID_gost2012PKey512:
ctx->cert->pub->pkey_algo = "ecrdsa";
- else
+ break;
+ case OID_id_ecPublicKey:
+ ctx->cert->pub->pkey_algo = "sm2";
+ break;
+ default:
return -ENOPKG;
+ }
/* Discard the BIT STRING metadata */
if (vlen < 1 || *(const u8 *)value != 0)
pr_devel("==>%s()\n", __func__);
+ sig->data = cert->tbs;
+ sig->data_size = cert->tbs_size;
+
if (!cert->pub->pkey_algo)
cert->unsupported_key = true;
*/
#include <crypto/algapi.h>
-#include <crypto/cbc.h>
#include <crypto/internal/skcipher.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/log2.h>
#include <linux/module.h>
-static inline void crypto_cbc_encrypt_one(struct crypto_skcipher *tfm,
- const u8 *src, u8 *dst)
+static int crypto_cbc_encrypt_segment(struct skcipher_walk *walk,
+ struct crypto_skcipher *skcipher)
{
- crypto_cipher_encrypt_one(skcipher_cipher_simple(tfm), dst, src);
+ unsigned int bsize = crypto_skcipher_blocksize(skcipher);
+ void (*fn)(struct crypto_tfm *, u8 *, const u8 *);
+ unsigned int nbytes = walk->nbytes;
+ u8 *src = walk->src.virt.addr;
+ u8 *dst = walk->dst.virt.addr;
+ struct crypto_cipher *cipher;
+ struct crypto_tfm *tfm;
+ u8 *iv = walk->iv;
+
+ cipher = skcipher_cipher_simple(skcipher);
+ tfm = crypto_cipher_tfm(cipher);
+ fn = crypto_cipher_alg(cipher)->cia_encrypt;
+
+ do {
+ crypto_xor(iv, src, bsize);
+ fn(tfm, dst, iv);
+ memcpy(iv, dst, bsize);
+
+ src += bsize;
+ dst += bsize;
+ } while ((nbytes -= bsize) >= bsize);
+
+ return nbytes;
+}
+
+static int crypto_cbc_encrypt_inplace(struct skcipher_walk *walk,
+ struct crypto_skcipher *skcipher)
+{
+ unsigned int bsize = crypto_skcipher_blocksize(skcipher);
+ void (*fn)(struct crypto_tfm *, u8 *, const u8 *);
+ unsigned int nbytes = walk->nbytes;
+ u8 *src = walk->src.virt.addr;
+ struct crypto_cipher *cipher;
+ struct crypto_tfm *tfm;
+ u8 *iv = walk->iv;
+
+ cipher = skcipher_cipher_simple(skcipher);
+ tfm = crypto_cipher_tfm(cipher);
+ fn = crypto_cipher_alg(cipher)->cia_encrypt;
+
+ do {
+ crypto_xor(src, iv, bsize);
+ fn(tfm, src, src);
+ iv = src;
+
+ src += bsize;
+ } while ((nbytes -= bsize) >= bsize);
+
+ memcpy(walk->iv, iv, bsize);
+
+ return nbytes;
}
static int crypto_cbc_encrypt(struct skcipher_request *req)
{
- return crypto_cbc_encrypt_walk(req, crypto_cbc_encrypt_one);
+ struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+ struct skcipher_walk walk;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, false);
+
+ while (walk.nbytes) {
+ if (walk.src.virt.addr == walk.dst.virt.addr)
+ err = crypto_cbc_encrypt_inplace(&walk, skcipher);
+ else
+ err = crypto_cbc_encrypt_segment(&walk, skcipher);
+ err = skcipher_walk_done(&walk, err);
+ }
+
+ return err;
+}
+
+static int crypto_cbc_decrypt_segment(struct skcipher_walk *walk,
+ struct crypto_skcipher *skcipher)
+{
+ unsigned int bsize = crypto_skcipher_blocksize(skcipher);
+ void (*fn)(struct crypto_tfm *, u8 *, const u8 *);
+ unsigned int nbytes = walk->nbytes;
+ u8 *src = walk->src.virt.addr;
+ u8 *dst = walk->dst.virt.addr;
+ struct crypto_cipher *cipher;
+ struct crypto_tfm *tfm;
+ u8 *iv = walk->iv;
+
+ cipher = skcipher_cipher_simple(skcipher);
+ tfm = crypto_cipher_tfm(cipher);
+ fn = crypto_cipher_alg(cipher)->cia_decrypt;
+
+ do {
+ fn(tfm, dst, src);
+ crypto_xor(dst, iv, bsize);
+ iv = src;
+
+ src += bsize;
+ dst += bsize;
+ } while ((nbytes -= bsize) >= bsize);
+
+ memcpy(walk->iv, iv, bsize);
+
+ return nbytes;
}
-static inline void crypto_cbc_decrypt_one(struct crypto_skcipher *tfm,
- const u8 *src, u8 *dst)
+static int crypto_cbc_decrypt_inplace(struct skcipher_walk *walk,
+ struct crypto_skcipher *skcipher)
{
- crypto_cipher_decrypt_one(skcipher_cipher_simple(tfm), dst, src);
+ unsigned int bsize = crypto_skcipher_blocksize(skcipher);
+ void (*fn)(struct crypto_tfm *, u8 *, const u8 *);
+ unsigned int nbytes = walk->nbytes;
+ u8 *src = walk->src.virt.addr;
+ u8 last_iv[MAX_CIPHER_BLOCKSIZE];
+ struct crypto_cipher *cipher;
+ struct crypto_tfm *tfm;
+
+ cipher = skcipher_cipher_simple(skcipher);
+ tfm = crypto_cipher_tfm(cipher);
+ fn = crypto_cipher_alg(cipher)->cia_decrypt;
+
+ /* Start of the last block. */
+ src += nbytes - (nbytes & (bsize - 1)) - bsize;
+ memcpy(last_iv, src, bsize);
+
+ for (;;) {
+ fn(tfm, src, src);
+ if ((nbytes -= bsize) < bsize)
+ break;
+ crypto_xor(src, src - bsize, bsize);
+ src -= bsize;
+ }
+
+ crypto_xor(src, walk->iv, bsize);
+ memcpy(walk->iv, last_iv, bsize);
+
+ return nbytes;
}
static int crypto_cbc_decrypt(struct skcipher_request *req)
{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct skcipher_walk walk;
int err;
err = skcipher_walk_virt(&walk, req, false);
while (walk.nbytes) {
- err = crypto_cbc_decrypt_blocks(&walk, tfm,
- crypto_cbc_decrypt_one);
+ if (walk.src.virt.addr == walk.dst.virt.addr)
+ err = crypto_cbc_decrypt_inplace(&walk, skcipher);
+ else
+ err = crypto_cbc_decrypt_segment(&walk, skcipher);
err = skcipher_walk_done(&walk, err);
}
* pages = {},
* month = {June},
*}
- * Used by the iSCSI driver, possibly others, and derived from the
+ * Used by the iSCSI driver, possibly others, and derived from
* the iscsi-crc.c module of the linux-iscsi driver at
* http://linux-iscsi.sourceforge.net.
*
};
/*
- * Steps through buffer one byte at at time, calculates reflected
+ * Steps through buffer one byte at a time, calculates reflected
* crc using table.
*/
};
/*
- * Steps through buffer one byte at at time, calculates reflected
+ * Steps through buffer one byte at a time, calculates reflected
* crc using table.
*/
#include <linux/err.h>
#include <linux/delay.h>
+#include <linux/device.h>
#include <crypto/engine.h>
#include <uapi/linux/sched/types.h>
#include "internal.h"
* crypto-engine queue.
* @dev: the device attached with one hardware engine
* @retry_support: whether hardware has support for retry mechanism
- * @cbk_do_batch: pointer to a callback function to be invoked when executing a
+ * @cbk_do_batch: pointer to a callback function to be invoked when executing
* a batch of requests.
* This has the form:
* callback(struct crypto_engine *engine)
#include <crypto/internal/akcipher.h>
#include <crypto/akcipher.h>
#include <linux/oid_registry.h>
+#include <linux/scatterlist.h>
#include "ecrdsa_params.asn1.h"
#include "ecrdsa_pub_key.asn1.h"
#include "ecc.h"
#include <crypto/algapi.h>
#include <linux/completion.h>
-#include <linux/mm.h>
-#include <linux/highmem.h>
-#include <linux/interrupt.h>
-#include <linux/init.h>
#include <linux/list.h>
#include <linux/module.h>
-#include <linux/kernel.h>
#include <linux/notifier.h>
+#include <linux/numa.h>
+#include <linux/refcount.h>
#include <linux/rwsem.h>
-#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/types.h>
struct crypto_instance;
struct crypto_template;
blocking_notifier_call_chain(&crypto_chain, val, v);
}
+static inline void crypto_yield(u32 flags)
+{
+ if (flags & CRYPTO_TFM_REQ_MAY_SLEEP)
+ cond_resched();
+}
+
#endif /* _CRYPTO_INTERNAL_H */
* DAMAGE.
*/
+#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/fips.h>
#include <linux/time.h>
-#include <linux/crypto.h>
#include <crypto/internal/rng.h>
#include "jitterentropy.h"
static int c_show(struct seq_file *m, void *p)
{
struct crypto_alg *alg = list_entry(p, struct crypto_alg, cra_list);
-
+
seq_printf(m, "name : %s\n", alg->cra_name);
seq_printf(m, "driver : %s\n", alg->cra_driver_name);
seq_printf(m, "module : %s\n", module_name(alg->cra_module));
alg->cra_type->show(m, alg);
goto out;
}
-
+
switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
case CRYPTO_ALG_TYPE_CIPHER:
seq_printf(m, "type : cipher\n");
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/random.h>
+#include <linux/scatterlist.h>
/*
* Hash algorithm OIDs plus ASN.1 DER wrappings [RFC4880 sec 5.2.2].
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SM2 asymmetric public-key algorithm
+ * as specified by OSCCA GM/T 0003.1-2012 -- 0003.5-2012 SM2 and
+ * described at https://tools.ietf.org/html/draft-shen-sm2-ecdsa-02
+ *
+ * Copyright (c) 2020, Alibaba Group.
+ * Authors: Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
+ */
+
+#include <linux/module.h>
+#include <linux/mpi.h>
+#include <crypto/internal/akcipher.h>
+#include <crypto/akcipher.h>
+#include <crypto/hash.h>
+#include <crypto/sm3_base.h>
+#include <crypto/rng.h>
+#include <crypto/sm2.h>
+#include "sm2signature.asn1.h"
+
+#define MPI_NBYTES(m) ((mpi_get_nbits(m) + 7) / 8)
+
+struct ecc_domain_parms {
+ const char *desc; /* Description of the curve. */
+ unsigned int nbits; /* Number of bits. */
+ unsigned int fips:1; /* True if this is a FIPS140-2 approved curve */
+
+ /* The model describing this curve. This is mainly used to select
+ * the group equation.
+ */
+ enum gcry_mpi_ec_models model;
+
+ /* The actual ECC dialect used. This is used for curve specific
+ * optimizations and to select encodings etc.
+ */
+ enum ecc_dialects dialect;
+
+ const char *p; /* The prime defining the field. */
+ const char *a, *b; /* The coefficients. For Twisted Edwards
+ * Curves b is used for d. For Montgomery
+ * Curves (a,b) has ((A-2)/4,B^-1).
+ */
+ const char *n; /* The order of the base point. */
+ const char *g_x, *g_y; /* Base point. */
+ unsigned int h; /* Cofactor. */
+};
+
+static const struct ecc_domain_parms sm2_ecp = {
+ .desc = "sm2p256v1",
+ .nbits = 256,
+ .fips = 0,
+ .model = MPI_EC_WEIERSTRASS,
+ .dialect = ECC_DIALECT_STANDARD,
+ .p = "0xfffffffeffffffffffffffffffffffffffffffff00000000ffffffffffffffff",
+ .a = "0xfffffffeffffffffffffffffffffffffffffffff00000000fffffffffffffffc",
+ .b = "0x28e9fa9e9d9f5e344d5a9e4bcf6509a7f39789f515ab8f92ddbcbd414d940e93",
+ .n = "0xfffffffeffffffffffffffffffffffff7203df6b21c6052b53bbf40939d54123",
+ .g_x = "0x32c4ae2c1f1981195f9904466a39c9948fe30bbff2660be1715a4589334c74c7",
+ .g_y = "0xbc3736a2f4f6779c59bdcee36b692153d0a9877cc62a474002df32e52139f0a0",
+ .h = 1
+};
+
+static int sm2_ec_ctx_init(struct mpi_ec_ctx *ec)
+{
+ const struct ecc_domain_parms *ecp = &sm2_ecp;
+ MPI p, a, b;
+ MPI x, y;
+ int rc = -EINVAL;
+
+ p = mpi_scanval(ecp->p);
+ a = mpi_scanval(ecp->a);
+ b = mpi_scanval(ecp->b);
+ if (!p || !a || !b)
+ goto free_p;
+
+ x = mpi_scanval(ecp->g_x);
+ y = mpi_scanval(ecp->g_y);
+ if (!x || !y)
+ goto free;
+
+ rc = -ENOMEM;
+ /* mpi_ec_setup_elliptic_curve */
+ ec->G = mpi_point_new(0);
+ if (!ec->G)
+ goto free;
+
+ mpi_set(ec->G->x, x);
+ mpi_set(ec->G->y, y);
+ mpi_set_ui(ec->G->z, 1);
+
+ rc = -EINVAL;
+ ec->n = mpi_scanval(ecp->n);
+ if (!ec->n) {
+ mpi_point_release(ec->G);
+ goto free;
+ }
+
+ ec->h = ecp->h;
+ ec->name = ecp->desc;
+ mpi_ec_init(ec, ecp->model, ecp->dialect, 0, p, a, b);
+
+ rc = 0;
+
+free:
+ mpi_free(x);
+ mpi_free(y);
+free_p:
+ mpi_free(p);
+ mpi_free(a);
+ mpi_free(b);
+
+ return rc;
+}
+
+static void sm2_ec_ctx_deinit(struct mpi_ec_ctx *ec)
+{
+ mpi_ec_deinit(ec);
+
+ memset(ec, 0, sizeof(*ec));
+}
+
+static int sm2_ec_ctx_reset(struct mpi_ec_ctx *ec)
+{
+ sm2_ec_ctx_deinit(ec);
+ return sm2_ec_ctx_init(ec);
+}
+
+/* RESULT must have been initialized and is set on success to the
+ * point given by VALUE.
+ */
+static int sm2_ecc_os2ec(MPI_POINT result, MPI value)
+{
+ int rc;
+ size_t n;
+ const unsigned char *buf;
+ unsigned char *buf_memory;
+ MPI x, y;
+
+ n = (mpi_get_nbits(value)+7)/8;
+ buf_memory = kmalloc(n, GFP_KERNEL);
+ rc = mpi_print(GCRYMPI_FMT_USG, buf_memory, n, &n, value);
+ if (rc) {
+ kfree(buf_memory);
+ return rc;
+ }
+ buf = buf_memory;
+
+ if (n < 1) {
+ kfree(buf_memory);
+ return -EINVAL;
+ }
+ if (*buf != 4) {
+ kfree(buf_memory);
+ return -EINVAL; /* No support for point compression. */
+ }
+ if (((n-1)%2)) {
+ kfree(buf_memory);
+ return -EINVAL;
+ }
+ n = (n-1)/2;
+ x = mpi_read_raw_data(buf + 1, n);
+ if (!x) {
+ kfree(buf_memory);
+ return -ENOMEM;
+ }
+ y = mpi_read_raw_data(buf + 1 + n, n);
+ kfree(buf_memory);
+ if (!y) {
+ mpi_free(x);
+ return -ENOMEM;
+ }
+
+ mpi_normalize(x);
+ mpi_normalize(y);
+
+ mpi_set(result->x, x);
+ mpi_set(result->y, y);
+ mpi_set_ui(result->z, 1);
+
+ mpi_free(x);
+ mpi_free(y);
+
+ return 0;
+}
+
+struct sm2_signature_ctx {
+ MPI sig_r;
+ MPI sig_s;
+};
+
+int sm2_get_signature_r(void *context, size_t hdrlen, unsigned char tag,
+ const void *value, size_t vlen)
+{
+ struct sm2_signature_ctx *sig = context;
+
+ if (!value || !vlen)
+ return -EINVAL;
+
+ sig->sig_r = mpi_read_raw_data(value, vlen);
+ if (!sig->sig_r)
+ return -ENOMEM;
+
+ return 0;
+}
+
+int sm2_get_signature_s(void *context, size_t hdrlen, unsigned char tag,
+ const void *value, size_t vlen)
+{
+ struct sm2_signature_ctx *sig = context;
+
+ if (!value || !vlen)
+ return -EINVAL;
+
+ sig->sig_s = mpi_read_raw_data(value, vlen);
+ if (!sig->sig_s)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static int sm2_z_digest_update(struct shash_desc *desc,
+ MPI m, unsigned int pbytes)
+{
+ static const unsigned char zero[32];
+ unsigned char *in;
+ unsigned int inlen;
+
+ in = mpi_get_buffer(m, &inlen, NULL);
+ if (!in)
+ return -EINVAL;
+
+ if (inlen < pbytes) {
+ /* padding with zero */
+ crypto_sm3_update(desc, zero, pbytes - inlen);
+ crypto_sm3_update(desc, in, inlen);
+ } else if (inlen > pbytes) {
+ /* skip the starting zero */
+ crypto_sm3_update(desc, in + inlen - pbytes, pbytes);
+ } else {
+ crypto_sm3_update(desc, in, inlen);
+ }
+
+ kfree(in);
+ return 0;
+}
+
+static int sm2_z_digest_update_point(struct shash_desc *desc,
+ MPI_POINT point, struct mpi_ec_ctx *ec, unsigned int pbytes)
+{
+ MPI x, y;
+ int ret = -EINVAL;
+
+ x = mpi_new(0);
+ y = mpi_new(0);
+
+ if (!mpi_ec_get_affine(x, y, point, ec) &&
+ !sm2_z_digest_update(desc, x, pbytes) &&
+ !sm2_z_digest_update(desc, y, pbytes))
+ ret = 0;
+
+ mpi_free(x);
+ mpi_free(y);
+ return ret;
+}
+
+int sm2_compute_z_digest(struct crypto_akcipher *tfm,
+ const unsigned char *id, size_t id_len,
+ unsigned char dgst[SM3_DIGEST_SIZE])
+{
+ struct mpi_ec_ctx *ec = akcipher_tfm_ctx(tfm);
+ uint16_t bits_len;
+ unsigned char entl[2];
+ SHASH_DESC_ON_STACK(desc, NULL);
+ unsigned int pbytes;
+
+ if (id_len > (USHRT_MAX / 8) || !ec->Q)
+ return -EINVAL;
+
+ bits_len = (uint16_t)(id_len * 8);
+ entl[0] = bits_len >> 8;
+ entl[1] = bits_len & 0xff;
+
+ pbytes = MPI_NBYTES(ec->p);
+
+ /* ZA = H256(ENTLA | IDA | a | b | xG | yG | xA | yA) */
+ sm3_base_init(desc);
+ crypto_sm3_update(desc, entl, 2);
+ crypto_sm3_update(desc, id, id_len);
+
+ if (sm2_z_digest_update(desc, ec->a, pbytes) ||
+ sm2_z_digest_update(desc, ec->b, pbytes) ||
+ sm2_z_digest_update_point(desc, ec->G, ec, pbytes) ||
+ sm2_z_digest_update_point(desc, ec->Q, ec, pbytes))
+ return -EINVAL;
+
+ crypto_sm3_final(desc, dgst);
+ return 0;
+}
+EXPORT_SYMBOL(sm2_compute_z_digest);
+
+static int _sm2_verify(struct mpi_ec_ctx *ec, MPI hash, MPI sig_r, MPI sig_s)
+{
+ int rc = -EINVAL;
+ struct gcry_mpi_point sG, tP;
+ MPI t = NULL;
+ MPI x1 = NULL, y1 = NULL;
+
+ mpi_point_init(&sG);
+ mpi_point_init(&tP);
+ x1 = mpi_new(0);
+ y1 = mpi_new(0);
+ t = mpi_new(0);
+
+ /* r, s in [1, n-1] */
+ if (mpi_cmp_ui(sig_r, 1) < 0 || mpi_cmp(sig_r, ec->n) > 0 ||
+ mpi_cmp_ui(sig_s, 1) < 0 || mpi_cmp(sig_s, ec->n) > 0) {
+ goto leave;
+ }
+
+ /* t = (r + s) % n, t == 0 */
+ mpi_addm(t, sig_r, sig_s, ec->n);
+ if (mpi_cmp_ui(t, 0) == 0)
+ goto leave;
+
+ /* sG + tP = (x1, y1) */
+ rc = -EBADMSG;
+ mpi_ec_mul_point(&sG, sig_s, ec->G, ec);
+ mpi_ec_mul_point(&tP, t, ec->Q, ec);
+ mpi_ec_add_points(&sG, &sG, &tP, ec);
+ if (mpi_ec_get_affine(x1, y1, &sG, ec))
+ goto leave;
+
+ /* R = (e + x1) % n */
+ mpi_addm(t, hash, x1, ec->n);
+
+ /* check R == r */
+ rc = -EKEYREJECTED;
+ if (mpi_cmp(t, sig_r))
+ goto leave;
+
+ rc = 0;
+
+leave:
+ mpi_point_free_parts(&sG);
+ mpi_point_free_parts(&tP);
+ mpi_free(x1);
+ mpi_free(y1);
+ mpi_free(t);
+
+ return rc;
+}
+
+static int sm2_verify(struct akcipher_request *req)
+{
+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+ struct mpi_ec_ctx *ec = akcipher_tfm_ctx(tfm);
+ unsigned char *buffer;
+ struct sm2_signature_ctx sig;
+ MPI hash;
+ int ret;
+
+ if (unlikely(!ec->Q))
+ return -EINVAL;
+
+ buffer = kmalloc(req->src_len + req->dst_len, GFP_KERNEL);
+ if (!buffer)
+ return -ENOMEM;
+
+ sg_pcopy_to_buffer(req->src,
+ sg_nents_for_len(req->src, req->src_len + req->dst_len),
+ buffer, req->src_len + req->dst_len, 0);
+
+ sig.sig_r = NULL;
+ sig.sig_s = NULL;
+ ret = asn1_ber_decoder(&sm2signature_decoder, &sig,
+ buffer, req->src_len);
+ if (ret)
+ goto error;
+
+ ret = -ENOMEM;
+ hash = mpi_read_raw_data(buffer + req->src_len, req->dst_len);
+ if (!hash)
+ goto error;
+
+ ret = _sm2_verify(ec, hash, sig.sig_r, sig.sig_s);
+
+ mpi_free(hash);
+error:
+ mpi_free(sig.sig_r);
+ mpi_free(sig.sig_s);
+ kfree(buffer);
+ return ret;
+}
+
+static int sm2_set_pub_key(struct crypto_akcipher *tfm,
+ const void *key, unsigned int keylen)
+{
+ struct mpi_ec_ctx *ec = akcipher_tfm_ctx(tfm);
+ MPI a;
+ int rc;
+
+ rc = sm2_ec_ctx_reset(ec);
+ if (rc)
+ return rc;
+
+ ec->Q = mpi_point_new(0);
+ if (!ec->Q)
+ return -ENOMEM;
+
+ /* include the uncompressed flag '0x04' */
+ rc = -ENOMEM;
+ a = mpi_read_raw_data(key, keylen);
+ if (!a)
+ goto error;
+
+ mpi_normalize(a);
+ rc = sm2_ecc_os2ec(ec->Q, a);
+ mpi_free(a);
+ if (rc)
+ goto error;
+
+ return 0;
+
+error:
+ mpi_point_release(ec->Q);
+ ec->Q = NULL;
+ return rc;
+}
+
+static unsigned int sm2_max_size(struct crypto_akcipher *tfm)
+{
+ /* Unlimited max size */
+ return PAGE_SIZE;
+}
+
+static int sm2_init_tfm(struct crypto_akcipher *tfm)
+{
+ struct mpi_ec_ctx *ec = akcipher_tfm_ctx(tfm);
+
+ return sm2_ec_ctx_init(ec);
+}
+
+static void sm2_exit_tfm(struct crypto_akcipher *tfm)
+{
+ struct mpi_ec_ctx *ec = akcipher_tfm_ctx(tfm);
+
+ sm2_ec_ctx_deinit(ec);
+}
+
+static struct akcipher_alg sm2 = {
+ .verify = sm2_verify,
+ .set_pub_key = sm2_set_pub_key,
+ .max_size = sm2_max_size,
+ .init = sm2_init_tfm,
+ .exit = sm2_exit_tfm,
+ .base = {
+ .cra_name = "sm2",
+ .cra_driver_name = "sm2-generic",
+ .cra_priority = 100,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct mpi_ec_ctx),
+ },
+};
+
+static int sm2_init(void)
+{
+ return crypto_register_akcipher(&sm2);
+}
+
+static void sm2_exit(void)
+{
+ crypto_unregister_akcipher(&sm2);
+}
+
+subsys_initcall(sm2_init);
+module_exit(sm2_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
+MODULE_DESCRIPTION("SM2 generic algorithm");
+MODULE_ALIAS_CRYPTO("sm2-generic");
--- /dev/null
+Sm2Signature ::= SEQUENCE {
+ sig_r INTEGER ({ sm2_get_signature_r }),
+ sig_s INTEGER ({ sm2_get_signature_s })
+}
}
EXPORT_SYMBOL(crypto_sm3_update);
-static int sm3_final(struct shash_desc *desc, u8 *out)
+int crypto_sm3_final(struct shash_desc *desc, u8 *out)
{
sm3_base_do_finalize(desc, sm3_generic_block_fn);
return sm3_base_finish(desc, out);
}
+EXPORT_SYMBOL(crypto_sm3_final);
int crypto_sm3_finup(struct shash_desc *desc, const u8 *data,
unsigned int len, u8 *hash)
{
sm3_base_do_update(desc, data, len, sm3_generic_block_fn);
- return sm3_final(desc, hash);
+ return crypto_sm3_final(desc, hash);
}
EXPORT_SYMBOL(crypto_sm3_finup);
.digestsize = SM3_DIGEST_SIZE,
.init = sm3_base_init,
.update = crypto_sm3_update,
- .final = sm3_final,
+ .final = crypto_sm3_final,
.finup = crypto_sm3_finup,
.descsize = sizeof(struct sm3_state),
.base = {
static u32 mask;
static int mode;
static u32 num_mb = 8;
+static unsigned int klen;
static char *tvmem[TVMEMSIZE];
static const char *check[] = {
ret = do_one_aead_op(cur->req, ret);
if (ret) {
- pr_err("calculating auth failed failed (%d)\n",
+ pr_err("calculating auth failed (%d)\n",
ret);
break;
}
crypto_aead_encrypt(req));
if (ret) {
- pr_err("calculating auth failed failed (%d)\n",
+ pr_err("calculating auth failed (%d)\n",
ret);
break;
}
goto out;
}
- if (speed[i].klen)
- crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen);
+ if (klen)
+ crypto_ahash_setkey(tfm, tvmem[0], klen);
for (k = 0; k < num_mb; k++)
ahash_request_set_crypt(data[k].req, data[k].sg,
break;
}
- if (speed[i].klen)
- crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen);
+ if (klen)
+ crypto_ahash_setkey(tfm, tvmem[0], klen);
pr_info("test%3u "
"(%5u byte blocks,%5u bytes per update,%4u updates): ",
if (mode > 300 && mode < 400) break;
fallthrough;
case 318:
- test_hash_speed("ghash-generic", sec, hash_speed_template_16);
+ klen = 16;
+ test_hash_speed("ghash", sec, generic_hash_speed_template);
if (mode > 300 && mode < 400) break;
fallthrough;
case 319:
"(defaults to zero which uses CPU cycles instead)");
module_param(num_mb, uint, 0000);
MODULE_PARM_DESC(num_mb, "Number of concurrent requests to be used in mb speed tests (defaults to 8)");
+module_param(klen, uint, 0);
+MODULE_PARM_DESC(klen, "Key length (defaults to 0)");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Quick & dirty crypto testing module");
struct hash_speed {
unsigned int blen; /* buffer length */
unsigned int plen; /* per-update length */
- unsigned int klen; /* key length */
};
/*
{ .blen = 0, .plen = 0, }
};
-static struct hash_speed hash_speed_template_16[] = {
- { .blen = 16, .plen = 16, .klen = 16, },
- { .blen = 64, .plen = 16, .klen = 16, },
- { .blen = 64, .plen = 64, .klen = 16, },
- { .blen = 256, .plen = 16, .klen = 16, },
- { .blen = 256, .plen = 64, .klen = 16, },
- { .blen = 256, .plen = 256, .klen = 16, },
- { .blen = 1024, .plen = 16, .klen = 16, },
- { .blen = 1024, .plen = 256, .klen = 16, },
- { .blen = 1024, .plen = 1024, .klen = 16, },
- { .blen = 2048, .plen = 16, .klen = 16, },
- { .blen = 2048, .plen = 256, .klen = 16, },
- { .blen = 2048, .plen = 1024, .klen = 16, },
- { .blen = 2048, .plen = 2048, .klen = 16, },
- { .blen = 4096, .plen = 16, .klen = 16, },
- { .blen = 4096, .plen = 256, .klen = 16, },
- { .blen = 4096, .plen = 1024, .klen = 16, },
- { .blen = 4096, .plen = 4096, .klen = 16, },
- { .blen = 8192, .plen = 16, .klen = 16, },
- { .blen = 8192, .plen = 256, .klen = 16, },
- { .blen = 8192, .plen = 1024, .klen = 16, },
- { .blen = 8192, .plen = 4096, .klen = 16, },
- { .blen = 8192, .plen = 8192, .klen = 16, },
-
- /* End marker */
- { .blen = 0, .plen = 0, .klen = 0, }
-};
-
static struct hash_speed poly1305_speed_template[] = {
{ .blen = 96, .plen = 16, },
{ .blen = 96, .plen = 32, },
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/string.h>
+#include <linux/uio.h>
#include <crypto/rng.h>
#include <crypto/drbg.h>
#include <crypto/akcipher.h>
key = kmalloc(vecs->key_len + sizeof(u32) * 2 + vecs->param_len,
GFP_KERNEL);
if (!key)
- goto free_xbuf;
+ goto free_req;
memcpy(key, vecs->key, vecs->key_len);
ptr = key + vecs->key_len;
ptr = test_pack_u32(ptr, vecs->algo);
else
err = crypto_akcipher_set_priv_key(tfm, key, vecs->key_len);
if (err)
- goto free_req;
+ goto free_key;
/*
* First run test which do not require a private key, such as
out_len_max = crypto_akcipher_maxsize(tfm);
outbuf_enc = kzalloc(out_len_max, GFP_KERNEL);
if (!outbuf_enc)
- goto free_req;
+ goto free_key;
if (!vecs->siggen_sigver_test) {
m = vecs->m;
op = "verify";
}
+ err = -E2BIG;
if (WARN_ON(m_size > PAGE_SIZE))
goto free_all;
memcpy(xbuf[0], m, m_size);
pr_err("alg: akcipher: %s test failed. err %d\n", op, err);
goto free_all;
}
- if (!vecs->siggen_sigver_test) {
+ if (!vecs->siggen_sigver_test && c) {
if (req->dst_len != c_size) {
pr_err("alg: akcipher: %s test failed. Invalid output len\n",
op);
goto free_all;
}
+ if (!vecs->siggen_sigver_test && !c) {
+ c = outbuf_enc;
+ c_size = req->dst_len;
+ }
+
+ err = -E2BIG;
op = vecs->siggen_sigver_test ? "sign" : "decrypt";
if (WARN_ON(c_size > PAGE_SIZE))
goto free_all;
free_all:
kfree(outbuf_dec);
kfree(outbuf_enc);
+free_key:
+ kfree(key);
free_req:
akcipher_request_free(req);
- kfree(key);
free_xbuf:
testmgr_free_buf(xbuf);
return err;
.suite = {
.hash = __VECS(sha512_tv_template)
}
+ }, {
+ .alg = "sm2",
+ .test = alg_test_akcipher,
+ .suite = {
+ .akcipher = __VECS(sm2_tv_template)
+ }
}, {
.alg = "sm3",
.test = alg_test_hash,
},
};
+/*
+ * SM2 test vectors.
+ */
+static const struct akcipher_testvec sm2_tv_template[] = {
+ { /* Generated from openssl */
+ .key =
+ "\x04"
+ "\x8e\xa0\x33\x69\x91\x7e\x3d\xec\xad\x8e\xf0\x45\x5e\x13\x3e\x68"
+ "\x5b\x8c\xab\x5c\xc6\xc8\x50\xdf\x91\x00\xe0\x24\x73\x4d\x31\xf2"
+ "\x2e\xc0\xd5\x6b\xee\xda\x98\x93\xec\xd8\x36\xaa\xb9\xcf\x63\x82"
+ "\xef\xa7\x1a\x03\xed\x16\xba\x74\xb8\x8b\xf9\xe5\x70\x39\xa4\x70",
+ .key_len = 65,
+ .param_len = 0,
+ .c =
+ "\x30\x45"
+ "\x02\x20"
+ "\x70\xab\xb6\x7d\xd6\x54\x80\x64\x42\x7e\x2d\x05\x08\x36\xc9\x96"
+ "\x25\xc2\xbb\xff\x08\xe5\x43\x15\x5e\xf3\x06\xd9\x2b\x2f\x0a\x9f"
+ "\x02\x21"
+ "\x00"
+ "\xbf\x21\x5f\x7e\x5d\x3f\x1a\x4d\x8f\x84\xc2\xe9\xa6\x4c\xa4\x18"
+ "\xb2\xb8\x46\xf4\x32\x96\xfa\x57\xc6\x29\xd4\x89\xae\xcc\xda\xdb",
+ .c_size = 71,
+ .algo = OID_SM2_with_SM3,
+ .m =
+ "\x47\xa7\xbf\xd3\xda\xc4\x79\xee\xda\x8b\x4f\xe8\x40\x94\xd4\x32"
+ "\x8f\xf1\xcd\x68\x4d\xbd\x9b\x1d\xe0\xd8\x9a\x5d\xad\x85\x47\x5c",
+ .m_size = 32,
+ .public_key_vec = true,
+ .siggen_sigver_test = true,
+ },
+ { /* From libgcrypt */
+ .key =
+ "\x04"
+ "\x87\x59\x38\x9a\x34\xaa\xad\x07\xec\xf4\xe0\xc8\xc2\x65\x0a\x44"
+ "\x59\xc8\xd9\x26\xee\x23\x78\x32\x4e\x02\x61\xc5\x25\x38\xcb\x47"
+ "\x75\x28\x10\x6b\x1e\x0b\x7c\x8d\xd5\xff\x29\xa9\xc8\x6a\x89\x06"
+ "\x56\x56\xeb\x33\x15\x4b\xc0\x55\x60\x91\xef\x8a\xc9\xd1\x7d\x78",
+ .key_len = 65,
+ .param_len = 0,
+ .c =
+ "\x30\x44"
+ "\x02\x20"
+ "\xd9\xec\xef\xe8\x5f\xee\x3c\x59\x57\x8e\x5b\xab\xb3\x02\xe1\x42"
+ "\x4b\x67\x2c\x0b\x26\xb6\x51\x2c\x3e\xfc\xc6\x49\xec\xfe\x89\xe5"
+ "\x02\x20"
+ "\x43\x45\xd0\xa5\xff\xe5\x13\x27\x26\xd0\xec\x37\xad\x24\x1e\x9a"
+ "\x71\x9a\xa4\x89\xb0\x7e\x0f\xc4\xbb\x2d\x50\xd0\xe5\x7f\x7a\x68",
+ .c_size = 70,
+ .algo = OID_SM2_with_SM3,
+ .m =
+ "\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x00"
+ "\x12\x34\x56\x78\x9a\xbc\xde\xf0\x12\x34\x56\x78\x9a\xbc\xde\xf0",
+ .m_size = 32,
+ .public_key_vec = true,
+ .siggen_sigver_test = true,
+ },
+};
+
/* Example vectors below taken from
* http://www.oscca.gov.cn/UpFile/20101222141857786.pdf
*
/* Set of all registered templates. */
static struct xor_block_template *__initdata template_list;
-#define BENCH_SIZE (PAGE_SIZE)
+#ifndef MODULE
+static void __init do_xor_register(struct xor_block_template *tmpl)
+{
+ tmpl->next = template_list;
+ template_list = tmpl;
+}
+
+static int __init register_xor_blocks(void)
+{
+ active_template = XOR_SELECT_TEMPLATE(NULL);
+
+ if (!active_template) {
+#define xor_speed do_xor_register
+ // register all the templates and pick the first as the default
+ XOR_TRY_TEMPLATES;
+#undef xor_speed
+ active_template = template_list;
+ }
+ return 0;
+}
+#endif
+
+#define BENCH_SIZE 4096
+#define REPS 800U
static void __init
do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2)
{
int speed;
- unsigned long now, j;
- int i, count, max;
+ int i, j;
+ ktime_t min, start, diff;
tmpl->next = template_list;
template_list = tmpl;
preempt_disable();
- /*
- * Count the number of XORs done during a whole jiffy, and use
- * this to calculate the speed of checksumming. We use a 2-page
- * allocation to have guaranteed color L1-cache layout.
- */
- max = 0;
- for (i = 0; i < 5; i++) {
- j = jiffies;
- count = 0;
- while ((now = jiffies) == j)
- cpu_relax();
- while (time_before(jiffies, now + 1)) {
+ min = (ktime_t)S64_MAX;
+ for (i = 0; i < 3; i++) {
+ start = ktime_get();
+ for (j = 0; j < REPS; j++) {
mb(); /* prevent loop optimzation */
tmpl->do_2(BENCH_SIZE, b1, b2);
mb();
- count++;
- mb();
}
- if (count > max)
- max = count;
+ diff = ktime_sub(ktime_get(), start);
+ if (diff < min)
+ min = diff;
}
preempt_enable();
- speed = max * (HZ * BENCH_SIZE / 1024);
+ // bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s]
+ speed = (1000 * REPS * BENCH_SIZE) / (unsigned int)ktime_to_ns(min);
tmpl->speed = speed;
- printk(KERN_INFO " %-10s: %5d.%03d MB/sec\n", tmpl->name,
- speed / 1000, speed % 1000);
+ pr_info(" %-16s: %5d MB/sec\n", tmpl->name, speed);
}
static int __init
#define xor_speed(templ) do_xor_speed((templ), b1, b2)
printk(KERN_INFO "xor: measuring software checksum speed\n");
+ template_list = NULL;
XOR_TRY_TEMPLATES;
fastest = template_list;
for (f = fastest; f; f = f->next)
if (f->speed > fastest->speed)
fastest = f;
- printk(KERN_INFO "xor: using function: %s (%d.%03d MB/sec)\n",
- fastest->name, fastest->speed / 1000, fastest->speed % 1000);
+ pr_info("xor: using function: %s (%d MB/sec)\n",
+ fastest->name, fastest->speed);
#undef xor_speed
MODULE_LICENSE("GPL");
+#ifndef MODULE
/* when built-in xor.o must initialize before drivers/md/md.o */
-core_initcall(calibrate_xor_blocks);
+core_initcall(register_xor_blocks);
+#endif
+
+module_init(calibrate_xor_blocks);
module_exit(xor_exit);
return (fwspec && fwspec->ops) ? fwspec->ops : NULL;
}
-static inline int iort_add_device_replay(const struct iommu_ops *ops,
- struct device *dev)
+static inline int iort_add_device_replay(struct device *dev)
{
int err = 0;
*/
if (!err) {
ops = iort_fwspec_iommu_ops(dev);
- err = iort_add_device_replay(ops, dev);
+ err = iort_add_device_replay(dev);
}
/* Ignore all other errors apart from EPROBE_DEFER */
}
#else
-static inline const struct iommu_ops *iort_fwspec_iommu_ops(struct device *dev)
-{ return NULL; }
-static inline int iort_add_device_replay(const struct iommu_ops *ops,
- struct device *dev)
-{ return 0; }
int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head)
{ return 0; }
const struct iommu_ops *iort_iommu_configure_id(struct device *dev,
static bool lapic_timer_needs_broadcast(struct acpi_processor *pr,
struct acpi_processor_cx *cx)
{
+ return false;
}
#endif
{ PCI_VDEVICE(INTEL, 0x1f3f), board_ahci_avn }, /* Avoton RAID */
{ PCI_VDEVICE(INTEL, 0x2823), board_ahci }, /* Wellsburg RAID */
{ PCI_VDEVICE(INTEL, 0x2827), board_ahci }, /* Wellsburg RAID */
+ { PCI_VDEVICE(INTEL, 0x43d4), board_ahci }, /* Rocket Lake PCH-H RAID */
+ { PCI_VDEVICE(INTEL, 0x43d5), board_ahci }, /* Rocket Lake PCH-H RAID */
+ { PCI_VDEVICE(INTEL, 0x43d6), board_ahci }, /* Rocket Lake PCH-H RAID */
+ { PCI_VDEVICE(INTEL, 0x43d7), board_ahci }, /* Rocket Lake PCH-H RAID */
{ PCI_VDEVICE(INTEL, 0x8d02), board_ahci }, /* Wellsburg AHCI */
{ PCI_VDEVICE(INTEL, 0x8d04), board_ahci }, /* Wellsburg RAID */
{ PCI_VDEVICE(INTEL, 0x8d06), board_ahci }, /* Wellsburg RAID */
as default lpm_policy */
AHCI_HFLAG_SUSPEND_PHYS = (1 << 26), /* handle PHYs during
suspend/resume */
+ AHCI_HFLAG_IGN_NOTSUPP_POWER_ON = (1 << 27), /* ignore -EOPNOTSUPP
+ from phy_power_on() */
/* ap->flags bits */
static const struct ahci_mvebu_plat_data ahci_mvebu_armada_3700_plat_data = {
.plat_config = ahci_mvebu_armada_3700_config,
- .flags = AHCI_HFLAG_SUSPEND_PHYS,
+ .flags = AHCI_HFLAG_SUSPEND_PHYS | AHCI_HFLAG_IGN_NOTSUPP_POWER_ON,
};
static const struct of_device_id ahci_mvebu_of_match[] = {
* Tang Yuantian <Yuantian.Tang@freescale.com>
*/
+#include <linux/acpi.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pm.h>
};
MODULE_DEVICE_TABLE(of, ahci_qoriq_of_match);
+static const struct acpi_device_id ahci_qoriq_acpi_match[] = {
+ {"NXP0004", .driver_data = (kernel_ulong_t)AHCI_LX2160A},
+ { }
+};
+MODULE_DEVICE_TABLE(acpi, ahci_qoriq_acpi_match);
+
static int ahci_qoriq_hardreset(struct ata_link *link, unsigned int *class,
unsigned long deadline)
{
static int ahci_qoriq_probe(struct platform_device *pdev)
{
struct device_node *np = pdev->dev.of_node;
+ const struct acpi_device_id *acpi_id;
struct device *dev = &pdev->dev;
struct ahci_host_priv *hpriv;
struct ahci_qoriq_priv *qoriq_priv;
return PTR_ERR(hpriv);
of_id = of_match_node(ahci_qoriq_of_match, np);
- if (!of_id)
+ acpi_id = acpi_match_device(ahci_qoriq_acpi_match, &pdev->dev);
+ if (!(of_id || acpi_id))
return -ENODEV;
qoriq_priv = devm_kzalloc(dev, sizeof(*qoriq_priv), GFP_KERNEL);
if (!qoriq_priv)
return -ENOMEM;
- qoriq_priv->type = (enum ahci_qoriq_type)of_id->data;
+ if (of_id)
+ qoriq_priv->type = (enum ahci_qoriq_type)of_id->data;
+ else
+ qoriq_priv->type = (enum ahci_qoriq_type)acpi_id->driver_data;
if (unlikely(!ecc_initialized)) {
res = platform_get_resource_byname(pdev,
}
}
- qoriq_priv->is_dmacoherent = of_dma_is_coherent(np);
+ if (device_get_dma_attr(&pdev->dev) == DEV_DMA_COHERENT)
+ qoriq_priv->is_dmacoherent = true;
rc = ahci_platform_enable_resources(hpriv);
if (rc)
.driver = {
.name = DRV_NAME,
.of_match_table = ahci_qoriq_of_match,
+ .acpi_match_table = ahci_qoriq_acpi_match,
.pm = &ahci_qoriq_pm_ops,
},
};
}
rc = phy_power_on(hpriv->phys[i]);
- if (rc) {
+ if (rc && !(rc == -EOPNOTSUPP && (hpriv->flags & AHCI_HFLAG_IGN_NOTSUPP_POWER_ON))) {
phy_exit(hpriv->phys[i]);
goto disable_phys;
}
case 1:
ppi[0] = &cmd_info[4];
ppi[1] = &cmd_info[4];
- /* FALL THRU */
+ fallthrough;
/* Early revs have no CNTRL_CH0 */
case 2:
case 0:
struct ahci_host_priv *hpriv = host->private_data;
void __iomem *mmio = hpriv->mmio;
u32 ctl;
- int rc;
if (hpriv->flags & AHCI_HFLAG_NO_SUSPEND) {
dev_err(dev, "firmware update required for suspend/resume\n");
writel(ctl, mmio + HOST_CTL);
readl(mmio + HOST_CTL); /* flush */
- rc = ata_host_suspend(host, PMSG_SUSPEND);
- if (rc)
- return rc;
-
- return 0;
+ return ata_host_suspend(host, PMSG_SUSPEND);
}
static int ahci_highbank_resume(struct device *dev)
return pfn_to_nid(pfn);
}
+static int do_register_memory_block_under_node(int nid,
+ struct memory_block *mem_blk)
+{
+ int ret;
+
+ /*
+ * If this memory block spans multiple nodes, we only indicate
+ * the last processed node.
+ */
+ mem_blk->nid = nid;
+
+ ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj,
+ &mem_blk->dev.kobj,
+ kobject_name(&mem_blk->dev.kobj));
+ if (ret)
+ return ret;
+
+ return sysfs_create_link_nowarn(&mem_blk->dev.kobj,
+ &node_devices[nid]->dev.kobj,
+ kobject_name(&node_devices[nid]->dev.kobj));
+}
+
/* register memory section under specified node if it spans that node */
-static int register_mem_sect_under_node(struct memory_block *mem_blk,
- void *arg)
+static int register_mem_block_under_node_early(struct memory_block *mem_blk,
+ void *arg)
{
unsigned long memory_block_pfns = memory_block_size_bytes() / PAGE_SIZE;
unsigned long start_pfn = section_nr_to_pfn(mem_blk->start_section_nr);
unsigned long end_pfn = start_pfn + memory_block_pfns - 1;
- int ret, nid = *(int *)arg;
+ int nid = *(int *)arg;
unsigned long pfn;
for (pfn = start_pfn; pfn <= end_pfn; pfn++) {
}
/*
- * We need to check if page belongs to nid only for the boot
- * case, during hotplug we know that all pages in the memory
- * block belong to the same node.
- */
- if (system_state == SYSTEM_BOOTING) {
- page_nid = get_nid_for_pfn(pfn);
- if (page_nid < 0)
- continue;
- if (page_nid != nid)
- continue;
- }
-
- /*
- * If this memory block spans multiple nodes, we only indicate
- * the last processed node.
+ * We need to check if page belongs to nid only at the boot
+ * case because node's ranges can be interleaved.
*/
- mem_blk->nid = nid;
-
- ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj,
- &mem_blk->dev.kobj,
- kobject_name(&mem_blk->dev.kobj));
- if (ret)
- return ret;
+ page_nid = get_nid_for_pfn(pfn);
+ if (page_nid < 0)
+ continue;
+ if (page_nid != nid)
+ continue;
- return sysfs_create_link_nowarn(&mem_blk->dev.kobj,
- &node_devices[nid]->dev.kobj,
- kobject_name(&node_devices[nid]->dev.kobj));
+ return do_register_memory_block_under_node(nid, mem_blk);
}
/* mem section does not span the specified node */
return 0;
}
+/*
+ * During hotplug we know that all pages in the memory block belong to the same
+ * node.
+ */
+static int register_mem_block_under_node_hotplug(struct memory_block *mem_blk,
+ void *arg)
+{
+ int nid = *(int *)arg;
+
+ return do_register_memory_block_under_node(nid, mem_blk);
+}
+
/*
* Unregister a memory block device under the node it spans. Memory blocks
* with multiple nodes cannot be offlined and therefore also never be removed.
kobject_name(&node_devices[mem_blk->nid]->dev.kobj));
}
-int link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn)
+int link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn,
+ enum meminit_context context)
{
+ walk_memory_blocks_func_t func;
+
+ if (context == MEMINIT_HOTPLUG)
+ func = register_mem_block_under_node_hotplug;
+ else
+ func = register_mem_block_under_node_early;
+
return walk_memory_blocks(PFN_PHYS(start_pfn),
PFN_PHYS(end_pfn - start_pfn), (void *)&nid,
- register_mem_sect_under_node);
+ func);
}
#ifdef CONFIG_HUGETLBFS
# subsystems should select the appropriate symbols.
config REGMAP
- default y if (REGMAP_I2C || REGMAP_SPI || REGMAP_SPMI || REGMAP_W1 || REGMAP_AC97 || REGMAP_MMIO || REGMAP_IRQ || REGMAP_SOUNDWIRE || REGMAP_SCCB || REGMAP_I3C)
+ default y if (REGMAP_I2C || REGMAP_SPI || REGMAP_SPMI || REGMAP_W1 || REGMAP_AC97 || REGMAP_MMIO || REGMAP_IRQ || REGMAP_SOUNDWIRE || REGMAP_SCCB || REGMAP_I3C || REGMAP_SPI_AVMM)
select IRQ_DOMAIN if REGMAP_IRQ
bool
config REGMAP_I3C
tristate
depends on I3C
+
+config REGMAP_SPI_AVMM
+ tristate
+ depends on SPI
obj-$(CONFIG_REGMAP_SOUNDWIRE) += regmap-sdw.o
obj-$(CONFIG_REGMAP_SCCB) += regmap-sccb.o
obj-$(CONFIG_REGMAP_I3C) += regmap-i3c.o
+obj-$(CONFIG_REGMAP_SPI_AVMM) += regmap-spi-avmm.o
void *selector_work_buf; /* Scratch buffer used for selector */
struct hwspinlock *hwlock;
+
+ /* if set, the regmap core can sleep */
+ bool can_sleep;
};
struct regcache_ops {
#ifdef CONFIG_DEBUG_FS
extern void regmap_debugfs_initcall(void);
-extern void regmap_debugfs_init(struct regmap *map, const char *name);
+extern void regmap_debugfs_init(struct regmap *map);
extern void regmap_debugfs_exit(struct regmap *map);
static inline void regmap_debugfs_disable(struct regmap *map)
#else
static inline void regmap_debugfs_initcall(void) { }
-static inline void regmap_debugfs_init(struct regmap *map, const char *name) { }
+static inline void regmap_debugfs_init(struct regmap *map) { }
static inline void regmap_debugfs_exit(struct regmap *map) { }
static inline void regmap_debugfs_disable(struct regmap *map) { }
#endif
int regcache_lookup_reg(struct regmap *map, unsigned int reg);
int _regmap_raw_write(struct regmap *map, unsigned int reg,
- const void *val, size_t val_len);
+ const void *val, size_t val_len, bool noinc);
void regmap_async_complete_cb(struct regmap_async *async, int ret);
map->cache_bypass = true;
- ret = _regmap_raw_write(map, base, *data, count * val_bytes);
+ ret = _regmap_raw_write(map, base, *data, count * val_bytes, false);
if (ret)
dev_err(map->dev, "Unable to sync registers %#x-%#x. %d\n",
base, cur - map->reg_stride, ret);
struct regmap_debugfs_node {
struct regmap *map;
- const char *name;
struct list_head link;
};
{
/* Calculate the length of a fixed format */
if (!map->debugfs_tot_len) {
- map->debugfs_reg_len = regmap_calc_reg_len(map->max_register),
+ map->debugfs_reg_len = regmap_calc_reg_len(map->max_register);
map->debugfs_val_len = 2 * map->format.val_bytes;
map->debugfs_tot_len = map->debugfs_reg_len +
map->debugfs_val_len + 3; /* : \n */
.write = regmap_cache_bypass_write_file,
};
-void regmap_debugfs_init(struct regmap *map, const char *name)
+void regmap_debugfs_init(struct regmap *map)
{
struct rb_node *next;
struct regmap_range_node *range_node;
const char *devname = "dummy";
+ const char *name = map->name;
/*
* Userspace can initiate reads from the hardware over debugfs.
if (!node)
return;
node->map = map;
- node->name = name;
mutex_lock(®map_debugfs_early_lock);
list_add(&node->link, ®map_debugfs_early_list);
mutex_unlock(®map_debugfs_early_lock);
mutex_lock(®map_debugfs_early_lock);
list_for_each_entry_safe(node, tmp, ®map_debugfs_early_list, link) {
- regmap_debugfs_init(node->map, node->name);
+ regmap_debugfs_init(node->map);
list_del(&node->link);
kfree(node);
}
ret = regmap_write(map, reg, ~d->mask_buf[i]);
else
ret = regmap_write(map, reg, d->mask_buf[i]);
+ if (d->chip->clear_ack) {
+ if (d->chip->ack_invert && !ret)
+ ret = regmap_write(map, reg,
+ d->mask_buf[i]);
+ else if (!ret)
+ ret = regmap_write(map, reg,
+ ~d->mask_buf[i]);
+ }
if (ret != 0)
dev_err(d->map->dev, "Failed to ack 0x%x: %d\n",
reg, ret);
if (data->status_buf[i] && (chip->ack_base || chip->use_ack)) {
reg = chip->ack_base +
(i * map->reg_stride * data->irq_reg_stride);
- ret = regmap_write(map, reg, data->status_buf[i]);
+ if (chip->ack_invert)
+ ret = regmap_write(map, reg,
+ ~data->status_buf[i]);
+ else
+ ret = regmap_write(map, reg,
+ data->status_buf[i]);
+ if (chip->clear_ack) {
+ if (chip->ack_invert && !ret)
+ ret = regmap_write(map, reg,
+ data->status_buf[i]);
+ else if (!ret)
+ ret = regmap_write(map, reg,
+ ~data->status_buf[i]);
+ }
if (ret != 0)
dev_err(map->dev, "Failed to ack 0x%x: %d\n",
reg, ret);
else
ret = regmap_write(map, reg,
d->status_buf[i] & d->mask_buf[i]);
+ if (chip->clear_ack) {
+ if (chip->ack_invert && !ret)
+ ret = regmap_write(map, reg,
+ (d->status_buf[i] &
+ d->mask_buf[i]));
+ else if (!ret)
+ ret = regmap_write(map, reg,
+ ~(d->status_buf[i] &
+ d->mask_buf[i]));
+ }
if (ret != 0) {
dev_err(map->dev, "Failed to ack 0x%x: %d\n",
reg, ret);
// Copyright(c) 2015-17 Intel Corporation.
#include <linux/device.h>
-#include <linux/mod_devicetable.h>
#include <linux/module.h>
#include <linux/soundwire/sdw.h>
#include "internal.h"
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+//
+// Register map access API - SPI AVMM support
+//
+// Copyright (C) 2018-2020 Intel Corporation. All rights reserved.
+
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/spi/spi.h>
+
+/*
+ * This driver implements the regmap operations for a generic SPI
+ * master to access the registers of the spi slave chip which has an
+ * Avalone bus in it.
+ *
+ * The "SPI slave to Avalon Master Bridge" (spi-avmm) IP should be integrated
+ * in the spi slave chip. The IP acts as a bridge to convert encoded streams of
+ * bytes from the host to the internal register read/write on Avalon bus. In
+ * order to issue register access requests to the slave chip, the host should
+ * send formatted bytes that conform to the transfer protocol.
+ * The transfer protocol contains 3 layers: transaction layer, packet layer
+ * and physical layer.
+ *
+ * Reference Documents could be found at:
+ * https://www.intel.com/content/www/us/en/programmable/documentation/sfo1400787952932.html
+ *
+ * Chapter "SPI Slave/JTAG to Avalon Master Bridge Cores" is a general
+ * introduction to the protocol.
+ *
+ * Chapter "Avalon Packets to Transactions Converter Core" describes
+ * the transaction layer.
+ *
+ * Chapter "Avalon-ST Bytes to Packets and Packets to Bytes Converter Cores"
+ * describes the packet layer.
+ *
+ * Chapter "Avalon-ST Serial Peripheral Interface Core" describes the
+ * physical layer.
+ *
+ *
+ * When host issues a regmap read/write, the driver will transform the request
+ * to byte stream layer by layer. It formats the register addr, value and
+ * length to the transaction layer request, then converts the request to packet
+ * layer bytes stream and then to physical layer bytes stream. Finally the
+ * driver sends the formatted byte stream over SPI bus to the slave chip.
+ *
+ * The spi-avmm IP on the slave chip decodes the byte stream and initiates
+ * register read/write on its internal Avalon bus, and then encodes the
+ * response to byte stream and sends back to host.
+ *
+ * The driver receives the byte stream, reverses the 3 layers transformation,
+ * and finally gets the response value (read out data for register read,
+ * successful written size for register write).
+ */
+
+#define PKT_SOP 0x7a
+#define PKT_EOP 0x7b
+#define PKT_CHANNEL 0x7c
+#define PKT_ESC 0x7d
+
+#define PHY_IDLE 0x4a
+#define PHY_ESC 0x4d
+
+#define TRANS_CODE_WRITE 0x0
+#define TRANS_CODE_SEQ_WRITE 0x4
+#define TRANS_CODE_READ 0x10
+#define TRANS_CODE_SEQ_READ 0x14
+#define TRANS_CODE_NO_TRANS 0x7f
+
+#define SPI_AVMM_XFER_TIMEOUT (msecs_to_jiffies(200))
+
+/* slave's register addr is 32 bits */
+#define SPI_AVMM_REG_SIZE 4UL
+/* slave's register value is 32 bits */
+#define SPI_AVMM_VAL_SIZE 4UL
+
+/*
+ * max rx size could be larger. But considering the buffer consuming,
+ * it is proper that we limit 1KB xfer at max.
+ */
+#define MAX_READ_CNT 256UL
+#define MAX_WRITE_CNT 1UL
+
+struct trans_req_header {
+ u8 code;
+ u8 rsvd;
+ __be16 size;
+ __be32 addr;
+} __packed;
+
+struct trans_resp_header {
+ u8 r_code;
+ u8 rsvd;
+ __be16 size;
+} __packed;
+
+#define TRANS_REQ_HD_SIZE (sizeof(struct trans_req_header))
+#define TRANS_RESP_HD_SIZE (sizeof(struct trans_resp_header))
+
+/*
+ * In transaction layer,
+ * the write request format is: Transaction request header + data
+ * the read request format is: Transaction request header
+ * the write response format is: Transaction response header
+ * the read response format is: pure data, no Transaction response header
+ */
+#define TRANS_WR_TX_SIZE(n) (TRANS_REQ_HD_SIZE + SPI_AVMM_VAL_SIZE * (n))
+#define TRANS_RD_TX_SIZE TRANS_REQ_HD_SIZE
+#define TRANS_TX_MAX TRANS_WR_TX_SIZE(MAX_WRITE_CNT)
+
+#define TRANS_RD_RX_SIZE(n) (SPI_AVMM_VAL_SIZE * (n))
+#define TRANS_WR_RX_SIZE TRANS_RESP_HD_SIZE
+#define TRANS_RX_MAX TRANS_RD_RX_SIZE(MAX_READ_CNT)
+
+/* tx & rx share one transaction layer buffer */
+#define TRANS_BUF_SIZE ((TRANS_TX_MAX > TRANS_RX_MAX) ? \
+ TRANS_TX_MAX : TRANS_RX_MAX)
+
+/*
+ * In tx phase, the host prepares all the phy layer bytes of a request in the
+ * phy buffer and sends them in a batch.
+ *
+ * The packet layer and physical layer defines several special chars for
+ * various purpose, when a transaction layer byte hits one of these special
+ * chars, it should be escaped. The escape rule is, "Escape char first,
+ * following the byte XOR'ed with 0x20".
+ *
+ * This macro defines the max possible length of the phy data. In the worst
+ * case, all transaction layer bytes need to be escaped (so the data length
+ * doubles), plus 4 special chars (SOP, CHANNEL, CHANNEL_NUM, EOP). Finally
+ * we should make sure the length is aligned to SPI BPW.
+ */
+#define PHY_TX_MAX ALIGN(2 * TRANS_TX_MAX + 4, 4)
+
+/*
+ * Unlike tx, phy rx is affected by possible PHY_IDLE bytes from slave, the max
+ * length of the rx bit stream is unpredictable. So the driver reads the words
+ * one by one, and parses each word immediately into transaction layer buffer.
+ * Only one word length of phy buffer is used for rx.
+ */
+#define PHY_BUF_SIZE PHY_TX_MAX
+
+/**
+ * struct spi_avmm_bridge - SPI slave to AVMM bus master bridge
+ *
+ * @spi: spi slave associated with this bridge.
+ * @word_len: bytes of word for spi transfer.
+ * @trans_len: length of valid data in trans_buf.
+ * @phy_len: length of valid data in phy_buf.
+ * @trans_buf: the bridge buffer for transaction layer data.
+ * @phy_buf: the bridge buffer for physical layer data.
+ * @swap_words: the word swapping cb for phy data. NULL if not needed.
+ *
+ * As a device's registers are implemented on the AVMM bus address space, it
+ * requires the driver to issue formatted requests to spi slave to AVMM bus
+ * master bridge to perform register access.
+ */
+struct spi_avmm_bridge {
+ struct spi_device *spi;
+ unsigned char word_len;
+ unsigned int trans_len;
+ unsigned int phy_len;
+ /* bridge buffer used in translation between protocol layers */
+ char trans_buf[TRANS_BUF_SIZE];
+ char phy_buf[PHY_BUF_SIZE];
+ void (*swap_words)(char *buf, unsigned int len);
+};
+
+static void br_swap_words_32(char *buf, unsigned int len)
+{
+ u32 *p = (u32 *)buf;
+ unsigned int count;
+
+ count = len / 4;
+ while (count--) {
+ *p = swab32p(p);
+ p++;
+ }
+}
+
+/*
+ * Format transaction layer data in br->trans_buf according to the register
+ * access request, Store valid transaction layer data length in br->trans_len.
+ */
+static int br_trans_tx_prepare(struct spi_avmm_bridge *br, bool is_read, u32 reg,
+ u32 *wr_val, u32 count)
+{
+ struct trans_req_header *header;
+ unsigned int trans_len;
+ u8 code;
+ __le32 *data;
+ int i;
+
+ if (is_read) {
+ if (count == 1)
+ code = TRANS_CODE_READ;
+ else
+ code = TRANS_CODE_SEQ_READ;
+ } else {
+ if (count == 1)
+ code = TRANS_CODE_WRITE;
+ else
+ code = TRANS_CODE_SEQ_WRITE;
+ }
+
+ header = (struct trans_req_header *)br->trans_buf;
+ header->code = code;
+ header->rsvd = 0;
+ header->size = cpu_to_be16((u16)count * SPI_AVMM_VAL_SIZE);
+ header->addr = cpu_to_be32(reg);
+
+ trans_len = TRANS_REQ_HD_SIZE;
+
+ if (!is_read) {
+ trans_len += SPI_AVMM_VAL_SIZE * count;
+ if (trans_len > sizeof(br->trans_buf))
+ return -ENOMEM;
+
+ data = (__le32 *)(br->trans_buf + TRANS_REQ_HD_SIZE);
+
+ for (i = 0; i < count; i++)
+ *data++ = cpu_to_le32(*wr_val++);
+ }
+
+ /* Store valid trans data length for next layer */
+ br->trans_len = trans_len;
+
+ return 0;
+}
+
+/*
+ * Convert transaction layer data (in br->trans_buf) to phy layer data, store
+ * them in br->phy_buf. Pad the phy_buf aligned with SPI's BPW. Store valid phy
+ * layer data length in br->phy_len.
+ *
+ * phy_buf len should be aligned with SPI's BPW. Spare bytes should be padded
+ * with PHY_IDLE, then the slave will just drop them.
+ *
+ * The driver will not simply pad 4a at the tail. The concern is that driver
+ * will not store MISO data during tx phase, if the driver pads 4a at the tail,
+ * it is possible that if the slave is fast enough to response at the padding
+ * time. As a result these rx bytes are lost. In the following case, 7a,7c,00
+ * will lost.
+ * MOSI ...|7a|7c|00|10| |00|00|04|02| |4b|7d|5a|7b| |40|4a|4a|4a| |XX|XX|...
+ * MISO ...|4a|4a|4a|4a| |4a|4a|4a|4a| |4a|4a|4a|4a| |4a|7a|7c|00| |78|56|...
+ *
+ * So the driver moves EOP and bytes after EOP to the end of the aligned size,
+ * then fill the hole with PHY_IDLE. As following:
+ * before pad ...|7a|7c|00|10| |00|00|04|02| |4b|7d|5a|7b| |40|
+ * after pad ...|7a|7c|00|10| |00|00|04|02| |4b|7d|5a|4a| |4a|4a|7b|40|
+ * Then if the slave will not get the entire packet before the tx phase is
+ * over, it can't responsed to anything either.
+ */
+static int br_pkt_phy_tx_prepare(struct spi_avmm_bridge *br)
+{
+ char *tb, *tb_end, *pb, *pb_limit, *pb_eop = NULL;
+ unsigned int aligned_phy_len, move_size;
+ bool need_esc = false;
+
+ tb = br->trans_buf;
+ tb_end = tb + br->trans_len;
+ pb = br->phy_buf;
+ pb_limit = pb + ARRAY_SIZE(br->phy_buf);
+
+ *pb++ = PKT_SOP;
+
+ /*
+ * The driver doesn't support multiple channels so the channel number
+ * is always 0.
+ */
+ *pb++ = PKT_CHANNEL;
+ *pb++ = 0x0;
+
+ for (; pb < pb_limit && tb < tb_end; pb++) {
+ if (need_esc) {
+ *pb = *tb++ ^ 0x20;
+ need_esc = false;
+ continue;
+ }
+
+ /* EOP should be inserted before the last valid char */
+ if (tb == tb_end - 1 && !pb_eop) {
+ *pb = PKT_EOP;
+ pb_eop = pb;
+ continue;
+ }
+
+ /*
+ * insert an ESCAPE char if the data value equals any special
+ * char.
+ */
+ switch (*tb) {
+ case PKT_SOP:
+ case PKT_EOP:
+ case PKT_CHANNEL:
+ case PKT_ESC:
+ *pb = PKT_ESC;
+ need_esc = true;
+ break;
+ case PHY_IDLE:
+ case PHY_ESC:
+ *pb = PHY_ESC;
+ need_esc = true;
+ break;
+ default:
+ *pb = *tb++;
+ break;
+ }
+ }
+
+ /* The phy buffer is used out but transaction layer data remains */
+ if (tb < tb_end)
+ return -ENOMEM;
+
+ /* Store valid phy data length for spi transfer */
+ br->phy_len = pb - br->phy_buf;
+
+ if (br->word_len == 1)
+ return 0;
+
+ /* Do phy buf padding if word_len > 1 byte. */
+ aligned_phy_len = ALIGN(br->phy_len, br->word_len);
+ if (aligned_phy_len > sizeof(br->phy_buf))
+ return -ENOMEM;
+
+ if (aligned_phy_len == br->phy_len)
+ return 0;
+
+ /* move EOP and bytes after EOP to the end of aligned size */
+ move_size = pb - pb_eop;
+ memmove(&br->phy_buf[aligned_phy_len - move_size], pb_eop, move_size);
+
+ /* fill the hole with PHY_IDLEs */
+ memset(pb_eop, PHY_IDLE, aligned_phy_len - br->phy_len);
+
+ /* update the phy data length */
+ br->phy_len = aligned_phy_len;
+
+ return 0;
+}
+
+/*
+ * In tx phase, the slave only returns PHY_IDLE (0x4a). So the driver will
+ * ignore rx in tx phase.
+ */
+static int br_do_tx(struct spi_avmm_bridge *br)
+{
+ /* reorder words for spi transfer */
+ if (br->swap_words)
+ br->swap_words(br->phy_buf, br->phy_len);
+
+ /* send all data in phy_buf */
+ return spi_write(br->spi, br->phy_buf, br->phy_len);
+}
+
+/*
+ * This function read the rx byte stream from SPI word by word and convert
+ * them to transaction layer data in br->trans_buf. It also stores the length
+ * of rx transaction layer data in br->trans_len
+ *
+ * The slave may send an unknown number of PHY_IDLEs in rx phase, so we cannot
+ * prepare a fixed length buffer to receive all of the rx data in a batch. We
+ * have to read word by word and convert them to transaction layer data at
+ * once.
+ */
+static int br_do_rx_and_pkt_phy_parse(struct spi_avmm_bridge *br)
+{
+ bool eop_found = false, channel_found = false, esc_found = false;
+ bool valid_word = false, last_try = false;
+ struct device *dev = &br->spi->dev;
+ char *pb, *tb_limit, *tb = NULL;
+ unsigned long poll_timeout;
+ int ret, i;
+
+ tb_limit = br->trans_buf + ARRAY_SIZE(br->trans_buf);
+ pb = br->phy_buf;
+ poll_timeout = jiffies + SPI_AVMM_XFER_TIMEOUT;
+ while (tb < tb_limit) {
+ ret = spi_read(br->spi, pb, br->word_len);
+ if (ret)
+ return ret;
+
+ /* reorder the word back */
+ if (br->swap_words)
+ br->swap_words(pb, br->word_len);
+
+ valid_word = false;
+ for (i = 0; i < br->word_len; i++) {
+ /* drop everything before first SOP */
+ if (!tb && pb[i] != PKT_SOP)
+ continue;
+
+ /* drop PHY_IDLE */
+ if (pb[i] == PHY_IDLE)
+ continue;
+
+ valid_word = true;
+
+ /*
+ * We don't support multiple channels, so error out if
+ * a non-zero channel number is found.
+ */
+ if (channel_found) {
+ if (pb[i] != 0) {
+ dev_err(dev, "%s channel num != 0\n",
+ __func__);
+ return -EFAULT;
+ }
+
+ channel_found = false;
+ continue;
+ }
+
+ switch (pb[i]) {
+ case PKT_SOP:
+ /*
+ * reset the parsing if a second SOP appears.
+ */
+ tb = br->trans_buf;
+ eop_found = false;
+ channel_found = false;
+ esc_found = false;
+ break;
+ case PKT_EOP:
+ /*
+ * No special char is expected after ESC char.
+ * No special char (except ESC & PHY_IDLE) is
+ * expected after EOP char.
+ *
+ * The special chars are all dropped.
+ */
+ if (esc_found || eop_found)
+ return -EFAULT;
+
+ eop_found = true;
+ break;
+ case PKT_CHANNEL:
+ if (esc_found || eop_found)
+ return -EFAULT;
+
+ channel_found = true;
+ break;
+ case PKT_ESC:
+ case PHY_ESC:
+ if (esc_found)
+ return -EFAULT;
+
+ esc_found = true;
+ break;
+ default:
+ /* Record the normal byte in trans_buf. */
+ if (esc_found) {
+ *tb++ = pb[i] ^ 0x20;
+ esc_found = false;
+ } else {
+ *tb++ = pb[i];
+ }
+
+ /*
+ * We get the last normal byte after EOP, it is
+ * time we finish. Normally the function should
+ * return here.
+ */
+ if (eop_found) {
+ br->trans_len = tb - br->trans_buf;
+ return 0;
+ }
+ }
+ }
+
+ if (valid_word) {
+ /* update poll timeout when we get valid word */
+ poll_timeout = jiffies + SPI_AVMM_XFER_TIMEOUT;
+ last_try = false;
+ } else {
+ /*
+ * We timeout when rx keeps invalid for some time. But
+ * it is possible we are scheduled out for long time
+ * after a spi_read. So when we are scheduled in, a SW
+ * timeout happens. But actually HW may have worked fine and
+ * has been ready long time ago. So we need to do an extra
+ * read, if we get a valid word then we could continue rx,
+ * otherwise real a HW issue happens.
+ */
+ if (last_try)
+ return -ETIMEDOUT;
+
+ if (time_after(jiffies, poll_timeout))
+ last_try = true;
+ }
+ }
+
+ /*
+ * We have used out all transfer layer buffer but cannot find the end
+ * of the byte stream.
+ */
+ dev_err(dev, "%s transfer buffer is full but rx doesn't end\n",
+ __func__);
+
+ return -EFAULT;
+}
+
+/*
+ * For read transactions, the avmm bus will directly return register values
+ * without transaction response header.
+ */
+static int br_rd_trans_rx_parse(struct spi_avmm_bridge *br,
+ u32 *val, unsigned int expected_count)
+{
+ unsigned int i, trans_len = br->trans_len;
+ __le32 *data;
+
+ if (expected_count * SPI_AVMM_VAL_SIZE != trans_len)
+ return -EFAULT;
+
+ data = (__le32 *)br->trans_buf;
+ for (i = 0; i < expected_count; i++)
+ *val++ = le32_to_cpu(*data++);
+
+ return 0;
+}
+
+/*
+ * For write transactions, the slave will return a transaction response
+ * header.
+ */
+static int br_wr_trans_rx_parse(struct spi_avmm_bridge *br,
+ unsigned int expected_count)
+{
+ unsigned int trans_len = br->trans_len;
+ struct trans_resp_header *resp;
+ u8 code;
+ u16 val_len;
+
+ if (trans_len != TRANS_RESP_HD_SIZE)
+ return -EFAULT;
+
+ resp = (struct trans_resp_header *)br->trans_buf;
+
+ code = resp->r_code ^ 0x80;
+ val_len = be16_to_cpu(resp->size);
+ if (!val_len || val_len != expected_count * SPI_AVMM_VAL_SIZE)
+ return -EFAULT;
+
+ /* error out if the trans code doesn't align with the val size */
+ if ((val_len == SPI_AVMM_VAL_SIZE && code != TRANS_CODE_WRITE) ||
+ (val_len > SPI_AVMM_VAL_SIZE && code != TRANS_CODE_SEQ_WRITE))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int do_reg_access(void *context, bool is_read, unsigned int reg,
+ unsigned int *value, unsigned int count)
+{
+ struct spi_avmm_bridge *br = context;
+ int ret;
+
+ /* invalidate bridge buffers first */
+ br->trans_len = 0;
+ br->phy_len = 0;
+
+ ret = br_trans_tx_prepare(br, is_read, reg, value, count);
+ if (ret)
+ return ret;
+
+ ret = br_pkt_phy_tx_prepare(br);
+ if (ret)
+ return ret;
+
+ ret = br_do_tx(br);
+ if (ret)
+ return ret;
+
+ ret = br_do_rx_and_pkt_phy_parse(br);
+ if (ret)
+ return ret;
+
+ if (is_read)
+ return br_rd_trans_rx_parse(br, value, count);
+ else
+ return br_wr_trans_rx_parse(br, count);
+}
+
+static int regmap_spi_avmm_gather_write(void *context,
+ const void *reg_buf, size_t reg_len,
+ const void *val_buf, size_t val_len)
+{
+ if (reg_len != SPI_AVMM_REG_SIZE)
+ return -EINVAL;
+
+ if (!IS_ALIGNED(val_len, SPI_AVMM_VAL_SIZE))
+ return -EINVAL;
+
+ return do_reg_access(context, false, *(u32 *)reg_buf, (u32 *)val_buf,
+ val_len / SPI_AVMM_VAL_SIZE);
+}
+
+static int regmap_spi_avmm_write(void *context, const void *data, size_t bytes)
+{
+ if (bytes < SPI_AVMM_REG_SIZE + SPI_AVMM_VAL_SIZE)
+ return -EINVAL;
+
+ return regmap_spi_avmm_gather_write(context, data, SPI_AVMM_REG_SIZE,
+ data + SPI_AVMM_REG_SIZE,
+ bytes - SPI_AVMM_REG_SIZE);
+}
+
+static int regmap_spi_avmm_read(void *context,
+ const void *reg_buf, size_t reg_len,
+ void *val_buf, size_t val_len)
+{
+ if (reg_len != SPI_AVMM_REG_SIZE)
+ return -EINVAL;
+
+ if (!IS_ALIGNED(val_len, SPI_AVMM_VAL_SIZE))
+ return -EINVAL;
+
+ return do_reg_access(context, true, *(u32 *)reg_buf, val_buf,
+ (val_len / SPI_AVMM_VAL_SIZE));
+}
+
+static struct spi_avmm_bridge *
+spi_avmm_bridge_ctx_gen(struct spi_device *spi)
+{
+ struct spi_avmm_bridge *br;
+
+ if (!spi)
+ return ERR_PTR(-ENODEV);
+
+ /* Only support BPW == 8 or 32 now. Try 32 BPW first. */
+ spi->mode = SPI_MODE_1;
+ spi->bits_per_word = 32;
+ if (spi_setup(spi)) {
+ spi->bits_per_word = 8;
+ if (spi_setup(spi))
+ return ERR_PTR(-EINVAL);
+ }
+
+ br = kzalloc(sizeof(*br), GFP_KERNEL);
+ if (!br)
+ return ERR_PTR(-ENOMEM);
+
+ br->spi = spi;
+ br->word_len = spi->bits_per_word / 8;
+ if (br->word_len == 4) {
+ /*
+ * The protocol requires little endian byte order but MSB
+ * first. So driver needs to swap the byte order word by word
+ * if word length > 1.
+ */
+ br->swap_words = br_swap_words_32;
+ }
+
+ return br;
+}
+
+static void spi_avmm_bridge_ctx_free(void *context)
+{
+ kfree(context);
+}
+
+static const struct regmap_bus regmap_spi_avmm_bus = {
+ .write = regmap_spi_avmm_write,
+ .gather_write = regmap_spi_avmm_gather_write,
+ .read = regmap_spi_avmm_read,
+ .reg_format_endian_default = REGMAP_ENDIAN_NATIVE,
+ .val_format_endian_default = REGMAP_ENDIAN_NATIVE,
+ .max_raw_read = SPI_AVMM_VAL_SIZE * MAX_READ_CNT,
+ .max_raw_write = SPI_AVMM_VAL_SIZE * MAX_WRITE_CNT,
+ .free_context = spi_avmm_bridge_ctx_free,
+};
+
+struct regmap *__regmap_init_spi_avmm(struct spi_device *spi,
+ const struct regmap_config *config,
+ struct lock_class_key *lock_key,
+ const char *lock_name)
+{
+ struct spi_avmm_bridge *bridge;
+ struct regmap *map;
+
+ bridge = spi_avmm_bridge_ctx_gen(spi);
+ if (IS_ERR(bridge))
+ return ERR_CAST(bridge);
+
+ map = __regmap_init(&spi->dev, ®map_spi_avmm_bus,
+ bridge, config, lock_key, lock_name);
+ if (IS_ERR(map)) {
+ spi_avmm_bridge_ctx_free(bridge);
+ return ERR_CAST(map);
+ }
+
+ return map;
+}
+EXPORT_SYMBOL_GPL(__regmap_init_spi_avmm);
+
+struct regmap *__devm_regmap_init_spi_avmm(struct spi_device *spi,
+ const struct regmap_config *config,
+ struct lock_class_key *lock_key,
+ const char *lock_name)
+{
+ struct spi_avmm_bridge *bridge;
+ struct regmap *map;
+
+ bridge = spi_avmm_bridge_ctx_gen(spi);
+ if (IS_ERR(bridge))
+ return ERR_CAST(bridge);
+
+ map = __devm_regmap_init(&spi->dev, ®map_spi_avmm_bus,
+ bridge, config, lock_key, lock_name);
+ if (IS_ERR(map)) {
+ spi_avmm_bridge_ctx_free(bridge);
+ return ERR_CAST(map);
+ }
+
+ return map;
+}
+EXPORT_SYMBOL_GPL(__devm_regmap_init_spi_avmm);
+
+MODULE_LICENSE("GPL v2");
return true;
}
+static void regmap_format_12_20_write(struct regmap *map,
+ unsigned int reg, unsigned int val)
+{
+ u8 *out = map->work_buf;
+
+ out[0] = reg >> 4;
+ out[1] = (reg << 4) | (val >> 16);
+ out[2] = val >> 8;
+ out[3] = val;
+}
+
+
static void regmap_format_2_6_write(struct regmap *map,
unsigned int reg, unsigned int val)
{
kfree(map->selector_work_buf);
}
+static int regmap_set_name(struct regmap *map, const struct regmap_config *config)
+{
+ if (config->name) {
+ const char *name = kstrdup_const(config->name, GFP_KERNEL);
+
+ if (!name)
+ return -ENOMEM;
+
+ kfree_const(map->name);
+ map->name = name;
+ }
+
+ return 0;
+}
+
int regmap_attach_dev(struct device *dev, struct regmap *map,
const struct regmap_config *config)
{
struct regmap **m;
+ int ret;
map->dev = dev;
- regmap_debugfs_init(map, config->name);
+ ret = regmap_set_name(map, config);
+ if (ret)
+ return ret;
+
+ regmap_debugfs_init(map);
/* Add a devres resource for dev_get_regmap() */
m = devres_alloc(dev_get_regmap_release, sizeof(*m), GFP_KERNEL);
goto err;
}
- if (config->name) {
- map->name = kstrdup_const(config->name, GFP_KERNEL);
- if (!map->name) {
- ret = -ENOMEM;
- goto err_map;
- }
- }
+ ret = regmap_set_name(map, config);
+ if (ret)
+ goto err_map;
+
+ ret = -EINVAL; /* Later error paths rely on this */
if (config->disable_locking) {
map->lock = map->unlock = regmap_lock_unlock_none;
+ map->can_sleep = config->can_sleep;
regmap_debugfs_disable(map);
} else if (config->lock && config->unlock) {
map->lock = config->lock;
map->unlock = config->unlock;
map->lock_arg = config->lock_arg;
+ map->can_sleep = config->can_sleep;
} else if (config->use_hwlock) {
map->hwlock = hwspin_lock_request_specific(config->hwlock_id);
if (!map->hwlock) {
mutex_init(&map->mutex);
map->lock = regmap_lock_mutex;
map->unlock = regmap_unlock_mutex;
+ map->can_sleep = true;
lockdep_set_class_and_name(&map->mutex,
lock_key, lock_name);
}
}
break;
+ case 12:
+ switch (config->val_bits) {
+ case 20:
+ map->format.format_write = regmap_format_12_20_write;
+ break;
+ default:
+ goto err_hwlock;
+ }
+ break;
+
case 8:
map->format.format_reg = regmap_format_8;
break;
if (ret != 0)
goto err_regcache;
} else {
- regmap_debugfs_init(map, config->name);
+ regmap_debugfs_init(map);
}
return map;
}
EXPORT_SYMBOL_GPL(devm_regmap_field_alloc);
+
+/**
+ * regmap_field_bulk_alloc() - Allocate and initialise a bulk register field.
+ *
+ * @regmap: regmap bank in which this register field is located.
+ * @rm_field: regmap register fields within the bank.
+ * @reg_field: Register fields within the bank.
+ * @num_fields: Number of register fields.
+ *
+ * The return value will be an -ENOMEM on error or zero for success.
+ * Newly allocated regmap_fields should be freed by calling
+ * regmap_field_bulk_free()
+ */
+int regmap_field_bulk_alloc(struct regmap *regmap,
+ struct regmap_field **rm_field,
+ struct reg_field *reg_field,
+ int num_fields)
+{
+ struct regmap_field *rf;
+ int i;
+
+ rf = kcalloc(num_fields, sizeof(*rf), GFP_KERNEL);
+ if (!rf)
+ return -ENOMEM;
+
+ for (i = 0; i < num_fields; i++) {
+ regmap_field_init(&rf[i], regmap, reg_field[i]);
+ rm_field[i] = &rf[i];
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(regmap_field_bulk_alloc);
+
+/**
+ * devm_regmap_field_bulk_alloc() - Allocate and initialise a bulk register
+ * fields.
+ *
+ * @dev: Device that will be interacted with
+ * @regmap: regmap bank in which this register field is located.
+ * @rm_field: regmap register fields within the bank.
+ * @reg_field: Register fields within the bank.
+ * @num_fields: Number of register fields.
+ *
+ * The return value will be an -ENOMEM on error or zero for success.
+ * Newly allocated regmap_fields will be automatically freed by the
+ * device management code.
+ */
+int devm_regmap_field_bulk_alloc(struct device *dev,
+ struct regmap *regmap,
+ struct regmap_field **rm_field,
+ struct reg_field *reg_field,
+ int num_fields)
+{
+ struct regmap_field *rf;
+ int i;
+
+ rf = devm_kcalloc(dev, num_fields, sizeof(*rf), GFP_KERNEL);
+ if (!rf)
+ return -ENOMEM;
+
+ for (i = 0; i < num_fields; i++) {
+ regmap_field_init(&rf[i], regmap, reg_field[i]);
+ rm_field[i] = &rf[i];
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(devm_regmap_field_bulk_alloc);
+
+/**
+ * regmap_field_bulk_free() - Free register field allocated using
+ * regmap_field_bulk_alloc.
+ *
+ * @field: regmap fields which should be freed.
+ */
+void regmap_field_bulk_free(struct regmap_field *field)
+{
+ kfree(field);
+}
+EXPORT_SYMBOL_GPL(regmap_field_bulk_free);
+
+/**
+ * devm_regmap_field_bulk_free() - Free a bulk register field allocated using
+ * devm_regmap_field_bulk_alloc.
+ *
+ * @dev: Device that will be interacted with
+ * @field: regmap field which should be freed.
+ *
+ * Free register field allocated using devm_regmap_field_bulk_alloc(). Usually
+ * drivers need not call this function, as the memory allocated via devm
+ * will be freed as per device-driver life-cycle.
+ */
+void devm_regmap_field_bulk_free(struct device *dev,
+ struct regmap_field *field)
+{
+ devm_kfree(dev, field);
+}
+EXPORT_SYMBOL_GPL(devm_regmap_field_bulk_free);
+
/**
* devm_regmap_field_free() - Free a register field allocated using
* devm_regmap_field_alloc.
*/
int regmap_reinit_cache(struct regmap *map, const struct regmap_config *config)
{
+ int ret;
+
regcache_exit(map);
regmap_debugfs_exit(map);
map->readable_noinc_reg = config->readable_noinc_reg;
map->cache_type = config->cache_type;
- regmap_debugfs_init(map, config->name);
+ ret = regmap_set_name(map, config);
+ if (ret)
+ return ret;
+
+ regmap_debugfs_init(map);
map->cache_bypass = false;
map->cache_only = false;
}
if (map->hwlock)
hwspin_lock_free(map->hwlock);
+ if (map->lock == regmap_lock_mutex)
+ mutex_destroy(&map->mutex);
kfree_const(map->name);
kfree(map->patch);
kfree(map);
}
static int _regmap_raw_write_impl(struct regmap *map, unsigned int reg,
- const void *val, size_t val_len)
+ const void *val, size_t val_len, bool noinc)
{
struct regmap_range_node *range;
unsigned long flags;
win_residue, val_len / map->format.val_bytes);
ret = _regmap_raw_write_impl(map, reg, val,
win_residue *
- map->format.val_bytes);
+ map->format.val_bytes, noinc);
if (ret != 0)
return ret;
win_residue = range->window_len - win_offset;
}
- ret = _regmap_select_page(map, ®, range, val_num);
+ ret = _regmap_select_page(map, ®, range, noinc ? 1 : val_num);
if (ret != 0)
return ret;
}
map->work_buf +
map->format.reg_bytes +
map->format.pad_bytes,
- map->format.val_bytes);
+ map->format.val_bytes,
+ false);
}
static inline void *_regmap_map_get_context(struct regmap *map)
EXPORT_SYMBOL_GPL(regmap_write_async);
int _regmap_raw_write(struct regmap *map, unsigned int reg,
- const void *val, size_t val_len)
+ const void *val, size_t val_len, bool noinc)
{
size_t val_bytes = map->format.val_bytes;
size_t val_count = val_len / val_bytes;
/* Write as many bytes as possible with chunk_size */
for (i = 0; i < chunk_count; i++) {
- ret = _regmap_raw_write_impl(map, reg, val, chunk_bytes);
+ ret = _regmap_raw_write_impl(map, reg, val, chunk_bytes, noinc);
if (ret)
return ret;
/* Write remaining bytes */
if (val_len)
- ret = _regmap_raw_write_impl(map, reg, val, val_len);
+ ret = _regmap_raw_write_impl(map, reg, val, val_len, noinc);
return ret;
}
map->lock(map->lock_arg);
- ret = _regmap_raw_write(map, reg, val, val_len);
+ ret = _regmap_raw_write(map, reg, val, val_len, false);
map->unlock(map->lock_arg);
write_len = map->max_raw_write;
else
write_len = val_len;
- ret = _regmap_raw_write(map, reg, val, write_len);
+ ret = _regmap_raw_write(map, reg, val, write_len, true);
if (ret)
goto out_unlock;
val = ((u8 *)val) + write_len;
if (ret != 0)
return ret;
- if (regs[i].delay_us)
- udelay(regs[i].delay_us);
+ if (regs[i].delay_us) {
+ if (map->can_sleep)
+ fsleep(regs[i].delay_us);
+ else
+ udelay(regs[i].delay_us);
+ }
base += n;
n = 0;
if (ret != 0)
return ret;
- if (regs[i].delay_us)
- udelay(regs[i].delay_us);
+ if (regs[i].delay_us) {
+ if (map->can_sleep)
+ fsleep(regs[i].delay_us);
+ else
+ udelay(regs[i].delay_us);
+ }
}
return 0;
}
map->async = true;
- ret = _regmap_raw_write(map, reg, val, val_len);
+ ret = _regmap_raw_write(map, reg, val, val_len, false);
map->async = false;
EXPORT_SYMBOL_GPL(regmap_raw_write_async);
static int _regmap_raw_read(struct regmap *map, unsigned int reg, void *val,
- unsigned int val_len)
+ unsigned int val_len, bool noinc)
{
struct regmap_range_node *range;
int ret;
range = _regmap_range_lookup(map, reg);
if (range) {
ret = _regmap_select_page(map, ®, range,
- val_len / map->format.val_bytes);
+ noinc ? 1 : val_len / map->format.val_bytes);
if (ret != 0)
return ret;
}
if (!map->format.parse_val)
return -EINVAL;
- ret = _regmap_raw_read(map, reg, work_val, map->format.val_bytes);
+ ret = _regmap_raw_read(map, reg, work_val, map->format.val_bytes, false);
if (ret == 0)
*val = map->format.parse_val(work_val);
/* Read bytes that fit into whole chunks */
for (i = 0; i < chunk_count; i++) {
- ret = _regmap_raw_read(map, reg, val, chunk_bytes);
+ ret = _regmap_raw_read(map, reg, val, chunk_bytes, false);
if (ret != 0)
goto out;
/* Read remaining bytes */
if (val_len) {
- ret = _regmap_raw_read(map, reg, val, val_len);
+ ret = _regmap_raw_read(map, reg, val, val_len, false);
if (ret != 0)
goto out;
}
read_len = map->max_raw_read;
else
read_len = val_len;
- ret = _regmap_raw_read(map, reg, val, read_len);
+ ret = _regmap_raw_read(map, reg, val, read_len, true);
if (ret)
goto out_unlock;
val = ((u8 *)val) + read_len;
if (!get_ldev(device))
return 0; /* no disk, no metadata, no bitmap to manipulate bits in */
- nr_sectors = drbd_get_capacity(device->this_bdev);
+ nr_sectors = get_capacity(device->vdisk);
esector = sector + (size >> 9) - 1;
if (!expect(sector < nr_sectors))
sector_t p_size; /* partner's disk size */
struct request_queue *rq_queue;
- struct block_device *this_bdev;
struct gendisk *vdisk;
unsigned long last_reattach_jif;
p->d_size = cpu_to_be64(d_size);
p->u_size = cpu_to_be64(u_size);
- p->c_size = cpu_to_be64(trigger_reply ? 0 : drbd_get_capacity(device->this_bdev));
+ if (trigger_reply)
+ p->c_size = 0;
+ else
+ p->c_size = cpu_to_be64(get_capacity(device->vdisk));
p->max_bio_size = cpu_to_be32(max_bio_size);
p->queue_order_type = cpu_to_be16(q_order_type);
p->dds_flags = cpu_to_be16(flags);
* put_page(); and would cause either a VM_BUG directly, or
* __page_cache_release a page that would actually still be referenced
* by someone, leading to some obscure delayed Oops somewhere else. */
- if (drbd_disable_sendpage || (page_count(page) < 1) || PageSlab(page))
+ if (drbd_disable_sendpage || !sendpage_ok(page))
return _drbd_no_send_page(peer_device, page, offset, size, msg_flags);
msg_flags |= MSG_NOSIGNAL;
device->local_max_bio_size = DRBD_MAX_BIO_SIZE_SAFE;
}
-static void _drbd_set_my_capacity(struct drbd_device *device, sector_t size)
-{
- /* set_capacity(device->this_bdev->bd_disk, size); */
- set_capacity(device->vdisk, size);
- device->this_bdev->bd_inode->i_size = (loff_t)size << 9;
-}
-
void drbd_set_my_capacity(struct drbd_device *device, sector_t size)
{
char ppb[10];
- _drbd_set_my_capacity(device, size);
+
+ set_capacity(device->vdisk, size);
+ revalidate_disk_size(device->vdisk, false);
+
drbd_info(device, "size = %s (%llu KB)\n",
ppsize(ppb, size>>1), (unsigned long long)size>>1);
}
}
D_ASSERT(device, first_peer_device(device)->connection->net_conf == NULL);
- _drbd_set_my_capacity(device, 0);
+ set_capacity(device->vdisk, 0);
+ revalidate_disk_size(device->vdisk, false);
if (device->bitmap) {
/* maybe never allocated. */
drbd_bm_resize(device, 0, 1);
/* cleanup stuff that may have been allocated during
* device (re-)configuration or state changes */
- if (device->this_bdev)
- bdput(device->this_bdev);
-
drbd_backing_dev_free(device, device->ldev);
device->ldev = NULL;
sprintf(disk->disk_name, "drbd%d", minor);
disk->private_data = device;
- device->this_bdev = bdget(MKDEV(DRBD_MAJOR, minor));
- /* we have no partitions. we contain only ourselves. */
- device->this_bdev->bd_contains = device->this_bdev;
-
blk_queue_write_cache(q, true, true);
/* Setting the max_hw_sectors to an odd value of 8kibyte here
This triggers a max_bio_size message upon first attach or connect */
memset(buffer, 0, sizeof(*buffer));
- buffer->la_size_sect = cpu_to_be64(drbd_get_capacity(device->this_bdev));
+ buffer->la_size_sect = cpu_to_be64(get_capacity(device->vdisk));
for (i = UI_CURRENT; i < UI_SIZE; i++)
buffer->uuid[i] = cpu_to_be64(device->ldev->md.uuid[i]);
buffer->flags = cpu_to_be32(device->ldev->md.flags);
/* Update device->ldev->md.la_size_sect,
* since we updated it on metadata. */
- device->ldev->md.la_size_sect = drbd_get_capacity(device->this_bdev);
+ device->ldev->md.la_size_sect = get_capacity(device->vdisk);
drbd_md_put_buffer(device);
out:
goto err_out;
}
- if (drbd_get_capacity(device->this_bdev) != size ||
+ if (get_capacity(device->vdisk) != size ||
drbd_bm_capacity(device) != size) {
int err;
err = drbd_bm_resize(device, size, !(flags & DDSF_NO_RESYNC));
/* Make sure the new disk is big enough
* (we may currently be R_PRIMARY with no local disk...) */
- if (drbd_get_max_capacity(nbc) <
- drbd_get_capacity(device->this_bdev)) {
+ if (drbd_get_max_capacity(nbc) < get_capacity(device->vdisk)) {
retcode = ERR_DISK_TOO_SMALL;
goto fail;
}
s->dev_disk_flags = md->flags;
put_ldev(device);
}
- s->dev_size = drbd_get_capacity(device->this_bdev);
+ s->dev_size = get_capacity(device->vdisk);
s->dev_read = device->read_cnt;
s->dev_write = device->writ_cnt;
s->dev_al_writes = device->al_writ_cnt;
if (nla_put_u32(skb, T_sib_reason, sib ? sib->sib_reason : SIB_GET_STATUS_REPLY) ||
nla_put_u32(skb, T_current_state, device->state.i) ||
nla_put_u64_0pad(skb, T_ed_uuid, device->ed_uuid) ||
- nla_put_u64_0pad(skb, T_capacity,
- drbd_get_capacity(device->this_bdev)) ||
+ nla_put_u64_0pad(skb, T_capacity, get_capacity(device->vdisk)) ||
nla_put_u64_0pad(skb, T_send_cnt, device->send_cnt) ||
nla_put_u64_0pad(skb, T_recv_cnt, device->recv_cnt) ||
nla_put_u64_0pad(skb, T_read_cnt, device->read_cnt) ||
struct packet_info *pi) __must_hold(local)
{
struct drbd_device *device = peer_device->device;
- const sector_t capacity = drbd_get_capacity(device->this_bdev);
+ const sector_t capacity = get_capacity(device->vdisk);
struct drbd_peer_request *peer_req;
struct page *page;
int digest_size, err;
bool drbd_rs_c_min_rate_throttle(struct drbd_device *device)
{
- struct gendisk *disk = device->ldev->backing_bdev->bd_contains->bd_disk;
+ struct gendisk *disk = device->ldev->backing_bdev->bd_disk;
unsigned long db, dt, dbdt;
unsigned int c_min_rate;
int curr_events;
if (!peer_device)
return -EIO;
device = peer_device->device;
- capacity = drbd_get_capacity(device->this_bdev);
+ capacity = get_capacity(device->vdisk);
sector = be64_to_cpu(p->sector);
size = be32_to_cpu(p->blksize);
if (!peer_device)
return config_unknown_volume(connection, pi);
device = peer_device->device;
- cur_size = drbd_get_capacity(device->this_bdev);
+ cur_size = get_capacity(device->vdisk);
p_size = be64_to_cpu(p->d_size);
p_usize = be64_to_cpu(p->u_size);
}
if (device->state.conn > C_WF_REPORT_PARAMS) {
- if (be64_to_cpu(p->c_size) !=
- drbd_get_capacity(device->this_bdev) || ldsc) {
+ if (be64_to_cpu(p->c_size) != get_capacity(device->vdisk) ||
+ ldsc) {
/* we have different sizes, probably peer
* needs to know my new size... */
drbd_send_sizes(peer_device, 0, ddsf);
if (device->state.disk != D_INCONSISTENT)
return false;
esector = sector + (size >> 9) - 1;
- nr_sectors = drbd_get_capacity(device->this_bdev);
+ nr_sectors = get_capacity(device->vdisk);
D_ASSERT(device, sector < nr_sectors);
D_ASSERT(device, esector < nr_sectors);
struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL;
unsigned long bit;
sector_t sector;
- const sector_t capacity = drbd_get_capacity(device->this_bdev);
+ const sector_t capacity = get_capacity(device->vdisk);
int max_bio_size;
int number, rollback_i, size;
int align, requeue = 0;
{
int number, i, size;
sector_t sector;
- const sector_t capacity = drbd_get_capacity(device->this_bdev);
+ const sector_t capacity = get_capacity(device->vdisk);
bool stop_sector_reached = false;
if (unlikely(cancel))
void drbd_rs_controller_reset(struct drbd_device *device)
{
- struct gendisk *disk = device->ldev->backing_bdev->bd_contains->bd_disk;
+ struct gendisk *disk = device->ldev->backing_bdev->bd_disk;
struct fifo_buffer *plan;
atomic_set(&device->rs_sect_in, 0);
If unsure, say Y.
+config HW_RANDOM_INGENIC_TRNG
+ tristate "Ingenic True Random Number Generator support"
+ depends on HW_RANDOM
+ depends on MACH_X1830
+ default HW_RANDOM
+ help
+ This driver provides kernel-side support for the True Random Number Generator
+ hardware found in ingenic X1830 SoC. YSH & ATIL CU1830-Neo uses X1830 SoC.
+
+ To compile this driver as a module, choose M here: the
+ module will be called ingenic-trng.
+
+ If unsure, say Y.
+
config HW_RANDOM_NOMADIK
tristate "ST-Ericsson Nomadik Random Number Generator support"
depends on ARCH_NOMADIK
will be called cctrng.
If unsure, say 'N'.
+config HW_RANDOM_XIPHERA
+ tristate "Xiphera FPGA based True Random Number Generator support"
+ depends on HAS_IOMEM
+ help
+ This driver provides kernel-side support for Xiphera True Random
+ Number Generator Intellectual Property Core.
+
+ To compile this driver as a module, choose M here: the
+ module will be called xiphera-trng.
+
endif # HW_RANDOM
config UML_RANDOM
obj-$(CONFIG_HW_RANDOM_MXC_RNGA) += mxc-rnga.o
obj-$(CONFIG_HW_RANDOM_IMX_RNGC) += imx-rngc.o
obj-$(CONFIG_HW_RANDOM_INGENIC_RNG) += ingenic-rng.o
+obj-$(CONFIG_HW_RANDOM_INGENIC_TRNG) += ingenic-trng.o
obj-$(CONFIG_HW_RANDOM_OCTEON) += octeon-rng.o
obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o
obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o
obj-$(CONFIG_HW_RANDOM_OPTEE) += optee-rng.o
obj-$(CONFIG_HW_RANDOM_NPCM) += npcm-rng.o
obj-$(CONFIG_HW_RANDOM_CCTRNG) += cctrng.o
+obj-$(CONFIG_HW_RANDOM_XIPHERA) += xiphera-trng.o
int rc = 0;
clk = devm_clk_get_optional(dev, NULL);
- if (IS_ERR(clk)) {
- if (PTR_ERR(clk) != -EPROBE_DEFER)
- dev_err(dev, "Error getting clock: %pe\n", clk);
- return PTR_ERR(clk);
- }
+ if (IS_ERR(clk))
+ return dev_err_probe(dev, PTR_ERR(clk),
+ "Error getting clock\n");
+
drvdata->clk = clk;
rc = clk_prepare_enable(drvdata->clk);
rngc->rng.init = imx_rngc_init;
rngc->rng.read = imx_rngc_read;
rngc->rng.cleanup = imx_rngc_cleanup;
+ rngc->rng.quality = 19;
rngc->dev = &pdev->dev;
platform_set_drvdata(pdev, rngc);
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Ingenic True Random Number Generator driver
+ * Copyright (c) 2019 漆鹏振 (Qi Pengzhen) <aric.pzqi@ingenic.com>
+ * Copyright (c) 2020 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/hw_random.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+/* DTRNG register offsets */
+#define TRNG_REG_CFG_OFFSET 0x00
+#define TRNG_REG_RANDOMNUM_OFFSET 0x04
+#define TRNG_REG_STATUS_OFFSET 0x08
+
+/* bits within the CFG register */
+#define CFG_RDY_CLR BIT(12)
+#define CFG_INT_MASK BIT(11)
+#define CFG_GEN_EN BIT(0)
+
+/* bits within the STATUS register */
+#define STATUS_RANDOM_RDY BIT(0)
+
+struct ingenic_trng {
+ void __iomem *base;
+ struct clk *clk;
+ struct hwrng rng;
+};
+
+static int ingenic_trng_init(struct hwrng *rng)
+{
+ struct ingenic_trng *trng = container_of(rng, struct ingenic_trng, rng);
+ unsigned int ctrl;
+
+ ctrl = readl(trng->base + TRNG_REG_CFG_OFFSET);
+ ctrl |= CFG_GEN_EN;
+ writel(ctrl, trng->base + TRNG_REG_CFG_OFFSET);
+
+ return 0;
+}
+
+static void ingenic_trng_cleanup(struct hwrng *rng)
+{
+ struct ingenic_trng *trng = container_of(rng, struct ingenic_trng, rng);
+ unsigned int ctrl;
+
+ ctrl = readl(trng->base + TRNG_REG_CFG_OFFSET);
+ ctrl &= ~CFG_GEN_EN;
+ writel(ctrl, trng->base + TRNG_REG_CFG_OFFSET);
+}
+
+static int ingenic_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait)
+{
+ struct ingenic_trng *trng = container_of(rng, struct ingenic_trng, rng);
+ u32 *data = buf;
+ u32 status;
+ int ret;
+
+ ret = readl_poll_timeout(trng->base + TRNG_REG_STATUS_OFFSET, status,
+ status & STATUS_RANDOM_RDY, 10, 1000);
+ if (ret == -ETIMEDOUT) {
+ pr_err("%s: Wait for DTRNG data ready timeout\n", __func__);
+ return ret;
+ }
+
+ *data = readl(trng->base + TRNG_REG_RANDOMNUM_OFFSET);
+
+ return 4;
+}
+
+static int ingenic_trng_probe(struct platform_device *pdev)
+{
+ struct ingenic_trng *trng;
+ int ret;
+
+ trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL);
+ if (!trng)
+ return -ENOMEM;
+
+ trng->base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(trng->base)) {
+ pr_err("%s: Failed to map DTRNG registers\n", __func__);
+ ret = PTR_ERR(trng->base);
+ return PTR_ERR(trng->base);
+ }
+
+ trng->clk = devm_clk_get(&pdev->dev, NULL);
+ if (IS_ERR(trng->clk)) {
+ ret = PTR_ERR(trng->clk);
+ pr_crit("%s: Cannot get DTRNG clock\n", __func__);
+ return PTR_ERR(trng->clk);
+ }
+
+ ret = clk_prepare_enable(trng->clk);
+ if (ret) {
+ pr_crit("%s: Unable to enable DTRNG clock\n", __func__);
+ return ret;
+ }
+
+ trng->rng.name = pdev->name;
+ trng->rng.init = ingenic_trng_init;
+ trng->rng.cleanup = ingenic_trng_cleanup;
+ trng->rng.read = ingenic_trng_read;
+
+ ret = hwrng_register(&trng->rng);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to register hwrng\n");
+ return ret;
+ }
+
+ platform_set_drvdata(pdev, trng);
+
+ dev_info(&pdev->dev, "Ingenic DTRNG driver registered\n");
+ return 0;
+}
+
+static int ingenic_trng_remove(struct platform_device *pdev)
+{
+ struct ingenic_trng *trng = platform_get_drvdata(pdev);
+ unsigned int ctrl;
+
+ hwrng_unregister(&trng->rng);
+
+ ctrl = readl(trng->base + TRNG_REG_CFG_OFFSET);
+ ctrl &= ~CFG_GEN_EN;
+ writel(ctrl, trng->base + TRNG_REG_CFG_OFFSET);
+
+ clk_disable_unprepare(trng->clk);
+
+ return 0;
+}
+
+static const struct of_device_id ingenic_trng_of_match[] = {
+ { .compatible = "ingenic,x1830-dtrng" },
+ { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, ingenic_trng_of_match);
+
+static struct platform_driver ingenic_trng_driver = {
+ .probe = ingenic_trng_probe,
+ .remove = ingenic_trng_remove,
+ .driver = {
+ .name = "ingenic-trng",
+ .of_match_table = ingenic_trng_of_match,
+ },
+};
+
+module_platform_driver(ingenic_trng_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("漆鹏振 (Qi Pengzhen) <aric.pzqi@ingenic.com>");
+MODULE_AUTHOR("周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>");
+MODULE_DESCRIPTION("Ingenic True Random Number Generator driver");
int err = -ENODEV;
int i;
struct pci_dev *dev = NULL;
- void __iomem *mem = mem;
+ void __iomem *mem;
u8 hw_status;
struct intel_rng_hw *intel_rng_hw;
return PTR_ERR(priv->base);
}
- priv->rng.name = "iproc-rng200",
- priv->rng.read = iproc_rng200_read,
- priv->rng.init = iproc_rng200_init,
- priv->rng.cleanup = iproc_rng200_cleanup,
+ priv->rng.name = "iproc-rng200";
+ priv->rng.read = iproc_rng200_read;
+ priv->rng.init = iproc_rng200_init;
+ priv->rng.cleanup = iproc_rng200_cleanup;
/* Register driver */
ret = devm_hwrng_register(dev, &priv->rng);
mxc_rng->dev = &pdev->dev;
mxc_rng->rng.name = "mxc-rnga";
mxc_rng->rng.init = mxc_rnga_init;
- mxc_rng->rng.cleanup = mxc_rnga_cleanup,
- mxc_rng->rng.data_present = mxc_rnga_data_present,
- mxc_rng->rng.data_read = mxc_rnga_data_read,
+ mxc_rng->rng.cleanup = mxc_rnga_cleanup;
+ mxc_rng->rng.data_present = mxc_rnga_data_present;
+ mxc_rng->rng.data_read = mxc_rnga_data_read;
mxc_rng->clk = devm_clk_get(&pdev->dev, NULL);
if (IS_ERR(mxc_rng->clk)) {
pm_runtime_get_sync((struct device *)priv->rng.priv);
- while (max >= sizeof(u32)) {
+ while (max) {
if (wait) {
- if (readl_poll_timeout(priv->base + NPCM_RNGCS_REG,
+ if (readb_poll_timeout(priv->base + NPCM_RNGCS_REG,
ready,
ready & NPCM_RNG_DATA_VALID,
NPCM_RNG_POLL_USEC,
NPCM_RNG_TIMEOUT_USEC))
break;
} else {
- if ((readl(priv->base + NPCM_RNGCS_REG) &
+ if ((readb(priv->base + NPCM_RNGCS_REG) &
NPCM_RNG_DATA_VALID) == 0)
break;
}
- *(u32 *)buf = readl(priv->base + NPCM_RNGD_REG);
- retval += sizeof(u32);
- buf += sizeof(u32);
- max -= sizeof(u32);
+ *(u8 *)buf = readb(priv->base + NPCM_RNGD_REG);
+ retval++;
+ buf++;
+ max--;
}
pm_runtime_mark_last_busy((struct device *)priv->rng.priv);
if (max > MAX_ENTROPY_REQ_SZ)
max = MAX_ENTROPY_REQ_SZ;
- while (read == 0) {
+ while (read < max) {
rng_size = get_optee_rng_data(pvt_data, data, (max - read));
data += rng_size;
read += rng_size;
- if (wait) {
- if (timeout-- == 0)
+ if (wait && pvt_data->data_rate) {
+ if ((timeout-- == 0) || (read == max))
return read;
msleep((1000 * (max - read)) / pvt_data->data_rate);
} else {
dev_set_drvdata(dev, priv);
- priv->rng.name = dev_driver_string(dev),
+ priv->rng.name = dev_driver_string(dev);
#ifndef CONFIG_PM
- priv->rng.init = stm32_rng_init,
- priv->rng.cleanup = stm32_rng_cleanup,
+ priv->rng.init = stm32_rng_init;
+ priv->rng.cleanup = stm32_rng_cleanup;
#endif
- priv->rng.read = stm32_rng_read,
+ priv->rng.read = stm32_rng_read;
priv->rng.priv = (unsigned long) dev;
priv->rng.quality = 900;
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2020 Xiphera Ltd. */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/hw_random.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/delay.h>
+
+#define CONTROL_REG 0x00000000
+#define STATUS_REG 0x00000004
+#define RAND_REG 0x00000000
+
+#define HOST_TO_TRNG_RESET 0x00000001
+#define HOST_TO_TRNG_RELEASE_RESET 0x00000002
+#define HOST_TO_TRNG_ENABLE 0x80000000
+#define HOST_TO_TRNG_ZEROIZE 0x80000004
+#define HOST_TO_TRNG_ACK_ZEROIZE 0x80000008
+#define HOST_TO_TRNG_READ 0x8000000F
+
+/* trng statuses */
+#define TRNG_ACK_RESET 0x000000AC
+#define TRNG_SUCCESSFUL_STARTUP 0x00000057
+#define TRNG_FAILED_STARTUP 0x000000FA
+#define TRNG_NEW_RAND_AVAILABLE 0x000000ED
+
+struct xiphera_trng {
+ void __iomem *mem;
+ struct hwrng rng;
+};
+
+static int xiphera_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait)
+{
+ struct xiphera_trng *trng = container_of(rng, struct xiphera_trng, rng);
+ int ret = 0;
+
+ while (max >= sizeof(u32)) {
+ /* check for data */
+ if (readl(trng->mem + STATUS_REG) == TRNG_NEW_RAND_AVAILABLE) {
+ *(u32 *)buf = readl(trng->mem + RAND_REG);
+ /*
+ * Inform the trng of the read
+ * and re-enable it to produce a new random number
+ */
+ writel(HOST_TO_TRNG_READ, trng->mem + CONTROL_REG);
+ writel(HOST_TO_TRNG_ENABLE, trng->mem + CONTROL_REG);
+ ret += sizeof(u32);
+ buf += sizeof(u32);
+ max -= sizeof(u32);
+ } else {
+ break;
+ }
+ }
+ return ret;
+}
+
+static int xiphera_trng_probe(struct platform_device *pdev)
+{
+ int ret;
+ struct xiphera_trng *trng;
+ struct device *dev = &pdev->dev;
+ struct resource *res;
+
+ trng = devm_kzalloc(dev, sizeof(*trng), GFP_KERNEL);
+ if (!trng)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ trng->mem = devm_ioremap_resource(dev, res);
+ if (IS_ERR(trng->mem))
+ return PTR_ERR(trng->mem);
+
+ /*
+ * the trng needs to be reset first which might not happen in time,
+ * hence we incorporate a small delay to ensure proper behaviour
+ */
+ writel(HOST_TO_TRNG_RESET, trng->mem + CONTROL_REG);
+ usleep_range(100, 200);
+
+ if (readl(trng->mem + STATUS_REG) != TRNG_ACK_RESET) {
+ /*
+ * there is a small chance the trng is just not ready yet,
+ * so we try one more time. If the second time fails, we give up
+ */
+ usleep_range(100, 200);
+ if (readl(trng->mem + STATUS_REG) != TRNG_ACK_RESET) {
+ dev_err(dev, "failed to reset the trng ip\n");
+ return -ENODEV;
+ }
+ }
+
+ /*
+ * once again, to ensure proper behaviour we sleep
+ * for a while after zeroizing the trng
+ */
+ writel(HOST_TO_TRNG_RELEASE_RESET, trng->mem + CONTROL_REG);
+ writel(HOST_TO_TRNG_ENABLE, trng->mem + CONTROL_REG);
+ writel(HOST_TO_TRNG_ZEROIZE, trng->mem + CONTROL_REG);
+ msleep(20);
+
+ if (readl(trng->mem + STATUS_REG) != TRNG_SUCCESSFUL_STARTUP) {
+ /* diagnose the reason for the failure */
+ if (readl(trng->mem + STATUS_REG) == TRNG_FAILED_STARTUP) {
+ dev_err(dev, "trng ip startup-tests failed\n");
+ return -ENODEV;
+ }
+ dev_err(dev, "startup-tests yielded no response\n");
+ return -ENODEV;
+ }
+
+ writel(HOST_TO_TRNG_ACK_ZEROIZE, trng->mem + CONTROL_REG);
+
+ trng->rng.name = pdev->name;
+ trng->rng.read = xiphera_trng_read;
+ trng->rng.quality = 900;
+
+ ret = devm_hwrng_register(dev, &trng->rng);
+ if (ret) {
+ dev_err(dev, "failed to register rng device: %d\n", ret);
+ return ret;
+ }
+
+ platform_set_drvdata(pdev, trng);
+
+ return 0;
+}
+
+static const struct of_device_id xiphera_trng_of_match[] = {
+ { .compatible = "xiphera,xip8001b-trng", },
+ {},
+};
+MODULE_DEVICE_TABLE(of, xiphera_trng_of_match);
+
+static struct platform_driver xiphera_trng_driver = {
+ .driver = {
+ .name = "xiphera-trng",
+ .of_match_table = xiphera_trng_of_match,
+ },
+ .probe = xiphera_trng_probe,
+};
+
+module_platform_driver(xiphera_trng_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Atte Tommiska");
+MODULE_DESCRIPTION("Xiphera FPGA-based true random number generator driver");
If you have a H1 secure module running Cr50 firmware on SPI bus,
say Yes and it will be accessible from within Linux.
+config TCG_TIS_SYNQUACER
+ tristate "TPM Interface Specification 1.2 Interface / TPM 2.0 FIFO Interface (MMIO - SynQuacer)"
+ depends on ARCH_SYNQUACER
+ select TCG_TIS_CORE
+ help
+ If you have a TPM security chip that is compliant with the
+ TCG TIS 1.2 TPM specification (TPM1.2) or the TCG PTP FIFO
+ specification (TPM2.0) say Yes and it will be accessible from
+ within Linux on Socionext SynQuacer platform.
+ To compile this driver as a module, choose M here;
+ the module will be called tpm_tis_synquacer.
+
config TCG_TIS_I2C_ATMEL
tristate "TPM Interface Specification 1.2 Interface (I2C - Atmel)"
depends on I2C
tpm-$(CONFIG_OF) += eventlog/of.o
obj-$(CONFIG_TCG_TIS_CORE) += tpm_tis_core.o
obj-$(CONFIG_TCG_TIS) += tpm_tis.o
+obj-$(CONFIG_TCG_TIS_SYNQUACER) += tpm_tis_synquacer.o
obj-$(CONFIG_TCG_TIS_SPI) += tpm_tis_spi.o
tpm_tis_spi-y := tpm_tis_spi_main.o
out = (struct tpm_readpubek_out *)&tpm_buf.data[10];
str +=
sprintf(str,
- "Algorithm: %02X %02X %02X %02X\n"
- "Encscheme: %02X %02X\n"
- "Sigscheme: %02X %02X\n"
- "Parameters: %02X %02X %02X %02X "
- "%02X %02X %02X %02X "
- "%02X %02X %02X %02X\n"
+ "Algorithm: %4ph\n"
+ "Encscheme: %2ph\n"
+ "Sigscheme: %2ph\n"
+ "Parameters: %12ph\n"
"Modulus length: %d\n"
"Modulus:\n",
- out->algorithm[0], out->algorithm[1], out->algorithm[2],
- out->algorithm[3],
- out->encscheme[0], out->encscheme[1],
- out->sigscheme[0], out->sigscheme[1],
- out->parameters[0], out->parameters[1],
- out->parameters[2], out->parameters[3],
- out->parameters[4], out->parameters[5],
- out->parameters[6], out->parameters[7],
- out->parameters[8], out->parameters[9],
- out->parameters[10], out->parameters[11],
+ out->algorithm,
+ out->encscheme,
+ out->sigscheme,
+ out->parameters,
be32_to_cpu(out->keysize));
- for (i = 0; i < 256; i++) {
- str += sprintf(str, "%02X ", out->modulus[i]);
- if ((i + 1) % 16 == 0)
- str += sprintf(str, "\n");
- }
+ for (i = 0; i < 256; i += 16)
+ str += sprintf(str, "%16ph\n", &out->modulus[i]);
out_buf:
tpm_buf_destroy(&tpm_buf);
if (rc < 0)
return 0;
+ if (unlikely((status & TPM_STS_READ_ZERO) != 0)) {
+ /*
+ * If this trips, the chances are the read is
+ * returning 0xff because the locality hasn't been
+ * acquired. Usually because tpm_try_get_ops() hasn't
+ * been called before doing a TPM operation.
+ */
+ WARN_ONCE(1, "TPM returned invalid status\n");
+ return 0;
+ }
+
return status;
}
TPM_STS_GO = 0x20,
TPM_STS_DATA_AVAIL = 0x10,
TPM_STS_DATA_EXPECT = 0x08,
+ TPM_STS_READ_ZERO = 0x23, /* bits that must be zero on read */
};
enum tis_int_flags {
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Linaro Ltd.
+ *
+ * This device driver implements MMIO TPM on SynQuacer Platform.
+ */
+#include <linux/acpi.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/kernel.h>
+#include "tpm.h"
+#include "tpm_tis_core.h"
+
+/*
+ * irq > 0 means: use irq $irq;
+ * irq = 0 means: autoprobe for an irq;
+ * irq = -1 means: no irq support
+ */
+struct tpm_tis_synquacer_info {
+ struct resource res;
+ int irq;
+};
+
+struct tpm_tis_synquacer_phy {
+ struct tpm_tis_data priv;
+ void __iomem *iobase;
+};
+
+static inline struct tpm_tis_synquacer_phy *to_tpm_tis_tcg_phy(struct tpm_tis_data *data)
+{
+ return container_of(data, struct tpm_tis_synquacer_phy, priv);
+}
+
+static int tpm_tis_synquacer_read_bytes(struct tpm_tis_data *data, u32 addr,
+ u16 len, u8 *result)
+{
+ struct tpm_tis_synquacer_phy *phy = to_tpm_tis_tcg_phy(data);
+
+ while (len--)
+ *result++ = ioread8(phy->iobase + addr);
+
+ return 0;
+}
+
+static int tpm_tis_synquacer_write_bytes(struct tpm_tis_data *data, u32 addr,
+ u16 len, const u8 *value)
+{
+ struct tpm_tis_synquacer_phy *phy = to_tpm_tis_tcg_phy(data);
+
+ while (len--)
+ iowrite8(*value++, phy->iobase + addr);
+
+ return 0;
+}
+
+static int tpm_tis_synquacer_read16_bw(struct tpm_tis_data *data,
+ u32 addr, u16 *result)
+{
+ struct tpm_tis_synquacer_phy *phy = to_tpm_tis_tcg_phy(data);
+
+ /*
+ * Due to the limitation of SPI controller on SynQuacer,
+ * 16/32 bits access must be done in byte-wise and descending order.
+ */
+ *result = (ioread8(phy->iobase + addr + 1) << 8) |
+ (ioread8(phy->iobase + addr));
+
+ return 0;
+}
+
+static int tpm_tis_synquacer_read32_bw(struct tpm_tis_data *data,
+ u32 addr, u32 *result)
+{
+ struct tpm_tis_synquacer_phy *phy = to_tpm_tis_tcg_phy(data);
+
+ /*
+ * Due to the limitation of SPI controller on SynQuacer,
+ * 16/32 bits access must be done in byte-wise and descending order.
+ */
+ *result = (ioread8(phy->iobase + addr + 3) << 24) |
+ (ioread8(phy->iobase + addr + 2) << 16) |
+ (ioread8(phy->iobase + addr + 1) << 8) |
+ (ioread8(phy->iobase + addr));
+
+ return 0;
+}
+
+static int tpm_tis_synquacer_write32_bw(struct tpm_tis_data *data,
+ u32 addr, u32 value)
+{
+ struct tpm_tis_synquacer_phy *phy = to_tpm_tis_tcg_phy(data);
+
+ /*
+ * Due to the limitation of SPI controller on SynQuacer,
+ * 16/32 bits access must be done in byte-wise and descending order.
+ */
+ iowrite8(value >> 24, phy->iobase + addr + 3);
+ iowrite8(value >> 16, phy->iobase + addr + 2);
+ iowrite8(value >> 8, phy->iobase + addr + 1);
+ iowrite8(value, phy->iobase + addr);
+
+ return 0;
+}
+
+static const struct tpm_tis_phy_ops tpm_tcg_bw = {
+ .read_bytes = tpm_tis_synquacer_read_bytes,
+ .write_bytes = tpm_tis_synquacer_write_bytes,
+ .read16 = tpm_tis_synquacer_read16_bw,
+ .read32 = tpm_tis_synquacer_read32_bw,
+ .write32 = tpm_tis_synquacer_write32_bw,
+};
+
+static int tpm_tis_synquacer_init(struct device *dev,
+ struct tpm_tis_synquacer_info *tpm_info)
+{
+ struct tpm_tis_synquacer_phy *phy;
+
+ phy = devm_kzalloc(dev, sizeof(struct tpm_tis_synquacer_phy), GFP_KERNEL);
+ if (phy == NULL)
+ return -ENOMEM;
+
+ phy->iobase = devm_ioremap_resource(dev, &tpm_info->res);
+ if (IS_ERR(phy->iobase))
+ return PTR_ERR(phy->iobase);
+
+ return tpm_tis_core_init(dev, &phy->priv, tpm_info->irq, &tpm_tcg_bw,
+ ACPI_HANDLE(dev));
+}
+
+static SIMPLE_DEV_PM_OPS(tpm_tis_synquacer_pm, tpm_pm_suspend, tpm_tis_resume);
+
+static int tpm_tis_synquacer_probe(struct platform_device *pdev)
+{
+ struct tpm_tis_synquacer_info tpm_info = {};
+ struct resource *res;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (res == NULL) {
+ dev_err(&pdev->dev, "no memory resource defined\n");
+ return -ENODEV;
+ }
+ tpm_info.res = *res;
+
+ tpm_info.irq = -1;
+
+ return tpm_tis_synquacer_init(&pdev->dev, &tpm_info);
+}
+
+static int tpm_tis_synquacer_remove(struct platform_device *pdev)
+{
+ struct tpm_chip *chip = dev_get_drvdata(&pdev->dev);
+
+ tpm_chip_unregister(chip);
+ tpm_tis_remove(chip);
+
+ return 0;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id tis_synquacer_of_platform_match[] = {
+ {.compatible = "socionext,synquacer-tpm-mmio"},
+ {},
+};
+MODULE_DEVICE_TABLE(of, tis_synquacer_of_platform_match);
+#endif
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id tpm_synquacer_acpi_tbl[] = {
+ { "SCX0009" },
+ {},
+};
+MODULE_DEVICE_TABLE(acpi, tpm_synquacer_acpi_tbl);
+#endif
+
+static struct platform_driver tis_synquacer_drv = {
+ .probe = tpm_tis_synquacer_probe,
+ .remove = tpm_tis_synquacer_remove,
+ .driver = {
+ .name = "tpm_tis_synquacer",
+ .pm = &tpm_tis_synquacer_pm,
+ .of_match_table = of_match_ptr(tis_synquacer_of_platform_match),
+ .acpi_match_table = ACPI_PTR(tpm_synquacer_acpi_tbl),
+ },
+};
+
+static int __init tpm_tis_synquacer_module_init(void)
+{
+ int rc;
+
+ rc = platform_driver_register(&tis_synquacer_drv);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
+static void __exit tpm_tis_synquacer_module_exit(void)
+{
+ platform_driver_unregister(&tis_synquacer_drv);
+}
+
+module_init(tpm_tis_synquacer_module_init);
+module_exit(tpm_tis_synquacer_module_exit);
+MODULE_DESCRIPTION("TPM MMIO Driver for Socionext SynQuacer platform");
+MODULE_LICENSE("GPL");
GATE(CLK_PCIE, "pcie", "aclk133", GATE_IP_FSYS, 14, 0, 0),
GATE(CLK_SMMU_PCIE, "smmu_pcie", "aclk133", GATE_IP_FSYS, 18, 0, 0),
GATE(CLK_MODEMIF, "modemif", "aclk100", GATE_IP_PERIL, 28, 0, 0),
- GATE(CLK_CHIPID, "chipid", "aclk100", E4210_GATE_IP_PERIR, 0, 0, 0),
+ GATE(CLK_CHIPID, "chipid", "aclk100", E4210_GATE_IP_PERIR, 0, CLK_IGNORE_UNUSED, 0),
GATE(CLK_SYSREG, "sysreg", "aclk100", E4210_GATE_IP_PERIR, 0,
CLK_IGNORE_UNUSED, 0),
GATE(CLK_HDMI_CEC, "hdmi_cec", "aclk100", E4210_GATE_IP_PERIR, 11, 0,
0),
GATE(CLK_TSADC, "tsadc", "aclk133", E4X12_GATE_BUS_FSYS1, 16, 0, 0),
GATE(CLK_MIPI_HSI, "mipi_hsi", "aclk133", GATE_IP_FSYS, 10, 0, 0),
- GATE(CLK_CHIPID, "chipid", "aclk100", E4X12_GATE_IP_PERIR, 0, 0, 0),
+ GATE(CLK_CHIPID, "chipid", "aclk100", E4X12_GATE_IP_PERIR, 0, CLK_IGNORE_UNUSED, 0),
GATE(CLK_SYSREG, "sysreg", "aclk100", E4X12_GATE_IP_PERIR, 1,
CLK_IGNORE_UNUSED, 0),
GATE(CLK_HDMI_CEC, "hdmi_cec", "aclk100", E4X12_GATE_IP_PERIR, 11, 0,
* main G3D clock enablement status.
*/
clk_prepare_enable(__clk_lookup("mout_sw_aclk_g3d"));
+ /*
+ * Keep top BPLL mux enabled permanently to ensure that DRAM operates
+ * properly.
+ */
+ clk_prepare_enable(__clk_lookup("mout_bpll"));
samsung_clk_of_add_provider(np, ctx);
}
{ STRATIX10_EMAC_B_FREE_CLK, "emacb_free_clk", NULL, emacb_free_mux, ARRAY_SIZE(emacb_free_mux),
0, 0, 2, 0xB0, 1},
{ STRATIX10_EMAC_PTP_FREE_CLK, "emac_ptp_free_clk", NULL, emac_ptp_free_mux,
- ARRAY_SIZE(emac_ptp_free_mux), 0, 0, 4, 0xB0, 2},
+ ARRAY_SIZE(emac_ptp_free_mux), 0, 0, 2, 0xB0, 2},
{ STRATIX10_GPIO_DB_FREE_CLK, "gpio_db_free_clk", NULL, gpio_db_free_mux,
ARRAY_SIZE(gpio_db_free_mux), 0, 0, 0, 0xB0, 3},
{ STRATIX10_SDMMC_FREE_CLK, "sdmmc_free_clk", NULL, sdmmc_free_mux,
unsigned long flags = 0;
unsigned long input_rate;
- if (clk_pll_is_enabled(hw))
- return 0;
-
input_rate = clk_hw_get_rate(clk_hw_get_parent(hw));
if (_get_table_rate(hw, &sel, pll->params->fixed_rate, input_rate))
pll_writel(val, PLLE_SS_CTRL, pll);
udelay(1);
- /* Enable hw control of xusb brick pll */
+ /* Enable HW control of XUSB brick PLL */
val = pll_readl_misc(pll);
val &= ~PLLE_MISC_IDDQ_SW_CTRL;
pll_writel_misc(val, pll);
val |= XUSBIO_PLL_CFG0_SEQ_ENABLE;
pll_writel(val, XUSBIO_PLL_CFG0, pll);
- /* Enable hw control of SATA pll */
+ /* Enable HW control of SATA PLL */
val = pll_readl(SATA_PLL_CFG0, pll);
val &= ~SATA_PLL_CFG0_PADPLL_RESET_SWCTL;
val |= SATA_PLL_CFG0_PADPLL_USE_LOCKDET;
#include <linux/io.h>
#include <linux/slab.h>
+#include "clk.h"
+
#define CLK_SOURCE_EMC 0x19c
#define CLK_SOURCE_EMC_2X_CLK_SRC GENMASK(31, 29)
#define CLK_SOURCE_EMC_MC_EMC_SAME_FREQ BIT(16)
return PTR_ERR(clk);
}
- ret = ENXIO;
+ ret = -ENXIO;
base = of_iomap(node, 0);
if (!base) {
pr_err("failed to map registers for clockevent\n");
ce->clkevt.rating = 200;
ce->clkevt.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT;
ce->clkevt.cpumask = cpu_possible_mask;
- ce->clkevt.set_state_shutdown = mps2_timer_shutdown,
- ce->clkevt.set_state_periodic = mps2_timer_set_periodic,
- ce->clkevt.set_state_oneshot = mps2_timer_shutdown,
+ ce->clkevt.set_state_shutdown = mps2_timer_shutdown;
+ ce->clkevt.set_state_periodic = mps2_timer_set_periodic;
+ ce->clkevt.set_state_oneshot = mps2_timer_shutdown;
ce->clkevt.set_next_event = mps2_timer_set_next_event;
/* Ensure timer is disabled */
clr = TIMER0_25MHZ;
local_timer_ctrl_clrset(clr, set);
- evt->name = "armada_370_xp_per_cpu_tick",
+ evt->name = "armada_370_xp_per_cpu_tick";
evt->features = CLOCK_EVT_FEAT_ONESHOT |
CLOCK_EVT_FEAT_PERIODIC;
- evt->shift = 32,
- evt->rating = 300,
- evt->set_next_event = armada_370_xp_clkevt_next_event,
+ evt->shift = 32;
+ evt->rating = 300;
+ evt->set_next_event = armada_370_xp_clkevt_next_event;
evt->set_state_shutdown = armada_370_xp_clkevt_shutdown;
evt->set_state_periodic = armada_370_xp_clkevt_set_periodic;
evt->set_state_oneshot = armada_370_xp_clkevt_shutdown;
#ifdef CONFIG_RISCV_M_MODE
u64 __iomem *clint_time_val;
+EXPORT_SYMBOL(clint_time_val);
#endif
static void clint_send_ipi(const struct cpumask *target)
void __iomem *base = timer_of_base(to_timer_of(ce));
writel_relaxed(GX6605S_STATUS_CLR, base + TIMER_STATUS);
+ writel_relaxed(0, base + TIMER_INI);
ce->event_handler(ce);
*
* Every SP804 contains two identical timers.
*/
+#define NR_TIMERS 2
#define TIMER_1_BASE 0x00
#define TIMER_2_BASE 0x20
#define TIMER_RIS 0x10 /* CVR ro */
#define TIMER_MIS 0x14 /* CVR ro */
#define TIMER_BGLOAD 0x18 /* CVR rw */
+
+struct sp804_timer {
+ int load;
+ int load_h;
+ int value;
+ int value_h;
+ int ctrl;
+ int intclr;
+ int ris;
+ int mis;
+ int bgload;
+ int bgload_h;
+ int timer_base[NR_TIMERS];
+ int width;
+};
+
+struct sp804_clkevt {
+ void __iomem *base;
+ void __iomem *load;
+ void __iomem *load_h;
+ void __iomem *value;
+ void __iomem *value_h;
+ void __iomem *ctrl;
+ void __iomem *intclr;
+ void __iomem *ris;
+ void __iomem *mis;
+ void __iomem *bgload;
+ void __iomem *bgload_h;
+ unsigned long reload;
+ int width;
+};
#include <linux/of_irq.h>
#include <linux/sched_clock.h>
-#include <clocksource/timer-sp804.h>
-
#include "timer-sp.h"
-static long __init sp804_get_clock_rate(struct clk *clk)
+/* Hisilicon 64-bit timer(a variant of ARM SP804) */
+#define HISI_TIMER_1_BASE 0x00
+#define HISI_TIMER_2_BASE 0x40
+#define HISI_TIMER_LOAD 0x00
+#define HISI_TIMER_LOAD_H 0x04
+#define HISI_TIMER_VALUE 0x08
+#define HISI_TIMER_VALUE_H 0x0c
+#define HISI_TIMER_CTRL 0x10
+#define HISI_TIMER_INTCLR 0x14
+#define HISI_TIMER_RIS 0x18
+#define HISI_TIMER_MIS 0x1c
+#define HISI_TIMER_BGLOAD 0x20
+#define HISI_TIMER_BGLOAD_H 0x24
+
+
+struct sp804_timer __initdata arm_sp804_timer = {
+ .load = TIMER_LOAD,
+ .value = TIMER_VALUE,
+ .ctrl = TIMER_CTRL,
+ .intclr = TIMER_INTCLR,
+ .timer_base = {TIMER_1_BASE, TIMER_2_BASE},
+ .width = 32,
+};
+
+struct sp804_timer __initdata hisi_sp804_timer = {
+ .load = HISI_TIMER_LOAD,
+ .load_h = HISI_TIMER_LOAD_H,
+ .value = HISI_TIMER_VALUE,
+ .value_h = HISI_TIMER_VALUE_H,
+ .ctrl = HISI_TIMER_CTRL,
+ .intclr = HISI_TIMER_INTCLR,
+ .timer_base = {HISI_TIMER_1_BASE, HISI_TIMER_2_BASE},
+ .width = 64,
+};
+
+static struct sp804_clkevt sp804_clkevt[NR_TIMERS];
+
+static long __init sp804_get_clock_rate(struct clk *clk, const char *name)
{
long rate;
int err;
+ if (!clk)
+ clk = clk_get_sys("sp804", name);
+ if (IS_ERR(clk)) {
+ pr_err("sp804: %s clock not found: %ld\n", name, PTR_ERR(clk));
+ return PTR_ERR(clk);
+ }
+
err = clk_prepare(clk);
if (err) {
pr_err("sp804: clock failed to prepare: %d\n", err);
return rate;
}
-static void __iomem *sched_clock_base;
-
-static u64 notrace sp804_read(void)
+static struct sp804_clkevt * __init sp804_clkevt_get(void __iomem *base)
{
- return ~readl_relaxed(sched_clock_base + TIMER_VALUE);
+ int i;
+
+ for (i = 0; i < NR_TIMERS; i++) {
+ if (sp804_clkevt[i].base == base)
+ return &sp804_clkevt[i];
+ }
+
+ /* It's impossible to reach here */
+ WARN_ON(1);
+
+ return NULL;
}
-void __init sp804_timer_disable(void __iomem *base)
+static struct sp804_clkevt *sched_clkevt;
+
+static u64 notrace sp804_read(void)
{
- writel(0, base + TIMER_CTRL);
+ return ~readl_relaxed(sched_clkevt->value);
}
-int __init __sp804_clocksource_and_sched_clock_init(void __iomem *base,
- const char *name,
- struct clk *clk,
- int use_sched_clock)
+int __init sp804_clocksource_and_sched_clock_init(void __iomem *base,
+ const char *name,
+ struct clk *clk,
+ int use_sched_clock)
{
long rate;
+ struct sp804_clkevt *clkevt;
- if (!clk) {
- clk = clk_get_sys("sp804", name);
- if (IS_ERR(clk)) {
- pr_err("sp804: clock not found: %d\n",
- (int)PTR_ERR(clk));
- return PTR_ERR(clk);
- }
- }
-
- rate = sp804_get_clock_rate(clk);
+ rate = sp804_get_clock_rate(clk, name);
if (rate < 0)
return -EINVAL;
- /* setup timer 0 as free-running clocksource */
- writel(0, base + TIMER_CTRL);
- writel(0xffffffff, base + TIMER_LOAD);
- writel(0xffffffff, base + TIMER_VALUE);
+ clkevt = sp804_clkevt_get(base);
+
+ writel(0, clkevt->ctrl);
+ writel(0xffffffff, clkevt->load);
+ writel(0xffffffff, clkevt->value);
+ if (clkevt->width == 64) {
+ writel(0xffffffff, clkevt->load_h);
+ writel(0xffffffff, clkevt->value_h);
+ }
writel(TIMER_CTRL_32BIT | TIMER_CTRL_ENABLE | TIMER_CTRL_PERIODIC,
- base + TIMER_CTRL);
+ clkevt->ctrl);
- clocksource_mmio_init(base + TIMER_VALUE, name,
+ clocksource_mmio_init(clkevt->value, name,
rate, 200, 32, clocksource_mmio_readl_down);
if (use_sched_clock) {
- sched_clock_base = base;
+ sched_clkevt = clkevt;
sched_clock_register(sp804_read, 32, rate);
}
}
-static void __iomem *clkevt_base;
-static unsigned long clkevt_reload;
+static struct sp804_clkevt *common_clkevt;
/*
* IRQ handler for the timer
struct clock_event_device *evt = dev_id;
/* clear the interrupt */
- writel(1, clkevt_base + TIMER_INTCLR);
+ writel(1, common_clkevt->intclr);
evt->event_handler(evt);
static inline void timer_shutdown(struct clock_event_device *evt)
{
- writel(0, clkevt_base + TIMER_CTRL);
+ writel(0, common_clkevt->ctrl);
}
static int sp804_shutdown(struct clock_event_device *evt)
TIMER_CTRL_PERIODIC | TIMER_CTRL_ENABLE;
timer_shutdown(evt);
- writel(clkevt_reload, clkevt_base + TIMER_LOAD);
- writel(ctrl, clkevt_base + TIMER_CTRL);
+ writel(common_clkevt->reload, common_clkevt->load);
+ writel(ctrl, common_clkevt->ctrl);
return 0;
}
unsigned long ctrl = TIMER_CTRL_32BIT | TIMER_CTRL_IE |
TIMER_CTRL_ONESHOT | TIMER_CTRL_ENABLE;
- writel(next, clkevt_base + TIMER_LOAD);
- writel(ctrl, clkevt_base + TIMER_CTRL);
+ writel(next, common_clkevt->load);
+ writel(ctrl, common_clkevt->ctrl);
return 0;
}
.rating = 300,
};
-int __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struct clk *clk, const char *name)
+int __init sp804_clockevents_init(void __iomem *base, unsigned int irq,
+ struct clk *clk, const char *name)
{
struct clock_event_device *evt = &sp804_clockevent;
long rate;
- if (!clk)
- clk = clk_get_sys("sp804", name);
- if (IS_ERR(clk)) {
- pr_err("sp804: %s clock not found: %d\n", name,
- (int)PTR_ERR(clk));
- return PTR_ERR(clk);
- }
-
- rate = sp804_get_clock_rate(clk);
+ rate = sp804_get_clock_rate(clk, name);
if (rate < 0)
return -EINVAL;
- clkevt_base = base;
- clkevt_reload = DIV_ROUND_CLOSEST(rate, HZ);
+ common_clkevt = sp804_clkevt_get(base);
+ common_clkevt->reload = DIV_ROUND_CLOSEST(rate, HZ);
evt->name = name;
evt->irq = irq;
evt->cpumask = cpu_possible_mask;
- writel(0, base + TIMER_CTRL);
+ writel(0, common_clkevt->ctrl);
if (request_irq(irq, sp804_timer_interrupt, IRQF_TIMER | IRQF_IRQPOLL,
"timer", &sp804_clockevent))
return 0;
}
-static int __init sp804_of_init(struct device_node *np)
+static void __init sp804_clkevt_init(struct sp804_timer *timer, void __iomem *base)
+{
+ int i;
+
+ for (i = 0; i < NR_TIMERS; i++) {
+ void __iomem *timer_base;
+ struct sp804_clkevt *clkevt;
+
+ timer_base = base + timer->timer_base[i];
+ clkevt = &sp804_clkevt[i];
+ clkevt->base = timer_base;
+ clkevt->load = timer_base + timer->load;
+ clkevt->load_h = timer_base + timer->load_h;
+ clkevt->value = timer_base + timer->value;
+ clkevt->value_h = timer_base + timer->value_h;
+ clkevt->ctrl = timer_base + timer->ctrl;
+ clkevt->intclr = timer_base + timer->intclr;
+ clkevt->width = timer->width;
+ }
+}
+
+static int __init sp804_of_init(struct device_node *np, struct sp804_timer *timer)
{
static bool initialized = false;
void __iomem *base;
+ void __iomem *timer1_base;
+ void __iomem *timer2_base;
int irq, ret = -EINVAL;
u32 irq_num = 0;
struct clk *clk1, *clk2;
if (!base)
return -ENXIO;
+ timer1_base = base + timer->timer_base[0];
+ timer2_base = base + timer->timer_base[1];
+
/* Ensure timers are disabled */
- writel(0, base + TIMER_CTRL);
- writel(0, base + TIMER_2_BASE + TIMER_CTRL);
+ writel(0, timer1_base + timer->ctrl);
+ writel(0, timer2_base + timer->ctrl);
if (initialized || !of_device_is_available(np)) {
ret = -EINVAL;
if (irq <= 0)
goto err;
+ sp804_clkevt_init(timer, base);
+
of_property_read_u32(np, "arm,sp804-has-irq", &irq_num);
if (irq_num == 2) {
- ret = __sp804_clockevents_init(base + TIMER_2_BASE, irq, clk2, name);
+ ret = sp804_clockevents_init(timer2_base, irq, clk2, name);
if (ret)
goto err;
- ret = __sp804_clocksource_and_sched_clock_init(base, name, clk1, 1);
+ ret = sp804_clocksource_and_sched_clock_init(timer1_base,
+ name, clk1, 1);
if (ret)
goto err;
} else {
- ret = __sp804_clockevents_init(base, irq, clk1 , name);
+ ret = sp804_clockevents_init(timer1_base, irq, clk1, name);
if (ret)
goto err;
- ret =__sp804_clocksource_and_sched_clock_init(base + TIMER_2_BASE,
- name, clk2, 1);
+ ret = sp804_clocksource_and_sched_clock_init(timer2_base,
+ name, clk2, 1);
if (ret)
goto err;
}
iounmap(base);
return ret;
}
-TIMER_OF_DECLARE(sp804, "arm,sp804", sp804_of_init);
+
+static int __init arm_sp804_of_init(struct device_node *np)
+{
+ return sp804_of_init(np, &arm_sp804_timer);
+}
+TIMER_OF_DECLARE(sp804, "arm,sp804", arm_sp804_of_init);
+
+static int __init hisi_sp804_of_init(struct device_node *np)
+{
+ return sp804_of_init(np, &hisi_sp804_timer);
+}
+TIMER_OF_DECLARE(hisi_sp804, "hisilicon,sp804", hisi_sp804_of_init);
static int __init integrator_cp_of_init(struct device_node *np)
{
}
/* Ensure timer is disabled */
- writel(0, base + TIMER_CTRL);
+ writel(0, base + arm_sp804_timer.ctrl);
if (init_count == 2 || !of_device_is_available(np))
goto err;
+ sp804_clkevt_init(&arm_sp804_timer, base);
+
if (!init_count) {
- ret = __sp804_clocksource_and_sched_clock_init(base, name, clk, 0);
+ ret = sp804_clocksource_and_sched_clock_init(base,
+ name, clk, 0);
if (ret)
goto err;
} else {
if (irq <= 0)
goto err;
- ret = __sp804_clockevents_init(base, irq, clk, name);
+ ret = sp804_clockevents_init(base, irq, clk, name);
if (ret)
goto err;
}
return !(tidr >> 16);
}
+static void dmtimer_systimer_enable(struct dmtimer_systimer *t)
+{
+ u32 val;
+
+ if (dmtimer_systimer_revision1(t))
+ val = DMTIMER_TYPE1_ENABLE;
+ else
+ val = DMTIMER_TYPE2_ENABLE;
+
+ writel_relaxed(val, t->base + t->sysc);
+}
+
+static void dmtimer_systimer_disable(struct dmtimer_systimer *t)
+{
+ if (!dmtimer_systimer_revision1(t))
+ return;
+
+ writel_relaxed(DMTIMER_TYPE1_DISABLE, t->base + t->sysc);
+}
+
static int __init dmtimer_systimer_type1_reset(struct dmtimer_systimer *t)
{
void __iomem *syss = t->base + OMAP_TIMER_V1_SYS_STAT_OFFSET;
int ret;
u32 l;
+ dmtimer_systimer_enable(t);
writel_relaxed(BIT(1) | BIT(2), t->base + t->ifctrl);
ret = readl_poll_timeout_atomic(syss, l, l & BIT(0), 100,
DMTIMER_RESET_WAIT);
void __iomem *sysc = t->base + t->sysc;
u32 l;
+ dmtimer_systimer_enable(t);
l = readl_relaxed(sysc);
l |= BIT(0);
writel_relaxed(l, sysc);
return 0;
}
-static void dmtimer_systimer_enable(struct dmtimer_systimer *t)
-{
- u32 val;
-
- if (dmtimer_systimer_revision1(t))
- val = DMTIMER_TYPE1_ENABLE;
- else
- val = DMTIMER_TYPE2_ENABLE;
-
- writel_relaxed(val, t->base + t->sysc);
-}
-
-static void dmtimer_systimer_disable(struct dmtimer_systimer *t)
-{
- if (!dmtimer_systimer_revision1(t))
- return;
-
- writel_relaxed(DMTIMER_TYPE1_DISABLE, t->base + t->sysc);
-}
-
static int __init dmtimer_systimer_setup(struct device_node *np,
struct dmtimer_systimer *t)
{
t->wakeup = regbase + _OMAP_TIMER_WAKEUP_EN_OFFSET;
t->ifctrl = regbase + _OMAP_TIMER_IF_CTRL_OFFSET;
- dmtimer_systimer_enable(t);
dmtimer_systimer_reset(t);
+ dmtimer_systimer_enable(t);
pr_debug("dmtimer rev %08x sysc %08x\n", readl_relaxed(t->base),
readl_relaxed(t->base + t->sysc));
cpufreq_unregister_driver(intel_pstate_driver);
intel_pstate_driver_cleanup();
+ return 0;
}
if (size == 6 && !strncmp(buf, "active", size)) {
return -1;
/* Do runtime PM to manage a hierarchical CPU toplogy. */
- pm_runtime_put_sync_suspend(pd_dev);
+ RCU_NONIDLE(pm_runtime_put_sync_suspend(pd_dev));
state = psci_get_domain_state();
if (!state)
ret = psci_cpu_suspend_enter(state) ? -1 : idx;
- pm_runtime_get_sync(pd_dev);
+ RCU_NONIDLE(pm_runtime_get_sync(pd_dev));
cpu_pm_exit();
time_start = ns_to_ktime(local_clock());
- /*
- * trace_suspend_resume() called by tick_freeze() for the last CPU
- * executing it contains RCU usage regarded as invalid in the idle
- * context, so tell RCU about that.
- */
tick_freeze();
/*
* The state used here cannot be a "coupled" one, because the "coupled"
target_state->enter_s2idle(dev, drv, index);
if (WARN_ON_ONCE(!irqs_disabled()))
local_irq_disable();
- /*
- * timekeeping_resume() that will be called by tick_unfreeze() for the
- * first CPU executing it calls functions containing RCU read-side
- * critical sections, so tell RCU about that.
- */
if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))
rcu_idle_exit();
tick_unfreeze();
select CRYPTO_AES
select CRYPTO_AES_ARM64
select CRYPTO_ALGAPI
+ select CRYPTO_AUTHENC
select CRYPTO_SHA1
select CRYPTO_SHA256
select CRYPTO_SHA512
This will create /sys/kernel/debug/sun8i-ce/stats for displaying
the number of requests per flow and per algorithm.
+config CRYPTO_DEV_SUN8I_CE_HASH
+ bool "Enable support for hash on sun8i-ce"
+ depends on CRYPTO_DEV_SUN8I_CE
+ select MD5
+ select SHA1
+ select SHA256
+ select SHA512
+ help
+ Say y to enable support for hash algorithms.
+
+config CRYPTO_DEV_SUN8I_CE_PRNG
+ bool "Support for Allwinner Crypto Engine PRNG"
+ depends on CRYPTO_DEV_SUN8I_CE
+ select CRYPTO_RNG
+ help
+ Select this option if you want to provide kernel-side support for
+ the Pseudo-Random Number Generator found in the Crypto Engine.
+
+config CRYPTO_DEV_SUN8I_CE_TRNG
+ bool "Support for Allwinner Crypto Engine TRNG"
+ depends on CRYPTO_DEV_SUN8I_CE
+ select HW_RANDOM
+ help
+ Select this option if you want to provide kernel-side support for
+ the True Random Number Generator found in the Crypto Engine.
+
config CRYPTO_DEV_SUN8I_SS
tristate "Support for Allwinner Security System cryptographic offloader"
select CRYPTO_SKCIPHER
Say y to enable sun8i-ss debug stats.
This will create /sys/kernel/debug/sun8i-ss/stats for displaying
the number of requests per flow and per algorithm.
+
+config CRYPTO_DEV_SUN8I_SS_PRNG
+ bool "Support for Allwinner Security System PRNG"
+ depends on CRYPTO_DEV_SUN8I_SS
+ select CRYPTO_RNG
+ help
+ Select this option if you want to provide kernel-side support for
+ the Pseudo-Random Number Generator found in the Security System.
+
+config CRYPTO_DEV_SUN8I_SS_HASH
+ bool "Enable support for hash on sun8i-ss"
+ depends on CRYPTO_DEV_SUN8I_SS
+ select MD5
+ select SHA1
+ select SHA256
+ help
+ Say y to enable support for hash algorithms.
* You could find the datasheet in Documentation/arm/sunxi.rst
*/
#include "sun4i-ss.h"
+#include <asm/unaligned.h>
#include <linux/scatterlist.h>
/* This is a totally arbitrary value */
struct sg_mapping_iter mi;
int in_r, err = 0;
size_t copied = 0;
- __le32 wb = 0;
+ u32 wb = 0;
dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x",
__func__, crypto_tfm_alg_name(areq->base.tfm),
nbw = op->len - 4 * nwait;
if (nbw) {
- wb = cpu_to_le32(*(u32 *)(op->buf + nwait * 4));
+ wb = le32_to_cpup((__le32 *)(op->buf + nwait * 4));
wb &= GENMASK((nbw * 8) - 1, 0);
op->byte_count += nbw;
/* write the remaining bytes of the nbw buffer */
wb |= ((1 << 7) << (nbw * 8));
- bf[j++] = le32_to_cpu(wb);
+ ((__le32 *)bf)[j++] = cpu_to_le32(wb);
/*
* number of space to pad to obtain 64o minus 8(size) minus 4 (final 1)
/* Get the hash from the device */
if (op->mode == SS_OP_SHA1) {
for (i = 0; i < 5; i++) {
+ v = readl(ss->base + SS_MD0 + i * 4);
if (ss->variant->sha1_in_be)
- v = cpu_to_le32(readl(ss->base + SS_MD0 + i * 4));
+ put_unaligned_le32(v, areq->result + i * 4);
else
- v = cpu_to_be32(readl(ss->base + SS_MD0 + i * 4));
- memcpy(areq->result + i * 4, &v, 4);
+ put_unaligned_be32(v, areq->result + i * 4);
}
} else {
for (i = 0; i < 4; i++) {
- v = cpu_to_le32(readl(ss->base + SS_MD0 + i * 4));
- memcpy(areq->result + i * 4, &v, 4);
+ v = readl(ss->base + SS_MD0 + i * 4);
+ put_unaligned_le32(v, areq->result + i * 4);
}
}
obj-$(CONFIG_CRYPTO_DEV_SUN8I_CE) += sun8i-ce.o
sun8i-ce-y += sun8i-ce-core.o sun8i-ce-cipher.o
+sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_HASH) += sun8i-ce-hash.o
+sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG) += sun8i-ce-prng.o
+sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) += sun8i-ce-trng.o
return err;
}
-static int sun8i_ce_cipher(struct skcipher_request *areq)
+static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req)
{
+ struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base);
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
struct sun8i_ce_dev *ce = op->ce;
struct ce_task *cet;
struct scatterlist *sg;
unsigned int todo, len, offset, ivsize;
- dma_addr_t addr_iv = 0, addr_key = 0;
- void *backup_iv = NULL;
u32 common, sym;
int flow, i;
int nr_sgs = 0;
common |= rctx->op_dir | CE_COMM_INT;
cet->t_common_ctl = cpu_to_le32(common);
/* CTS and recent CE (H6) need length in bytes, in word otherwise */
- if (ce->variant->has_t_dlen_in_bytes)
+ if (ce->variant->cipher_t_dlen_in_bytes)
cet->t_dlen = cpu_to_le32(areq->cryptlen);
else
cet->t_dlen = cpu_to_le32(areq->cryptlen / 4);
cet->t_sym_ctl = cpu_to_le32(sym);
cet->t_asym_ctl = 0;
- addr_key = dma_map_single(ce->dev, op->key, op->keylen, DMA_TO_DEVICE);
- cet->t_key = cpu_to_le32(addr_key);
- if (dma_mapping_error(ce->dev, addr_key)) {
+ rctx->addr_key = dma_map_single(ce->dev, op->key, op->keylen, DMA_TO_DEVICE);
+ if (dma_mapping_error(ce->dev, rctx->addr_key)) {
dev_err(ce->dev, "Cannot DMA MAP KEY\n");
err = -EFAULT;
goto theend;
}
+ cet->t_key = cpu_to_le32(rctx->addr_key);
ivsize = crypto_skcipher_ivsize(tfm);
if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) {
- chan->ivlen = ivsize;
- chan->bounce_iv = kzalloc(ivsize, GFP_KERNEL | GFP_DMA);
- if (!chan->bounce_iv) {
+ rctx->ivlen = ivsize;
+ rctx->bounce_iv = kzalloc(ivsize, GFP_KERNEL | GFP_DMA);
+ if (!rctx->bounce_iv) {
err = -ENOMEM;
goto theend_key;
}
if (rctx->op_dir & CE_DECRYPTION) {
- backup_iv = kzalloc(ivsize, GFP_KERNEL);
- if (!backup_iv) {
+ rctx->backup_iv = kzalloc(ivsize, GFP_KERNEL);
+ if (!rctx->backup_iv) {
err = -ENOMEM;
goto theend_key;
}
offset = areq->cryptlen - ivsize;
- scatterwalk_map_and_copy(backup_iv, areq->src, offset,
- ivsize, 0);
+ scatterwalk_map_and_copy(rctx->backup_iv, areq->src,
+ offset, ivsize, 0);
}
- memcpy(chan->bounce_iv, areq->iv, ivsize);
- addr_iv = dma_map_single(ce->dev, chan->bounce_iv, chan->ivlen,
- DMA_TO_DEVICE);
- cet->t_iv = cpu_to_le32(addr_iv);
- if (dma_mapping_error(ce->dev, addr_iv)) {
+ memcpy(rctx->bounce_iv, areq->iv, ivsize);
+ rctx->addr_iv = dma_map_single(ce->dev, rctx->bounce_iv, rctx->ivlen,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(ce->dev, rctx->addr_iv)) {
dev_err(ce->dev, "Cannot DMA MAP IV\n");
err = -ENOMEM;
goto theend_iv;
}
+ cet->t_iv = cpu_to_le32(rctx->addr_iv);
}
if (areq->src == areq->dst) {
}
chan->timeout = areq->cryptlen;
- err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(areq->base.tfm));
+ rctx->nr_sgs = nr_sgs;
+ rctx->nr_sgd = nr_sgd;
+ return 0;
theend_sgs:
if (areq->src == areq->dst) {
theend_iv:
if (areq->iv && ivsize > 0) {
- if (addr_iv)
- dma_unmap_single(ce->dev, addr_iv, chan->ivlen,
- DMA_TO_DEVICE);
+ if (rctx->addr_iv)
+ dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE);
offset = areq->cryptlen - ivsize;
if (rctx->op_dir & CE_DECRYPTION) {
- memcpy(areq->iv, backup_iv, ivsize);
- kfree_sensitive(backup_iv);
+ memcpy(areq->iv, rctx->backup_iv, ivsize);
+ kfree_sensitive(rctx->backup_iv);
} else {
scatterwalk_map_and_copy(areq->iv, areq->dst, offset,
ivsize, 0);
}
- kfree(chan->bounce_iv);
+ kfree(rctx->bounce_iv);
}
theend_key:
- dma_unmap_single(ce->dev, addr_key, op->keylen, DMA_TO_DEVICE);
+ dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE);
theend:
return err;
}
-static int sun8i_ce_handle_cipher_request(struct crypto_engine *engine, void *areq)
+static int sun8i_ce_cipher_run(struct crypto_engine *engine, void *areq)
{
- int err;
struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(breq);
+ struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun8i_ce_dev *ce = op->ce;
+ struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(breq);
+ int flow, err;
- err = sun8i_ce_cipher(breq);
+ flow = rctx->flow;
+ err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(breq->base.tfm));
crypto_finalize_skcipher_request(engine, breq, err);
+ return 0;
+}
+
+static int sun8i_ce_cipher_unprepare(struct crypto_engine *engine, void *async_req)
+{
+ struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base);
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+ struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+ struct sun8i_ce_dev *ce = op->ce;
+ struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
+ struct sun8i_ce_flow *chan;
+ struct ce_task *cet;
+ unsigned int ivsize, offset;
+ int nr_sgs = rctx->nr_sgs;
+ int nr_sgd = rctx->nr_sgd;
+ int flow;
+
+ flow = rctx->flow;
+ chan = &ce->chanlist[flow];
+ cet = chan->tl;
+ ivsize = crypto_skcipher_ivsize(tfm);
+
+ if (areq->src == areq->dst) {
+ dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_BIDIRECTIONAL);
+ } else {
+ if (nr_sgs > 0)
+ dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE);
+ dma_unmap_sg(ce->dev, areq->dst, nr_sgd, DMA_FROM_DEVICE);
+ }
+
+ if (areq->iv && ivsize > 0) {
+ if (cet->t_iv)
+ dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE);
+ offset = areq->cryptlen - ivsize;
+ if (rctx->op_dir & CE_DECRYPTION) {
+ memcpy(areq->iv, rctx->backup_iv, ivsize);
+ kfree_sensitive(rctx->backup_iv);
+ } else {
+ scatterwalk_map_and_copy(areq->iv, areq->dst, offset,
+ ivsize, 0);
+ }
+ kfree(rctx->bounce_iv);
+ }
+
+ dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE);
return 0;
}
crypto_tfm_alg_driver_name(&sktfm->base),
crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)));
- op->enginectx.op.do_one_request = sun8i_ce_handle_cipher_request;
- op->enginectx.op.prepare_request = NULL;
- op->enginectx.op.unprepare_request = NULL;
+ op->enginectx.op.do_one_request = sun8i_ce_cipher_run;
+ op->enginectx.op.prepare_request = sun8i_ce_cipher_prepare;
+ op->enginectx.op.unprepare_request = sun8i_ce_cipher_unprepare;
err = pm_runtime_get_sync(op->ce->dev);
if (err < 0)
{
struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
- if (op->key) {
- memzero_explicit(op->key, op->keylen);
- kfree(op->key);
- }
+ kfree_sensitive(op->key);
crypto_free_skcipher(op->fallback_tfm);
pm_runtime_put_sync_suspend(op->ce->dev);
}
dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen);
return -EINVAL;
}
- if (op->key) {
- memzero_explicit(op->key, op->keylen);
- kfree(op->key);
- }
+ kfree_sensitive(op->key);
op->keylen = keylen;
op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
if (!op->key)
if (err)
return err;
- if (op->key) {
- memzero_explicit(op->key, op->keylen);
- kfree(op->key);
- }
+ kfree_sensitive(op->key);
op->keylen = keylen;
op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
if (!op->key)
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/reset.h>
+#include <crypto/internal/rng.h>
#include <crypto/internal/skcipher.h>
#include "sun8i-ce.h"
static const struct ce_variant ce_h3_variant = {
.alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
},
+ .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
+ CE_ALG_SHA384, CE_ALG_SHA512
+ },
.op_mode = { CE_OP_ECB, CE_OP_CBC
},
.ce_clks = {
{ "bus", 0, 200000000 },
{ "mod", 50000000, 0 },
- }
+ },
+ .esr = ESR_H3,
+ .prng = CE_ALG_PRNG,
+ .trng = CE_ID_NOTSUPP,
};
static const struct ce_variant ce_h5_variant = {
.alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
},
+ .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
+ CE_ID_NOTSUPP, CE_ID_NOTSUPP
+ },
.op_mode = { CE_OP_ECB, CE_OP_CBC
},
.ce_clks = {
{ "bus", 0, 200000000 },
{ "mod", 300000000, 0 },
- }
+ },
+ .esr = ESR_H5,
+ .prng = CE_ALG_PRNG,
+ .trng = CE_ID_NOTSUPP,
};
static const struct ce_variant ce_h6_variant = {
.alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
},
+ .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
+ CE_ALG_SHA384, CE_ALG_SHA512
+ },
.op_mode = { CE_OP_ECB, CE_OP_CBC
},
- .has_t_dlen_in_bytes = true,
+ .cipher_t_dlen_in_bytes = true,
+ .hash_t_dlen_in_bits = true,
+ .prng_t_dlen_in_bytes = true,
+ .trng_t_dlen_in_bytes = true,
.ce_clks = {
{ "bus", 0, 200000000 },
{ "mod", 300000000, 0 },
{ "ram", 0, 400000000 },
- }
+ },
+ .esr = ESR_H6,
+ .prng = CE_ALG_PRNG_V2,
+ .trng = CE_ALG_TRNG_V2,
};
static const struct ce_variant ce_a64_variant = {
.alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
},
+ .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
+ CE_ID_NOTSUPP, CE_ID_NOTSUPP
+ },
.op_mode = { CE_OP_ECB, CE_OP_CBC
},
.ce_clks = {
{ "bus", 0, 200000000 },
{ "mod", 300000000, 0 },
- }
+ },
+ .esr = ESR_A64,
+ .prng = CE_ALG_PRNG,
+ .trng = CE_ID_NOTSUPP,
};
static const struct ce_variant ce_r40_variant = {
.alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
},
+ .alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
+ CE_ID_NOTSUPP, CE_ID_NOTSUPP
+ },
.op_mode = { CE_OP_ECB, CE_OP_CBC
},
.ce_clks = {
{ "bus", 0, 200000000 },
{ "mod", 300000000, 0 },
- }
+ },
+ .esr = ESR_R40,
+ .prng = CE_ALG_PRNG,
+ .trng = CE_ID_NOTSUPP,
};
/*
* sun8i_ce_get_engine_number() get the next channel slot
* This is a simple round-robin way of getting the next channel
+ * The flow 3 is reserve for xRNG operations
*/
int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce)
{
- return atomic_inc_return(&ce->flow) % MAXFLOW;
+ return atomic_inc_return(&ce->flow) % (MAXFLOW - 1);
}
int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name)
{
u32 v;
int err = 0;
+ struct ce_task *cet = ce->chanlist[flow].tl;
#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
ce->chanlist[flow].stat_req++;
/* Be sure all data is written before enabling the task */
wmb();
- v = 1 | (ce->chanlist[flow].tl->t_common_ctl & 0x7F) << 8;
+ /* Only H6 needs to write a part of t_common_ctl along with "1", but since it is ignored
+ * on older SoCs, we have no reason to complicate things.
+ */
+ v = 1 | ((le32_to_cpu(ce->chanlist[flow].tl->t_common_ctl) & 0x7F) << 8);
writel(v, ce->base + CE_TLR);
mutex_unlock(&ce->mlock);
msecs_to_jiffies(ce->chanlist[flow].timeout));
if (ce->chanlist[flow].status == 0) {
- dev_err(ce->dev, "DMA timeout for %s\n", name);
+ dev_err(ce->dev, "DMA timeout for %s (tm=%d) on flow %d\n", name,
+ ce->chanlist[flow].timeout, flow);
err = -EFAULT;
}
/* No need to lock for this read, the channel is locked so
* nothing could modify the error value for this channel
*/
v = readl(ce->base + CE_ESR);
- if (v) {
+ switch (ce->variant->esr) {
+ case ESR_H3:
+ /* Sadly, the error bit is not per flow */
+ if (v) {
+ dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow);
+ err = -EFAULT;
+ print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4,
+ cet, sizeof(struct ce_task), false);
+ }
+ if (v & CE_ERR_ALGO_NOTSUP)
+ dev_err(ce->dev, "CE ERROR: algorithm not supported\n");
+ if (v & CE_ERR_DATALEN)
+ dev_err(ce->dev, "CE ERROR: data length error\n");
+ if (v & CE_ERR_KEYSRAM)
+ dev_err(ce->dev, "CE ERROR: keysram access error for AES\n");
+ break;
+ case ESR_A64:
+ case ESR_H5:
+ case ESR_R40:
v >>= (flow * 4);
+ v &= 0xF;
+ if (v) {
+ dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow);
+ err = -EFAULT;
+ print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4,
+ cet, sizeof(struct ce_task), false);
+ }
+ if (v & CE_ERR_ALGO_NOTSUP)
+ dev_err(ce->dev, "CE ERROR: algorithm not supported\n");
+ if (v & CE_ERR_DATALEN)
+ dev_err(ce->dev, "CE ERROR: data length error\n");
+ if (v & CE_ERR_KEYSRAM)
+ dev_err(ce->dev, "CE ERROR: keysram access error for AES\n");
+ break;
+ case ESR_H6:
+ v >>= (flow * 8);
v &= 0xFF;
if (v) {
dev_err(ce->dev, "CE ERROR: %x for flow %x\n", v, flow);
err = -EFAULT;
+ print_hex_dump(KERN_INFO, "TASK: ", DUMP_PREFIX_NONE, 16, 4,
+ cet, sizeof(struct ce_task), false);
}
if (v & CE_ERR_ALGO_NOTSUP)
dev_err(ce->dev, "CE ERROR: algorithm not supported\n");
dev_err(ce->dev, "CE ERROR: keysram access error for AES\n");
if (v & CE_ERR_ADDR_INVALID)
dev_err(ce->dev, "CE ERROR: address invalid\n");
- }
+ if (v & CE_ERR_KEYLADDER)
+ dev_err(ce->dev, "CE ERROR: key ladder configuration error\n");
+ break;
+ }
return err;
}
.decrypt = sun8i_ce_skdecrypt,
}
},
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH
+{ .type = CRYPTO_ALG_TYPE_AHASH,
+ .ce_algo_id = CE_ID_HASH_MD5,
+ .alg.hash = {
+ .init = sun8i_ce_hash_init,
+ .update = sun8i_ce_hash_update,
+ .final = sun8i_ce_hash_final,
+ .finup = sun8i_ce_hash_finup,
+ .digest = sun8i_ce_hash_digest,
+ .export = sun8i_ce_hash_export,
+ .import = sun8i_ce_hash_import,
+ .halg = {
+ .digestsize = MD5_DIGEST_SIZE,
+ .statesize = sizeof(struct md5_state),
+ .base = {
+ .cra_name = "md5",
+ .cra_driver_name = "md5-sun8i-ce",
+ .cra_priority = 300,
+ .cra_alignmask = 3,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ce_hash_crainit,
+ .cra_exit = sun8i_ce_hash_craexit,
+ }
+ }
+ }
+},
+{ .type = CRYPTO_ALG_TYPE_AHASH,
+ .ce_algo_id = CE_ID_HASH_SHA1,
+ .alg.hash = {
+ .init = sun8i_ce_hash_init,
+ .update = sun8i_ce_hash_update,
+ .final = sun8i_ce_hash_final,
+ .finup = sun8i_ce_hash_finup,
+ .digest = sun8i_ce_hash_digest,
+ .export = sun8i_ce_hash_export,
+ .import = sun8i_ce_hash_import,
+ .halg = {
+ .digestsize = SHA1_DIGEST_SIZE,
+ .statesize = sizeof(struct sha1_state),
+ .base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "sha1-sun8i-ce",
+ .cra_priority = 300,
+ .cra_alignmask = 3,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ce_hash_crainit,
+ .cra_exit = sun8i_ce_hash_craexit,
+ }
+ }
+ }
+},
+{ .type = CRYPTO_ALG_TYPE_AHASH,
+ .ce_algo_id = CE_ID_HASH_SHA224,
+ .alg.hash = {
+ .init = sun8i_ce_hash_init,
+ .update = sun8i_ce_hash_update,
+ .final = sun8i_ce_hash_final,
+ .finup = sun8i_ce_hash_finup,
+ .digest = sun8i_ce_hash_digest,
+ .export = sun8i_ce_hash_export,
+ .import = sun8i_ce_hash_import,
+ .halg = {
+ .digestsize = SHA224_DIGEST_SIZE,
+ .statesize = sizeof(struct sha256_state),
+ .base = {
+ .cra_name = "sha224",
+ .cra_driver_name = "sha224-sun8i-ce",
+ .cra_priority = 300,
+ .cra_alignmask = 3,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ce_hash_crainit,
+ .cra_exit = sun8i_ce_hash_craexit,
+ }
+ }
+ }
+},
+{ .type = CRYPTO_ALG_TYPE_AHASH,
+ .ce_algo_id = CE_ID_HASH_SHA256,
+ .alg.hash = {
+ .init = sun8i_ce_hash_init,
+ .update = sun8i_ce_hash_update,
+ .final = sun8i_ce_hash_final,
+ .finup = sun8i_ce_hash_finup,
+ .digest = sun8i_ce_hash_digest,
+ .export = sun8i_ce_hash_export,
+ .import = sun8i_ce_hash_import,
+ .halg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .statesize = sizeof(struct sha256_state),
+ .base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "sha256-sun8i-ce",
+ .cra_priority = 300,
+ .cra_alignmask = 3,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ce_hash_crainit,
+ .cra_exit = sun8i_ce_hash_craexit,
+ }
+ }
+ }
+},
+{ .type = CRYPTO_ALG_TYPE_AHASH,
+ .ce_algo_id = CE_ID_HASH_SHA384,
+ .alg.hash = {
+ .init = sun8i_ce_hash_init,
+ .update = sun8i_ce_hash_update,
+ .final = sun8i_ce_hash_final,
+ .finup = sun8i_ce_hash_finup,
+ .digest = sun8i_ce_hash_digest,
+ .export = sun8i_ce_hash_export,
+ .import = sun8i_ce_hash_import,
+ .halg = {
+ .digestsize = SHA384_DIGEST_SIZE,
+ .statesize = sizeof(struct sha512_state),
+ .base = {
+ .cra_name = "sha384",
+ .cra_driver_name = "sha384-sun8i-ce",
+ .cra_priority = 300,
+ .cra_alignmask = 3,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ce_hash_crainit,
+ .cra_exit = sun8i_ce_hash_craexit,
+ }
+ }
+ }
+},
+{ .type = CRYPTO_ALG_TYPE_AHASH,
+ .ce_algo_id = CE_ID_HASH_SHA512,
+ .alg.hash = {
+ .init = sun8i_ce_hash_init,
+ .update = sun8i_ce_hash_update,
+ .final = sun8i_ce_hash_final,
+ .finup = sun8i_ce_hash_finup,
+ .digest = sun8i_ce_hash_digest,
+ .export = sun8i_ce_hash_export,
+ .import = sun8i_ce_hash_import,
+ .halg = {
+ .digestsize = SHA512_DIGEST_SIZE,
+ .statesize = sizeof(struct sha512_state),
+ .base = {
+ .cra_name = "sha512",
+ .cra_driver_name = "sha512-sun8i-ce",
+ .cra_priority = 300,
+ .cra_alignmask = 3,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sun8i_ce_hash_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ce_hash_crainit,
+ .cra_exit = sun8i_ce_hash_craexit,
+ }
+ }
+ }
+},
+#endif
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG
+{
+ .type = CRYPTO_ALG_TYPE_RNG,
+ .alg.rng = {
+ .base = {
+ .cra_name = "stdrng",
+ .cra_driver_name = "sun8i-ce-prng",
+ .cra_priority = 300,
+ .cra_ctxsize = sizeof(struct sun8i_ce_rng_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ce_prng_init,
+ .cra_exit = sun8i_ce_prng_exit,
+ },
+ .generate = sun8i_ce_prng_generate,
+ .seed = sun8i_ce_prng_seed,
+ .seedsize = PRNG_SEED_SIZE,
+ }
+},
+#endif
};
#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
-static int sun8i_ce_dbgfs_read(struct seq_file *seq, void *v)
+static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v)
{
struct sun8i_ce_dev *ce = seq->private;
- int i;
+ unsigned int i;
for (i = 0; i < MAXFLOW; i++)
seq_printf(seq, "Channel %d: nreq %lu\n", i, ce->chanlist[i].stat_req);
ce_algs[i].alg.skcipher.base.cra_name,
ce_algs[i].stat_req, ce_algs[i].stat_fb);
break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ seq_printf(seq, "%s %s %lu %lu\n",
+ ce_algs[i].alg.hash.halg.base.cra_driver_name,
+ ce_algs[i].alg.hash.halg.base.cra_name,
+ ce_algs[i].stat_req, ce_algs[i].stat_fb);
+ break;
+ case CRYPTO_ALG_TYPE_RNG:
+ seq_printf(seq, "%s %s %lu %lu\n",
+ ce_algs[i].alg.rng.base.cra_driver_name,
+ ce_algs[i].alg.rng.base.cra_name,
+ ce_algs[i].stat_req, ce_algs[i].stat_bytes);
+ break;
}
}
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG
+ seq_printf(seq, "HWRNG %lu %lu\n",
+ ce->hwrng_stat_req, ce->hwrng_stat_bytes);
+#endif
return 0;
}
-static int sun8i_ce_dbgfs_open(struct inode *inode, struct file *file)
-{
- return single_open(file, sun8i_ce_dbgfs_read, inode->i_private);
-}
-
-static const struct file_operations sun8i_ce_debugfs_fops = {
- .owner = THIS_MODULE,
- .open = sun8i_ce_dbgfs_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(sun8i_ce_debugfs);
#endif
static void sun8i_ce_free_chanlist(struct sun8i_ce_dev *ce, int i)
static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce)
{
- int ce_method, err, id, i;
+ int ce_method, err, id;
+ unsigned int i;
for (i = 0; i < ARRAY_SIZE(ce_algs); i++) {
ce_algs[i].ce = ce;
return err;
}
break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ id = ce_algs[i].ce_algo_id;
+ ce_method = ce->variant->alg_hash[id];
+ if (ce_method == CE_ID_NOTSUPP) {
+ dev_info(ce->dev,
+ "DEBUG: Algo of %s not supported\n",
+ ce_algs[i].alg.hash.halg.base.cra_name);
+ ce_algs[i].ce = NULL;
+ break;
+ }
+ dev_info(ce->dev, "Register %s\n",
+ ce_algs[i].alg.hash.halg.base.cra_name);
+ err = crypto_register_ahash(&ce_algs[i].alg.hash);
+ if (err) {
+ dev_err(ce->dev, "ERROR: Fail to register %s\n",
+ ce_algs[i].alg.hash.halg.base.cra_name);
+ ce_algs[i].ce = NULL;
+ return err;
+ }
+ break;
+ case CRYPTO_ALG_TYPE_RNG:
+ if (ce->variant->prng == CE_ID_NOTSUPP) {
+ dev_info(ce->dev,
+ "DEBUG: Algo of %s not supported\n",
+ ce_algs[i].alg.rng.base.cra_name);
+ ce_algs[i].ce = NULL;
+ break;
+ }
+ dev_info(ce->dev, "Register %s\n",
+ ce_algs[i].alg.rng.base.cra_name);
+ err = crypto_register_rng(&ce_algs[i].alg.rng);
+ if (err) {
+ dev_err(ce->dev, "Fail to register %s\n",
+ ce_algs[i].alg.rng.base.cra_name);
+ ce_algs[i].ce = NULL;
+ }
+ break;
default:
ce_algs[i].ce = NULL;
dev_err(ce->dev, "ERROR: tried to register an unknown algo\n");
static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce)
{
- int i;
+ unsigned int i;
for (i = 0; i < ARRAY_SIZE(ce_algs); i++) {
if (!ce_algs[i].ce)
ce_algs[i].alg.skcipher.base.cra_name);
crypto_unregister_skcipher(&ce_algs[i].alg.skcipher);
break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ dev_info(ce->dev, "Unregister %d %s\n", i,
+ ce_algs[i].alg.hash.halg.base.cra_name);
+ crypto_unregister_ahash(&ce_algs[i].alg.hash);
+ break;
+ case CRYPTO_ALG_TYPE_RNG:
+ dev_info(ce->dev, "Unregister %d %s\n", i,
+ ce_algs[i].alg.rng.base.cra_name);
+ crypto_unregister_rng(&ce_algs[i].alg.rng);
+ break;
}
}
}
return irq;
ce->reset = devm_reset_control_get(&pdev->dev, NULL);
- if (IS_ERR(ce->reset)) {
- if (PTR_ERR(ce->reset) == -EPROBE_DEFER)
- return PTR_ERR(ce->reset);
- dev_err(&pdev->dev, "No reset control found\n");
- return PTR_ERR(ce->reset);
- }
+ if (IS_ERR(ce->reset))
+ return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset),
+ "No reset control found\n");
mutex_init(&ce->mlock);
+ mutex_init(&ce->rnglock);
err = sun8i_ce_allocate_chanlist(ce);
if (err)
if (err < 0)
goto error_alg;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG
+ sun8i_ce_hwrng_register(ce);
+#endif
+
v = readl(ce->base + CE_CTR);
v >>= CE_DIE_ID_SHIFT;
v &= CE_DIE_ID_MASK;
{
struct sun8i_ce_dev *ce = platform_get_drvdata(pdev);
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG
+ sun8i_ce_hwrng_unregister(ce);
+#endif
+
sun8i_ce_unregister_algs(ce);
#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * sun8i-ce-hash.c - hardware cryptographic offloader for
+ * Allwinner H3/A64/H5/H2+/H6/R40 SoC
+ *
+ * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>
+ *
+ * This file add support for MD5 and SHA1/SHA224/SHA256/SHA384/SHA512.
+ *
+ * You could find the datasheet in Documentation/arm/sunxi/README
+ */
+#include <linux/dma-mapping.h>
+#include <linux/pm_runtime.h>
+#include <linux/scatterlist.h>
+#include <crypto/internal/hash.h>
+#include <crypto/sha.h>
+#include <crypto/md5.h>
+#include "sun8i-ce.h"
+
+int sun8i_ce_hash_crainit(struct crypto_tfm *tfm)
+{
+ struct sun8i_ce_hash_tfm_ctx *op = crypto_tfm_ctx(tfm);
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg);
+ struct sun8i_ce_alg_template *algt;
+ int err;
+
+ memset(op, 0, sizeof(struct sun8i_ce_hash_tfm_ctx));
+
+ algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash);
+ op->ce = algt->ce;
+
+ op->enginectx.op.do_one_request = sun8i_ce_hash_run;
+ op->enginectx.op.prepare_request = NULL;
+ op->enginectx.op.unprepare_request = NULL;
+
+ /* FALLBACK */
+ op->fallback_tfm = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(op->fallback_tfm)) {
+ dev_err(algt->ce->dev, "Fallback driver could no be loaded\n");
+ return PTR_ERR(op->fallback_tfm);
+ }
+
+ if (algt->alg.hash.halg.statesize < crypto_ahash_statesize(op->fallback_tfm))
+ algt->alg.hash.halg.statesize = crypto_ahash_statesize(op->fallback_tfm);
+
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct sun8i_ce_hash_reqctx) +
+ crypto_ahash_reqsize(op->fallback_tfm));
+
+ dev_info(op->ce->dev, "Fallback for %s is %s\n",
+ crypto_tfm_alg_driver_name(tfm),
+ crypto_tfm_alg_driver_name(&op->fallback_tfm->base));
+ err = pm_runtime_get_sync(op->ce->dev);
+ if (err < 0)
+ goto error_pm;
+ return 0;
+error_pm:
+ pm_runtime_put_noidle(op->ce->dev);
+ crypto_free_ahash(op->fallback_tfm);
+ return err;
+}
+
+void sun8i_ce_hash_craexit(struct crypto_tfm *tfm)
+{
+ struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_tfm_ctx(tfm);
+
+ crypto_free_ahash(tfmctx->fallback_tfm);
+ pm_runtime_put_sync_suspend(tfmctx->ce->dev);
+}
+
+int sun8i_ce_hash_init(struct ahash_request *areq)
+{
+ struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+
+ memset(rctx, 0, sizeof(struct sun8i_ce_hash_reqctx));
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ return crypto_ahash_init(&rctx->fallback_req);
+}
+
+int sun8i_ce_hash_export(struct ahash_request *areq, void *out)
+{
+ struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ return crypto_ahash_export(&rctx->fallback_req, out);
+}
+
+int sun8i_ce_hash_import(struct ahash_request *areq, const void *in)
+{
+ struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ return crypto_ahash_import(&rctx->fallback_req, in);
+}
+
+int sun8i_ce_hash_final(struct ahash_request *areq)
+{
+ struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
+ struct sun8i_ce_alg_template *algt;
+#endif
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags &
+ CRYPTO_TFM_REQ_MAY_SLEEP;
+ rctx->fallback_req.result = areq->result;
+
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
+ algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash);
+ algt->stat_fb++;
+#endif
+
+ return crypto_ahash_final(&rctx->fallback_req);
+}
+
+int sun8i_ce_hash_update(struct ahash_request *areq)
+{
+ struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags &
+ CRYPTO_TFM_REQ_MAY_SLEEP;
+ rctx->fallback_req.nbytes = areq->nbytes;
+ rctx->fallback_req.src = areq->src;
+
+ return crypto_ahash_update(&rctx->fallback_req);
+}
+
+int sun8i_ce_hash_finup(struct ahash_request *areq)
+{
+ struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
+ struct sun8i_ce_alg_template *algt;
+#endif
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags &
+ CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ rctx->fallback_req.nbytes = areq->nbytes;
+ rctx->fallback_req.src = areq->src;
+ rctx->fallback_req.result = areq->result;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
+ algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash);
+ algt->stat_fb++;
+#endif
+
+ return crypto_ahash_finup(&rctx->fallback_req);
+}
+
+static int sun8i_ce_hash_digest_fb(struct ahash_request *areq)
+{
+ struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ce_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
+ struct sun8i_ce_alg_template *algt;
+#endif
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags &
+ CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ rctx->fallback_req.nbytes = areq->nbytes;
+ rctx->fallback_req.src = areq->src;
+ rctx->fallback_req.result = areq->result;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
+ algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash);
+ algt->stat_fb++;
+#endif
+
+ return crypto_ahash_digest(&rctx->fallback_req);
+}
+
+static bool sun8i_ce_hash_need_fallback(struct ahash_request *areq)
+{
+ struct scatterlist *sg;
+
+ if (areq->nbytes == 0)
+ return true;
+ /* we need to reserve one SG for padding one */
+ if (sg_nents(areq->src) > MAX_SG - 1)
+ return true;
+ sg = areq->src;
+ while (sg) {
+ if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32)))
+ return true;
+ sg = sg_next(sg);
+ }
+ return false;
+}
+
+int sun8i_ce_hash_digest(struct ahash_request *areq)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
+ struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct sun8i_ce_alg_template *algt;
+ struct sun8i_ce_dev *ce;
+ struct crypto_engine *engine;
+ struct scatterlist *sg;
+ int nr_sgs, e, i;
+
+ if (sun8i_ce_hash_need_fallback(areq))
+ return sun8i_ce_hash_digest_fb(areq);
+
+ nr_sgs = sg_nents(areq->src);
+ if (nr_sgs > MAX_SG - 1)
+ return sun8i_ce_hash_digest_fb(areq);
+
+ for_each_sg(areq->src, sg, nr_sgs, i) {
+ if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32)))
+ return sun8i_ce_hash_digest_fb(areq);
+ }
+
+ algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash);
+ ce = algt->ce;
+
+ e = sun8i_ce_get_engine_number(ce);
+ rctx->flow = e;
+ engine = ce->chanlist[e].engine;
+
+ return crypto_transfer_hash_request_to_engine(engine, areq);
+}
+
+int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq)
+{
+ struct ahash_request *areq = container_of(breq, struct ahash_request, base);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
+ struct sun8i_ce_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct sun8i_ce_alg_template *algt;
+ struct sun8i_ce_dev *ce;
+ struct sun8i_ce_flow *chan;
+ struct ce_task *cet;
+ struct scatterlist *sg;
+ int nr_sgs, flow, err;
+ unsigned int len;
+ u32 common;
+ u64 byte_count;
+ __le32 *bf;
+ void *buf;
+ int j, i, todo;
+ int nbw = 0;
+ u64 fill, min_fill;
+ __be64 *bebits;
+ __le64 *lebits;
+ void *result;
+ u64 bs;
+ int digestsize;
+ dma_addr_t addr_res, addr_pad;
+
+ algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash);
+ ce = algt->ce;
+
+ bs = algt->alg.hash.halg.base.cra_blocksize;
+ digestsize = algt->alg.hash.halg.digestsize;
+ if (digestsize == SHA224_DIGEST_SIZE)
+ digestsize = SHA256_DIGEST_SIZE;
+ if (digestsize == SHA384_DIGEST_SIZE)
+ digestsize = SHA512_DIGEST_SIZE;
+
+ /* the padding could be up to two block. */
+ buf = kzalloc(bs * 2, GFP_KERNEL | GFP_DMA);
+ if (!buf)
+ return -ENOMEM;
+ bf = (__le32 *)buf;
+
+ result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA);
+ if (!result)
+ return -ENOMEM;
+
+ flow = rctx->flow;
+ chan = &ce->chanlist[flow];
+
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
+ algt->stat_req++;
+#endif
+ dev_dbg(ce->dev, "%s %s len=%d\n", __func__, crypto_tfm_alg_name(areq->base.tfm), areq->nbytes);
+
+ cet = chan->tl;
+ memset(cet, 0, sizeof(struct ce_task));
+
+ cet->t_id = cpu_to_le32(flow);
+ common = ce->variant->alg_hash[algt->ce_algo_id];
+ common |= CE_COMM_INT;
+ cet->t_common_ctl = cpu_to_le32(common);
+
+ cet->t_sym_ctl = 0;
+ cet->t_asym_ctl = 0;
+
+ nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE);
+ if (nr_sgs <= 0 || nr_sgs > MAX_SG) {
+ dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);
+ err = -EINVAL;
+ goto theend;
+ }
+
+ len = areq->nbytes;
+ for_each_sg(areq->src, sg, nr_sgs, i) {
+ cet->t_src[i].addr = cpu_to_le32(sg_dma_address(sg));
+ todo = min(len, sg_dma_len(sg));
+ cet->t_src[i].len = cpu_to_le32(todo / 4);
+ len -= todo;
+ }
+ if (len > 0) {
+ dev_err(ce->dev, "remaining len %d\n", len);
+ err = -EINVAL;
+ goto theend;
+ }
+ addr_res = dma_map_single(ce->dev, result, digestsize, DMA_FROM_DEVICE);
+ cet->t_dst[0].addr = cpu_to_le32(addr_res);
+ cet->t_dst[0].len = cpu_to_le32(digestsize / 4);
+ if (dma_mapping_error(ce->dev, addr_res)) {
+ dev_err(ce->dev, "DMA map dest\n");
+ err = -EINVAL;
+ goto theend;
+ }
+
+ byte_count = areq->nbytes;
+ j = 0;
+ bf[j++] = cpu_to_le32(0x80);
+
+ if (bs == 64) {
+ fill = 64 - (byte_count % 64);
+ min_fill = 2 * sizeof(u32) + (nbw ? 0 : sizeof(u32));
+ } else {
+ fill = 128 - (byte_count % 128);
+ min_fill = 4 * sizeof(u32) + (nbw ? 0 : sizeof(u32));
+ }
+
+ if (fill < min_fill)
+ fill += bs;
+
+ j += (fill - min_fill) / sizeof(u32);
+
+ switch (algt->ce_algo_id) {
+ case CE_ID_HASH_MD5:
+ lebits = (__le64 *)&bf[j];
+ *lebits = cpu_to_le64(byte_count << 3);
+ j += 2;
+ break;
+ case CE_ID_HASH_SHA1:
+ case CE_ID_HASH_SHA224:
+ case CE_ID_HASH_SHA256:
+ bebits = (__be64 *)&bf[j];
+ *bebits = cpu_to_be64(byte_count << 3);
+ j += 2;
+ break;
+ case CE_ID_HASH_SHA384:
+ case CE_ID_HASH_SHA512:
+ bebits = (__be64 *)&bf[j];
+ *bebits = cpu_to_be64(byte_count >> 61);
+ j += 2;
+ bebits = (__be64 *)&bf[j];
+ *bebits = cpu_to_be64(byte_count << 3);
+ j += 2;
+ break;
+ }
+
+ addr_pad = dma_map_single(ce->dev, buf, j * 4, DMA_TO_DEVICE);
+ cet->t_src[i].addr = cpu_to_le32(addr_pad);
+ cet->t_src[i].len = cpu_to_le32(j);
+ if (dma_mapping_error(ce->dev, addr_pad)) {
+ dev_err(ce->dev, "DMA error on padding SG\n");
+ err = -EINVAL;
+ goto theend;
+ }
+
+ if (ce->variant->hash_t_dlen_in_bits)
+ cet->t_dlen = cpu_to_le32((areq->nbytes + j * 4) * 8);
+ else
+ cet->t_dlen = cpu_to_le32(areq->nbytes / 4 + j);
+
+ chan->timeout = areq->nbytes;
+
+ err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(areq->base.tfm));
+
+ dma_unmap_single(ce->dev, addr_pad, j * 4, DMA_TO_DEVICE);
+ dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE);
+ dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE);
+
+ kfree(buf);
+
+ memcpy(areq->result, result, algt->alg.hash.halg.digestsize);
+ kfree(result);
+theend:
+ crypto_finalize_hash_request(engine, breq, err);
+ return 0;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * sun8i-ce-prng.c - hardware cryptographic offloader for
+ * Allwinner H3/A64/H5/H2+/H6/R40 SoC
+ *
+ * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>
+ *
+ * This file handle the PRNG
+ *
+ * You could find a link for the datasheet in Documentation/arm/sunxi/README
+ */
+#include "sun8i-ce.h"
+#include <linux/dma-mapping.h>
+#include <linux/pm_runtime.h>
+#include <crypto/internal/rng.h>
+
+int sun8i_ce_prng_init(struct crypto_tfm *tfm)
+{
+ struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ memset(ctx, 0, sizeof(struct sun8i_ce_rng_tfm_ctx));
+ return 0;
+}
+
+void sun8i_ce_prng_exit(struct crypto_tfm *tfm)
+{
+ struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ memzero_explicit(ctx->seed, ctx->slen);
+ kfree(ctx->seed);
+ ctx->seed = NULL;
+ ctx->slen = 0;
+}
+
+int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed,
+ unsigned int slen)
+{
+ struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm);
+
+ if (ctx->seed && ctx->slen != slen) {
+ memzero_explicit(ctx->seed, ctx->slen);
+ kfree(ctx->seed);
+ ctx->slen = 0;
+ ctx->seed = NULL;
+ }
+ if (!ctx->seed)
+ ctx->seed = kmalloc(slen, GFP_KERNEL | GFP_DMA);
+ if (!ctx->seed)
+ return -ENOMEM;
+
+ memcpy(ctx->seed, seed, slen);
+ ctx->slen = slen;
+
+ return 0;
+}
+
+int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src,
+ unsigned int slen, u8 *dst, unsigned int dlen)
+{
+ struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm);
+ struct rng_alg *alg = crypto_rng_alg(tfm);
+ struct sun8i_ce_alg_template *algt;
+ struct sun8i_ce_dev *ce;
+ dma_addr_t dma_iv, dma_dst;
+ int err = 0;
+ int flow = 3;
+ unsigned int todo;
+ struct sun8i_ce_flow *chan;
+ struct ce_task *cet;
+ u32 common, sym;
+ void *d;
+
+ algt = container_of(alg, struct sun8i_ce_alg_template, alg.rng);
+ ce = algt->ce;
+
+ if (ctx->slen == 0) {
+ dev_err(ce->dev, "not seeded\n");
+ return -EINVAL;
+ }
+
+ /* we want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE */
+ todo = dlen + ctx->slen + PRNG_DATA_SIZE * 2;
+ todo -= todo % PRNG_DATA_SIZE;
+
+ d = kzalloc(todo, GFP_KERNEL | GFP_DMA);
+ if (!d) {
+ err = -ENOMEM;
+ goto err_mem;
+ }
+
+ dev_dbg(ce->dev, "%s PRNG slen=%u dlen=%u todo=%u multi=%u\n", __func__,
+ slen, dlen, todo, todo / PRNG_DATA_SIZE);
+
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
+ algt->stat_req++;
+ algt->stat_bytes += todo;
+#endif
+
+ dma_iv = dma_map_single(ce->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE);
+ if (dma_mapping_error(ce->dev, dma_iv)) {
+ dev_err(ce->dev, "Cannot DMA MAP IV\n");
+ goto err_iv;
+ }
+
+ dma_dst = dma_map_single(ce->dev, d, todo, DMA_FROM_DEVICE);
+ if (dma_mapping_error(ce->dev, dma_dst)) {
+ dev_err(ce->dev, "Cannot DMA MAP DST\n");
+ err = -EFAULT;
+ goto err_dst;
+ }
+
+ err = pm_runtime_get_sync(ce->dev);
+ if (err < 0) {
+ pm_runtime_put_noidle(ce->dev);
+ goto err_pm;
+ }
+
+ mutex_lock(&ce->rnglock);
+ chan = &ce->chanlist[flow];
+
+ cet = &chan->tl[0];
+ memset(cet, 0, sizeof(struct ce_task));
+
+ cet->t_id = cpu_to_le32(flow);
+ common = ce->variant->prng | CE_COMM_INT;
+ cet->t_common_ctl = cpu_to_le32(common);
+
+ /* recent CE (H6) need length in bytes, in word otherwise */
+ if (ce->variant->prng_t_dlen_in_bytes)
+ cet->t_dlen = cpu_to_le32(todo);
+ else
+ cet->t_dlen = cpu_to_le32(todo / 4);
+
+ sym = PRNG_LD;
+ cet->t_sym_ctl = cpu_to_le32(sym);
+ cet->t_asym_ctl = 0;
+
+ cet->t_key = cpu_to_le32(dma_iv);
+ cet->t_iv = cpu_to_le32(dma_iv);
+
+ cet->t_dst[0].addr = cpu_to_le32(dma_dst);
+ cet->t_dst[0].len = cpu_to_le32(todo / 4);
+ ce->chanlist[flow].timeout = 2000;
+
+ err = sun8i_ce_run_task(ce, 3, "PRNG");
+ mutex_unlock(&ce->rnglock);
+
+ pm_runtime_put(ce->dev);
+
+err_pm:
+ dma_unmap_single(ce->dev, dma_dst, todo, DMA_FROM_DEVICE);
+err_dst:
+ dma_unmap_single(ce->dev, dma_iv, ctx->slen, DMA_TO_DEVICE);
+
+ if (!err) {
+ memcpy(dst, d, dlen);
+ memcpy(ctx->seed, d + dlen, ctx->slen);
+ }
+ memzero_explicit(d, todo);
+err_iv:
+ kfree(d);
+err_mem:
+ return err;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * sun8i-ce-trng.c - hardware cryptographic offloader for
+ * Allwinner H3/A64/H5/H2+/H6/R40 SoC
+ *
+ * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>
+ *
+ * This file handle the TRNG
+ *
+ * You could find a link for the datasheet in Documentation/arm/sunxi/README
+ */
+#include "sun8i-ce.h"
+#include <linux/dma-mapping.h>
+#include <linux/pm_runtime.h>
+#include <linux/hw_random.h>
+/*
+ * Note that according to the algorithm ID, 2 versions of the TRNG exists,
+ * The first present in H3/H5/R40/A64 and the second present in H6.
+ * This file adds support for both, but only the second is working
+ * reliabily according to rngtest.
+ **/
+
+static int sun8i_ce_trng_read(struct hwrng *rng, void *data, size_t max, bool wait)
+{
+ struct sun8i_ce_dev *ce;
+ dma_addr_t dma_dst;
+ int err = 0;
+ int flow = 3;
+ unsigned int todo;
+ struct sun8i_ce_flow *chan;
+ struct ce_task *cet;
+ u32 common;
+ void *d;
+
+ ce = container_of(rng, struct sun8i_ce_dev, trng);
+
+ /* round the data length to a multiple of 32*/
+ todo = max + 32;
+ todo -= todo % 32;
+
+ d = kzalloc(todo, GFP_KERNEL | GFP_DMA);
+ if (!d)
+ return -ENOMEM;
+
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
+ ce->hwrng_stat_req++;
+ ce->hwrng_stat_bytes += todo;
+#endif
+
+ dma_dst = dma_map_single(ce->dev, d, todo, DMA_FROM_DEVICE);
+ if (dma_mapping_error(ce->dev, dma_dst)) {
+ dev_err(ce->dev, "Cannot DMA MAP DST\n");
+ err = -EFAULT;
+ goto err_dst;
+ }
+
+ err = pm_runtime_get_sync(ce->dev);
+ if (err < 0) {
+ pm_runtime_put_noidle(ce->dev);
+ goto err_pm;
+ }
+
+ mutex_lock(&ce->rnglock);
+ chan = &ce->chanlist[flow];
+
+ cet = &chan->tl[0];
+ memset(cet, 0, sizeof(struct ce_task));
+
+ cet->t_id = cpu_to_le32(flow);
+ common = ce->variant->trng | CE_COMM_INT;
+ cet->t_common_ctl = cpu_to_le32(common);
+
+ /* recent CE (H6) need length in bytes, in word otherwise */
+ if (ce->variant->trng_t_dlen_in_bytes)
+ cet->t_dlen = cpu_to_le32(todo);
+ else
+ cet->t_dlen = cpu_to_le32(todo / 4);
+
+ cet->t_sym_ctl = 0;
+ cet->t_asym_ctl = 0;
+
+ cet->t_dst[0].addr = cpu_to_le32(dma_dst);
+ cet->t_dst[0].len = cpu_to_le32(todo / 4);
+ ce->chanlist[flow].timeout = todo;
+
+ err = sun8i_ce_run_task(ce, 3, "TRNG");
+ mutex_unlock(&ce->rnglock);
+
+ pm_runtime_put(ce->dev);
+
+err_pm:
+ dma_unmap_single(ce->dev, dma_dst, todo, DMA_FROM_DEVICE);
+
+ if (!err) {
+ memcpy(data, d, max);
+ err = max;
+ }
+ memzero_explicit(d, todo);
+err_dst:
+ kfree(d);
+ return err;
+}
+
+int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce)
+{
+ int ret;
+
+ if (ce->variant->trng == CE_ID_NOTSUPP) {
+ dev_info(ce->dev, "TRNG not supported\n");
+ return 0;
+ }
+ ce->trng.name = "sun8i Crypto Engine TRNG";
+ ce->trng.read = sun8i_ce_trng_read;
+ ce->trng.quality = 1000;
+
+ ret = hwrng_register(&ce->trng);
+ if (ret)
+ dev_err(ce->dev, "Fail to register the TRNG\n");
+ return ret;
+}
+
+void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce)
+{
+ if (ce->variant->trng == CE_ID_NOTSUPP)
+ return;
+ hwrng_unregister(&ce->trng);
+}
#include <linux/atomic.h>
#include <linux/debugfs.h>
#include <linux/crypto.h>
+#include <linux/hw_random.h>
+#include <crypto/internal/hash.h>
+#include <crypto/md5.h>
+#include <crypto/rng.h>
+#include <crypto/sha.h>
/* CE Registers */
#define CE_TDQ 0x00
#define CE_ALG_AES 0
#define CE_ALG_DES 1
#define CE_ALG_3DES 2
+#define CE_ALG_MD5 16
+#define CE_ALG_SHA1 17
+#define CE_ALG_SHA224 18
+#define CE_ALG_SHA256 19
+#define CE_ALG_SHA384 20
+#define CE_ALG_SHA512 21
+#define CE_ALG_TRNG 48
+#define CE_ALG_PRNG 49
+#define CE_ALG_TRNG_V2 0x1c
+#define CE_ALG_PRNG_V2 0x1d
/* Used in ce_variant */
#define CE_ID_NOTSUPP 0xFF
#define CE_ID_CIPHER_DES3 2
#define CE_ID_CIPHER_MAX 3
+#define CE_ID_HASH_MD5 0
+#define CE_ID_HASH_SHA1 1
+#define CE_ID_HASH_SHA224 2
+#define CE_ID_HASH_SHA256 3
+#define CE_ID_HASH_SHA384 4
+#define CE_ID_HASH_SHA512 5
+#define CE_ID_HASH_MAX 6
+
#define CE_ID_OP_ECB 0
#define CE_ID_OP_CBC 1
#define CE_ID_OP_MAX 2
#define CE_ERR_ADDR_INVALID BIT(5)
#define CE_ERR_KEYLADDER BIT(6)
+#define ESR_H3 0
+#define ESR_A64 1
+#define ESR_R40 2
+#define ESR_H5 3
+#define ESR_H6 4
+
+#define PRNG_DATA_SIZE (160 / 8)
+#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8)
+#define PRNG_LD BIT(17)
+
#define CE_DIE_ID_SHIFT 16
#define CE_DIE_ID_MASK 0x07
* struct ce_variant - Describe CE capability for each variant hardware
* @alg_cipher: list of supported ciphers. for each CE_ID_ this will give the
* coresponding CE_ALG_XXX value
+ * @alg_hash: list of supported hashes. for each CE_ID_ this will give the
+ * corresponding CE_ALG_XXX value
* @op_mode: list of supported block modes
- * @has_t_dlen_in_bytes: Does the request size for cipher is in
+ * @cipher_t_dlen_in_bytes: Does the request size for cipher is in
+ * bytes or words
+ * @hash_t_dlen_in_bytes: Does the request size for hash is in
+ * bits or words
+ * @prng_t_dlen_in_bytes: Does the request size for PRNG is in
+ * bytes or words
+ * @trng_t_dlen_in_bytes: Does the request size for TRNG is in
* bytes or words
* @ce_clks: list of clocks needed by this variant
+ * @esr: The type of error register
+ * @prng: The CE_ALG_XXX value for the PRNG
+ * @trng: The CE_ALG_XXX value for the TRNG
*/
struct ce_variant {
char alg_cipher[CE_ID_CIPHER_MAX];
+ char alg_hash[CE_ID_HASH_MAX];
u32 op_mode[CE_ID_OP_MAX];
- bool has_t_dlen_in_bytes;
+ bool cipher_t_dlen_in_bytes;
+ bool hash_t_dlen_in_bits;
+ bool prng_t_dlen_in_bytes;
+ bool trng_t_dlen_in_bytes;
struct ce_clock ce_clks[CE_MAX_CLOCKS];
+ int esr;
+ unsigned char prng;
+ unsigned char trng;
};
struct sginfo {
/*
* struct sun8i_ce_flow - Information used by each flow
* @engine: ptr to the crypto_engine for this flow
- * @bounce_iv: buffer which contain the IV
- * @ivlen: size of bounce_iv
* @complete: completion for the current task on this flow
* @status: set to 1 by interrupt if task is done
* @t_phy: Physical address of task
*/
struct sun8i_ce_flow {
struct crypto_engine *engine;
- void *bounce_iv;
- unsigned int ivlen;
struct completion complete;
int status;
dma_addr_t t_phy;
* @reset: pointer to reset controller
* @dev: the platform device
* @mlock: Control access to device registers
+ * @rnglock: Control access to the RNG (dedicated channel 3)
* @chanlist: array of all flow
* @flow: flow to use in next request
* @variant: pointer to variant specific data
struct reset_control *reset;
struct device *dev;
struct mutex mlock;
+ struct mutex rnglock;
struct sun8i_ce_flow *chanlist;
atomic_t flow;
const struct ce_variant *variant;
struct dentry *dbgfs_dir;
struct dentry *dbgfs_stats;
#endif
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG
+ struct hwrng trng;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
+ unsigned long hwrng_stat_req;
+ unsigned long hwrng_stat_bytes;
+#endif
+#endif
};
/*
* struct sun8i_cipher_req_ctx - context for a skcipher request
* @op_dir: direction (encrypt vs decrypt) for this request
* @flow: the flow to use for this request
+ * @backup_iv: buffer which contain the next IV to store
+ * @bounce_iv: buffer which contain the IV
+ * @ivlen: size of bounce_iv
+ * @nr_sgs: The number of source SG (as given by dma_map_sg())
+ * @nr_sgd: The number of destination SG (as given by dma_map_sg())
+ * @addr_iv: The IV addr returned by dma_map_single, need to unmap later
+ * @addr_key: The key addr returned by dma_map_single, need to unmap later
* @fallback_req: request struct for invoking the fallback skcipher TFM
*/
struct sun8i_cipher_req_ctx {
u32 op_dir;
int flow;
+ void *backup_iv;
+ void *bounce_iv;
+ unsigned int ivlen;
+ int nr_sgs;
+ int nr_sgd;
+ dma_addr_t addr_iv;
+ dma_addr_t addr_key;
struct skcipher_request fallback_req; // keep at the end
};
struct crypto_skcipher *fallback_tfm;
};
+/*
+ * struct sun8i_ce_hash_tfm_ctx - context for an ahash TFM
+ * @enginectx: crypto_engine used by this TFM
+ * @ce: pointer to the private data of driver handling this TFM
+ * @fallback_tfm: pointer to the fallback TFM
+ */
+struct sun8i_ce_hash_tfm_ctx {
+ struct crypto_engine_ctx enginectx;
+ struct sun8i_ce_dev *ce;
+ struct crypto_ahash *fallback_tfm;
+};
+
+/*
+ * struct sun8i_ce_hash_reqctx - context for an ahash request
+ * @fallback_req: pre-allocated fallback request
+ * @flow: the flow to use for this request
+ */
+struct sun8i_ce_hash_reqctx {
+ struct ahash_request fallback_req;
+ int flow;
+};
+
+/*
+ * struct sun8i_ce_prng_ctx - context for PRNG TFM
+ * @seed: The seed to use
+ * @slen: The size of the seed
+ */
+struct sun8i_ce_rng_tfm_ctx {
+ void *seed;
+ unsigned int slen;
+};
+
/*
* struct sun8i_ce_alg_template - crypto_alg template
* @type: the CRYPTO_ALG_TYPE for this template
* @alg: one of sub struct must be used
* @stat_req: number of request done on this template
* @stat_fb: number of request which has fallbacked
+ * @stat_bytes: total data size done by this template
*/
struct sun8i_ce_alg_template {
u32 type;
struct sun8i_ce_dev *ce;
union {
struct skcipher_alg skcipher;
+ struct ahash_alg hash;
+ struct rng_alg rng;
} alg;
#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
unsigned long stat_req;
unsigned long stat_fb;
+ unsigned long stat_bytes;
#endif
};
int sun8i_ce_get_engine_number(struct sun8i_ce_dev *ce);
int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name);
+
+int sun8i_ce_hash_crainit(struct crypto_tfm *tfm);
+void sun8i_ce_hash_craexit(struct crypto_tfm *tfm);
+int sun8i_ce_hash_init(struct ahash_request *areq);
+int sun8i_ce_hash_export(struct ahash_request *areq, void *out);
+int sun8i_ce_hash_import(struct ahash_request *areq, const void *in);
+int sun8i_ce_hash(struct ahash_request *areq);
+int sun8i_ce_hash_final(struct ahash_request *areq);
+int sun8i_ce_hash_update(struct ahash_request *areq);
+int sun8i_ce_hash_finup(struct ahash_request *areq);
+int sun8i_ce_hash_digest(struct ahash_request *areq);
+int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq);
+
+int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src,
+ unsigned int slen, u8 *dst, unsigned int dlen);
+int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen);
+void sun8i_ce_prng_exit(struct crypto_tfm *tfm);
+int sun8i_ce_prng_init(struct crypto_tfm *tfm);
+
+int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce);
+void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce);
obj-$(CONFIG_CRYPTO_DEV_SUN8I_SS) += sun8i-ss.o
sun8i-ss-y += sun8i-ss-core.o sun8i-ss-cipher.o
+sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG) += sun8i-ss-prng.o
+sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_HASH) += sun8i-ss-hash.o
offset = areq->cryptlen - ivsize;
if (rctx->op_dir & SS_DECRYPTION) {
memcpy(areq->iv, backup_iv, ivsize);
- memzero_explicit(backup_iv, ivsize);
kfree_sensitive(backup_iv);
} else {
scatterwalk_map_and_copy(areq->iv, areq->dst, offset,
{
struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
- if (op->key) {
- memzero_explicit(op->key, op->keylen);
- kfree(op->key);
- }
+ kfree_sensitive(op->key);
crypto_free_skcipher(op->fallback_tfm);
pm_runtime_put_sync(op->ss->dev);
}
dev_dbg(ss->dev, "ERROR: Invalid keylen %u\n", keylen);
return -EINVAL;
}
- if (op->key) {
- memzero_explicit(op->key, op->keylen);
- kfree(op->key);
- }
+ kfree_sensitive(op->key);
op->keylen = keylen;
op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
if (!op->key)
return -EINVAL;
}
- if (op->key) {
- memzero_explicit(op->key, op->keylen);
- kfree(op->key);
- }
+ kfree_sensitive(op->key);
op->keylen = keylen;
op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
if (!op->key)
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/reset.h>
+#include <crypto/internal/rng.h>
#include <crypto/internal/skcipher.h>
#include "sun8i-ss.h"
static const struct ss_variant ss_a83t_variant = {
.alg_cipher = { SS_ALG_AES, SS_ALG_DES, SS_ALG_3DES,
},
+ .alg_hash = { SS_ALG_MD5, SS_ALG_SHA1, SS_ALG_SHA224, SS_ALG_SHA256,
+ },
.op_mode = { SS_OP_ECB, SS_OP_CBC,
},
.ss_clks = {
const char *name)
{
int flow = rctx->flow;
- u32 v = 1;
+ u32 v = SS_START;
int i;
#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
.decrypt = sun8i_ss_skdecrypt,
}
},
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG
+{
+ .type = CRYPTO_ALG_TYPE_RNG,
+ .alg.rng = {
+ .base = {
+ .cra_name = "stdrng",
+ .cra_driver_name = "sun8i-ss-prng",
+ .cra_priority = 300,
+ .cra_ctxsize = sizeof(struct sun8i_ss_rng_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ss_prng_init,
+ .cra_exit = sun8i_ss_prng_exit,
+ },
+ .generate = sun8i_ss_prng_generate,
+ .seed = sun8i_ss_prng_seed,
+ .seedsize = PRNG_SEED_SIZE,
+ }
+},
+#endif
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH
+{ .type = CRYPTO_ALG_TYPE_AHASH,
+ .ss_algo_id = SS_ID_HASH_MD5,
+ .alg.hash = {
+ .init = sun8i_ss_hash_init,
+ .update = sun8i_ss_hash_update,
+ .final = sun8i_ss_hash_final,
+ .finup = sun8i_ss_hash_finup,
+ .digest = sun8i_ss_hash_digest,
+ .export = sun8i_ss_hash_export,
+ .import = sun8i_ss_hash_import,
+ .halg = {
+ .digestsize = MD5_DIGEST_SIZE,
+ .statesize = sizeof(struct md5_state),
+ .base = {
+ .cra_name = "md5",
+ .cra_driver_name = "md5-sun8i-ss",
+ .cra_priority = 300,
+ .cra_alignmask = 3,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ss_hash_crainit,
+ .cra_exit = sun8i_ss_hash_craexit,
+ }
+ }
+ }
+},
+{ .type = CRYPTO_ALG_TYPE_AHASH,
+ .ss_algo_id = SS_ID_HASH_SHA1,
+ .alg.hash = {
+ .init = sun8i_ss_hash_init,
+ .update = sun8i_ss_hash_update,
+ .final = sun8i_ss_hash_final,
+ .finup = sun8i_ss_hash_finup,
+ .digest = sun8i_ss_hash_digest,
+ .export = sun8i_ss_hash_export,
+ .import = sun8i_ss_hash_import,
+ .halg = {
+ .digestsize = SHA1_DIGEST_SIZE,
+ .statesize = sizeof(struct sha1_state),
+ .base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "sha1-sun8i-ss",
+ .cra_priority = 300,
+ .cra_alignmask = 3,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ss_hash_crainit,
+ .cra_exit = sun8i_ss_hash_craexit,
+ }
+ }
+ }
+},
+{ .type = CRYPTO_ALG_TYPE_AHASH,
+ .ss_algo_id = SS_ID_HASH_SHA224,
+ .alg.hash = {
+ .init = sun8i_ss_hash_init,
+ .update = sun8i_ss_hash_update,
+ .final = sun8i_ss_hash_final,
+ .finup = sun8i_ss_hash_finup,
+ .digest = sun8i_ss_hash_digest,
+ .export = sun8i_ss_hash_export,
+ .import = sun8i_ss_hash_import,
+ .halg = {
+ .digestsize = SHA224_DIGEST_SIZE,
+ .statesize = sizeof(struct sha256_state),
+ .base = {
+ .cra_name = "sha224",
+ .cra_driver_name = "sha224-sun8i-ss",
+ .cra_priority = 300,
+ .cra_alignmask = 3,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ss_hash_crainit,
+ .cra_exit = sun8i_ss_hash_craexit,
+ }
+ }
+ }
+},
+{ .type = CRYPTO_ALG_TYPE_AHASH,
+ .ss_algo_id = SS_ID_HASH_SHA256,
+ .alg.hash = {
+ .init = sun8i_ss_hash_init,
+ .update = sun8i_ss_hash_update,
+ .final = sun8i_ss_hash_final,
+ .finup = sun8i_ss_hash_finup,
+ .digest = sun8i_ss_hash_digest,
+ .export = sun8i_ss_hash_export,
+ .import = sun8i_ss_hash_import,
+ .halg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .statesize = sizeof(struct sha256_state),
+ .base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "sha256-sun8i-ss",
+ .cra_priority = 300,
+ .cra_alignmask = 3,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct sun8i_ss_hash_tfm_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = sun8i_ss_hash_crainit,
+ .cra_exit = sun8i_ss_hash_craexit,
+ }
+ }
+ }
+},
+#endif
};
#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
-static int sun8i_ss_dbgfs_read(struct seq_file *seq, void *v)
+static int sun8i_ss_debugfs_show(struct seq_file *seq, void *v)
{
struct sun8i_ss_dev *ss = seq->private;
- int i;
+ unsigned int i;
for (i = 0; i < MAXFLOW; i++)
seq_printf(seq, "Channel %d: nreq %lu\n", i, ss->flows[i].stat_req);
continue;
switch (ss_algs[i].type) {
case CRYPTO_ALG_TYPE_SKCIPHER:
- seq_printf(seq, "%s %s %lu %lu\n",
+ seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
ss_algs[i].alg.skcipher.base.cra_driver_name,
ss_algs[i].alg.skcipher.base.cra_name,
ss_algs[i].stat_req, ss_algs[i].stat_fb);
break;
+ case CRYPTO_ALG_TYPE_RNG:
+ seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n",
+ ss_algs[i].alg.rng.base.cra_driver_name,
+ ss_algs[i].alg.rng.base.cra_name,
+ ss_algs[i].stat_req, ss_algs[i].stat_bytes);
+ break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
+ ss_algs[i].alg.hash.halg.base.cra_driver_name,
+ ss_algs[i].alg.hash.halg.base.cra_name,
+ ss_algs[i].stat_req, ss_algs[i].stat_fb);
+ break;
}
}
return 0;
}
-static int sun8i_ss_dbgfs_open(struct inode *inode, struct file *file)
-{
- return single_open(file, sun8i_ss_dbgfs_read, inode->i_private);
-}
-
-static const struct file_operations sun8i_ss_debugfs_fops = {
- .owner = THIS_MODULE,
- .open = sun8i_ss_dbgfs_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(sun8i_ss_debugfs);
#endif
static void sun8i_ss_free_flows(struct sun8i_ss_dev *ss, int i)
static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss)
{
- int ss_method, err, id, i;
+ int ss_method, err, id;
+ unsigned int i;
for (i = 0; i < ARRAY_SIZE(ss_algs); i++) {
ss_algs[i].ss = ss;
return err;
}
break;
+ case CRYPTO_ALG_TYPE_RNG:
+ err = crypto_register_rng(&ss_algs[i].alg.rng);
+ if (err) {
+ dev_err(ss->dev, "Fail to register %s\n",
+ ss_algs[i].alg.rng.base.cra_name);
+ ss_algs[i].ss = NULL;
+ }
+ break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ id = ss_algs[i].ss_algo_id;
+ ss_method = ss->variant->alg_hash[id];
+ if (ss_method == SS_ID_NOTSUPP) {
+ dev_info(ss->dev,
+ "DEBUG: Algo of %s not supported\n",
+ ss_algs[i].alg.hash.halg.base.cra_name);
+ ss_algs[i].ss = NULL;
+ break;
+ }
+ dev_info(ss->dev, "Register %s\n",
+ ss_algs[i].alg.hash.halg.base.cra_name);
+ err = crypto_register_ahash(&ss_algs[i].alg.hash);
+ if (err) {
+ dev_err(ss->dev, "ERROR: Fail to register %s\n",
+ ss_algs[i].alg.hash.halg.base.cra_name);
+ ss_algs[i].ss = NULL;
+ return err;
+ }
+ break;
default:
ss_algs[i].ss = NULL;
dev_err(ss->dev, "ERROR: tried to register an unknown algo\n");
static void sun8i_ss_unregister_algs(struct sun8i_ss_dev *ss)
{
- int i;
+ unsigned int i;
for (i = 0; i < ARRAY_SIZE(ss_algs); i++) {
if (!ss_algs[i].ss)
ss_algs[i].alg.skcipher.base.cra_name);
crypto_unregister_skcipher(&ss_algs[i].alg.skcipher);
break;
+ case CRYPTO_ALG_TYPE_RNG:
+ dev_info(ss->dev, "Unregister %d %s\n", i,
+ ss_algs[i].alg.rng.base.cra_name);
+ crypto_unregister_rng(&ss_algs[i].alg.rng);
+ break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ dev_info(ss->dev, "Unregister %d %s\n", i,
+ ss_algs[i].alg.hash.halg.base.cra_name);
+ crypto_unregister_ahash(&ss_algs[i].alg.hash);
+ break;
}
}
}
return irq;
ss->reset = devm_reset_control_get(&pdev->dev, NULL);
- if (IS_ERR(ss->reset)) {
- if (PTR_ERR(ss->reset) == -EPROBE_DEFER)
- return PTR_ERR(ss->reset);
- dev_err(&pdev->dev, "No reset control found\n");
- return PTR_ERR(ss->reset);
- }
+ if (IS_ERR(ss->reset))
+ return dev_err_probe(&pdev->dev, PTR_ERR(ss->reset),
+ "No reset control found\n");
mutex_init(&ss->mlock);
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * sun8i-ss-hash.c - hardware cryptographic offloader for
+ * Allwinner A80/A83T SoC
+ *
+ * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>
+ *
+ * This file add support for MD5 and SHA1/SHA224/SHA256.
+ *
+ * You could find the datasheet in Documentation/arm/sunxi.rst
+ */
+#include <linux/dma-mapping.h>
+#include <linux/pm_runtime.h>
+#include <linux/scatterlist.h>
+#include <crypto/internal/hash.h>
+#include <crypto/sha.h>
+#include <crypto/md5.h>
+#include "sun8i-ss.h"
+
+int sun8i_ss_hash_crainit(struct crypto_tfm *tfm)
+{
+ struct sun8i_ss_hash_tfm_ctx *op = crypto_tfm_ctx(tfm);
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg);
+ struct sun8i_ss_alg_template *algt;
+ int err;
+
+ memset(op, 0, sizeof(struct sun8i_ss_hash_tfm_ctx));
+
+ algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
+ op->ss = algt->ss;
+
+ op->enginectx.op.do_one_request = sun8i_ss_hash_run;
+ op->enginectx.op.prepare_request = NULL;
+ op->enginectx.op.unprepare_request = NULL;
+
+ /* FALLBACK */
+ op->fallback_tfm = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(op->fallback_tfm)) {
+ dev_err(algt->ss->dev, "Fallback driver could no be loaded\n");
+ return PTR_ERR(op->fallback_tfm);
+ }
+
+ if (algt->alg.hash.halg.statesize < crypto_ahash_statesize(op->fallback_tfm))
+ algt->alg.hash.halg.statesize = crypto_ahash_statesize(op->fallback_tfm);
+
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct sun8i_ss_hash_reqctx) +
+ crypto_ahash_reqsize(op->fallback_tfm));
+
+ dev_info(op->ss->dev, "Fallback for %s is %s\n",
+ crypto_tfm_alg_driver_name(tfm),
+ crypto_tfm_alg_driver_name(&op->fallback_tfm->base));
+ err = pm_runtime_get_sync(op->ss->dev);
+ if (err < 0)
+ goto error_pm;
+ return 0;
+error_pm:
+ pm_runtime_put_noidle(op->ss->dev);
+ crypto_free_ahash(op->fallback_tfm);
+ return err;
+}
+
+void sun8i_ss_hash_craexit(struct crypto_tfm *tfm)
+{
+ struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_tfm_ctx(tfm);
+
+ crypto_free_ahash(tfmctx->fallback_tfm);
+ pm_runtime_put_sync_suspend(tfmctx->ss->dev);
+}
+
+int sun8i_ss_hash_init(struct ahash_request *areq)
+{
+ struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+
+ memset(rctx, 0, sizeof(struct sun8i_ss_hash_reqctx));
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ return crypto_ahash_init(&rctx->fallback_req);
+}
+
+int sun8i_ss_hash_export(struct ahash_request *areq, void *out)
+{
+ struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ return crypto_ahash_export(&rctx->fallback_req, out);
+}
+
+int sun8i_ss_hash_import(struct ahash_request *areq, const void *in)
+{
+ struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ return crypto_ahash_import(&rctx->fallback_req, in);
+}
+
+int sun8i_ss_hash_final(struct ahash_request *areq)
+{
+ struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
+ struct sun8i_ss_alg_template *algt;
+#endif
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags &
+ CRYPTO_TFM_REQ_MAY_SLEEP;
+ rctx->fallback_req.result = areq->result;
+
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
+ algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
+ algt->stat_fb++;
+#endif
+
+ return crypto_ahash_final(&rctx->fallback_req);
+}
+
+int sun8i_ss_hash_update(struct ahash_request *areq)
+{
+ struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags &
+ CRYPTO_TFM_REQ_MAY_SLEEP;
+ rctx->fallback_req.nbytes = areq->nbytes;
+ rctx->fallback_req.src = areq->src;
+
+ return crypto_ahash_update(&rctx->fallback_req);
+}
+
+int sun8i_ss_hash_finup(struct ahash_request *areq)
+{
+ struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
+ struct sun8i_ss_alg_template *algt;
+#endif
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags &
+ CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ rctx->fallback_req.nbytes = areq->nbytes;
+ rctx->fallback_req.src = areq->src;
+ rctx->fallback_req.result = areq->result;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
+ algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
+ algt->stat_fb++;
+#endif
+
+ return crypto_ahash_finup(&rctx->fallback_req);
+}
+
+static int sun8i_ss_hash_digest_fb(struct ahash_request *areq)
+{
+ struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct sun8i_ss_hash_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
+ struct sun8i_ss_alg_template *algt;
+#endif
+
+ ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm);
+ rctx->fallback_req.base.flags = areq->base.flags &
+ CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ rctx->fallback_req.nbytes = areq->nbytes;
+ rctx->fallback_req.src = areq->src;
+ rctx->fallback_req.result = areq->result;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
+ algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
+ algt->stat_fb++;
+#endif
+
+ return crypto_ahash_digest(&rctx->fallback_req);
+}
+
+static int sun8i_ss_run_hash_task(struct sun8i_ss_dev *ss,
+ struct sun8i_ss_hash_reqctx *rctx,
+ const char *name)
+{
+ int flow = rctx->flow;
+ u32 v = SS_START;
+ int i;
+
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
+ ss->flows[flow].stat_req++;
+#endif
+
+ /* choose between stream0/stream1 */
+ if (flow)
+ v |= SS_FLOW1;
+ else
+ v |= SS_FLOW0;
+
+ v |= rctx->method;
+
+ for (i = 0; i < MAX_SG; i++) {
+ if (!rctx->t_dst[i].addr)
+ break;
+
+ mutex_lock(&ss->mlock);
+ if (i > 0) {
+ v |= BIT(17);
+ writel(rctx->t_dst[i - 1].addr, ss->base + SS_KEY_ADR_REG);
+ writel(rctx->t_dst[i - 1].addr, ss->base + SS_IV_ADR_REG);
+ }
+
+ dev_dbg(ss->dev,
+ "Processing SG %d on flow %d %s ctl=%x %d to %d method=%x src=%x dst=%x\n",
+ i, flow, name, v,
+ rctx->t_src[i].len, rctx->t_dst[i].len,
+ rctx->method, rctx->t_src[i].addr, rctx->t_dst[i].addr);
+
+ writel(rctx->t_src[i].addr, ss->base + SS_SRC_ADR_REG);
+ writel(rctx->t_dst[i].addr, ss->base + SS_DST_ADR_REG);
+ writel(rctx->t_src[i].len, ss->base + SS_LEN_ADR_REG);
+ writel(BIT(0) | BIT(1), ss->base + SS_INT_CTL_REG);
+
+ reinit_completion(&ss->flows[flow].complete);
+ ss->flows[flow].status = 0;
+ wmb();
+
+ writel(v, ss->base + SS_CTL_REG);
+ mutex_unlock(&ss->mlock);
+ wait_for_completion_interruptible_timeout(&ss->flows[flow].complete,
+ msecs_to_jiffies(2000));
+ if (ss->flows[flow].status == 0) {
+ dev_err(ss->dev, "DMA timeout for %s\n", name);
+ return -EFAULT;
+ }
+ }
+
+ return 0;
+}
+
+static bool sun8i_ss_hash_need_fallback(struct ahash_request *areq)
+{
+ struct scatterlist *sg;
+
+ if (areq->nbytes == 0)
+ return true;
+ /* we need to reserve one SG for the padding one */
+ if (sg_nents(areq->src) > MAX_SG - 1)
+ return true;
+ sg = areq->src;
+ while (sg) {
+ /* SS can operate hash only on full block size
+ * since SS support only MD5,sha1,sha224 and sha256, blocksize
+ * is always 64
+ * TODO: handle request if last SG is not len%64
+ * but this will need to copy data on a new SG of size=64
+ */
+ if (sg->length % 64 || !IS_ALIGNED(sg->offset, sizeof(u32)))
+ return true;
+ sg = sg_next(sg);
+ }
+ return false;
+}
+
+int sun8i_ss_hash_digest(struct ahash_request *areq)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
+ struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct sun8i_ss_alg_template *algt;
+ struct sun8i_ss_dev *ss;
+ struct crypto_engine *engine;
+ struct scatterlist *sg;
+ int nr_sgs, e, i;
+
+ if (sun8i_ss_hash_need_fallback(areq))
+ return sun8i_ss_hash_digest_fb(areq);
+
+ nr_sgs = sg_nents(areq->src);
+ if (nr_sgs > MAX_SG - 1)
+ return sun8i_ss_hash_digest_fb(areq);
+
+ for_each_sg(areq->src, sg, nr_sgs, i) {
+ if (sg->length % 4 || !IS_ALIGNED(sg->offset, sizeof(u32)))
+ return sun8i_ss_hash_digest_fb(areq);
+ }
+
+ algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
+ ss = algt->ss;
+
+ e = sun8i_ss_get_engine_number(ss);
+ rctx->flow = e;
+ engine = ss->flows[e].engine;
+
+ return crypto_transfer_hash_request_to_engine(engine, areq);
+}
+
+/* sun8i_ss_hash_run - run an ahash request
+ * Send the data of the request to the SS along with an extra SG with padding
+ */
+int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq)
+{
+ struct ahash_request *areq = container_of(breq, struct ahash_request, base);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg);
+ struct sun8i_ss_hash_reqctx *rctx = ahash_request_ctx(areq);
+ struct sun8i_ss_alg_template *algt;
+ struct sun8i_ss_dev *ss;
+ struct scatterlist *sg;
+ int nr_sgs, err, digestsize;
+ unsigned int len;
+ u64 fill, min_fill, byte_count;
+ void *pad, *result;
+ int j, i, todo;
+ __be64 *bebits;
+ __le64 *lebits;
+ dma_addr_t addr_res, addr_pad;
+ __le32 *bf;
+
+ algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash);
+ ss = algt->ss;
+
+ digestsize = algt->alg.hash.halg.digestsize;
+ if (digestsize == SHA224_DIGEST_SIZE)
+ digestsize = SHA256_DIGEST_SIZE;
+
+ /* the padding could be up to two block. */
+ pad = kzalloc(algt->alg.hash.halg.base.cra_blocksize * 2, GFP_KERNEL | GFP_DMA);
+ if (!pad)
+ return -ENOMEM;
+ bf = (__le32 *)pad;
+
+ result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA);
+ if (!result)
+ return -ENOMEM;
+
+ for (i = 0; i < MAX_SG; i++) {
+ rctx->t_dst[i].addr = 0;
+ rctx->t_dst[i].len = 0;
+ }
+
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
+ algt->stat_req++;
+#endif
+
+ rctx->method = ss->variant->alg_hash[algt->ss_algo_id];
+
+ nr_sgs = dma_map_sg(ss->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE);
+ if (nr_sgs <= 0 || nr_sgs > MAX_SG) {
+ dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs);
+ err = -EINVAL;
+ goto theend;
+ }
+
+ addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE);
+ if (dma_mapping_error(ss->dev, addr_res)) {
+ dev_err(ss->dev, "DMA map dest\n");
+ err = -EINVAL;
+ goto theend;
+ }
+
+ len = areq->nbytes;
+ for_each_sg(areq->src, sg, nr_sgs, i) {
+ rctx->t_src[i].addr = sg_dma_address(sg);
+ todo = min(len, sg_dma_len(sg));
+ rctx->t_src[i].len = todo / 4;
+ len -= todo;
+ rctx->t_dst[i].addr = addr_res;
+ rctx->t_dst[i].len = digestsize / 4;
+ }
+ if (len > 0) {
+ dev_err(ss->dev, "remaining len %d\n", len);
+ err = -EINVAL;
+ goto theend;
+ }
+
+ byte_count = areq->nbytes;
+ j = 0;
+ bf[j++] = cpu_to_le32(0x80);
+
+ fill = 64 - (byte_count % 64);
+ min_fill = 3 * sizeof(u32);
+
+ if (fill < min_fill)
+ fill += 64;
+
+ j += (fill - min_fill) / sizeof(u32);
+
+ switch (algt->ss_algo_id) {
+ case SS_ID_HASH_MD5:
+ lebits = (__le64 *)&bf[j];
+ *lebits = cpu_to_le64(byte_count << 3);
+ j += 2;
+ break;
+ case SS_ID_HASH_SHA1:
+ case SS_ID_HASH_SHA224:
+ case SS_ID_HASH_SHA256:
+ bebits = (__be64 *)&bf[j];
+ *bebits = cpu_to_be64(byte_count << 3);
+ j += 2;
+ break;
+ }
+
+ addr_pad = dma_map_single(ss->dev, pad, j * 4, DMA_TO_DEVICE);
+ rctx->t_src[i].addr = addr_pad;
+ rctx->t_src[i].len = j;
+ rctx->t_dst[i].addr = addr_res;
+ rctx->t_dst[i].len = digestsize / 4;
+ if (dma_mapping_error(ss->dev, addr_pad)) {
+ dev_err(ss->dev, "DMA error on padding SG\n");
+ err = -EINVAL;
+ goto theend;
+ }
+
+ err = sun8i_ss_run_hash_task(ss, rctx, crypto_tfm_alg_name(areq->base.tfm));
+
+ dma_unmap_single(ss->dev, addr_pad, j * 4, DMA_TO_DEVICE);
+ dma_unmap_sg(ss->dev, areq->src, nr_sgs, DMA_TO_DEVICE);
+ dma_unmap_single(ss->dev, addr_res, digestsize, DMA_FROM_DEVICE);
+
+ kfree(pad);
+
+ memcpy(areq->result, result, algt->alg.hash.halg.digestsize);
+ kfree(result);
+theend:
+ crypto_finalize_hash_request(engine, breq, err);
+ return 0;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * sun8i-ss-prng.c - hardware cryptographic offloader for
+ * Allwinner A80/A83T SoC
+ *
+ * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>
+ *
+ * This file handle the PRNG found in the SS
+ *
+ * You could find a link for the datasheet in Documentation/arm/sunxi.rst
+ */
+#include "sun8i-ss.h"
+#include <linux/dma-mapping.h>
+#include <linux/pm_runtime.h>
+#include <crypto/internal/rng.h>
+
+int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed,
+ unsigned int slen)
+{
+ struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm);
+
+ if (ctx->seed && ctx->slen != slen) {
+ memzero_explicit(ctx->seed, ctx->slen);
+ kfree(ctx->seed);
+ ctx->slen = 0;
+ ctx->seed = NULL;
+ }
+ if (!ctx->seed)
+ ctx->seed = kmalloc(slen, GFP_KERNEL | GFP_DMA);
+ if (!ctx->seed)
+ return -ENOMEM;
+
+ memcpy(ctx->seed, seed, slen);
+ ctx->slen = slen;
+
+ return 0;
+}
+
+int sun8i_ss_prng_init(struct crypto_tfm *tfm)
+{
+ struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ memset(ctx, 0, sizeof(struct sun8i_ss_rng_tfm_ctx));
+ return 0;
+}
+
+void sun8i_ss_prng_exit(struct crypto_tfm *tfm)
+{
+ struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ memzero_explicit(ctx->seed, ctx->slen);
+ kfree(ctx->seed);
+ ctx->seed = NULL;
+ ctx->slen = 0;
+}
+
+int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
+ unsigned int slen, u8 *dst, unsigned int dlen)
+{
+ struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm);
+ struct rng_alg *alg = crypto_rng_alg(tfm);
+ struct sun8i_ss_alg_template *algt;
+ struct sun8i_ss_dev *ss;
+ dma_addr_t dma_iv, dma_dst;
+ unsigned int todo;
+ int err = 0;
+ int flow;
+ void *d;
+ u32 v;
+
+ algt = container_of(alg, struct sun8i_ss_alg_template, alg.rng);
+ ss = algt->ss;
+
+ if (ctx->slen == 0) {
+ dev_err(ss->dev, "The PRNG is not seeded\n");
+ return -EINVAL;
+ }
+
+ /* The SS does not give an updated seed, so we need to get a new one.
+ * So we will ask for an extra PRNG_SEED_SIZE data.
+ * We want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE
+ */
+ todo = dlen + PRNG_SEED_SIZE + PRNG_DATA_SIZE;
+ todo -= todo % PRNG_DATA_SIZE;
+
+ d = kzalloc(todo, GFP_KERNEL | GFP_DMA);
+ if (!d)
+ return -ENOMEM;
+
+ flow = sun8i_ss_get_engine_number(ss);
+
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
+ algt->stat_req++;
+ algt->stat_bytes += todo;
+#endif
+
+ v = SS_ALG_PRNG | SS_PRNG_CONTINUE | SS_START;
+ if (flow)
+ v |= SS_FLOW1;
+ else
+ v |= SS_FLOW0;
+
+ dma_iv = dma_map_single(ss->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE);
+ if (dma_mapping_error(ss->dev, dma_iv)) {
+ dev_err(ss->dev, "Cannot DMA MAP IV\n");
+ return -EFAULT;
+ }
+
+ dma_dst = dma_map_single(ss->dev, d, todo, DMA_FROM_DEVICE);
+ if (dma_mapping_error(ss->dev, dma_dst)) {
+ dev_err(ss->dev, "Cannot DMA MAP DST\n");
+ err = -EFAULT;
+ goto err_iv;
+ }
+
+ err = pm_runtime_get_sync(ss->dev);
+ if (err < 0) {
+ pm_runtime_put_noidle(ss->dev);
+ goto err_pm;
+ }
+ err = 0;
+
+ mutex_lock(&ss->mlock);
+ writel(dma_iv, ss->base + SS_IV_ADR_REG);
+ /* the PRNG act badly (failing rngtest) without SS_KEY_ADR_REG set */
+ writel(dma_iv, ss->base + SS_KEY_ADR_REG);
+ writel(dma_dst, ss->base + SS_DST_ADR_REG);
+ writel(todo / 4, ss->base + SS_LEN_ADR_REG);
+
+ reinit_completion(&ss->flows[flow].complete);
+ ss->flows[flow].status = 0;
+ /* Be sure all data is written before enabling the task */
+ wmb();
+
+ writel(v, ss->base + SS_CTL_REG);
+
+ wait_for_completion_interruptible_timeout(&ss->flows[flow].complete,
+ msecs_to_jiffies(todo));
+ if (ss->flows[flow].status == 0) {
+ dev_err(ss->dev, "DMA timeout for PRNG (size=%u)\n", todo);
+ err = -EFAULT;
+ }
+ /* Since cipher and hash use the linux/cryptoengine and that we have
+ * a cryptoengine per flow, we are sure that they will issue only one
+ * request per flow.
+ * Since the cryptoengine wait for completion before submitting a new
+ * one, the mlock could be left just after the final writel.
+ * But cryptoengine cannot handle crypto_rng, so we need to be sure
+ * nothing will use our flow.
+ * The easiest way is to grab mlock until the hardware end our requests.
+ * We could have used a per flow lock, but this would increase
+ * complexity.
+ * The drawback is that no request could be handled for the other flow.
+ */
+ mutex_unlock(&ss->mlock);
+
+ pm_runtime_put(ss->dev);
+
+err_pm:
+ dma_unmap_single(ss->dev, dma_dst, todo, DMA_FROM_DEVICE);
+err_iv:
+ dma_unmap_single(ss->dev, dma_iv, ctx->slen, DMA_TO_DEVICE);
+
+ if (!err) {
+ memcpy(dst, d, dlen);
+ /* Update seed */
+ memcpy(ctx->seed, d + dlen, ctx->slen);
+ }
+ memzero_explicit(d, todo);
+ kfree(d);
+
+ return err;
+}
#include <crypto/aes.h>
#include <crypto/des.h>
#include <crypto/engine.h>
+#include <crypto/rng.h>
#include <crypto/skcipher.h>
#include <linux/atomic.h>
#include <linux/debugfs.h>
#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+#include <crypto/md5.h>
+#include <crypto/sha.h>
+
+#define SS_START 1
#define SS_ENCRYPTION 0
#define SS_DECRYPTION BIT(6)
#define SS_ALG_AES 0
#define SS_ALG_DES (1 << 2)
#define SS_ALG_3DES (2 << 2)
+#define SS_ALG_MD5 (3 << 2)
+#define SS_ALG_PRNG (4 << 2)
+#define SS_ALG_SHA1 (6 << 2)
+#define SS_ALG_SHA224 (7 << 2)
+#define SS_ALG_SHA256 (8 << 2)
#define SS_CTL_REG 0x00
#define SS_INT_CTL_REG 0x04
#define SS_OP_ECB 0
#define SS_OP_CBC (1 << 13)
+#define SS_ID_HASH_MD5 0
+#define SS_ID_HASH_SHA1 1
+#define SS_ID_HASH_SHA224 2
+#define SS_ID_HASH_SHA256 3
+#define SS_ID_HASH_MAX 4
+
#define SS_FLOW0 BIT(30)
#define SS_FLOW1 BIT(31)
+#define SS_PRNG_CONTINUE BIT(18)
+
#define MAX_SG 8
#define MAXFLOW 2
#define SS_DIE_ID_SHIFT 20
#define SS_DIE_ID_MASK 0x07
+#define PRNG_DATA_SIZE (160 / 8)
+#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8)
+
/*
* struct ss_clock - Describe clocks used by sun8i-ss
* @name: Name of clock needed by this variant
* struct ss_variant - Describe SS capability for each variant hardware
* @alg_cipher: list of supported ciphers. for each SS_ID_ this will give the
* coresponding SS_ALG_XXX value
+ * @alg_hash: list of supported hashes. for each SS_ID_ this will give the
+ * corresponding SS_ALG_XXX value
* @op_mode: list of supported block modes
- * @ss_clks! list of clock needed by this variant
+ * @ss_clks: list of clock needed by this variant
*/
struct ss_variant {
char alg_cipher[SS_ID_CIPHER_MAX];
+ char alg_hash[SS_ID_HASH_MAX];
u32 op_mode[SS_ID_OP_MAX];
struct ss_clock ss_clks[SS_MAX_CLOCKS];
};
* @keylen: len of the key
* @ss: pointer to the private data of driver handling this TFM
* @fallback_tfm: pointer to the fallback TFM
+ *
+ * enginectx must be the first element
*/
struct sun8i_cipher_tfm_ctx {
struct crypto_engine_ctx enginectx;
struct crypto_skcipher *fallback_tfm;
};
+/*
+ * struct sun8i_ss_prng_ctx - context for PRNG TFM
+ * @seed: The seed to use
+ * @slen: The size of the seed
+ */
+struct sun8i_ss_rng_tfm_ctx {
+ void *seed;
+ unsigned int slen;
+};
+
+/*
+ * struct sun8i_ss_hash_tfm_ctx - context for an ahash TFM
+ * @enginectx: crypto_engine used by this TFM
+ * @fallback_tfm: pointer to the fallback TFM
+ * @ss: pointer to the private data of driver handling this TFM
+ *
+ * enginectx must be the first element
+ */
+struct sun8i_ss_hash_tfm_ctx {
+ struct crypto_engine_ctx enginectx;
+ struct crypto_ahash *fallback_tfm;
+ struct sun8i_ss_dev *ss;
+};
+
+/*
+ * struct sun8i_ss_hash_reqctx - context for an ahash request
+ * @t_src: list of DMA address and size for source SGs
+ * @t_dst: list of DMA address and size for destination SGs
+ * @fallback_req: pre-allocated fallback request
+ * @method: the register value for the algorithm used by this request
+ * @flow: the flow to use for this request
+ */
+struct sun8i_ss_hash_reqctx {
+ struct sginfo t_src[MAX_SG];
+ struct sginfo t_dst[MAX_SG];
+ struct ahash_request fallback_req;
+ u32 method;
+ int flow;
+};
+
/*
* struct sun8i_ss_alg_template - crypto_alg template
* @type: the CRYPTO_ALG_TYPE for this template
* @alg: one of sub struct must be used
* @stat_req: number of request done on this template
* @stat_fb: number of request which has fallbacked
+ * @stat_bytes: total data size done by this template
*/
struct sun8i_ss_alg_template {
u32 type;
struct sun8i_ss_dev *ss;
union {
struct skcipher_alg skcipher;
+ struct rng_alg rng;
+ struct ahash_alg hash;
} alg;
#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
unsigned long stat_req;
unsigned long stat_fb;
+ unsigned long stat_bytes;
#endif
};
int sun8i_ss_get_engine_number(struct sun8i_ss_dev *ss);
int sun8i_ss_run_task(struct sun8i_ss_dev *ss, struct sun8i_cipher_req_ctx *rctx, const char *name);
+int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
+ unsigned int slen, u8 *dst, unsigned int dlen);
+int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen);
+int sun8i_ss_prng_init(struct crypto_tfm *tfm);
+void sun8i_ss_prng_exit(struct crypto_tfm *tfm);
+
+int sun8i_ss_hash_crainit(struct crypto_tfm *tfm);
+void sun8i_ss_hash_craexit(struct crypto_tfm *tfm);
+int sun8i_ss_hash_init(struct ahash_request *areq);
+int sun8i_ss_hash_export(struct ahash_request *areq, void *out);
+int sun8i_ss_hash_import(struct ahash_request *areq, const void *in);
+int sun8i_ss_hash_final(struct ahash_request *areq);
+int sun8i_ss_hash_update(struct ahash_request *areq);
+int sun8i_ss_hash_finup(struct ahash_request *areq);
+int sun8i_ss_hash_digest(struct ahash_request *areq);
+int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq);
sa->sa_command_1.w = 0;
sa->sa_command_1.bf.crypto_mode31 = (cm & 4) >> 2;
sa->sa_command_1.bf.crypto_mode9_8 = cm & 3;
- sa->sa_command_1.bf.feedback_mode = cfb,
+ sa->sa_command_1.bf.feedback_mode = cfb;
sa->sa_command_1.bf.sa_rev = 1;
sa->sa_command_1.bf.hmac_muting = hmac_mc;
sa->sa_command_1.bf.extended_seq_num = esn;
#include <linux/ratelimit.h>
#include <linux/mutex.h>
+#include <linux/scatterlist.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/rng.h>
unsigned int keyivlen, ivsize, offset, tloffset;
dma_addr_t phykeyiv;
void *backup_iv = NULL, *bkeyiv;
- __le32 v;
+ u32 v;
algt = container_of(alg, struct meson_alg_template, alg.skcipher);
{
struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
- if (op->key) {
- memzero_explicit(op->key, op->keylen);
- kfree(op->key);
- }
+ kfree_sensitive(op->key);
crypto_free_skcipher(op->fallback_tfm);
}
dev_dbg(mc->dev, "ERROR: Invalid keylen %u\n", keylen);
return -EINVAL;
}
- if (op->key) {
- memzero_explicit(op->key, op->keylen);
- kfree(op->key);
- }
+ kfree_sensitive(op->key);
op->keylen = keylen;
op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
if (!op->key)
};
#ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG
-static int meson_dbgfs_read(struct seq_file *seq, void *v)
+static int meson_debugfs_show(struct seq_file *seq, void *v)
{
struct meson_dev *mc = seq->private;
int i;
}
return 0;
}
-
-static int meson_dbgfs_open(struct inode *inode, struct file *file)
-{
- return single_open(file, meson_dbgfs_read, inode->i_private);
-}
-
-static const struct file_operations meson_debugfs_fops = {
- .owner = THIS_MODULE,
- .open = meson_dbgfs_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(meson_debugfs);
#endif
static void meson_free_chanlist(struct meson_dev *mc, int i)
/* Write incr32(J0) into IV. */
j0_lsw = j0[3];
- j0[3] = cpu_to_be32(be32_to_cpu(j0[3]) + 1);
+ be32_add_cpu(&j0[3], 1);
atmel_aes_write_block(dd, AES_IVR(0), j0);
j0[3] = j0_lsw;
{
alg->base.cra_priority = ATMEL_TDES_PRIORITY;
alg->base.cra_flags = CRYPTO_ALG_ASYNC;
- alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx),
+ alg->base.cra_ctxsize = sizeof(struct atmel_tdes_ctx);
alg->base.cra_module = THIS_MODULE;
alg->init = atmel_tdes_init_tfm;
return -EFAULT;
}
- if (ctx->cipher.alg == CIPHER_ALG_RC4)
- /* Add buffer to catch 260-byte SUPDT field for RC4 */
- sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak, SPU_SUPDT_LEN);
-
if (stat_pad_len)
sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
u8 local_iv_ctr[MAX_IV_SIZE];
u32 stat_pad_len; /* num bytes to align status field */
u32 pad_len; /* total length of all padding */
- bool update_key = false;
struct brcm_message *mssg; /* mailbox message */
/* number of entries in src and dst sg in mailbox message. */
}
}
- if (ctx->cipher.alg == CIPHER_ALG_RC4) {
- rx_frag_num++;
- if (chunk_start) {
- /*
- * for non-first RC4 chunks, use SUPDT from previous
- * response as key for this chunk.
- */
- cipher_parms.key_buf = rctx->msg_buf.c.supdt_tweak;
- update_key = true;
- cipher_parms.type = CIPHER_TYPE_UPDT;
- } else if (!rctx->is_encrypt) {
- /*
- * First RC4 chunk. For decrypt, key in pre-built msg
- * header may have been changed if encrypt required
- * multiple chunks. So revert the key to the
- * ctx->enckey value.
- */
- update_key = true;
- cipher_parms.type = CIPHER_TYPE_INIT;
- }
- }
-
if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
flow_log("max_payload infinite\n");
else
memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr,
sizeof(rctx->msg_buf.bcm_spu_req_hdr));
- /*
- * Pass SUPDT field as key. Key field in finish() call is only used
- * when update_key has been set above for RC4. Will be ignored in
- * all other cases.
- */
spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
ctx->spu_req_hdr_len, !(rctx->is_encrypt),
- &cipher_parms, update_key, chunksize);
+ &cipher_parms, chunksize);
atomic64_add(chunksize, &iproc_priv.bytes_out);
__func__, rctx->total_received, payload_len);
dump_sg(req->dst, rctx->total_received, payload_len);
- if (ctx->cipher.alg == CIPHER_ALG_RC4)
- packet_dump(" supdt ", rctx->msg_buf.c.supdt_tweak,
- SPU_SUPDT_LEN);
rctx->total_received += payload_len;
if (rctx->total_received == rctx->total_todo) {
return 0;
}
-static int rc4_setkey(struct crypto_skcipher *cipher, const u8 *key,
- unsigned int keylen)
-{
- struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
- int i;
-
- ctx->enckeylen = ARC4_MAX_KEY_SIZE + ARC4_STATE_SIZE;
-
- ctx->enckey[0] = 0x00; /* 0x00 */
- ctx->enckey[1] = 0x00; /* i */
- ctx->enckey[2] = 0x00; /* 0x00 */
- ctx->enckey[3] = 0x00; /* j */
- for (i = 0; i < ARC4_MAX_KEY_SIZE; i++)
- ctx->enckey[i + ARC4_STATE_SIZE] = key[i % keylen];
-
- ctx->cipher_type = CIPHER_TYPE_INIT;
-
- return 0;
-}
-
static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key,
unsigned int keylen)
{
case CIPHER_ALG_AES:
err = aes_setkey(cipher, key, keylen);
break;
- case CIPHER_ALG_RC4:
- err = rc4_setkey(cipher, key, keylen);
- break;
default:
pr_err("%s() Error: unknown cipher alg\n", __func__);
err = -EINVAL;
if (err)
return err;
- /* RC4 already populated ctx->enkey */
- if (ctx->cipher.alg != CIPHER_ALG_RC4) {
- memcpy(ctx->enckey, key, keylen);
- ctx->enckeylen = keylen;
- }
+ memcpy(ctx->enckey, key, keylen);
+ ctx->enckeylen = keylen;
+
/* SPU needs XTS keys in the reverse order the crypto API presents */
if ((ctx->cipher.alg == CIPHER_ALG_AES) &&
(ctx->cipher.mode == CIPHER_MODE_XTS)) {
goto badkey;
}
break;
- case CIPHER_ALG_RC4:
- ctx->cipher_type = CIPHER_TYPE_INIT;
- break;
default:
pr_err("%s() Error: Unknown cipher alg\n", __func__);
return -EINVAL;
ctx->enckeylen = keylen;
ctx->authkeylen = 0;
- memcpy(ctx->enckey, key, ctx->enckeylen);
switch (ctx->enckeylen) {
case AES_KEYSIZE_128:
goto badkey;
}
+ memcpy(ctx->enckey, key, ctx->enckeylen);
+
flow_log(" enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
ctx->authkeylen);
flow_dump(" enc: ", ctx->enckey, ctx->enckeylen);
struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
flow_log("%s\n", __func__);
+
+ if (keylen < GCM_ESP_SALT_SIZE)
+ return -EINVAL;
+
ctx->salt_len = GCM_ESP_SALT_SIZE;
ctx->salt_offset = GCM_ESP_SALT_OFFSET;
memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
flow_log("%s\n", __func__);
+
+ if (keylen < GCM_ESP_SALT_SIZE)
+ return -EINVAL;
+
ctx->salt_len = GCM_ESP_SALT_SIZE;
ctx->salt_offset = GCM_ESP_SALT_OFFSET;
memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
flow_log("%s\n", __func__);
+
+ if (keylen < CCM_ESP_SALT_SIZE)
+ return -EINVAL;
+
ctx->salt_len = CCM_ESP_SALT_SIZE;
ctx->salt_offset = CCM_ESP_SALT_OFFSET;
memcpy(ctx->salt, key + keylen - CCM_ESP_SALT_SIZE, CCM_ESP_SALT_SIZE);
},
/* SKCIPHER algorithms. */
- {
- .type = CRYPTO_ALG_TYPE_SKCIPHER,
- .alg.skcipher = {
- .base.cra_name = "ecb(arc4)",
- .base.cra_driver_name = "ecb-arc4-iproc",
- .base.cra_blocksize = ARC4_BLOCK_SIZE,
- .min_keysize = ARC4_MIN_KEY_SIZE,
- .max_keysize = ARC4_MAX_KEY_SIZE,
- .ivsize = 0,
- },
- .cipher_info = {
- .alg = CIPHER_ALG_RC4,
- .mode = CIPHER_MODE_NONE,
- },
- .auth_info = {
- .alg = HASH_ALG_NONE,
- .mode = HASH_MODE_NONE,
- },
- },
{
.type = CRYPTO_ALG_TYPE_SKCIPHER,
.alg.skcipher = {
static int spu_register_skcipher(struct iproc_alg_s *driver_alg)
{
- struct spu_hw *spu = &iproc_priv.spu;
struct skcipher_alg *crypto = &driver_alg->alg.skcipher;
int err;
- /* SPU2 does not support RC4 */
- if ((driver_alg->cipher_info.alg == CIPHER_ALG_RC4) &&
- (spu->spu_type == SPU_TYPE_SPU2))
- return 0;
-
crypto->base.cra_module = THIS_MODULE;
crypto->base.cra_priority = cipher_pri;
crypto->base.cra_alignmask = 0;
u16 spu_req_hdr_len,
unsigned int is_inbound,
struct spu_cipher_parms *cipher_parms,
- bool update_key,
unsigned int data_size);
void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding,
u32 hash_pad_len, enum hash_alg auth_alg,
cipher_key_len = 24;
name = "3DES";
break;
- case CIPHER_ALG_RC4:
- cipher_key_len = 260;
- name = "ARC4";
- break;
case CIPHER_ALG_AES:
switch (cipher_type) {
case CIPHER_TYPE_AES128:
* @spu_req_hdr_len: Length in bytes of the SPU request header
* @isInbound: 0 encrypt, 1 decrypt
* @cipher_parms: Parameters describing cipher operation to be performed
- * @update_key: If true, rewrite the cipher key in SCTX
* @data_size: Length of the data in the BD field
*
* Assumes much of the header was already filled in at setkey() time in
* spum_cipher_req_init().
- * spum_cipher_req_init() fills in the encryption key. For RC4, when submitting
- * a request for a non-first chunk, we use the 260-byte SUPDT field from the
- * previous response as the key. update_key is true for this case. Unused in all
- * other cases.
+ * spum_cipher_req_init() fills in the encryption key.
*/
void spum_cipher_req_finish(u8 *spu_hdr,
u16 spu_req_hdr_len,
unsigned int is_inbound,
struct spu_cipher_parms *cipher_parms,
- bool update_key,
unsigned int data_size)
{
struct SPUHEADER *spuh;
flow_log(" in: %u\n", is_inbound);
flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg,
cipher_parms->type);
- if (update_key) {
- flow_log(" cipher key len: %u\n", cipher_parms->key_len);
- flow_dump(" key: ", cipher_parms->key_buf,
- cipher_parms->key_len);
- }
/*
* In XTS mode, API puts "i" parameter (block tweak) in IV. For
else
cipher_bits &= ~CIPHER_INBOUND;
- /* update encryption key for RC4 on non-first chunk */
- if (update_key) {
- spuh->sa.cipher_flags |=
- cipher_parms->type << CIPHER_TYPE_SHIFT;
- memcpy(spuh + 1, cipher_parms->key_buf, cipher_parms->key_len);
- }
-
if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len)
/* cipher iv provided so put it in here */
memcpy(bdesc_ptr - cipher_parms->iv_len, cipher_parms->iv_buf,
u16 spu_req_hdr_len,
unsigned int is_inbound,
struct spu_cipher_parms *cipher_parms,
- bool update_key,
unsigned int data_size);
void spum_request_pad(u8 *pad_start,
* @spu_req_hdr_len: Length in bytes of the SPU request header
* @isInbound: 0 encrypt, 1 decrypt
* @cipher_parms: Parameters describing cipher operation to be performed
- * @update_key: If true, rewrite the cipher key in SCTX
* @data_size: Length of the data in the BD field
*
* Assumes much of the header was already filled in at setkey() time in
* spu_cipher_req_init().
- * spu_cipher_req_init() fills in the encryption key. For RC4, when submitting a
- * request for a non-first chunk, we use the 260-byte SUPDT field from the
- * previous response as the key. update_key is true for this case. Unused in all
- * other cases.
+ * spu_cipher_req_init() fills in the encryption key.
*/
void spu2_cipher_req_finish(u8 *spu_hdr,
u16 spu_req_hdr_len,
unsigned int is_inbound,
struct spu_cipher_parms *cipher_parms,
- bool update_key,
unsigned int data_size)
{
struct SPU2_FMD *fmd;
flow_log(" in: %u\n", is_inbound);
flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg,
cipher_parms->type);
- if (update_key) {
- flow_log(" cipher key len: %u\n", cipher_parms->key_len);
- flow_dump(" key: ", cipher_parms->key_buf,
- cipher_parms->key_len);
- }
flow_log(" iv len: %d\n", cipher_parms->iv_len);
flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
flow_log(" data_size: %u\n", data_size);
u16 spu_req_hdr_len,
unsigned int is_inbound,
struct spu_cipher_parms *cipher_parms,
- bool update_key,
unsigned int data_size);
void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len,
enum hash_alg auth_alg, enum hash_mode auth_mode,
select CRYPTO_AUTHENC
select CRYPTO_SKCIPHER
select CRYPTO_LIB_DES
+ select CRYPTO_XTS
help
Selecting this will offload crypto for users of the
scatterlist crypto API (such as the linux native IPSec
select CRYPTO_AUTHENC
select CRYPTO_SKCIPHER
select CRYPTO_DES
+ select CRYPTO_XTS
help
Selecting this will use CAAM Queue Interface (QI) for sending
& receiving crypto jobs to/from CAAM. This gives better performance
select CRYPTO_AEAD
select CRYPTO_HASH
select CRYPTO_DES
+ select CRYPTO_XTS
help
CAAM driver for QorIQ Data Path Acceleration Architecture 2.
It handles DPSECI DPAA2 objects that sit on the Management Complex
ccflags-y += -DCONFIG_CAAM_QI
endif
+caam-$(CONFIG_DEBUG_FS) += debugfs.o
+
obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o
dpaa2_caam-y := caamalg_qi2.o dpseci.o
#include "key_gen.h"
#include "caamalg_desc.h"
#include <crypto/engine.h>
+#include <crypto/xts.h>
+#include <asm/unaligned.h>
/*
* crypto alg
struct alginfo adata;
struct alginfo cdata;
unsigned int authsize;
+ bool xts_key_fallback;
+ struct crypto_skcipher *fallback;
};
struct caam_skcipher_req_ctx {
struct skcipher_edesc *edesc;
+ struct skcipher_request fallback_req;
};
struct caam_aead_req_ctx {
{
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct device *jrdev = ctx->jrdev;
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
u32 *desc;
+ int err;
- if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) {
+ err = xts_verify_key(skcipher, key, keylen);
+ if (err) {
dev_dbg(jrdev, "key size mismatch\n");
- return -EINVAL;
+ return err;
+ }
+
+ if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
+ ctx->xts_key_fallback = true;
+
+ if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) {
+ err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
+ if (err)
+ return err;
}
ctx->cdata.keylen = keylen;
return ret;
}
+static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
+{
+ struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+ unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
+
+ return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
+}
+
static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
{
struct skcipher_edesc *edesc;
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct device *jrdev = ctx->jrdev;
struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
u32 *desc;
int ret = 0;
- if (!req->cryptlen)
+ /*
+ * XTS is expected to return an error even for input length = 0
+ * Note that the case input length < block size will be caught during
+ * HW offloading and return an error.
+ */
+ if (!req->cryptlen && !ctx->fallback)
return 0;
+ if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) ||
+ ctx->xts_key_fallback)) {
+ struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
+
+ skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
+ skcipher_request_set_callback(&rctx->fallback_req,
+ req->base.flags,
+ req->base.complete,
+ req->base.data);
+ skcipher_request_set_crypt(&rctx->fallback_req, req->src,
+ req->dst, req->cryptlen, req->iv);
+
+ return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) :
+ crypto_skcipher_decrypt(&rctx->fallback_req);
+ }
+
/* allocate extended descriptor */
edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
if (IS_ERR(edesc))
.base = {
.cra_name = "xts(aes)",
.cra_driver_name = "xts-aes-caam",
+ .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = AES_BLOCK_SIZE,
},
.setkey = xts_skcipher_setkey,
struct caam_skcipher_alg *caam_alg =
container_of(alg, typeof(*caam_alg), skcipher);
struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
-
- crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx));
+ u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
+ int ret = 0;
ctx->enginectx.op.do_one_request = skcipher_do_one_req;
- return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam,
- false);
+ if (alg_aai == OP_ALG_AAI_XTS) {
+ const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
+ struct crypto_skcipher *fallback;
+
+ fallback = crypto_alloc_skcipher(tfm_name, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(fallback)) {
+ dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n",
+ tfm_name, PTR_ERR(fallback));
+ return PTR_ERR(fallback);
+ }
+
+ ctx->fallback = fallback;
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) +
+ crypto_skcipher_reqsize(fallback));
+ } else {
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx));
+ }
+
+ ret = caam_init_common(ctx, &caam_alg->caam, false);
+ if (ret && ctx->fallback)
+ crypto_free_skcipher(ctx->fallback);
+
+ return ret;
}
static int caam_aead_init(struct crypto_aead *tfm)
static void caam_cra_exit(struct crypto_skcipher *tfm)
{
- caam_exit_common(crypto_skcipher_ctx(tfm));
+ struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ if (ctx->fallback)
+ crypto_free_skcipher(ctx->fallback);
+ caam_exit_common(ctx);
}
static void caam_aead_exit(struct crypto_aead *tfm)
alg->base.cra_module = THIS_MODULE;
alg->base.cra_priority = CAAM_CRA_PRIORITY;
alg->base.cra_ctxsize = sizeof(struct caam_ctx);
- alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
- CRYPTO_ALG_KERN_DRIVER_ONLY;
+ alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_ALG_KERN_DRIVER_ONLY);
alg->init = caam_cra_init;
alg->exit = caam_cra_exit;
* with OP_ALG_AAI_HMAC_PRECOMP.
* @ivsize: initialization vector size
* @icvsize: integrity check value (ICV) size (truncated or full)
+ * @geniv: whether to generate Encrypted Chain IV
* @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
* @nonce: pointer to rfc3686 nonce
* @ctx1_iv_off: IV offset in CONTEXT1 register
set_jump_tgt_here(desc, key_jump_cmd);
/*
- * create sequence for loading the sector index
- * Upper 8B of IV - will be used as sector index
- * Lower 8B of IV - will be discarded
+ * create sequence for loading the sector index / 16B tweak value
+ * Lower 8B of IV - sector index / tweak lower half
+ * Upper 8B of IV - upper half of 16B tweak
*/
append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
(0x20 << LDST_OFFSET_SHIFT));
- append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
+ append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
+ (0x30 << LDST_OFFSET_SHIFT));
/* Load operation */
append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
/* Perform operation */
skcipher_append_src_dst(desc);
- /* Store upper 8B of IV */
+ /* Store lower 8B and upper 8B of IV */
append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
(0x20 << LDST_OFFSET_SHIFT));
+ append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
+ (0x30 << LDST_OFFSET_SHIFT));
print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
": ", DUMP_PREFIX_ADDRESS, 16, 4,
set_jump_tgt_here(desc, key_jump_cmd);
/*
- * create sequence for loading the sector index
- * Upper 8B of IV - will be used as sector index
- * Lower 8B of IV - will be discarded
+ * create sequence for loading the sector index / 16B tweak value
+ * Lower 8B of IV - sector index / tweak lower half
+ * Upper 8B of IV - upper half of 16B tweak
*/
append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
(0x20 << LDST_OFFSET_SHIFT));
- append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
-
+ append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
+ (0x30 << LDST_OFFSET_SHIFT));
/* Load operation */
append_dec_op1(desc, cdata->algtype);
/* Perform operation */
skcipher_append_src_dst(desc);
- /* Store upper 8B of IV */
+ /* Store lower 8B and upper 8B of IV */
append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
(0x20 << LDST_OFFSET_SHIFT));
+ append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
+ (0x30 << LDST_OFFSET_SHIFT));
print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
#include "qi.h"
#include "jr.h"
#include "caamalg_desc.h"
+#include <crypto/xts.h>
+#include <asm/unaligned.h>
/*
* crypto alg
struct device *qidev;
spinlock_t lock; /* Protects multiple init of driver context */
struct caam_drv_ctx *drv_ctx[NUM_OP];
+ bool xts_key_fallback;
+ struct crypto_skcipher *fallback;
+};
+
+struct caam_skcipher_req_ctx {
+ struct skcipher_request fallback_req;
};
static int aead_set_sh_desc(struct crypto_aead *aead)
{
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct device *jrdev = ctx->jrdev;
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
int ret = 0;
+ int err;
- if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) {
+ err = xts_verify_key(skcipher, key, keylen);
+ if (err) {
dev_dbg(jrdev, "key size mismatch\n");
- return -EINVAL;
+ return err;
+ }
+
+ if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
+ ctx->xts_key_fallback = true;
+
+ if (ctrlpriv->era <= 8 || ctx->xts_key_fallback) {
+ err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
+ if (err)
+ return err;
}
ctx->cdata.keylen = keylen;
return edesc;
}
+static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
+{
+ struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+ unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
+
+ return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
+}
+
static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
{
struct skcipher_edesc *edesc;
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
int ret;
- if (!req->cryptlen)
+ /*
+ * XTS is expected to return an error even for input length = 0
+ * Note that the case input length < block size will be caught during
+ * HW offloading and return an error.
+ */
+ if (!req->cryptlen && !ctx->fallback)
return 0;
+ if (ctx->fallback && ((ctrlpriv->era <= 8 && xts_skcipher_ivsize(req)) ||
+ ctx->xts_key_fallback)) {
+ struct caam_skcipher_req_ctx *rctx = skcipher_request_ctx(req);
+
+ skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
+ skcipher_request_set_callback(&rctx->fallback_req,
+ req->base.flags,
+ req->base.complete,
+ req->base.data);
+ skcipher_request_set_crypt(&rctx->fallback_req, req->src,
+ req->dst, req->cryptlen, req->iv);
+
+ return encrypt ? crypto_skcipher_encrypt(&rctx->fallback_req) :
+ crypto_skcipher_decrypt(&rctx->fallback_req);
+ }
+
if (unlikely(caam_congested))
return -EAGAIN;
.base = {
.cra_name = "xts(aes)",
.cra_driver_name = "xts-aes-caam-qi",
+ .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = AES_BLOCK_SIZE,
},
.setkey = xts_skcipher_setkey,
struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
struct caam_skcipher_alg *caam_alg =
container_of(alg, typeof(*caam_alg), skcipher);
+ struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
+ u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
+ int ret = 0;
+
+ if (alg_aai == OP_ALG_AAI_XTS) {
+ const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
+ struct crypto_skcipher *fallback;
+
+ fallback = crypto_alloc_skcipher(tfm_name, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(fallback)) {
+ dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n",
+ tfm_name, PTR_ERR(fallback));
+ return PTR_ERR(fallback);
+ }
+
+ ctx->fallback = fallback;
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_skcipher_req_ctx) +
+ crypto_skcipher_reqsize(fallback));
+ }
+
+ ret = caam_init_common(ctx, &caam_alg->caam, false);
+ if (ret && ctx->fallback)
+ crypto_free_skcipher(ctx->fallback);
- return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam,
- false);
+ return ret;
}
static int caam_aead_init(struct crypto_aead *tfm)
static void caam_cra_exit(struct crypto_skcipher *tfm)
{
- caam_exit_common(crypto_skcipher_ctx(tfm));
+ struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ if (ctx->fallback)
+ crypto_free_skcipher(ctx->fallback);
+ caam_exit_common(ctx);
}
static void caam_aead_exit(struct crypto_aead *tfm)
alg->base.cra_module = THIS_MODULE;
alg->base.cra_priority = CAAM_CRA_PRIORITY;
alg->base.cra_ctxsize = sizeof(struct caam_ctx);
- alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
- CRYPTO_ALG_KERN_DRIVER_ONLY;
+ alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_ALG_KERN_DRIVER_ONLY);
alg->init = caam_cra_init;
alg->exit = caam_cra_exit;
#include <linux/fsl/mc.h>
#include <soc/fsl/dpaa2-io.h>
#include <soc/fsl/dpaa2-fd.h>
+#include <crypto/xts.h>
+#include <asm/unaligned.h>
#define CAAM_CRA_PRIORITY 2000
};
/**
- * caam_ctx - per-session context
+ * struct caam_ctx - per-session context
* @flc: Flow Contexts array
* @key: [authentication key], encryption key
* @flc_dma: I/O virtual addresses of the Flow Contexts
struct alginfo adata;
struct alginfo cdata;
unsigned int authsize;
+ bool xts_key_fallback;
+ struct crypto_skcipher *fallback;
};
static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv,
{
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct device *dev = ctx->dev;
+ struct dpaa2_caam_priv *priv = dev_get_drvdata(dev);
struct caam_flc *flc;
u32 *desc;
+ int err;
- if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) {
+ err = xts_verify_key(skcipher, key, keylen);
+ if (err) {
dev_dbg(dev, "key size mismatch\n");
- return -EINVAL;
+ return err;
+ }
+
+ if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256)
+ ctx->xts_key_fallback = true;
+
+ if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) {
+ err = crypto_skcipher_setkey(ctx->fallback, key, keylen);
+ if (err)
+ return err;
}
ctx->cdata.keylen = keylen;
skcipher_request_complete(req, ecode);
}
+static inline bool xts_skcipher_ivsize(struct skcipher_request *req)
+{
+ struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+ unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
+
+ return !!get_unaligned((u64 *)(req->iv + (ivsize / 2)));
+}
+
static int skcipher_encrypt(struct skcipher_request *req)
{
struct skcipher_edesc *edesc;
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct caam_request *caam_req = skcipher_request_ctx(req);
+ struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
int ret;
- if (!req->cryptlen)
+ /*
+ * XTS is expected to return an error even for input length = 0
+ * Note that the case input length < block size will be caught during
+ * HW offloading and return an error.
+ */
+ if (!req->cryptlen && !ctx->fallback)
return 0;
+ if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
+ ctx->xts_key_fallback)) {
+ skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
+ skcipher_request_set_callback(&caam_req->fallback_req,
+ req->base.flags,
+ req->base.complete,
+ req->base.data);
+ skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
+ req->dst, req->cryptlen, req->iv);
+
+ return crypto_skcipher_encrypt(&caam_req->fallback_req);
+ }
+
/* allocate extended descriptor */
edesc = skcipher_edesc_alloc(req);
if (IS_ERR(edesc))
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
struct caam_request *caam_req = skcipher_request_ctx(req);
+ struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev);
int ret;
- if (!req->cryptlen)
+ /*
+ * XTS is expected to return an error even for input length = 0
+ * Note that the case input length < block size will be caught during
+ * HW offloading and return an error.
+ */
+ if (!req->cryptlen && !ctx->fallback)
return 0;
+
+ if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) ||
+ ctx->xts_key_fallback)) {
+ skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback);
+ skcipher_request_set_callback(&caam_req->fallback_req,
+ req->base.flags,
+ req->base.complete,
+ req->base.data);
+ skcipher_request_set_crypt(&caam_req->fallback_req, req->src,
+ req->dst, req->cryptlen, req->iv);
+
+ return crypto_skcipher_decrypt(&caam_req->fallback_req);
+ }
+
/* allocate extended descriptor */
edesc = skcipher_edesc_alloc(req);
if (IS_ERR(edesc))
struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
struct caam_skcipher_alg *caam_alg =
container_of(alg, typeof(*caam_alg), skcipher);
+ struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
+ u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
+ int ret = 0;
- crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request));
- return caam_cra_init(crypto_skcipher_ctx(tfm), &caam_alg->caam, false);
+ if (alg_aai == OP_ALG_AAI_XTS) {
+ const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
+ struct crypto_skcipher *fallback;
+
+ fallback = crypto_alloc_skcipher(tfm_name, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(fallback)) {
+ dev_err(ctx->dev, "Failed to allocate %s fallback: %ld\n",
+ tfm_name, PTR_ERR(fallback));
+ return PTR_ERR(fallback);
+ }
+
+ ctx->fallback = fallback;
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request) +
+ crypto_skcipher_reqsize(fallback));
+ } else {
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct caam_request));
+ }
+
+ ret = caam_cra_init(ctx, &caam_alg->caam, false);
+ if (ret && ctx->fallback)
+ crypto_free_skcipher(ctx->fallback);
+
+ return ret;
}
static int caam_cra_init_aead(struct crypto_aead *tfm)
static void caam_cra_exit(struct crypto_skcipher *tfm)
{
- caam_exit_common(crypto_skcipher_ctx(tfm));
+ struct caam_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ if (ctx->fallback)
+ crypto_free_skcipher(ctx->fallback);
+ caam_exit_common(ctx);
}
static void caam_cra_exit_aead(struct crypto_aead *tfm)
.base = {
.cra_name = "xts(aes)",
.cra_driver_name = "xts-aes-caam-qi2",
+ .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = AES_BLOCK_SIZE,
},
.setkey = xts_skcipher_setkey,
alg->base.cra_module = THIS_MODULE;
alg->base.cra_priority = CAAM_CRA_PRIORITY;
alg->base.cra_ctxsize = sizeof(struct caam_ctx);
- alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
- CRYPTO_ALG_KERN_DRIVER_ONLY;
+ alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
+ CRYPTO_ALG_KERN_DRIVER_ONLY);
alg->init = caam_cra_init_skcipher;
alg->exit = caam_cra_exit;
};
/**
- * caam_hash_ctx - ahash per-session context
+ * struct caam_hash_ctx - ahash per-session context
* @flc: Flow Contexts array
* @key: authentication key
* @flc_dma: I/O virtual addresses of the Flow Contexts
/* DPIO */
err = dpaa2_dpseci_dpio_setup(priv);
if (err) {
- if (err != -EPROBE_DEFER)
- dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n");
+ dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n");
goto err_dpio_setup;
}
#include <linux/netdevice.h>
#include "dpseci.h"
#include "desc_constr.h"
+#include <crypto/skcipher.h>
#define DPAA2_CAAM_STORE_SIZE 16
/* NAPI weight *must* be a multiple of the store size. */
void (*cbk)(void *ctx, u32 err);
void *ctx;
void *edesc;
+ struct skcipher_request fallback_req;
};
/**
#include <linux/fsl/mc.h>
#include "compat.h"
+#include "debugfs.h"
#include "regs.h"
#include "intern.h"
#include "jr.h"
kfree(desc);
- if (!ret)
- ret = devm_add_action_or_reset(ctrldev, devm_deinstantiate_rng,
- ctrldev);
+ if (ret)
+ return ret;
- return ret;
+ return devm_add_action_or_reset(ctrldev, devm_deinstantiate_rng, ctrldev);
}
/*
* by u-boot.
* In case this property is not passed an attempt to retrieve the CAAM
* era via register reads will be made.
- **/
+ *
+ * @ctrl: controller region
+ */
static int caam_get_era(struct caam_ctrl __iomem *ctrl)
{
struct device_node *caam_node;
return devm_add_action_or_reset(dev, disable_clocks, ctrlpriv);
}
-#ifdef CONFIG_DEBUG_FS
static void caam_remove_debugfs(void *root)
{
debugfs_remove_recursive(root);
}
-#endif
#ifdef CONFIG_FSL_MC_BUS
static bool check_version(struct fsl_mc_version *mc_version, u32 major,
struct device_node *nprop, *np;
struct caam_ctrl __iomem *ctrl;
struct caam_drv_private *ctrlpriv;
-#ifdef CONFIG_DEBUG_FS
- struct caam_perfmon *perfmon;
struct dentry *dfs_root;
-#endif
u32 scfgr, comp_params;
u8 rng_vid;
int pg_size;
ctrlpriv->era = caam_get_era(ctrl);
ctrlpriv->domain = iommu_get_domain_for_dev(dev);
-#ifdef CONFIG_DEBUG_FS
- /*
- * FIXME: needs better naming distinction, as some amalgamation of
- * "caam" and nprop->full_name. The OF name isn't distinctive,
- * but does separate instances
- */
- perfmon = (struct caam_perfmon __force *)&ctrl->perfmon;
-
dfs_root = debugfs_create_dir(dev_name(dev), NULL);
- ret = devm_add_action_or_reset(dev, caam_remove_debugfs, dfs_root);
- if (ret)
- return ret;
+ if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+ ret = devm_add_action_or_reset(dev, caam_remove_debugfs,
+ dfs_root);
+ if (ret)
+ return ret;
+ }
- ctrlpriv->ctl = debugfs_create_dir("ctl", dfs_root);
-#endif
+ caam_debugfs_init(ctrlpriv, dfs_root);
/* Check to see if (DPAA 1.x) QI present. If so, enable */
if (ctrlpriv->qi_present && !caam_dpaa2) {
dev_info(dev, "job rings = %d, qi = %d\n",
ctrlpriv->total_jobrs, ctrlpriv->qi_present);
-#ifdef CONFIG_DEBUG_FS
- debugfs_create_file("rq_dequeued", S_IRUSR | S_IRGRP | S_IROTH,
- ctrlpriv->ctl, &perfmon->req_dequeued,
- &caam_fops_u64_ro);
- debugfs_create_file("ob_rq_encrypted", S_IRUSR | S_IRGRP | S_IROTH,
- ctrlpriv->ctl, &perfmon->ob_enc_req,
- &caam_fops_u64_ro);
- debugfs_create_file("ib_rq_decrypted", S_IRUSR | S_IRGRP | S_IROTH,
- ctrlpriv->ctl, &perfmon->ib_dec_req,
- &caam_fops_u64_ro);
- debugfs_create_file("ob_bytes_encrypted", S_IRUSR | S_IRGRP | S_IROTH,
- ctrlpriv->ctl, &perfmon->ob_enc_bytes,
- &caam_fops_u64_ro);
- debugfs_create_file("ob_bytes_protected", S_IRUSR | S_IRGRP | S_IROTH,
- ctrlpriv->ctl, &perfmon->ob_prot_bytes,
- &caam_fops_u64_ro);
- debugfs_create_file("ib_bytes_decrypted", S_IRUSR | S_IRGRP | S_IROTH,
- ctrlpriv->ctl, &perfmon->ib_dec_bytes,
- &caam_fops_u64_ro);
- debugfs_create_file("ib_bytes_validated", S_IRUSR | S_IRGRP | S_IROTH,
- ctrlpriv->ctl, &perfmon->ib_valid_bytes,
- &caam_fops_u64_ro);
-
- /* Controller level - global status values */
- debugfs_create_file("fault_addr", S_IRUSR | S_IRGRP | S_IROTH,
- ctrlpriv->ctl, &perfmon->faultaddr,
- &caam_fops_u32_ro);
- debugfs_create_file("fault_detail", S_IRUSR | S_IRGRP | S_IROTH,
- ctrlpriv->ctl, &perfmon->faultdetail,
- &caam_fops_u32_ro);
- debugfs_create_file("fault_status", S_IRUSR | S_IRGRP | S_IROTH,
- ctrlpriv->ctl, &perfmon->status,
- &caam_fops_u32_ro);
-
- /* Internal covering keys (useful in non-secure mode only) */
- ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0];
- ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
- debugfs_create_blob("kek", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl,
- &ctrlpriv->ctl_kek_wrap);
-
- ctrlpriv->ctl_tkek_wrap.data = (__force void *)&ctrlpriv->ctrl->tkek[0];
- ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
- debugfs_create_blob("tkek", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl,
- &ctrlpriv->ctl_tkek_wrap);
-
- ctrlpriv->ctl_tdsk_wrap.data = (__force void *)&ctrlpriv->ctrl->tdsk[0];
- ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32);
- debugfs_create_blob("tdsk", S_IRUSR | S_IRGRP | S_IROTH, ctrlpriv->ctl,
- &ctrlpriv->ctl_tdsk_wrap);
-#endif
-
ret = devm_of_platform_populate(dev);
if (ret)
dev_err(dev, "JR platform devices creation error\n");
--- /dev/null
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/* Copyright 2019 NXP */
+
+#include <linux/debugfs.h>
+#include "compat.h"
+#include "debugfs.h"
+#include "regs.h"
+#include "intern.h"
+
+static int caam_debugfs_u64_get(void *data, u64 *val)
+{
+ *val = caam64_to_cpu(*(u64 *)data);
+ return 0;
+}
+
+static int caam_debugfs_u32_get(void *data, u64 *val)
+{
+ *val = caam32_to_cpu(*(u32 *)data);
+ return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n");
+DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n");
+
+#ifdef CONFIG_CAAM_QI
+/*
+ * This is a counter for the number of times the congestion group (where all
+ * the request and response queueus are) reached congestion. Incremented
+ * each time the congestion callback is called with congested == true.
+ */
+static u64 times_congested;
+
+void caam_debugfs_qi_congested(void)
+{
+ times_congested++;
+}
+
+void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv)
+{
+ debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
+ ×_congested, &caam_fops_u64_ro);
+}
+#endif
+
+void caam_debugfs_init(struct caam_drv_private *ctrlpriv, struct dentry *root)
+{
+ struct caam_perfmon *perfmon;
+
+ /*
+ * FIXME: needs better naming distinction, as some amalgamation of
+ * "caam" and nprop->full_name. The OF name isn't distinctive,
+ * but does separate instances
+ */
+ perfmon = (struct caam_perfmon __force *)&ctrlpriv->ctrl->perfmon;
+
+ ctrlpriv->ctl = debugfs_create_dir("ctl", root);
+
+ debugfs_create_file("rq_dequeued", 0444, ctrlpriv->ctl,
+ &perfmon->req_dequeued, &caam_fops_u64_ro);
+ debugfs_create_file("ob_rq_encrypted", 0444, ctrlpriv->ctl,
+ &perfmon->ob_enc_req, &caam_fops_u64_ro);
+ debugfs_create_file("ib_rq_decrypted", 0444, ctrlpriv->ctl,
+ &perfmon->ib_dec_req, &caam_fops_u64_ro);
+ debugfs_create_file("ob_bytes_encrypted", 0444, ctrlpriv->ctl,
+ &perfmon->ob_enc_bytes, &caam_fops_u64_ro);
+ debugfs_create_file("ob_bytes_protected", 0444, ctrlpriv->ctl,
+ &perfmon->ob_prot_bytes, &caam_fops_u64_ro);
+ debugfs_create_file("ib_bytes_decrypted", 0444, ctrlpriv->ctl,
+ &perfmon->ib_dec_bytes, &caam_fops_u64_ro);
+ debugfs_create_file("ib_bytes_validated", 0444, ctrlpriv->ctl,
+ &perfmon->ib_valid_bytes, &caam_fops_u64_ro);
+
+ /* Controller level - global status values */
+ debugfs_create_file("fault_addr", 0444, ctrlpriv->ctl,
+ &perfmon->faultaddr, &caam_fops_u32_ro);
+ debugfs_create_file("fault_detail", 0444, ctrlpriv->ctl,
+ &perfmon->faultdetail, &caam_fops_u32_ro);
+ debugfs_create_file("fault_status", 0444, ctrlpriv->ctl,
+ &perfmon->status, &caam_fops_u32_ro);
+
+ /* Internal covering keys (useful in non-secure mode only) */
+ ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0];
+ ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
+ debugfs_create_blob("kek", 0444, ctrlpriv->ctl,
+ &ctrlpriv->ctl_kek_wrap);
+
+ ctrlpriv->ctl_tkek_wrap.data = (__force void *)&ctrlpriv->ctrl->tkek[0];
+ ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
+ debugfs_create_blob("tkek", 0444, ctrlpriv->ctl,
+ &ctrlpriv->ctl_tkek_wrap);
+
+ ctrlpriv->ctl_tdsk_wrap.data = (__force void *)&ctrlpriv->ctrl->tdsk[0];
+ ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32);
+ debugfs_create_blob("tdsk", 0444, ctrlpriv->ctl,
+ &ctrlpriv->ctl_tdsk_wrap);
+}
--- /dev/null
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
+/* Copyright 2019 NXP */
+
+#ifndef CAAM_DEBUGFS_H
+#define CAAM_DEBUGFS_H
+
+struct dentry;
+struct caam_drv_private;
+
+#ifdef CONFIG_DEBUG_FS
+void caam_debugfs_init(struct caam_drv_private *ctrlpriv, struct dentry *root);
+#else
+static inline void caam_debugfs_init(struct caam_drv_private *ctrlpriv,
+ struct dentry *root)
+{}
+#endif
+
+#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_CAAM_QI)
+void caam_debugfs_qi_congested(void);
+void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv);
+#else
+static inline void caam_debugfs_qi_congested(void) {}
+static inline void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv) {}
+#endif
+
+#endif /* CAAM_DEBUGFS_H */
return 0;
}
-static int dpseci_dbg_fqs_open(struct inode *inode, struct file *file)
-{
- int err;
- struct dpaa2_caam_priv *priv;
-
- priv = (struct dpaa2_caam_priv *)inode->i_private;
-
- err = single_open(file, dpseci_dbg_fqs_show, priv);
- if (err < 0)
- dev_err(priv->dev, "single_open() failed\n");
-
- return err;
-}
-
-static const struct file_operations dpseci_dbg_fq_ops = {
- .open = dpseci_dbg_fqs_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(dpseci_dbg_fqs);
void dpaa2_dpseci_debugfs_init(struct dpaa2_caam_priv *priv)
{
priv->dfs_root = debugfs_create_dir(dev_name(priv->dev), NULL);
debugfs_create_file("fq_stats", 0444, priv->dfs_root, priv,
- &dpseci_dbg_fq_ops);
+ &dpseci_dbg_fqs_fops);
}
void dpaa2_dpseci_debugfs_exit(struct dpaa2_caam_priv *priv)
#endif /* CONFIG_CAAM_QI */
-#ifdef CONFIG_DEBUG_FS
-static int caam_debugfs_u64_get(void *data, u64 *val)
-{
- *val = caam64_to_cpu(*(u64 *)data);
- return 0;
-}
-
-static int caam_debugfs_u32_get(void *data, u64 *val)
-{
- *val = caam32_to_cpu(*(u32 *)data);
- return 0;
-}
-
-DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n");
-DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n");
-#endif
-
static inline u64 caam_get_dma_mask(struct device *dev)
{
struct device_node *nprop = dev->of_node;
/**
* caam_jr_free() - Free the Job Ring
- * @rdev - points to the dev that identifies the Job ring to
+ * @rdev: points to the dev that identifies the Job ring to
* be released.
**/
void caam_jr_free(struct device *rdev)
* of this request. This has the form:
* callback(struct device *dev, u32 *desc, u32 stat, void *arg)
* where:
- * @dev: contains the job ring device that processed this
+ * dev: contains the job ring device that processed this
* response.
- * @desc: descriptor that initiated the request, same as
+ * desc: descriptor that initiated the request, same as
* "desc" being argued to caam_jr_enqueue().
- * @status: untranslated status received from CAAM. See the
+ * status: untranslated status received from CAAM. See the
* reference manual for a detailed description of
* error meaning, or see the JRSTA definitions in the
* register header file
- * @areq: optional pointer to an argument passed with the
+ * areq: optional pointer to an argument passed with the
* original request
* @areq: optional pointer to a user argument for use at callback
* time.
#include <linux/kthread.h>
#include <soc/fsl/qman.h>
+#include "debugfs.h"
#include "regs.h"
#include "qi.h"
#include "desc.h"
bool caam_congested __read_mostly;
EXPORT_SYMBOL(caam_congested);
-#ifdef CONFIG_DEBUG_FS
-/*
- * This is a counter for the number of times the congestion group (where all
- * the request and response queueus are) reached congestion. Incremented
- * each time the congestion callback is called with congested == true.
- */
-static u64 times_congested;
-#endif
-
/*
* This is a a cache of buffers, from which the users of CAAM QI driver
* can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than
caam_congested = congested;
if (congested) {
-#ifdef CONFIG_DEBUG_FS
- times_congested++;
-#endif
+ caam_debugfs_qi_congested();
+
pr_debug_ratelimited("CAAM entered congestion\n");
} else {
return -ENOMEM;
}
-#ifdef CONFIG_DEBUG_FS
- debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
- ×_congested, &caam_fops_u64_ro);
-#endif
+ caam_debugfs_qi_init(ctrlpriv);
err = devm_add_action_or_reset(qidev, caam_qi_shutdown, ctrlpriv);
if (err)
static inline int cav_register_algs(void)
{
- int err = 0;
-
- err = crypto_register_skciphers(algs, ARRAY_SIZE(algs));
- if (err)
- return err;
-
- return 0;
+ return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
}
static inline void cav_unregister_algs(void)
err = pci_request_mem_regions(pdev, nitrox_driver_name);
if (err) {
pci_disable_device(pdev);
+ dev_err(&pdev->dev, "Failed to request mem regions!\n");
return err;
}
pci_set_master(pdev);
#include <linux/debugfs.h>
/* Displays ZIP device statistics */
-static int zip_show_stats(struct seq_file *s, void *unused)
+static int zip_stats_show(struct seq_file *s, void *unused)
{
u64 val = 0ull;
u64 avg_chunk = 0ull, avg_cr = 0ull;
}
/* Clears stats data */
-static int zip_clear_stats(struct seq_file *s, void *unused)
+static int zip_clear_show(struct seq_file *s, void *unused)
{
int index = 0;
};
/* Prints registers' contents */
-static int zip_print_regs(struct seq_file *s, void *unused)
+static int zip_regs_show(struct seq_file *s, void *unused)
{
u64 val = 0;
int i = 0, index = 0;
return 0;
}
-static int zip_stats_open(struct inode *inode, struct file *file)
-{
- return single_open(file, zip_show_stats, NULL);
-}
-
-static const struct file_operations zip_stats_fops = {
- .owner = THIS_MODULE,
- .open = zip_stats_open,
- .read = seq_read,
- .release = single_release,
-};
-
-static int zip_clear_open(struct inode *inode, struct file *file)
-{
- return single_open(file, zip_clear_stats, NULL);
-}
-
-static const struct file_operations zip_clear_fops = {
- .owner = THIS_MODULE,
- .open = zip_clear_open,
- .read = seq_read,
- .release = single_release,
-};
-
-static int zip_regs_open(struct inode *inode, struct file *file)
-{
- return single_open(file, zip_print_regs, NULL);
-}
-
-static const struct file_operations zip_regs_fops = {
- .owner = THIS_MODULE,
- .open = zip_regs_open,
- .read = seq_read,
- .release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(zip_stats);
+DEFINE_SHOW_ATTRIBUTE(zip_clear);
+DEFINE_SHOW_ATTRIBUTE(zip_regs);
/* Root directory for thunderx_zip debugfs entry */
static struct dentry *zip_debugfs_root;
* Author: Gary R Hook <gary.hook@amd.com>
*/
+#include <linux/dma-mapping.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/interrupt.h>
break;
default:
ret = -EINVAL;
- goto e_ctx;
+ goto e_data;
}
} else {
/* Stash the context */
switch (size) {
case CC_AES_128_BIT_KEY_SIZE:
case CC_AES_192_BIT_KEY_SIZE:
- if (ctx_p->cipher_mode != DRV_CIPHER_XTS &&
- ctx_p->cipher_mode != DRV_CIPHER_BITLOCKER)
+ if (ctx_p->cipher_mode != DRV_CIPHER_XTS)
return 0;
break;
case CC_AES_256_BIT_KEY_SIZE:
case (CC_AES_192_BIT_KEY_SIZE * 2):
case (CC_AES_256_BIT_KEY_SIZE * 2):
if (ctx_p->cipher_mode == DRV_CIPHER_XTS ||
- ctx_p->cipher_mode == DRV_CIPHER_ESSIV ||
- ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER)
+ ctx_p->cipher_mode == DRV_CIPHER_ESSIV)
return 0;
break;
default:
case DRV_CIPHER_ECB:
case DRV_CIPHER_CBC:
case DRV_CIPHER_ESSIV:
- case DRV_CIPHER_BITLOCKER:
if (IS_ALIGNED(size, AES_BLOCK_SIZE))
return 0;
break;
}
if (ctx_p->cipher_mode == DRV_CIPHER_XTS ||
- ctx_p->cipher_mode == DRV_CIPHER_ESSIV ||
- ctx_p->cipher_mode == DRV_CIPHER_BITLOCKER) {
+ ctx_p->cipher_mode == DRV_CIPHER_ESSIV) {
if (hki.hw_key1 == hki.hw_key2) {
dev_err(dev, "Illegal hw key numbers (%d,%d)\n",
hki.hw_key1, hki.hw_key2);
break;
case DRV_CIPHER_XTS:
case DRV_CIPHER_ESSIV:
- case DRV_CIPHER_BITLOCKER:
/* IV */
hw_desc_init(&desc[*seq_size]);
set_setup_mode(&desc[*seq_size], SETUP_WRITE_STATE1);
break;
case DRV_CIPHER_XTS:
case DRV_CIPHER_ESSIV:
- case DRV_CIPHER_BITLOCKER:
break;
default:
dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode);
dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr;
unsigned int key_len = (ctx_p->keylen / 2);
dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr;
- unsigned int du_size = nbytes;
unsigned int key_offset = key_len;
- struct cc_crypto_alg *cc_alg =
- container_of(tfm->__crt_alg, struct cc_crypto_alg,
- skcipher_alg.base);
-
- if (cc_alg->data_unit)
- du_size = cc_alg->data_unit;
-
switch (cipher_mode) {
case DRV_CIPHER_ECB:
break;
break;
case DRV_CIPHER_XTS:
case DRV_CIPHER_ESSIV:
- case DRV_CIPHER_BITLOCKER:
if (cipher_mode == DRV_CIPHER_ESSIV)
key_len = SHA256_DIGEST_SIZE;
(key_dma_addr + key_offset),
key_len, NS_BIT);
}
- set_xex_data_unit_size(&desc[*seq_size], du_size);
+ set_xex_data_unit_size(&desc[*seq_size], nbytes);
set_flow_mode(&desc[*seq_size], S_DIN_to_AES2);
set_key_size_aes(&desc[*seq_size], key_len);
set_setup_mode(&desc[*seq_size], SETUP_LOAD_XEX_KEY);
break;
case DRV_CIPHER_XTS:
case DRV_CIPHER_ESSIV:
- case DRV_CIPHER_BITLOCKER:
/* Load AES key */
hw_desc_init(&desc[*seq_size]);
set_cipher_mode(&desc[*seq_size], cipher_mode);
.std_body = CC_STD_NIST,
.sec_func = true,
},
- {
- .name = "xts512(paes)",
- .driver_name = "xts-paes-du512-ccree",
- .blocksize = 1,
- .template_skcipher = {
- .setkey = cc_cipher_sethkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = CC_HW_KEY_SIZE,
- .max_keysize = CC_HW_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_XTS,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 512,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- .sec_func = true,
- },
- {
- .name = "xts4096(paes)",
- .driver_name = "xts-paes-du4096-ccree",
- .blocksize = 1,
- .template_skcipher = {
- .setkey = cc_cipher_sethkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = CC_HW_KEY_SIZE,
- .max_keysize = CC_HW_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_XTS,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 4096,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- .sec_func = true,
- },
{
.name = "essiv(cbc(paes),sha256)",
.driver_name = "essiv-paes-ccree",
.std_body = CC_STD_NIST,
.sec_func = true,
},
- {
- .name = "essiv512(cbc(paes),sha256)",
- .driver_name = "essiv-paes-du512-ccree",
- .blocksize = AES_BLOCK_SIZE,
- .template_skcipher = {
- .setkey = cc_cipher_sethkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = CC_HW_KEY_SIZE,
- .max_keysize = CC_HW_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_ESSIV,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 512,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- .sec_func = true,
- },
- {
- .name = "essiv4096(cbc(paes),sha256)",
- .driver_name = "essiv-paes-du4096-ccree",
- .blocksize = AES_BLOCK_SIZE,
- .template_skcipher = {
- .setkey = cc_cipher_sethkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = CC_HW_KEY_SIZE,
- .max_keysize = CC_HW_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_ESSIV,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 4096,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- .sec_func = true,
- },
- {
- .name = "bitlocker(paes)",
- .driver_name = "bitlocker-paes-ccree",
- .blocksize = AES_BLOCK_SIZE,
- .template_skcipher = {
- .setkey = cc_cipher_sethkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = CC_HW_KEY_SIZE,
- .max_keysize = CC_HW_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_BITLOCKER,
- .flow_mode = S_DIN_to_AES,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- .sec_func = true,
- },
- {
- .name = "bitlocker512(paes)",
- .driver_name = "bitlocker-paes-du512-ccree",
- .blocksize = AES_BLOCK_SIZE,
- .template_skcipher = {
- .setkey = cc_cipher_sethkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = CC_HW_KEY_SIZE,
- .max_keysize = CC_HW_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_BITLOCKER,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 512,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- .sec_func = true,
- },
- {
- .name = "bitlocker4096(paes)",
- .driver_name = "bitlocker-paes-du4096-ccree",
- .blocksize = AES_BLOCK_SIZE,
- .template_skcipher = {
- .setkey = cc_cipher_sethkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = CC_HW_KEY_SIZE,
- .max_keysize = CC_HW_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_BITLOCKER,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 4096,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- .sec_func = true,
- },
{
.name = "ecb(paes)",
.driver_name = "ecb-paes-ccree",
.min_hw_rev = CC_HW_REV_630,
.std_body = CC_STD_NIST,
},
- {
- .name = "xts512(aes)",
- .driver_name = "xts-aes-du512-ccree",
- .blocksize = 1,
- .template_skcipher = {
- .setkey = cc_cipher_setkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE * 2,
- .max_keysize = AES_MAX_KEY_SIZE * 2,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_XTS,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 512,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- },
- {
- .name = "xts4096(aes)",
- .driver_name = "xts-aes-du4096-ccree",
- .blocksize = 1,
- .template_skcipher = {
- .setkey = cc_cipher_setkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE * 2,
- .max_keysize = AES_MAX_KEY_SIZE * 2,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_XTS,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 4096,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- },
{
.name = "essiv(cbc(aes),sha256)",
.driver_name = "essiv-aes-ccree",
.min_hw_rev = CC_HW_REV_712,
.std_body = CC_STD_NIST,
},
- {
- .name = "essiv512(cbc(aes),sha256)",
- .driver_name = "essiv-aes-du512-ccree",
- .blocksize = AES_BLOCK_SIZE,
- .template_skcipher = {
- .setkey = cc_cipher_setkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_ESSIV,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 512,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- },
- {
- .name = "essiv4096(cbc(aes),sha256)",
- .driver_name = "essiv-aes-du4096-ccree",
- .blocksize = AES_BLOCK_SIZE,
- .template_skcipher = {
- .setkey = cc_cipher_setkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_ESSIV,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 4096,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- },
- {
- .name = "bitlocker(aes)",
- .driver_name = "bitlocker-aes-ccree",
- .blocksize = AES_BLOCK_SIZE,
- .template_skcipher = {
- .setkey = cc_cipher_setkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE * 2,
- .max_keysize = AES_MAX_KEY_SIZE * 2,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_BITLOCKER,
- .flow_mode = S_DIN_to_AES,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- },
- {
- .name = "bitlocker512(aes)",
- .driver_name = "bitlocker-aes-du512-ccree",
- .blocksize = AES_BLOCK_SIZE,
- .template_skcipher = {
- .setkey = cc_cipher_setkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE * 2,
- .max_keysize = AES_MAX_KEY_SIZE * 2,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_BITLOCKER,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 512,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- },
- {
- .name = "bitlocker4096(aes)",
- .driver_name = "bitlocker-aes-du4096-ccree",
- .blocksize = AES_BLOCK_SIZE,
- .template_skcipher = {
- .setkey = cc_cipher_setkey,
- .encrypt = cc_cipher_encrypt,
- .decrypt = cc_cipher_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE * 2,
- .max_keysize = AES_MAX_KEY_SIZE * 2,
- .ivsize = AES_BLOCK_SIZE,
- },
- .cipher_mode = DRV_CIPHER_BITLOCKER,
- .flow_mode = S_DIN_to_AES,
- .data_unit = 4096,
- .min_hw_rev = CC_HW_REV_712,
- .std_body = CC_STD_NIST,
- },
{
.name = "ecb(aes)",
.driver_name = "ecb-aes-ccree",
t_alg->cipher_mode = tmpl->cipher_mode;
t_alg->flow_mode = tmpl->flow_mode;
- t_alg->data_unit = tmpl->data_unit;
return t_alg;
}
DRV_CIPHER_CBC_CTS = 11,
DRV_CIPHER_GCTR = 12,
DRV_CIPHER_ESSIV = 13,
- DRV_CIPHER_BITLOCKER = 14,
DRV_CIPHER_RESERVE32B = S32_MAX
};
new_drvdata->plat_dev = plat_dev;
clk = devm_clk_get_optional(dev, NULL);
- if (IS_ERR(clk)) {
- if (PTR_ERR(clk) != -EPROBE_DEFER)
- dev_err(dev, "Error getting clock: %pe\n", clk);
- return PTR_ERR(clk);
- }
+ if (IS_ERR(clk))
+ return dev_err_probe(dev, PTR_ERR(clk), "Error getting clock\n");
new_drvdata->clk = clk;
new_drvdata->coherent = of_dma_is_coherent(np);
int cipher_mode;
int flow_mode; /* Note: currently, refers to the cipher mode only. */
int auth_mode;
- unsigned int data_unit;
struct cc_drvdata *drvdata;
struct skcipher_alg skcipher_alg;
struct aead_alg aead_alg;
int cc_pm_get(struct device *dev)
{
int rc = pm_runtime_get_sync(dev);
+ if (rc < 0) {
+ pm_runtime_put_noidle(dev);
+ return rc;
+ }
- return (rc == 1 ? 0 : rc);
+ return 0;
}
void cc_pm_put_suspend(struct device *dev)
static int chcr_dev_move(struct uld_ctx *u_ctx)
{
- mutex_lock(&drv_data.drv_mutex);
+ mutex_lock(&drv_data.drv_mutex);
if (drv_data.last_dev == u_ctx) {
if (list_is_last(&drv_data.last_dev->entry, &drv_data.act_dev))
drv_data.last_dev = list_first_entry(&drv_data.act_dev,
dev->pk_clk_freq) * 256;
dev->rng.name = dev->name;
- dev->rng.data_present = hifn_rng_data_present,
- dev->rng.data_read = hifn_rng_data_read,
+ dev->rng.data_present = hifn_rng_data_present;
+ dev->rng.data_read = hifn_rng_data_read;
dev->rng.priv = (unsigned long)dev;
return hwrng_register(&dev->rng);
int idx;
dma_addr_t addr;
- addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
+ addr = dma_map_page(&dev->pdev->dev, page, offset, size,
+ DMA_TO_DEVICE);
idx = dma->srci;
int idx;
dma_addr_t addr;
- addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
+ addr = dma_map_page(&dev->pdev->dev, page, offset, size,
+ DMA_FROM_DEVICE);
idx = dma->dsti;
dma->dstr[idx].p = __cpu_to_le32(addr);
return err;
pci_set_master(pdev);
- err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+ err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
if (err)
goto err_out_disable_pci_device;
}
}
- dev->desc_virt = pci_zalloc_consistent(pdev, sizeof(struct hifn_dma),
- &dev->desc_dma);
+ dev->desc_virt = dma_alloc_coherent(&pdev->dev,
+ sizeof(struct hifn_dma),
+ &dev->desc_dma, GFP_KERNEL);
if (!dev->desc_virt) {
dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n");
err = -ENOMEM;
free_irq(dev->irq, dev);
tasklet_kill(&dev->tasklet);
err_out_free_desc:
- pci_free_consistent(pdev, sizeof(struct hifn_dma),
- dev->desc_virt, dev->desc_dma);
+ dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt,
+ dev->desc_dma);
err_out_unmap_bars:
for (i = 0; i < 3; ++i)
hifn_flush(dev);
- pci_free_consistent(pdev, sizeof(struct hifn_dma),
- dev->desc_virt, dev->desc_dma);
+ dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma),
+ dev->desc_virt, dev->desc_dma);
for (i = 0; i < 3; ++i)
if (dev->bar[i])
iounmap(dev->bar[i]);
unsigned int freq;
int err;
- /* HIFN supports only 32-bit addresses */
- BUILD_BUG_ON(sizeof(dma_addr_t) != 4);
-
if (strncmp(hifn_pll_ref, "ext", 3) &&
strncmp(hifn_pll_ref, "pci", 3)) {
pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext");
* Just relevant for PF.
*/
struct hpre_debug {
- struct dentry *debug_root;
struct hpre_dfx dfx[HPRE_DFX_FILE_NUM];
struct hpre_debugfs_file files[HPRE_DEBUGFS_FILE_NUM];
};
struct timespec64 req_time;
};
-static DEFINE_MUTEX(hpre_alg_lock);
-static unsigned int hpre_active_devs;
-
static int hpre_alloc_req_id(struct hpre_ctx *ctx)
{
unsigned long flags;
hpre_req->dst = NULL;
dma_dir = DMA_FROM_DEVICE;
}
- *tmp = dma_map_single(dev, sg_virt(data),
- len, dma_dir);
+ *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir);
if (unlikely(dma_mapping_error(dev, *tmp))) {
dev_err(dev, "dma map data err!\n");
return -ENOMEM;
((is_dh && !is_src) || !is_dh))
ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp);
else
- ret = hpre_prepare_dma_buf(hpre_req, data, len,
- is_src, &tmp);
+ ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp);
+
if (unlikely(ret))
return ret;
if (src) {
if (req->src)
- dma_free_coherent(dev, ctx->key_sz,
- req->src, tmp);
+ dma_free_coherent(dev, ctx->key_sz, req->src, tmp);
else
- dma_unmap_single(dev, tmp,
- ctx->key_sz, DMA_TO_DEVICE);
+ dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE);
}
tmp = le64_to_cpu(sqe->out);
h_req->areq.dh = kreq;
msg = &h_req->req;
memset(msg, 0, sizeof(*msg));
- msg->key = cpu_to_le64((u64)ctx->dh.dma_xa_p);
+ msg->key = cpu_to_le64(ctx->dh.dma_xa_p);
}
msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT);
ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1);
if (unlikely(ret))
goto clear_all;
+ } else {
+ msg->in = cpu_to_le64(ctx->dh.dma_g);
}
ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1);
return ret;
msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT);
- msg->key = cpu_to_le64((u64)ctx->rsa.dma_pubkey);
+ msg->key = cpu_to_le64(ctx->rsa.dma_pubkey);
ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
if (unlikely(ret))
return ret;
if (ctx->crt_g2_mode) {
- msg->key = cpu_to_le64((u64)ctx->rsa.dma_crt_prikey);
+ msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey);
msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
HPRE_ALG_NC_CRT);
} else {
- msg->key = cpu_to_le64((u64)ctx->rsa.dma_prikey);
+ msg->key = cpu_to_le64(ctx->rsa.dma_prikey);
msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
HPRE_ALG_NC_NCRT);
}
int hpre_algs_register(void)
{
- int ret = 0;
-
- mutex_lock(&hpre_alg_lock);
- if (++hpre_active_devs == 1) {
- rsa.base.cra_flags = 0;
- ret = crypto_register_akcipher(&rsa);
- if (ret)
- goto unlock;
+ int ret;
+
+ rsa.base.cra_flags = 0;
+ ret = crypto_register_akcipher(&rsa);
+ if (ret)
+ return ret;
#ifdef CONFIG_CRYPTO_DH
- ret = crypto_register_kpp(&dh);
- if (ret) {
- crypto_unregister_akcipher(&rsa);
- goto unlock;
- }
+ ret = crypto_register_kpp(&dh);
+ if (ret)
+ crypto_unregister_akcipher(&rsa);
#endif
- }
-unlock:
- mutex_unlock(&hpre_alg_lock);
return ret;
}
void hpre_algs_unregister(void)
{
- mutex_lock(&hpre_alg_lock);
- if (--hpre_active_devs == 0) {
- crypto_unregister_akcipher(&rsa);
+ crypto_unregister_akcipher(&rsa);
#ifdef CONFIG_CRYPTO_DH
- crypto_unregister_kpp(&dh);
+ crypto_unregister_kpp(&dh);
#endif
- }
- mutex_unlock(&hpre_alg_lock);
}
#define HPRE_SQE_MASK_OFFSET 8
#define HPRE_SQE_MASK_LEN 24
-static struct hisi_qm_list hpre_devices;
static const char hpre_name[] = "hisi_hpre";
static struct dentry *hpre_debugfs_root;
static const struct pci_device_id hpre_dev_ids[] = {
const char *msg;
};
+static struct hisi_qm_list hpre_devices = {
+ .register_to_crypto = hpre_algs_register,
+ .unregister_from_crypto = hpre_algs_unregister,
+};
+
static const char * const hpre_debug_file_name[] = {
[HPRE_CURRENT_QM] = "current_qm",
[HPRE_CLEAR_ENABLE] = "rdclr_en",
static u32 pf_q_num = HPRE_PF_DEF_Q_NUM;
module_param_cb(pf_q_num, &hpre_pf_q_num_ops, &pf_q_num, 0444);
-MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(1-1024)");
+MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(2-1024)");
static const struct kernel_param_ops vfs_num_ops = {
.set = vfs_num_set,
if (ret)
dev_warn(&pdev->dev, "init debugfs fail!\n");
- hisi_qm_add_to_list(qm, &hpre_devices);
-
- ret = hpre_algs_register();
+ ret = hisi_qm_alg_register(qm, &hpre_devices);
if (ret < 0) {
pci_err(pdev, "fail to register algs to crypto!\n");
goto err_with_qm_start;
if (qm->fun_type == QM_HW_PF && vfs_num) {
ret = hisi_qm_sriov_enable(pdev, vfs_num);
if (ret < 0)
- goto err_with_crypto_register;
+ goto err_with_alg_register;
}
return 0;
-err_with_crypto_register:
- hpre_algs_unregister();
+err_with_alg_register:
+ hisi_qm_alg_unregister(qm, &hpre_devices);
err_with_qm_start:
- hisi_qm_del_from_list(qm, &hpre_devices);
hpre_debugfs_exit(qm);
- hisi_qm_stop(qm);
+ hisi_qm_stop(qm, QM_NORMAL);
err_with_err_init:
hisi_qm_dev_err_uninit(qm);
static void hpre_remove(struct pci_dev *pdev)
{
- struct hpre *hpre = pci_get_drvdata(pdev);
- struct hisi_qm *qm = &hpre->qm;
+ struct hisi_qm *qm = pci_get_drvdata(pdev);
int ret;
- hpre_algs_unregister();
- hisi_qm_del_from_list(qm, &hpre_devices);
+ hisi_qm_wait_task_finish(qm, &hpre_devices);
+ hisi_qm_alg_unregister(qm, &hpre_devices);
if (qm->fun_type == QM_HW_PF && qm->vfs_num) {
- ret = hisi_qm_sriov_disable(pdev);
+ ret = hisi_qm_sriov_disable(pdev, qm->is_frozen);
if (ret) {
pci_err(pdev, "Disable SRIOV fail!\n");
return;
}
hpre_debugfs_exit(qm);
- hisi_qm_stop(qm);
+ hisi_qm_stop(qm, QM_NORMAL);
hisi_qm_dev_err_uninit(qm);
hisi_qm_uninit(qm);
}
.sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ?
hisi_qm_sriov_configure : NULL,
.err_handler = &hpre_err_handler,
+ .shutdown = hisi_qm_dev_shutdown,
};
static void hpre_register_debugfs(void)
#define QM_DBG_TMP_BUF_LEN 22
#define QM_PCI_COMMAND_INVALID ~0
+#define WAIT_PERIOD 20
+#define REMOVE_WAIT_DELAY 10
#define QM_SQE_ADDR_MASK GENMASK(7, 0)
+#define QM_EQ_DEPTH (1024 * 2)
#define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \
(((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \
qp = qm_to_hisi_qp(qm, eqe);
qm_poll_qp(qp, qm);
- if (qm->status.eq_head == QM_Q_DEPTH - 1) {
+ if (qm->status.eq_head == QM_EQ_DEPTH - 1) {
qm->status.eqc_phase = !qm->status.eqc_phase;
eqe = qm->eqe;
qm->status.eq_head = 0;
qm->status.eq_head++;
}
- if (eqe_num == QM_Q_DEPTH / 2 - 1) {
+ if (eqe_num == QM_EQ_DEPTH / 2 - 1) {
eqe_num = 0;
qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
}
qp_status->sq_tail = 0;
qp_status->cq_head = 0;
qp_status->cqc_phase = true;
- atomic_set(&qp_status->flags, 0);
+ atomic_set(&qp_status->used, 0);
}
static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,
return 0;
}
-static int qm_regs_open(struct inode *inode, struct file *file)
-{
- return single_open(file, qm_regs_show, inode->i_private);
-}
-
-static const struct file_operations qm_regs_fops = {
- .owner = THIS_MODULE,
- .open = qm_regs_open,
- .read = seq_read,
- .release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(qm_regs);
static ssize_t qm_cmd_read(struct file *filp, char __user *buffer,
size_t count, loff_t *pos)
return -EINVAL;
ret = kstrtou32(s, 0, &xeqe_id);
- if (ret || xeqe_id >= QM_Q_DEPTH) {
+ if (ret)
+ return -EINVAL;
+
+ if (!strcmp(name, "EQE") && xeqe_id >= QM_EQ_DEPTH) {
+ dev_err(dev, "Please input eqe num (0-%d)", QM_EQ_DEPTH - 1);
+ return -EINVAL;
+ } else if (!strcmp(name, "AEQE") && xeqe_id >= QM_Q_DEPTH) {
dev_err(dev, "Please input aeqe num (0-%d)", QM_Q_DEPTH - 1);
return -EINVAL;
}
static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf)
{
struct device *dev = &qm->pdev->dev;
- char *presult, *s;
+ char *presult, *s, *s_tmp;
int ret;
s = kstrdup(cmd_buf, GFP_KERNEL);
if (!s)
return -ENOMEM;
+ s_tmp = s;
presult = strsep(&s, " ");
if (!presult) {
- kfree(s);
- return -EINVAL;
+ ret = -EINVAL;
+ goto err_buffer_free;
}
if (!strcmp(presult, "sqc"))
if (ret)
dev_info(dev, "Please echo help\n");
- kfree(s);
+err_buffer_free:
+ kfree(s_tmp);
return ret;
}
struct hisi_qp_status *qp_status = &qp->qp_status;
u16 sq_tail = qp_status->sq_tail;
- if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH))
+ if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH - 1))
return NULL;
return qp->sqe + sq_tail * qp->qm->sqe_size;
if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP ||
atomic_read(&qp->qm->status.flags) == QM_STOP ||
qp->is_resetting)) {
- dev_info(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
+ dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
return -EAGAIN;
}
return 0;
}
+/**
+ * qm_frozen() - Try to froze QM to cut continuous queue request. If
+ * there is user on the QM, return failure without doing anything.
+ * @qm: The qm needed to be fronzen.
+ *
+ * This function frozes QM, then we can do SRIOV disabling.
+ */
+static int qm_frozen(struct hisi_qm *qm)
+{
+ down_write(&qm->qps_lock);
+
+ if (qm->is_frozen) {
+ up_write(&qm->qps_lock);
+ return 0;
+ }
+
+ if (!qm->qp_in_used) {
+ qm->qp_in_used = qm->qp_num;
+ qm->is_frozen = true;
+ up_write(&qm->qps_lock);
+ return 0;
+ }
+
+ up_write(&qm->qps_lock);
+
+ return -EBUSY;
+}
+
+static int qm_try_frozen_vfs(struct pci_dev *pdev,
+ struct hisi_qm_list *qm_list)
+{
+ struct hisi_qm *qm, *vf_qm;
+ struct pci_dev *dev;
+ int ret = 0;
+
+ if (!qm_list || !pdev)
+ return -EINVAL;
+
+ /* Try to frozen all the VFs as disable SRIOV */
+ mutex_lock(&qm_list->lock);
+ list_for_each_entry(qm, &qm_list->list, list) {
+ dev = qm->pdev;
+ if (dev == pdev)
+ continue;
+ if (pci_physfn(dev) == pdev) {
+ vf_qm = pci_get_drvdata(dev);
+ ret = qm_frozen(vf_qm);
+ if (ret)
+ goto frozen_fail;
+ }
+ }
+
+frozen_fail:
+ mutex_unlock(&qm_list->lock);
+
+ return ret;
+}
+
+/**
+ * hisi_qm_wait_task_finish() - Wait until the task is finished
+ * when removing the driver.
+ * @qm: The qm needed to wait for the task to finish.
+ * @qm_list: The list of all available devices.
+ */
+void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
+{
+ while (qm_frozen(qm) ||
+ ((qm->fun_type == QM_HW_PF) &&
+ qm_try_frozen_vfs(qm->pdev, qm_list))) {
+ msleep(WAIT_PERIOD);
+ }
+
+ udelay(REMOVE_WAIT_DELAY);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish);
+
/**
* hisi_qm_get_free_qp_num() - Get free number of qp in qm.
* @qm: The qm which want to get free qp.
} while (0)
idr_init(&qm->qp_idr);
- qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_Q_DEPTH) +
+ qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) +
QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) +
QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) +
QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num);
if (!qm->qdma.va)
return -ENOMEM;
- QM_INIT_BUF(qm, eqe, QM_Q_DEPTH);
+ QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH);
QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH);
QM_INIT_BUF(qm, sqc, qm->qp_num);
QM_INIT_BUF(qm, cqc, qm->qp_num);
mutex_init(&qm->mailbox_lock);
init_rwsem(&qm->qps_lock);
qm->qp_in_used = 0;
+ qm->is_frozen = false;
}
/**
eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma));
if (qm->ver == QM_HW_V1)
eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE);
- eqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT));
+ eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT));
ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0);
dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE);
kfree(eqc);
/**
* hisi_qm_stop() - Stop a qm.
* @qm: The qm which will be stopped.
+ * @r: The reason to stop qm.
*
* This function stops qm and its qps, then qm can not accept request.
* Related resources are not released at this state, we can use hisi_qm_start
* to let qm start again.
*/
-int hisi_qm_stop(struct hisi_qm *qm)
+int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
{
struct device *dev = &qm->pdev->dev;
int ret = 0;
down_write(&qm->qps_lock);
+ qm->status.stop_reason = r;
if (!qm_avail_state(qm, QM_STOP)) {
ret = -EPERM;
goto err_unlock;
/**
* hisi_qm_sriov_disable - disable virtual functions
- * @pdev: the PCI device
+ * @pdev: the PCI device.
+ * @is_frozen: true when all the VFs are frozen.
*
- * Return failure if there are VFs assigned already.
+ * Return failure if there are VFs assigned already or VF is in used.
*/
-int hisi_qm_sriov_disable(struct pci_dev *pdev)
+int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen)
{
struct hisi_qm *qm = pci_get_drvdata(pdev);
return -EPERM;
}
- /* remove in hpre_pci_driver will be called to free VF resources */
+ /* While VF is in used, SRIOV cannot be disabled. */
+ if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) {
+ pci_err(pdev, "Task is using its VF!\n");
+ return -EBUSY;
+ }
+
pci_disable_sriov(pdev);
return qm_clear_vft_config(qm);
}
int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
if (num_vfs == 0)
- return hisi_qm_sriov_disable(pdev);
+ return hisi_qm_sriov_disable(pdev, 0);
else
return hisi_qm_sriov_enable(pdev, num_vfs);
}
return 0;
}
-static int qm_vf_reset_prepare(struct hisi_qm *qm)
+static int qm_vf_reset_prepare(struct hisi_qm *qm,
+ enum qm_stop_reason stop_reason)
{
struct hisi_qm_list *qm_list = qm->qm_list;
- int stop_reason = qm->status.stop_reason;
struct pci_dev *pdev = qm->pdev;
struct pci_dev *virtfn;
struct hisi_qm *vf_qm;
continue;
if (pci_physfn(virtfn) == pdev) {
- vf_qm->status.stop_reason = stop_reason;
- ret = hisi_qm_stop(vf_qm);
+ /* save VFs PCIE BAR configuration */
+ pci_save_state(virtfn);
+
+ ret = hisi_qm_stop(vf_qm, stop_reason);
if (ret)
goto stop_fail;
}
}
if (qm->vfs_num) {
- ret = qm_vf_reset_prepare(qm);
+ ret = qm_vf_reset_prepare(qm, QM_SOFT_RESET);
if (ret) {
pci_err(pdev, "Fails to stop VFs!\n");
return ret;
}
}
- qm->status.stop_reason = QM_SOFT_RESET;
- ret = hisi_qm_stop(qm);
+ ret = hisi_qm_stop(qm, QM_SOFT_RESET);
if (ret) {
pci_err(pdev, "Fails to stop QM!\n");
return ret;
continue;
if (pci_physfn(virtfn) == pdev) {
+ /* enable VFs PCIE BAR configuration */
+ pci_restore_state(virtfn);
+
ret = qm_restart(vf_qm);
if (ret)
goto restart_fail;
}
if (qm->vfs_num) {
- ret = qm_vf_reset_prepare(qm);
+ ret = qm_vf_reset_prepare(qm, QM_FLR);
if (ret) {
pci_err(pdev, "Failed to prepare reset, ret = %d.\n",
ret);
}
}
- ret = hisi_qm_stop(qm);
+ ret = hisi_qm_stop(qm, QM_FLR);
if (ret) {
pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret);
return;
return ret;
}
+/**
+ * hisi_qm_dev_shutdown() - Shutdown device.
+ * @pdev: The device will be shutdown.
+ *
+ * This function will stop qm when OS shutdown or rebooting.
+ */
+void hisi_qm_dev_shutdown(struct pci_dev *pdev)
+{
+ struct hisi_qm *qm = pci_get_drvdata(pdev);
+ int ret;
+
+ ret = hisi_qm_stop(qm, QM_NORMAL);
+ if (ret)
+ dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n");
+}
+EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown);
+
static void hisi_qm_controller_reset(struct work_struct *rst_work)
{
struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work);
}
+/**
+ * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list.
+ * @qm: The qm needs add.
+ * @qm_list: The qm list.
+ *
+ * This function adds qm to qm list, and will register algorithm to
+ * crypto when the qm list is empty.
+ */
+int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
+{
+ int flag = 0;
+ int ret = 0;
+
+ mutex_lock(&qm_list->lock);
+ if (list_empty(&qm_list->list))
+ flag = 1;
+ list_add_tail(&qm->list, &qm_list->list);
+ mutex_unlock(&qm_list->lock);
+
+ if (flag) {
+ ret = qm_list->register_to_crypto();
+ if (ret) {
+ mutex_lock(&qm_list->lock);
+ list_del(&qm->list);
+ mutex_unlock(&qm_list->lock);
+ }
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(hisi_qm_alg_register);
+
+/**
+ * hisi_qm_alg_unregister() - Unregister alg from crypto and delete qm from
+ * qm list.
+ * @qm: The qm needs delete.
+ * @qm_list: The qm list.
+ *
+ * This function deletes qm from qm list, and will unregister algorithm
+ * from crypto when the qm list is empty.
+ */
+void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
+{
+ mutex_lock(&qm_list->lock);
+ list_del(&qm->list);
+ mutex_unlock(&qm_list->lock);
+
+ if (list_empty(&qm_list->list))
+ qm_list->unregister_from_crypto();
+}
+EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister);
+
/**
* hisi_qm_init() - Initialize configures about qm.
* @qm: The qm needing init.
#define QM_BASE_CE QM_ECC_1BIT
#define QM_Q_DEPTH 1024
-
+#define QM_MIN_QNUM 2
#define HISI_ACC_SGL_SGE_NR_MAX 255
/* page number for queue file region */
struct hisi_qm_list {
struct mutex lock;
struct list_head list;
+ int (*register_to_crypto)(void);
+ void (*unregister_from_crypto)(void);
};
struct hisi_qm {
const char *algs;
bool use_sva;
+ bool is_frozen;
resource_size_t phys_base;
resource_size_t phys_size;
struct uacce_device *uacce;
}
ret = kstrtou32(val, 10, &n);
- if (ret || !n || n > q_num)
+ if (ret || n < QM_MIN_QNUM || n > q_num)
return -EINVAL;
return param_set_int(val, kp);
mutex_init(&qm_list->lock);
}
-static inline void hisi_qm_add_to_list(struct hisi_qm *qm,
- struct hisi_qm_list *qm_list)
-{
- mutex_lock(&qm_list->lock);
- list_add_tail(&qm->list, &qm_list->list);
- mutex_unlock(&qm_list->lock);
-}
-
-static inline void hisi_qm_del_from_list(struct hisi_qm *qm,
- struct hisi_qm_list *qm_list)
-{
- mutex_lock(&qm_list->lock);
- list_del(&qm->list);
- mutex_unlock(&qm_list->lock);
-}
-
int hisi_qm_init(struct hisi_qm *qm);
void hisi_qm_uninit(struct hisi_qm *qm);
int hisi_qm_start(struct hisi_qm *qm);
-int hisi_qm_stop(struct hisi_qm *qm);
+int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r);
struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type);
int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg);
int hisi_qm_stop_qp(struct hisi_qp *qp);
enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev);
void hisi_qm_debug_regs_clear(struct hisi_qm *qm);
int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs);
-int hisi_qm_sriov_disable(struct pci_dev *pdev);
+int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen);
int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs);
void hisi_qm_dev_err_init(struct hisi_qm *qm);
void hisi_qm_dev_err_uninit(struct hisi_qm *qm);
int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num,
u8 alg_type, int node, struct hisi_qp **qps);
void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num);
+void hisi_qm_dev_shutdown(struct pci_dev *pdev);
+void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list);
+int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list);
+void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list);
#endif
#define SEC_SQE_AEAD_FLAG 3
#define SEC_SQE_DONE 0x1
-static atomic_t sec_active_devs;
-
/* Get an en/de-cipher queue cyclically to balance load over queues of TFM */
static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req)
{
ret = sec_alloc_pbuf_resource(dev, res);
if (ret) {
dev_err(dev, "fail to alloc pbuf dma resource!\n");
- goto alloc_fail;
+ goto alloc_pbuf_fail;
}
}
return 0;
+alloc_pbuf_fail:
+ if (ctx->alg_type == SEC_AEAD)
+ sec_free_mac_resource(dev, qp_ctx->res);
alloc_fail:
sec_free_civ_resource(dev, res);
ctx->fake_req_limit = QM_Q_DEPTH >> 1;
ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx),
GFP_KERNEL);
- if (!ctx->qp_ctx)
- return -ENOMEM;
+ if (!ctx->qp_ctx) {
+ ret = -ENOMEM;
+ goto err_destroy_qps;
+ }
for (i = 0; i < sec->ctx_q_num; i++) {
ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0);
}
return 0;
+
err_sec_release_qp_ctx:
for (i = i - 1; i >= 0; i--)
sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
- sec_destroy_qps(ctx->qps, sec->ctx_q_num);
kfree(ctx->qp_ctx);
+err_destroy_qps:
+ sec_destroy_qps(ctx->qps, sec->ctx_q_num);
+
return ret;
}
int sec_register_to_crypto(void)
{
- int ret = 0;
+ int ret;
/* To avoid repeat register */
- if (atomic_add_return(1, &sec_active_devs) == 1) {
- ret = crypto_register_skciphers(sec_skciphers,
- ARRAY_SIZE(sec_skciphers));
- if (ret)
- return ret;
-
- ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads));
- if (ret)
- goto reg_aead_fail;
- }
-
- return ret;
-
-reg_aead_fail:
- crypto_unregister_skciphers(sec_skciphers, ARRAY_SIZE(sec_skciphers));
+ ret = crypto_register_skciphers(sec_skciphers,
+ ARRAY_SIZE(sec_skciphers));
+ if (ret)
+ return ret;
+ ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads));
+ if (ret)
+ crypto_unregister_skciphers(sec_skciphers,
+ ARRAY_SIZE(sec_skciphers));
return ret;
}
void sec_unregister_from_crypto(void)
{
- if (atomic_sub_return(1, &sec_active_devs) == 0) {
- crypto_unregister_skciphers(sec_skciphers,
- ARRAY_SIZE(sec_skciphers));
- crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads));
- }
+ crypto_unregister_skciphers(sec_skciphers,
+ ARRAY_SIZE(sec_skciphers));
+ crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads));
}
static const char sec_name[] = "hisi_sec2";
static struct dentry *sec_debugfs_root;
-static struct hisi_qm_list sec_devices;
+
+static struct hisi_qm_list sec_devices = {
+ .register_to_crypto = sec_register_to_crypto,
+ .unregister_from_crypto = sec_unregister_from_crypto,
+};
static const struct sec_hw_error sec_hw_errors[] = {
{.int_msk = BIT(0), .msg = "sec_axi_rresp_err_rint"},
static u32 pf_q_num = SEC_PF_DEF_Q_NUM;
module_param_cb(pf_q_num, &sec_pf_q_num_ops, &pf_q_num, 0444);
-MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 0-4096, v2 0-1024)");
+MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)");
static int sec_ctx_q_num_set(const char *val, const struct kernel_param *kp)
{
if (ret)
pci_warn(pdev, "Failed to init debugfs!\n");
- hisi_qm_add_to_list(qm, &sec_devices);
-
- ret = sec_register_to_crypto();
+ ret = hisi_qm_alg_register(qm, &sec_devices);
if (ret < 0) {
pr_err("Failed to register driver to crypto.\n");
- goto err_remove_from_list;
+ goto err_qm_stop;
}
if (qm->fun_type == QM_HW_PF && vfs_num) {
ret = hisi_qm_sriov_enable(pdev, vfs_num);
if (ret < 0)
- goto err_crypto_unregister;
+ goto err_alg_unregister;
}
return 0;
-err_crypto_unregister:
- sec_unregister_from_crypto();
+err_alg_unregister:
+ hisi_qm_alg_unregister(qm, &sec_devices);
-err_remove_from_list:
- hisi_qm_del_from_list(qm, &sec_devices);
+err_qm_stop:
sec_debugfs_exit(qm);
- hisi_qm_stop(qm);
+ hisi_qm_stop(qm, QM_NORMAL);
err_probe_uninit:
sec_probe_uninit(qm);
static void sec_remove(struct pci_dev *pdev)
{
- struct sec_dev *sec = pci_get_drvdata(pdev);
- struct hisi_qm *qm = &sec->qm;
-
- sec_unregister_from_crypto();
-
- hisi_qm_del_from_list(qm, &sec_devices);
+ struct hisi_qm *qm = pci_get_drvdata(pdev);
+ hisi_qm_wait_task_finish(qm, &sec_devices);
+ hisi_qm_alg_unregister(qm, &sec_devices);
if (qm->fun_type == QM_HW_PF && qm->vfs_num)
- hisi_qm_sriov_disable(pdev);
+ hisi_qm_sriov_disable(pdev, qm->is_frozen);
sec_debugfs_exit(qm);
- (void)hisi_qm_stop(qm);
+ (void)hisi_qm_stop(qm, QM_NORMAL);
if (qm->fun_type == QM_HW_PF)
sec_debug_regs_clear(qm);
.remove = sec_remove,
.err_handler = &sec_err_handler,
.sriov_configure = hisi_qm_sriov_configure,
+ .shutdown = hisi_qm_dev_shutdown,
};
static void sec_register_debugfs(void)
#include <linux/list.h>
#include "../qm.h"
-/* hisi_zip_sqe dw3 */
-#define HZIP_BD_STATUS_M GENMASK(7, 0)
-/* hisi_zip_sqe dw7 */
-#define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0)
-/* hisi_zip_sqe dw8 */
-#define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0)
-/* hisi_zip_sqe dw9 */
-#define HZIP_REQ_TYPE_M GENMASK(7, 0)
-#define HZIP_ALG_TYPE_ZLIB 0x02
-#define HZIP_ALG_TYPE_GZIP 0x03
-#define HZIP_BUF_TYPE_M GENMASK(11, 8)
-#define HZIP_PBUFFER 0x0
-#define HZIP_SGL 0x1
-
enum hisi_zip_error_type {
/* negative compression */
HZIP_NC_ERR = 0x0d,
struct hisi_zip {
struct hisi_qm qm;
- struct list_head list;
struct hisi_zip_ctrl *ctrl;
struct hisi_zip_dfx dfx;
};
#include <linux/scatterlist.h>
#include "zip.h"
+/* hisi_zip_sqe dw3 */
+#define HZIP_BD_STATUS_M GENMASK(7, 0)
+/* hisi_zip_sqe dw7 */
+#define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0)
+/* hisi_zip_sqe dw8 */
+#define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0)
+/* hisi_zip_sqe dw9 */
+#define HZIP_REQ_TYPE_M GENMASK(7, 0)
+#define HZIP_ALG_TYPE_ZLIB 0x02
+#define HZIP_ALG_TYPE_GZIP 0x03
+#define HZIP_BUF_TYPE_M GENMASK(11, 8)
+#define HZIP_PBUFFER 0x0
+#define HZIP_SGL 0x1
+
#define HZIP_ZLIB_HEAD_SIZE 2
#define HZIP_GZIP_HEAD_SIZE 10
#define GZIP_HEAD_FLG_SHIFT 3
#define GZIP_HEAD_FEXTRA_SHIFT 10
-#define GZIP_HEAD_FEXTRA_XLEN 2
+#define GZIP_HEAD_FEXTRA_XLEN 2UL
#define GZIP_HEAD_FHCRC_SIZE 2
-#define HZIP_CTX_Q_NUM 2
#define HZIP_GZIP_HEAD_BUF 256
#define HZIP_ALG_PRIORITY 300
#define HZIP_SGL_SGE_NR 10
static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c};
-static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {0x1f, 0x8b, 0x08, 0x0, 0x0,
- 0x0, 0x0, 0x0, 0x0, 0x03};
+static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = {
+ 0x1f, 0x8b, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x03
+};
+
enum hisi_zip_alg_type {
HZIP_ALG_TYPE_COMP = 0,
HZIP_ALG_TYPE_DECOMP = 1,
};
+enum {
+ HZIP_QPC_COMP,
+ HZIP_QPC_DECOMP,
+ HZIP_CTX_Q_NUM
+};
+
#define COMP_NAME_TO_TYPE(alg_name) \
(!strcmp((alg_name), "zlib-deflate") ? HZIP_ALG_TYPE_ZLIB : \
!strcmp((alg_name), "gzip") ? HZIP_ALG_TYPE_GZIP : 0) \
struct hisi_zip_req {
struct acomp_req *req;
- int sskip;
- int dskip;
+ u32 sskip;
+ u32 dskip;
struct hisi_acc_hw_sgl *hw_src;
struct hisi_acc_hw_sgl *hw_dst;
dma_addr_t dma_src;
dma_addr_t dma_dst;
- int req_id;
+ u16 req_id;
};
struct hisi_zip_req_q {
};
struct hisi_zip_ctx {
-#define QPC_COMP 0
-#define QPC_DECOMP 1
struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM];
};
static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type,
dma_addr_t s_addr, dma_addr_t d_addr, u32 slen,
- u32 dlen, int sskip, int dskip)
+ u32 dlen, u32 sskip, u32 dskip)
{
memset(sqe, 0, sizeof(struct hisi_zip_sqe));
ret = hisi_qm_start_qp(qp, 0);
if (ret < 0) {
- dev_err(dev, "start qp failed!\n");
+ dev_err(dev, "failed to start qp (%d)!\n", ret);
return ret;
}
ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node);
if (ret) {
- pr_err("Can not create zip qps!\n");
+ pr_err("failed to create zip qps (%d)!\n", ret);
return -ENODEV;
}
return 0;
err_free_loop1:
- kfree(ctx->qp_ctx[QPC_DECOMP].req_q.req_bitmap);
+ kfree(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap);
err_free_loop0:
- kfree(ctx->qp_ctx[QPC_COMP].req_q.q);
+ kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q);
err_free_bitmap:
- kfree(ctx->qp_ctx[QPC_COMP].req_q.req_bitmap);
+ kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap);
return ret;
}
return 0;
err_free_sgl_pool0:
- hisi_acc_free_sgl_pool(&ctx->qp_ctx[QPC_COMP].qp->qm->pdev->dev,
- ctx->qp_ctx[QPC_COMP].sgl_pool);
+ hisi_acc_free_sgl_pool(&ctx->qp_ctx[HZIP_QPC_COMP].qp->qm->pdev->dev,
+ ctx->qp_ctx[HZIP_QPC_COMP].sgl_pool);
return -ENOMEM;
}
atomic64_inc(&dfx->recv_cnt);
status = sqe->dw3 & HZIP_BD_STATUS_M;
-
if (status != 0 && status != HZIP_NC_ERR) {
dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n",
(qp->alg_type == 0) ? "" : "de", qp->qp_id, status,
{
const char *alg_name = crypto_tfm_alg_name(&tfm->base);
struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base);
+ struct device *dev;
int ret;
ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node);
- if (ret)
+ if (ret) {
+ pr_err("failed to init ctx (%d)!\n", ret);
return ret;
+ }
+
+ dev = &ctx->qp_ctx[0].qp->qm->pdev->dev;
ret = hisi_zip_create_req_q(ctx);
- if (ret)
+ if (ret) {
+ dev_err(dev, "failed to create request queue (%d)!\n", ret);
goto err_ctx_exit;
+ }
ret = hisi_zip_create_sgl_pool(ctx);
- if (ret)
+ if (ret) {
+ dev_err(dev, "failed to create sgl pool (%d)!\n", ret);
goto err_release_req_q;
+ }
hisi_zip_set_acomp_cb(ctx, hisi_zip_acomp_cb);
int ret;
ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size);
- if (ret != head_size)
+ if (ret != head_size) {
+ pr_err("the head size of buffer is wrong (%d)!\n", ret);
return -ENOMEM;
+ }
return head_size;
}
-static size_t get_gzip_head_size(struct scatterlist *sgl)
+static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl)
{
char buf[HZIP_GZIP_HEAD_BUF];
return __get_gzip_head_size(buf);
}
-static size_t get_comp_head_size(struct scatterlist *src, u8 req_type)
+static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type)
{
+ if (!acomp_req->src || !acomp_req->slen)
+ return -EINVAL;
+
+ if ((req_type == HZIP_ALG_TYPE_GZIP) &&
+ (acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT))
+ return -EINVAL;
+
switch (req_type) {
case HZIP_ALG_TYPE_ZLIB:
return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB);
case HZIP_ALG_TYPE_GZIP:
- return get_gzip_head_size(src);
+ return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP);
default:
pr_err("request type does not support!\n");
return -EINVAL;
if (req_id >= req_q->size) {
write_unlock(&req_q->req_lock);
dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n");
- return ERR_PTR(-EBUSY);
+ return ERR_PTR(-EAGAIN);
}
set_bit(req_id, req_q->req_bitmap);
struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool;
struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
struct hisi_zip_sqe zip_sqe;
- dma_addr_t input;
- dma_addr_t output;
+ dma_addr_t input, output;
int ret;
if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen)
req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool,
req->req_id << 1, &input);
- if (IS_ERR(req->hw_src))
+ if (IS_ERR(req->hw_src)) {
+ dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n",
+ PTR_ERR(req->hw_src));
return PTR_ERR(req->hw_src);
+ }
req->dma_src = input;
req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool,
&output);
if (IS_ERR(req->hw_dst)) {
ret = PTR_ERR(req->hw_dst);
+ dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n",
+ ret);
goto err_unmap_input;
}
req->dma_dst = output;
ret = hisi_qp_send(qp, &zip_sqe);
if (ret < 0) {
atomic64_inc(&dfx->send_busy_cnt);
+ ret = -EAGAIN;
+ dev_dbg_ratelimited(dev, "failed to send request!\n");
goto err_unmap_output;
}
static int hisi_zip_acompress(struct acomp_req *acomp_req)
{
struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
- struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_COMP];
+ struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP];
+ struct device *dev = &qp_ctx->qp->qm->pdev->dev;
struct hisi_zip_req *req;
int head_size;
int ret;
/* let's output compression head now */
head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type);
- if (head_size < 0)
- return -ENOMEM;
+ if (head_size < 0) {
+ dev_err_ratelimited(dev, "failed to add comp head (%d)!\n",
+ head_size);
+ return head_size;
+ }
- req = hisi_zip_create_req(acomp_req, qp_ctx, (size_t)head_size, true);
+ req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true);
if (IS_ERR(req))
return PTR_ERR(req);
ret = hisi_zip_do_work(req, qp_ctx);
- if (ret != -EINPROGRESS)
+ if (ret != -EINPROGRESS) {
+ dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret);
hisi_zip_remove_req(qp_ctx, req);
+ }
return ret;
}
static int hisi_zip_adecompress(struct acomp_req *acomp_req)
{
struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
- struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[QPC_DECOMP];
+ struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP];
+ struct device *dev = &qp_ctx->qp->qm->pdev->dev;
struct hisi_zip_req *req;
- size_t head_size;
- int ret;
+ int head_size, ret;
- head_size = get_comp_head_size(acomp_req->src, qp_ctx->qp->req_type);
+ head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type);
+ if (head_size < 0) {
+ dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n",
+ head_size);
+ return head_size;
+ }
req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false);
if (IS_ERR(req))
return PTR_ERR(req);
ret = hisi_zip_do_work(req, qp_ctx);
- if (ret != -EINPROGRESS)
+ if (ret != -EINPROGRESS) {
+ dev_info_ratelimited(dev, "failed to do decompress (%d)!\n",
+ ret);
hisi_zip_remove_req(qp_ctx, req);
+ }
return ret;
}
int hisi_zip_register_to_crypto(void)
{
- int ret = 0;
+ int ret;
ret = crypto_register_acomp(&hisi_zip_acomp_zlib);
if (ret) {
- pr_err("Zlib acomp algorithm registration failed\n");
+ pr_err("failed to register to zlib (%d)!\n", ret);
return ret;
}
ret = crypto_register_acomp(&hisi_zip_acomp_gzip);
if (ret) {
- pr_err("Gzip acomp algorithm registration failed\n");
+ pr_err("failed to register to gzip (%d)!\n", ret);
crypto_unregister_acomp(&hisi_zip_acomp_zlib);
}
#define PCI_DEVICE_ID_ZIP_PF 0xa250
#define PCI_DEVICE_ID_ZIP_VF 0xa251
-#define HZIP_VF_NUM 63
#define HZIP_QUEUE_NUM_V1 4096
#define HZIP_QUEUE_NUM_V2 1024
#define DECOMP3_ENABLE BIT(5)
#define DECOMP4_ENABLE BIT(6)
#define DECOMP5_ENABLE BIT(7)
-#define ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \
+#define HZIP_ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \
DECOMP0_ENABLE | DECOMP1_ENABLE | \
DECOMP2_ENABLE | DECOMP3_ENABLE | \
DECOMP4_ENABLE | DECOMP5_ENABLE)
-#define DECOMP_CHECK_ENABLE BIT(16)
+#define HZIP_DECOMP_CHECK_ENABLE BIT(16)
#define HZIP_FSM_MAX_CNT 0x301008
#define HZIP_PORT_ARCA_CHE_0 0x301040
#define HZIP_PORT_ARCA_CHE_1 0x301044
#define HZIP_PORT_AWCA_CHE_0 0x301060
#define HZIP_PORT_AWCA_CHE_1 0x301064
-#define CACHE_ALL_EN 0xffffffff
+#define HZIP_CACHE_ALL_EN 0xffffffff
#define HZIP_BD_RUSER_32_63 0x301110
#define HZIP_SGL_RUSER_32_63 0x30111c
#define HZIP_PF_DEF_Q_BASE 0
#define HZIP_SOFT_CTRL_CNT_CLR_CE 0x301000
-#define SOFT_CTRL_CNT_CLR_CE_BIT BIT(0)
+#define HZIP_SOFT_CTRL_CNT_CLR_CE_BIT BIT(0)
#define HZIP_SOFT_CTRL_ZIP_CONTROL 0x30100C
#define HZIP_AXI_SHUTDOWN_ENABLE BIT(14)
#define HZIP_WR_PORT BIT(11)
#define HZIP_SQE_MASK_OFFSET 64
#define HZIP_SQE_MASK_LEN 48
+#define HZIP_CNT_CLR_CE_EN BIT(0)
+#define HZIP_RO_CNT_CLR_CE_EN BIT(2)
+#define HZIP_RD_CNT_CLR_CE_EN (HZIP_CNT_CLR_CE_EN | \
+ HZIP_RO_CNT_CLR_CE_EN)
+
static const char hisi_zip_name[] = "hisi_zip";
static struct dentry *hzip_debugfs_root;
-static struct hisi_qm_list zip_devices;
struct hisi_zip_hw_error {
u32 int_msk;
u32 offset;
};
+static struct hisi_qm_list zip_devices = {
+ .register_to_crypto = hisi_zip_register_to_crypto,
+ .unregister_from_crypto = hisi_zip_unregister_from_crypto,
+};
+
static struct zip_dfx_item zip_dfx_files[] = {
{"send_cnt", offsetof(struct hisi_zip_dfx, send_cnt)},
{"recv_cnt", offsetof(struct hisi_zip_dfx, recv_cnt)},
*/
struct hisi_zip_ctrl {
struct hisi_zip *hisi_zip;
- struct dentry *debug_root;
struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM];
};
static u32 pf_q_num = HZIP_PF_DEF_Q_NUM;
module_param_cb(pf_q_num, &pf_q_num_ops, &pf_q_num, 0444);
-MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 1-4096, v2 1-1024)");
+MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)");
static const struct kernel_param_ops vfs_num_ops = {
.set = vfs_num_set,
/* qm cache */
writel(AXI_M_CFG, base + QM_AXI_M_CFG);
writel(AXI_M_CFG_ENABLE, base + QM_AXI_M_CFG_ENABLE);
+
/* disable FLR triggered by BME(bus master enable) */
writel(PEH_AXUSER_CFG, base + QM_PEH_AXUSER_CFG);
writel(PEH_AXUSER_CFG_ENABLE, base + QM_PEH_AXUSER_CFG_ENABLE);
/* cache */
- writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0);
- writel(CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1);
- writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0);
- writel(CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1);
+ writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_0);
+ writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_ARCA_CHE_1);
+ writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_0);
+ writel(HZIP_CACHE_ALL_EN, base + HZIP_PORT_AWCA_CHE_1);
/* user domain configurations */
writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63);
}
/* let's open all compression/decompression cores */
- writel(DECOMP_CHECK_ENABLE | ALL_COMP_DECOMP_EN,
+ writel(HZIP_DECOMP_CHECK_ENABLE | HZIP_ALL_COMP_DECOMP_EN,
base + HZIP_CLOCK_GATE_CTRL);
- /* enable sqc writeback */
+ /* enable sqc,cqc writeback */
writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE |
CQC_CACHE_WB_ENABLE | FIELD_PREP(SQC_CACHE_WB_THRD, 1) |
FIELD_PREP(CQC_CACHE_WB_THRD, 1), base + QM_CACHE_CTL);
writel(0x1, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB);
writel(0x0, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB);
writel(HZIP_CORE_INT_RAS_NFE_ENABLE,
- qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
+ qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB);
/* enable ZIP hw error interrupts */
writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG);
if (val > qm->vfs_num)
return -EINVAL;
- /* Calculate curr_qm_qp_num and store */
+ /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
if (val == 0) {
qm->debug.curr_qm_qp_num = qm->qp_num;
} else {
struct hisi_qm *qm = file_to_qm(file);
return readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &
- SOFT_CTRL_CNT_CLR_CE_BIT;
+ HZIP_SOFT_CTRL_CNT_CLR_CE_BIT;
}
static int clear_enable_write(struct ctrl_debug_file *file, u32 val)
return -EINVAL;
tmp = (readl(qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE) &
- ~SOFT_CTRL_CNT_CLR_CE_BIT) | val;
+ ~HZIP_SOFT_CTRL_CNT_CLR_CE_BIT) | val;
writel(tmp, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
return 0;
}
-static ssize_t ctrl_debug_read(struct file *filp, char __user *buf,
- size_t count, loff_t *pos)
+static ssize_t hisi_zip_ctrl_debug_read(struct file *filp, char __user *buf,
+ size_t count, loff_t *pos)
{
struct ctrl_debug_file *file = filp->private_data;
char tbuf[HZIP_BUF_SIZE];
return -EINVAL;
}
spin_unlock_irq(&file->lock);
- ret = sprintf(tbuf, "%u\n", val);
+ ret = scnprintf(tbuf, sizeof(tbuf), "%u\n", val);
return simple_read_from_buffer(buf, count, pos, tbuf, ret);
}
-static ssize_t ctrl_debug_write(struct file *filp, const char __user *buf,
- size_t count, loff_t *pos)
+static ssize_t hisi_zip_ctrl_debug_write(struct file *filp,
+ const char __user *buf,
+ size_t count, loff_t *pos)
{
struct ctrl_debug_file *file = filp->private_data;
char tbuf[HZIP_BUF_SIZE];
static const struct file_operations ctrl_debug_fops = {
.owner = THIS_MODULE,
.open = simple_open,
- .read = ctrl_debug_read,
- .write = ctrl_debug_write,
+ .read = hisi_zip_ctrl_debug_read,
+ .write = hisi_zip_ctrl_debug_write,
};
-
static int zip_debugfs_atomic64_set(void *data, u64 val)
{
if (val)
DEFINE_DEBUGFS_ATTRIBUTE(zip_atomic64_ops, zip_debugfs_atomic64_get,
zip_debugfs_atomic64_set, "%llu\n");
-static int hisi_zip_core_debug_init(struct hisi_zip_ctrl *ctrl)
+static int hisi_zip_core_debug_init(struct hisi_qm *qm)
{
- struct hisi_zip *hisi_zip = ctrl->hisi_zip;
- struct hisi_qm *qm = &hisi_zip->qm;
struct device *dev = &qm->pdev->dev;
struct debugfs_regset32 *regset;
struct dentry *tmp_d;
for (i = 0; i < HZIP_CORE_NUM; i++) {
if (i < HZIP_COMP_CORE_NUM)
- sprintf(buf, "comp_core%d", i);
+ scnprintf(buf, sizeof(buf), "comp_core%d", i);
else
- sprintf(buf, "decomp_core%d", i - HZIP_COMP_CORE_NUM);
+ scnprintf(buf, sizeof(buf), "decomp_core%d",
+ i - HZIP_COMP_CORE_NUM);
regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
if (!regset)
regset->nregs = ARRAY_SIZE(hzip_dfx_regs);
regset->base = qm->io_base + core_offsets[i];
- tmp_d = debugfs_create_dir(buf, ctrl->debug_root);
+ tmp_d = debugfs_create_dir(buf, qm->debug.debug_root);
debugfs_create_regset32("regs", 0444, tmp_d, regset);
}
for (i = 0; i < ARRAY_SIZE(zip_dfx_files); i++) {
data = (atomic64_t *)((uintptr_t)dfx + zip_dfx_files[i].offset);
debugfs_create_file(zip_dfx_files[i].name,
- 0644,
- tmp_dir,
- data,
- &zip_atomic64_ops);
+ 0644, tmp_dir, data,
+ &zip_atomic64_ops);
}
}
-static int hisi_zip_ctrl_debug_init(struct hisi_zip_ctrl *ctrl)
+static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm)
{
+ struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm);
int i;
for (i = HZIP_CURRENT_QM; i < HZIP_DEBUG_FILE_NUM; i++) {
- spin_lock_init(&ctrl->files[i].lock);
- ctrl->files[i].ctrl = ctrl;
- ctrl->files[i].index = i;
+ spin_lock_init(&zip->ctrl->files[i].lock);
+ zip->ctrl->files[i].ctrl = zip->ctrl;
+ zip->ctrl->files[i].index = i;
debugfs_create_file(ctrl_debug_file_name[i], 0600,
- ctrl->debug_root, ctrl->files + i,
+ qm->debug.debug_root,
+ zip->ctrl->files + i,
&ctrl_debug_fops);
}
- return hisi_zip_core_debug_init(ctrl);
+ return hisi_zip_core_debug_init(qm);
}
-static int hisi_zip_debugfs_init(struct hisi_zip *hisi_zip)
+static int hisi_zip_debugfs_init(struct hisi_qm *qm)
{
- struct hisi_qm *qm = &hisi_zip->qm;
struct device *dev = &qm->pdev->dev;
struct dentry *dev_d;
int ret;
goto failed_to_create;
if (qm->fun_type == QM_HW_PF) {
- hisi_zip->ctrl->debug_root = dev_d;
- ret = hisi_zip_ctrl_debug_init(hisi_zip->ctrl);
+ ret = hisi_zip_ctrl_debug_init(qm);
if (ret)
goto failed_to_create;
}
return ret;
}
-static void hisi_zip_debug_regs_clear(struct hisi_zip *hisi_zip)
+/* hisi_zip_debug_regs_clear() - clear the zip debug regs */
+static void hisi_zip_debug_regs_clear(struct hisi_qm *qm)
{
- struct hisi_qm *qm = &hisi_zip->qm;
+ int i, j;
+ /* clear current_qm */
writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
+
+ /* enable register read_clear bit */
+ writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
+ for (i = 0; i < ARRAY_SIZE(core_offsets); i++)
+ for (j = 0; j < ARRAY_SIZE(hzip_dfx_regs); j++)
+ readl(qm->io_base + core_offsets[i] +
+ hzip_dfx_regs[j].offset);
+
+ /* disable register read_clear bit */
writel(0x0, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
hisi_qm_debug_regs_clear(qm);
}
-static void hisi_zip_debugfs_exit(struct hisi_zip *hisi_zip)
+static void hisi_zip_debugfs_exit(struct hisi_qm *qm)
{
- struct hisi_qm *qm = &hisi_zip->qm;
-
debugfs_remove_recursive(qm->debug.debug_root);
- if (qm->fun_type == QM_HW_PF)
- hisi_zip_debug_regs_clear(hisi_zip);
+ if (qm->fun_type == QM_HW_PF) {
+ hisi_zip_debug_regs_clear(qm);
+ qm->debug.curr_qm_qp_num = 0;
+ }
}
static void hisi_zip_log_hw_error(struct hisi_qm *qm, u32 err_sts)
while (err->msg) {
if (err->int_msk & err_sts) {
dev_err(dev, "%s [error status=0x%x] found\n",
- err->msg, err->int_msk);
+ err->msg, err->int_msk);
if (err->int_msk & HZIP_CORE_INT_STATUS_M_ECC) {
err_val = readl(qm->io_base +
dev_err(dev, "hisi-zip multi ecc sram num=0x%x\n",
((err_val >>
HZIP_SRAM_ECC_ERR_NUM_SHIFT) & 0xFF));
- dev_err(dev, "hisi-zip multi ecc sram addr=0x%x\n",
- (err_val >>
- HZIP_SRAM_ECC_ERR_ADDR_SHIFT));
}
}
err++;
hisi_zip_set_user_domain_and_cache(qm);
hisi_qm_dev_err_init(qm);
- hisi_zip_debug_regs_clear(hisi_zip);
+ hisi_zip_debug_regs_clear(qm);
return 0;
}
if (qm->fun_type == QM_HW_PF) {
qm->qp_base = HZIP_PF_DEF_Q_BASE;
qm->qp_num = pf_q_num;
+ qm->debug.curr_qm_qp_num = pf_q_num;
qm->qm_list = &zip_devices;
} else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) {
/*
ret = hisi_qm_start(qm);
if (ret)
- goto err_qm_uninit;
+ goto err_dev_err_uninit;
- ret = hisi_zip_debugfs_init(hisi_zip);
+ ret = hisi_zip_debugfs_init(qm);
if (ret)
- dev_err(&pdev->dev, "Failed to init debugfs (%d)!\n", ret);
+ pci_err(pdev, "failed to init debugfs (%d)!\n", ret);
- hisi_qm_add_to_list(qm, &zip_devices);
+ ret = hisi_qm_alg_register(qm, &zip_devices);
+ if (ret < 0) {
+ pci_err(pdev, "failed to register driver to crypto!\n");
+ goto err_qm_stop;
+ }
if (qm->uacce) {
ret = uacce_register(qm->uacce);
- if (ret)
- goto err_qm_uninit;
+ if (ret) {
+ pci_err(pdev, "failed to register uacce (%d)!\n", ret);
+ goto err_qm_alg_unregister;
+ }
}
if (qm->fun_type == QM_HW_PF && vfs_num > 0) {
ret = hisi_qm_sriov_enable(pdev, vfs_num);
if (ret < 0)
- goto err_remove_from_list;
+ goto err_qm_alg_unregister;
}
return 0;
-err_remove_from_list:
- hisi_qm_del_from_list(qm, &zip_devices);
- hisi_zip_debugfs_exit(hisi_zip);
- hisi_qm_stop(qm);
+err_qm_alg_unregister:
+ hisi_qm_alg_unregister(qm, &zip_devices);
+
+err_qm_stop:
+ hisi_zip_debugfs_exit(qm);
+ hisi_qm_stop(qm, QM_NORMAL);
+
+err_dev_err_uninit:
+ hisi_qm_dev_err_uninit(qm);
+
err_qm_uninit:
hisi_qm_uninit(qm);
static void hisi_zip_remove(struct pci_dev *pdev)
{
- struct hisi_zip *hisi_zip = pci_get_drvdata(pdev);
- struct hisi_qm *qm = &hisi_zip->qm;
+ struct hisi_qm *qm = pci_get_drvdata(pdev);
- if (qm->fun_type == QM_HW_PF && qm->vfs_num)
- hisi_qm_sriov_disable(pdev);
+ hisi_qm_wait_task_finish(qm, &zip_devices);
+ hisi_qm_alg_unregister(qm, &zip_devices);
- hisi_zip_debugfs_exit(hisi_zip);
- hisi_qm_stop(qm);
+ if (qm->fun_type == QM_HW_PF && qm->vfs_num)
+ hisi_qm_sriov_disable(pdev, qm->is_frozen);
+ hisi_zip_debugfs_exit(qm);
+ hisi_qm_stop(qm, QM_NORMAL);
hisi_qm_dev_err_uninit(qm);
hisi_qm_uninit(qm);
- hisi_qm_del_from_list(qm, &zip_devices);
}
static const struct pci_error_handlers hisi_zip_err_handler = {
.sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ?
hisi_qm_sriov_configure : NULL,
.err_handler = &hisi_zip_err_handler,
+ .shutdown = hisi_qm_dev_shutdown,
};
static void hisi_zip_register_debugfs(void)
ret = pci_register_driver(&hisi_zip_pci_driver);
if (ret < 0) {
+ hisi_zip_unregister_debugfs();
pr_err("Failed to register pci driver.\n");
- goto err_pci;
}
- ret = hisi_zip_register_to_crypto();
- if (ret < 0) {
- pr_err("Failed to register driver to crypto.\n");
- goto err_crypto;
- }
-
- return 0;
-
-err_crypto:
- pci_unregister_driver(&hisi_zip_pci_driver);
-err_pci:
- hisi_zip_unregister_debugfs();
-
return ret;
}
static void __exit hisi_zip_exit(void)
{
- hisi_zip_unregister_from_crypto();
pci_unregister_driver(&hisi_zip_pci_driver);
hisi_zip_unregister_debugfs();
}
*/
#include <linux/clk.h>
+#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/interrupt.h>
#include <linux/io.h>
/* Enable access to all IFPP program memories */
writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN,
EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
+
+ /* bypass the OCE, if present */
+ if (priv->flags & EIP197_OCE)
+ writel(EIP197_DEBUG_OCE_BYPASS, EIP197_PE(priv) +
+ EIP197_PE_DEBUG(pe));
}
}
hwopt = readl(EIP197_GLOBAL(priv) + EIP197_OPTIONS);
hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS);
+ priv->hwconfig.icever = 0;
+ priv->hwconfig.ocever = 0;
+ priv->hwconfig.psever = 0;
if (priv->flags & SAFEXCEL_HW_EIP197) {
/* EIP197 */
peopt = readl(EIP197_PE(priv) + EIP197_PE_OPTIONS(0));
EIP197_N_RINGS_MASK;
if (hiaopt & EIP197_HIA_OPT_HAS_PE_ARB)
priv->flags |= EIP197_PE_ARB;
- if (EIP206_OPT_ICE_TYPE(peopt) == 1)
+ if (EIP206_OPT_ICE_TYPE(peopt) == 1) {
priv->flags |= EIP197_ICE;
+ /* Detect ICE EIP207 class. engine and version */
+ version = readl(EIP197_PE(priv) +
+ EIP197_PE_ICE_VERSION(0));
+ if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {
+ dev_err(dev, "EIP%d: ICE EIP207 not detected.\n",
+ peid);
+ return -ENODEV;
+ }
+ priv->hwconfig.icever = EIP197_VERSION_MASK(version);
+ }
+ if (EIP206_OPT_OCE_TYPE(peopt) == 1) {
+ priv->flags |= EIP197_OCE;
+ /* Detect EIP96PP packet stream editor and version */
+ version = readl(EIP197_PE(priv) + EIP197_PE_PSE_VERSION(0));
+ if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
+ dev_err(dev, "EIP%d: EIP96PP not detected.\n", peid);
+ return -ENODEV;
+ }
+ priv->hwconfig.psever = EIP197_VERSION_MASK(version);
+ /* Detect OCE EIP207 class. engine and version */
+ version = readl(EIP197_PE(priv) +
+ EIP197_PE_ICE_VERSION(0));
+ if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {
+ dev_err(dev, "EIP%d: OCE EIP207 not detected.\n",
+ peid);
+ return -ENODEV;
+ }
+ priv->hwconfig.ocever = EIP197_VERSION_MASK(version);
+ }
/* If not a full TRC, then assume simple TRC */
if (!(hwopt & EIP197_OPT_HAS_TRC))
priv->flags |= EIP197_SIMPLE_TRC;
EIP197_PE_EIP96_OPTIONS(0));
/* Print single info line describing what we just detected */
- dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x,alg:%08x\n",
+ dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x(alg:%08x)/%x/%x/%x\n",
peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes,
priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic,
priv->hwconfig.hiaver, priv->hwconfig.hwdataw,
priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize,
priv->hwconfig.ppver, priv->hwconfig.pever,
- priv->hwconfig.algo_flags);
+ priv->hwconfig.algo_flags, priv->hwconfig.icever,
+ priv->hwconfig.ocever, priv->hwconfig.psever);
safexcel_configure(priv);
#include <crypto/algapi.h>
#include <crypto/internal/hash.h>
#include <crypto/sha.h>
+#include <crypto/sha3.h>
#include <crypto/skcipher.h>
+#include <linux/types.h>
#define EIP197_HIA_VERSION_BE 0xca35
#define EIP197_HIA_VERSION_LE 0x35ca
#define EIP96_VERSION_LE 0x9f60
#define EIP201_VERSION_LE 0x36c9
#define EIP206_VERSION_LE 0x31ce
+#define EIP207_VERSION_LE 0x30cf
#define EIP197_REG_LO16(reg) (reg & 0xffff)
#define EIP197_REG_HI16(reg) ((reg >> 16) & 0xffff)
#define EIP197_VERSION_MASK(reg) ((reg >> 16) & 0xfff)
/* EIP206 OPTIONS ENCODING */
#define EIP206_OPT_ICE_TYPE(n) ((n>>8)&3)
+#define EIP206_OPT_OCE_TYPE(n) ((n>>10)&3)
/* EIP197 OPTIONS ENCODING */
#define EIP197_OPT_HAS_TRC BIT(31)
#define EIP197_PE_ICE_FPP_CTRL(n) (0x0d80 + (0x2000 * (n)))
#define EIP197_PE_ICE_PPTF_CTRL(n) (0x0e00 + (0x2000 * (n)))
#define EIP197_PE_ICE_RAM_CTRL(n) (0x0ff0 + (0x2000 * (n)))
+#define EIP197_PE_ICE_VERSION(n) (0x0ffc + (0x2000 * (n)))
#define EIP197_PE_EIP96_TOKEN_CTRL(n) (0x1000 + (0x2000 * (n)))
#define EIP197_PE_EIP96_FUNCTION_EN(n) (0x1004 + (0x2000 * (n)))
#define EIP197_PE_EIP96_CONTEXT_CTRL(n) (0x1008 + (0x2000 * (n)))
#define EIP197_PE_EIP96_FUNCTION2_EN(n) (0x1030 + (0x2000 * (n)))
#define EIP197_PE_EIP96_OPTIONS(n) (0x13f8 + (0x2000 * (n)))
#define EIP197_PE_EIP96_VERSION(n) (0x13fc + (0x2000 * (n)))
+#define EIP197_PE_OCE_VERSION(n) (0x1bfc + (0x2000 * (n)))
#define EIP197_PE_OUT_DBUF_THRES(n) (0x1c00 + (0x2000 * (n)))
#define EIP197_PE_OUT_TBUF_THRES(n) (0x1d00 + (0x2000 * (n)))
+#define EIP197_PE_PSE_VERSION(n) (0x1efc + (0x2000 * (n)))
+#define EIP197_PE_DEBUG(n) (0x1ff4 + (0x2000 * (n)))
#define EIP197_PE_OPTIONS(n) (0x1ff8 + (0x2000 * (n)))
#define EIP197_PE_VERSION(n) (0x1ffc + (0x2000 * (n)))
#define EIP197_MST_CTRL 0xfff4
/* EIP197_PE_EIP96_TOKEN_CTRL2 */
#define EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE BIT(3)
+/* EIP197_PE_DEBUG */
+#define EIP197_DEBUG_OCE_BYPASS BIT(1)
+
/* EIP197_STRC_CONFIG */
#define EIP197_STRC_CONFIG_INIT BIT(31)
#define EIP197_STRC_CONFIG_LARGE_REC(s) (s<<8)
EIP197_PE_ARB = BIT(2),
EIP197_ICE = BIT(3),
EIP197_SIMPLE_TRC = BIT(4),
+ EIP197_OCE = BIT(5),
};
struct safexcel_hwconfig {
int hwver;
int hiaver;
int ppver;
+ int icever;
int pever;
+ int ocever;
+ int psever;
int hwdataw;
int hwcfsize;
int hwrfsize;
struct crypto_async_request *req, bool *complete,
int *ret);
struct safexcel_context_record *ctxr;
+ struct safexcel_crypto_priv *priv;
dma_addr_t ctxr_dma;
+ union {
+ __le32 le[SHA3_512_BLOCK_SIZE / 4];
+ __be32 be[SHA3_512_BLOCK_SIZE / 4];
+ u32 word[SHA3_512_BLOCK_SIZE / 4];
+ u8 byte[SHA3_512_BLOCK_SIZE];
+ } ipad, opad;
+
int ring;
bool needs_inv;
bool exit_inv;
inline struct crypto_async_request *
safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring);
void safexcel_inv_complete(struct crypto_async_request *req, int error);
-int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen,
- void *istate, void *ostate);
+int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key,
+ unsigned int keylen, const char *alg,
+ unsigned int state_sz);
/* available algorithms */
extern struct safexcel_alg_template safexcel_alg_ecb_des;
/* All the below is AEAD specific */
u32 hash_alg;
u32 state_sz;
- __be32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)];
- __be32 opad[SHA512_DIGEST_SIZE / sizeof(u32)];
struct crypto_cipher *hkaes;
struct crypto_aead *fback;
{
struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
struct crypto_aes_ctx aes;
int ret, i;
{
struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_ahash_export_state istate, ostate;
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
struct crypto_authenc_keys keys;
struct crypto_aes_ctx aes;
int err = -EINVAL, i;
+ const char *alg;
if (unlikely(crypto_authenc_extractkeys(&keys, key, len)))
goto badkey;
/* Auth key */
switch (ctx->hash_alg) {
case CONTEXT_CONTROL_CRYPTO_ALG_SHA1:
- if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey,
- keys.authkeylen, &istate, &ostate))
- goto badkey;
+ alg = "safexcel-sha1";
break;
case CONTEXT_CONTROL_CRYPTO_ALG_SHA224:
- if (safexcel_hmac_setkey("safexcel-sha224", keys.authkey,
- keys.authkeylen, &istate, &ostate))
- goto badkey;
+ alg = "safexcel-sha224";
break;
case CONTEXT_CONTROL_CRYPTO_ALG_SHA256:
- if (safexcel_hmac_setkey("safexcel-sha256", keys.authkey,
- keys.authkeylen, &istate, &ostate))
- goto badkey;
+ alg = "safexcel-sha256";
break;
case CONTEXT_CONTROL_CRYPTO_ALG_SHA384:
- if (safexcel_hmac_setkey("safexcel-sha384", keys.authkey,
- keys.authkeylen, &istate, &ostate))
- goto badkey;
+ alg = "safexcel-sha384";
break;
case CONTEXT_CONTROL_CRYPTO_ALG_SHA512:
- if (safexcel_hmac_setkey("safexcel-sha512", keys.authkey,
- keys.authkeylen, &istate, &ostate))
- goto badkey;
+ alg = "safexcel-sha512";
break;
case CONTEXT_CONTROL_CRYPTO_ALG_SM3:
- if (safexcel_hmac_setkey("safexcel-sm3", keys.authkey,
- keys.authkeylen, &istate, &ostate))
- goto badkey;
+ alg = "safexcel-sm3";
break;
default:
dev_err(priv->dev, "aead: unsupported hash algorithm\n");
goto badkey;
}
- if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma &&
- (memcmp(ctx->ipad, istate.state, ctx->state_sz) ||
- memcmp(ctx->opad, ostate.state, ctx->state_sz)))
- ctx->base.needs_inv = true;
+ if (safexcel_hmac_setkey(&ctx->base, keys.authkey, keys.authkeylen,
+ alg, ctx->state_sz))
+ goto badkey;
/* Now copy the keys into the context */
for (i = 0; i < keys.enckeylen / sizeof(u32); i++)
ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]);
ctx->key_len = keys.enckeylen;
- memcpy(ctx->ipad, &istate.state, ctx->state_sz);
- memcpy(ctx->opad, &ostate.state, ctx->state_sz);
-
memzero_explicit(&keys, sizeof(keys));
return 0;
struct safexcel_cipher_req *sreq,
struct safexcel_command_desc *cdesc)
{
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int ctrl_size = ctx->key_len / sizeof(u32);
cdesc->control_data.control1 = ctx->mode;
struct skcipher_request *areq = skcipher_request_cast(base);
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
struct safexcel_command_desc *cdesc;
struct safexcel_command_desc *first_cdesc = NULL;
struct safexcel_result_desc *rdesc, *first_rdesc = NULL;
totlen_dst += digestsize;
memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
- ctx->ipad, ctx->state_sz);
+ &ctx->base.ipad, ctx->state_sz);
if (!ctx->xcm)
memcpy(ctx->base.ctxr->data + (ctx->key_len +
- ctx->state_sz) / sizeof(u32), ctx->opad,
+ ctx->state_sz) / sizeof(u32), &ctx->base.opad,
ctx->state_sz);
} else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) &&
(sreq->direction == SAFEXCEL_DECRYPT)) {
int ring, int *commands, int *results)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int ret;
ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring);
struct skcipher_request *req = skcipher_request_cast(async);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int ret;
BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
struct safexcel_cipher_req *sreq = aead_request_ctx(req);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int ret;
BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
struct safexcel_inv_result *result)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int ring = ctx->base.ring;
init_completion(&result->completion);
enum safexcel_cipher_direction dir)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int ret, ring;
sreq->needs_inv = false;
crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
sizeof(struct safexcel_cipher_req));
- ctx->priv = tmpl->priv;
+ ctx->base.priv = tmpl->priv;
ctx->base.send = safexcel_skcipher_send;
ctx->base.handle_result = safexcel_skcipher_handle_result;
static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int ret;
if (safexcel_cipher_cra_exit(tfm))
static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
{
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int ret;
if (safexcel_cipher_cra_exit(tfm))
{
struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
struct crypto_aes_ctx aes;
int ret, i;
unsigned int keylen;
unsigned int len)
{
struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int ret;
ret = verify_skcipher_des_key(ctfm, key);
const u8 *key, unsigned int len)
{
struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int err;
err = verify_skcipher_des3_key(ctfm, key);
crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
sizeof(struct safexcel_cipher_req));
- ctx->priv = tmpl->priv;
+ ctx->base.priv = tmpl->priv;
ctx->alg = SAFEXCEL_AES; /* default */
ctx->blocksz = AES_BLOCK_SIZE;
{
struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
struct crypto_aes_ctx aes;
int ret, i;
unsigned int keylen;
{
struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
struct crypto_aes_ctx aes;
u32 hashkey[AES_BLOCK_SIZE >> 2];
int ret, i;
if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
- if (be32_to_cpu(ctx->ipad[i]) != hashkey[i]) {
+ if (be32_to_cpu(ctx->base.ipad.be[i]) != hashkey[i]) {
ctx->base.needs_inv = true;
break;
}
}
for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++)
- ctx->ipad[i] = cpu_to_be32(hashkey[i]);
+ ctx->base.ipad.be[i] = cpu_to_be32(hashkey[i]);
memzero_explicit(hashkey, AES_BLOCK_SIZE);
memzero_explicit(&aes, sizeof(aes));
{
struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
struct crypto_aes_ctx aes;
int ret, i;
for (i = 0; i < len / sizeof(u32); i++) {
ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
- ctx->ipad[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] =
+ ctx->base.ipad.be[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] =
cpu_to_be32(aes.key_enc[i]);
}
static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx,
const u8 *key)
{
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma)
if (memcmp(ctx->key, key, CHACHA_KEY_SIZE))
{
struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
if (len != SM4_KEY_SIZE)
return -EINVAL;
struct safexcel_ahash_ctx {
struct safexcel_context base;
- struct safexcel_crypto_priv *priv;
u32 alg;
u8 key_sz;
bool fb_init_done;
bool fb_do_setkey;
- __le32 ipad[SHA3_512_BLOCK_SIZE / sizeof(__le32)];
- __le32 opad[SHA3_512_BLOCK_SIZE / sizeof(__le32)];
-
struct crypto_cipher *kaes;
struct crypto_ahash *fback;
struct crypto_shash *shpre;
struct safexcel_ahash_req *req,
struct safexcel_command_desc *cdesc)
{
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
u64 count = 0;
cdesc->control_data.control0 = ctx->alg;
*/
if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) {
if (req->xcbcmac)
- memcpy(ctx->base.ctxr->data, ctx->ipad, ctx->key_sz);
+ memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz);
else
memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
} else { /* HMAC */
/* Need outer digest for HMAC finalization */
memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),
- ctx->opad, req->state_sz);
+ &ctx->base.opad, req->state_sz);
/* Single pass HMAC - no digest count */
cdesc->control_data.control0 |=
memcpy(sreq->cache, sreq->state,
crypto_ahash_digestsize(ahash));
- memcpy(sreq->state, ctx->opad, sreq->digest_sz);
+ memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz);
sreq->len = sreq->block_sz +
crypto_ahash_digestsize(ahash);
struct ahash_request *areq = ahash_request_cast(async);
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
struct safexcel_result_desc *rdesc;
struct scatterlist *sg;
// 10- padding for XCBCMAC & CMAC
req->cache[cache_len + skip] = 0x80;
// HW will use K2 iso K3 - compensate!
- for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++)
- ((__be32 *)req->cache)[i] ^=
- cpu_to_be32(le32_to_cpu(
- ctx->ipad[i] ^ ctx->ipad[i + 4]));
+ for (i = 0; i < AES_BLOCK_SIZE / 4; i++) {
+ u32 *cache = (void *)req->cache;
+ u32 *ipad = ctx->base.ipad.word;
+ u32 x;
+
+ x = ipad[i] ^ ipad[i + 4];
+ cache[i] ^= swab(x);
+ }
}
cache_len = AES_BLOCK_SIZE;
queued = queued + extra;
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
int ret;
- ret = safexcel_invalidate_cache(async, ctx->priv,
+ ret = safexcel_invalidate_cache(async, ctx->base.priv,
ctx->base.ctxr_dma, ring);
if (unlikely(ret))
return ret;
static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
{
struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);
struct safexcel_ahash_req *rctx = ahash_request_ctx(req);
struct safexcel_inv_result result = {};
{
struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct safexcel_ahash_req *req = ahash_request_ctx(areq);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int ret, ring;
req->needs_inv = false;
/* invalidate for HMAC finish with odigest changed */
(req->finish && req->hmac &&
memcmp(ctx->base.ctxr->data + (req->state_sz>>2),
- ctx->opad, req->state_sz))))
+ &ctx->base.opad, req->state_sz))))
/*
* We're still setting needs_inv here, even though it is
* cleared right away, because the needs_inv flag can be
ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 &&
req->len == sizeof(u32) && !areq->nbytes)) {
/* Zero length CRC32 */
- memcpy(areq->result, ctx->ipad, sizeof(u32));
+ memcpy(areq->result, &ctx->base.ipad, sizeof(u32));
return 0;
} else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE &&
!areq->nbytes)) {
/* Zero length (X)CBC/CMAC */
int i;
- for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++)
- ((__be32 *)areq->result)[i] =
- cpu_to_be32(le32_to_cpu(ctx->ipad[i + 4]));//K3
+ for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
+ u32 *result = (void *)areq->result;
+
+ /* K3 */
+ result[i] = swab(ctx->base.ipad.word[i + 4]);
+ }
areq->result[0] ^= 0x80; // 10- padding
crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result);
return 0;
container_of(__crypto_ahash_alg(tfm->__crt_alg),
struct safexcel_alg_template, alg.ahash);
- ctx->priv = tmpl->priv;
+ ctx->base.priv = tmpl->priv;
ctx->base.send = safexcel_ahash_send;
ctx->base.handle_result = safexcel_handle_result;
ctx->fb_do_setkey = false;
static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
{
struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = ctx->base.priv;
int ret;
/* context not allocated, skip invalidation */
memset(req, 0, sizeof(*req));
/* Start from ipad precompute */
- memcpy(req->state, ctx->ipad, SHA1_DIGEST_SIZE);
+ memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE);
/* Already processed the key^ipad part now! */
req->len = SHA1_BLOCK_SIZE;
req->processed = SHA1_BLOCK_SIZE;
}
/* Avoid leaking */
- memzero_explicit(keydup, keylen);
- kfree(keydup);
+ kfree_sensitive(keydup);
if (ret)
return ret;
return crypto_ahash_export(areq, state);
}
-int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen,
- void *istate, void *ostate)
+static int __safexcel_hmac_setkey(const char *alg, const u8 *key,
+ unsigned int keylen,
+ void *istate, void *ostate)
{
struct ahash_request *areq;
struct crypto_ahash *tfm;
return ret;
}
-static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
- unsigned int keylen, const char *alg,
- unsigned int state_sz)
+int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key,
+ unsigned int keylen, const char *alg,
+ unsigned int state_sz)
{
- struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
- struct safexcel_crypto_priv *priv = ctx->priv;
+ struct safexcel_crypto_priv *priv = base->priv;
struct safexcel_ahash_export_state istate, ostate;
int ret;
- ret = safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
+ ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
if (ret)
return ret;
- if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr &&
- (memcmp(ctx->ipad, istate.state, state_sz) ||
- memcmp(ctx->opad, ostate.state, state_sz)))
- ctx->base.needs_inv = true;
+ if (priv->flags & EIP197_TRC_CACHE && base->ctxr &&
+ (memcmp(&base->ipad, istate.state, state_sz) ||
+ memcmp(&base->opad, ostate.state, state_sz)))
+ base->needs_inv = true;
- memcpy(ctx->ipad, &istate.state, state_sz);
- memcpy(ctx->opad, &ostate.state, state_sz);
+ memcpy(&base->ipad, &istate.state, state_sz);
+ memcpy(&base->opad, &ostate.state, state_sz);
return 0;
}
+static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen, const char *alg,
+ unsigned int state_sz)
+{
+ struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
+
+ return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz);
+}
+
static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int keylen)
{
memset(req, 0, sizeof(*req));
/* Start from ipad precompute */
- memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE);
+ memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
/* Already processed the key^ipad part now! */
req->len = SHA256_BLOCK_SIZE;
req->processed = SHA256_BLOCK_SIZE;
memset(req, 0, sizeof(*req));
/* Start from ipad precompute */
- memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE);
+ memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
/* Already processed the key^ipad part now! */
req->len = SHA256_BLOCK_SIZE;
req->processed = SHA256_BLOCK_SIZE;
memset(req, 0, sizeof(*req));
/* Start from ipad precompute */
- memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE);
+ memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
/* Already processed the key^ipad part now! */
req->len = SHA512_BLOCK_SIZE;
req->processed = SHA512_BLOCK_SIZE;
memset(req, 0, sizeof(*req));
/* Start from ipad precompute */
- memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE);
+ memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
/* Already processed the key^ipad part now! */
req->len = SHA512_BLOCK_SIZE;
req->processed = SHA512_BLOCK_SIZE;
memset(req, 0, sizeof(*req));
/* Start from ipad precompute */
- memcpy(req->state, ctx->ipad, MD5_DIGEST_SIZE);
+ memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE);
/* Already processed the key^ipad part now! */
req->len = MD5_HMAC_BLOCK_SIZE;
req->processed = MD5_HMAC_BLOCK_SIZE;
int ret = safexcel_ahash_cra_init(tfm);
/* Default 'key' is all zeroes */
- memset(ctx->ipad, 0, sizeof(u32));
+ memset(&ctx->base.ipad, 0, sizeof(u32));
return ret;
}
memset(req, 0, sizeof(*req));
/* Start from loaded key */
- req->state[0] = (__force __le32)le32_to_cpu(~ctx->ipad[0]);
+ req->state[0] = cpu_to_le32(~ctx->base.ipad.word[0]);
/* Set processed to non-zero to enable invalidation detection */
req->len = sizeof(u32);
req->processed = sizeof(u32);
if (keylen != sizeof(u32))
return -EINVAL;
- memcpy(ctx->ipad, key, sizeof(u32));
+ memcpy(&ctx->base.ipad, key, sizeof(u32));
return 0;
}
memset(req, 0, sizeof(*req));
/* Start from loaded keys */
- memcpy(req->state, ctx->ipad, ctx->key_sz);
+ memcpy(req->state, &ctx->base.ipad, ctx->key_sz);
/* Set processed to non-zero to enable invalidation detection */
req->len = AES_BLOCK_SIZE;
req->processed = AES_BLOCK_SIZE;
if (ret)
return ret;
- memset(ctx->ipad, 0, 2 * AES_BLOCK_SIZE);
+ memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE);
for (i = 0; i < len / sizeof(u32); i++)
- ctx->ipad[i + 8] = (__force __le32)cpu_to_be32(aes.key_enc[i]);
+ ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]);
if (len == AES_KEYSIZE_192) {
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE,
"\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3");
for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++)
- ctx->ipad[i] =
- cpu_to_le32((__force u32)cpu_to_be32(key_tmp[i]));
+ ctx->base.ipad.word[i] = swab(key_tmp[i]);
crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
return ret;
for (i = 0; i < len / sizeof(u32); i++)
- ctx->ipad[i + 8] =
- cpu_to_le32((__force u32)cpu_to_be32(aes.key_enc[i]));
+ ctx->base.ipad.word[i + 8] = swab(aes.key_enc[i]);
/* precompute the CMAC key material */
crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
/* end of code borrowed from crypto/cmac.c */
for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++)
- ctx->ipad[i] = (__force __le32)cpu_to_be32(((u32 *)consts)[i]);
+ ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]);
if (len == AES_KEYSIZE_192) {
ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
memset(req, 0, sizeof(*req));
/* Start from ipad precompute */
- memcpy(req->state, ctx->ipad, SM3_DIGEST_SIZE);
+ memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE);
/* Already processed the key^ipad part now! */
req->len = SM3_BLOCK_SIZE;
req->processed = SM3_BLOCK_SIZE;
/* Set fallback cipher HMAC key */
u8 key[SHA3_224_BLOCK_SIZE];
- memcpy(key, ctx->ipad,
+ memcpy(key, &ctx->base.ipad,
crypto_ahash_blocksize(ctx->fback) / 2);
memcpy(key +
crypto_ahash_blocksize(ctx->fback) / 2,
- ctx->opad,
+ &ctx->base.opad,
crypto_ahash_blocksize(ctx->fback) / 2);
ret = crypto_ahash_setkey(ctx->fback, key,
crypto_ahash_blocksize(ctx->fback));
* first using our fallback cipher
*/
ret = crypto_shash_digest(ctx->shdesc, key, keylen,
- (u8 *)ctx->ipad);
+ ctx->base.ipad.byte);
keylen = crypto_shash_digestsize(ctx->shpre);
/*
*/
if (keylen > crypto_ahash_blocksize(tfm) / 2)
/* Buffers overlap, need to use memmove iso memcpy! */
- memmove(ctx->opad,
- (u8 *)ctx->ipad +
+ memmove(&ctx->base.opad,
+ ctx->base.ipad.byte +
crypto_ahash_blocksize(tfm) / 2,
keylen - crypto_ahash_blocksize(tfm) / 2);
} else {
* to match the existing HMAC driver infrastructure.
*/
if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
- memcpy(ctx->ipad, key, keylen);
+ memcpy(&ctx->base.ipad, key, keylen);
} else {
- memcpy(ctx->ipad, key,
+ memcpy(&ctx->base.ipad, key,
crypto_ahash_blocksize(tfm) / 2);
- memcpy(ctx->opad,
+ memcpy(&ctx->base.opad,
key + crypto_ahash_blocksize(tfm) / 2,
keylen - crypto_ahash_blocksize(tfm) / 2);
}
/* Pad key with zeroes */
if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
- memset((u8 *)ctx->ipad + keylen, 0,
+ memset(ctx->base.ipad.byte + keylen, 0,
crypto_ahash_blocksize(tfm) / 2 - keylen);
- memset(ctx->opad, 0, crypto_ahash_blocksize(tfm) / 2);
+ memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2);
} else {
- memset((u8 *)ctx->opad + keylen -
+ memset(ctx->base.opad.byte + keylen -
crypto_ahash_blocksize(tfm) / 2, 0,
crypto_ahash_blocksize(tfm) - keylen);
}
memset(req, 0, sizeof(*req));
/* Copy (half of) the key */
- memcpy(req->state, ctx->ipad, SHA3_224_BLOCK_SIZE / 2);
+ memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2);
/* Start of HMAC should have len == processed == blocksize */
req->len = SHA3_224_BLOCK_SIZE;
req->processed = SHA3_224_BLOCK_SIZE;
memset(req, 0, sizeof(*req));
/* Copy (half of) the key */
- memcpy(req->state, ctx->ipad, SHA3_256_BLOCK_SIZE / 2);
+ memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2);
/* Start of HMAC should have len == processed == blocksize */
req->len = SHA3_256_BLOCK_SIZE;
req->processed = SHA3_256_BLOCK_SIZE;
memset(req, 0, sizeof(*req));
/* Copy (half of) the key */
- memcpy(req->state, ctx->ipad, SHA3_384_BLOCK_SIZE / 2);
+ memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2);
/* Start of HMAC should have len == processed == blocksize */
req->len = SHA3_384_BLOCK_SIZE;
req->processed = SHA3_384_BLOCK_SIZE;
memset(req, 0, sizeof(*req));
/* Copy (half of) the key */
- memcpy(req->state, ctx->ipad, SHA3_512_BLOCK_SIZE / 2);
+ memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2);
/* Start of HMAC should have len == processed == blocksize */
req->len = SHA3_512_BLOCK_SIZE;
req->processed = SHA3_512_BLOCK_SIZE;
rdesc->particle_size = len;
rdesc->rsvd0 = 0;
- rdesc->descriptor_overflow = 0;
- rdesc->buffer_overflow = 0;
+ rdesc->descriptor_overflow = 1; /* assume error */
+ rdesc->buffer_overflow = 1; /* assume error */
rdesc->last_seg = last;
rdesc->first_seg = first;
rdesc->result_size = EIP197_RD64_RESULT_SIZE;
rdesc->data_lo = lower_32_bits(data);
rdesc->data_hi = upper_32_bits(data);
- /* Clear length & error code in result token */
+ /* Clear length in result token */
rtoken->packet_length = 0;
- rtoken->error_code = 0;
+ /* Assume errors - HW will clear if not the case */
+ rtoken->error_code = 0x7fff;
return rdesc;
}
if (crypt_virt) {
dma_free_coherent(dev,
- NPE_QLEN_TOTAL * sizeof( struct crypt_ctl),
+ NPE_QLEN * sizeof(struct crypt_ctl),
crypt_virt, crypt_phys);
}
}
struct device *dev = &pdev->dev;
struct mv_cesa_dev *cesa;
struct mv_cesa_engine *engines;
- struct resource *res;
int irq, ret, i, cpu;
u32 sram_size;
spin_lock_init(&cesa->lock);
- res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs");
- cesa->regs = devm_ioremap_resource(dev, res);
+ cesa->regs = devm_platform_ioremap_resource_byname(pdev, "regs");
if (IS_ERR(cesa->regs))
return PTR_ERR(cesa->regs);
#ifndef __MARVELL_CESA_H__
#define __MARVELL_CESA_H__
-#include <crypto/algapi.h>
-#include <crypto/hash.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/skcipher.h>
-#include <linux/crypto.h>
+#include <linux/dma-direction.h>
#include <linux/dmapool.h>
#define CESA_ENGINE_OFF(i) (((i) * 0x2000))
* Context associated to a cipher operation.
*/
struct mv_cesa_skcipher_op_ctx {
- u32 key[8];
+ __le32 key[8];
u32 iv[4];
};
*/
struct mv_cesa_hash_op_ctx {
u32 iv[16];
- u32 hash[8];
+ __le32 hash[8];
};
/**
*/
struct mv_cesa_tdma_desc {
__le32 byte_cnt;
- __le32 src;
- __le32 dst;
+ union {
+ __le32 src;
+ dma_addr_t src_dma;
+ };
+ union {
+ __le32 dst;
+ dma_addr_t dst_dma;
+ };
__le32 next_dma;
/* Software state */
*/
struct mv_cesa_hmac_ctx {
struct mv_cesa_ctx base;
- u32 iv[16];
+ __be32 iv[16];
};
/**
#include <crypto/aes.h>
#include <crypto/internal/des.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
#include "cesa.h"
remaining = (ctx->aes.key_length - 16) / 4;
offset = ctx->aes.key_length + 24 - remaining;
for (i = 0; i < remaining; i++)
- ctx->aes.key_dec[4 + i] =
- cpu_to_le32(ctx->aes.key_enc[offset + i]);
+ ctx->aes.key_dec[4 + i] = ctx->aes.key_enc[offset + i];
return 0;
}
#include <crypto/hmac.h>
#include <crypto/md5.h>
#include <crypto/sha.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
#include "cesa.h"
CESA_SA_DATA_SRAM_OFFSET + len,
new_cache_ptr);
} else {
- len += mv_cesa_ahash_pad_req(creq,
- engine->sram + len +
- CESA_SA_DATA_SRAM_OFFSET);
+ i = mv_cesa_ahash_pad_req(creq, creq->cache);
+ len += i;
+ memcpy_toio(engine->sram + len +
+ CESA_SA_DATA_SRAM_OFFSET,
+ creq->cache, i);
}
if (frag_mode == CESA_SA_DESC_CFG_LAST_FRAG)
*/
data = creq->base.chain.last->op->ctx.hash.hash;
for (i = 0; i < digsize / 4; i++)
- creq->state[i] = cpu_to_le32(data[i]);
+ creq->state[i] = le32_to_cpu(data[i]);
memcpy(ahashreq->result, data, digsize);
} else {
return ret;
for (i = 0; i < ARRAY_SIZE(istate.hash); i++)
- ctx->iv[i] = be32_to_cpu(istate.hash[i]);
+ ctx->iv[i] = cpu_to_be32(istate.hash[i]);
for (i = 0; i < ARRAY_SIZE(ostate.hash); i++)
- ctx->iv[i + 8] = be32_to_cpu(ostate.hash[i]);
+ ctx->iv[i + 8] = cpu_to_be32(ostate.hash[i]);
return 0;
}
return ret;
for (i = 0; i < ARRAY_SIZE(istate.state); i++)
- ctx->iv[i] = be32_to_cpu(istate.state[i]);
+ ctx->iv[i] = cpu_to_be32(istate.state[i]);
for (i = 0; i < ARRAY_SIZE(ostate.state); i++)
- ctx->iv[i + 8] = be32_to_cpu(ostate.state[i]);
+ ctx->iv[i + 8] = cpu_to_be32(ostate.state[i]);
return 0;
}
return ret;
for (i = 0; i < ARRAY_SIZE(istate.state); i++)
- ctx->iv[i] = be32_to_cpu(istate.state[i]);
+ ctx->iv[i] = cpu_to_be32(istate.state[i]);
for (i = 0; i < ARRAY_SIZE(ostate.state); i++)
- ctx->iv[i + 8] = be32_to_cpu(ostate.state[i]);
+ ctx->iv[i + 8] = cpu_to_be32(ostate.state[i]);
return 0;
}
for (tdma = dreq->chain.first; tdma; tdma = tdma->next) {
if (tdma->flags & CESA_TDMA_DST_IN_SRAM)
- tdma->dst = cpu_to_le32(tdma->dst + engine->sram_dma);
+ tdma->dst = cpu_to_le32(tdma->dst_dma + engine->sram_dma);
if (tdma->flags & CESA_TDMA_SRC_IN_SRAM)
- tdma->src = cpu_to_le32(tdma->src + engine->sram_dma);
+ tdma->src = cpu_to_le32(tdma->src_dma + engine->sram_dma);
if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP)
mv_cesa_adjust_op(engine, tdma->op);
*/
if (!(last->flags & CESA_TDMA_BREAK_CHAIN) &&
!(dreq->chain.first->flags & CESA_TDMA_SET_STATE))
- last->next_dma = dreq->chain.first->cur_dma;
+ last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma);
}
}
return -EIO;
tdma->byte_cnt = cpu_to_le32(size | BIT(31));
- tdma->src = src;
- tdma->dst = op_desc->src;
+ tdma->src_dma = src;
+ tdma->dst_dma = op_desc->src_dma;
tdma->op = op_desc->op;
flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM);
tdma->op = op;
tdma->byte_cnt = cpu_to_le32(size | BIT(31));
tdma->src = cpu_to_le32(dma_handle);
- tdma->dst = CESA_SA_CFG_SRAM_OFFSET;
+ tdma->dst_dma = CESA_SA_CFG_SRAM_OFFSET;
tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP;
return op;
return PTR_ERR(tdma);
tdma->byte_cnt = cpu_to_le32(size | BIT(31));
- tdma->src = src;
- tdma->dst = dst;
+ tdma->src_dma = src;
+ tdma->dst_dma = dst;
flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM);
tdma->flags = flags | CESA_TDMA_DATA;
static int create_sysfs_eng_grps_info(struct device *dev,
struct otx_cpt_eng_grp_info *eng_grp)
{
- int ret;
-
eng_grp->info_attr.show = eng_grp_info_show;
eng_grp->info_attr.store = NULL;
eng_grp->info_attr.attr.name = eng_grp->sysfs_info_name;
eng_grp->info_attr.attr.mode = 0440;
sysfs_attr_init(&eng_grp->info_attr.attr);
- ret = device_create_file(dev, &eng_grp->info_attr);
- if (ret)
- return ret;
-
- return 0;
+ return device_create_file(dev, &eng_grp->info_attr);
}
static void ucode_unload(struct device *dev, struct otx_cpt_ucode *ucode)
struct mtk_aes_ctr_ctx {
struct mtk_aes_base_ctx base;
- u32 iv[AES_BLOCK_SIZE / sizeof(u32)];
+ __be32 iv[AES_BLOCK_SIZE / sizeof(u32)];
size_t offset;
struct scatterlist src[2];
struct scatterlist dst[2];
sg->length += dma->remainder;
}
-static inline void mtk_aes_write_state_le(__le32 *dst, const u32 *src, u32 size)
-{
- int i;
-
- for (i = 0; i < SIZE_IN_WORDS(size); i++)
- dst[i] = cpu_to_le32(src[i]);
-}
-
-static inline void mtk_aes_write_state_be(__be32 *dst, const u32 *src, u32 size)
-{
- int i;
-
- for (i = 0; i < SIZE_IN_WORDS(size); i++)
- dst[i] = cpu_to_be32(src[i]);
-}
-
static inline int mtk_aes_complete(struct mtk_cryp *cryp,
struct mtk_aes_rec *aes,
int err)
/* Prepare enough space for authenticated tag */
if (aes->flags & AES_FLAGS_GCM)
- res->hdr += AES_BLOCK_SIZE;
+ le32_add_cpu(&res->hdr, AES_BLOCK_SIZE);
/*
* Make sure that all changes to the DMA ring are done before we
return;
}
- mtk_aes_write_state_le(info->state + ctx->keylen, (void *)req->iv,
- AES_BLOCK_SIZE);
+ memcpy(info->state + ctx->keylen, req->iv, AES_BLOCK_SIZE);
ctr:
- info->tfm[0] += AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE));
+ le32_add_cpu(&info->tfm[0],
+ le32_to_cpu(AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE))));
info->tfm[1] |= AES_TFM_FULL_IV;
info->cmd[cnt++] = AES_CMD2;
ecb:
scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset));
/* Write IVs into transform state buffer. */
- mtk_aes_write_state_le(ctx->info.state + ctx->keylen, cctx->iv,
- AES_BLOCK_SIZE);
+ memcpy(ctx->info.state + ctx->keylen, cctx->iv, AES_BLOCK_SIZE);
if (unlikely(fragmented)) {
/*
}
ctx->keylen = SIZE_IN_WORDS(keylen);
- mtk_aes_write_state_le(ctx->key, (const u32 *)key, keylen);
+ memcpy(ctx->key, key, keylen);
return 0;
}
static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp,
struct mtk_aes_rec *aes)
{
- u32 status = cryp->ring[aes->id]->res_prev->ct;
+ __le32 status = cryp->ring[aes->id]->res_prev->ct;
return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ?
-EBADMSG : 0);
u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
u32 cnt = 0;
- ctx->ct_hdr = AES_CT_CTRL_HDR | len;
+ ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len);
info->cmd[cnt++] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen);
info->cmd[cnt++] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen);
info->tfm[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | AES_TFM_3IV |
AES_TFM_ENC_HASH;
- mtk_aes_write_state_le(info->state + ctx->keylen + SIZE_IN_WORDS(
- AES_BLOCK_SIZE), (const u32 *)req->iv, ivsize);
+ memcpy(info->state + ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE),
+ req->iv, ivsize);
}
static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
u32 keylen)
{
struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead);
- u8 hash[AES_BLOCK_SIZE] __aligned(4) = {};
+ union {
+ u32 x32[SIZE_IN_WORDS(AES_BLOCK_SIZE)];
+ u8 x8[AES_BLOCK_SIZE];
+ } hash = {};
struct crypto_aes_ctx aes_ctx;
int err;
+ int i;
switch (keylen) {
case AES_KEYSIZE_128:
if (err)
return err;
- aes_encrypt(&aes_ctx, hash, hash);
+ aes_encrypt(&aes_ctx, hash.x8, hash.x8);
memzero_explicit(&aes_ctx, sizeof(aes_ctx));
- mtk_aes_write_state_le(ctx->key, (const u32 *)key, keylen);
- mtk_aes_write_state_be(ctx->key + ctx->keylen, (const u32 *)hash,
- AES_BLOCK_SIZE);
+ memcpy(ctx->key, key, keylen);
+
+ /* Why do we need to do this? */
+ for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++)
+ hash.x32[i] = swab32(hash.x32[i]);
+
+ memcpy(ctx->key + ctx->keylen, &hash, AES_BLOCK_SIZE);
return 0;
}
static int mtk_dfe_dse_reset(struct mtk_cryp *cryp)
{
- int err;
-
/* Reset DSE/DFE and correct system priorities for all rings. */
writel(MTK_DFSE_THR_CTRL_RESET, cryp->base + DFE_THR_CTRL);
writel(0, cryp->base + DFE_PRIO_0);
writel(0, cryp->base + DSE_PRIO_2);
writel(0, cryp->base + DSE_PRIO_3);
- err = mtk_dfe_dse_state_check(cryp);
- if (err)
- return err;
-
- return 0;
+ return mtk_dfe_dse_state_check(cryp);
}
static void mtk_cmd_desc_ring_setup(struct mtk_cryp *cryp,
static int mtk_desc_ring_alloc(struct mtk_cryp *cryp)
{
struct mtk_ring **ring = cryp->ring;
- int i, err = ENOMEM;
+ int i;
for (i = 0; i < MTK_RING_MAX; i++) {
ring[i] = kzalloc(sizeof(**ring), GFP_KERNEL);
return 0;
err_cleanup:
- for (; i--; ) {
+ do {
dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ,
ring[i]->res_base, ring[i]->res_dma);
dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ,
ring[i]->cmd_base, ring[i]->cmd_dma);
kfree(ring[i]);
- }
- return err;
+ } while (i--);
+ return -ENOMEM;
}
static int mtk_crypto_probe(struct platform_device *pdev)
static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len)
{
u32 index, padlen;
- u64 bits[2];
+ __be64 bits[2];
u64 size = ctx->digcnt;
size += ctx->bufcnt;
struct n2_ahash_alg {
struct list_head entry;
const u8 *hash_zero;
- const u32 *hash_init;
+ const u8 *hash_init;
u8 hw_op_hashsz;
u8 digest_size;
u8 auth_type;
u8 aes[AES_MAX_KEY_SIZE];
u8 des[DES_KEY_SIZE];
u8 des3[3 * DES_KEY_SIZE];
- u8 arc4[258]; /* S-box, X, Y */
} key;
};
return 0;
}
-static int n2_arc4_setkey(struct crypto_skcipher *skcipher, const u8 *key,
- unsigned int keylen)
-{
- struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
- struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm);
- struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher);
- u8 *s = ctx->key.arc4;
- u8 *x = s + 256;
- u8 *y = x + 1;
- int i, j, k;
-
- ctx->enc_type = n2alg->enc_type;
-
- j = k = 0;
- *x = 0;
- *y = 0;
- for (i = 0; i < 256; i++)
- s[i] = i;
- for (i = 0; i < 256; i++) {
- u8 a = s[i];
- j = (j + key[k] + a) & 0xff;
- s[i] = s[j];
- s[j] = a;
- if (++k >= keylen)
- k = 0;
- }
-
- return 0;
-}
-
static inline int skcipher_descriptor_len(int nbytes, unsigned int block_size)
{
int this_len = nbytes;
};
static const struct n2_skcipher_tmpl skcipher_tmpls[] = {
- /* ARC4: only ECB is supported (chaining bits ignored) */
- { .name = "ecb(arc4)",
- .drv_name = "ecb-arc4",
- .block_size = 1,
- .enc_type = (ENC_TYPE_ALG_RC4_STREAM |
- ENC_TYPE_CHAINING_ECB),
- .skcipher = {
- .min_keysize = 1,
- .max_keysize = 256,
- .setkey = n2_arc4_setkey,
- .encrypt = n2_encrypt_ecb,
- .decrypt = n2_decrypt_ecb,
- },
- },
-
/* DES: ECB CBC and CFB are supported */
{ .name = "ecb(des)",
.drv_name = "ecb-des",
struct n2_hash_tmpl {
const char *name;
const u8 *hash_zero;
- const u32 *hash_init;
+ const u8 *hash_init;
u8 hw_op_hashsz;
u8 digest_size;
u8 block_size;
u8 hmac_type;
};
-static const u32 n2_md5_init[MD5_HASH_WORDS] = {
+static const __le32 n2_md5_init[MD5_HASH_WORDS] = {
cpu_to_le32(MD5_H0),
cpu_to_le32(MD5_H1),
cpu_to_le32(MD5_H2),
static const struct n2_hash_tmpl hash_tmpls[] = {
{ .name = "md5",
.hash_zero = md5_zero_message_hash,
- .hash_init = n2_md5_init,
+ .hash_init = (u8 *)n2_md5_init,
.auth_type = AUTH_TYPE_MD5,
.hmac_type = AUTH_TYPE_HMAC_MD5,
.hw_op_hashsz = MD5_DIGEST_SIZE,
.block_size = MD5_HMAC_BLOCK_SIZE },
{ .name = "sha1",
.hash_zero = sha1_zero_message_hash,
- .hash_init = n2_sha1_init,
+ .hash_init = (u8 *)n2_sha1_init,
.auth_type = AUTH_TYPE_SHA1,
.hmac_type = AUTH_TYPE_HMAC_SHA1,
.hw_op_hashsz = SHA1_DIGEST_SIZE,
.block_size = SHA1_BLOCK_SIZE },
{ .name = "sha256",
.hash_zero = sha256_zero_message_hash,
- .hash_init = n2_sha256_init,
+ .hash_init = (u8 *)n2_sha256_init,
.auth_type = AUTH_TYPE_SHA256,
.hmac_type = AUTH_TYPE_HMAC_SHA256,
.hw_op_hashsz = SHA256_DIGEST_SIZE,
.block_size = SHA256_BLOCK_SIZE },
{ .name = "sha224",
.hash_zero = sha224_zero_message_hash,
- .hash_init = n2_sha224_init,
+ .hash_init = (u8 *)n2_sha224_init,
.auth_type = AUTH_TYPE_SHA256,
.hmac_type = AUTH_TYPE_RESERVED,
.hw_op_hashsz = SHA256_DIGEST_SIZE,
#include <crypto/hash.h>
#include <crypto/hmac.h>
#include <crypto/internal/hash.h>
+#include <crypto/engine.h>
#define MD5_DIGEST_SIZE 16
#define DEFAULT_AUTOSUSPEND_DELAY 1000
/* mostly device flags */
-#define FLAGS_BUSY 0
#define FLAGS_FINAL 1
#define FLAGS_DMA_ACTIVE 2
#define FLAGS_OUTPUT_READY 3
struct omap_sham_reqctx {
struct omap_sham_dev *dd;
unsigned long flags;
- unsigned long op;
+ u8 op;
u8 digest[SHA512_DIGEST_SIZE] OMAP_ALIGNED;
size_t digcnt;
};
struct omap_sham_ctx {
+ struct crypto_engine_ctx enginectx;
unsigned long flags;
/* fallback stuff */
struct device *dev;
void __iomem *io_base;
int irq;
- spinlock_t lock;
int err;
struct dma_chan *dma_lch;
struct tasklet_struct done_task;
int fallback_sz;
struct crypto_queue queue;
struct ahash_request *req;
+ struct crypto_engine *engine;
const struct omap_sham_pdata *pdata;
};
.lock = __SPIN_LOCK_UNLOCKED(sham.lock),
};
+static int omap_sham_enqueue(struct ahash_request *req, unsigned int op);
+static void omap_sham_finish_req(struct ahash_request *req, int err);
+
static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset)
{
return __raw_readl(dd->io_base + offset);
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
u32 val, mask;
+ if (likely(ctx->digcnt))
+ omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt);
+
/*
* Setting ALGO_CONST only for the first iteration and
* CLOSE_HASH only for the last one. Note that flags mode bits
return 0;
}
-static int omap_sham_prepare_request(struct ahash_request *req, bool update)
+static int omap_sham_prepare_request(struct crypto_engine *engine, void *areq)
{
+ struct ahash_request *req = container_of(areq, struct ahash_request,
+ base);
struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
int bs;
int ret;
unsigned int nbytes;
bool final = rctx->flags & BIT(FLAGS_FINUP);
+ bool update = rctx->op == OP_UPDATE;
int hash_later;
bs = get_block_size(rctx);
struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
int err;
bool final = (ctx->flags & BIT(FLAGS_FINUP)) &&
- !(dd->flags & BIT(FLAGS_HUGE));
+ !(dd->flags & BIT(FLAGS_HUGE));
dev_dbg(dd->dev, "update_req: total: %u, digcnt: %zd, final: %d",
ctx->total, ctx->digcnt, final);
return err;
}
+static int omap_sham_hash_one_req(struct crypto_engine *engine, void *areq)
+{
+ struct ahash_request *req = container_of(areq, struct ahash_request,
+ base);
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ struct omap_sham_dev *dd = ctx->dd;
+ int err;
+ bool final = (ctx->flags & BIT(FLAGS_FINUP)) &&
+ !(dd->flags & BIT(FLAGS_HUGE));
+
+ dev_dbg(dd->dev, "hash-one: op: %u, total: %u, digcnt: %zd, final: %d",
+ ctx->op, ctx->total, ctx->digcnt, final);
+
+ dd->req = req;
+
+ err = omap_sham_hw_init(dd);
+ if (err)
+ return err;
+
+ if (ctx->digcnt)
+ dd->pdata->copy_hash(req, 0);
+
+ if (ctx->op == OP_UPDATE)
+ err = omap_sham_update_req(dd);
+ else if (ctx->op == OP_FINAL)
+ err = omap_sham_final_req(dd);
+
+ if (err != -EINPROGRESS)
+ omap_sham_finish_req(req, err);
+
+ return 0;
+}
+
static int omap_sham_finish_hmac(struct ahash_request *req)
{
struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
ctx->sg = NULL;
- dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED));
+ dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED) |
+ BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) |
+ BIT(FLAGS_OUTPUT_READY));
+
+ if (!err)
+ dd->pdata->copy_hash(req, 1);
if (dd->flags & BIT(FLAGS_HUGE)) {
- dd->flags &= ~(BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) |
- BIT(FLAGS_OUTPUT_READY) | BIT(FLAGS_HUGE));
- omap_sham_prepare_request(req, ctx->op == OP_UPDATE);
- if (ctx->op == OP_UPDATE || (dd->flags & BIT(FLAGS_HUGE))) {
- err = omap_sham_update_req(dd);
- if (err != -EINPROGRESS &&
- (ctx->flags & BIT(FLAGS_FINUP)))
- err = omap_sham_final_req(dd);
- } else if (ctx->op == OP_FINAL) {
- omap_sham_final_req(dd);
- }
+ /* Re-enqueue the request */
+ omap_sham_enqueue(req, ctx->op);
return;
}
if (!err) {
- dd->pdata->copy_hash(req, 1);
if (test_bit(FLAGS_FINAL, &dd->flags))
err = omap_sham_finish(req);
} else {
}
/* atomic operation is not needed here */
- dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) |
+ dd->flags &= ~(BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) |
BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY));
pm_runtime_mark_last_busy(dd->dev);
ctx->offset = 0;
- if (req->base.complete)
- req->base.complete(&req->base, err);
+ crypto_finalize_hash_request(dd->engine, req, err);
}
static int omap_sham_handle_queue(struct omap_sham_dev *dd,
struct ahash_request *req)
{
- struct crypto_async_request *async_req, *backlog;
- struct omap_sham_reqctx *ctx;
- unsigned long flags;
- int err = 0, ret = 0;
-
-retry:
- spin_lock_irqsave(&dd->lock, flags);
- if (req)
- ret = ahash_enqueue_request(&dd->queue, req);
- if (test_bit(FLAGS_BUSY, &dd->flags)) {
- spin_unlock_irqrestore(&dd->lock, flags);
- return ret;
- }
- backlog = crypto_get_backlog(&dd->queue);
- async_req = crypto_dequeue_request(&dd->queue);
- if (async_req)
- set_bit(FLAGS_BUSY, &dd->flags);
- spin_unlock_irqrestore(&dd->lock, flags);
-
- if (!async_req)
- return ret;
-
- if (backlog)
- backlog->complete(backlog, -EINPROGRESS);
-
- req = ahash_request_cast(async_req);
- dd->req = req;
- ctx = ahash_request_ctx(req);
-
- err = omap_sham_prepare_request(req, ctx->op == OP_UPDATE);
- if (err || !ctx->total)
- goto err1;
-
- dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
- ctx->op, req->nbytes);
-
- err = omap_sham_hw_init(dd);
- if (err)
- goto err1;
-
- if (ctx->digcnt)
- /* request has changed - restore hash */
- dd->pdata->copy_hash(req, 0);
-
- if (ctx->op == OP_UPDATE || (dd->flags & BIT(FLAGS_HUGE))) {
- err = omap_sham_update_req(dd);
- if (err != -EINPROGRESS && (ctx->flags & BIT(FLAGS_FINUP)))
- /* no final() after finup() */
- err = omap_sham_final_req(dd);
- } else if (ctx->op == OP_FINAL) {
- err = omap_sham_final_req(dd);
- }
-err1:
- dev_dbg(dd->dev, "exit, err: %d\n", err);
-
- if (err != -EINPROGRESS) {
- /* done_task will not finish it, so do it here */
- omap_sham_finish_req(req, err);
- req = NULL;
-
- /*
- * Execute next request immediately if there is anything
- * in queue.
- */
- goto retry;
- }
-
- return ret;
+ return crypto_transfer_hash_request_to_engine(dd->engine, req);
}
static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
}
+ tctx->enginectx.op.do_one_request = omap_sham_hash_one_req;
+ tctx->enginectx.op.prepare_request = omap_sham_prepare_request;
+ tctx->enginectx.op.unprepare_request = NULL;
+
return 0;
}
dev_dbg(dd->dev, "%s: flags=%lx\n", __func__, dd->flags);
- if (!test_bit(FLAGS_BUSY, &dd->flags)) {
- omap_sham_handle_queue(dd, NULL);
- return;
- }
-
if (test_bit(FLAGS_CPU, &dd->flags)) {
if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags))
goto finish;
dev_dbg(dd->dev, "update done: err: %d\n", err);
/* finish curent request */
omap_sham_finish_req(dd->req, err);
-
- /* If we are not busy, process next req */
- if (!test_bit(FLAGS_BUSY, &dd->flags))
- omap_sham_handle_queue(dd, NULL);
}
static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd)
{
- if (!test_bit(FLAGS_BUSY, &dd->flags)) {
- dev_warn(dd->dev, "Interrupt when no active requests.\n");
- } else {
- set_bit(FLAGS_OUTPUT_READY, &dd->flags);
- tasklet_schedule(&dd->done_task);
- }
+ set_bit(FLAGS_OUTPUT_READY, &dd->flags);
+ tasklet_schedule(&dd->done_task);
return IRQ_HANDLED;
}
struct omap_sham_dev *dd = dev_get_drvdata(dev);
ssize_t status;
long value;
- unsigned long flags;
status = kstrtol(buf, 0, &value);
if (status)
* than current size, it will just not accept new entries until
* it has shrank enough.
*/
- spin_lock_irqsave(&dd->lock, flags);
dd->queue.max_qlen = value;
- spin_unlock_irqrestore(&dd->lock, flags);
return size;
}
platform_set_drvdata(pdev, dd);
INIT_LIST_HEAD(&dd->list);
- spin_lock_init(&dd->lock);
tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd);
crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH);
list_add_tail(&dd->list, &sham.dev_list);
spin_unlock(&sham.lock);
+ dd->engine = crypto_engine_alloc_init(dev, 1);
+ if (!dd->engine) {
+ err = -ENOMEM;
+ goto err_engine;
+ }
+
+ err = crypto_engine_start(dd->engine);
+ if (err)
+ goto err_engine_start;
+
for (i = 0; i < dd->pdata->algs_info_size; i++) {
if (dd->pdata->algs_info[i].registered)
break;
for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
crypto_unregister_ahash(
&dd->pdata->algs_info[i].algs_list[j]);
+err_engine_start:
+ crypto_engine_exit(dd->engine);
+err_engine:
+ spin_lock(&sham.lock);
+ list_del(&dd->list);
+ spin_unlock(&sham.lock);
err_pm:
pm_runtime_disable(dev);
if (!dd->polling_mode)
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
+#include <linux/mm.h>
#include <linux/percpu.h>
#include <linux/smp.h>
#include <linux/slab.h>
goto err_clk_put;
}
- ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh);
- if (ret)
- goto err_clk_disable;
-
-
/*
* Use an IRQ threshold of 50% as a default. This seems to be a
* reasonable trade off of latency against throughput but can be
*/
engine->stat_irq_thresh = (engine->fifo_sz / 2);
+ ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh);
+ if (ret)
+ goto err_clk_disable;
+
/*
* Configure the interrupts. We only use the STAT_CNT interrupt as we
* only submit a new packet for processing when we complete another in
#include <adf_cfg.h>
#include "adf_c3xxx_hw_data.h"
-#define ADF_SYSTEM_DEVICE(device_id) \
- {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
-
static const struct pci_device_id adf_pci_tbl[] = {
- ADF_SYSTEM_DEVICE(ADF_C3XXX_PCI_DEVICE_ID),
- {0,}
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX), },
+ { }
};
MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
if (accel_dev->hw_device) {
switch (accel_pci_dev->pci_dev->device) {
- case ADF_C3XXX_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
adf_clean_hw_data_c3xxx(accel_dev->hw_device);
break;
default:
int ret;
switch (ent->device) {
- case ADF_C3XXX_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
break;
default:
dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
}
/* Create dev top level debugfs entry */
- snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
- ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
- pdev->bus->number, PCI_SLOT(pdev->devfn),
- PCI_FUNC(pdev->devfn));
+ snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX,
+ hw_data->dev_class->name, pci_name(pdev));
accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
}
pci_set_master(pdev);
- if (adf_enable_aer(accel_dev, &adf_driver)) {
+ if (adf_enable_aer(accel_dev)) {
dev_err(&pdev->dev, "Failed to enable aer\n");
ret = -EFAULT;
goto out_err_free_reg;
#include <adf_cfg.h>
#include "adf_c3xxxvf_hw_data.h"
-#define ADF_SYSTEM_DEVICE(device_id) \
- {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
-
static const struct pci_device_id adf_pci_tbl[] = {
- ADF_SYSTEM_DEVICE(ADF_C3XXXIOV_PCI_DEVICE_ID),
- {0,}
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF), },
+ { }
};
MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
if (accel_dev->hw_device) {
switch (accel_pci_dev->pci_dev->device) {
- case ADF_C3XXXIOV_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF:
adf_clean_hw_data_c3xxxiov(accel_dev->hw_device);
break;
default:
int ret;
switch (ent->device) {
- case ADF_C3XXXIOV_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF:
break;
default:
dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
accel_pci_dev->sku = hw_data->get_sku(hw_data);
/* Create dev top level debugfs entry */
- snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
- ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
- pdev->bus->number, PCI_SLOT(pdev->devfn),
- PCI_FUNC(pdev->devfn));
+ snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX,
+ hw_data->dev_class->name, pci_name(pdev));
accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
#include <adf_cfg.h>
#include "adf_c62x_hw_data.h"
-#define ADF_SYSTEM_DEVICE(device_id) \
- {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
-
static const struct pci_device_id adf_pci_tbl[] = {
- ADF_SYSTEM_DEVICE(ADF_C62X_PCI_DEVICE_ID),
- {0,}
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X), },
+ { }
};
MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
if (accel_dev->hw_device) {
switch (accel_pci_dev->pci_dev->device) {
- case ADF_C62X_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_C62X:
adf_clean_hw_data_c62x(accel_dev->hw_device);
break;
default:
int ret;
switch (ent->device) {
- case ADF_C62X_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_C62X:
break;
default:
dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
}
/* Create dev top level debugfs entry */
- snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
- ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
- pdev->bus->number, PCI_SLOT(pdev->devfn),
- PCI_FUNC(pdev->devfn));
+ snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX,
+ hw_data->dev_class->name, pci_name(pdev));
accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
}
pci_set_master(pdev);
- if (adf_enable_aer(accel_dev, &adf_driver)) {
+ if (adf_enable_aer(accel_dev)) {
dev_err(&pdev->dev, "Failed to enable aer\n");
ret = -EFAULT;
goto out_err_free_reg;
#include <adf_cfg.h>
#include "adf_c62xvf_hw_data.h"
-#define ADF_SYSTEM_DEVICE(device_id) \
- {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
-
static const struct pci_device_id adf_pci_tbl[] = {
- ADF_SYSTEM_DEVICE(ADF_C62XIOV_PCI_DEVICE_ID),
- {0,}
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X_VF), },
+ { }
};
MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
if (accel_dev->hw_device) {
switch (accel_pci_dev->pci_dev->device) {
- case ADF_C62XIOV_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_C62X_VF:
adf_clean_hw_data_c62xiov(accel_dev->hw_device);
break;
default:
int ret;
switch (ent->device) {
- case ADF_C62XIOV_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_C62X_VF:
break;
default:
dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
accel_pci_dev->sku = hw_data->get_sku(hw_data);
/* Create dev top level debugfs entry */
- snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
- ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
- pdev->bus->number, PCI_SLOT(pdev->devfn),
- PCI_FUNC(pdev->devfn));
+ snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX,
+ hw_data->dev_class->name, pci_name(pdev));
accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
#define ADF_C62XVF_DEVICE_NAME "c6xxvf"
#define ADF_C3XXX_DEVICE_NAME "c3xxx"
#define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf"
-#define ADF_DH895XCC_PCI_DEVICE_ID 0x435
-#define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443
-#define ADF_C62X_PCI_DEVICE_ID 0x37c8
-#define ADF_C62XIOV_PCI_DEVICE_ID 0x37c9
-#define ADF_C3XXX_PCI_DEVICE_ID 0x19e2
-#define ADF_C3XXXIOV_PCI_DEVICE_ID 0x19e3
#define ADF_ERRSOU3 (0x3A000 + 0x0C)
#define ADF_ERRSOU5 (0x3A000 + 0xD8)
#define ADF_DEVICE_FUSECTL_OFFSET 0x40
/**
* adf_enable_aer() - Enable Advance Error Reporting for acceleration device
* @accel_dev: Pointer to acceleration device.
- * @adf: PCI device driver owning the given acceleration device.
*
* Function enables PCI Advance Error Reporting for the
* QAT acceleration device accel_dev.
*
* Return: 0 on success, error code otherwise.
*/
-int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf)
+int adf_enable_aer(struct adf_accel_dev *accel_dev)
{
struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
+ struct pci_driver *pdrv = pdev->driver;
- adf->err_handler = &adf_err_handler;
+ pdrv->err_handler = &adf_err_handler;
pci_enable_pcie_error_reporting(pdev);
return 0;
}
.show = qat_dev_cfg_show
};
-static int qat_dev_cfg_open(struct inode *inode, struct file *file)
-{
- int ret = seq_open(file, &qat_dev_cfg_sops);
-
- if (!ret) {
- struct seq_file *seq_f = file->private_data;
-
- seq_f->private = inode->i_private;
- }
- return ret;
-}
-
-static const struct file_operations qat_dev_cfg_fops = {
- .open = qat_dev_cfg_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = seq_release
-};
+DEFINE_SEQ_ATTRIBUTE(qat_dev_cfg);
/**
* adf_cfg_dev_add() - Create an acceleration device configuration table.
int adf_ae_start(struct adf_accel_dev *accel_dev);
int adf_ae_stop(struct adf_accel_dev *accel_dev);
-int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf);
+int adf_enable_aer(struct adf_accel_dev *accel_dev);
void adf_disable_aer(struct adf_accel_dev *accel_dev);
void adf_reset_sbr(struct adf_accel_dev *accel_dev);
void adf_reset_flr(struct adf_accel_dev *accel_dev);
static int __init adf_register_ctl_device_driver(void)
{
- mutex_init(&adf_ctl_lock);
-
if (adf_chr_drv_create())
goto err_chr_dev;
/**
* adf_devmgr_pci_to_accel_dev() - Get accel_dev associated with the pci_dev.
- * @accel_dev: Pointer to pci device.
+ * @pci_dev: Pointer to pci device.
*
* Function returns acceleration device associated with the given pci device.
* To be used by QAT device specific drivers.
/**
* adf_sriov_configure() - Enable SRIOV for the device
* @pdev: Pointer to pci device.
+ * @numvfs: Number of virtual functions (VFs) to enable.
+ *
+ * Note that the @numvfs parameter is ignored and all VFs supported by the
+ * device are enabled due to the design of the hardware.
*
* Function enables SRIOV for the pci device.
*
- * Return: 0 on success, error code otherwise.
+ * Return: number of VFs enabled on success, error code otherwise.
*/
int adf_sriov_configure(struct pci_dev *pdev, int numvfs)
{
mutex_unlock(&ring_read_lock);
}
-static const struct seq_operations adf_ring_sops = {
+static const struct seq_operations adf_ring_debug_sops = {
.start = adf_ring_start,
.next = adf_ring_next,
.stop = adf_ring_stop,
.show = adf_ring_show
};
-static int adf_ring_open(struct inode *inode, struct file *file)
-{
- int ret = seq_open(file, &adf_ring_sops);
-
- if (!ret) {
- struct seq_file *seq_f = file->private_data;
-
- seq_f->private = inode->i_private;
- }
- return ret;
-}
-
-static const struct file_operations adf_ring_debug_fops = {
- .open = adf_ring_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = seq_release
-};
+DEFINE_SEQ_ATTRIBUTE(adf_ring_debug);
int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name)
{
mutex_unlock(&bank_read_lock);
}
-static const struct seq_operations adf_bank_sops = {
+static const struct seq_operations adf_bank_debug_sops = {
.start = adf_bank_start,
.next = adf_bank_next,
.stop = adf_bank_stop,
.show = adf_bank_show
};
-static int adf_bank_open(struct inode *inode, struct file *file)
-{
- int ret = seq_open(file, &adf_bank_sops);
-
- if (!ret) {
- struct seq_file *seq_f = file->private_data;
-
- seq_f->private = inode->i_private;
- }
- return ret;
-}
-
-static const struct file_operations adf_bank_debug_fops = {
- .open = adf_bank_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = seq_release
-};
+DEFINE_SEQ_ATTRIBUTE(adf_bank_debug);
int adf_bank_debugfs_add(struct adf_etr_bank_data *bank)
{
struct icp_qat_fw_la_bulk_req *msg;
int digst_size = crypto_aead_authsize(aead_tfm);
int ret, ctr = 0;
+ u32 cipher_len;
+
+ cipher_len = areq->cryptlen - digst_size;
+ if (cipher_len % AES_BLOCK_SIZE != 0)
+ return -EINVAL;
ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);
if (unlikely(ret))
qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
- cipher_param->cipher_length = areq->cryptlen - digst_size;
+ cipher_param->cipher_length = cipher_len;
cipher_param->cipher_offset = areq->assoclen;
memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
u8 *iv = areq->iv;
int ret, ctr = 0;
+ if (areq->cryptlen % AES_BLOCK_SIZE != 0)
+ return -EINVAL;
+
ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);
if (unlikely(ret))
return ret;
/* Copyright(c) 2014 - 2020 Intel Corporation */
#include <linux/slab.h>
#include <linux/delay.h>
+#include <linux/pci_ids.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"
unsigned int csr_val;
int times = 30;
- if (handle->pci_dev->device != ADF_DH895XCC_PCI_DEVICE_ID)
+ if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC)
return 0;
csr_val = ADF_CSR_RD(csr_addr, 0);
(void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v +
LOCAL_TO_XFER_REG_OFFSET);
handle->pci_dev = pci_info->pci_dev;
- if (handle->pci_dev->device == ADF_DH895XCC_PCI_DEVICE_ID) {
+ if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_DH895XCC) {
sram_bar =
&pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
handle->hal_sram_addr_v = sram_bar->virt_addr;
}
handle->fw_auth = (handle->pci_dev->device ==
- ADF_DH895XCC_PCI_DEVICE_ID) ? false : true;
+ PCI_DEVICE_ID_INTEL_QAT_DH895XCC) ? false : true;
handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
if (!handle->hal_handle)
goto out_hal_handle;
#include <linux/ctype.h>
#include <linux/kernel.h>
#include <linux/delay.h>
+#include <linux/pci_ids.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"
#include "icp_qat_uclo.h"
qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle)
{
switch (handle->pci_dev->device) {
- case ADF_DH895XCC_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
return ICP_QAT_AC_895XCC_DEV_TYPE;
- case ADF_C62X_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_C62X:
return ICP_QAT_AC_C62X_DEV_TYPE;
- case ADF_C3XXX_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
return ICP_QAT_AC_C3XXX_DEV_TYPE;
default:
pr_err("QAT: unsupported device 0x%x\n",
status = qat_uclo_auth_fw(handle, desc);
qat_uclo_ummap_auth_fw(handle, &desc);
} else {
- if (handle->pci_dev->device == ADF_C3XXX_PCI_DEVICE_ID) {
+ if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_C3XXX) {
pr_err("QAT: C3XXX doesn't support unsigned MMP\n");
return -EINVAL;
}
#include <adf_cfg.h>
#include "adf_dh895xcc_hw_data.h"
-#define ADF_SYSTEM_DEVICE(device_id) \
- {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
-
static const struct pci_device_id adf_pci_tbl[] = {
- ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID),
- {0,}
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC), },
+ { }
};
MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
if (accel_dev->hw_device) {
switch (accel_pci_dev->pci_dev->device) {
- case ADF_DH895XCC_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
adf_clean_hw_data_dh895xcc(accel_dev->hw_device);
break;
default:
int ret;
switch (ent->device) {
- case ADF_DH895XCC_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
break;
default:
dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
}
/* Create dev top level debugfs entry */
- snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
- ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
- pdev->bus->number, PCI_SLOT(pdev->devfn),
- PCI_FUNC(pdev->devfn));
+ snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX,
+ hw_data->dev_class->name, pci_name(pdev));
accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
}
pci_set_master(pdev);
- if (adf_enable_aer(accel_dev, &adf_driver)) {
+ if (adf_enable_aer(accel_dev)) {
dev_err(&pdev->dev, "Failed to enable aer\n");
ret = -EFAULT;
goto out_err_free_reg;
#include <adf_cfg.h>
#include "adf_dh895xccvf_hw_data.h"
-#define ADF_SYSTEM_DEVICE(device_id) \
- {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
-
static const struct pci_device_id adf_pci_tbl[] = {
- ADF_SYSTEM_DEVICE(ADF_DH895XCCIOV_PCI_DEVICE_ID),
- {0,}
+ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF), },
+ { }
};
MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
if (accel_dev->hw_device) {
switch (accel_pci_dev->pci_dev->device) {
- case ADF_DH895XCCIOV_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF:
adf_clean_hw_data_dh895xcciov(accel_dev->hw_device);
break;
default:
int ret;
switch (ent->device) {
- case ADF_DH895XCCIOV_PCI_DEVICE_ID:
+ case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF:
break;
default:
dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device);
accel_pci_dev->sku = hw_data->get_sku(hw_data);
/* Create dev top level debugfs entry */
- snprintf(name, sizeof(name), "%s%s_%02x:%02d.%d",
- ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name,
- pdev->bus->number, PCI_SLOT(pdev->devfn),
- PCI_FUNC(pdev->devfn));
+ snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX,
+ hw_data->dev_class->name, pci_name(pdev));
accel_dev->debugfs_dir = debugfs_create_dir(name, NULL);
*/
#include <linux/clk.h>
+#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/mod_devicetable.h>
*/
#include <linux/device.h>
+#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <crypto/internal/hash.h>
*/
#include <linux/device.h>
+#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/moduleparam.h>
#include <linux/types.h>
#include <linux/acpi.h>
#include <linux/clk.h>
#include <linux/crypto.h>
+#include <linux/io.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/platform_device.h>
*/
#include "rk3288_crypto.h"
+#include <linux/dma-mapping.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/of.h>
#include <crypto/algapi.h>
#include <linux/interrupt.h>
#include <linux/delay.h>
+#include <linux/scatterlist.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/skcipher.h>
*
* Some ideas are from marvell/cesa.c and s5p-sss.c driver.
*/
+#include <linux/device.h>
#include "rk3288_crypto.h"
/*
*
* Some ideas are from marvell-cesa.c and s5p-sss.c driver.
*/
+#include <linux/device.h>
#include "rk3288_crypto.h"
#define RK_CRYPTO_DEC BIT(0)
* struct s5p_aes_dev - Crypto device state container
* @dev: Associated device
* @clk: Clock for accessing hardware
+ * @pclk: APB bus clock necessary to access the hardware
* @ioaddr: Mapped IO memory region
* @aes_ioaddr: Per-varian offset for AES block IO memory
* @irq_fc: Feed control interrupt line
* @engine: Bits for selecting type of HASH in SSS block
* @sg: sg for DMA transfer
* @sg_len: Length of sg for DMA transfer
- * @sgl[]: sg for joining buffer and req->src scatterlist
+ * @sgl: sg for joining buffer and req->src scatterlist
* @skip: Skip offset in req->src for current op
* @total: Total number of bytes for current request
* @finup: Keep state for finup or final.
* @error: Keep track of error.
* @bufcnt: Number of bytes holded in buffer[]
- * @buffer[]: For byte(s) from end of req->src in UPDATE op
+ * @buffer: For byte(s) from end of req->src in UPDATE op
*/
struct s5p_hash_reqctx {
struct s5p_aes_dev *dd;
* s5p_hash_prepare_sgs() - prepare sg for processing
* @ctx: request context
* @sg: source scatterlist request
- * @nbytes: number of bytes to process from sg
+ * @new_len: number of bytes to process from sg
* @final: final flag
*
* Check two conditions: (1) if buffers in sg have len aligned data, and (2)
}
pdata->clk = devm_clk_get(dev, variant->clk_names[0]);
- if (IS_ERR(pdata->clk)) {
- dev_err(dev, "failed to find secss clock %s\n",
- variant->clk_names[0]);
- return -ENOENT;
- }
+ if (IS_ERR(pdata->clk))
+ return dev_err_probe(dev, PTR_ERR(pdata->clk),
+ "failed to find secss clock %s\n",
+ variant->clk_names[0]);
err = clk_prepare_enable(pdata->clk);
if (err < 0) {
if (variant->clk_names[1]) {
pdata->pclk = devm_clk_get(dev, variant->clk_names[1]);
if (IS_ERR(pdata->pclk)) {
- dev_err(dev, "failed to find clock %s\n",
- variant->clk_names[1]);
- err = -ENOENT;
+ err = dev_err_probe(dev, PTR_ERR(pdata->pclk),
+ "failed to find clock %s\n",
+ variant->clk_names[1]);
goto err_clk;
}
tasklet_kill(&pdata->tasklet);
err_irq:
- if (pdata->pclk)
- clk_disable_unprepare(pdata->pclk);
+ clk_disable_unprepare(pdata->pclk);
err_clk:
clk_disable_unprepare(pdata->clk);
pdata->use_hash = false;
}
- if (pdata->pclk)
- clk_disable_unprepare(pdata->pclk);
+ clk_disable_unprepare(pdata->pclk);
clk_disable_unprepare(pdata->clk);
s5p_dev = NULL;
bool registered;
};
+/**
+ * struct sa_mapped_sg: scatterlist information for tx and rx
+ * @mapped: Set to true if the @sgt is mapped
+ * @dir: mapping direction used for @sgt
+ * @split_sg: Set if the sg is split and needs to be freed up
+ * @static_sg: Static scatterlist entry for overriding data
+ * @sgt: scatterlist table for DMA API use
+ */
+struct sa_mapped_sg {
+ bool mapped;
+ enum dma_data_direction dir;
+ struct scatterlist static_sg;
+ struct scatterlist *split_sg;
+ struct sg_table sgt;
+};
/**
* struct sa_rx_data: RX Packet miscellaneous data place holder
* @req: crypto request data pointer
* @ddev: pointer to the DMA device
* @tx_in: dma_async_tx_descriptor pointer for rx channel
- * @split_src_sg: Set if the src sg is split and needs to be freed up
- * @split_dst_sg: Set if the dst sg is split and needs to be freed up
+ * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping
* @enc: Flag indicating either encryption or decryption
* @enc_iv_size: Initialisation vector size
* @iv_idx: Initialisation vector index
- * @rx_sg: Static scatterlist entry for overriding RX data
- * @tx_sg: Static scatterlist entry for overriding TX data
- * @src: Source data pointer
- * @dst: Destination data pointer
*/
struct sa_rx_data {
void *req;
struct device *ddev;
struct dma_async_tx_descriptor *tx_in;
- struct scatterlist *split_src_sg;
- struct scatterlist *split_dst_sg;
+ struct sa_mapped_sg mapped_sg[2];
u8 enc;
u8 enc_iv_size;
u8 iv_idx;
- struct scatterlist rx_sg;
- struct scatterlist tx_sg;
- struct scatterlist *src;
- struct scatterlist *dst;
};
/**
return sa_cipher_setkey(tfm, key, keylen, &ad);
}
+static void sa_sync_from_device(struct sa_rx_data *rxd)
+{
+ struct sg_table *sgt;
+
+ if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL)
+ sgt = &rxd->mapped_sg[0].sgt;
+ else
+ sgt = &rxd->mapped_sg[1].sgt;
+
+ dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE);
+}
+
+static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) {
+ struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i];
+
+ if (mapped_sg->mapped) {
+ dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt,
+ mapped_sg->dir, 0);
+ kfree(mapped_sg->split_sg);
+ }
+ }
+
+ kfree(rxd);
+}
+
static void sa_aes_dma_in_callback(void *data)
{
struct sa_rx_data *rxd = (struct sa_rx_data *)data;
struct skcipher_request *req;
- int sglen;
u32 *result;
__be32 *mdptr;
size_t ml, pl;
int i;
- enum dma_data_direction dir_src;
- bool diff_dst;
+ sa_sync_from_device(rxd);
req = container_of(rxd->req, struct skcipher_request, base);
- sglen = sg_nents_for_len(req->src, req->cryptlen);
-
- diff_dst = (req->src != req->dst) ? true : false;
- dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
if (req->iv) {
mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl,
result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]);
}
- dma_unmap_sg(rxd->ddev, req->src, sglen, dir_src);
- kfree(rxd->split_src_sg);
-
- if (diff_dst) {
- sglen = sg_nents_for_len(req->dst, req->cryptlen);
-
- dma_unmap_sg(rxd->ddev, req->dst, sglen,
- DMA_FROM_DEVICE);
- kfree(rxd->split_dst_sg);
- }
-
- kfree(rxd);
+ sa_free_sa_rx_data(rxd);
skcipher_request_complete(req, 0);
}
struct device *ddev;
struct dma_chan *dma_rx;
int sg_nents, src_nents, dst_nents;
- int mapped_src_nents, mapped_dst_nents;
struct scatterlist *src, *dst;
size_t pl, ml, split_size;
struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec;
u32 *mdptr;
bool diff_dst;
enum dma_data_direction dir_src;
+ struct sa_mapped_sg *mapped_sg;
gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
GFP_KERNEL : GFP_ATOMIC;
dma_rx = pdata->dma_rx1;
ddev = dma_rx->device->dev;
+ rxd->ddev = ddev;
memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
split_size = req->size;
+ mapped_sg = &rxd->mapped_sg[0];
if (sg_nents == 1 && split_size <= req->src->length) {
- src = &rxd->rx_sg;
+ src = &mapped_sg->static_sg;
+ src_nents = 1;
sg_init_table(src, 1);
sg_set_page(src, sg_page(req->src), split_size,
req->src->offset);
- src_nents = 1;
- dma_map_sg(ddev, src, sg_nents, dir_src);
+
+ mapped_sg->sgt.sgl = src;
+ mapped_sg->sgt.orig_nents = src_nents;
+ ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
+ if (ret)
+ return ret;
+
+ mapped_sg->dir = dir_src;
+ mapped_sg->mapped = true;
} else {
- mapped_src_nents = dma_map_sg(ddev, req->src, sg_nents,
- dir_src);
- ret = sg_split(req->src, mapped_src_nents, 0, 1, &split_size,
- &src, &src_nents, gfp_flags);
+ mapped_sg->sgt.sgl = req->src;
+ mapped_sg->sgt.orig_nents = sg_nents;
+ ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
+ if (ret)
+ return ret;
+
+ mapped_sg->dir = dir_src;
+ mapped_sg->mapped = true;
+
+ ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1,
+ &split_size, &src, &src_nents, gfp_flags);
if (ret) {
- src_nents = sg_nents;
- src = req->src;
+ src_nents = mapped_sg->sgt.nents;
+ src = mapped_sg->sgt.sgl;
} else {
- rxd->split_src_sg = src;
+ mapped_sg->split_sg = src;
}
}
+ dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE);
+
if (!diff_dst) {
dst_nents = src_nents;
dst = src;
} else {
dst_nents = sg_nents_for_len(req->dst, req->size);
+ mapped_sg = &rxd->mapped_sg[1];
if (dst_nents == 1 && split_size <= req->dst->length) {
- dst = &rxd->tx_sg;
+ dst = &mapped_sg->static_sg;
+ dst_nents = 1;
sg_init_table(dst, 1);
sg_set_page(dst, sg_page(req->dst), split_size,
req->dst->offset);
- dst_nents = 1;
- dma_map_sg(ddev, dst, dst_nents, DMA_FROM_DEVICE);
+
+ mapped_sg->sgt.sgl = dst;
+ mapped_sg->sgt.orig_nents = dst_nents;
+ ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
+ DMA_FROM_DEVICE, 0);
+ if (ret)
+ goto err_cleanup;
+
+ mapped_sg->dir = DMA_FROM_DEVICE;
+ mapped_sg->mapped = true;
} else {
- mapped_dst_nents = dma_map_sg(ddev, req->dst, dst_nents,
- DMA_FROM_DEVICE);
- ret = sg_split(req->dst, mapped_dst_nents, 0, 1,
- &split_size, &dst, &dst_nents,
+ mapped_sg->sgt.sgl = req->dst;
+ mapped_sg->sgt.orig_nents = dst_nents;
+ ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
+ DMA_FROM_DEVICE, 0);
+ if (ret)
+ goto err_cleanup;
+
+ mapped_sg->dir = DMA_FROM_DEVICE;
+ mapped_sg->mapped = true;
+
+ ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents,
+ 0, 1, &split_size, &dst, &dst_nents,
gfp_flags);
if (ret) {
- dst_nents = dst_nents;
- dst = req->dst;
+ dst_nents = mapped_sg->sgt.nents;
+ dst = mapped_sg->sgt.sgl;
} else {
- rxd->split_dst_sg = dst;
+ mapped_sg->split_sg = dst;
}
}
}
- if (unlikely(src_nents != sg_nents)) {
- dev_warn_ratelimited(sa_k3_dev, "failed to map tx pkt\n");
- ret = -EIO;
- goto err_cleanup;
- }
-
rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents,
DMA_DEV_TO_MEM,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
rxd->req = (void *)req->base;
rxd->enc = req->enc;
- rxd->ddev = ddev;
- rxd->src = src;
- rxd->dst = dst;
rxd->iv_idx = req->ctx->iv_idx;
rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size;
rxd->tx_in->callback = req->callback;
return -EINPROGRESS;
err_cleanup:
- dma_unmap_sg(ddev, req->src, sg_nents, DMA_TO_DEVICE);
- kfree(rxd->split_src_sg);
-
- if (req->src != req->dst) {
- dst_nents = sg_nents_for_len(req->dst, req->size);
- dma_unmap_sg(ddev, req->dst, dst_nents, DMA_FROM_DEVICE);
- kfree(rxd->split_dst_sg);
- }
-
- kfree(rxd);
+ sa_free_sa_rx_data(rxd);
return ret;
}
struct ahash_request *req;
struct crypto_ahash *tfm;
unsigned int authsize;
- int i, sg_nents;
+ int i;
size_t ml, pl;
u32 *result;
__be32 *mdptr;
+ sa_sync_from_device(rxd);
req = container_of(rxd->req, struct ahash_request, base);
tfm = crypto_ahash_reqtfm(req);
authsize = crypto_ahash_digestsize(tfm);
for (i = 0; i < (authsize / 4); i++)
result[i] = be32_to_cpu(mdptr[i + 4]);
- sg_nents = sg_nents_for_len(req->src, req->nbytes);
- dma_unmap_sg(rxd->ddev, req->src, sg_nents, DMA_FROM_DEVICE);
-
- kfree(rxd->split_src_sg);
-
- kfree(rxd);
+ sa_free_sa_rx_data(rxd);
ahash_request_complete(req, 0);
}
struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
- dev_dbg(sa_k3_dev, "init: digest size: %d, rctx=%llx\n",
- crypto_ahash_digestsize(tfm), (u64)rctx);
+ dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n",
+ crypto_ahash_digestsize(tfm), rctx);
ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
rctx->fallback_req.base.flags =
unsigned int authsize;
u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
size_t pl, ml;
- int i, sglen;
+ int i;
int err = 0;
u16 auth_len;
u32 *mdptr;
- bool diff_dst;
- enum dma_data_direction dir_src;
+ sa_sync_from_device(rxd);
req = container_of(rxd->req, struct aead_request, base);
tfm = crypto_aead_reqtfm(req);
start = req->assoclen + req->cryptlen;
authsize = crypto_aead_authsize(tfm);
- diff_dst = (req->src != req->dst) ? true : false;
- dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;
-
mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
for (i = 0; i < (authsize / 4); i++)
mdptr[i + 4] = swab32(mdptr[i + 4]);
auth_len = req->assoclen + req->cryptlen;
- if (!rxd->enc)
- auth_len -= authsize;
-
- sglen = sg_nents_for_len(rxd->src, auth_len);
- dma_unmap_sg(rxd->ddev, rxd->src, sglen, dir_src);
- kfree(rxd->split_src_sg);
-
- if (diff_dst) {
- sglen = sg_nents_for_len(rxd->dst, auth_len);
- dma_unmap_sg(rxd->ddev, rxd->dst, sglen, DMA_FROM_DEVICE);
- kfree(rxd->split_dst_sg);
- }
if (rxd->enc) {
scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
1);
} else {
+ auth_len -= authsize;
start -= authsize;
scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
0);
err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0;
}
- kfree(rxd);
+ sa_free_sa_rx_data(rxd);
aead_request_complete(req, err);
}
return ret;
dd->dma_rx1 = dma_request_chan(dd->dev, "rx1");
- if (IS_ERR(dd->dma_rx1)) {
- if (PTR_ERR(dd->dma_rx1) != -EPROBE_DEFER)
- dev_err(dd->dev, "Unable to request rx1 DMA channel\n");
- return PTR_ERR(dd->dma_rx1);
- }
+ if (IS_ERR(dd->dma_rx1))
+ return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1),
+ "Unable to request rx1 DMA channel\n");
dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
if (IS_ERR(dd->dma_rx2)) {
dma_release_channel(dd->dma_rx1);
- if (PTR_ERR(dd->dma_rx2) != -EPROBE_DEFER)
- dev_err(dd->dev, "Unable to request rx2 DMA channel\n");
- return PTR_ERR(dd->dma_rx2);
+ return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2),
+ "Unable to request rx2 DMA channel\n");
}
dd->dma_tx = dma_request_chan(dd->dev, "tx");
if (IS_ERR(dd->dma_tx)) {
- if (PTR_ERR(dd->dma_tx) != -EPROBE_DEFER)
- dev_err(dd->dev, "Unable to request tx DMA channel\n");
- ret = PTR_ERR(dd->dma_tx);
+ ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx),
+ "Unable to request tx DMA channel\n");
goto err_dma_tx;
}
pm_runtime_enable(dev);
ret = pm_runtime_get_sync(dev);
- if (ret) {
+ if (ret < 0) {
dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__,
ret);
return ret;
#include <crypto/sha.h>
#include <linux/clk.h>
-#include <linux/crypto.h>
+#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/irq.h>
tristate "Support for STM32 crc accelerators"
depends on ARCH_STM32
select CRYPTO_HASH
+ select CRC32
help
This enables support for the CRC32 hw accelerator which can be found
on STMicroelectronics STM32 SOC.
#include <linux/bitrev.h>
#include <linux/clk.h>
+#include <linux/crc32.h>
#include <linux/crc32poly.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mod_devicetable.h>
#include <linux/platform_device.h>
struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc);
struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
struct stm32_crc *crc;
- unsigned long flags;
crc = stm32_crc_get_next_crc();
if (!crc)
pm_runtime_get_sync(crc->dev);
- spin_lock_irqsave(&crc->lock, flags);
+ if (!spin_trylock(&crc->lock)) {
+ /* Hardware is busy, calculate crc32 by software */
+ if (mctx->poly == CRC32_POLY_LE)
+ ctx->partial = crc32_le(ctx->partial, d8, length);
+ else
+ ctx->partial = __crc32c_le(ctx->partial, d8, length);
+
+ goto pm_out;
+ }
/*
* Restore previously calculated CRC for this context as init value
/* Store partial result */
ctx->partial = readl_relaxed(crc->regs + CRC_DR);
- spin_unlock_irqrestore(&crc->lock, flags);
+ spin_unlock(&crc->lock);
+pm_out:
pm_runtime_mark_last_busy(crc->dev);
pm_runtime_put_autosuspend(crc->dev);
return burst_update(desc, d8, length);
/* Digest first bytes not 32bit aligned at first pass in the loop */
- size = min(length,
- burst_sz + (unsigned int)d8 - ALIGN_DOWN((unsigned int)d8,
- sizeof(u32)));
+ size = min_t(size_t, length, burst_sz + (size_t)d8 -
+ ALIGN_DOWN((size_t)d8, sizeof(u32)));
for (rem_sz = length, cur = d8; rem_sz;
rem_sz -= size, cur += size, size = min(rem_sz, burst_sz)) {
ret = burst_update(desc, cur, size);
struct crypto_engine_ctx enginectx;
struct stm32_cryp *cryp;
int keylen;
- u32 key[AES_KEYSIZE_256 / sizeof(u32)];
+ __be32 key[AES_KEYSIZE_256 / sizeof(u32)];
unsigned long flags;
};
return 0;
}
-static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, u32 *iv)
+static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv)
{
if (!iv)
return;
- stm32_cryp_write(cryp, CRYP_IV0LR, cpu_to_be32(*iv++));
- stm32_cryp_write(cryp, CRYP_IV0RR, cpu_to_be32(*iv++));
+ stm32_cryp_write(cryp, CRYP_IV0LR, be32_to_cpu(*iv++));
+ stm32_cryp_write(cryp, CRYP_IV0RR, be32_to_cpu(*iv++));
if (is_aes(cryp)) {
- stm32_cryp_write(cryp, CRYP_IV1LR, cpu_to_be32(*iv++));
- stm32_cryp_write(cryp, CRYP_IV1RR, cpu_to_be32(*iv++));
+ stm32_cryp_write(cryp, CRYP_IV1LR, be32_to_cpu(*iv++));
+ stm32_cryp_write(cryp, CRYP_IV1RR, be32_to_cpu(*iv++));
}
}
static void stm32_cryp_get_iv(struct stm32_cryp *cryp)
{
struct skcipher_request *req = cryp->req;
- u32 *tmp = (void *)req->iv;
+ __be32 *tmp = (void *)req->iv;
if (!tmp)
return;
int r_id;
if (is_des(c)) {
- stm32_cryp_write(c, CRYP_K1LR, cpu_to_be32(c->ctx->key[0]));
- stm32_cryp_write(c, CRYP_K1RR, cpu_to_be32(c->ctx->key[1]));
+ stm32_cryp_write(c, CRYP_K1LR, be32_to_cpu(c->ctx->key[0]));
+ stm32_cryp_write(c, CRYP_K1RR, be32_to_cpu(c->ctx->key[1]));
} else {
r_id = CRYP_K3RR;
for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4)
stm32_cryp_write(c, r_id,
- cpu_to_be32(c->ctx->key[i - 1]));
+ be32_to_cpu(c->ctx->key[i - 1]));
}
}
static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg)
{
int ret;
- u32 iv[4];
+ __be32 iv[4];
/* Phase 1 : init */
memcpy(iv, cryp->areq->iv, 12);
{
int ret;
u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE];
+ __be32 *bd;
u32 *d;
unsigned int i, textlen;
memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
iv[AES_BLOCK_SIZE - 1] = 1;
- stm32_cryp_hw_write_iv(cryp, (u32 *)iv);
+ stm32_cryp_hw_write_iv(cryp, (__be32 *)iv);
/* Build B0 */
memcpy(b0, iv, AES_BLOCK_SIZE);
/* Write B0 */
d = (u32 *)b0;
+ bd = (__be32 *)b0;
for (i = 0; i < AES_BLOCK_32; i++) {
+ u32 xd = d[i];
+
if (!cryp->caps->padding_wa)
- *d = cpu_to_be32(*d);
- stm32_cryp_write(cryp, CRYP_DIN, *d++);
+ xd = be32_to_cpu(bd[i]);
+ stm32_cryp_write(cryp, CRYP_DIN, xd);
}
/* Wait for end of processing */
case CR_TDES_CBC:
case CR_AES_CBC:
case CR_AES_CTR:
- stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->req->iv);
+ stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv);
break;
default:
/* GCM: write aad and payload size (in bits) */
size_bit = cryp->areq->assoclen * 8;
if (cryp->caps->swap_final)
- size_bit = cpu_to_be32(size_bit);
+ size_bit = (__force u32)cpu_to_be32(size_bit);
stm32_cryp_write(cryp, CRYP_DIN, 0);
stm32_cryp_write(cryp, CRYP_DIN, size_bit);
cryp->areq->cryptlen - AES_BLOCK_SIZE;
size_bit *= 8;
if (cryp->caps->swap_final)
- size_bit = cpu_to_be32(size_bit);
+ size_bit = (__force u32)cpu_to_be32(size_bit);
stm32_cryp_write(cryp, CRYP_DIN, 0);
stm32_cryp_write(cryp, CRYP_DIN, size_bit);
/* CCM: write CTR0 */
u8 iv[AES_BLOCK_SIZE];
u32 *iv32 = (u32 *)iv;
+ __be32 *biv;
+
+ biv = (void *)iv;
memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE);
memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
for (i = 0; i < AES_BLOCK_32; i++) {
+ u32 xiv = iv32[i];
+
if (!cryp->caps->padding_wa)
- *iv32 = cpu_to_be32(*iv32);
- stm32_cryp_write(cryp, CRYP_DIN, *iv32++);
+ xiv = be32_to_cpu(biv[i]);
+ stm32_cryp_write(cryp, CRYP_DIN, xiv);
}
}
#include <linux/clk.h>
#include <linux/crypto.h>
#include <linux/delay.h>
+#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/interrupt.h>
#include <linux/io.h>
static void stm32_hash_copy_hash(struct ahash_request *req)
{
struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req);
- u32 *hash = (u32 *)rctx->digest;
+ __be32 *hash = (void *)rctx->digest;
unsigned int i, hashsize;
switch (rctx->flags & HASH_FLAGS_ALGO_MASK) {
}
for (i = 0; i < hashsize / sizeof(u32); i++)
- hash[i] = be32_to_cpu(stm32_hash_read(rctx->hdev,
+ hash[i] = cpu_to_be32(stm32_hash_read(rctx->hdev,
HASH_HREG(i)));
}
}
hdev->clk = devm_clk_get(&pdev->dev, NULL);
- if (IS_ERR(hdev->clk)) {
- if (PTR_ERR(hdev->clk) != -EPROBE_DEFER) {
- dev_err(dev, "failed to get clock for hash (%lu)\n",
- PTR_ERR(hdev->clk));
- }
-
- return PTR_ERR(hdev->clk);
- }
+ if (IS_ERR(hdev->clk))
+ return dev_err_probe(dev, PTR_ERR(hdev->clk),
+ "failed to get clock for hash\n");
ret = clk_prepare_enable(hdev->clk);
if (ret) {
struct talitos_private *priv = dev_get_drvdata(dev);
int err;
- priv->rng.name = dev_driver_string(dev),
- priv->rng.init = talitos_rng_init,
- priv->rng.data_present = talitos_rng_data_present,
- priv->rng.data_read = talitos_rng_data_read,
+ priv->rng.name = dev_driver_string(dev);
+ priv->rng.init = talitos_rng_init;
+ priv->rng.data_present = talitos_rng_data_present;
+ priv->rng.data_read = talitos_rng_data_read;
priv->rng.priv = (unsigned long)dev;
err = hwrng_register(&priv->rng);
#include <linux/clk.h>
#include <linux/completion.h>
-#include <linux/crypto.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/irqreturn.h>
+#include <linux/kernel.h>
#include <linux/klist.h>
#include <linux/module.h>
#include <linux/mod_devicetable.h>
#include <linux/platform_data/dma-ste-dma40.h>
#include <crypto/aes.h>
-#include <crypto/algapi.h>
#include <crypto/ctr.h>
#include <crypto/internal/des.h>
#include <crypto/internal/skcipher.h>
static struct cryp_driver_data driver_data;
-/**
- * uint8p_to_uint32_be - 4*uint8 to uint32 big endian
- * @in: Data to convert.
- */
-static inline u32 uint8p_to_uint32_be(u8 *in)
-{
- u32 *data = (u32 *)in;
-
- return cpu_to_be32p(data);
-}
-
/**
* swap_bits_in_byte - mirror the bits in a byte
* @b: the byte to be mirrored
int i;
int status = 0;
int num_of_regs = ctx->blocksize / 8;
+ __be32 *civ = (__be32 *)ctx->iv;
u32 iv[AES_BLOCK_SIZE / 4];
dev_dbg(device_data->dev, "[%s]", __func__);
}
for (i = 0; i < ctx->blocksize / 4; i++)
- iv[i] = uint8p_to_uint32_be(ctx->iv + i*4);
+ iv[i] = be32_to_cpup(civ + i);
for (i = 0; i < num_of_regs; i++) {
status = cfg_iv(device_data, iv[i*2], iv[i*2+1],
int i;
int num_of_regs = ctx->keylen / 8;
u32 swapped_key[CRYP_MAX_KEY_SIZE / 4];
+ __be32 *ckey = (__be32 *)ctx->key;
int cryp_error = 0;
dev_dbg(ctx->device->dev, "[%s]", __func__);
if (mode_is_aes(ctx->config.algomode)) {
- swap_words_in_key_and_bits_in_byte((u8 *)ctx->key,
+ swap_words_in_key_and_bits_in_byte((u8 *)ckey,
(u8 *)swapped_key,
ctx->keylen);
} else {
for (i = 0; i < ctx->keylen / 4; i++)
- swapped_key[i] = uint8p_to_uint32_be(ctx->key + i*4);
+ swapped_key[i] = be32_to_cpup(ckey + i);
}
for (i = 0; i < num_of_regs; i++) {
cryp_error = set_key(ctx->device,
- *(((u32 *)swapped_key)+i*2),
- *(((u32 *)swapped_key)+i*2+1),
+ swapped_key[i * 2],
+ swapped_key[i * 2 + 1],
(enum cryp_key_reg_index) i);
if (cryp_error != 0) {
#include <linux/clk.h>
#include <linux/device.h>
+#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/io.h>
struct hash_ctx *ctx = crypto_ahash_ctx(tfm);
struct hash_req_ctx *req_ctx = ahash_request_ctx(req);
struct crypto_hash_walk walk;
- int msg_length = crypto_hash_walk_first(req, &walk);
-
- /* Empty message ("") is correct indata */
- if (msg_length == 0)
- return ret;
+ int msg_length;
index = req_ctx->state.index;
buffer = (u8 *)req_ctx->state.buffer;
+ ret = hash_get_device_data(ctx, &device_data);
+ if (ret)
+ return ret;
+
+ msg_length = crypto_hash_walk_first(req, &walk);
+
+ /* Empty message ("") is correct indata */
+ if (msg_length == 0) {
+ ret = 0;
+ goto release_dev;
+ }
+
/* Check if ctx->state.length + msg_length
overflows */
if (msg_length > (req_ctx->state.length.low_word + msg_length) &&
HASH_HIGH_WORD_MAX_VAL == req_ctx->state.length.high_word) {
pr_err("%s: HASH_MSG_LENGTH_OVERFLOW!\n", __func__);
- return -EPERM;
+ ret = crypto_hash_walk_done(&walk, -EPERM);
+ goto release_dev;
}
- ret = hash_get_device_data(ctx, &device_data);
- if (ret)
- return ret;
-
/* Main loop */
while (0 != msg_length) {
data_buffer = walk.data;
if (ret) {
dev_err(device_data->dev, "%s: hash_internal_hw_update() failed!\n",
__func__);
- goto out;
+ crypto_hash_walk_done(&walk, ret);
+ goto release_dev;
}
msg_length = crypto_hash_walk_done(&walk, 0);
dev_dbg(device_data->dev, "%s: indata length=%d, bin=%d\n",
__func__, req_ctx->state.index, req_ctx->state.bit_index);
-out:
+release_dev:
release_hash_device(device_data);
return ret;
select CRYPTO_AEAD
select CRYPTO_SKCIPHER
select CRYPTO_ENGINE
- default m
help
This driver provides support for virtio crypto device. If you
choose 'M' here, this module will be called virtio_crypto.
#include <crypto/internal/aead.h>
#include <crypto/scatterwalk.h>
+#include <linux/dma-mapping.h>
#include <linux/module.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
struct devfreq *p_devfreq = NULL;
unsigned long cur_freq, min_freq, max_freq;
unsigned int polling_ms;
+ unsigned int timer;
- seq_printf(s, "%-30s %-30s %-15s %10s %12s %12s %12s\n",
+ seq_printf(s, "%-30s %-30s %-15s %-10s %10s %12s %12s %12s\n",
"dev",
"parent_dev",
"governor",
+ "timer",
"polling_ms",
"cur_freq_Hz",
"min_freq_Hz",
"max_freq_Hz");
- seq_printf(s, "%30s %30s %15s %10s %12s %12s %12s\n",
+ seq_printf(s, "%30s %30s %15s %10s %10s %12s %12s %12s\n",
"------------------------------",
"------------------------------",
"---------------",
"----------",
+ "----------",
"------------",
"------------",
"------------");
cur_freq = devfreq->previous_freq;
get_freq_range(devfreq, &min_freq, &max_freq);
polling_ms = devfreq->profile->polling_ms;
+ timer = devfreq->profile->timer;
mutex_unlock(&devfreq->lock);
seq_printf(s,
- "%-30s %-30s %-15s %10d %12ld %12ld %12ld\n",
+ "%-30s %-30s %-15s %-10s %10d %12ld %12ld %12ld\n",
dev_name(&devfreq->dev),
p_devfreq ? dev_name(&p_devfreq->dev) : "null",
devfreq->governor_name,
+ polling_ms ? timer_name[timer] : "null",
polling_ms,
cur_freq,
min_freq,
rate = clk_round_rate(tegra->emc_clock, ULONG_MAX);
if (rate < 0) {
dev_err(&pdev->dev, "Failed to round clock rate: %ld\n", rate);
- return rate;
+ err = rate;
+ goto disable_clk;
}
tegra->max_freq = rate / KHZ;
dev_pm_opp_remove_all_dynamic(&pdev->dev);
reset_control_reset(tegra->reset);
+disable_clk:
clk_disable_unprepare(tegra->clock);
return err;
struct dma_buf *dmabuf;
dmabuf = dentry->d_fsdata;
+ if (unlikely(!dmabuf))
+ return;
BUG_ON(dmabuf->vmapping_counter);
* @nr_channels: number of channels under test
* @lock: access protection to the fields of this structure
* @did_init: module has been initialized completely
+ * @last_error: test has faced configuration issues
*/
static struct dmatest_info {
/* Test parameters */
/* Internal state */
struct list_head channels;
unsigned int nr_channels;
+ int last_error;
struct mutex lock;
bool did_init;
} test_info = {
return ret;
} else if (dmatest_run) {
if (!is_threaded_test_pending(info)) {
- pr_info("No channels configured, continue with any\n");
- if (!is_threaded_test_run(info))
- stop_threaded_test(info);
- add_threaded_test(info);
+ /*
+ * We have nothing to run. This can be due to:
+ */
+ ret = info->last_error;
+ if (ret) {
+ /* 1) Misconfiguration */
+ pr_err("Channel misconfigured, can't continue\n");
+ mutex_unlock(&info->lock);
+ return ret;
+ } else {
+ /* 2) We rely on defaults */
+ pr_info("No channels configured, continue with any\n");
+ if (!is_threaded_test_run(info))
+ stop_threaded_test(info);
+ add_threaded_test(info);
+ }
}
start_threaded_tests(info);
} else {
struct dmatest_info *info = &test_info;
struct dmatest_chan *dtc;
char chan_reset_val[20];
- int ret = 0;
+ int ret;
mutex_lock(&info->lock);
ret = param_set_copystring(val, kp);
goto add_chan_err;
}
+ info->last_error = ret;
mutex_unlock(&info->lock);
return ret;
add_chan_err:
param_set_copystring(chan_reset_val, kp);
+ info->last_error = ret;
mutex_unlock(&info->lock);
return ret;
In addition, there are two control files, inject_read and inject_write,
which trigger the DRAM ECC Read and Write respectively.
+config EDAC_AL_MC
+ tristate "Amazon's Annapurna Lab Memory Controller"
+ depends on (ARCH_ALPINE || COMPILE_TEST)
+ help
+ Support for error detection and correction for Amazon's Annapurna
+ Labs Alpine chips which allow 1 bit correction and 2 bits detection.
+
config EDAC_AMD76X
tristate "AMD 76x (760, 762, 768)"
depends on PCI && X86_32
edac_mce_amd-y := mce_amd.o
obj-$(CONFIG_EDAC_DECODE_MCE) += edac_mce_amd.o
+obj-$(CONFIG_EDAC_AL_MC) += al_mc_edac.o
obj-$(CONFIG_EDAC_AMD76X) += amd76x_edac.o
obj-$(CONFIG_EDAC_CPC925) += cpc925_edac.o
obj-$(CONFIG_EDAC_I5000) += i5000_edac.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ */
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/edac.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include "edac_module.h"
+
+/* Registers Offset */
+#define AL_MC_ECC_CFG 0x70
+#define AL_MC_ECC_CLEAR 0x7c
+#define AL_MC_ECC_ERR_COUNT 0x80
+#define AL_MC_ECC_CE_ADDR0 0x84
+#define AL_MC_ECC_CE_ADDR1 0x88
+#define AL_MC_ECC_UE_ADDR0 0xa4
+#define AL_MC_ECC_UE_ADDR1 0xa8
+#define AL_MC_ECC_CE_SYND0 0x8c
+#define AL_MC_ECC_CE_SYND1 0x90
+#define AL_MC_ECC_CE_SYND2 0x94
+#define AL_MC_ECC_UE_SYND0 0xac
+#define AL_MC_ECC_UE_SYND1 0xb0
+#define AL_MC_ECC_UE_SYND2 0xb4
+
+/* Registers Fields */
+#define AL_MC_ECC_CFG_SCRUB_DISABLED BIT(4)
+
+#define AL_MC_ECC_CLEAR_UE_COUNT BIT(3)
+#define AL_MC_ECC_CLEAR_CE_COUNT BIT(2)
+#define AL_MC_ECC_CLEAR_UE_ERR BIT(1)
+#define AL_MC_ECC_CLEAR_CE_ERR BIT(0)
+
+#define AL_MC_ECC_ERR_COUNT_UE GENMASK(31, 16)
+#define AL_MC_ECC_ERR_COUNT_CE GENMASK(15, 0)
+
+#define AL_MC_ECC_CE_ADDR0_RANK GENMASK(25, 24)
+#define AL_MC_ECC_CE_ADDR0_ROW GENMASK(17, 0)
+
+#define AL_MC_ECC_CE_ADDR1_BG GENMASK(25, 24)
+#define AL_MC_ECC_CE_ADDR1_BANK GENMASK(18, 16)
+#define AL_MC_ECC_CE_ADDR1_COLUMN GENMASK(11, 0)
+
+#define AL_MC_ECC_UE_ADDR0_RANK GENMASK(25, 24)
+#define AL_MC_ECC_UE_ADDR0_ROW GENMASK(17, 0)
+
+#define AL_MC_ECC_UE_ADDR1_BG GENMASK(25, 24)
+#define AL_MC_ECC_UE_ADDR1_BANK GENMASK(18, 16)
+#define AL_MC_ECC_UE_ADDR1_COLUMN GENMASK(11, 0)
+
+#define DRV_NAME "al_mc_edac"
+#define AL_MC_EDAC_MSG_MAX 256
+
+struct al_mc_edac {
+ void __iomem *mmio_base;
+ spinlock_t lock;
+ int irq_ce;
+ int irq_ue;
+};
+
+static void prepare_msg(char *message, size_t buffer_size,
+ enum hw_event_mc_err_type type,
+ u8 rank, u32 row, u8 bg, u8 bank, u16 column,
+ u32 syn0, u32 syn1, u32 syn2)
+{
+ snprintf(message, buffer_size,
+ "%s rank=0x%x row=0x%x bg=0x%x bank=0x%x col=0x%x syn0: 0x%x syn1: 0x%x syn2: 0x%x",
+ type == HW_EVENT_ERR_UNCORRECTED ? "UE" : "CE",
+ rank, row, bg, bank, column, syn0, syn1, syn2);
+}
+
+static int handle_ce(struct mem_ctl_info *mci)
+{
+ u32 eccerrcnt, ecccaddr0, ecccaddr1, ecccsyn0, ecccsyn1, ecccsyn2, row;
+ struct al_mc_edac *al_mc = mci->pvt_info;
+ char msg[AL_MC_EDAC_MSG_MAX];
+ u16 ce_count, column;
+ unsigned long flags;
+ u8 rank, bg, bank;
+
+ eccerrcnt = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_ERR_COUNT);
+ ce_count = FIELD_GET(AL_MC_ECC_ERR_COUNT_CE, eccerrcnt);
+ if (!ce_count)
+ return 0;
+
+ ecccaddr0 = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_CE_ADDR0);
+ ecccaddr1 = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_CE_ADDR1);
+ ecccsyn0 = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_CE_SYND0);
+ ecccsyn1 = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_CE_SYND1);
+ ecccsyn2 = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_CE_SYND2);
+
+ writel_relaxed(AL_MC_ECC_CLEAR_CE_COUNT | AL_MC_ECC_CLEAR_CE_ERR,
+ al_mc->mmio_base + AL_MC_ECC_CLEAR);
+
+ dev_dbg(mci->pdev, "eccuaddr0=0x%08x eccuaddr1=0x%08x\n",
+ ecccaddr0, ecccaddr1);
+
+ rank = FIELD_GET(AL_MC_ECC_CE_ADDR0_RANK, ecccaddr0);
+ row = FIELD_GET(AL_MC_ECC_CE_ADDR0_ROW, ecccaddr0);
+
+ bg = FIELD_GET(AL_MC_ECC_CE_ADDR1_BG, ecccaddr1);
+ bank = FIELD_GET(AL_MC_ECC_CE_ADDR1_BANK, ecccaddr1);
+ column = FIELD_GET(AL_MC_ECC_CE_ADDR1_COLUMN, ecccaddr1);
+
+ prepare_msg(msg, sizeof(msg), HW_EVENT_ERR_CORRECTED,
+ rank, row, bg, bank, column,
+ ecccsyn0, ecccsyn1, ecccsyn2);
+
+ spin_lock_irqsave(&al_mc->lock, flags);
+ edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci,
+ ce_count, 0, 0, 0, 0, 0, -1, mci->ctl_name, msg);
+ spin_unlock_irqrestore(&al_mc->lock, flags);
+
+ return ce_count;
+}
+
+static int handle_ue(struct mem_ctl_info *mci)
+{
+ u32 eccerrcnt, eccuaddr0, eccuaddr1, eccusyn0, eccusyn1, eccusyn2, row;
+ struct al_mc_edac *al_mc = mci->pvt_info;
+ char msg[AL_MC_EDAC_MSG_MAX];
+ u16 ue_count, column;
+ unsigned long flags;
+ u8 rank, bg, bank;
+
+ eccerrcnt = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_ERR_COUNT);
+ ue_count = FIELD_GET(AL_MC_ECC_ERR_COUNT_UE, eccerrcnt);
+ if (!ue_count)
+ return 0;
+
+ eccuaddr0 = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_UE_ADDR0);
+ eccuaddr1 = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_UE_ADDR1);
+ eccusyn0 = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_UE_SYND0);
+ eccusyn1 = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_UE_SYND1);
+ eccusyn2 = readl_relaxed(al_mc->mmio_base + AL_MC_ECC_UE_SYND2);
+
+ writel_relaxed(AL_MC_ECC_CLEAR_UE_COUNT | AL_MC_ECC_CLEAR_UE_ERR,
+ al_mc->mmio_base + AL_MC_ECC_CLEAR);
+
+ dev_dbg(mci->pdev, "eccuaddr0=0x%08x eccuaddr1=0x%08x\n",
+ eccuaddr0, eccuaddr1);
+
+ rank = FIELD_GET(AL_MC_ECC_UE_ADDR0_RANK, eccuaddr0);
+ row = FIELD_GET(AL_MC_ECC_UE_ADDR0_ROW, eccuaddr0);
+
+ bg = FIELD_GET(AL_MC_ECC_UE_ADDR1_BG, eccuaddr1);
+ bank = FIELD_GET(AL_MC_ECC_UE_ADDR1_BANK, eccuaddr1);
+ column = FIELD_GET(AL_MC_ECC_UE_ADDR1_COLUMN, eccuaddr1);
+
+ prepare_msg(msg, sizeof(msg), HW_EVENT_ERR_UNCORRECTED,
+ rank, row, bg, bank, column,
+ eccusyn0, eccusyn1, eccusyn2);
+
+ spin_lock_irqsave(&al_mc->lock, flags);
+ edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci,
+ ue_count, 0, 0, 0, 0, 0, -1, mci->ctl_name, msg);
+ spin_unlock_irqrestore(&al_mc->lock, flags);
+
+ return ue_count;
+}
+
+static void al_mc_edac_check(struct mem_ctl_info *mci)
+{
+ struct al_mc_edac *al_mc = mci->pvt_info;
+
+ if (al_mc->irq_ue <= 0)
+ handle_ue(mci);
+
+ if (al_mc->irq_ce <= 0)
+ handle_ce(mci);
+}
+
+static irqreturn_t al_mc_edac_irq_handler_ue(int irq, void *info)
+{
+ struct platform_device *pdev = info;
+ struct mem_ctl_info *mci = platform_get_drvdata(pdev);
+
+ if (handle_ue(mci))
+ return IRQ_HANDLED;
+ return IRQ_NONE;
+}
+
+static irqreturn_t al_mc_edac_irq_handler_ce(int irq, void *info)
+{
+ struct platform_device *pdev = info;
+ struct mem_ctl_info *mci = platform_get_drvdata(pdev);
+
+ if (handle_ce(mci))
+ return IRQ_HANDLED;
+ return IRQ_NONE;
+}
+
+static enum scrub_type get_scrub_mode(void __iomem *mmio_base)
+{
+ u32 ecccfg0;
+
+ ecccfg0 = readl(mmio_base + AL_MC_ECC_CFG);
+
+ if (FIELD_GET(AL_MC_ECC_CFG_SCRUB_DISABLED, ecccfg0))
+ return SCRUB_NONE;
+ else
+ return SCRUB_HW_SRC;
+}
+
+static void devm_al_mc_edac_free(void *data)
+{
+ edac_mc_free(data);
+}
+
+static void devm_al_mc_edac_del(void *data)
+{
+ edac_mc_del_mc(data);
+}
+
+static int al_mc_edac_probe(struct platform_device *pdev)
+{
+ struct edac_mc_layer layers[1];
+ struct mem_ctl_info *mci;
+ struct al_mc_edac *al_mc;
+ void __iomem *mmio_base;
+ struct dimm_info *dimm;
+ int ret;
+
+ mmio_base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(mmio_base)) {
+ dev_err(&pdev->dev, "failed to ioremap memory (%ld)\n",
+ PTR_ERR(mmio_base));
+ return PTR_ERR(mmio_base);
+ }
+
+ layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
+ layers[0].size = 1;
+ layers[0].is_virt_csrow = false;
+ mci = edac_mc_alloc(0, ARRAY_SIZE(layers), layers,
+ sizeof(struct al_mc_edac));
+ if (!mci)
+ return -ENOMEM;
+
+ ret = devm_add_action(&pdev->dev, devm_al_mc_edac_free, mci);
+ if (ret) {
+ edac_mc_free(mci);
+ return ret;
+ }
+
+ platform_set_drvdata(pdev, mci);
+ al_mc = mci->pvt_info;
+
+ al_mc->mmio_base = mmio_base;
+
+ al_mc->irq_ue = of_irq_get_byname(pdev->dev.of_node, "ue");
+ if (al_mc->irq_ue <= 0)
+ dev_dbg(&pdev->dev,
+ "no IRQ defined for UE - falling back to polling\n");
+
+ al_mc->irq_ce = of_irq_get_byname(pdev->dev.of_node, "ce");
+ if (al_mc->irq_ce <= 0)
+ dev_dbg(&pdev->dev,
+ "no IRQ defined for CE - falling back to polling\n");
+
+ /*
+ * In case both interrupts (ue/ce) are to be found, use interrupt mode.
+ * In case none of the interrupt are foud, use polling mode.
+ * In case only one interrupt is found, use interrupt mode for it but
+ * keep polling mode enable for the other.
+ */
+ if (al_mc->irq_ue <= 0 || al_mc->irq_ce <= 0) {
+ edac_op_state = EDAC_OPSTATE_POLL;
+ mci->edac_check = al_mc_edac_check;
+ } else {
+ edac_op_state = EDAC_OPSTATE_INT;
+ }
+
+ spin_lock_init(&al_mc->lock);
+
+ mci->mtype_cap = MEM_FLAG_DDR3 | MEM_FLAG_DDR4;
+ mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
+ mci->edac_cap = EDAC_FLAG_SECDED;
+ mci->mod_name = DRV_NAME;
+ mci->ctl_name = "al_mc";
+ mci->pdev = &pdev->dev;
+ mci->scrub_mode = get_scrub_mode(mmio_base);
+
+ dimm = *mci->dimms;
+ dimm->grain = 1;
+
+ ret = edac_mc_add_mc(mci);
+ if (ret < 0) {
+ dev_err(&pdev->dev,
+ "fail to add memory controller device (%d)\n",
+ ret);
+ return ret;
+ }
+
+ ret = devm_add_action(&pdev->dev, devm_al_mc_edac_del, &pdev->dev);
+ if (ret) {
+ edac_mc_del_mc(&pdev->dev);
+ return ret;
+ }
+
+ if (al_mc->irq_ue > 0) {
+ ret = devm_request_irq(&pdev->dev,
+ al_mc->irq_ue,
+ al_mc_edac_irq_handler_ue,
+ IRQF_SHARED,
+ pdev->name,
+ pdev);
+ if (ret != 0) {
+ dev_err(&pdev->dev,
+ "failed to request UE IRQ %d (%d)\n",
+ al_mc->irq_ue, ret);
+ return ret;
+ }
+ }
+
+ if (al_mc->irq_ce > 0) {
+ ret = devm_request_irq(&pdev->dev,
+ al_mc->irq_ce,
+ al_mc_edac_irq_handler_ce,
+ IRQF_SHARED,
+ pdev->name,
+ pdev);
+ if (ret != 0) {
+ dev_err(&pdev->dev,
+ "failed to request CE IRQ %d (%d)\n",
+ al_mc->irq_ce, ret);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+static const struct of_device_id al_mc_edac_of_match[] = {
+ { .compatible = "amazon,al-mc-edac", },
+ {},
+};
+
+MODULE_DEVICE_TABLE(of, al_mc_edac_of_match);
+
+static struct platform_driver al_mc_edac_driver = {
+ .probe = al_mc_edac_probe,
+ .driver = {
+ .name = DRV_NAME,
+ .of_match_table = al_mc_edac_of_match,
+ },
+};
+
+module_platform_driver(al_mc_edac_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Talel Shenhar");
+MODULE_DESCRIPTION("Amazon's Annapurna Lab's Memory Controller EDAC Driver");
break;
case 0x19:
+ if (pvt->model >= 0x20 && pvt->model <= 0x2f) {
+ fam_type = &family_types[F17_M70H_CPUS];
+ pvt->ops = &family_types[F17_M70H_CPUS].ops;
+ fam_type->ctl_name = "F19h_M20h";
+ break;
+ }
fam_type = &family_types[F19_CPUS];
pvt->ops = &family_types[F19_CPUS].ops;
family_types[F19_CPUS].ctl_name = "F19h";
/* register interrupt handler */
irq = platform_get_irq(pdev, 0);
dev_dbg(&pdev->dev, "got irq %d\n", irq);
- if (!irq)
- return -ENODEV;
+ if (irq < 0)
+ return irq;
rc = devm_request_irq(&pdev->dev, irq, mcr_isr, IRQF_TRIGGER_HIGH,
DRV_NAME, ctx);
.probe = aspeed_probe,
.remove = aspeed_remove
};
-
-
-static int __init aspeed_init(void)
-{
- return platform_driver_register(&aspeed_driver);
-}
-
-
-static void __exit aspeed_exit(void)
-{
- platform_driver_unregister(&aspeed_driver);
-}
-
-
-module_init(aspeed_init);
-module_exit(aspeed_exit);
-
+module_platform_driver(aspeed_driver);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Stefan Schaeckeler <sschaeck@cisco.com>");
* Implement support for the e7520, E7525, e7320 and i3100 memory controllers.
*
* Datasheets:
- * http://www.intel.in/content/www/in/en/chipsets/e7525-memory-controller-hub-datasheet.html
+ * https://www.intel.in/content/www/in/en/chipsets/e7525-memory-controller-hub-datasheet.html
* ftp://download.intel.com/design/intarch/datashts/31345803.pdf
*
* Written by Tom Zimmerman
struct device_attribute *mattr, char *data)
{
struct dimm_info *dimm = to_dimm(dev);
+ ssize_t count;
- return edac_dimm_info_location(dimm, data, PAGE_SIZE);
+ count = edac_dimm_info_location(dimm, data, PAGE_SIZE);
+ count += scnprintf(data + count, PAGE_SIZE - count, "\n");
+
+ return count;
}
static ssize_t dimmdev_label_show(struct device *dev,
char *data)
{
struct mem_ctl_info *mci = to_mci(dev);
- int i;
+ int len = PAGE_SIZE;
char *p = data;
+ int i, n;
for (i = 0; i < mci->n_layers; i++) {
- p += sprintf(p, "%s %d ",
- edac_layer_name[mci->layers[i].type],
- mci->layers[i].size - 1);
+ n = scnprintf(p, len, "%s %d ",
+ edac_layer_name[mci->layers[i].type],
+ mci->layers[i].size - 1);
+ len -= n;
+ if (len <= 0)
+ goto out;
+
+ p += n;
}
+ p += scnprintf(p, len, "\n");
+out:
return p - data;
}
*
* Copyright (c) 2013 by Mauro Carvalho Chehab
*
- * Red Hat Inc. http://www.redhat.com
+ * Red Hat Inc. https://www.redhat.com
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
p += sprintf(p, "rank:%d ", mem_err->rank);
if (mem_err->validation_bits & CPER_MEM_VALID_BANK)
p += sprintf(p, "bank:%d ", mem_err->bank);
- if (mem_err->validation_bits & CPER_MEM_VALID_ROW)
- p += sprintf(p, "row:%d ", mem_err->row);
+ if (mem_err->validation_bits & CPER_MEM_VALID_BANK_GROUP)
+ p += sprintf(p, "bank_group:%d ",
+ mem_err->bank >> CPER_MEM_BANK_GROUP_SHIFT);
+ if (mem_err->validation_bits & CPER_MEM_VALID_BANK_ADDRESS)
+ p += sprintf(p, "bank_address:%d ",
+ mem_err->bank & CPER_MEM_BANK_ADDRESS_MASK);
+ if (mem_err->validation_bits & (CPER_MEM_VALID_ROW | CPER_MEM_VALID_ROW_EXT)) {
+ u32 row = mem_err->row;
+
+ row |= cper_get_mem_extension(mem_err->validation_bits, mem_err->extended);
+ p += sprintf(p, "row:%d ", row);
+ }
if (mem_err->validation_bits & CPER_MEM_VALID_COLUMN)
p += sprintf(p, "col:%d ", mem_err->column);
if (mem_err->validation_bits & CPER_MEM_VALID_BIT_POSITION)
strcpy(e->label, dimm->label);
}
}
+ if (mem_err->validation_bits & CPER_MEM_VALID_CHIP_ID)
+ p += sprintf(p, "chipID: %d ",
+ mem_err->extended >> CPER_MEM_CHIP_ID_SHIFT);
if (p > e->location)
*(p - 1) = '\0';
PCI_DEVICE_ID_INTEL_5100_19, 0);
if (!einj) {
ret = -ENODEV;
- goto bail_einj;
+ goto bail_mc_free;
}
rc = pci_enable_device(einj);
if (rc < 0) {
ret = rc;
- goto bail_disable_einj;
+ goto bail_einj;
}
-
mci->pdev = &pdev->dev;
priv = mci->pvt_info;
bail_scrub:
priv->scrub_enable = 0;
cancel_delayed_work_sync(&(priv->i5100_scrubbing));
- edac_mc_free(mci);
-
-bail_disable_einj:
pci_disable_device(einj);
bail_einj:
pci_dev_put(einj);
+bail_mc_free:
+ edac_mc_free(mci);
+
bail_disable_ch1:
pci_disable_device(ch1mm);
* Ben Woodard <woodard@redhat.com>
* Mauro Carvalho Chehab
*
- * Red Hat Inc. http://www.redhat.com
+ * Red Hat Inc. https://www.redhat.com
*
* Forked and adapted from the i5000_edac driver which was
* written by Douglas Thompson Linux Networx <norsk5@xmission.com>
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ben Woodard <woodard@redhat.com>");
MODULE_AUTHOR("Mauro Carvalho Chehab");
-MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
+MODULE_AUTHOR("Red Hat Inc. (https://www.redhat.com)");
MODULE_DESCRIPTION("MC Driver for Intel I5400 memory controllers - "
I5400_REVISION);
* Copyright (c) 2010 by:
* Mauro Carvalho Chehab
*
- * Red Hat Inc. http://www.redhat.com
+ * Red Hat Inc. https://www.redhat.com
*
* Intel 7300 Chipset Memory Controller Hub (MCH) - Datasheet
* http://www.intel.com/Assets/PDF/datasheet/318082.pdf
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Mauro Carvalho Chehab");
-MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
+MODULE_AUTHOR("Red Hat Inc. (https://www.redhat.com)");
MODULE_DESCRIPTION("MC Driver for Intel I7300 memory controllers - "
I7300_REVISION);
* Copyright (c) 2009-2010 by:
* Mauro Carvalho Chehab
*
- * Red Hat Inc. http://www.redhat.com
+ * Red Hat Inc. https://www.redhat.com
*
* Forked and adapted from the i5400_edac driver
*
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Mauro Carvalho Chehab");
-MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
+MODULE_AUTHOR("Red Hat Inc. (https://www.redhat.com)");
MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
I7CORE_REVISION);
* Since the DRAM controller is on the cpu chip, we can use its PCI device
* id to identify these processors.
*
- * PCI DRAM controller device ids (Taken from The PCI ID Repository - http://pci-ids.ucw.cz/)
+ * PCI DRAM controller device ids (Taken from The PCI ID Repository - https://pci-ids.ucw.cz/)
*
* 0108: Xeon E3-1200 Processor Family DRAM Controller
* 010c: Xeon E3-1200/2nd Generation Core Processor Family DRAM Controller
* 3e..: 8th/9th Gen Core Processor Host Bridge/DRAM Registers
*
* Based on Intel specification:
- * http://www.intel.com/content/dam/www/public/us/en/documents/datasheets/xeon-e3-1200v3-vol-2-datasheet.pdf
+ * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/xeon-e3-1200v3-vol-2-datasheet.pdf
* http://www.intel.com/content/www/us/en/processors/xeon/xeon-e3-1200-family-vol-2-datasheet.html
- * http://www.intel.com/content/www/us/en/processors/core/7th-gen-core-family-mobile-h-processor-lines-datasheet-vol-2.html
+ * https://www.intel.com/content/www/us/en/processors/core/7th-gen-core-family-mobile-h-processor-lines-datasheet-vol-2.html
* https://www.intel.com/content/www/us/en/products/docs/processors/core/8th-gen-core-family-datasheet-vol-2.html
*
* According to the above datasheet (p.16):
"L2 BTB Multi-Match Error",
"L2 Cache Response Poison Error",
"System Read Data Error",
+ "Hardware Assertion Error",
+ "L1-TLB Multi-Hit",
+ "L2-TLB Multi-Hit",
+ "BSR Parity Error",
+ "CT MCE",
};
static const char * const smca_l2_mce_desc[] = {
"Fetch address FIFO parity error",
"Patch RAM data parity error",
"Patch RAM sequencer parity error",
- "Micro-op buffer parity error"
+ "Micro-op buffer parity error",
+ "Hardware Assertion MCA Error",
};
static const char * const smca_ex_mce_desc[] = {
"Scheduling queue parity error",
"Branch buffer queue parity error",
"Hardware Assertion error",
+ "Spec Map parity error",
+ "Retire Map parity error",
};
static const char * const smca_fp_mce_desc[] = {
"Instruction Tag Cache Bank A ECC or parity error",
"Instruction Tag Cache Bank B ECC or parity error",
"System Hub Read Buffer ECC or parity error",
+ "PHY RAM ECC error",
};
static const char * const smca_mp5_mce_desc[] = {
pr_emerg(HW_ERR "%s Ext. Error Code: %d", ip_name, xec);
/* Only print the decode of valid error codes */
- if (xec < smca_mce_descs[bank_type].num_descs &&
- (hwid->xec_bitmap & BIT_ULL(xec))) {
+ if (xec < smca_mce_descs[bank_type].num_descs)
pr_cont(", %s.\n", smca_mce_descs[bank_type].descs[xec]);
- }
if (bank_type == SMCA_UMC && xec == 0 && decode_dram_ecc)
decode_dram_ecc(cpu_to_node(m->extcpu), m);
static enum dev_type __ibridge_get_width(u32 mtr)
{
- enum dev_type type;
+ enum dev_type type = DEV_UNKNOWN;
switch (mtr) {
- case 3:
- type = DEV_UNKNOWN;
- break;
case 2:
type = DEV_X16;
break;
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Mauro Carvalho Chehab");
-MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
+MODULE_AUTHOR("Red Hat Inc. (https://www.redhat.com)");
MODULE_DESCRIPTION("MC Driver for Intel Sandy Bridge and Ivy Bridge memory controllers - "
SBRIDGE_REVISION);
DEBUGFS_STRUCT(inject_ecc, 0200, thunderx_lmc_inject_ecc_write, NULL);
DEBUGFS_STRUCT(int_w1c, 0400, NULL, thunderx_lmc_int_read);
-struct debugfs_entry *lmc_dfs_ents[] = {
+static struct debugfs_entry *lmc_dfs_ents[] = {
&debugfs_mask0,
&debugfs_mask2,
&debugfs_parity_test,
// SPDX-License-Identifier: GPL-2.0
/*
- * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/
+ * Copyright (C) 2017 Texas Instruments Incorporated - https://www.ti.com/
*
* Texas Instruments DDR3 ECC error correction and detection driver
*
/* add EMIF ECC error handler */
error_irq = platform_get_irq(pdev, 0);
- if (!error_irq) {
+ if (error_irq < 0) {
+ ret = error_irq;
edac_printk(KERN_ERR, EDAC_MOD_NAME,
"EMIF irq number not defined.\n");
goto err;
int first_error;
};
-#define CROSSCALL_INIT(arg, event) (arg.event = event, \
- arg.first_error = 0, \
- atomic_set(&arg.errors, 0))
+#define CROSSCALL_INIT(arg, event) \
+ do { \
+ arg.event = event; \
+ arg.first_error = 0; \
+ atomic_set(&arg.errors, 0); \
+ } while (0)
+
+static inline int sdei_do_local_call(smp_call_func_t fn,
+ struct sdei_event *event)
+{
+ struct sdei_crosscall_args arg;
+
+ CROSSCALL_INIT(arg, event);
+ fn(&arg);
-static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
+ return arg.first_error;
+}
+
+static inline int sdei_do_cross_call(smp_call_func_t fn,
+ struct sdei_event *event)
{
struct sdei_crosscall_args arg;
return -ENOMEM;
}
- /* Not an error value ... */
- return sdei_err;
-}
-
-/*
- * If x0 is any of these values, then the call failed, use sdei_to_linux_errno()
- * to translate.
- */
-static int sdei_is_err(struct arm_smccc_res *res)
-{
- switch (res->a0) {
- case SDEI_NOT_SUPPORTED:
- case SDEI_INVALID_PARAMETERS:
- case SDEI_DENIED:
- case SDEI_PENDING:
- case SDEI_OUT_OF_RESOURCE:
- return true;
- }
-
- return false;
+ return 0;
}
static int invoke_sdei_fn(unsigned long function_id, unsigned long arg0,
unsigned long arg3, unsigned long arg4,
u64 *result)
{
- int err = 0;
+ int err;
struct arm_smccc_res res;
if (sdei_firmware_call) {
sdei_firmware_call(function_id, arg0, arg1, arg2, arg3, arg4,
&res);
- if (sdei_is_err(&res))
- err = sdei_to_linux_errno(res.a0);
+ err = sdei_to_linux_errno(res.a0);
} else {
/*
* !sdei_firmware_call means we failed to probe or called
lockdep_assert_held(&sdei_events_lock);
event = kzalloc(sizeof(*event), GFP_KERNEL);
- if (!event)
- return ERR_PTR(-ENOMEM);
+ if (!event) {
+ err = -ENOMEM;
+ goto fail;
+ }
INIT_LIST_HEAD(&event->list);
event->event_num = event_num;
err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
&result);
- if (err) {
- kfree(event);
- return ERR_PTR(err);
- }
+ if (err)
+ goto fail;
event->priority = result;
err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_TYPE,
&result);
- if (err) {
- kfree(event);
- return ERR_PTR(err);
- }
+ if (err)
+ goto fail;
event->type = result;
if (event->type == SDEI_EVENT_TYPE_SHARED) {
reg = kzalloc(sizeof(*reg), GFP_KERNEL);
if (!reg) {
- kfree(event);
- return ERR_PTR(-ENOMEM);
+ err = -ENOMEM;
+ goto fail;
}
- reg->event_num = event_num;
+ reg->event_num = event->event_num;
reg->priority = event->priority;
reg->callback = cb;
regs = alloc_percpu(struct sdei_registered_event);
if (!regs) {
- kfree(event);
- return ERR_PTR(-ENOMEM);
+ err = -ENOMEM;
+ goto fail;
}
for_each_possible_cpu(cpu) {
spin_unlock(&sdei_list_lock);
return event;
+
+fail:
+ kfree(event);
+ return ERR_PTR(err);
}
static void sdei_event_destroy_llocked(struct sdei_event *event)
sdei_cross_call_return(arg, err);
}
-static int _sdei_event_unregister(struct sdei_event *event)
-{
- lockdep_assert_held(&sdei_events_lock);
-
- if (event->type == SDEI_EVENT_TYPE_SHARED)
- return sdei_api_event_unregister(event->event_num);
-
- return sdei_do_cross_call(_local_event_unregister, event);
-}
-
int sdei_event_unregister(u32 event_num)
{
int err;
mutex_lock(&sdei_events_lock);
event = sdei_event_find(event_num);
- do {
- if (!event) {
- pr_warn("Event %u not registered\n", event_num);
- err = -ENOENT;
- break;
- }
+ if (!event) {
+ pr_warn("Event %u not registered\n", event_num);
+ err = -ENOENT;
+ goto unlock;
+ }
- spin_lock(&sdei_list_lock);
- event->reregister = false;
- event->reenable = false;
- spin_unlock(&sdei_list_lock);
+ spin_lock(&sdei_list_lock);
+ event->reregister = false;
+ event->reenable = false;
+ spin_unlock(&sdei_list_lock);
- err = _sdei_event_unregister(event);
- if (err)
- break;
+ if (event->type == SDEI_EVENT_TYPE_SHARED)
+ err = sdei_api_event_unregister(event->event_num);
+ else
+ err = sdei_do_cross_call(_local_event_unregister, event);
- sdei_event_destroy(event);
- } while (0);
+ if (err)
+ goto unlock;
+
+ sdei_event_destroy(event);
+unlock:
mutex_unlock(&sdei_events_lock);
return err;
if (event->type != SDEI_EVENT_TYPE_SHARED)
continue;
- err = _sdei_event_unregister(event);
+ err = sdei_api_event_unregister(event->event_num);
if (err)
break;
}
sdei_cross_call_return(arg, err);
}
-static int _sdei_event_register(struct sdei_event *event)
-{
- int err;
-
- lockdep_assert_held(&sdei_events_lock);
-
- if (event->type == SDEI_EVENT_TYPE_SHARED)
- return sdei_api_event_register(event->event_num,
- sdei_entry_point,
- event->registered,
- SDEI_EVENT_REGISTER_RM_ANY, 0);
-
- err = sdei_do_cross_call(_local_event_register, event);
- if (err)
- sdei_do_cross_call(_local_event_unregister, event);
-
- return err;
-}
-
int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
{
int err;
WARN_ON(in_nmi());
mutex_lock(&sdei_events_lock);
- do {
- if (sdei_event_find(event_num)) {
- pr_warn("Event %u already registered\n", event_num);
- err = -EBUSY;
- break;
- }
-
- event = sdei_event_create(event_num, cb, arg);
- if (IS_ERR(event)) {
- err = PTR_ERR(event);
- pr_warn("Failed to create event %u: %d\n", event_num,
- err);
- break;
- }
-
- cpus_read_lock();
- err = _sdei_event_register(event);
- if (err) {
- sdei_event_destroy(event);
- pr_warn("Failed to register event %u: %d\n", event_num,
- err);
- } else {
- spin_lock(&sdei_list_lock);
- event->reregister = true;
- spin_unlock(&sdei_list_lock);
- }
- cpus_read_unlock();
- } while (0);
- mutex_unlock(&sdei_events_lock);
-
- return err;
-}
-
-static int sdei_reregister_event_llocked(struct sdei_event *event)
-{
- int err;
-
- lockdep_assert_held(&sdei_events_lock);
- lockdep_assert_held(&sdei_list_lock);
+ if (sdei_event_find(event_num)) {
+ pr_warn("Event %u already registered\n", event_num);
+ err = -EBUSY;
+ goto unlock;
+ }
- err = _sdei_event_register(event);
- if (err) {
- pr_err("Failed to re-register event %u\n", event->event_num);
- sdei_event_destroy_llocked(event);
- return err;
+ event = sdei_event_create(event_num, cb, arg);
+ if (IS_ERR(event)) {
+ err = PTR_ERR(event);
+ pr_warn("Failed to create event %u: %d\n", event_num, err);
+ goto unlock;
}
- if (event->reenable) {
- if (event->type == SDEI_EVENT_TYPE_SHARED)
- err = sdei_api_event_enable(event->event_num);
- else
- err = sdei_do_cross_call(_local_event_enable, event);
+ cpus_read_lock();
+ if (event->type == SDEI_EVENT_TYPE_SHARED) {
+ err = sdei_api_event_register(event->event_num,
+ sdei_entry_point,
+ event->registered,
+ SDEI_EVENT_REGISTER_RM_ANY, 0);
+ } else {
+ err = sdei_do_cross_call(_local_event_register, event);
+ if (err)
+ sdei_do_cross_call(_local_event_unregister, event);
}
- if (err)
- pr_err("Failed to re-enable event %u\n", event->event_num);
+ if (err) {
+ sdei_event_destroy(event);
+ pr_warn("Failed to register event %u: %d\n", event_num, err);
+ goto cpu_unlock;
+ }
+ spin_lock(&sdei_list_lock);
+ event->reregister = true;
+ spin_unlock(&sdei_list_lock);
+cpu_unlock:
+ cpus_read_unlock();
+unlock:
+ mutex_unlock(&sdei_events_lock);
return err;
}
continue;
if (event->reregister) {
- err = sdei_reregister_event_llocked(event);
- if (err)
+ err = sdei_api_event_register(event->event_num,
+ sdei_entry_point, event->registered,
+ SDEI_EVENT_REGISTER_RM_ANY, 0);
+ if (err) {
+ pr_err("Failed to re-register event %u\n",
+ event->event_num);
+ sdei_event_destroy_llocked(event);
break;
+ }
+ }
+
+ if (event->reenable) {
+ err = sdei_api_event_enable(event->event_num);
+ if (err) {
+ pr_err("Failed to re-enable event %u\n",
+ event->event_num);
+ break;
+ }
}
}
spin_unlock(&sdei_list_lock);
static int sdei_cpuhp_down(unsigned int cpu)
{
struct sdei_event *event;
- struct sdei_crosscall_args arg;
+ int err;
/* un-register private events */
spin_lock(&sdei_list_lock);
if (event->type == SDEI_EVENT_TYPE_SHARED)
continue;
- CROSSCALL_INIT(arg, event);
- /* call the cross-call function locally... */
- _local_event_unregister(&arg);
- if (arg.first_error)
+ err = sdei_do_local_call(_local_event_unregister, event);
+ if (err) {
pr_err("Failed to unregister event %u: %d\n",
- event->event_num, arg.first_error);
+ event->event_num, err);
+ }
}
spin_unlock(&sdei_list_lock);
static int sdei_cpuhp_up(unsigned int cpu)
{
struct sdei_event *event;
- struct sdei_crosscall_args arg;
+ int err;
/* re-register/enable private events */
spin_lock(&sdei_list_lock);
continue;
if (event->reregister) {
- CROSSCALL_INIT(arg, event);
- /* call the cross-call function locally... */
- _local_event_register(&arg);
- if (arg.first_error)
+ err = sdei_do_local_call(_local_event_register, event);
+ if (err) {
pr_err("Failed to re-register event %u: %d\n",
- event->event_num, arg.first_error);
+ event->event_num, err);
+ }
}
if (event->reenable) {
- CROSSCALL_INIT(arg, event);
- _local_event_enable(&arg);
- if (arg.first_error)
+ err = sdei_do_local_call(_local_event_enable, event);
+ if (err) {
pr_err("Failed to re-enable event %u: %d\n",
- event->event_num, arg.first_error);
+ event->event_num, err);
+ }
}
}
spin_unlock(&sdei_list_lock);
}
pr_warn("invalid \"method\" property: %s\n", method);
- } else if (IS_ENABLED(CONFIG_ACPI) && !acpi_disabled) {
+ } else if (!acpi_disabled) {
if (acpi_psci_use_hvc()) {
sdei_firmware_call = &sdei_smccc_hvc;
return SMCCC_CONDUIT_HVC;
return 0;
err = sdei_api_get_version(&ver);
- if (err == -EOPNOTSUPP)
- pr_err("advertised but not implemented in platform firmware\n");
if (err) {
pr_err("Failed to get SDEI version: %d\n", err);
sdei_mark_interface_broken();
static int __init sdei_init(void)
{
- int ret = platform_driver_register(&sdei_driver);
-
- if (!ret && sdei_present_acpi()) {
- struct platform_device *pdev;
-
- pdev = platform_device_register_simple(sdei_driver.driver.name,
- 0, NULL, 0);
- if (IS_ERR(pdev))
- pr_info("Failed to register ACPI:SDEI platform device %ld\n",
- PTR_ERR(pdev));
+ struct platform_device *pdev;
+ int ret;
+
+ ret = platform_driver_register(&sdei_driver);
+ if (ret || !sdei_present_acpi())
+ return ret;
+
+ pdev = platform_device_register_simple(sdei_driver.driver.name,
+ 0, NULL, 0);
+ if (IS_ERR(pdev)) {
+ ret = PTR_ERR(pdev);
+ platform_driver_unregister(&sdei_driver);
+ pr_info("Failed to register ACPI:SDEI platform device %d\n",
+ ret);
}
return ret;
config EFI_VARS
tristate "EFI Variable Support via sysfs"
- depends on EFI
+ depends on EFI && (X86 || IA64)
default n
help
If you say Y here, you are able to get EFI (Extensible Firmware
Interface) variable information via sysfs. You may read,
write, create, and destroy EFI variables through this interface.
-
- Note that using this driver in concert with efibootmgr requires
- at least test release version 0.5.0-test3 or later, which is
- available from:
- <http://linux.dell.com/efibootmgr/testing/efibootmgr-0.5.0-test3.tar.gz>
-
- Subsequent efibootmgr releases may be found at:
- <http://github.com/vathpela/efibootmgr>
+ Note that this driver is only retained for compatibility with
+ legacy users: new users should use the efivarfs filesystem
+ instead.
config EFI_ESRT
bool
config EFI_VARS_PSTORE
tristate "Register efivars backend for pstore"
- depends on EFI_VARS && PSTORE
+ depends on PSTORE
default y
help
Say Y here to enable use efivars as a backend to pstore. This
config EFI_BOOTLOADER_CONTROL
tristate "EFI Bootloader Control"
- depends on EFI_VARS
default n
help
This module installs a reboot hook, such that if reboot() is
config EFI_CUSTOM_SSDT_OVERLAYS
bool "Load custom ACPI SSDT overlay from an EFI variable"
- depends on EFI_VARS && ACPI
+ depends on EFI && ACPI
default ACPI_TABLE_UPGRADE
help
Allow loading of an ACPI SSDT overlay from an EFI variable specified
obj-$(CONFIG_APPLE_PROPERTIES) += apple-properties.o
obj-$(CONFIG_EFI_RCI2_TABLE) += rci2-table.o
obj-$(CONFIG_EFI_EMBEDDED_FIRMWARE) += embedded-firmware.o
+obj-$(CONFIG_LOAD_UEFI_KEYS) += mokvar-table.o
fake_map-y += fake_mem.o
fake_map-$(CONFIG_X86) += x86_fake_mem.o
-arm-obj-$(CONFIG_EFI) := arm-init.o arm-runtime.o
+arm-obj-$(CONFIG_EFI) := efi-init.o arm-runtime.o
obj-$(CONFIG_ARM) += $(arm-obj-y)
obj-$(CONFIG_ARM64) += $(arm-obj-y)
obj-$(CONFIG_EFI_CAPSULE_LOADER) += capsule-loader.o
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Extensible Firmware Interface
- *
- * Based on Extensible Firmware Interface Specification version 2.4
- *
- * Copyright (C) 2013 - 2015 Linaro Ltd.
- */
-
-#define pr_fmt(fmt) "efi: " fmt
-
-#include <linux/efi.h>
-#include <linux/fwnode.h>
-#include <linux/init.h>
-#include <linux/memblock.h>
-#include <linux/mm_types.h>
-#include <linux/of.h>
-#include <linux/of_address.h>
-#include <linux/of_fdt.h>
-#include <linux/platform_device.h>
-#include <linux/screen_info.h>
-
-#include <asm/efi.h>
-
-static int __init is_memory(efi_memory_desc_t *md)
-{
- if (md->attribute & (EFI_MEMORY_WB|EFI_MEMORY_WT|EFI_MEMORY_WC))
- return 1;
- return 0;
-}
-
-/*
- * Translate a EFI virtual address into a physical address: this is necessary,
- * as some data members of the EFI system table are virtually remapped after
- * SetVirtualAddressMap() has been called.
- */
-static phys_addr_t __init efi_to_phys(unsigned long addr)
-{
- efi_memory_desc_t *md;
-
- for_each_efi_memory_desc(md) {
- if (!(md->attribute & EFI_MEMORY_RUNTIME))
- continue;
- if (md->virt_addr == 0)
- /* no virtual mapping has been installed by the stub */
- break;
- if (md->virt_addr <= addr &&
- (addr - md->virt_addr) < (md->num_pages << EFI_PAGE_SHIFT))
- return md->phys_addr + addr - md->virt_addr;
- }
- return addr;
-}
-
-static __initdata unsigned long screen_info_table = EFI_INVALID_TABLE_ADDR;
-static __initdata unsigned long cpu_state_table = EFI_INVALID_TABLE_ADDR;
-
-static const efi_config_table_type_t arch_tables[] __initconst = {
- {LINUX_EFI_ARM_SCREEN_INFO_TABLE_GUID, &screen_info_table},
- {LINUX_EFI_ARM_CPU_STATE_TABLE_GUID, &cpu_state_table},
- {}
-};
-
-static void __init init_screen_info(void)
-{
- struct screen_info *si;
-
- if (IS_ENABLED(CONFIG_ARM) &&
- screen_info_table != EFI_INVALID_TABLE_ADDR) {
- si = early_memremap_ro(screen_info_table, sizeof(*si));
- if (!si) {
- pr_err("Could not map screen_info config table\n");
- return;
- }
- screen_info = *si;
- early_memunmap(si, sizeof(*si));
-
- /* dummycon on ARM needs non-zero values for columns/lines */
- screen_info.orig_video_cols = 80;
- screen_info.orig_video_lines = 25;
- }
-
- if (screen_info.orig_video_isVGA == VIDEO_TYPE_EFI &&
- memblock_is_map_memory(screen_info.lfb_base))
- memblock_mark_nomap(screen_info.lfb_base, screen_info.lfb_size);
-}
-
-static int __init uefi_init(u64 efi_system_table)
-{
- efi_config_table_t *config_tables;
- efi_system_table_t *systab;
- size_t table_size;
- int retval;
-
- systab = early_memremap_ro(efi_system_table, sizeof(efi_system_table_t));
- if (systab == NULL) {
- pr_warn("Unable to map EFI system table.\n");
- return -ENOMEM;
- }
-
- set_bit(EFI_BOOT, &efi.flags);
- if (IS_ENABLED(CONFIG_64BIT))
- set_bit(EFI_64BIT, &efi.flags);
-
- retval = efi_systab_check_header(&systab->hdr, 2);
- if (retval)
- goto out;
-
- efi.runtime = systab->runtime;
- efi.runtime_version = systab->hdr.revision;
-
- efi_systab_report_header(&systab->hdr, efi_to_phys(systab->fw_vendor));
-
- table_size = sizeof(efi_config_table_t) * systab->nr_tables;
- config_tables = early_memremap_ro(efi_to_phys(systab->tables),
- table_size);
- if (config_tables == NULL) {
- pr_warn("Unable to map EFI config table array.\n");
- retval = -ENOMEM;
- goto out;
- }
- retval = efi_config_parse_tables(config_tables, systab->nr_tables,
- IS_ENABLED(CONFIG_ARM) ? arch_tables
- : NULL);
-
- early_memunmap(config_tables, table_size);
-out:
- early_memunmap(systab, sizeof(efi_system_table_t));
- return retval;
-}
-
-/*
- * Return true for regions that can be used as System RAM.
- */
-static __init int is_usable_memory(efi_memory_desc_t *md)
-{
- switch (md->type) {
- case EFI_LOADER_CODE:
- case EFI_LOADER_DATA:
- case EFI_ACPI_RECLAIM_MEMORY:
- case EFI_BOOT_SERVICES_CODE:
- case EFI_BOOT_SERVICES_DATA:
- case EFI_CONVENTIONAL_MEMORY:
- case EFI_PERSISTENT_MEMORY:
- /*
- * Special purpose memory is 'soft reserved', which means it
- * is set aside initially, but can be hotplugged back in or
- * be assigned to the dax driver after boot.
- */
- if (efi_soft_reserve_enabled() &&
- (md->attribute & EFI_MEMORY_SP))
- return false;
-
- /*
- * According to the spec, these regions are no longer reserved
- * after calling ExitBootServices(). However, we can only use
- * them as System RAM if they can be mapped writeback cacheable.
- */
- return (md->attribute & EFI_MEMORY_WB);
- default:
- break;
- }
- return false;
-}
-
-static __init void reserve_regions(void)
-{
- efi_memory_desc_t *md;
- u64 paddr, npages, size;
-
- if (efi_enabled(EFI_DBG))
- pr_info("Processing EFI memory map:\n");
-
- /*
- * Discard memblocks discovered so far: if there are any at this
- * point, they originate from memory nodes in the DT, and UEFI
- * uses its own memory map instead.
- */
- memblock_dump_all();
- memblock_remove(0, PHYS_ADDR_MAX);
-
- for_each_efi_memory_desc(md) {
- paddr = md->phys_addr;
- npages = md->num_pages;
-
- if (efi_enabled(EFI_DBG)) {
- char buf[64];
-
- pr_info(" 0x%012llx-0x%012llx %s\n",
- paddr, paddr + (npages << EFI_PAGE_SHIFT) - 1,
- efi_md_typeattr_format(buf, sizeof(buf), md));
- }
-
- memrange_efi_to_native(&paddr, &npages);
- size = npages << PAGE_SHIFT;
-
- if (is_memory(md)) {
- early_init_dt_add_memory_arch(paddr, size);
-
- if (!is_usable_memory(md))
- memblock_mark_nomap(paddr, size);
-
- /* keep ACPI reclaim memory intact for kexec etc. */
- if (md->type == EFI_ACPI_RECLAIM_MEMORY)
- memblock_reserve(paddr, size);
- }
- }
-}
-
-void __init efi_init(void)
-{
- struct efi_memory_map_data data;
- u64 efi_system_table;
-
- /* Grab UEFI information placed in FDT by stub */
- efi_system_table = efi_get_fdt_params(&data);
- if (!efi_system_table)
- return;
-
- if (efi_memmap_init_early(&data) < 0) {
- /*
- * If we are booting via UEFI, the UEFI memory map is the only
- * description of memory we have, so there is little point in
- * proceeding if we cannot access it.
- */
- panic("Unable to map EFI memory map.\n");
- }
-
- WARN(efi.memmap.desc_version != 1,
- "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
- efi.memmap.desc_version);
-
- if (uefi_init(efi_system_table) < 0) {
- efi_memmap_unmap();
- return;
- }
-
- reserve_regions();
- efi_esrt_init();
-
- memblock_reserve(data.phys_map & PAGE_MASK,
- PAGE_ALIGN(data.size + (data.phys_map & ~PAGE_MASK)));
-
- init_screen_info();
-
-#ifdef CONFIG_ARM
- /* ARM does not permit early mappings to persist across paging_init() */
- efi_memmap_unmap();
-
- if (cpu_state_table != EFI_INVALID_TABLE_ADDR) {
- struct efi_arm_entry_state *state;
- bool dump_state = true;
-
- state = early_memremap_ro(cpu_state_table,
- sizeof(struct efi_arm_entry_state));
- if (state == NULL) {
- pr_warn("Unable to map CPU entry state table.\n");
- return;
- }
-
- if ((state->sctlr_before_ebs & 1) == 0)
- pr_warn(FW_BUG "EFI stub was entered with MMU and Dcache disabled, please fix your firmware!\n");
- else if ((state->sctlr_after_ebs & 1) == 0)
- pr_warn(FW_BUG "ExitBootServices() returned with MMU and Dcache disabled, please fix your firmware!\n");
- else
- dump_state = false;
-
- if (dump_state || efi_enabled(EFI_DBG)) {
- pr_info("CPSR at EFI stub entry : 0x%08x\n", state->cpsr_before_ebs);
- pr_info("SCTLR at EFI stub entry : 0x%08x\n", state->sctlr_before_ebs);
- pr_info("CPSR after ExitBootServices() : 0x%08x\n", state->cpsr_after_ebs);
- pr_info("SCTLR after ExitBootServices(): 0x%08x\n", state->sctlr_after_ebs);
- }
- early_memunmap(state, sizeof(struct efi_arm_entry_state));
- }
-#endif
-}
-
-static bool efifb_overlaps_pci_range(const struct of_pci_range *range)
-{
- u64 fb_base = screen_info.lfb_base;
-
- if (screen_info.capabilities & VIDEO_CAPABILITY_64BIT_BASE)
- fb_base |= (u64)(unsigned long)screen_info.ext_lfb_base << 32;
-
- return fb_base >= range->cpu_addr &&
- fb_base < (range->cpu_addr + range->size);
-}
-
-static struct device_node *find_pci_overlap_node(void)
-{
- struct device_node *np;
-
- for_each_node_by_type(np, "pci") {
- struct of_pci_range_parser parser;
- struct of_pci_range range;
- int err;
-
- err = of_pci_range_parser_init(&parser, np);
- if (err) {
- pr_warn("of_pci_range_parser_init() failed: %d\n", err);
- continue;
- }
-
- for_each_of_pci_range(&parser, &range)
- if (efifb_overlaps_pci_range(&range))
- return np;
- }
- return NULL;
-}
-
-/*
- * If the efifb framebuffer is backed by a PCI graphics controller, we have
- * to ensure that this relation is expressed using a device link when
- * running in DT mode, or the probe order may be reversed, resulting in a
- * resource reservation conflict on the memory window that the efifb
- * framebuffer steals from the PCIe host bridge.
- */
-static int efifb_add_links(const struct fwnode_handle *fwnode,
- struct device *dev)
-{
- struct device_node *sup_np;
- struct device *sup_dev;
-
- sup_np = find_pci_overlap_node();
-
- /*
- * If there's no PCI graphics controller backing the efifb, we are
- * done here.
- */
- if (!sup_np)
- return 0;
-
- sup_dev = get_dev_from_fwnode(&sup_np->fwnode);
- of_node_put(sup_np);
-
- /*
- * Return -ENODEV if the PCI graphics controller device hasn't been
- * registered yet. This ensures that efifb isn't allowed to probe
- * and this function is retried again when new devices are
- * registered.
- */
- if (!sup_dev)
- return -ENODEV;
-
- /*
- * If this fails, retrying this function at a later point won't
- * change anything. So, don't return an error after this.
- */
- if (!device_link_add(dev, sup_dev, fw_devlink_get_flags()))
- dev_warn(dev, "device_link_add() failed\n");
-
- put_device(sup_dev);
-
- return 0;
-}
-
-static const struct fwnode_operations efifb_fwnode_ops = {
- .add_links = efifb_add_links,
-};
-
-static struct fwnode_handle efifb_fwnode = {
- .ops = &efifb_fwnode_ops,
-};
-
-static int __init register_gop_device(void)
-{
- struct platform_device *pd;
- int err;
-
- if (screen_info.orig_video_isVGA != VIDEO_TYPE_EFI)
- return 0;
-
- pd = platform_device_alloc("efi-framebuffer", 0);
- if (!pd)
- return -ENOMEM;
-
- if (IS_ENABLED(CONFIG_PCI))
- pd->dev.fwnode = &efifb_fwnode;
-
- err = platform_device_add_data(pd, &screen_info, sizeof(screen_info));
- if (err)
- return err;
-
- return platform_device_add(pd);
-}
-subsys_initcall(register_gop_device);
n += scnprintf(msg + n, len - n, "rank: %d ", mem->rank);
if (mem->validation_bits & CPER_MEM_VALID_BANK)
n += scnprintf(msg + n, len - n, "bank: %d ", mem->bank);
+ if (mem->validation_bits & CPER_MEM_VALID_BANK_GROUP)
+ n += scnprintf(msg + n, len - n, "bank_group: %d ",
+ mem->bank >> CPER_MEM_BANK_GROUP_SHIFT);
+ if (mem->validation_bits & CPER_MEM_VALID_BANK_ADDRESS)
+ n += scnprintf(msg + n, len - n, "bank_address: %d ",
+ mem->bank & CPER_MEM_BANK_ADDRESS_MASK);
if (mem->validation_bits & CPER_MEM_VALID_DEVICE)
n += scnprintf(msg + n, len - n, "device: %d ", mem->device);
- if (mem->validation_bits & CPER_MEM_VALID_ROW)
- n += scnprintf(msg + n, len - n, "row: %d ", mem->row);
+ if (mem->validation_bits & (CPER_MEM_VALID_ROW | CPER_MEM_VALID_ROW_EXT)) {
+ u32 row = mem->row;
+
+ row |= cper_get_mem_extension(mem->validation_bits, mem->extended);
+ n += scnprintf(msg + n, len - n, "row: %d ", row);
+ }
if (mem->validation_bits & CPER_MEM_VALID_COLUMN)
n += scnprintf(msg + n, len - n, "column: %d ", mem->column);
if (mem->validation_bits & CPER_MEM_VALID_BIT_POSITION)
if (mem->validation_bits & CPER_MEM_VALID_TARGET_ID)
scnprintf(msg + n, len - n, "target_id: 0x%016llx ",
mem->target_id);
+ if (mem->validation_bits & CPER_MEM_VALID_CHIP_ID)
+ scnprintf(msg + n, len - n, "chip_id: %d ",
+ mem->extended >> CPER_MEM_CHIP_ID_SHIFT);
msg[n] = '\0';
return n;
cmem->requestor_id = mem->requestor_id;
cmem->responder_id = mem->responder_id;
cmem->target_id = mem->target_id;
+ cmem->extended = mem->extended;
cmem->rank = mem->rank;
cmem->mem_array_handle = mem->mem_array_handle;
cmem->mem_dev_handle = mem->mem_dev_handle;
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Extensible Firmware Interface
+ *
+ * Based on Extensible Firmware Interface Specification version 2.4
+ *
+ * Copyright (C) 2013 - 2015 Linaro Ltd.
+ */
+
+#define pr_fmt(fmt) "efi: " fmt
+
+#include <linux/efi.h>
+#include <linux/fwnode.h>
+#include <linux/init.h>
+#include <linux/memblock.h>
+#include <linux/mm_types.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_fdt.h>
+#include <linux/platform_device.h>
+#include <linux/screen_info.h>
+
+#include <asm/efi.h>
+
+static int __init is_memory(efi_memory_desc_t *md)
+{
+ if (md->attribute & (EFI_MEMORY_WB|EFI_MEMORY_WT|EFI_MEMORY_WC))
+ return 1;
+ return 0;
+}
+
+/*
+ * Translate a EFI virtual address into a physical address: this is necessary,
+ * as some data members of the EFI system table are virtually remapped after
+ * SetVirtualAddressMap() has been called.
+ */
+static phys_addr_t __init efi_to_phys(unsigned long addr)
+{
+ efi_memory_desc_t *md;
+
+ for_each_efi_memory_desc(md) {
+ if (!(md->attribute & EFI_MEMORY_RUNTIME))
+ continue;
+ if (md->virt_addr == 0)
+ /* no virtual mapping has been installed by the stub */
+ break;
+ if (md->virt_addr <= addr &&
+ (addr - md->virt_addr) < (md->num_pages << EFI_PAGE_SHIFT))
+ return md->phys_addr + addr - md->virt_addr;
+ }
+ return addr;
+}
+
+static __initdata unsigned long screen_info_table = EFI_INVALID_TABLE_ADDR;
+static __initdata unsigned long cpu_state_table = EFI_INVALID_TABLE_ADDR;
+
+static const efi_config_table_type_t arch_tables[] __initconst = {
+ {LINUX_EFI_ARM_SCREEN_INFO_TABLE_GUID, &screen_info_table},
+ {LINUX_EFI_ARM_CPU_STATE_TABLE_GUID, &cpu_state_table},
+ {}
+};
+
+static void __init init_screen_info(void)
+{
+ struct screen_info *si;
+
+ if (IS_ENABLED(CONFIG_ARM) &&
+ screen_info_table != EFI_INVALID_TABLE_ADDR) {
+ si = early_memremap_ro(screen_info_table, sizeof(*si));
+ if (!si) {
+ pr_err("Could not map screen_info config table\n");
+ return;
+ }
+ screen_info = *si;
+ early_memunmap(si, sizeof(*si));
+
+ /* dummycon on ARM needs non-zero values for columns/lines */
+ screen_info.orig_video_cols = 80;
+ screen_info.orig_video_lines = 25;
+ }
+
+ if (screen_info.orig_video_isVGA == VIDEO_TYPE_EFI &&
+ memblock_is_map_memory(screen_info.lfb_base))
+ memblock_mark_nomap(screen_info.lfb_base, screen_info.lfb_size);
+}
+
+static int __init uefi_init(u64 efi_system_table)
+{
+ efi_config_table_t *config_tables;
+ efi_system_table_t *systab;
+ size_t table_size;
+ int retval;
+
+ systab = early_memremap_ro(efi_system_table, sizeof(efi_system_table_t));
+ if (systab == NULL) {
+ pr_warn("Unable to map EFI system table.\n");
+ return -ENOMEM;
+ }
+
+ set_bit(EFI_BOOT, &efi.flags);
+ if (IS_ENABLED(CONFIG_64BIT))
+ set_bit(EFI_64BIT, &efi.flags);
+
+ retval = efi_systab_check_header(&systab->hdr, 2);
+ if (retval)
+ goto out;
+
+ efi.runtime = systab->runtime;
+ efi.runtime_version = systab->hdr.revision;
+
+ efi_systab_report_header(&systab->hdr, efi_to_phys(systab->fw_vendor));
+
+ table_size = sizeof(efi_config_table_t) * systab->nr_tables;
+ config_tables = early_memremap_ro(efi_to_phys(systab->tables),
+ table_size);
+ if (config_tables == NULL) {
+ pr_warn("Unable to map EFI config table array.\n");
+ retval = -ENOMEM;
+ goto out;
+ }
+ retval = efi_config_parse_tables(config_tables, systab->nr_tables,
+ IS_ENABLED(CONFIG_ARM) ? arch_tables
+ : NULL);
+
+ early_memunmap(config_tables, table_size);
+out:
+ early_memunmap(systab, sizeof(efi_system_table_t));
+ return retval;
+}
+
+/*
+ * Return true for regions that can be used as System RAM.
+ */
+static __init int is_usable_memory(efi_memory_desc_t *md)
+{
+ switch (md->type) {
+ case EFI_LOADER_CODE:
+ case EFI_LOADER_DATA:
+ case EFI_ACPI_RECLAIM_MEMORY:
+ case EFI_BOOT_SERVICES_CODE:
+ case EFI_BOOT_SERVICES_DATA:
+ case EFI_CONVENTIONAL_MEMORY:
+ case EFI_PERSISTENT_MEMORY:
+ /*
+ * Special purpose memory is 'soft reserved', which means it
+ * is set aside initially, but can be hotplugged back in or
+ * be assigned to the dax driver after boot.
+ */
+ if (efi_soft_reserve_enabled() &&
+ (md->attribute & EFI_MEMORY_SP))
+ return false;
+
+ /*
+ * According to the spec, these regions are no longer reserved
+ * after calling ExitBootServices(). However, we can only use
+ * them as System RAM if they can be mapped writeback cacheable.
+ */
+ return (md->attribute & EFI_MEMORY_WB);
+ default:
+ break;
+ }
+ return false;
+}
+
+static __init void reserve_regions(void)
+{
+ efi_memory_desc_t *md;
+ u64 paddr, npages, size;
+
+ if (efi_enabled(EFI_DBG))
+ pr_info("Processing EFI memory map:\n");
+
+ /*
+ * Discard memblocks discovered so far: if there are any at this
+ * point, they originate from memory nodes in the DT, and UEFI
+ * uses its own memory map instead.
+ */
+ memblock_dump_all();
+ memblock_remove(0, PHYS_ADDR_MAX);
+
+ for_each_efi_memory_desc(md) {
+ paddr = md->phys_addr;
+ npages = md->num_pages;
+
+ if (efi_enabled(EFI_DBG)) {
+ char buf[64];
+
+ pr_info(" 0x%012llx-0x%012llx %s\n",
+ paddr, paddr + (npages << EFI_PAGE_SHIFT) - 1,
+ efi_md_typeattr_format(buf, sizeof(buf), md));
+ }
+
+ memrange_efi_to_native(&paddr, &npages);
+ size = npages << PAGE_SHIFT;
+
+ if (is_memory(md)) {
+ early_init_dt_add_memory_arch(paddr, size);
+
+ if (!is_usable_memory(md))
+ memblock_mark_nomap(paddr, size);
+
+ /* keep ACPI reclaim memory intact for kexec etc. */
+ if (md->type == EFI_ACPI_RECLAIM_MEMORY)
+ memblock_reserve(paddr, size);
+ }
+ }
+}
+
+void __init efi_init(void)
+{
+ struct efi_memory_map_data data;
+ u64 efi_system_table;
+
+ /* Grab UEFI information placed in FDT by stub */
+ efi_system_table = efi_get_fdt_params(&data);
+ if (!efi_system_table)
+ return;
+
+ if (efi_memmap_init_early(&data) < 0) {
+ /*
+ * If we are booting via UEFI, the UEFI memory map is the only
+ * description of memory we have, so there is little point in
+ * proceeding if we cannot access it.
+ */
+ panic("Unable to map EFI memory map.\n");
+ }
+
+ WARN(efi.memmap.desc_version != 1,
+ "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
+ efi.memmap.desc_version);
+
+ if (uefi_init(efi_system_table) < 0) {
+ efi_memmap_unmap();
+ return;
+ }
+
+ reserve_regions();
+ efi_esrt_init();
+ efi_mokvar_table_init();
+
+ memblock_reserve(data.phys_map & PAGE_MASK,
+ PAGE_ALIGN(data.size + (data.phys_map & ~PAGE_MASK)));
+
+ init_screen_info();
+
+#ifdef CONFIG_ARM
+ /* ARM does not permit early mappings to persist across paging_init() */
+ efi_memmap_unmap();
+
+ if (cpu_state_table != EFI_INVALID_TABLE_ADDR) {
+ struct efi_arm_entry_state *state;
+ bool dump_state = true;
+
+ state = early_memremap_ro(cpu_state_table,
+ sizeof(struct efi_arm_entry_state));
+ if (state == NULL) {
+ pr_warn("Unable to map CPU entry state table.\n");
+ return;
+ }
+
+ if ((state->sctlr_before_ebs & 1) == 0)
+ pr_warn(FW_BUG "EFI stub was entered with MMU and Dcache disabled, please fix your firmware!\n");
+ else if ((state->sctlr_after_ebs & 1) == 0)
+ pr_warn(FW_BUG "ExitBootServices() returned with MMU and Dcache disabled, please fix your firmware!\n");
+ else
+ dump_state = false;
+
+ if (dump_state || efi_enabled(EFI_DBG)) {
+ pr_info("CPSR at EFI stub entry : 0x%08x\n", state->cpsr_before_ebs);
+ pr_info("SCTLR at EFI stub entry : 0x%08x\n", state->sctlr_before_ebs);
+ pr_info("CPSR after ExitBootServices() : 0x%08x\n", state->cpsr_after_ebs);
+ pr_info("SCTLR after ExitBootServices(): 0x%08x\n", state->sctlr_after_ebs);
+ }
+ early_memunmap(state, sizeof(struct efi_arm_entry_state));
+ }
+#endif
+}
+
+static bool efifb_overlaps_pci_range(const struct of_pci_range *range)
+{
+ u64 fb_base = screen_info.lfb_base;
+
+ if (screen_info.capabilities & VIDEO_CAPABILITY_64BIT_BASE)
+ fb_base |= (u64)(unsigned long)screen_info.ext_lfb_base << 32;
+
+ return fb_base >= range->cpu_addr &&
+ fb_base < (range->cpu_addr + range->size);
+}
+
+static struct device_node *find_pci_overlap_node(void)
+{
+ struct device_node *np;
+
+ for_each_node_by_type(np, "pci") {
+ struct of_pci_range_parser parser;
+ struct of_pci_range range;
+ int err;
+
+ err = of_pci_range_parser_init(&parser, np);
+ if (err) {
+ pr_warn("of_pci_range_parser_init() failed: %d\n", err);
+ continue;
+ }
+
+ for_each_of_pci_range(&parser, &range)
+ if (efifb_overlaps_pci_range(&range))
+ return np;
+ }
+ return NULL;
+}
+
+/*
+ * If the efifb framebuffer is backed by a PCI graphics controller, we have
+ * to ensure that this relation is expressed using a device link when
+ * running in DT mode, or the probe order may be reversed, resulting in a
+ * resource reservation conflict on the memory window that the efifb
+ * framebuffer steals from the PCIe host bridge.
+ */
+static int efifb_add_links(const struct fwnode_handle *fwnode,
+ struct device *dev)
+{
+ struct device_node *sup_np;
+ struct device *sup_dev;
+
+ sup_np = find_pci_overlap_node();
+
+ /*
+ * If there's no PCI graphics controller backing the efifb, we are
+ * done here.
+ */
+ if (!sup_np)
+ return 0;
+
+ sup_dev = get_dev_from_fwnode(&sup_np->fwnode);
+ of_node_put(sup_np);
+
+ /*
+ * Return -ENODEV if the PCI graphics controller device hasn't been
+ * registered yet. This ensures that efifb isn't allowed to probe
+ * and this function is retried again when new devices are
+ * registered.
+ */
+ if (!sup_dev)
+ return -ENODEV;
+
+ /*
+ * If this fails, retrying this function at a later point won't
+ * change anything. So, don't return an error after this.
+ */
+ if (!device_link_add(dev, sup_dev, fw_devlink_get_flags()))
+ dev_warn(dev, "device_link_add() failed\n");
+
+ put_device(sup_dev);
+
+ return 0;
+}
+
+static const struct fwnode_operations efifb_fwnode_ops = {
+ .add_links = efifb_add_links,
+};
+
+static struct fwnode_handle efifb_fwnode = {
+ .ops = &efifb_fwnode_ops,
+};
+
+static int __init register_gop_device(void)
+{
+ struct platform_device *pd;
+ int err;
+
+ if (screen_info.orig_video_isVGA != VIDEO_TYPE_EFI)
+ return 0;
+
+ pd = platform_device_alloc("efi-framebuffer", 0);
+ if (!pd)
+ return -ENOMEM;
+
+ if (IS_ENABLED(CONFIG_PCI))
+ pd->dev.fwnode = &efifb_fwnode;
+
+ err = platform_device_add_data(pd, &screen_info, sizeof(screen_info));
+ if (err)
+ return err;
+
+ return platform_device_add(pd);
+}
+subsys_initcall(register_gop_device);
#define DUMP_NAME_LEN 66
+#define EFIVARS_DATA_SIZE_MAX 1024
+
static bool efivars_pstore_disable =
IS_ENABLED(CONFIG_EFI_VARS_PSTORE_DEFAULT_DISABLE);
EFI_VARIABLE_BOOTSERVICE_ACCESS | \
EFI_VARIABLE_RUNTIME_ACCESS)
+static LIST_HEAD(efi_pstore_list);
+static DECLARE_WORK(efivar_work, NULL);
+
static int efi_pstore_open(struct pstore_info *psi)
{
psi->data = NULL;
if (entry->deleting) {
list_del(&entry->list);
efivar_entry_iter_end();
- efivar_unregister(entry);
+ kfree(entry);
if (efivar_entry_iter_begin())
return -EINTR;
} else if (turn_off_scanning)
{
struct efivar_entry **pos = (struct efivar_entry **)&record->psi->data;
struct efivar_entry *entry, *n;
- struct list_head *head = &efivar_sysfs_list;
+ struct list_head *head = &efi_pstore_list;
int size = 0;
int ret;
ret = efivar_entry_set_safe(efi_name, vendor, PSTORE_EFI_ATTRIBUTES,
preemptible(), record->size, record->psi->buf);
- if (record->reason == KMSG_DUMP_OOPS)
- efivar_run_worker();
+ if (record->reason == KMSG_DUMP_OOPS && try_module_get(THIS_MODULE))
+ if (!schedule_work(&efivar_work))
+ module_put(THIS_MODULE);
return ret;
};
if (efivar_entry_iter_begin())
return -EINTR;
- found = __efivar_entry_iter(efi_pstore_erase_func, &efivar_sysfs_list,
+ found = __efivar_entry_iter(efi_pstore_erase_func, &efi_pstore_list,
efi_name, &entry);
efivar_entry_iter_end();
if (found && !entry->scanning)
- efivar_unregister(entry);
+ kfree(entry);
return found ? 0 : -ENOENT;
}
.erase = efi_pstore_erase,
};
+static int efi_pstore_callback(efi_char16_t *name, efi_guid_t vendor,
+ unsigned long name_size, void *data)
+{
+ struct efivar_entry *entry;
+ int ret;
+
+ entry = kzalloc(sizeof(*entry), GFP_KERNEL);
+ if (!entry)
+ return -ENOMEM;
+
+ memcpy(entry->var.VariableName, name, name_size);
+ entry->var.VendorGuid = vendor;
+
+ ret = efivar_entry_add(entry, &efi_pstore_list);
+ if (ret)
+ kfree(entry);
+
+ return ret;
+}
+
+static int efi_pstore_update_entry(efi_char16_t *name, efi_guid_t vendor,
+ unsigned long name_size, void *data)
+{
+ struct efivar_entry *entry = data;
+
+ if (efivar_entry_find(name, vendor, &efi_pstore_list, false))
+ return 0;
+
+ memcpy(entry->var.VariableName, name, name_size);
+ memcpy(&(entry->var.VendorGuid), &vendor, sizeof(efi_guid_t));
+
+ return 1;
+}
+
+static void efi_pstore_update_entries(struct work_struct *work)
+{
+ struct efivar_entry *entry;
+ int err;
+
+ /* Add new sysfs entries */
+ while (1) {
+ entry = kzalloc(sizeof(*entry), GFP_KERNEL);
+ if (!entry)
+ return;
+
+ err = efivar_init(efi_pstore_update_entry, entry,
+ false, &efi_pstore_list);
+ if (!err)
+ break;
+
+ efivar_entry_add(entry, &efi_pstore_list);
+ }
+
+ kfree(entry);
+ module_put(THIS_MODULE);
+}
+
static __init int efivars_pstore_init(void)
{
+ int ret;
+
if (!efivars_kobject() || !efivar_supports_writes())
return 0;
if (efivars_pstore_disable)
return 0;
+ ret = efivar_init(efi_pstore_callback, NULL, true, &efi_pstore_list);
+ if (ret)
+ return ret;
+
efi_pstore_info.buf = kmalloc(4096, GFP_KERNEL);
if (!efi_pstore_info.buf)
return -ENOMEM;
efi_pstore_info.bufsize = 0;
}
+ INIT_WORK(&efivar_work, efi_pstore_update_entries);
+
return 0;
}
.esrt = EFI_INVALID_TABLE_ADDR,
.tpm_log = EFI_INVALID_TABLE_ADDR,
.tpm_final_log = EFI_INVALID_TABLE_ADDR,
+#ifdef CONFIG_LOAD_UEFI_KEYS
+ .mokvar_table = EFI_INVALID_TABLE_ADDR,
+#endif
};
EXPORT_SYMBOL(efi);
{EFI_RT_PROPERTIES_TABLE_GUID, &rt_prop, "RTPROP" },
#ifdef CONFIG_EFI_RCI2_TABLE
{DELLEMC_EFI_RCI2_TABLE_GUID, &rci2_table_phys },
+#endif
+#ifdef CONFIG_LOAD_UEFI_KEYS
+ {LINUX_EFI_MOK_VARIABLE_TABLE_GUID, &efi.mokvar_table, "MOKvar" },
#endif
{},
};
vendor);
}
-static __initdata char memory_type_name[][20] = {
+static __initdata char memory_type_name[][13] = {
"Reserved",
"Loader Code",
"Loader Data",
"Boot Data",
"Runtime Code",
"Runtime Data",
- "Conventional Memory",
- "Unusable Memory",
- "ACPI Reclaim Memory",
- "ACPI Memory NVS",
- "Memory Mapped I/O",
- "MMIO Port Space",
+ "Conventional",
+ "Unusable",
+ "ACPI Reclaim",
+ "ACPI Mem NVS",
+ "MMIO",
+ "MMIO Port",
"PAL Code",
- "Persistent Memory",
+ "Persistent",
};
char * __init efi_md_typeattr_format(char *buf, size_t size,
if (attr & ~(EFI_MEMORY_UC | EFI_MEMORY_WC | EFI_MEMORY_WT |
EFI_MEMORY_WB | EFI_MEMORY_UCE | EFI_MEMORY_RO |
EFI_MEMORY_WP | EFI_MEMORY_RP | EFI_MEMORY_XP |
- EFI_MEMORY_NV | EFI_MEMORY_SP |
+ EFI_MEMORY_NV | EFI_MEMORY_SP | EFI_MEMORY_CPU_CRYPTO |
EFI_MEMORY_RUNTIME | EFI_MEMORY_MORE_RELIABLE))
snprintf(pos, size, "|attr=0x%016llx]",
(unsigned long long)attr);
else
snprintf(pos, size,
- "|%3s|%2s|%2s|%2s|%2s|%2s|%2s|%2s|%3s|%2s|%2s|%2s|%2s]",
- attr & EFI_MEMORY_RUNTIME ? "RUN" : "",
- attr & EFI_MEMORY_MORE_RELIABLE ? "MR" : "",
- attr & EFI_MEMORY_SP ? "SP" : "",
- attr & EFI_MEMORY_NV ? "NV" : "",
- attr & EFI_MEMORY_XP ? "XP" : "",
- attr & EFI_MEMORY_RP ? "RP" : "",
- attr & EFI_MEMORY_WP ? "WP" : "",
- attr & EFI_MEMORY_RO ? "RO" : "",
- attr & EFI_MEMORY_UCE ? "UCE" : "",
- attr & EFI_MEMORY_WB ? "WB" : "",
- attr & EFI_MEMORY_WT ? "WT" : "",
- attr & EFI_MEMORY_WC ? "WC" : "",
- attr & EFI_MEMORY_UC ? "UC" : "");
+ "|%3s|%2s|%2s|%2s|%2s|%2s|%2s|%2s|%2s|%3s|%2s|%2s|%2s|%2s]",
+ attr & EFI_MEMORY_RUNTIME ? "RUN" : "",
+ attr & EFI_MEMORY_MORE_RELIABLE ? "MR" : "",
+ attr & EFI_MEMORY_CPU_CRYPTO ? "CC" : "",
+ attr & EFI_MEMORY_SP ? "SP" : "",
+ attr & EFI_MEMORY_NV ? "NV" : "",
+ attr & EFI_MEMORY_XP ? "XP" : "",
+ attr & EFI_MEMORY_RP ? "RP" : "",
+ attr & EFI_MEMORY_WP ? "WP" : "",
+ attr & EFI_MEMORY_RO ? "RO" : "",
+ attr & EFI_MEMORY_UCE ? "UCE" : "",
+ attr & EFI_MEMORY_WB ? "WB" : "",
+ attr & EFI_MEMORY_WT ? "WT" : "",
+ attr & EFI_MEMORY_WC ? "WC" : "",
+ attr & EFI_MEMORY_UC ? "UC" : "");
return buf;
}
MODULE_DESCRIPTION("sysfs interface to EFI Variables");
MODULE_LICENSE("GPL");
MODULE_VERSION(EFIVARS_VERSION);
-MODULE_ALIAS("platform:efivars");
-LIST_HEAD(efivar_sysfs_list);
-EXPORT_SYMBOL_GPL(efivar_sysfs_list);
+static LIST_HEAD(efivar_sysfs_list);
static struct kset *efivars_kset;
return error;
}
-static int efivar_update_sysfs_entry(efi_char16_t *name, efi_guid_t vendor,
- unsigned long name_size, void *data)
-{
- struct efivar_entry *entry = data;
-
- if (efivar_entry_find(name, vendor, &efivar_sysfs_list, false))
- return 0;
-
- memcpy(entry->var.VariableName, name, name_size);
- memcpy(&(entry->var.VendorGuid), &vendor, sizeof(efi_guid_t));
-
- return 1;
-}
-
-static void efivar_update_sysfs_entries(struct work_struct *work)
-{
- struct efivar_entry *entry;
- int err;
-
- /* Add new sysfs entries */
- while (1) {
- entry = kzalloc(sizeof(*entry), GFP_KERNEL);
- if (!entry)
- return;
-
- err = efivar_init(efivar_update_sysfs_entry, entry,
- false, &efivar_sysfs_list);
- if (!err)
- break;
-
- efivar_create_sysfs_entry(entry);
- }
-
- kfree(entry);
-}
-
static int efivars_sysfs_callback(efi_char16_t *name, efi_guid_t vendor,
unsigned long name_size, void *data)
{
kset_unregister(efivars_kset);
}
-int efivars_sysfs_init(void)
+static int efivars_sysfs_init(void)
{
struct kobject *parent_kobj = efivars_kobject();
int error = 0;
return error;
}
- INIT_WORK(&efivar_work, efivar_update_sysfs_entries);
-
return 0;
}
-EXPORT_SYMBOL_GPL(efivars_sysfs_init);
module_init(efivars_sysfs_init);
module_exit(efivars_sysfs_exit);
# arm64 uses the full KBUILD_CFLAGS so it's necessary to explicitly
# disable the stackleak plugin
cflags-$(CONFIG_ARM64) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \
- -fpie $(DISABLE_STACKLEAK_PLUGIN)
+ -fpie $(DISABLE_STACKLEAK_PLUGIN) \
+ $(call cc-option,-mbranch-protection=none)
cflags-$(CONFIG_ARM) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \
-fno-builtin -fpic \
$(call cc-option,-mno-single-pic-base)
cflags-$(CONFIG_EFI_GENERIC_STUB) += -I$(srctree)/scripts/dtc/libfdt
KBUILD_CFLAGS := $(cflags-y) -Os -DDISABLE_BRANCH_PROFILING \
- -include $(srctree)/drivers/firmware/efi/libstub/hidden.h \
+ -include $(srctree)/include/linux/hidden.h \
-D__NO_FORTIFY \
-ffreestanding \
-fno-stack-protector \
lib-$(CONFIG_ARM64) += arm64-stub.o
lib-$(CONFIG_X86) += x86-stub.o
CFLAGS_arm32-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET)
-CFLAGS_arm64-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET)
+
+# Even when -mbranch-protection=none is set, Clang will generate a
+# .note.gnu.property for code-less object files (like lib/ctype.c),
+# so work around this by explicitly removing the unwanted section.
+# https://bugs.llvm.org/show_bug.cgi?id=46480
+STUBCOPY_FLAGS-y += --remove-section=.note.gnu.property
#
# For x86, bootloaders like systemd-boot or grub-efi do not zero-initialize the
efi_bs_call(free_pool, si);
}
-static efi_status_t reserve_kernel_base(unsigned long dram_base,
- unsigned long *reserve_addr,
- unsigned long *reserve_size)
-{
- efi_physical_addr_t alloc_addr;
- efi_memory_desc_t *memory_map;
- unsigned long nr_pages, map_size, desc_size, buff_size;
- efi_status_t status;
- unsigned long l;
-
- struct efi_boot_memmap map = {
- .map = &memory_map,
- .map_size = &map_size,
- .desc_size = &desc_size,
- .desc_ver = NULL,
- .key_ptr = NULL,
- .buff_size = &buff_size,
- };
-
- /*
- * Reserve memory for the uncompressed kernel image. This is
- * all that prevents any future allocations from conflicting
- * with the kernel. Since we can't tell from the compressed
- * image how much DRAM the kernel actually uses (due to BSS
- * size uncertainty) we allocate the maximum possible size.
- * Do this very early, as prints can cause memory allocations
- * that may conflict with this.
- */
- alloc_addr = dram_base + MAX_UNCOMP_KERNEL_SIZE;
- nr_pages = MAX_UNCOMP_KERNEL_SIZE / EFI_PAGE_SIZE;
- status = efi_bs_call(allocate_pages, EFI_ALLOCATE_MAX_ADDRESS,
- EFI_BOOT_SERVICES_DATA, nr_pages, &alloc_addr);
- if (status == EFI_SUCCESS) {
- if (alloc_addr == dram_base) {
- *reserve_addr = alloc_addr;
- *reserve_size = MAX_UNCOMP_KERNEL_SIZE;
- return EFI_SUCCESS;
- }
- /*
- * If we end up here, the allocation succeeded but starts below
- * dram_base. This can only occur if the real base of DRAM is
- * not a multiple of 128 MB, in which case dram_base will have
- * been rounded up. Since this implies that a part of the region
- * was already occupied, we need to fall through to the code
- * below to ensure that the existing allocations don't conflict.
- * For this reason, we use EFI_BOOT_SERVICES_DATA above and not
- * EFI_LOADER_DATA, which we wouldn't able to distinguish from
- * allocations that we want to disallow.
- */
- }
-
- /*
- * If the allocation above failed, we may still be able to proceed:
- * if the only allocations in the region are of types that will be
- * released to the OS after ExitBootServices(), the decompressor can
- * safely overwrite them.
- */
- status = efi_get_memory_map(&map);
- if (status != EFI_SUCCESS) {
- efi_err("reserve_kernel_base(): Unable to retrieve memory map.\n");
- return status;
- }
-
- for (l = 0; l < map_size; l += desc_size) {
- efi_memory_desc_t *desc;
- u64 start, end;
-
- desc = (void *)memory_map + l;
- start = desc->phys_addr;
- end = start + desc->num_pages * EFI_PAGE_SIZE;
-
- /* Skip if entry does not intersect with region */
- if (start >= dram_base + MAX_UNCOMP_KERNEL_SIZE ||
- end <= dram_base)
- continue;
-
- switch (desc->type) {
- case EFI_BOOT_SERVICES_CODE:
- case EFI_BOOT_SERVICES_DATA:
- /* Ignore types that are released to the OS anyway */
- continue;
-
- case EFI_CONVENTIONAL_MEMORY:
- /* Skip soft reserved conventional memory */
- if (efi_soft_reserve_enabled() &&
- (desc->attribute & EFI_MEMORY_SP))
- continue;
-
- /*
- * Reserve the intersection between this entry and the
- * region.
- */
- start = max(start, (u64)dram_base);
- end = min(end, (u64)dram_base + MAX_UNCOMP_KERNEL_SIZE);
-
- status = efi_bs_call(allocate_pages,
- EFI_ALLOCATE_ADDRESS,
- EFI_LOADER_DATA,
- (end - start) / EFI_PAGE_SIZE,
- &start);
- if (status != EFI_SUCCESS) {
- efi_err("reserve_kernel_base(): alloc failed.\n");
- goto out;
- }
- break;
-
- case EFI_LOADER_CODE:
- case EFI_LOADER_DATA:
- /*
- * These regions may be released and reallocated for
- * another purpose (including EFI_RUNTIME_SERVICE_DATA)
- * at any time during the execution of the OS loader,
- * so we cannot consider them as safe.
- */
- default:
- /*
- * Treat any other allocation in the region as unsafe */
- status = EFI_OUT_OF_RESOURCES;
- goto out;
- }
- }
-
- status = EFI_SUCCESS;
-out:
- efi_bs_call(free_pool, memory_map);
- return status;
-}
-
efi_status_t handle_kernel_image(unsigned long *image_addr,
unsigned long *image_size,
unsigned long *reserve_addr,
unsigned long *reserve_size,
- unsigned long dram_base,
efi_loaded_image_t *image)
{
- unsigned long kernel_base;
+ const int slack = TEXT_OFFSET - 5 * PAGE_SIZE;
+ int alloc_size = MAX_UNCOMP_KERNEL_SIZE + EFI_PHYS_ALIGN;
+ unsigned long alloc_base, kernel_base;
efi_status_t status;
- /* use a 16 MiB aligned base for the decompressed kernel */
- kernel_base = round_up(dram_base, SZ_16M) + TEXT_OFFSET;
-
/*
- * Note that some platforms (notably, the Raspberry Pi 2) put
- * spin-tables and other pieces of firmware at the base of RAM,
- * abusing the fact that the window of TEXT_OFFSET bytes at the
- * base of the kernel image is only partially used at the moment.
- * (Up to 5 pages are used for the swapper page tables)
+ * Allocate space for the decompressed kernel as low as possible.
+ * The region should be 16 MiB aligned, but the first 'slack' bytes
+ * are not used by Linux, so we allow those to be occupied by the
+ * firmware.
*/
- status = reserve_kernel_base(kernel_base - 5 * PAGE_SIZE, reserve_addr,
- reserve_size);
+ status = efi_low_alloc_above(alloc_size, EFI_PAGE_SIZE, &alloc_base, 0x0);
if (status != EFI_SUCCESS) {
efi_err("Unable to allocate memory for uncompressed kernel.\n");
return status;
}
- *image_addr = kernel_base;
+ if ((alloc_base % EFI_PHYS_ALIGN) > slack) {
+ /*
+ * More than 'slack' bytes are already occupied at the base of
+ * the allocation, so we need to advance to the next 16 MiB block.
+ */
+ kernel_base = round_up(alloc_base, EFI_PHYS_ALIGN);
+ efi_info("Free memory starts at 0x%lx, setting kernel_base to 0x%lx\n",
+ alloc_base, kernel_base);
+ } else {
+ kernel_base = round_down(alloc_base, EFI_PHYS_ALIGN);
+ }
+
+ *reserve_addr = kernel_base + slack;
+ *reserve_size = MAX_UNCOMP_KERNEL_SIZE;
+
+ /* now free the parts that we will not use */
+ if (*reserve_addr > alloc_base) {
+ efi_bs_call(free_pages, alloc_base,
+ (*reserve_addr - alloc_base) / EFI_PAGE_SIZE);
+ alloc_size -= *reserve_addr - alloc_base;
+ }
+ efi_bs_call(free_pages, *reserve_addr + MAX_UNCOMP_KERNEL_SIZE,
+ (alloc_size - MAX_UNCOMP_KERNEL_SIZE) / EFI_PAGE_SIZE);
+
+ *image_addr = kernel_base + TEXT_OFFSET;
*image_size = 0;
+
+ efi_debug("image addr == 0x%lx, reserve_addr == 0x%lx\n",
+ *image_addr, *reserve_addr);
+
return EFI_SUCCESS;
}
unsigned long *image_size,
unsigned long *reserve_addr,
unsigned long *reserve_size,
- unsigned long dram_base,
efi_loaded_image_t *image)
{
efi_status_t status;
status = efi_get_random_bytes(sizeof(phys_seed),
(u8 *)&phys_seed);
if (status == EFI_NOT_FOUND) {
- efi_info("EFI_RNG_PROTOCOL unavailable, no randomness supplied\n");
+ efi_info("EFI_RNG_PROTOCOL unavailable, KASLR will be disabled\n");
+ efi_nokaslr = true;
} else if (status != EFI_SUCCESS) {
- efi_err("efi_get_random_bytes() failed\n");
- return status;
+ efi_err("efi_get_random_bytes() failed (0x%lx), KASLR will be disabled\n",
+ status);
+ efi_nokaslr = true;
}
} else {
efi_info("KASLR disabled on kernel command line\n");
kernel_size = _edata - _text;
kernel_memsize = kernel_size + (_end - _edata);
- *reserve_size = kernel_memsize + TEXT_OFFSET % min_kimg_align();
+ *reserve_size = kernel_memsize;
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && phys_seed != 0) {
/*
}
if (status != EFI_SUCCESS) {
- if (IS_ALIGNED((u64)_text - TEXT_OFFSET, min_kimg_align())) {
+ if (IS_ALIGNED((u64)_text, min_kimg_align())) {
/*
* Just execute from wherever we were loaded by the
* UEFI PE/COFF loader if the alignment is suitable.
}
}
- *image_addr = *reserve_addr + TEXT_OFFSET % min_kimg_align();
+ *image_addr = *reserve_addr;
memcpy((void *)*image_addr, _text, kernel_size);
return EFI_SUCCESS;
return EFI_SUCCESS;
}
+/*
+ * The EFI_LOAD_OPTION descriptor has the following layout:
+ * u32 Attributes;
+ * u16 FilePathListLength;
+ * u16 Description[];
+ * efi_device_path_protocol_t FilePathList[];
+ * u8 OptionalData[];
+ *
+ * This function validates and unpacks the variable-size data fields.
+ */
+static
+bool efi_load_option_unpack(efi_load_option_unpacked_t *dest,
+ const efi_load_option_t *src, size_t size)
+{
+ const void *pos;
+ u16 c;
+ efi_device_path_protocol_t header;
+ const efi_char16_t *description;
+ const efi_device_path_protocol_t *file_path_list;
+
+ if (size < offsetof(efi_load_option_t, variable_data))
+ return false;
+ pos = src->variable_data;
+ size -= offsetof(efi_load_option_t, variable_data);
+
+ if ((src->attributes & ~EFI_LOAD_OPTION_MASK) != 0)
+ return false;
+
+ /* Scan description. */
+ description = pos;
+ do {
+ if (size < sizeof(c))
+ return false;
+ c = *(const u16 *)pos;
+ pos += sizeof(c);
+ size -= sizeof(c);
+ } while (c != L'\0');
+
+ /* Scan file_path_list. */
+ file_path_list = pos;
+ do {
+ if (size < sizeof(header))
+ return false;
+ header = *(const efi_device_path_protocol_t *)pos;
+ if (header.length < sizeof(header))
+ return false;
+ if (size < header.length)
+ return false;
+ pos += header.length;
+ size -= header.length;
+ } while ((header.type != EFI_DEV_END_PATH && header.type != EFI_DEV_END_PATH2) ||
+ (header.sub_type != EFI_DEV_END_ENTIRE));
+ if (pos != (const void *)file_path_list + src->file_path_list_length)
+ return false;
+
+ dest->attributes = src->attributes;
+ dest->file_path_list_length = src->file_path_list_length;
+ dest->description = description;
+ dest->file_path_list = file_path_list;
+ dest->optional_data_size = size;
+ dest->optional_data = size ? pos : NULL;
+
+ return true;
+}
+
+/*
+ * At least some versions of Dell firmware pass the entire contents of the
+ * Boot#### variable, i.e. the EFI_LOAD_OPTION descriptor, rather than just the
+ * OptionalData field.
+ *
+ * Detect this case and extract OptionalData.
+ */
+void efi_apply_loadoptions_quirk(const void **load_options, int *load_options_size)
+{
+ const efi_load_option_t *load_option = *load_options;
+ efi_load_option_unpacked_t load_option_unpacked;
+
+ if (!IS_ENABLED(CONFIG_X86))
+ return;
+ if (!load_option)
+ return;
+ if (*load_options_size < sizeof(*load_option))
+ return;
+ if ((load_option->attributes & ~EFI_LOAD_OPTION_BOOT_MASK) != 0)
+ return;
+
+ if (!efi_load_option_unpack(&load_option_unpacked, load_option, *load_options_size))
+ return;
+
+ efi_warn_once(FW_BUG "LoadOptions is an EFI_LOAD_OPTION descriptor\n");
+ efi_warn_once(FW_BUG "Using OptionalData as a workaround\n");
+
+ *load_options = load_option_unpacked.optional_data;
+ *load_options_size = load_option_unpacked.optional_data_size;
+}
+
/*
* Convert the unicode UEFI command line to ASCII to pass to kernel.
* Size of memory allocated return in *cmd_line_len.
{
const u16 *s2;
unsigned long cmdline_addr = 0;
- int options_chars = efi_table_attr(image, load_options_size) / 2;
+ int options_chars = efi_table_attr(image, load_options_size);
const u16 *options = efi_table_attr(image, load_options);
int options_bytes = 0, safe_options_bytes = 0; /* UTF-8 bytes */
bool in_quote = false;
efi_status_t status;
+ efi_apply_loadoptions_quirk((const void **)&options, &options_chars);
+ options_chars /= sizeof(*options);
+
if (options) {
s2 = options;
while (options_bytes < COMMAND_LINE_SIZE && options_chars--) {
efi_err("Failed to install memreserve config table!\n");
}
-static unsigned long get_dram_base(void)
-{
- efi_status_t status;
- unsigned long map_size, buff_size;
- unsigned long membase = EFI_ERROR;
- struct efi_memory_map map;
- efi_memory_desc_t *md;
- struct efi_boot_memmap boot_map;
-
- boot_map.map = (efi_memory_desc_t **)&map.map;
- boot_map.map_size = &map_size;
- boot_map.desc_size = &map.desc_size;
- boot_map.desc_ver = NULL;
- boot_map.key_ptr = NULL;
- boot_map.buff_size = &buff_size;
-
- status = efi_get_memory_map(&boot_map);
- if (status != EFI_SUCCESS)
- return membase;
-
- map.map_end = map.map + map_size;
-
- for_each_efi_memory_desc_in_map(&map, md) {
- if (md->attribute & EFI_MEMORY_WB) {
- if (membase > md->phys_addr)
- membase = md->phys_addr;
- }
- }
-
- efi_bs_call(free_pool, map.map);
-
- return membase;
-}
-
/*
* EFI entry point for the arm/arm64 EFI stubs. This is the entrypoint
* that is described in the PE/COFF header. Most of the code is the same
efi_status_t status;
unsigned long image_addr;
unsigned long image_size = 0;
- unsigned long dram_base;
/* addr/point and size pairs for memory management*/
unsigned long initrd_addr = 0;
unsigned long initrd_size = 0;
goto fail;
}
- dram_base = get_dram_base();
- if (dram_base == EFI_ERROR) {
- efi_err("Failed to find DRAM base\n");
- status = EFI_LOAD_ERROR;
- goto fail;
- }
-
/*
* Get the command line from EFI, using the LOADED_IMAGE
* protocol. We are going to copy the command line into the
status = handle_kernel_image(&image_addr, &image_size,
&reserve_addr,
&reserve_size,
- dram_base, image);
+ image);
if (status != EFI_SUCCESS) {
efi_err("Failed to relocate kernel\n");
goto fail_free_screeninfo;
efi_info("Generating empty DTB\n");
if (!efi_noinitrd) {
- max_addr = efi_get_max_initrd_addr(dram_base, image_addr);
+ max_addr = efi_get_max_initrd_addr(image_addr);
status = efi_load_initrd(image, &initrd_addr, &initrd_size,
ULONG_MAX, max_addr);
if (status != EFI_SUCCESS)
install_memreserve_table();
status = allocate_new_fdt_and_exit_boot(handle, &fdt_addr,
- efi_get_max_fdt_addr(dram_base),
+ efi_get_max_fdt_addr(image_addr),
initrd_addr, initrd_size,
cmdline_ptr, fdt_addr, fdt_size);
if (status != EFI_SUCCESS)
#include <linux/types.h>
#include <asm/efi.h>
-/* error code which can't be mistaken for valid address */
-#define EFI_ERROR (~0UL)
-
/*
* __init annotations should not be used in the EFI stub, since the code is
* either included in the decompressor (x86, ARM) where they have no effect,
#define efi_info(fmt, ...) \
efi_printk(KERN_INFO fmt, ##__VA_ARGS__)
+#define efi_warn(fmt, ...) \
+ efi_printk(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__)
#define efi_err(fmt, ...) \
efi_printk(KERN_ERR "ERROR: " fmt, ##__VA_ARGS__)
#define efi_debug(fmt, ...) \
efi_printk(KERN_DEBUG "DEBUG: " fmt, ##__VA_ARGS__)
+#define efi_printk_once(fmt, ...) \
+({ \
+ static bool __print_once; \
+ bool __ret_print_once = !__print_once; \
+ \
+ if (!__print_once) { \
+ __print_once = true; \
+ efi_printk(fmt, ##__VA_ARGS__); \
+ } \
+ __ret_print_once; \
+})
+
+#define efi_info_once(fmt, ...) \
+ efi_printk_once(KERN_INFO fmt, ##__VA_ARGS__)
+#define efi_warn_once(fmt, ...) \
+ efi_printk_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__)
+#define efi_err_once(fmt, ...) \
+ efi_printk_once(KERN_ERR "ERROR: " fmt, ##__VA_ARGS__)
+#define efi_debug_once(fmt, ...) \
+ efi_printk_once(KERN_DEBUG "DEBUG: " fmt, ##__VA_ARGS__)
+
/* Helper macros for the usual case of using simple C variables: */
#ifndef fdt_setprop_inplace_var
#define fdt_setprop_inplace_var(fdt, node_offset, name, var) \
} mixed_mode;
};
+typedef struct {
+ u32 attributes;
+ u16 file_path_list_length;
+ u8 variable_data[];
+ // efi_char16_t description[];
+ // efi_device_path_protocol_t file_path_list[];
+ // u8 optional_data[];
+} __packed efi_load_option_t;
+
+#define EFI_LOAD_OPTION_ACTIVE 0x0001U
+#define EFI_LOAD_OPTION_FORCE_RECONNECT 0x0002U
+#define EFI_LOAD_OPTION_HIDDEN 0x0008U
+#define EFI_LOAD_OPTION_CATEGORY 0x1f00U
+#define EFI_LOAD_OPTION_CATEGORY_BOOT 0x0000U
+#define EFI_LOAD_OPTION_CATEGORY_APP 0x0100U
+
+#define EFI_LOAD_OPTION_BOOT_MASK \
+ (EFI_LOAD_OPTION_ACTIVE|EFI_LOAD_OPTION_HIDDEN|EFI_LOAD_OPTION_CATEGORY)
+#define EFI_LOAD_OPTION_MASK (EFI_LOAD_OPTION_FORCE_RECONNECT|EFI_LOAD_OPTION_BOOT_MASK)
+
+typedef struct {
+ u32 attributes;
+ u16 file_path_list_length;
+ const efi_char16_t *description;
+ const efi_device_path_protocol_t *file_path_list;
+ size_t optional_data_size;
+ const void *optional_data;
+} efi_load_option_unpacked_t;
+
void efi_pci_disable_bridge_busmaster(void);
typedef efi_status_t (*efi_exit_boot_map_processing)(
void efi_free(unsigned long size, unsigned long addr);
+void efi_apply_loadoptions_quirk(const void **load_options, int *load_options_size);
+
char *efi_convert_cmdline(efi_loaded_image_t *image, int *cmd_line_len);
efi_status_t efi_get_memory_map(struct efi_boot_memmap *map);
efi_status_t efi_allocate_pages_aligned(unsigned long size, unsigned long *addr,
unsigned long max, unsigned long align);
+efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align,
+ unsigned long *addr, unsigned long min);
+
efi_status_t efi_relocate_kernel(unsigned long *image_addr,
unsigned long image_size,
unsigned long alloc_size,
unsigned long *image_size,
unsigned long *reserve_addr,
unsigned long *reserve_size,
- unsigned long dram_base,
efi_loaded_image_t *image);
asmlinkage void __noreturn efi_enter_kernel(unsigned long entrypoint,
if (status)
goto fdt_set_fail;
- if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
+ if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && !efi_nokaslr) {
efi_status_t efi_status;
efi_status = efi_get_random_bytes(sizeof(fdt_val64),
status = fdt_setprop_var(fdt, node, "kaslr-seed", fdt_val64);
if (status)
goto fdt_set_fail;
- } else if (efi_status != EFI_NOT_FOUND) {
- return efi_status;
}
}
unsigned long *load_size)
{
const efi_char16_t *cmdline = image->load_options;
- int cmdline_len = image->load_options_size / 2;
+ int cmdline_len = image->load_options_size;
unsigned long efi_chunk_size = ULONG_MAX;
efi_file_protocol_t *volume = NULL;
efi_file_protocol_t *file;
if (!load_addr || !load_size)
return EFI_INVALID_PARAMETER;
+ efi_apply_loadoptions_quirk((const void **)&cmdline, &cmdline_len);
+ cmdline_len /= sizeof(*cmdline);
+
if (IS_ENABLED(CONFIG_X86) && !efi_nochunk)
efi_chunk_size = EFI_READ_CHUNK_SIZE;
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * To prevent the compiler from emitting GOT-indirected (and thus absolute)
- * references to any global symbols, override their visibility as 'hidden'
- */
-#pragma GCC visibility push(hidden)
*
* Return: status code
*/
-static efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align,
- unsigned long *addr, unsigned long min)
+efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align,
+ unsigned long *addr, unsigned long min)
{
unsigned long map_size, desc_size, buff_size;
efi_memory_desc_t *map;
*/
#include <linux/ctype.h>
+#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/string.h>
break;
default:
unreachable();
- };
+ }
return end;
}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * mokvar-table.c
+ *
+ * Copyright (c) 2020 Red Hat
+ * Author: Lenny Szubowicz <lszubowi@redhat.com>
+ *
+ * This module contains the kernel support for the Linux EFI Machine
+ * Owner Key (MOK) variable configuration table, which is identified by
+ * the LINUX_EFI_MOK_VARIABLE_TABLE_GUID.
+ *
+ * This EFI configuration table provides a more robust alternative to
+ * EFI volatile variables by which an EFI boot loader can pass the
+ * contents of the Machine Owner Key (MOK) certificate stores to the
+ * kernel during boot. If both the EFI MOK config table and corresponding
+ * EFI MOK variables are present, the table should be considered as
+ * more authoritative.
+ *
+ * This module includes code that validates and maps the EFI MOK table,
+ * if it's presence was detected very early in boot.
+ *
+ * Kernel interface routines are provided to walk through all the
+ * entries in the MOK config table or to search for a specific named
+ * entry.
+ *
+ * The contents of the individual named MOK config table entries are
+ * made available to user space via read-only sysfs binary files under:
+ *
+ * /sys/firmware/efi/mok-variables/
+ *
+ */
+#define pr_fmt(fmt) "mokvar: " fmt
+
+#include <linux/capability.h>
+#include <linux/efi.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/kobject.h>
+#include <linux/list.h>
+#include <linux/slab.h>
+
+#include <asm/early_ioremap.h>
+
+/*
+ * The LINUX_EFI_MOK_VARIABLE_TABLE_GUID config table is a packed
+ * sequence of struct efi_mokvar_table_entry, one for each named
+ * MOK variable. The sequence is terminated by an entry with a
+ * completely NULL name and 0 data size.
+ *
+ * efi_mokvar_table_size is set to the computed size of the
+ * MOK config table by efi_mokvar_table_init(). This will be
+ * non-zero if and only if the table if present and has been
+ * validated by efi_mokvar_table_init().
+ */
+static size_t efi_mokvar_table_size;
+
+/*
+ * efi_mokvar_table_va is the kernel virtual address at which the
+ * EFI MOK config table has been mapped by efi_mokvar_sysfs_init().
+ */
+static struct efi_mokvar_table_entry *efi_mokvar_table_va;
+
+/*
+ * Each /sys/firmware/efi/mok-variables/ sysfs file is represented by
+ * an instance of struct efi_mokvar_sysfs_attr on efi_mokvar_sysfs_list.
+ * bin_attr.private points to the associated EFI MOK config table entry.
+ *
+ * This list is created during boot and then remains unchanged.
+ * So no synchronization is currently required to walk the list.
+ */
+struct efi_mokvar_sysfs_attr {
+ struct bin_attribute bin_attr;
+ struct list_head node;
+};
+
+static LIST_HEAD(efi_mokvar_sysfs_list);
+static struct kobject *mokvar_kobj;
+
+/*
+ * efi_mokvar_table_init() - Early boot validation of EFI MOK config table
+ *
+ * If present, validate and compute the size of the EFI MOK variable
+ * configuration table. This table may be provided by an EFI boot loader
+ * as an alternative to ordinary EFI variables, due to platform-dependent
+ * limitations. The memory occupied by this table is marked as reserved.
+ *
+ * This routine must be called before efi_free_boot_services() in order
+ * to guarantee that it can mark the table as reserved.
+ *
+ * Implicit inputs:
+ * efi.mokvar_table: Physical address of EFI MOK variable config table
+ * or special value that indicates no such table.
+ *
+ * Implicit outputs:
+ * efi_mokvar_table_size: Computed size of EFI MOK variable config table.
+ * The table is considered present and valid if this
+ * is non-zero.
+ */
+void __init efi_mokvar_table_init(void)
+{
+ efi_memory_desc_t md;
+ void *va = NULL;
+ unsigned long cur_offset = 0;
+ unsigned long offset_limit;
+ unsigned long map_size = 0;
+ unsigned long map_size_needed = 0;
+ unsigned long size;
+ struct efi_mokvar_table_entry *mokvar_entry;
+ int err;
+
+ if (!efi_enabled(EFI_MEMMAP))
+ return;
+
+ if (efi.mokvar_table == EFI_INVALID_TABLE_ADDR)
+ return;
+ /*
+ * The EFI MOK config table must fit within a single EFI memory
+ * descriptor range.
+ */
+ err = efi_mem_desc_lookup(efi.mokvar_table, &md);
+ if (err) {
+ pr_warn("EFI MOKvar config table is not within the EFI memory map\n");
+ return;
+ }
+
+ offset_limit = efi_mem_desc_end(&md) - efi.mokvar_table;
+
+ /*
+ * Validate the MOK config table. Since there is no table header
+ * from which we could get the total size of the MOK config table,
+ * we compute the total size as we validate each variably sized
+ * entry, remapping as necessary.
+ */
+ err = -EINVAL;
+ while (cur_offset + sizeof(*mokvar_entry) <= offset_limit) {
+ mokvar_entry = va + cur_offset;
+ map_size_needed = cur_offset + sizeof(*mokvar_entry);
+ if (map_size_needed > map_size) {
+ if (va)
+ early_memunmap(va, map_size);
+ /*
+ * Map a little more than the fixed size entry
+ * header, anticipating some data. It's safe to
+ * do so as long as we stay within current memory
+ * descriptor.
+ */
+ map_size = min(map_size_needed + 2*EFI_PAGE_SIZE,
+ offset_limit);
+ va = early_memremap(efi.mokvar_table, map_size);
+ if (!va) {
+ pr_err("Failed to map EFI MOKvar config table pa=0x%lx, size=%lu.\n",
+ efi.mokvar_table, map_size);
+ return;
+ }
+ mokvar_entry = va + cur_offset;
+ }
+
+ /* Check for last sentinel entry */
+ if (mokvar_entry->name[0] == '\0') {
+ if (mokvar_entry->data_size != 0)
+ break;
+ err = 0;
+ break;
+ }
+
+ /* Sanity check that the name is null terminated */
+ size = strnlen(mokvar_entry->name,
+ sizeof(mokvar_entry->name));
+ if (size >= sizeof(mokvar_entry->name))
+ break;
+
+ /* Advance to the next entry */
+ cur_offset = map_size_needed + mokvar_entry->data_size;
+ }
+
+ if (va)
+ early_memunmap(va, map_size);
+ if (err) {
+ pr_err("EFI MOKvar config table is not valid\n");
+ return;
+ }
+ efi_mem_reserve(efi.mokvar_table, map_size_needed);
+ efi_mokvar_table_size = map_size_needed;
+}
+
+/*
+ * efi_mokvar_entry_next() - Get next entry in the EFI MOK config table
+ *
+ * mokvar_entry: Pointer to current EFI MOK config table entry
+ * or null. Null indicates get first entry.
+ * Passed by reference. This is updated to the
+ * same value as the return value.
+ *
+ * Returns: Pointer to next EFI MOK config table entry
+ * or null, if there are no more entries.
+ * Same value is returned in the mokvar_entry
+ * parameter.
+ *
+ * This routine depends on the EFI MOK config table being entirely
+ * mapped with it's starting virtual address in efi_mokvar_table_va.
+ */
+struct efi_mokvar_table_entry *efi_mokvar_entry_next(
+ struct efi_mokvar_table_entry **mokvar_entry)
+{
+ struct efi_mokvar_table_entry *mokvar_cur;
+ struct efi_mokvar_table_entry *mokvar_next;
+ size_t size_cur;
+
+ mokvar_cur = *mokvar_entry;
+ *mokvar_entry = NULL;
+
+ if (efi_mokvar_table_va == NULL)
+ return NULL;
+
+ if (mokvar_cur == NULL) {
+ mokvar_next = efi_mokvar_table_va;
+ } else {
+ if (mokvar_cur->name[0] == '\0')
+ return NULL;
+ size_cur = sizeof(*mokvar_cur) + mokvar_cur->data_size;
+ mokvar_next = (void *)mokvar_cur + size_cur;
+ }
+
+ if (mokvar_next->name[0] == '\0')
+ return NULL;
+
+ *mokvar_entry = mokvar_next;
+ return mokvar_next;
+}
+
+/*
+ * efi_mokvar_entry_find() - Find EFI MOK config entry by name
+ *
+ * name: Name of the entry to look for.
+ *
+ * Returns: Pointer to EFI MOK config table entry if found;
+ * null otherwise.
+ *
+ * This routine depends on the EFI MOK config table being entirely
+ * mapped with it's starting virtual address in efi_mokvar_table_va.
+ */
+struct efi_mokvar_table_entry *efi_mokvar_entry_find(const char *name)
+{
+ struct efi_mokvar_table_entry *mokvar_entry = NULL;
+
+ while (efi_mokvar_entry_next(&mokvar_entry)) {
+ if (!strncmp(name, mokvar_entry->name,
+ sizeof(mokvar_entry->name)))
+ return mokvar_entry;
+ }
+ return NULL;
+}
+
+/*
+ * efi_mokvar_sysfs_read() - sysfs binary file read routine
+ *
+ * Returns: Count of bytes read.
+ *
+ * Copy EFI MOK config table entry data for this mokvar sysfs binary file
+ * to the supplied buffer, starting at the specified offset into mokvar table
+ * entry data, for the specified count bytes. The copy is limited by the
+ * amount of data in this mokvar config table entry.
+ */
+static ssize_t efi_mokvar_sysfs_read(struct file *file, struct kobject *kobj,
+ struct bin_attribute *bin_attr, char *buf,
+ loff_t off, size_t count)
+{
+ struct efi_mokvar_table_entry *mokvar_entry = bin_attr->private;
+
+ if (!capable(CAP_SYS_ADMIN))
+ return 0;
+
+ if (off >= mokvar_entry->data_size)
+ return 0;
+ if (count > mokvar_entry->data_size - off)
+ count = mokvar_entry->data_size - off;
+
+ memcpy(buf, mokvar_entry->data + off, count);
+ return count;
+}
+
+/*
+ * efi_mokvar_sysfs_init() - Map EFI MOK config table and create sysfs
+ *
+ * Map the EFI MOK variable config table for run-time use by the kernel
+ * and create the sysfs entries in /sys/firmware/efi/mok-variables/
+ *
+ * This routine just returns if a valid EFI MOK variable config table
+ * was not found earlier during boot.
+ *
+ * This routine must be called during a "middle" initcall phase, i.e.
+ * after efi_mokvar_table_init() but before UEFI certs are loaded
+ * during late init.
+ *
+ * Implicit inputs:
+ * efi.mokvar_table: Physical address of EFI MOK variable config table
+ * or special value that indicates no such table.
+ *
+ * efi_mokvar_table_size: Computed size of EFI MOK variable config table.
+ * The table is considered present and valid if this
+ * is non-zero.
+ *
+ * Implicit outputs:
+ * efi_mokvar_table_va: Start virtual address of the EFI MOK config table.
+ */
+static int __init efi_mokvar_sysfs_init(void)
+{
+ void *config_va;
+ struct efi_mokvar_table_entry *mokvar_entry = NULL;
+ struct efi_mokvar_sysfs_attr *mokvar_sysfs = NULL;
+ int err = 0;
+
+ if (efi_mokvar_table_size == 0)
+ return -ENOENT;
+
+ config_va = memremap(efi.mokvar_table, efi_mokvar_table_size,
+ MEMREMAP_WB);
+ if (!config_va) {
+ pr_err("Failed to map EFI MOKvar config table\n");
+ return -ENOMEM;
+ }
+ efi_mokvar_table_va = config_va;
+
+ mokvar_kobj = kobject_create_and_add("mok-variables", efi_kobj);
+ if (!mokvar_kobj) {
+ pr_err("Failed to create EFI mok-variables sysfs entry\n");
+ return -ENOMEM;
+ }
+
+ while (efi_mokvar_entry_next(&mokvar_entry)) {
+ mokvar_sysfs = kzalloc(sizeof(*mokvar_sysfs), GFP_KERNEL);
+ if (!mokvar_sysfs) {
+ err = -ENOMEM;
+ break;
+ }
+
+ sysfs_bin_attr_init(&mokvar_sysfs->bin_attr);
+ mokvar_sysfs->bin_attr.private = mokvar_entry;
+ mokvar_sysfs->bin_attr.attr.name = mokvar_entry->name;
+ mokvar_sysfs->bin_attr.attr.mode = 0400;
+ mokvar_sysfs->bin_attr.size = mokvar_entry->data_size;
+ mokvar_sysfs->bin_attr.read = efi_mokvar_sysfs_read;
+
+ err = sysfs_create_bin_file(mokvar_kobj,
+ &mokvar_sysfs->bin_attr);
+ if (err)
+ break;
+
+ list_add_tail(&mokvar_sysfs->node, &efi_mokvar_sysfs_list);
+ }
+
+ if (err) {
+ pr_err("Failed to create some EFI mok-variables sysfs entries\n");
+ kfree(mokvar_sysfs);
+ }
+ return err;
+}
+device_initcall(efi_mokvar_sysfs_init);
*/
static DEFINE_SEMAPHORE(efivars_lock);
-static bool efivar_wq_enabled = true;
-DECLARE_WORK(efivar_work, NULL);
-EXPORT_SYMBOL_GPL(efivar_work);
-
static bool
validate_device_path(efi_char16_t *var_name, int match, u8 *buffer,
unsigned long len)
size_t i, len8 = len16 / sizeof(efi_char16_t);
char *str8;
- /*
- * Disable the workqueue since the algorithm it uses for
- * detecting new variables won't work with this buggy
- * implementation of GetNextVariableName().
- */
- efivar_wq_enabled = false;
-
str8 = kzalloc(len8, GFP_KERNEL);
if (!str8)
return;
* efivar_init - build the initial list of EFI variables
* @func: callback function to invoke for every variable
* @data: function-specific data to pass to @func
- * @atomic: do we need to execute the @func-loop atomically?
* @duplicates: error if we encounter duplicates on @head?
* @head: initialised head of variable list
*
}
EXPORT_SYMBOL_GPL(efivars_kobject);
-/**
- * efivar_run_worker - schedule the efivar worker thread
- */
-void efivar_run_worker(void)
-{
- if (efivar_wq_enabled)
- schedule_work(&efivar_work);
-}
-EXPORT_SYMBOL_GPL(efivar_run_worker);
-
/**
* efivars_register - register an efivars
* @efivars: efivars to register
help
Say Y here if you want to enable SMI callbacks for Google
platforms. This provides an interface for writing to and
- clearing the event log. If EFI_VARS is also enabled this
+ clearing the event log. If CONFIG_EFI is also enabled this
driver provides an interface for reading and writing NVRAM
variables.
return rc;
}
-#ifdef CONFIG_EFI_VARS
+#ifdef CONFIG_EFI
static struct efivars efivars;
.get_next_variable = gsmi_get_next_variable,
};
-#endif /* CONFIG_EFI_VARS */
+#endif /* CONFIG_EFI */
static ssize_t eventlog_write(struct file *filp, struct kobject *kobj,
struct bin_attribute *bin_attr,
goto out_remove_bin_file;
}
-#ifdef CONFIG_EFI_VARS
+#ifdef CONFIG_EFI
ret = efivars_register(&efivars, &efivar_ops, gsmi_kobj);
if (ret) {
printk(KERN_INFO "gsmi: Failed to register efivars\n");
unregister_die_notifier(&gsmi_die_notifier);
atomic_notifier_chain_unregister(&panic_notifier_list,
&gsmi_panic_notifier);
-#ifdef CONFIG_EFI_VARS
+#ifdef CONFIG_EFI
efivars_unregister(&efivars);
#endif
}
EXPORT_SYMBOL(qcom_scm_iommu_secure_ptbl_init);
+int qcom_scm_mem_protect_video_var(u32 cp_start, u32 cp_size,
+ u32 cp_nonpixel_start,
+ u32 cp_nonpixel_size)
+{
+ int ret;
+ struct qcom_scm_desc desc = {
+ .svc = QCOM_SCM_SVC_MP,
+ .cmd = QCOM_SCM_MP_VIDEO_VAR,
+ .arginfo = QCOM_SCM_ARGS(4, QCOM_SCM_VAL, QCOM_SCM_VAL,
+ QCOM_SCM_VAL, QCOM_SCM_VAL),
+ .args[0] = cp_start,
+ .args[1] = cp_size,
+ .args[2] = cp_nonpixel_start,
+ .args[3] = cp_nonpixel_size,
+ .owner = ARM_SMCCC_OWNER_SIP,
+ };
+ struct qcom_scm_res res;
+
+ ret = qcom_scm_call(__scm->dev, &desc, &res);
+
+ return ret ? : res.result[0];
+}
+EXPORT_SYMBOL(qcom_scm_mem_protect_video_var);
+
static int __qcom_scm_assign_mem(struct device *dev, phys_addr_t mem_region,
size_t mem_sz, phys_addr_t src, size_t src_sz,
phys_addr_t dest, size_t dest_sz)
#define QCOM_SCM_MP_RESTORE_SEC_CFG 0x02
#define QCOM_SCM_MP_IOMMU_SECURE_PTBL_SIZE 0x03
#define QCOM_SCM_MP_IOMMU_SECURE_PTBL_INIT 0x04
+#define QCOM_SCM_MP_VIDEO_VAR 0x08
#define QCOM_SCM_MP_ASSIGN 0x16
#define QCOM_SCM_SVC_OCMEM 0x0f
This ABI is deprecated. If you want to use GPIO from userspace,
use the character device /dev/gpiochipN with the appropriate
- ioctl() operations instead. The character device is always
- available.
+ ioctl() operations instead.
+
+config GPIO_CDEV
+ bool
+ prompt "Character device (/dev/gpiochipN) support" if EXPERT
+ default y
+ help
+ Say Y here to add the character device /dev/gpiochipN interface
+ for GPIOs. The character device allows userspace to control GPIOs
+ using ioctl() operations.
+
+ Only say N if you are sure that the GPIO character device is not
+ required.
+
+ If unsure, say Y.
+
+config GPIO_CDEV_V1
+ bool "Support GPIO ABI Version 1"
+ default y
+ depends on GPIO_CDEV
+ help
+ Say Y here to support version 1 of the GPIO CDEV ABI.
+
+ This ABI version is deprecated.
+ Please use the latest ABI for new developments.
+
+ If unsure, say Y.
config GPIO_GENERIC
depends on HAS_IOMEM # Only for IOMEM drivers
config GPIO_DWAPB
tristate "Synopsys DesignWare APB GPIO driver"
select GPIO_GENERIC
- select GENERIC_IRQ_CHIP
+ select GPIOLIB_IRQCHIP
help
Say Y or M here to build support for the Synopsys DesignWare APB
GPIO block.
select REGMAP_MMIO
config GPIO_MXC
- def_bool y
+ tristate "i.MX GPIO support"
depends on ARCH_MXC || COMPILE_TEST
select GPIO_GENERIC
select GENERIC_IRQ_CHIP
obj-$(CONFIG_GPIOLIB) += gpiolib.o
obj-$(CONFIG_GPIOLIB) += gpiolib-devres.o
obj-$(CONFIG_GPIOLIB) += gpiolib-legacy.o
-obj-$(CONFIG_GPIOLIB) += gpiolib-devprop.o
-obj-$(CONFIG_GPIOLIB) += gpiolib-cdev.o
obj-$(CONFIG_OF_GPIO) += gpiolib-of.o
+obj-$(CONFIG_GPIO_CDEV) += gpiolib-cdev.o
obj-$(CONFIG_GPIO_SYSFS) += gpiolib-sysfs.o
obj-$(CONFIG_GPIO_ACPI) += gpiolib-acpi.o
return gpiod_get_value(fwd->descs[offset]);
}
-static int gpio_fwd_get_multiple(struct gpio_chip *chip, unsigned long *mask,
+static int gpio_fwd_get_multiple(struct gpiochip_fwd *fwd, unsigned long *mask,
unsigned long *bits)
{
- struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
- unsigned long *values, flags = 0;
struct gpio_desc **descs;
+ unsigned long *values;
unsigned int i, j = 0;
int error;
- if (chip->can_sleep)
- mutex_lock(&fwd->mlock);
- else
- spin_lock_irqsave(&fwd->slock, flags);
-
/* Both values bitmap and desc pointers are stored in tmp[] */
values = &fwd->tmp[0];
descs = (void *)&fwd->tmp[BITS_TO_LONGS(fwd->chip.ngpio)];
descs[j++] = fwd->descs[i];
error = gpiod_get_array_value(j, descs, NULL, values);
- if (!error) {
- j = 0;
- for_each_set_bit(i, mask, fwd->chip.ngpio)
- __assign_bit(i, bits, test_bit(j++, values));
- }
+ if (error)
+ return error;
- if (chip->can_sleep)
+ j = 0;
+ for_each_set_bit(i, mask, fwd->chip.ngpio)
+ __assign_bit(i, bits, test_bit(j++, values));
+
+ return 0;
+}
+
+static int gpio_fwd_get_multiple_locked(struct gpio_chip *chip,
+ unsigned long *mask, unsigned long *bits)
+{
+ struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+ unsigned long flags;
+ int error;
+
+ if (chip->can_sleep) {
+ mutex_lock(&fwd->mlock);
+ error = gpio_fwd_get_multiple(fwd, mask, bits);
mutex_unlock(&fwd->mlock);
- else
+ } else {
+ spin_lock_irqsave(&fwd->slock, flags);
+ error = gpio_fwd_get_multiple(fwd, mask, bits);
spin_unlock_irqrestore(&fwd->slock, flags);
+ }
return error;
}
gpiod_set_value(fwd->descs[offset], value);
}
-static void gpio_fwd_set_multiple(struct gpio_chip *chip, unsigned long *mask,
+static void gpio_fwd_set_multiple(struct gpiochip_fwd *fwd, unsigned long *mask,
unsigned long *bits)
{
- struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
- unsigned long *values, flags = 0;
struct gpio_desc **descs;
+ unsigned long *values;
unsigned int i, j = 0;
- if (chip->can_sleep)
- mutex_lock(&fwd->mlock);
- else
- spin_lock_irqsave(&fwd->slock, flags);
-
/* Both values bitmap and desc pointers are stored in tmp[] */
values = &fwd->tmp[0];
descs = (void *)&fwd->tmp[BITS_TO_LONGS(fwd->chip.ngpio)];
}
gpiod_set_array_value(j, descs, NULL, values);
+}
- if (chip->can_sleep)
+static void gpio_fwd_set_multiple_locked(struct gpio_chip *chip,
+ unsigned long *mask, unsigned long *bits)
+{
+ struct gpiochip_fwd *fwd = gpiochip_get_data(chip);
+ unsigned long flags;
+
+ if (chip->can_sleep) {
+ mutex_lock(&fwd->mlock);
+ gpio_fwd_set_multiple(fwd, mask, bits);
mutex_unlock(&fwd->mlock);
- else
+ } else {
+ spin_lock_irqsave(&fwd->slock, flags);
+ gpio_fwd_set_multiple(fwd, mask, bits);
spin_unlock_irqrestore(&fwd->slock, flags);
+ }
}
static int gpio_fwd_set_config(struct gpio_chip *chip, unsigned int offset,
chip->direction_input = gpio_fwd_direction_input;
chip->direction_output = gpio_fwd_direction_output;
chip->get = gpio_fwd_get;
- chip->get_multiple = gpio_fwd_get_multiple;
+ chip->get_multiple = gpio_fwd_get_multiple_locked;
chip->set = gpio_fwd_set;
- chip->set_multiple = gpio_fwd_set_multiple;
+ chip->set_multiple = gpio_fwd_set_multiple_locked;
chip->base = -1;
chip->ngpio = ngpios;
fwd->descs = descs;
ret = (readl_relaxed(ptr) & AMD_FCH_GPIO_FLAG_DIRECTION);
spin_unlock_irqrestore(&priv->lock, flags);
- return ret ? GPIO_LINE_DIRECTION_IN : GPIO_LINE_DIRECTION_OUT;
+ return ret ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
}
static void amd_fch_gpio_set(struct gpio_chip *gc,
#include <linux/spinlock.h>
#include <linux/string.h>
-#define MAX_NR_SGPIO 80
+/*
+ * MAX_NR_HW_GPIO represents the number of actual hardware-supported GPIOs (ie,
+ * slots within the clocked serial GPIO data). Since each HW GPIO is both an
+ * input and an output, we provide MAX_NR_HW_GPIO * 2 lines on our gpiochip
+ * device.
+ *
+ * We use SGPIO_OUTPUT_OFFSET to define the split between the inputs and
+ * outputs; the inputs start at line 0, the outputs start at OUTPUT_OFFSET.
+ */
+#define MAX_NR_HW_SGPIO 80
+#define SGPIO_OUTPUT_OFFSET MAX_NR_HW_SGPIO
#define ASPEED_SGPIO_CTRL 0x54
struct clk *pclk;
spinlock_t lock;
void __iomem *base;
- uint32_t dir_in[3];
int irq;
+ int n_sgpio;
};
struct aspeed_sgpio_bank {
}
}
-#define GPIO_BANK(x) ((x) >> 5)
-#define GPIO_OFFSET(x) ((x) & 0x1f)
+#define GPIO_BANK(x) ((x % SGPIO_OUTPUT_OFFSET) >> 5)
+#define GPIO_OFFSET(x) ((x % SGPIO_OUTPUT_OFFSET) & 0x1f)
#define GPIO_BIT(x) BIT(GPIO_OFFSET(x))
static const struct aspeed_sgpio_bank *to_bank(unsigned int offset)
{
- unsigned int bank = GPIO_BANK(offset);
+ unsigned int bank;
+
+ bank = GPIO_BANK(offset);
WARN_ON(bank >= ARRAY_SIZE(aspeed_sgpio_banks));
return &aspeed_sgpio_banks[bank];
}
+static int aspeed_sgpio_init_valid_mask(struct gpio_chip *gc,
+ unsigned long *valid_mask, unsigned int ngpios)
+{
+ struct aspeed_sgpio *sgpio = gpiochip_get_data(gc);
+ int n = sgpio->n_sgpio;
+ int c = SGPIO_OUTPUT_OFFSET - n;
+
+ WARN_ON(ngpios < MAX_NR_HW_SGPIO * 2);
+
+ /* input GPIOs in the lower range */
+ bitmap_set(valid_mask, 0, n);
+ bitmap_clear(valid_mask, n, c);
+
+ /* output GPIOS above SGPIO_OUTPUT_OFFSET */
+ bitmap_set(valid_mask, SGPIO_OUTPUT_OFFSET, n);
+ bitmap_clear(valid_mask, SGPIO_OUTPUT_OFFSET + n, c);
+
+ return 0;
+}
+
+static void aspeed_sgpio_irq_init_valid_mask(struct gpio_chip *gc,
+ unsigned long *valid_mask, unsigned int ngpios)
+{
+ struct aspeed_sgpio *sgpio = gpiochip_get_data(gc);
+ int n = sgpio->n_sgpio;
+
+ WARN_ON(ngpios < MAX_NR_HW_SGPIO * 2);
+
+ /* input GPIOs in the lower range */
+ bitmap_set(valid_mask, 0, n);
+ bitmap_clear(valid_mask, n, ngpios - n);
+}
+
+static bool aspeed_sgpio_is_input(unsigned int offset)
+{
+ return offset < SGPIO_OUTPUT_OFFSET;
+}
+
static int aspeed_sgpio_get(struct gpio_chip *gc, unsigned int offset)
{
struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
const struct aspeed_sgpio_bank *bank = to_bank(offset);
unsigned long flags;
enum aspeed_sgpio_reg reg;
- bool is_input;
int rc = 0;
spin_lock_irqsave(&gpio->lock, flags);
- is_input = gpio->dir_in[GPIO_BANK(offset)] & GPIO_BIT(offset);
- reg = is_input ? reg_val : reg_rdata;
+ reg = aspeed_sgpio_is_input(offset) ? reg_val : reg_rdata;
rc = !!(ioread32(bank_reg(gpio, bank, reg)) & GPIO_BIT(offset));
spin_unlock_irqrestore(&gpio->lock, flags);
return rc;
}
-static void sgpio_set_value(struct gpio_chip *gc, unsigned int offset, int val)
+static int sgpio_set_value(struct gpio_chip *gc, unsigned int offset, int val)
{
struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
const struct aspeed_sgpio_bank *bank = to_bank(offset);
- void __iomem *addr;
+ void __iomem *addr_r, *addr_w;
u32 reg = 0;
- addr = bank_reg(gpio, bank, reg_val);
- reg = ioread32(addr);
+ if (aspeed_sgpio_is_input(offset))
+ return -EINVAL;
+
+ /* Since this is an output, read the cached value from rdata, then
+ * update val. */
+ addr_r = bank_reg(gpio, bank, reg_rdata);
+ addr_w = bank_reg(gpio, bank, reg_val);
+
+ reg = ioread32(addr_r);
if (val)
reg |= GPIO_BIT(offset);
else
reg &= ~GPIO_BIT(offset);
- iowrite32(reg, addr);
+ iowrite32(reg, addr_w);
+
+ return 0;
}
static void aspeed_sgpio_set(struct gpio_chip *gc, unsigned int offset, int val)
static int aspeed_sgpio_dir_in(struct gpio_chip *gc, unsigned int offset)
{
- struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
- unsigned long flags;
-
- spin_lock_irqsave(&gpio->lock, flags);
- gpio->dir_in[GPIO_BANK(offset)] |= GPIO_BIT(offset);
- spin_unlock_irqrestore(&gpio->lock, flags);
-
- return 0;
+ return aspeed_sgpio_is_input(offset) ? 0 : -EINVAL;
}
static int aspeed_sgpio_dir_out(struct gpio_chip *gc, unsigned int offset, int val)
{
struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
unsigned long flags;
+ int rc;
- spin_lock_irqsave(&gpio->lock, flags);
-
- gpio->dir_in[GPIO_BANK(offset)] &= ~GPIO_BIT(offset);
- sgpio_set_value(gc, offset, val);
+ /* No special action is required for setting the direction; we'll
+ * error-out in sgpio_set_value if this isn't an output GPIO */
+ spin_lock_irqsave(&gpio->lock, flags);
+ rc = sgpio_set_value(gc, offset, val);
spin_unlock_irqrestore(&gpio->lock, flags);
- return 0;
+ return rc;
}
static int aspeed_sgpio_get_direction(struct gpio_chip *gc, unsigned int offset)
{
- int dir_status;
- struct aspeed_sgpio *gpio = gpiochip_get_data(gc);
- unsigned long flags;
-
- spin_lock_irqsave(&gpio->lock, flags);
- dir_status = gpio->dir_in[GPIO_BANK(offset)] & GPIO_BIT(offset);
- spin_unlock_irqrestore(&gpio->lock, flags);
-
- return dir_status;
-
+ return !!aspeed_sgpio_is_input(offset);
}
static void irqd_to_aspeed_sgpio_data(struct irq_data *d,
irq = &gpio->chip.irq;
irq->chip = &aspeed_sgpio_irqchip;
+ irq->init_valid_mask = aspeed_sgpio_irq_init_valid_mask;
irq->handler = handle_bad_irq;
irq->default_type = IRQ_TYPE_NONE;
irq->parent_handler = aspeed_sgpio_irq_handler;
irq->parents = &gpio->irq;
irq->num_parents = 1;
- /* set IRQ settings and Enable Interrupt */
+ /* Apply default IRQ settings */
for (i = 0; i < ARRAY_SIZE(aspeed_sgpio_banks); i++) {
bank = &aspeed_sgpio_banks[i];
/* set falling or level-low irq */
iowrite32(0x00000000, bank_reg(gpio, bank, reg_irq_type0));
/* trigger type is edge */
iowrite32(0x00000000, bank_reg(gpio, bank, reg_irq_type1));
- /* dual edge trigger mode. */
- iowrite32(0xffffffff, bank_reg(gpio, bank, reg_irq_type2));
- /* enable irq */
- iowrite32(0xffffffff, bank_reg(gpio, bank, reg_irq_enable));
+ /* single edge trigger */
+ iowrite32(0x00000000, bank_reg(gpio, bank, reg_irq_type2));
}
return 0;
if (rc < 0) {
dev_err(&pdev->dev, "Could not read ngpios property\n");
return -EINVAL;
- } else if (nr_gpios > MAX_NR_SGPIO) {
+ } else if (nr_gpios > MAX_NR_HW_SGPIO) {
dev_err(&pdev->dev, "Number of GPIOs exceeds the maximum of %d: %d\n",
- MAX_NR_SGPIO, nr_gpios);
+ MAX_NR_HW_SGPIO, nr_gpios);
return -EINVAL;
}
+ gpio->n_sgpio = nr_gpios;
rc = of_property_read_u32(pdev->dev.of_node, "bus-frequency", &sgpio_freq);
if (rc < 0) {
spin_lock_init(&gpio->lock);
gpio->chip.parent = &pdev->dev;
- gpio->chip.ngpio = nr_gpios;
+ gpio->chip.ngpio = MAX_NR_HW_SGPIO * 2;
+ gpio->chip.init_valid_mask = aspeed_sgpio_init_valid_mask;
gpio->chip.direction_input = aspeed_sgpio_dir_in;
gpio->chip.direction_output = aspeed_sgpio_dir_out;
gpio->chip.get_direction = aspeed_sgpio_get_direction;
gpio->chip.label = dev_name(&pdev->dev);
gpio->chip.base = -1;
- /* set all SGPIO pins as input (1). */
- memset(gpio->dir_in, 0xff, sizeof(gpio->dir_in));
-
aspeed_sgpio_setup_irqs(gpio, pdev);
rc = devm_gpiochip_add_data(&pdev->dev, &gpio->chip, gpio);
static const struct aspeed_bank_props ast2600_bank_props[] = {
/* input output */
- {5, 0xffffffff, 0x0000ffff}, /* U/V/W/X */
- {6, 0xffff0000, 0x0fff0000}, /* Y/Z */
+ {5, 0xffffffff, 0xffffff00}, /* U/V/W/X */
+ {6, 0x0000ffff, 0x0000ffff}, /* Y/Z */
{ },
};
dev_err(dev, "Couldn't determine # GPIO banks\n");
return -ENOENT;
} else if (ret < 0) {
- if (ret != -EPROBE_DEFER)
- dev_err(dev, "Couldn't determine GPIO banks: (%pe)\n",
- ERR_PTR(ret));
- return ret;
+ return dev_err_probe(dev, ret, "Couldn't determine GPIO banks\n");
}
kona_gpio->num_bank = ret;
for (i = 0; i < nirq; i++) {
chips->irqs[i] = platform_get_irq(pdev, i);
- if (chips->irqs[i] < 0) {
- if (chips->irqs[i] != -EPROBE_DEFER)
- dev_info(dev, "IRQ not populated, err = %d\n",
- chips->irqs[i]);
- return chips->irqs[i];
- }
+ if (chips->irqs[i] < 0)
+ return dev_err_probe(dev, chips->irqs[i], "IRQ not populated\n");
}
chips->chip.label = dev_name(dev);
#include <linux/io.h>
#include <linux/ioport.h>
#include <linux/irq.h>
-#include <linux/irqdomain.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_address.h>
};
#endif
+struct dwapb_gpio_port_irqchip {
+ struct irq_chip irqchip;
+ unsigned int nr_irqs;
+ unsigned int irq[DWAPB_MAX_GPIOS];
+};
+
struct dwapb_gpio_port {
struct gpio_chip gc;
- bool is_registered;
+ struct dwapb_gpio_port_irqchip *pirq;
struct dwapb_gpio *gpio;
#ifdef CONFIG_PM_SLEEP
struct dwapb_context *ctx;
#endif
unsigned int idx;
};
+#define to_dwapb_gpio(_gc) \
+ (container_of(_gc, struct dwapb_gpio_port, gc)->gpio)
struct dwapb_gpio {
struct device *dev;
void __iomem *regs;
struct dwapb_gpio_port *ports;
unsigned int nr_ports;
- struct irq_domain *domain;
unsigned int flags;
struct reset_control *rst;
struct clk_bulk_data clks[DWAPB_NR_CLOCKS];
gc->write_reg(reg_base + gpio_reg_convert(gpio, offset), val);
}
-static int dwapb_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
-{
- struct dwapb_gpio_port *port = gpiochip_get_data(gc);
- struct dwapb_gpio *gpio = port->gpio;
-
- return irq_find_mapping(gpio->domain, offset);
-}
-
static struct dwapb_gpio_port *dwapb_offs_to_port(struct dwapb_gpio *gpio, unsigned int offs)
{
struct dwapb_gpio_port *port;
for (i = 0; i < gpio->nr_ports; i++) {
port = &gpio->ports[i];
- if (port->idx == offs / 32)
+ if (port->idx == offs / DWAPB_MAX_GPIOS)
return port;
}
pol = dwapb_read(gpio, GPIO_INT_POLARITY);
/* Just read the current value right out of the data register */
- val = gc->get(gc, offs % 32);
+ val = gc->get(gc, offs % DWAPB_MAX_GPIOS);
if (val)
pol &= ~BIT(offs);
else
static u32 dwapb_do_irq(struct dwapb_gpio *gpio)
{
+ struct gpio_chip *gc = &gpio->ports[0].gc;
unsigned long irq_status;
irq_hw_number_t hwirq;
irq_status = dwapb_read(gpio, GPIO_INTSTATUS);
- for_each_set_bit(hwirq, &irq_status, 32) {
- int gpio_irq = irq_find_mapping(gpio->domain, hwirq);
+ for_each_set_bit(hwirq, &irq_status, DWAPB_MAX_GPIOS) {
+ int gpio_irq = irq_find_mapping(gc->irq.domain, hwirq);
u32 irq_type = irq_get_trigger_type(gpio_irq);
generic_handle_irq(gpio_irq);
chained_irq_exit(chip, desc);
}
+static irqreturn_t dwapb_irq_handler_mfd(int irq, void *dev_id)
+{
+ return IRQ_RETVAL(dwapb_do_irq(dev_id));
+}
+
+static void dwapb_irq_ack(struct irq_data *d)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct dwapb_gpio *gpio = to_dwapb_gpio(gc);
+ u32 val = BIT(irqd_to_hwirq(d));
+ unsigned long flags;
+
+ spin_lock_irqsave(&gc->bgpio_lock, flags);
+ dwapb_write(gpio, GPIO_PORTA_EOI, val);
+ spin_unlock_irqrestore(&gc->bgpio_lock, flags);
+}
+
+static void dwapb_irq_mask(struct irq_data *d)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct dwapb_gpio *gpio = to_dwapb_gpio(gc);
+ unsigned long flags;
+ u32 val;
+
+ spin_lock_irqsave(&gc->bgpio_lock, flags);
+ val = dwapb_read(gpio, GPIO_INTMASK) | BIT(irqd_to_hwirq(d));
+ dwapb_write(gpio, GPIO_INTMASK, val);
+ spin_unlock_irqrestore(&gc->bgpio_lock, flags);
+}
+
+static void dwapb_irq_unmask(struct irq_data *d)
+{
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct dwapb_gpio *gpio = to_dwapb_gpio(gc);
+ unsigned long flags;
+ u32 val;
+
+ spin_lock_irqsave(&gc->bgpio_lock, flags);
+ val = dwapb_read(gpio, GPIO_INTMASK) & ~BIT(irqd_to_hwirq(d));
+ dwapb_write(gpio, GPIO_INTMASK, val);
+ spin_unlock_irqrestore(&gc->bgpio_lock, flags);
+}
+
static void dwapb_irq_enable(struct irq_data *d)
{
- struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
- struct dwapb_gpio *gpio = igc->private;
- struct gpio_chip *gc = &gpio->ports[0].gc;
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct dwapb_gpio *gpio = to_dwapb_gpio(gc);
unsigned long flags;
u32 val;
static void dwapb_irq_disable(struct irq_data *d)
{
- struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
- struct dwapb_gpio *gpio = igc->private;
- struct gpio_chip *gc = &gpio->ports[0].gc;
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct dwapb_gpio *gpio = to_dwapb_gpio(gc);
unsigned long flags;
u32 val;
static int dwapb_irq_set_type(struct irq_data *d, u32 type)
{
- struct irq_chip_generic *igc = irq_data_get_irq_chip_data(d);
- struct dwapb_gpio *gpio = igc->private;
- struct gpio_chip *gc = &gpio->ports[0].gc;
+ struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+ struct dwapb_gpio *gpio = to_dwapb_gpio(gc);
irq_hw_number_t bit = irqd_to_hwirq(d);
unsigned long level, polarity, flags;
break;
}
- irq_setup_alt_chip(d, type);
+ if (type & IRQ_TYPE_LEVEL_MASK)
+ irq_set_handler_locked(d, handle_level_irq);
+ else if (type & IRQ_TYPE_EDGE_BOTH)
+ irq_set_handler_locked(d, handle_edge_irq);
dwapb_write(gpio, GPIO_INTTYPE_LEVEL, level);
if (type != IRQ_TYPE_EDGE_BOTH)
return dwapb_gpio_set_debounce(gc, offset, debounce);
}
-static irqreturn_t dwapb_irq_handler_mfd(int irq, void *dev_id)
+static int dwapb_convert_irqs(struct dwapb_gpio_port_irqchip *pirq,
+ struct dwapb_port_property *pp)
{
- return IRQ_RETVAL(dwapb_do_irq(dev_id));
+ int i;
+
+ /* Group all available IRQs into an array of parental IRQs. */
+ for (i = 0; i < pp->ngpio; ++i) {
+ if (!pp->irq[i])
+ continue;
+
+ pirq->irq[pirq->nr_irqs++] = pp->irq[i];
+ }
+
+ return pirq->nr_irqs ? 0 : -ENOENT;
}
static void dwapb_configure_irqs(struct dwapb_gpio *gpio,
struct dwapb_gpio_port *port,
struct dwapb_port_property *pp)
{
+ struct dwapb_gpio_port_irqchip *pirq;
struct gpio_chip *gc = &port->gc;
- struct fwnode_handle *fwnode = pp->fwnode;
- struct irq_chip_generic *irq_gc = NULL;
- unsigned int ngpio = gc->ngpio;
- struct irq_chip_type *ct;
- irq_hw_number_t hwirq;
- int err, i;
-
- if (memchr_inv(pp->irq, 0, sizeof(pp->irq)) == NULL) {
- dev_warn(gpio->dev, "no IRQ for port%d\n", pp->idx);
- return;
- }
-
- gpio->domain = irq_domain_create_linear(fwnode, ngpio,
- &irq_generic_chip_ops, gpio);
- if (!gpio->domain)
- return;
+ struct gpio_irq_chip *girq;
+ int err;
- err = irq_alloc_domain_generic_chips(gpio->domain, ngpio, 2,
- DWAPB_DRIVER_NAME, handle_bad_irq,
- IRQ_NOREQUEST, 0,
- IRQ_GC_INIT_NESTED_LOCK);
- if (err) {
- dev_info(gpio->dev, "irq_alloc_domain_generic_chips failed\n");
- irq_domain_remove(gpio->domain);
- gpio->domain = NULL;
+ pirq = devm_kzalloc(gpio->dev, sizeof(*pirq), GFP_KERNEL);
+ if (!pirq)
return;
- }
- irq_gc = irq_get_domain_generic_chip(gpio->domain, 0);
- if (!irq_gc) {
- irq_domain_remove(gpio->domain);
- gpio->domain = NULL;
- return;
+ if (dwapb_convert_irqs(pirq, pp)) {
+ dev_warn(gpio->dev, "no IRQ for port%d\n", pp->idx);
+ goto err_kfree_pirq;
}
- irq_gc->reg_base = gpio->regs;
- irq_gc->private = gpio;
-
- for (i = 0; i < 2; i++) {
- ct = &irq_gc->chip_types[i];
- ct->chip.irq_ack = irq_gc_ack_set_bit;
- ct->chip.irq_mask = irq_gc_mask_set_bit;
- ct->chip.irq_unmask = irq_gc_mask_clr_bit;
- ct->chip.irq_set_type = dwapb_irq_set_type;
- ct->chip.irq_enable = dwapb_irq_enable;
- ct->chip.irq_disable = dwapb_irq_disable;
+ girq = &gc->irq;
+ girq->handler = handle_bad_irq;
+ girq->default_type = IRQ_TYPE_NONE;
+
+ port->pirq = pirq;
+ pirq->irqchip.name = DWAPB_DRIVER_NAME;
+ pirq->irqchip.irq_ack = dwapb_irq_ack;
+ pirq->irqchip.irq_mask = dwapb_irq_mask;
+ pirq->irqchip.irq_unmask = dwapb_irq_unmask;
+ pirq->irqchip.irq_set_type = dwapb_irq_set_type;
+ pirq->irqchip.irq_enable = dwapb_irq_enable;
+ pirq->irqchip.irq_disable = dwapb_irq_disable;
#ifdef CONFIG_PM_SLEEP
- ct->chip.irq_set_wake = dwapb_irq_set_wake;
+ pirq->irqchip.irq_set_wake = dwapb_irq_set_wake;
#endif
- ct->regs.ack = gpio_reg_convert(gpio, GPIO_PORTA_EOI);
- ct->regs.mask = gpio_reg_convert(gpio, GPIO_INTMASK);
- ct->type = IRQ_TYPE_LEVEL_MASK;
- }
-
- irq_gc->chip_types[0].type = IRQ_TYPE_LEVEL_MASK;
- irq_gc->chip_types[0].handler = handle_level_irq;
- irq_gc->chip_types[1].type = IRQ_TYPE_EDGE_BOTH;
- irq_gc->chip_types[1].handler = handle_edge_irq;
if (!pp->irq_shared) {
- int i;
-
- for (i = 0; i < pp->ngpio; i++) {
- if (pp->irq[i])
- irq_set_chained_handler_and_data(pp->irq[i],
- dwapb_irq_handler, gpio);
- }
+ girq->num_parents = pirq->nr_irqs;
+ girq->parents = pirq->irq;
+ girq->parent_handler_data = gpio;
+ girq->parent_handler = dwapb_irq_handler;
} else {
+ /* This will let us handle the parent IRQ in the driver */
+ girq->num_parents = 0;
+ girq->parents = NULL;
+ girq->parent_handler = NULL;
+
/*
* Request a shared IRQ since where MFD would have devices
* using the same irq pin
IRQF_SHARED, DWAPB_DRIVER_NAME, gpio);
if (err) {
dev_err(gpio->dev, "error requesting IRQ\n");
- irq_domain_remove(gpio->domain);
- gpio->domain = NULL;
- return;
+ goto err_kfree_pirq;
}
}
- for (hwirq = 0; hwirq < ngpio; hwirq++)
- irq_create_mapping(gpio->domain, hwirq);
-
- port->gc.to_irq = dwapb_gpio_to_irq;
-}
-
-static void dwapb_irq_teardown(struct dwapb_gpio *gpio)
-{
- struct dwapb_gpio_port *port = &gpio->ports[0];
- struct gpio_chip *gc = &port->gc;
- unsigned int ngpio = gc->ngpio;
- irq_hw_number_t hwirq;
-
- if (!gpio->domain)
- return;
+ girq->chip = &pirq->irqchip;
- for (hwirq = 0; hwirq < ngpio; hwirq++)
- irq_dispose_mapping(irq_find_mapping(gpio->domain, hwirq));
+ return;
- irq_domain_remove(gpio->domain);
- gpio->domain = NULL;
+err_kfree_pirq:
+ devm_kfree(gpio->dev, pirq);
}
static int dwapb_gpio_add_port(struct dwapb_gpio *gpio,
if (pp->idx == 0)
dwapb_configure_irqs(gpio, port, pp);
- err = gpiochip_add_data(&port->gc, port);
+ err = devm_gpiochip_add_data(gpio->dev, &port->gc, port);
if (err) {
dev_err(gpio->dev, "failed to register gpiochip for port%d\n",
port->idx);
return err;
}
- /* Add GPIO-signaled ACPI event support */
- acpi_gpiochip_request_interrupts(&port->gc);
-
- port->is_registered = true;
-
return 0;
}
-static void dwapb_gpio_unregister(struct dwapb_gpio *gpio)
-{
- unsigned int m;
-
- for (m = 0; m < gpio->nr_ports; ++m) {
- struct dwapb_gpio_port *port = &gpio->ports[m];
-
- if (!port->is_registered)
- continue;
-
- acpi_gpiochip_free_interrupts(&port->gc);
- gpiochip_remove(&port->gc);
- }
-}
-
static void dwapb_get_irq(struct device *dev, struct fwnode_handle *fwnode,
struct dwapb_port_property *pp)
{
return ERR_PTR(-EINVAL);
}
- if (fwnode_property_read_u32(fwnode, "snps,nr-gpios", &pp->ngpio)) {
+ if (fwnode_property_read_u32(fwnode, "ngpios", &pp->ngpio) &&
+ fwnode_property_read_u32(fwnode, "snps,nr-gpios", &pp->ngpio)) {
dev_info(dev,
"failed to get number of gpios for port%d\n",
i);
- pp->ngpio = 32;
+ pp->ngpio = DWAPB_MAX_GPIOS;
}
pp->irq_shared = false;
return pdata;
}
+static void dwapb_assert_reset(void *data)
+{
+ struct dwapb_gpio *gpio = data;
+
+ reset_control_assert(gpio->rst);
+}
+
+static int dwapb_get_reset(struct dwapb_gpio *gpio)
+{
+ int err;
+
+ gpio->rst = devm_reset_control_get_optional_shared(gpio->dev, NULL);
+ if (IS_ERR(gpio->rst)) {
+ dev_err(gpio->dev, "Cannot get reset descriptor\n");
+ return PTR_ERR(gpio->rst);
+ }
+
+ err = reset_control_deassert(gpio->rst);
+ if (err) {
+ dev_err(gpio->dev, "Cannot deassert reset lane\n");
+ return err;
+ }
+
+ return devm_add_action_or_reset(gpio->dev, dwapb_assert_reset, gpio);
+}
+
+static void dwapb_disable_clks(void *data)
+{
+ struct dwapb_gpio *gpio = data;
+
+ clk_bulk_disable_unprepare(DWAPB_NR_CLOCKS, gpio->clks);
+}
+
+static int dwapb_get_clks(struct dwapb_gpio *gpio)
+{
+ int err;
+
+ /* Optional bus and debounce clocks */
+ gpio->clks[0].id = "bus";
+ gpio->clks[1].id = "db";
+ err = devm_clk_bulk_get_optional(gpio->dev, DWAPB_NR_CLOCKS,
+ gpio->clks);
+ if (err) {
+ dev_err(gpio->dev, "Cannot get APB/Debounce clocks\n");
+ return err;
+ }
+
+ err = clk_bulk_prepare_enable(DWAPB_NR_CLOCKS, gpio->clks);
+ if (err) {
+ dev_err(gpio->dev, "Cannot enable APB/Debounce clocks\n");
+ return err;
+ }
+
+ return devm_add_action_or_reset(gpio->dev, dwapb_disable_clks, gpio);
+}
+
static const struct of_device_id dwapb_of_match[] = {
{ .compatible = "snps,dw-apb-gpio", .data = (void *)0},
{ .compatible = "apm,xgene-gpio-v2", .data = (void *)GPIO_REG_OFFSET_V2},
gpio->dev = &pdev->dev;
gpio->nr_ports = pdata->nports;
- gpio->rst = devm_reset_control_get_optional_shared(dev, NULL);
- if (IS_ERR(gpio->rst))
- return PTR_ERR(gpio->rst);
-
- reset_control_deassert(gpio->rst);
+ err = dwapb_get_reset(gpio);
+ if (err)
+ return err;
gpio->ports = devm_kcalloc(&pdev->dev, gpio->nr_ports,
sizeof(*gpio->ports), GFP_KERNEL);
if (IS_ERR(gpio->regs))
return PTR_ERR(gpio->regs);
- /* Optional bus and debounce clocks */
- gpio->clks[0].id = "bus";
- gpio->clks[1].id = "db";
- err = devm_clk_bulk_get_optional(&pdev->dev, DWAPB_NR_CLOCKS,
- gpio->clks);
- if (err) {
- dev_err(&pdev->dev, "Cannot get APB/Debounce clocks\n");
- return err;
- }
-
- err = clk_bulk_prepare_enable(DWAPB_NR_CLOCKS, gpio->clks);
- if (err) {
- dev_err(&pdev->dev, "Cannot enable APB/Debounce clocks\n");
+ err = dwapb_get_clks(gpio);
+ if (err)
return err;
- }
gpio->flags = (uintptr_t)device_get_match_data(dev);
for (i = 0; i < gpio->nr_ports; i++) {
err = dwapb_gpio_add_port(gpio, &pdata->properties[i], i);
if (err)
- goto out_unregister;
+ return err;
}
- platform_set_drvdata(pdev, gpio);
-
- return 0;
-
-out_unregister:
- dwapb_gpio_unregister(gpio);
- dwapb_irq_teardown(gpio);
- clk_bulk_disable_unprepare(DWAPB_NR_CLOCKS, gpio->clks);
-
- return err;
-}
-
-static int dwapb_gpio_remove(struct platform_device *pdev)
-{
- struct dwapb_gpio *gpio = platform_get_drvdata(pdev);
-
- dwapb_gpio_unregister(gpio);
- dwapb_irq_teardown(gpio);
- reset_control_assert(gpio->rst);
- clk_bulk_disable_unprepare(DWAPB_NR_CLOCKS, gpio->clks);
return 0;
}
.acpi_match_table = dwapb_acpi_match,
},
.probe = dwapb_gpio_probe,
- .remove = dwapb_gpio_remove,
};
module_platform_driver(dwapb_gpio_driver);
* Copyright (C) 2017 Bartosz Golaszewski <brgl@bgdev.pl>
*/
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
#include <linux/debugfs.h>
-#include <linux/gpio/consumer.h>
#include <linux/gpio/driver.h>
-#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/irq_sim.h>
#include <linux/platform_device.h>
#include <linux/property.h>
#include <linux/slab.h>
+#include <linux/string_helpers.h>
#include <linux/uaccess.h>
#include "gpiolib.h"
-#define GPIO_MOCKUP_NAME "gpio-mockup"
#define GPIO_MOCKUP_MAX_GC 10
/*
* We're storing two values per chip: the GPIO base and the number
* of GPIO lines.
*/
#define GPIO_MOCKUP_MAX_RANGES (GPIO_MOCKUP_MAX_GC * 2)
-/* Maximum of three properties + the sentinel. */
-#define GPIO_MOCKUP_MAX_PROP 4
-
-#define gpio_mockup_err(...) pr_err(GPIO_MOCKUP_NAME ": " __VA_ARGS__)
+/* Maximum of four properties + the sentinel. */
+#define GPIO_MOCKUP_MAX_PROP 5
/*
* struct gpio_pin_status - structure describing a GPIO status
debugfs_create_file(name, 0200, chip->dbg_dir, priv,
&gpio_mockup_debugfs_ops);
}
-
- return;
-}
-
-static int gpio_mockup_name_lines(struct device *dev,
- struct gpio_mockup_chip *chip)
-{
- struct gpio_chip *gc = &chip->gc;
- char **names;
- int i;
-
- names = devm_kcalloc(dev, gc->ngpio, sizeof(char *), GFP_KERNEL);
- if (!names)
- return -ENOMEM;
-
- for (i = 0; i < gc->ngpio; i++) {
- names[i] = devm_kasprintf(dev, GFP_KERNEL,
- "%s-%d", gc->label, i);
- if (!names[i])
- return -ENOMEM;
- }
-
- gc->names = (const char *const *)names;
-
- return 0;
}
static void gpio_mockup_dispose_mappings(void *data)
if (rv)
return rv;
- rv = device_property_read_string(dev, "chip-name", &name);
+ rv = device_property_read_string(dev, "chip-label", &name);
if (rv)
- name = NULL;
+ name = dev_name(dev);
chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
if (!chip)
return -ENOMEM;
- if (!name) {
- name = devm_kasprintf(dev, GFP_KERNEL,
- "%s-%c", pdev->name, pdev->id + 'A');
- if (!name)
- return -ENOMEM;
- }
-
mutex_init(&chip->lock);
gc = &chip->gc;
for (i = 0; i < gc->ngpio; i++)
chip->lines[i].dir = GPIO_LINE_DIRECTION_IN;
- if (device_property_read_bool(dev, "named-gpio-lines")) {
- rv = gpio_mockup_name_lines(dev, chip);
- if (rv)
- return rv;
- }
-
chip->irq_sim_domain = devm_irq_domain_create_sim(dev, NULL,
gc->ngpio);
if (IS_ERR(chip->irq_sim_domain))
static struct platform_driver gpio_mockup_driver = {
.driver = {
- .name = GPIO_MOCKUP_NAME,
+ .name = "gpio-mockup",
},
.probe = gpio_mockup_probe,
};
}
}
-static int __init gpio_mockup_init(void)
+static __init char **gpio_mockup_make_line_names(const char *label,
+ unsigned int num_lines)
+{
+ unsigned int i;
+ char **names;
+
+ names = kcalloc(num_lines + 1, sizeof(char *), GFP_KERNEL);
+ if (!names)
+ return NULL;
+
+ for (i = 0; i < num_lines; i++) {
+ names[i] = kasprintf(GFP_KERNEL, "%s-%u", label, i);
+ if (!names[i]) {
+ kfree_strarray(names, i);
+ return NULL;
+ }
+ }
+
+ return names;
+}
+
+static int __init gpio_mockup_register_chip(int idx)
{
struct property_entry properties[GPIO_MOCKUP_MAX_PROP];
- int i, prop, num_chips, err = 0, base;
struct platform_device_info pdevinfo;
struct platform_device *pdev;
+ char **line_names = NULL;
+ char chip_label[32];
+ int prop = 0, base;
u16 ngpio;
+ memset(properties, 0, sizeof(properties));
+ memset(&pdevinfo, 0, sizeof(pdevinfo));
+
+ snprintf(chip_label, sizeof(chip_label), "gpio-mockup-%c", idx + 'A');
+ properties[prop++] = PROPERTY_ENTRY_STRING("chip-label", chip_label);
+
+ base = gpio_mockup_range_base(idx);
+ if (base >= 0)
+ properties[prop++] = PROPERTY_ENTRY_U32("gpio-base", base);
+
+ ngpio = base < 0 ? gpio_mockup_range_ngpio(idx)
+ : gpio_mockup_range_ngpio(idx) - base;
+ properties[prop++] = PROPERTY_ENTRY_U16("nr-gpios", ngpio);
+
+ if (gpio_mockup_named_lines) {
+ line_names = gpio_mockup_make_line_names(chip_label, ngpio);
+ if (!line_names)
+ return -ENOMEM;
+
+ properties[prop++] = PROPERTY_ENTRY_STRING_ARRAY_LEN(
+ "gpio-line-names", line_names, ngpio);
+ }
+
+ pdevinfo.name = "gpio-mockup";
+ pdevinfo.id = idx;
+ pdevinfo.properties = properties;
+
+ pdev = platform_device_register_full(&pdevinfo);
+ kfree_strarray(line_names, ngpio);
+ if (IS_ERR(pdev)) {
+ pr_err("error registering device");
+ return PTR_ERR(pdev);
+ }
+
+ gpio_mockup_pdevs[idx] = pdev;
+
+ return 0;
+}
+
+static int __init gpio_mockup_init(void)
+{
+ int i, num_chips, err;
+
if ((gpio_mockup_num_ranges < 2) ||
(gpio_mockup_num_ranges % 2) ||
(gpio_mockup_num_ranges > GPIO_MOCKUP_MAX_RANGES))
err = platform_driver_register(&gpio_mockup_driver);
if (err) {
- gpio_mockup_err("error registering platform driver\n");
+ pr_err("error registering platform driver\n");
+ debugfs_remove_recursive(gpio_mockup_dbg_dir);
return err;
}
for (i = 0; i < num_chips; i++) {
- memset(properties, 0, sizeof(properties));
- memset(&pdevinfo, 0, sizeof(pdevinfo));
- prop = 0;
-
- base = gpio_mockup_range_base(i);
- if (base >= 0)
- properties[prop++] = PROPERTY_ENTRY_U32("gpio-base",
- base);
-
- ngpio = base < 0 ? gpio_mockup_range_ngpio(i)
- : gpio_mockup_range_ngpio(i) - base;
- properties[prop++] = PROPERTY_ENTRY_U16("nr-gpios", ngpio);
-
- if (gpio_mockup_named_lines)
- properties[prop++] = PROPERTY_ENTRY_BOOL(
- "named-gpio-lines");
-
- pdevinfo.name = GPIO_MOCKUP_NAME;
- pdevinfo.id = i;
- pdevinfo.properties = properties;
-
- pdev = platform_device_register_full(&pdevinfo);
- if (IS_ERR(pdev)) {
- gpio_mockup_err("error registering device");
+ err = gpio_mockup_register_chip(i);
+ if (err) {
platform_driver_unregister(&gpio_mockup_driver);
gpio_mockup_unregister_pdevs();
- return PTR_ERR(pdev);
+ debugfs_remove_recursive(gpio_mockup_dbg_dir);
+ return err;
}
-
- gpio_mockup_pdevs[i] = pdev;
}
return 0;
unsigned int irqn;
};
-/* The GPIO Input Buffer Enable register(GPIO_IBE) is used to
- * control the input enable of each individual GPIO port.
- * When an individual GPIO port’s direction is set to
- * input (GPIO_GPDIR[DRn=0]), the associated input enable must be
- * set (GPIOxGPIE[IEn]=1) to propagate the port value to the GPIO
- * Data Register.
- */
-static int ls1028a_gpio_dir_in_init(struct gpio_chip *gc)
-{
- unsigned long flags;
- struct mpc8xxx_gpio_chip *mpc8xxx_gc = gpiochip_get_data(gc);
-
- spin_lock_irqsave(&gc->bgpio_lock, flags);
-
- gc->write_reg(mpc8xxx_gc->regs + GPIO_IBE, 0xffffffff);
-
- spin_unlock_irqrestore(&gc->bgpio_lock, flags);
-
- return 0;
-}
-
/*
* This hardware has a big endian bit assignment such that GPIO line 0 is
* connected to bit 31, line 1 to bit 30 ... line 31 to bit 0.
};
struct mpc8xxx_gpio_devtype {
- int (*gpio_dir_in_init)(struct gpio_chip *chip);
int (*gpio_dir_out)(struct gpio_chip *, unsigned int, int);
int (*gpio_get)(struct gpio_chip *, unsigned int);
int (*irq_set_type)(struct irq_data *, unsigned int);
.irq_set_type = mpc512x_irq_set_type,
};
-static const struct mpc8xxx_gpio_devtype ls1028a_gpio_devtype = {
- .gpio_dir_in_init = ls1028a_gpio_dir_in_init,
- .irq_set_type = mpc8xxx_irq_set_type,
-};
-
static const struct mpc8xxx_gpio_devtype mpc5125_gpio_devtype = {
.gpio_dir_out = mpc5125_gpio_dir_out,
.irq_set_type = mpc512x_irq_set_type,
{ .compatible = "fsl,mpc5121-gpio", .data = &mpc512x_gpio_devtype, },
{ .compatible = "fsl,mpc5125-gpio", .data = &mpc5125_gpio_devtype, },
{ .compatible = "fsl,pq3-gpio", },
- { .compatible = "fsl,ls1028a-gpio", .data = &ls1028a_gpio_devtype, },
- { .compatible = "fsl,ls1088a-gpio", .data = &ls1028a_gpio_devtype, },
+ { .compatible = "fsl,ls1028a-gpio", },
+ { .compatible = "fsl,ls1088a-gpio", },
{ .compatible = "fsl,qoriq-gpio", },
{}
};
gc->to_irq = mpc8xxx_gpio_to_irq;
- if (of_device_is_compatible(np, "fsl,qoriq-gpio"))
+ /*
+ * The GPIO Input Buffer Enable register(GPIO_IBE) is used to control
+ * the input enable of each individual GPIO port. When an individual
+ * GPIO port’s direction is set to input (GPIO_GPDIR[DRn=0]), the
+ * associated input enable must be set (GPIOxGPIE[IEn]=1) to propagate
+ * the port value to the GPIO Data Register.
+ */
+ if (of_device_is_compatible(np, "fsl,qoriq-gpio") ||
+ of_device_is_compatible(np, "fsl,ls1028a-gpio") ||
+ of_device_is_compatible(np, "fsl,ls1088a-gpio"))
gc->write_reg(mpc8xxx_gc->regs + GPIO_IBE, 0xffffffff);
ret = gpiochip_add_data(gc, mpc8xxx_gc);
/* ack and mask all irqs */
gc->write_reg(mpc8xxx_gc->regs + GPIO_IER, 0xffffffff);
gc->write_reg(mpc8xxx_gc->regs + GPIO_IMR, 0);
- /* enable input buffer */
- if (devtype->gpio_dir_in_init)
- devtype->gpio_dir_in_init(gc);
ret = devm_request_irq(&pdev->dev, mpc8xxx_gc->irqn,
mpc8xxx_gpio_irq_cascade,
#include <linux/irq.h>
#include <linux/irqdomain.h>
#include <linux/irqchip/chained_irq.h>
+#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/syscore_ops.h>
{ .compatible = "fsl,imx7d-gpio", .data = &mxc_gpio_devtype[IMX35_GPIO], },
{ /* sentinel */ }
};
+MODULE_DEVICE_TABLE(of, mxc_gpio_dt_ids);
/*
* MX2 has one interrupt *for all* gpio ports. The list is used
return platform_driver_register(&mxc_gpio_driver);
}
subsys_initcall(gpio_mxc_init);
+
+MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>");
+MODULE_DESCRIPTION("i.MX GPIO Driver");
+MODULE_LICENSE("GPL");
if (bank->irq <= 0) {
if (!bank->irq)
bank->irq = -ENXIO;
- if (bank->irq != -EPROBE_DEFER)
- dev_err(dev,
- "can't get irq resource ret=%d\n", bank->irq);
- return bank->irq;
+ return dev_err_probe(dev, bank->irq, "can't get irq resource\n");
}
bank->chip.parent = dev;
return 0;
}
-static int omap_gpio_suspend(struct device *dev)
+static int __maybe_unused omap_gpio_suspend(struct device *dev)
{
struct gpio_bank *bank = dev_get_drvdata(dev);
return omap_gpio_runtime_suspend(dev);
}
-static int omap_gpio_resume(struct device *dev)
+static int __maybe_unused omap_gpio_resume(struct device *dev)
{
struct gpio_bank *bank = dev_get_drvdata(dev);
{ "pcal6416", 16 | PCA953X_TYPE | PCA_LATCH_INT, },
{ "pcal6524", 24 | PCA953X_TYPE | PCA_LATCH_INT, },
{ "pcal9535", 16 | PCA953X_TYPE | PCA_LATCH_INT, },
+ { "pcal9554b", 8 | PCA953X_TYPE | PCA_LATCH_INT, },
{ "pcal9555a", 16 | PCA953X_TYPE | PCA_LATCH_INT, },
{ "max7310", 8 | PCA953X_TYPE, },
int level;
bool ret;
+ bitmap_zero(pending, MAX_LINE);
+
mutex_lock(&chip->i2c_lock);
ret = pca953x_irq_pending(chip, pending);
mutex_unlock(&chip->i2c_lock);
- for_each_set_bit(level, pending, gc->ngpio)
- handle_nested_irq(irq_find_mapping(gc->irq.domain, level));
+ if (ret) {
+ ret = 0;
+
+ for_each_set_bit(level, pending, gc->ngpio) {
+ int nested_irq = irq_find_mapping(gc->irq.domain, level);
+
+ if (unlikely(nested_irq <= 0)) {
+ dev_warn_ratelimited(gc->parent, "unmapped interrupt %d\n", level);
+ continue;
+ }
+
+ handle_nested_irq(nested_irq);
+ ret = 1;
+ }
+ }
return IRQ_RETVAL(ret);
}
static int device_pca957x_init(struct pca953x_chip *chip, u32 invert)
{
DECLARE_BITMAP(val, MAX_LINE);
+ unsigned int i;
int ret;
ret = device_pca95xx_init(chip, invert);
goto out;
/* To enable register 6, 7 to control pull up and pull down */
- memset(val, 0x02, NBANK(chip));
+ for (i = 0; i < NBANK(chip); i++)
+ bitmap_set_value8(val, 0x02, i * BANK_SZ);
+
ret = pca953x_write_regs(chip, PCA957X_BKEN, val);
if (ret)
goto out;
chip->client = client;
reg = devm_regulator_get(&client->dev, "vcc");
- if (IS_ERR(reg)) {
- ret = PTR_ERR(reg);
- if (ret != -EPROBE_DEFER)
- dev_err(&client->dev, "reg get err: %d\n", ret);
- return ret;
- }
+ if (IS_ERR(reg))
+ return dev_err_probe(&client->dev, PTR_ERR(reg), "reg get err\n");
+
ret = regulator_enable(reg);
if (ret) {
dev_err(&client->dev, "reg en err: %d\n", ret);
{ .compatible = "nxp,pcal6416", .data = OF_953X(16, PCA_LATCH_INT), },
{ .compatible = "nxp,pcal6524", .data = OF_953X(24, PCA_LATCH_INT), },
{ .compatible = "nxp,pcal9535", .data = OF_953X(16, PCA_LATCH_INT), },
+ { .compatible = "nxp,pcal9554b", .data = OF_953X( 8, PCA_LATCH_INT), },
{ .compatible = "nxp,pcal9555a", .data = OF_953X(16, PCA_LATCH_INT), },
{ .compatible = "maxim,max7310", .data = OF_953X( 8, 0), },
return -ENOMEM;
gpio->load_gpio = devm_gpiod_get_optional(dev, "load", GPIOD_OUT_LOW);
- if (IS_ERR(gpio->load_gpio)) {
- ret = PTR_ERR(gpio->load_gpio);
- if (ret != -EPROBE_DEFER)
- dev_err(dev, "Unable to allocate load GPIO\n");
- return ret;
- }
+ if (IS_ERR(gpio->load_gpio))
+ return dev_err_probe(dev, PTR_ERR(gpio->load_gpio),
+ "Unable to allocate load GPIO\n");
mutex_init(&gpio->lock);
girq->chip = &ddata->ichip;
girq->default_type = IRQ_TYPE_NONE;
girq->handler = handle_level_irq;
+ girq->threaded = true;
ret = devm_gpiochip_add_data(dev, &ddata->gchip, NULL);
if (ret)
sprd_gpio_update(chip, offset, SPRD_GPIO_IS, 0);
sprd_gpio_update(chip, offset, SPRD_GPIO_IBE, 0);
sprd_gpio_update(chip, offset, SPRD_GPIO_IEV, 1);
+ sprd_gpio_update(chip, offset, SPRD_GPIO_IC, 1);
irq_set_handler_locked(data, handle_edge_irq);
break;
case IRQ_TYPE_EDGE_FALLING:
sprd_gpio_update(chip, offset, SPRD_GPIO_IS, 0);
sprd_gpio_update(chip, offset, SPRD_GPIO_IBE, 0);
sprd_gpio_update(chip, offset, SPRD_GPIO_IEV, 0);
+ sprd_gpio_update(chip, offset, SPRD_GPIO_IC, 1);
irq_set_handler_locked(data, handle_edge_irq);
break;
case IRQ_TYPE_EDGE_BOTH:
sprd_gpio_update(chip, offset, SPRD_GPIO_IS, 0);
sprd_gpio_update(chip, offset, SPRD_GPIO_IBE, 1);
+ sprd_gpio_update(chip, offset, SPRD_GPIO_IC, 1);
irq_set_handler_locked(data, handle_edge_irq);
break;
case IRQ_TYPE_LEVEL_HIGH:
#define XWAY_STP_4HZ BIT(23)
#define XWAY_STP_8HZ BIT(24)
#define XWAY_STP_10HZ (BIT(24) | BIT(23))
-#define XWAY_STP_SPEED_MASK (0xf << 23)
+#define XWAY_STP_SPEED_MASK (BIT(23) | BIT(24) | BIT(25) | BIT(26) | BIT(27))
+
+#define XWAY_STP_FPIS_VALUE BIT(21)
+#define XWAY_STP_FPIS_MASK (BIT(20) | BIT(21))
/* clock source for automatic update */
#define XWAY_STP_UPD_FPI BIT(31)
/* 2 groups of 3 bits can be driven by the phys */
#define XWAY_STP_PHY_MASK 0x7
#define XWAY_STP_PHY1_SHIFT 27
-#define XWAY_STP_PHY2_SHIFT 15
+#define XWAY_STP_PHY2_SHIFT 3
+#define XWAY_STP_PHY3_SHIFT 6
+#define XWAY_STP_PHY4_SHIFT 15
/* STP has 3 groups of 8 bits */
#define XWAY_STP_GROUP0 BIT(0)
u8 dsl; /* the 2 LSBs can be driven by the dsl core */
u8 phy1; /* 3 bits can be driven by phy1 */
u8 phy2; /* 3 bits can be driven by phy2 */
+ u8 phy3; /* 3 bits can be driven by phy3 */
+ u8 phy4; /* 3 bits can be driven by phy4 */
u8 reserved; /* mask out the hw driven bits in gpio_request */
};
else
chip->shadow &= ~BIT(gpio);
xway_stp_w32(chip->virt, chip->shadow, XWAY_STP_CPU0);
- xway_stp_w32_mask(chip->virt, 0, XWAY_STP_CON_SWU, XWAY_STP_CON0);
+ if (!chip->reserved)
+ xway_stp_w32_mask(chip->virt, 0, XWAY_STP_CON_SWU, XWAY_STP_CON0);
}
/**
chip->phy2 << XWAY_STP_PHY2_SHIFT,
XWAY_STP_CON1);
+ if (of_machine_is_compatible("lantiq,grx390")
+ || of_machine_is_compatible("lantiq,ar10")) {
+ xway_stp_w32_mask(chip->virt,
+ XWAY_STP_PHY_MASK << XWAY_STP_PHY3_SHIFT,
+ chip->phy3 << XWAY_STP_PHY3_SHIFT,
+ XWAY_STP_CON1);
+ }
+
+ if (of_machine_is_compatible("lantiq,grx390")) {
+ xway_stp_w32_mask(chip->virt,
+ XWAY_STP_PHY_MASK << XWAY_STP_PHY4_SHIFT,
+ chip->phy4 << XWAY_STP_PHY4_SHIFT,
+ XWAY_STP_CON1);
+ }
+
/* mask out the hw driven bits in gpio_request */
- chip->reserved = (chip->phy2 << 5) | (chip->phy1 << 2) | chip->dsl;
+ chip->reserved = (chip->phy4 << 11) | (chip->phy3 << 8) | (chip->phy2 << 5)
+ | (chip->phy1 << 2) | chip->dsl;
/*
* if we have pins that are driven by hw, we need to tell the stp what
* clock to use as a timer.
*/
- if (chip->reserved)
+ if (chip->reserved) {
xway_stp_w32_mask(chip->virt, XWAY_STP_UPD_MASK,
XWAY_STP_UPD_FPI, XWAY_STP_CON1);
+ xway_stp_w32_mask(chip->virt, XWAY_STP_SPEED_MASK,
+ XWAY_STP_10HZ, XWAY_STP_CON1);
+ xway_stp_w32_mask(chip->virt, XWAY_STP_FPIS_MASK,
+ XWAY_STP_FPIS_VALUE, XWAY_STP_CON1);
+ }
}
static int xway_stp_probe(struct platform_device *pdev)
/* find out which gpios are controlled by the phys */
if (of_machine_is_compatible("lantiq,ar9") ||
of_machine_is_compatible("lantiq,gr9") ||
- of_machine_is_compatible("lantiq,vr9")) {
+ of_machine_is_compatible("lantiq,vr9") ||
+ of_machine_is_compatible("lantiq,ar10") ||
+ of_machine_is_compatible("lantiq,grx390")) {
if (!of_property_read_u32(pdev->dev.of_node, "lantiq,phy1", &phy))
chip->phy1 = phy & XWAY_STP_PHY_MASK;
if (!of_property_read_u32(pdev->dev.of_node, "lantiq,phy2", &phy))
chip->phy2 = phy & XWAY_STP_PHY_MASK;
}
+ if (of_machine_is_compatible("lantiq,ar10") ||
+ of_machine_is_compatible("lantiq,grx390")) {
+ if (!of_property_read_u32(pdev->dev.of_node, "lantiq,phy3", &phy))
+ chip->phy3 = phy & XWAY_STP_PHY_MASK;
+ }
+
+ if (of_machine_is_compatible("lantiq,grx390")) {
+ if (!of_property_read_u32(pdev->dev.of_node, "lantiq,phy4", &phy))
+ chip->phy4 = phy & XWAY_STP_PHY_MASK;
+ }
+
/* check which edge trigger we should use, default to a falling edge */
if (!of_find_property(pdev->dev.of_node, "lantiq,rising", NULL))
chip->edge = XWAY_STP_FALLING;
* These registers are modified under the irq bus lock and cached to avoid
* unnecessary writes in bus_sync_unlock.
*/
-enum { REG_IBE, REG_IEV, REG_IS, REG_IE };
+enum { REG_IBE, REG_IEV, REG_IS, REG_IE, REG_DIRECT };
-#define CACHE_NR_REGS 4
+#define CACHE_NR_REGS 5
#define CACHE_NR_BANKS 3
struct tc3589x_gpio {
[REG_IEV] = TC3589x_GPIOIEV0,
[REG_IS] = TC3589x_GPIOIS0,
[REG_IE] = TC3589x_GPIOIE0,
+ [REG_DIRECT] = TC3589x_DIRECT0,
};
int i, j;
continue;
tc3589x_gpio->oldregs[i][j] = new;
- tc3589x_reg_write(tc3589x, regmap[i] + j * 8, new);
+ tc3589x_reg_write(tc3589x, regmap[i] + j, new);
}
}
int mask = BIT(offset % 8);
tc3589x_gpio->regs[REG_IE][regoffset] &= ~mask;
+ tc3589x_gpio->regs[REG_DIRECT][regoffset] |= mask;
}
static void tc3589x_gpio_irq_unmask(struct irq_data *d)
int mask = BIT(offset % 8);
tc3589x_gpio->regs[REG_IE][regoffset] |= mask;
+ tc3589x_gpio->regs[REG_DIRECT][regoffset] &= ~mask;
}
static struct irq_chip tc3589x_gpio_irq_chip = {
if (ret < 0)
return ret;
+ /* For tc35894, have to disable Direct KBD interrupts,
+ * else IRQST will always be 0x20, IRQN low level, can't
+ * clear the irq status.
+ * TODO: need more test on other tc3589x chip.
+ *
+ */
+ ret = tc3589x_reg_write(tc3589x, TC3589x_DKBDMSK,
+ TC3589x_DKBDMSK_ELINT | TC3589x_DKBDMSK_EINT);
+ if (ret < 0)
+ return ret;
+
ret = devm_request_threaded_irq(&pdev->dev,
irq, NULL, tc3589x_gpio_irq,
IRQF_ONESHOT, "tc3589x-gpio",
else
irq_set_handler_locked(data, handle_edge_irq);
- return irq_chip_set_type_parent(data, type);
+ if (data->parent_data)
+ return irq_chip_set_type_parent(data, type);
+
+ return 0;
+}
+
+static int tegra186_irq_set_wake(struct irq_data *data, unsigned int on)
+{
+ if (data->parent_data)
+ return irq_chip_set_wake_parent(data, on);
+
+ return 0;
}
static void tegra186_gpio_irq(struct irq_desc *desc)
gpio->intc.irq_mask = tegra186_irq_mask;
gpio->intc.irq_unmask = tegra186_irq_unmask;
gpio->intc.irq_set_type = tegra186_irq_set_type;
- gpio->intc.irq_set_wake = irq_chip_set_wake_parent;
+ gpio->intc.irq_set_wake = tegra186_irq_set_wake;
irq = &gpio->gpio.irq;
irq->chip = &gpio->intc;
/* Retrieve GPIO clock */
gpio->clk = devm_clk_get(&pdev->dev, NULL);
- if (IS_ERR(gpio->clk)) {
- if (PTR_ERR(gpio->clk) != -EPROBE_DEFER)
- dev_err(&pdev->dev, "input clock not found.\n");
- return PTR_ERR(gpio->clk);
- }
+ if (IS_ERR(gpio->clk))
+ return dev_err_probe(&pdev->dev, PTR_ERR(gpio->clk), "input clock not found.\n");
+
ret = clk_prepare_enable(gpio->clk);
if (ret) {
dev_err(&pdev->dev, "Unable to enable clock.\n");
return;
}
- if (!chip->names)
- devprop_gpiochip_set_names(chip, dev_fwnode(chip->parent));
-
acpi_gpiochip_request_regions(acpi_gpio);
acpi_gpiochip_scan_gpios(acpi_gpio);
acpi_walk_dep_device_list(handle);
// SPDX-License-Identifier: GPL-2.0
#include <linux/anon_inodes.h>
+#include <linux/atomic.h>
#include <linux/bitmap.h>
+#include <linux/build_bug.h>
#include <linux/cdev.h>
#include <linux/compat.h>
+#include <linux/compiler.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/file.h>
#include <linux/kernel.h>
#include <linux/kfifo.h>
#include <linux/module.h>
+#include <linux/mutex.h>
#include <linux/pinctrl/consumer.h>
#include <linux/poll.h>
#include <linux/spinlock.h>
#include <linux/timekeeping.h>
#include <linux/uaccess.h>
+#include <linux/workqueue.h>
#include <uapi/linux/gpio.h>
#include "gpiolib.h"
#include "gpiolib-cdev.h"
+/*
+ * Array sizes must ensure 64-bit alignment and not create holes in the
+ * struct packing.
+ */
+static_assert(IS_ALIGNED(GPIO_V2_LINES_MAX, 2));
+static_assert(IS_ALIGNED(GPIO_MAX_NAME_SIZE, 8));
+
+/*
+ * Check that uAPI structs are 64-bit aligned for 32/64-bit compatibility
+ */
+static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_attribute), 8));
+static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_config_attribute), 8));
+static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_config), 8));
+static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_request), 8));
+static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_info), 8));
+static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_info_changed), 8));
+static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_event), 8));
+static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_values), 8));
+
/* Character device interface to GPIO.
*
* The GPIO character device, /dev/gpiochipN, provides userspace an
* GPIO line handle management
*/
+#ifdef CONFIG_GPIO_CDEV_V1
/**
* struct linehandle_state - contains the state of a userspace handle
* @gdev: the GPIO device the handle pertains to
if (copy_from_user(&gcnf, ip, sizeof(gcnf)))
return -EFAULT;
- lflags = gcnf.flags;
- ret = linehandle_validate_flags(lflags);
- if (ret)
- return ret;
+ lflags = gcnf.flags;
+ ret = linehandle_validate_flags(lflags);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < lh->num_descs; i++) {
+ desc = lh->descs[i];
+ linehandle_flags_to_desc_flags(gcnf.flags, &desc->flags);
+
+ /*
+ * Lines have to be requested explicitly for input
+ * or output, else the line will be treated "as is".
+ */
+ if (lflags & GPIOHANDLE_REQUEST_OUTPUT) {
+ int val = !!gcnf.default_values[i];
+
+ ret = gpiod_direction_output(desc, val);
+ if (ret)
+ return ret;
+ } else if (lflags & GPIOHANDLE_REQUEST_INPUT) {
+ ret = gpiod_direction_input(desc);
+ if (ret)
+ return ret;
+ }
+
+ blocking_notifier_call_chain(&desc->gdev->notifier,
+ GPIO_V2_LINE_CHANGED_CONFIG,
+ desc);
+ }
+ return 0;
+}
+
+static long linehandle_ioctl(struct file *file, unsigned int cmd,
+ unsigned long arg)
+{
+ struct linehandle_state *lh = file->private_data;
+ void __user *ip = (void __user *)arg;
+ struct gpiohandle_data ghd;
+ DECLARE_BITMAP(vals, GPIOHANDLES_MAX);
+ int i;
+
+ if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) {
+ /* NOTE: It's ok to read values of output lines. */
+ int ret = gpiod_get_array_value_complex(false,
+ true,
+ lh->num_descs,
+ lh->descs,
+ NULL,
+ vals);
+ if (ret)
+ return ret;
+
+ memset(&ghd, 0, sizeof(ghd));
+ for (i = 0; i < lh->num_descs; i++)
+ ghd.values[i] = test_bit(i, vals);
+
+ if (copy_to_user(ip, &ghd, sizeof(ghd)))
+ return -EFAULT;
+
+ return 0;
+ } else if (cmd == GPIOHANDLE_SET_LINE_VALUES_IOCTL) {
+ /*
+ * All line descriptors were created at once with the same
+ * flags so just check if the first one is really output.
+ */
+ if (!test_bit(FLAG_IS_OUT, &lh->descs[0]->flags))
+ return -EPERM;
+
+ if (copy_from_user(&ghd, ip, sizeof(ghd)))
+ return -EFAULT;
+
+ /* Clamp all values to [0,1] */
+ for (i = 0; i < lh->num_descs; i++)
+ __assign_bit(i, vals, ghd.values[i]);
+
+ /* Reuse the array setting function */
+ return gpiod_set_array_value_complex(false,
+ true,
+ lh->num_descs,
+ lh->descs,
+ NULL,
+ vals);
+ } else if (cmd == GPIOHANDLE_SET_CONFIG_IOCTL) {
+ return linehandle_set_config(lh, ip);
+ }
+ return -EINVAL;
+}
+
+#ifdef CONFIG_COMPAT
+static long linehandle_ioctl_compat(struct file *file, unsigned int cmd,
+ unsigned long arg)
+{
+ return linehandle_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
+}
+#endif
+
+static void linehandle_free(struct linehandle_state *lh)
+{
+ int i;
+
+ for (i = 0; i < lh->num_descs; i++)
+ if (lh->descs[i])
+ gpiod_free(lh->descs[i]);
+ kfree(lh->label);
+ put_device(&lh->gdev->dev);
+ kfree(lh);
+}
+
+static int linehandle_release(struct inode *inode, struct file *file)
+{
+ linehandle_free(file->private_data);
+ return 0;
+}
+
+static const struct file_operations linehandle_fileops = {
+ .release = linehandle_release,
+ .owner = THIS_MODULE,
+ .llseek = noop_llseek,
+ .unlocked_ioctl = linehandle_ioctl,
+#ifdef CONFIG_COMPAT
+ .compat_ioctl = linehandle_ioctl_compat,
+#endif
+};
+
+static int linehandle_create(struct gpio_device *gdev, void __user *ip)
+{
+ struct gpiohandle_request handlereq;
+ struct linehandle_state *lh;
+ struct file *file;
+ int fd, i, ret;
+ u32 lflags;
+
+ if (copy_from_user(&handlereq, ip, sizeof(handlereq)))
+ return -EFAULT;
+ if ((handlereq.lines == 0) || (handlereq.lines > GPIOHANDLES_MAX))
+ return -EINVAL;
+
+ lflags = handlereq.flags;
+
+ ret = linehandle_validate_flags(lflags);
+ if (ret)
+ return ret;
+
+ lh = kzalloc(sizeof(*lh), GFP_KERNEL);
+ if (!lh)
+ return -ENOMEM;
+ lh->gdev = gdev;
+ get_device(&gdev->dev);
+
+ if (handlereq.consumer_label[0] != '\0') {
+ /* label is only initialized if consumer_label is set */
+ lh->label = kstrndup(handlereq.consumer_label,
+ sizeof(handlereq.consumer_label) - 1,
+ GFP_KERNEL);
+ if (!lh->label) {
+ ret = -ENOMEM;
+ goto out_free_lh;
+ }
+ }
+
+ lh->num_descs = handlereq.lines;
+
+ /* Request each GPIO */
+ for (i = 0; i < handlereq.lines; i++) {
+ u32 offset = handlereq.lineoffsets[i];
+ struct gpio_desc *desc = gpiochip_get_desc(gdev->chip, offset);
+
+ if (IS_ERR(desc)) {
+ ret = PTR_ERR(desc);
+ goto out_free_lh;
+ }
+
+ ret = gpiod_request(desc, lh->label);
+ if (ret)
+ goto out_free_lh;
+ lh->descs[i] = desc;
+ linehandle_flags_to_desc_flags(handlereq.flags, &desc->flags);
+
+ ret = gpiod_set_transitory(desc, false);
+ if (ret < 0)
+ goto out_free_lh;
+
+ /*
+ * Lines have to be requested explicitly for input
+ * or output, else the line will be treated "as is".
+ */
+ if (lflags & GPIOHANDLE_REQUEST_OUTPUT) {
+ int val = !!handlereq.default_values[i];
+
+ ret = gpiod_direction_output(desc, val);
+ if (ret)
+ goto out_free_lh;
+ } else if (lflags & GPIOHANDLE_REQUEST_INPUT) {
+ ret = gpiod_direction_input(desc);
+ if (ret)
+ goto out_free_lh;
+ }
+
+ blocking_notifier_call_chain(&desc->gdev->notifier,
+ GPIO_V2_LINE_CHANGED_REQUESTED, desc);
+
+ dev_dbg(&gdev->dev, "registered chardev handle for line %d\n",
+ offset);
+ }
+
+ fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC);
+ if (fd < 0) {
+ ret = fd;
+ goto out_free_lh;
+ }
+
+ file = anon_inode_getfile("gpio-linehandle",
+ &linehandle_fileops,
+ lh,
+ O_RDONLY | O_CLOEXEC);
+ if (IS_ERR(file)) {
+ ret = PTR_ERR(file);
+ goto out_put_unused_fd;
+ }
+
+ handlereq.fd = fd;
+ if (copy_to_user(ip, &handlereq, sizeof(handlereq))) {
+ /*
+ * fput() will trigger the release() callback, so do not go onto
+ * the regular error cleanup path here.
+ */
+ fput(file);
+ put_unused_fd(fd);
+ return -EFAULT;
+ }
+
+ fd_install(fd, file);
+
+ dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n",
+ lh->num_descs);
+
+ return 0;
+
+out_put_unused_fd:
+ put_unused_fd(fd);
+out_free_lh:
+ linehandle_free(lh);
+ return ret;
+}
+#endif /* CONFIG_GPIO_CDEV_V1 */
+
+/**
+ * struct line - contains the state of a requested line
+ * @desc: the GPIO descriptor for this line.
+ * @req: the corresponding line request
+ * @irq: the interrupt triggered in response to events on this GPIO
+ * @eflags: the edge flags, GPIO_V2_LINE_FLAG_EDGE_RISING and/or
+ * GPIO_V2_LINE_FLAG_EDGE_FALLING, indicating the edge detection applied
+ * @timestamp_ns: cache for the timestamp storing it between hardirq and
+ * IRQ thread, used to bring the timestamp close to the actual event
+ * @req_seqno: the seqno for the current edge event in the sequence of
+ * events for the corresponding line request. This is drawn from the @req.
+ * @line_seqno: the seqno for the current edge event in the sequence of
+ * events for this line.
+ * @work: the worker that implements software debouncing
+ * @sw_debounced: flag indicating if the software debouncer is active
+ * @level: the current debounced physical level of the line
+ */
+struct line {
+ struct gpio_desc *desc;
+ /*
+ * -- edge detector specific fields --
+ */
+ struct linereq *req;
+ unsigned int irq;
+ u64 eflags;
+ /*
+ * timestamp_ns and req_seqno are accessed only by
+ * edge_irq_handler() and edge_irq_thread(), which are themselves
+ * mutually exclusive, so no additional protection is necessary.
+ */
+ u64 timestamp_ns;
+ u32 req_seqno;
+ /*
+ * line_seqno is accessed by either edge_irq_thread() or
+ * debounce_work_func(), which are themselves mutually exclusive,
+ * so no additional protection is necessary.
+ */
+ u32 line_seqno;
+ /*
+ * -- debouncer specific fields --
+ */
+ struct delayed_work work;
+ /*
+ * sw_debounce is accessed by linereq_set_config(), which is the
+ * only setter, and linereq_get_values(), which can live with a
+ * slightly stale value.
+ */
+ unsigned int sw_debounced;
+ /*
+ * level is accessed by debounce_work_func(), which is the only
+ * setter, and linereq_get_values() which can live with a slightly
+ * stale value.
+ */
+ unsigned int level;
+};
+
+/**
+ * struct linereq - contains the state of a userspace line request
+ * @gdev: the GPIO device the line request pertains to
+ * @label: consumer label used to tag GPIO descriptors
+ * @num_lines: the number of lines in the lines array
+ * @wait: wait queue that handles blocking reads of events
+ * @event_buffer_size: the number of elements allocated in @events
+ * @events: KFIFO for the GPIO events
+ * @seqno: the sequence number for edge events generated on all lines in
+ * this line request. Note that this is not used when @num_lines is 1, as
+ * the line_seqno is then the same and is cheaper to calculate.
+ * @config_mutex: mutex for serializing ioctl() calls to ensure consistency
+ * of configuration, particularly multi-step accesses to desc flags.
+ * @lines: the lines held by this line request, with @num_lines elements.
+ */
+struct linereq {
+ struct gpio_device *gdev;
+ const char *label;
+ u32 num_lines;
+ wait_queue_head_t wait;
+ u32 event_buffer_size;
+ DECLARE_KFIFO_PTR(events, struct gpio_v2_line_event);
+ atomic_t seqno;
+ struct mutex config_mutex;
+ struct line lines[];
+};
+
+#define GPIO_V2_LINE_BIAS_FLAGS \
+ (GPIO_V2_LINE_FLAG_BIAS_PULL_UP | \
+ GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN | \
+ GPIO_V2_LINE_FLAG_BIAS_DISABLED)
+
+#define GPIO_V2_LINE_DIRECTION_FLAGS \
+ (GPIO_V2_LINE_FLAG_INPUT | \
+ GPIO_V2_LINE_FLAG_OUTPUT)
+
+#define GPIO_V2_LINE_DRIVE_FLAGS \
+ (GPIO_V2_LINE_FLAG_OPEN_DRAIN | \
+ GPIO_V2_LINE_FLAG_OPEN_SOURCE)
+
+#define GPIO_V2_LINE_EDGE_FLAGS \
+ (GPIO_V2_LINE_FLAG_EDGE_RISING | \
+ GPIO_V2_LINE_FLAG_EDGE_FALLING)
+
+#define GPIO_V2_LINE_VALID_FLAGS \
+ (GPIO_V2_LINE_FLAG_ACTIVE_LOW | \
+ GPIO_V2_LINE_DIRECTION_FLAGS | \
+ GPIO_V2_LINE_DRIVE_FLAGS | \
+ GPIO_V2_LINE_EDGE_FLAGS | \
+ GPIO_V2_LINE_BIAS_FLAGS)
+
+static void linereq_put_event(struct linereq *lr,
+ struct gpio_v2_line_event *le)
+{
+ bool overflow = false;
+
+ spin_lock(&lr->wait.lock);
+ if (kfifo_is_full(&lr->events)) {
+ overflow = true;
+ kfifo_skip(&lr->events);
+ }
+ kfifo_in(&lr->events, le, 1);
+ spin_unlock(&lr->wait.lock);
+ if (!overflow)
+ wake_up_poll(&lr->wait, EPOLLIN);
+ else
+ pr_debug_ratelimited("event FIFO is full - event dropped\n");
+}
+
+static irqreturn_t edge_irq_thread(int irq, void *p)
+{
+ struct line *line = p;
+ struct linereq *lr = line->req;
+ struct gpio_v2_line_event le;
+
+ /* Do not leak kernel stack to userspace */
+ memset(&le, 0, sizeof(le));
+
+ if (line->timestamp_ns) {
+ le.timestamp_ns = line->timestamp_ns;
+ } else {
+ /*
+ * We may be running from a nested threaded interrupt in
+ * which case we didn't get the timestamp from
+ * edge_irq_handler().
+ */
+ le.timestamp_ns = ktime_get_ns();
+ if (lr->num_lines != 1)
+ line->req_seqno = atomic_inc_return(&lr->seqno);
+ }
+ line->timestamp_ns = 0;
+
+ if (line->eflags == (GPIO_V2_LINE_FLAG_EDGE_RISING |
+ GPIO_V2_LINE_FLAG_EDGE_FALLING)) {
+ int level = gpiod_get_value_cansleep(line->desc);
+
+ if (level)
+ /* Emit low-to-high event */
+ le.id = GPIO_V2_LINE_EVENT_RISING_EDGE;
+ else
+ /* Emit high-to-low event */
+ le.id = GPIO_V2_LINE_EVENT_FALLING_EDGE;
+ } else if (line->eflags == GPIO_V2_LINE_FLAG_EDGE_RISING) {
+ /* Emit low-to-high event */
+ le.id = GPIO_V2_LINE_EVENT_RISING_EDGE;
+ } else if (line->eflags == GPIO_V2_LINE_FLAG_EDGE_FALLING) {
+ /* Emit high-to-low event */
+ le.id = GPIO_V2_LINE_EVENT_FALLING_EDGE;
+ } else {
+ return IRQ_NONE;
+ }
+ line->line_seqno++;
+ le.line_seqno = line->line_seqno;
+ le.seqno = (lr->num_lines == 1) ? le.line_seqno : line->req_seqno;
+ le.offset = gpio_chip_hwgpio(line->desc);
+
+ linereq_put_event(lr, &le);
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t edge_irq_handler(int irq, void *p)
+{
+ struct line *line = p;
+ struct linereq *lr = line->req;
+
+ /*
+ * Just store the timestamp in hardirq context so we get it as
+ * close in time as possible to the actual event.
+ */
+ line->timestamp_ns = ktime_get_ns();
+
+ if (lr->num_lines != 1)
+ line->req_seqno = atomic_inc_return(&lr->seqno);
+
+ return IRQ_WAKE_THREAD;
+}
+
+/*
+ * returns the current debounced logical value.
+ */
+static bool debounced_value(struct line *line)
+{
+ bool value;
+
+ /*
+ * minor race - debouncer may be stopped here, so edge_detector_stop()
+ * must leave the value unchanged so the following will read the level
+ * from when the debouncer was last running.
+ */
+ value = READ_ONCE(line->level);
+
+ if (test_bit(FLAG_ACTIVE_LOW, &line->desc->flags))
+ value = !value;
+
+ return value;
+}
+
+static irqreturn_t debounce_irq_handler(int irq, void *p)
+{
+ struct line *line = p;
+
+ mod_delayed_work(system_wq, &line->work,
+ usecs_to_jiffies(READ_ONCE(line->desc->debounce_period_us)));
+
+ return IRQ_HANDLED;
+}
+
+static void debounce_work_func(struct work_struct *work)
+{
+ struct gpio_v2_line_event le;
+ struct line *line = container_of(work, struct line, work.work);
+ struct linereq *lr;
+ int level;
+
+ level = gpiod_get_raw_value_cansleep(line->desc);
+ if (level < 0) {
+ pr_debug_ratelimited("debouncer failed to read line value\n");
+ return;
+ }
+
+ if (READ_ONCE(line->level) == level)
+ return;
+
+ WRITE_ONCE(line->level, level);
+
+ /* -- edge detection -- */
+ if (!line->eflags)
+ return;
+
+ /* switch from physical level to logical - if they differ */
+ if (test_bit(FLAG_ACTIVE_LOW, &line->desc->flags))
+ level = !level;
+
+ /* ignore edges that are not being monitored */
+ if (((line->eflags == GPIO_V2_LINE_FLAG_EDGE_RISING) && !level) ||
+ ((line->eflags == GPIO_V2_LINE_FLAG_EDGE_FALLING) && level))
+ return;
+
+ /* Do not leak kernel stack to userspace */
+ memset(&le, 0, sizeof(le));
+
+ lr = line->req;
+ le.timestamp_ns = ktime_get_ns();
+ le.offset = gpio_chip_hwgpio(line->desc);
+ line->line_seqno++;
+ le.line_seqno = line->line_seqno;
+ le.seqno = (lr->num_lines == 1) ?
+ le.line_seqno : atomic_inc_return(&lr->seqno);
+
+ if (level)
+ /* Emit low-to-high event */
+ le.id = GPIO_V2_LINE_EVENT_RISING_EDGE;
+ else
+ /* Emit high-to-low event */
+ le.id = GPIO_V2_LINE_EVENT_FALLING_EDGE;
+
+ linereq_put_event(lr, &le);
+}
+
+static int debounce_setup(struct line *line,
+ unsigned int debounce_period_us)
+{
+ unsigned long irqflags;
+ int ret, level, irq;
+
+ /* try hardware */
+ ret = gpiod_set_debounce(line->desc, debounce_period_us);
+ if (!ret) {
+ WRITE_ONCE(line->desc->debounce_period_us, debounce_period_us);
+ return ret;
+ }
+ if (ret != -ENOTSUPP)
+ return ret;
+
+ if (debounce_period_us) {
+ /* setup software debounce */
+ level = gpiod_get_raw_value_cansleep(line->desc);
+ if (level < 0)
+ return level;
+
+ irq = gpiod_to_irq(line->desc);
+ if (irq < 0)
+ return -ENXIO;
+
+ WRITE_ONCE(line->level, level);
+ irqflags = IRQF_TRIGGER_FALLING | IRQF_TRIGGER_RISING;
+ ret = request_irq(irq, debounce_irq_handler, irqflags,
+ line->req->label, line);
+ if (ret)
+ return ret;
+
+ WRITE_ONCE(line->sw_debounced, 1);
+ line->irq = irq;
+ }
+ return 0;
+}
+
+static bool gpio_v2_line_config_debounced(struct gpio_v2_line_config *lc,
+ unsigned int line_idx)
+{
+ unsigned int i;
+ u64 mask = BIT_ULL(line_idx);
+
+ for (i = 0; i < lc->num_attrs; i++) {
+ if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_DEBOUNCE) &&
+ (lc->attrs[i].mask & mask))
+ return true;
+ }
+ return false;
+}
+
+static u32 gpio_v2_line_config_debounce_period(struct gpio_v2_line_config *lc,
+ unsigned int line_idx)
+{
+ unsigned int i;
+ u64 mask = BIT_ULL(line_idx);
+
+ for (i = 0; i < lc->num_attrs; i++) {
+ if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_DEBOUNCE) &&
+ (lc->attrs[i].mask & mask))
+ return lc->attrs[i].attr.debounce_period_us;
+ }
+ return 0;
+}
+
+static void edge_detector_stop(struct line *line)
+{
+ if (line->irq) {
+ free_irq(line->irq, line);
+ line->irq = 0;
+ }
+
+ cancel_delayed_work_sync(&line->work);
+ WRITE_ONCE(line->sw_debounced, 0);
+ line->eflags = 0;
+ /* do not change line->level - see comment in debounced_value() */
+}
+
+static int edge_detector_setup(struct line *line,
+ struct gpio_v2_line_config *lc,
+ unsigned int line_idx,
+ u64 eflags)
+{
+ u32 debounce_period_us;
+ unsigned long irqflags = 0;
+ int irq, ret;
+
+ if (eflags && !kfifo_initialized(&line->req->events)) {
+ ret = kfifo_alloc(&line->req->events,
+ line->req->event_buffer_size, GFP_KERNEL);
+ if (ret)
+ return ret;
+ }
+ line->eflags = eflags;
+ if (gpio_v2_line_config_debounced(lc, line_idx)) {
+ debounce_period_us = gpio_v2_line_config_debounce_period(lc, line_idx);
+ ret = debounce_setup(line, debounce_period_us);
+ if (ret)
+ return ret;
+ WRITE_ONCE(line->desc->debounce_period_us, debounce_period_us);
+ }
+
+ /* detection disabled or sw debouncer will provide edge detection */
+ if (!eflags || READ_ONCE(line->sw_debounced))
+ return 0;
+
+ irq = gpiod_to_irq(line->desc);
+ if (irq < 0)
+ return -ENXIO;
+
+ if (eflags & GPIO_V2_LINE_FLAG_EDGE_RISING)
+ irqflags |= test_bit(FLAG_ACTIVE_LOW, &line->desc->flags) ?
+ IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING;
+ if (eflags & GPIO_V2_LINE_FLAG_EDGE_FALLING)
+ irqflags |= test_bit(FLAG_ACTIVE_LOW, &line->desc->flags) ?
+ IRQF_TRIGGER_RISING : IRQF_TRIGGER_FALLING;
+ irqflags |= IRQF_ONESHOT;
+
+ /* Request a thread to read the events */
+ ret = request_threaded_irq(irq, edge_irq_handler, edge_irq_thread,
+ irqflags, line->req->label, line);
+ if (ret)
+ return ret;
+
+ line->irq = irq;
+ return 0;
+}
+
+static int edge_detector_update(struct line *line,
+ struct gpio_v2_line_config *lc,
+ unsigned int line_idx,
+ u64 eflags, bool polarity_change)
+{
+ unsigned int debounce_period_us =
+ gpio_v2_line_config_debounce_period(lc, line_idx);
+
+ if ((line->eflags == eflags) && !polarity_change &&
+ (READ_ONCE(line->desc->debounce_period_us) == debounce_period_us))
+ return 0;
+
+ /* sw debounced and still will be...*/
+ if (debounce_period_us && READ_ONCE(line->sw_debounced)) {
+ line->eflags = eflags;
+ WRITE_ONCE(line->desc->debounce_period_us, debounce_period_us);
+ return 0;
+ }
+
+ /* reconfiguring edge detection or sw debounce being disabled */
+ if ((line->irq && !READ_ONCE(line->sw_debounced)) ||
+ (!debounce_period_us && READ_ONCE(line->sw_debounced)))
+ edge_detector_stop(line);
+
+ return edge_detector_setup(line, lc, line_idx, eflags);
+}
+
+static u64 gpio_v2_line_config_flags(struct gpio_v2_line_config *lc,
+ unsigned int line_idx)
+{
+ unsigned int i;
+ u64 mask = BIT_ULL(line_idx);
+
+ for (i = 0; i < lc->num_attrs; i++) {
+ if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_FLAGS) &&
+ (lc->attrs[i].mask & mask))
+ return lc->attrs[i].attr.flags;
+ }
+ return lc->flags;
+}
+
+static int gpio_v2_line_config_output_value(struct gpio_v2_line_config *lc,
+ unsigned int line_idx)
+{
+ unsigned int i;
+ u64 mask = BIT_ULL(line_idx);
+
+ for (i = 0; i < lc->num_attrs; i++) {
+ if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_OUTPUT_VALUES) &&
+ (lc->attrs[i].mask & mask))
+ return !!(lc->attrs[i].attr.values & mask);
+ }
+ return 0;
+}
+
+static int gpio_v2_line_flags_validate(u64 flags)
+{
+ /* Return an error if an unknown flag is set */
+ if (flags & ~GPIO_V2_LINE_VALID_FLAGS)
+ return -EINVAL;
+
+ /*
+ * Do not allow both INPUT and OUTPUT flags to be set as they are
+ * contradictory.
+ */
+ if ((flags & GPIO_V2_LINE_FLAG_INPUT) &&
+ (flags & GPIO_V2_LINE_FLAG_OUTPUT))
+ return -EINVAL;
+
+ /* Edge detection requires explicit input. */
+ if ((flags & GPIO_V2_LINE_EDGE_FLAGS) &&
+ !(flags & GPIO_V2_LINE_FLAG_INPUT))
+ return -EINVAL;
+
+ /*
+ * Do not allow OPEN_SOURCE and OPEN_DRAIN flags in a single
+ * request. If the hardware actually supports enabling both at the
+ * same time the electrical result would be disastrous.
+ */
+ if ((flags & GPIO_V2_LINE_FLAG_OPEN_DRAIN) &&
+ (flags & GPIO_V2_LINE_FLAG_OPEN_SOURCE))
+ return -EINVAL;
+
+ /* Drive requires explicit output direction. */
+ if ((flags & GPIO_V2_LINE_DRIVE_FLAGS) &&
+ !(flags & GPIO_V2_LINE_FLAG_OUTPUT))
+ return -EINVAL;
+
+ /* Bias requires explicit direction. */
+ if ((flags & GPIO_V2_LINE_BIAS_FLAGS) &&
+ !(flags & GPIO_V2_LINE_DIRECTION_FLAGS))
+ return -EINVAL;
+
+ /* Only one bias flag can be set. */
+ if (((flags & GPIO_V2_LINE_FLAG_BIAS_DISABLED) &&
+ (flags & (GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN |
+ GPIO_V2_LINE_FLAG_BIAS_PULL_UP))) ||
+ ((flags & GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN) &&
+ (flags & GPIO_V2_LINE_FLAG_BIAS_PULL_UP)))
+ return -EINVAL;
+
+ return 0;
+}
+
+static int gpio_v2_line_config_validate(struct gpio_v2_line_config *lc,
+ unsigned int num_lines)
+{
+ unsigned int i;
+ u64 flags;
+ int ret;
+
+ if (lc->num_attrs > GPIO_V2_LINE_NUM_ATTRS_MAX)
+ return -EINVAL;
+
+ if (memchr_inv(lc->padding, 0, sizeof(lc->padding)))
+ return -EINVAL;
+
+ for (i = 0; i < num_lines; i++) {
+ flags = gpio_v2_line_config_flags(lc, i);
+ ret = gpio_v2_line_flags_validate(flags);
+ if (ret)
+ return ret;
+
+ /* debounce requires explicit input */
+ if (gpio_v2_line_config_debounced(lc, i) &&
+ !(flags & GPIO_V2_LINE_FLAG_INPUT))
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static void gpio_v2_line_config_flags_to_desc_flags(u64 flags,
+ unsigned long *flagsp)
+{
+ assign_bit(FLAG_ACTIVE_LOW, flagsp,
+ flags & GPIO_V2_LINE_FLAG_ACTIVE_LOW);
+
+ if (flags & GPIO_V2_LINE_FLAG_OUTPUT)
+ set_bit(FLAG_IS_OUT, flagsp);
+ else if (flags & GPIO_V2_LINE_FLAG_INPUT)
+ clear_bit(FLAG_IS_OUT, flagsp);
+
+ assign_bit(FLAG_EDGE_RISING, flagsp,
+ flags & GPIO_V2_LINE_FLAG_EDGE_RISING);
+ assign_bit(FLAG_EDGE_FALLING, flagsp,
+ flags & GPIO_V2_LINE_FLAG_EDGE_FALLING);
+
+ assign_bit(FLAG_OPEN_DRAIN, flagsp,
+ flags & GPIO_V2_LINE_FLAG_OPEN_DRAIN);
+ assign_bit(FLAG_OPEN_SOURCE, flagsp,
+ flags & GPIO_V2_LINE_FLAG_OPEN_SOURCE);
+
+ assign_bit(FLAG_PULL_UP, flagsp,
+ flags & GPIO_V2_LINE_FLAG_BIAS_PULL_UP);
+ assign_bit(FLAG_PULL_DOWN, flagsp,
+ flags & GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN);
+ assign_bit(FLAG_BIAS_DISABLE, flagsp,
+ flags & GPIO_V2_LINE_FLAG_BIAS_DISABLED);
+}
+
+static long linereq_get_values(struct linereq *lr, void __user *ip)
+{
+ struct gpio_v2_line_values lv;
+ DECLARE_BITMAP(vals, GPIO_V2_LINES_MAX);
+ struct gpio_desc **descs;
+ unsigned int i, didx, num_get;
+ bool val;
+ int ret;
+
+ /* NOTE: It's ok to read values of output lines. */
+ if (copy_from_user(&lv, ip, sizeof(lv)))
+ return -EFAULT;
+
+ for (num_get = 0, i = 0; i < lr->num_lines; i++) {
+ if (lv.mask & BIT_ULL(i)) {
+ num_get++;
+ descs = &lr->lines[i].desc;
+ }
+ }
+
+ if (num_get == 0)
+ return -EINVAL;
+
+ if (num_get != 1) {
+ descs = kmalloc_array(num_get, sizeof(*descs), GFP_KERNEL);
+ if (!descs)
+ return -ENOMEM;
+ for (didx = 0, i = 0; i < lr->num_lines; i++) {
+ if (lv.mask & BIT_ULL(i)) {
+ descs[didx] = lr->lines[i].desc;
+ didx++;
+ }
+ }
+ }
+ ret = gpiod_get_array_value_complex(false, true, num_get,
+ descs, NULL, vals);
+
+ if (num_get != 1)
+ kfree(descs);
+ if (ret)
+ return ret;
+
+ lv.bits = 0;
+ for (didx = 0, i = 0; i < lr->num_lines; i++) {
+ if (lv.mask & BIT_ULL(i)) {
+ if (lr->lines[i].sw_debounced)
+ val = debounced_value(&lr->lines[i]);
+ else
+ val = test_bit(didx, vals);
+ if (val)
+ lv.bits |= BIT_ULL(i);
+ didx++;
+ }
+ }
+
+ if (copy_to_user(ip, &lv, sizeof(lv)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static long linereq_set_values_unlocked(struct linereq *lr,
+ struct gpio_v2_line_values *lv)
+{
+ DECLARE_BITMAP(vals, GPIO_V2_LINES_MAX);
+ struct gpio_desc **descs;
+ unsigned int i, didx, num_set;
+ int ret;
+
+ bitmap_zero(vals, GPIO_V2_LINES_MAX);
+ for (num_set = 0, i = 0; i < lr->num_lines; i++) {
+ if (lv->mask & BIT_ULL(i)) {
+ if (!test_bit(FLAG_IS_OUT, &lr->lines[i].desc->flags))
+ return -EPERM;
+ if (lv->bits & BIT_ULL(i))
+ __set_bit(num_set, vals);
+ num_set++;
+ descs = &lr->lines[i].desc;
+ }
+ }
+ if (num_set == 0)
+ return -EINVAL;
+
+ if (num_set != 1) {
+ /* build compacted desc array and values */
+ descs = kmalloc_array(num_set, sizeof(*descs), GFP_KERNEL);
+ if (!descs)
+ return -ENOMEM;
+ for (didx = 0, i = 0; i < lr->num_lines; i++) {
+ if (lv->mask & BIT_ULL(i)) {
+ descs[didx] = lr->lines[i].desc;
+ didx++;
+ }
+ }
+ }
+ ret = gpiod_set_array_value_complex(false, true, num_set,
+ descs, NULL, vals);
+
+ if (num_set != 1)
+ kfree(descs);
+ return ret;
+}
+
+static long linereq_set_values(struct linereq *lr, void __user *ip)
+{
+ struct gpio_v2_line_values lv;
+ int ret;
+
+ if (copy_from_user(&lv, ip, sizeof(lv)))
+ return -EFAULT;
+
+ mutex_lock(&lr->config_mutex);
- for (i = 0; i < lh->num_descs; i++) {
- desc = lh->descs[i];
- linehandle_flags_to_desc_flags(gcnf.flags, &desc->flags);
+ ret = linereq_set_values_unlocked(lr, &lv);
+
+ mutex_unlock(&lr->config_mutex);
+
+ return ret;
+}
+
+static long linereq_set_config_unlocked(struct linereq *lr,
+ struct gpio_v2_line_config *lc)
+{
+ struct gpio_desc *desc;
+ unsigned int i;
+ u64 flags;
+ bool polarity_change;
+ int ret;
+
+ for (i = 0; i < lr->num_lines; i++) {
+ desc = lr->lines[i].desc;
+ flags = gpio_v2_line_config_flags(lc, i);
+ polarity_change =
+ (!!test_bit(FLAG_ACTIVE_LOW, &desc->flags) !=
+ ((flags & GPIO_V2_LINE_FLAG_ACTIVE_LOW) != 0));
+ gpio_v2_line_config_flags_to_desc_flags(flags, &desc->flags);
/*
* Lines have to be requested explicitly for input
* or output, else the line will be treated "as is".
*/
- if (lflags & GPIOHANDLE_REQUEST_OUTPUT) {
- int val = !!gcnf.default_values[i];
+ if (flags & GPIO_V2_LINE_FLAG_OUTPUT) {
+ int val = gpio_v2_line_config_output_value(lc, i);
+ edge_detector_stop(&lr->lines[i]);
ret = gpiod_direction_output(desc, val);
if (ret)
return ret;
- } else if (lflags & GPIOHANDLE_REQUEST_INPUT) {
+ } else if (flags & GPIO_V2_LINE_FLAG_INPUT) {
ret = gpiod_direction_input(desc);
if (ret)
return ret;
+
+ ret = edge_detector_update(&lr->lines[i], lc, i,
+ flags & GPIO_V2_LINE_EDGE_FLAGS,
+ polarity_change);
+ if (ret)
+ return ret;
}
blocking_notifier_call_chain(&desc->gdev->notifier,
- GPIOLINE_CHANGED_CONFIG, desc);
+ GPIO_V2_LINE_CHANGED_CONFIG,
+ desc);
}
return 0;
}
-static long linehandle_ioctl(struct file *file, unsigned int cmd,
- unsigned long arg)
+static long linereq_set_config(struct linereq *lr, void __user *ip)
{
- struct linehandle_state *lh = file->private_data;
- void __user *ip = (void __user *)arg;
- struct gpiohandle_data ghd;
- DECLARE_BITMAP(vals, GPIOHANDLES_MAX);
- int i;
+ struct gpio_v2_line_config lc;
+ int ret;
- if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) {
- /* NOTE: It's ok to read values of output lines. */
- int ret = gpiod_get_array_value_complex(false,
- true,
- lh->num_descs,
- lh->descs,
- NULL,
- vals);
- if (ret)
- return ret;
+ if (copy_from_user(&lc, ip, sizeof(lc)))
+ return -EFAULT;
- memset(&ghd, 0, sizeof(ghd));
- for (i = 0; i < lh->num_descs; i++)
- ghd.values[i] = test_bit(i, vals);
+ ret = gpio_v2_line_config_validate(&lc, lr->num_lines);
+ if (ret)
+ return ret;
- if (copy_to_user(ip, &ghd, sizeof(ghd)))
- return -EFAULT;
+ mutex_lock(&lr->config_mutex);
- return 0;
- } else if (cmd == GPIOHANDLE_SET_LINE_VALUES_IOCTL) {
- /*
- * All line descriptors were created at once with the same
- * flags so just check if the first one is really output.
- */
- if (!test_bit(FLAG_IS_OUT, &lh->descs[0]->flags))
- return -EPERM;
+ ret = linereq_set_config_unlocked(lr, &lc);
- if (copy_from_user(&ghd, ip, sizeof(ghd)))
- return -EFAULT;
+ mutex_unlock(&lr->config_mutex);
- /* Clamp all values to [0,1] */
- for (i = 0; i < lh->num_descs; i++)
- __assign_bit(i, vals, ghd.values[i]);
+ return ret;
+}
+
+static long linereq_ioctl(struct file *file, unsigned int cmd,
+ unsigned long arg)
+{
+ struct linereq *lr = file->private_data;
+ void __user *ip = (void __user *)arg;
+
+ if (cmd == GPIO_V2_LINE_GET_VALUES_IOCTL)
+ return linereq_get_values(lr, ip);
+ else if (cmd == GPIO_V2_LINE_SET_VALUES_IOCTL)
+ return linereq_set_values(lr, ip);
+ else if (cmd == GPIO_V2_LINE_SET_CONFIG_IOCTL)
+ return linereq_set_config(lr, ip);
- /* Reuse the array setting function */
- return gpiod_set_array_value_complex(false,
- true,
- lh->num_descs,
- lh->descs,
- NULL,
- vals);
- } else if (cmd == GPIOHANDLE_SET_CONFIG_IOCTL) {
- return linehandle_set_config(lh, ip);
- }
return -EINVAL;
}
#ifdef CONFIG_COMPAT
-static long linehandle_ioctl_compat(struct file *file, unsigned int cmd,
- unsigned long arg)
+static long linereq_ioctl_compat(struct file *file, unsigned int cmd,
+ unsigned long arg)
{
- return linehandle_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
+ return linereq_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
}
#endif
-static void linehandle_free(struct linehandle_state *lh)
+static __poll_t linereq_poll(struct file *file,
+ struct poll_table_struct *wait)
{
- int i;
+ struct linereq *lr = file->private_data;
+ __poll_t events = 0;
- for (i = 0; i < lh->num_descs; i++)
- if (lh->descs[i])
- gpiod_free(lh->descs[i]);
- kfree(lh->label);
- put_device(&lh->gdev->dev);
- kfree(lh);
+ poll_wait(file, &lr->wait, wait);
+
+ if (!kfifo_is_empty_spinlocked_noirqsave(&lr->events,
+ &lr->wait.lock))
+ events = EPOLLIN | EPOLLRDNORM;
+
+ return events;
}
-static int linehandle_release(struct inode *inode, struct file *file)
+static ssize_t linereq_read(struct file *file,
+ char __user *buf,
+ size_t count,
+ loff_t *f_ps)
{
- linehandle_free(file->private_data);
+ struct linereq *lr = file->private_data;
+ struct gpio_v2_line_event le;
+ ssize_t bytes_read = 0;
+ int ret;
+
+ if (count < sizeof(le))
+ return -EINVAL;
+
+ do {
+ spin_lock(&lr->wait.lock);
+ if (kfifo_is_empty(&lr->events)) {
+ if (bytes_read) {
+ spin_unlock(&lr->wait.lock);
+ return bytes_read;
+ }
+
+ if (file->f_flags & O_NONBLOCK) {
+ spin_unlock(&lr->wait.lock);
+ return -EAGAIN;
+ }
+
+ ret = wait_event_interruptible_locked(lr->wait,
+ !kfifo_is_empty(&lr->events));
+ if (ret) {
+ spin_unlock(&lr->wait.lock);
+ return ret;
+ }
+ }
+
+ ret = kfifo_out(&lr->events, &le, 1);
+ spin_unlock(&lr->wait.lock);
+ if (ret != 1) {
+ /*
+ * This should never happen - we were holding the
+ * lock from the moment we learned the fifo is no
+ * longer empty until now.
+ */
+ ret = -EIO;
+ break;
+ }
+
+ if (copy_to_user(buf + bytes_read, &le, sizeof(le)))
+ return -EFAULT;
+ bytes_read += sizeof(le);
+ } while (count >= bytes_read + sizeof(le));
+
+ return bytes_read;
+}
+
+static void linereq_free(struct linereq *lr)
+{
+ unsigned int i;
+
+ for (i = 0; i < lr->num_lines; i++) {
+ edge_detector_stop(&lr->lines[i]);
+ if (lr->lines[i].desc)
+ gpiod_free(lr->lines[i].desc);
+ }
+ kfifo_free(&lr->events);
+ kfree(lr->label);
+ put_device(&lr->gdev->dev);
+ kfree(lr);
+}
+
+static int linereq_release(struct inode *inode, struct file *file)
+{
+ struct linereq *lr = file->private_data;
+
+ linereq_free(lr);
return 0;
}
-static const struct file_operations linehandle_fileops = {
- .release = linehandle_release,
+static const struct file_operations line_fileops = {
+ .release = linereq_release,
+ .read = linereq_read,
+ .poll = linereq_poll,
.owner = THIS_MODULE,
.llseek = noop_llseek,
- .unlocked_ioctl = linehandle_ioctl,
+ .unlocked_ioctl = linereq_ioctl,
#ifdef CONFIG_COMPAT
- .compat_ioctl = linehandle_ioctl_compat,
+ .compat_ioctl = linereq_ioctl_compat,
#endif
};
-static int linehandle_create(struct gpio_device *gdev, void __user *ip)
+static int linereq_create(struct gpio_device *gdev, void __user *ip)
{
- struct gpiohandle_request handlereq;
- struct linehandle_state *lh;
+ struct gpio_v2_line_request ulr;
+ struct gpio_v2_line_config *lc;
+ struct linereq *lr;
struct file *file;
- int fd, i, ret;
- u32 lflags;
+ u64 flags;
+ unsigned int i;
+ int fd, ret;
- if (copy_from_user(&handlereq, ip, sizeof(handlereq)))
+ if (copy_from_user(&ulr, ip, sizeof(ulr)))
return -EFAULT;
- if ((handlereq.lines == 0) || (handlereq.lines > GPIOHANDLES_MAX))
+
+ if ((ulr.num_lines == 0) || (ulr.num_lines > GPIO_V2_LINES_MAX))
return -EINVAL;
- lflags = handlereq.flags;
+ if (memchr_inv(ulr.padding, 0, sizeof(ulr.padding)))
+ return -EINVAL;
- ret = linehandle_validate_flags(lflags);
+ lc = &ulr.config;
+ ret = gpio_v2_line_config_validate(lc, ulr.num_lines);
if (ret)
return ret;
- lh = kzalloc(sizeof(*lh), GFP_KERNEL);
- if (!lh)
+ lr = kzalloc(struct_size(lr, lines, ulr.num_lines), GFP_KERNEL);
+ if (!lr)
return -ENOMEM;
- lh->gdev = gdev;
+
+ lr->gdev = gdev;
get_device(&gdev->dev);
- /* Make sure this is terminated */
- handlereq.consumer_label[sizeof(handlereq.consumer_label)-1] = '\0';
- if (strlen(handlereq.consumer_label)) {
- lh->label = kstrdup(handlereq.consumer_label,
- GFP_KERNEL);
- if (!lh->label) {
+ for (i = 0; i < ulr.num_lines; i++) {
+ lr->lines[i].req = lr;
+ WRITE_ONCE(lr->lines[i].sw_debounced, 0);
+ INIT_DELAYED_WORK(&lr->lines[i].work, debounce_work_func);
+ }
+
+ if (ulr.consumer[0] != '\0') {
+ /* label is only initialized if consumer is set */
+ lr->label = kstrndup(ulr.consumer, sizeof(ulr.consumer) - 1,
+ GFP_KERNEL);
+ if (!lr->label) {
ret = -ENOMEM;
- goto out_free_lh;
+ goto out_free_linereq;
}
}
- lh->num_descs = handlereq.lines;
+ mutex_init(&lr->config_mutex);
+ init_waitqueue_head(&lr->wait);
+ lr->event_buffer_size = ulr.event_buffer_size;
+ if (lr->event_buffer_size == 0)
+ lr->event_buffer_size = ulr.num_lines * 16;
+ else if (lr->event_buffer_size > GPIO_V2_LINES_MAX * 16)
+ lr->event_buffer_size = GPIO_V2_LINES_MAX * 16;
+
+ atomic_set(&lr->seqno, 0);
+ lr->num_lines = ulr.num_lines;
/* Request each GPIO */
- for (i = 0; i < handlereq.lines; i++) {
- u32 offset = handlereq.lineoffsets[i];
+ for (i = 0; i < ulr.num_lines; i++) {
+ u32 offset = ulr.offsets[i];
struct gpio_desc *desc = gpiochip_get_desc(gdev->chip, offset);
if (IS_ERR(desc)) {
ret = PTR_ERR(desc);
- goto out_free_lh;
+ goto out_free_linereq;
}
- ret = gpiod_request(desc, lh->label);
+ ret = gpiod_request(desc, lr->label);
if (ret)
- goto out_free_lh;
- lh->descs[i] = desc;
- linehandle_flags_to_desc_flags(handlereq.flags, &desc->flags);
+ goto out_free_linereq;
+
+ lr->lines[i].desc = desc;
+ flags = gpio_v2_line_config_flags(lc, i);
+ gpio_v2_line_config_flags_to_desc_flags(flags, &desc->flags);
ret = gpiod_set_transitory(desc, false);
if (ret < 0)
- goto out_free_lh;
+ goto out_free_linereq;
/*
* Lines have to be requested explicitly for input
* or output, else the line will be treated "as is".
*/
- if (lflags & GPIOHANDLE_REQUEST_OUTPUT) {
- int val = !!handlereq.default_values[i];
+ if (flags & GPIO_V2_LINE_FLAG_OUTPUT) {
+ int val = gpio_v2_line_config_output_value(lc, i);
ret = gpiod_direction_output(desc, val);
if (ret)
- goto out_free_lh;
- } else if (lflags & GPIOHANDLE_REQUEST_INPUT) {
+ goto out_free_linereq;
+ } else if (flags & GPIO_V2_LINE_FLAG_INPUT) {
ret = gpiod_direction_input(desc);
if (ret)
- goto out_free_lh;
+ goto out_free_linereq;
+
+ ret = edge_detector_setup(&lr->lines[i], lc, i,
+ flags & GPIO_V2_LINE_EDGE_FLAGS);
+ if (ret)
+ goto out_free_linereq;
}
blocking_notifier_call_chain(&desc->gdev->notifier,
- GPIOLINE_CHANGED_REQUESTED, desc);
+ GPIO_V2_LINE_CHANGED_REQUESTED, desc);
dev_dbg(&gdev->dev, "registered chardev handle for line %d\n",
offset);
fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC);
if (fd < 0) {
ret = fd;
- goto out_free_lh;
+ goto out_free_linereq;
}
- file = anon_inode_getfile("gpio-linehandle",
- &linehandle_fileops,
- lh,
+ file = anon_inode_getfile("gpio-line", &line_fileops, lr,
O_RDONLY | O_CLOEXEC);
if (IS_ERR(file)) {
ret = PTR_ERR(file);
goto out_put_unused_fd;
}
- handlereq.fd = fd;
- if (copy_to_user(ip, &handlereq, sizeof(handlereq))) {
+ ulr.fd = fd;
+ if (copy_to_user(ip, &ulr, sizeof(ulr))) {
/*
* fput() will trigger the release() callback, so do not go onto
* the regular error cleanup path here.
fd_install(fd, file);
dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n",
- lh->num_descs);
+ lr->num_lines);
return 0;
out_put_unused_fd:
put_unused_fd(fd);
-out_free_lh:
- linehandle_free(lh);
+out_free_linereq:
+ linereq_free(lr);
return ret;
}
+#ifdef CONFIG_GPIO_CDEV_V1
+
/*
* GPIO line event management
*/
return events;
}
+static ssize_t lineevent_get_size(void)
+{
+#if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
+ /* i386 has no padding after 'id' */
+ if (in_ia32_syscall()) {
+ struct compat_gpioeevent_data {
+ compat_u64 timestamp;
+ u32 id;
+ };
+
+ return sizeof(struct compat_gpioeevent_data);
+ }
+#endif
+ return sizeof(struct gpioevent_data);
+}
static ssize_t lineevent_read(struct file *file,
char __user *buf,
struct lineevent_state *le = file->private_data;
struct gpioevent_data ge;
ssize_t bytes_read = 0;
+ ssize_t ge_size;
int ret;
- if (count < sizeof(ge))
+ /*
+ * When compatible system call is being used the struct gpioevent_data,
+ * in case of at least ia32, has different size due to the alignment
+ * differences. Because we have first member 64 bits followed by one of
+ * 32 bits there is no gap between them. The only difference is the
+ * padding at the end of the data structure. Hence, we calculate the
+ * actual sizeof() and pass this as an argument to copy_to_user() to
+ * drop unneeded bytes from the output.
+ */
+ ge_size = lineevent_get_size();
+ if (count < ge_size)
return -EINVAL;
do {
break;
}
- if (copy_to_user(buf + bytes_read, &ge, sizeof(ge)))
+ if (copy_to_user(buf + bytes_read, &ge, ge_size))
return -EFAULT;
- bytes_read += sizeof(ge);
- } while (count >= bytes_read + sizeof(ge));
+ bytes_read += ge_size;
+ } while (count >= bytes_read + ge_size);
return bytes_read;
}
le->gdev = gdev;
get_device(&gdev->dev);
- /* Make sure this is terminated */
- eventreq.consumer_label[sizeof(eventreq.consumer_label)-1] = '\0';
- if (strlen(eventreq.consumer_label)) {
- le->label = kstrdup(eventreq.consumer_label,
- GFP_KERNEL);
+ if (eventreq.consumer_label[0] != '\0') {
+ /* label is only initialized if consumer_label is set */
+ le->label = kstrndup(eventreq.consumer_label,
+ sizeof(eventreq.consumer_label) - 1,
+ GFP_KERNEL);
if (!le->label) {
ret = -ENOMEM;
goto out_free_le;
goto out_free_le;
blocking_notifier_call_chain(&desc->gdev->notifier,
- GPIOLINE_CHANGED_REQUESTED, desc);
+ GPIO_V2_LINE_CHANGED_REQUESTED, desc);
irq = gpiod_to_irq(desc);
if (irq <= 0) {
return ret;
}
+static void gpio_v2_line_info_to_v1(struct gpio_v2_line_info *info_v2,
+ struct gpioline_info *info_v1)
+{
+ u64 flagsv2 = info_v2->flags;
+
+ memcpy(info_v1->name, info_v2->name, sizeof(info_v1->name));
+ memcpy(info_v1->consumer, info_v2->consumer, sizeof(info_v1->consumer));
+ info_v1->line_offset = info_v2->offset;
+ info_v1->flags = 0;
+
+ if (flagsv2 & GPIO_V2_LINE_FLAG_USED)
+ info_v1->flags |= GPIOLINE_FLAG_KERNEL;
+
+ if (flagsv2 & GPIO_V2_LINE_FLAG_OUTPUT)
+ info_v1->flags |= GPIOLINE_FLAG_IS_OUT;
+
+ if (flagsv2 & GPIO_V2_LINE_FLAG_ACTIVE_LOW)
+ info_v1->flags |= GPIOLINE_FLAG_ACTIVE_LOW;
+
+ if (flagsv2 & GPIO_V2_LINE_FLAG_OPEN_DRAIN)
+ info_v1->flags |= GPIOLINE_FLAG_OPEN_DRAIN;
+ if (flagsv2 & GPIO_V2_LINE_FLAG_OPEN_SOURCE)
+ info_v1->flags |= GPIOLINE_FLAG_OPEN_SOURCE;
+
+ if (flagsv2 & GPIO_V2_LINE_FLAG_BIAS_PULL_UP)
+ info_v1->flags |= GPIOLINE_FLAG_BIAS_PULL_UP;
+ if (flagsv2 & GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN)
+ info_v1->flags |= GPIOLINE_FLAG_BIAS_PULL_DOWN;
+ if (flagsv2 & GPIO_V2_LINE_FLAG_BIAS_DISABLED)
+ info_v1->flags |= GPIOLINE_FLAG_BIAS_DISABLE;
+}
+
+static void gpio_v2_line_info_changed_to_v1(
+ struct gpio_v2_line_info_changed *lic_v2,
+ struct gpioline_info_changed *lic_v1)
+{
+ gpio_v2_line_info_to_v1(&lic_v2->info, &lic_v1->info);
+ lic_v1->timestamp = lic_v2->timestamp_ns;
+ lic_v1->event_type = lic_v2->event_type;
+}
+
+#endif /* CONFIG_GPIO_CDEV_V1 */
+
static void gpio_desc_to_lineinfo(struct gpio_desc *desc,
- struct gpioline_info *info)
+ struct gpio_v2_line_info *info)
{
struct gpio_chip *gc = desc->gdev->chip;
bool ok_for_pinctrl;
unsigned long flags;
+ u32 debounce_period_us;
+ unsigned int num_attrs = 0;
+
+ memset(info, 0, sizeof(*info));
+ info->offset = gpio_chip_hwgpio(desc);
/*
* This function takes a mutex so we must check this before taking
* lock common to both frameworks?
*/
ok_for_pinctrl =
- pinctrl_gpio_can_use_line(gc->base + info->line_offset);
+ pinctrl_gpio_can_use_line(gc->base + info->offset);
spin_lock_irqsave(&gpio_lock, flags);
- if (desc->name) {
- strncpy(info->name, desc->name, sizeof(info->name));
- info->name[sizeof(info->name) - 1] = '\0';
- } else {
- info->name[0] = '\0';
- }
+ if (desc->name)
+ strscpy(info->name, desc->name, sizeof(info->name));
- if (desc->label) {
- strncpy(info->consumer, desc->label, sizeof(info->consumer));
- info->consumer[sizeof(info->consumer) - 1] = '\0';
- } else {
- info->consumer[0] = '\0';
- }
+ if (desc->label)
+ strscpy(info->consumer, desc->label, sizeof(info->consumer));
/*
* Userspace only need to know that the kernel is using this GPIO so
test_bit(FLAG_EXPORT, &desc->flags) ||
test_bit(FLAG_SYSFS, &desc->flags) ||
!ok_for_pinctrl)
- info->flags |= GPIOLINE_FLAG_KERNEL;
+ info->flags |= GPIO_V2_LINE_FLAG_USED;
+
if (test_bit(FLAG_IS_OUT, &desc->flags))
- info->flags |= GPIOLINE_FLAG_IS_OUT;
+ info->flags |= GPIO_V2_LINE_FLAG_OUTPUT;
+ else
+ info->flags |= GPIO_V2_LINE_FLAG_INPUT;
+
if (test_bit(FLAG_ACTIVE_LOW, &desc->flags))
- info->flags |= GPIOLINE_FLAG_ACTIVE_LOW;
+ info->flags |= GPIO_V2_LINE_FLAG_ACTIVE_LOW;
+
if (test_bit(FLAG_OPEN_DRAIN, &desc->flags))
- info->flags |= (GPIOLINE_FLAG_OPEN_DRAIN |
- GPIOLINE_FLAG_IS_OUT);
+ info->flags |= GPIO_V2_LINE_FLAG_OPEN_DRAIN;
if (test_bit(FLAG_OPEN_SOURCE, &desc->flags))
- info->flags |= (GPIOLINE_FLAG_OPEN_SOURCE |
- GPIOLINE_FLAG_IS_OUT);
+ info->flags |= GPIO_V2_LINE_FLAG_OPEN_SOURCE;
+
if (test_bit(FLAG_BIAS_DISABLE, &desc->flags))
- info->flags |= GPIOLINE_FLAG_BIAS_DISABLE;
+ info->flags |= GPIO_V2_LINE_FLAG_BIAS_DISABLED;
if (test_bit(FLAG_PULL_DOWN, &desc->flags))
- info->flags |= GPIOLINE_FLAG_BIAS_PULL_DOWN;
+ info->flags |= GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN;
if (test_bit(FLAG_PULL_UP, &desc->flags))
- info->flags |= GPIOLINE_FLAG_BIAS_PULL_UP;
+ info->flags |= GPIO_V2_LINE_FLAG_BIAS_PULL_UP;
+
+ if (test_bit(FLAG_EDGE_RISING, &desc->flags))
+ info->flags |= GPIO_V2_LINE_FLAG_EDGE_RISING;
+ if (test_bit(FLAG_EDGE_FALLING, &desc->flags))
+ info->flags |= GPIO_V2_LINE_FLAG_EDGE_FALLING;
+
+ debounce_period_us = READ_ONCE(desc->debounce_period_us);
+ if (debounce_period_us) {
+ info->attrs[num_attrs].id = GPIO_V2_LINE_ATTR_ID_DEBOUNCE;
+ info->attrs[num_attrs].debounce_period_us = debounce_period_us;
+ num_attrs++;
+ }
+ info->num_attrs = num_attrs;
spin_unlock_irqrestore(&gpio_lock, flags);
}
struct gpio_chardev_data {
struct gpio_device *gdev;
wait_queue_head_t wait;
- DECLARE_KFIFO(events, struct gpioline_info_changed, 32);
+ DECLARE_KFIFO(events, struct gpio_v2_line_info_changed, 32);
struct notifier_block lineinfo_changed_nb;
unsigned long *watched_lines;
+#ifdef CONFIG_GPIO_CDEV_V1
+ atomic_t watch_abi_version;
+#endif
};
+#ifdef CONFIG_GPIO_CDEV_V1
+/*
+ * returns 0 if the versions match, else the previously selected ABI version
+ */
+static int lineinfo_ensure_abi_version(struct gpio_chardev_data *cdata,
+ unsigned int version)
+{
+ int abiv = atomic_cmpxchg(&cdata->watch_abi_version, 0, version);
+
+ if (abiv == version)
+ return 0;
+
+ return abiv;
+}
+#endif
+
+static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip,
+ bool watch)
+{
+ struct gpio_desc *desc;
+ struct gpio_v2_line_info lineinfo;
+
+ if (copy_from_user(&lineinfo, ip, sizeof(lineinfo)))
+ return -EFAULT;
+
+ if (memchr_inv(lineinfo.padding, 0, sizeof(lineinfo.padding)))
+ return -EINVAL;
+
+ desc = gpiochip_get_desc(cdev->gdev->chip, lineinfo.offset);
+ if (IS_ERR(desc))
+ return PTR_ERR(desc);
+
+ if (watch) {
+#ifdef CONFIG_GPIO_CDEV_V1
+ if (lineinfo_ensure_abi_version(cdev, 2))
+ return -EPERM;
+#endif
+ if (test_and_set_bit(lineinfo.offset, cdev->watched_lines))
+ return -EBUSY;
+ }
+ gpio_desc_to_lineinfo(desc, &lineinfo);
+
+ if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) {
+ if (watch)
+ clear_bit(lineinfo.offset, cdev->watched_lines);
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
/*
* gpio_ioctl() - ioctl handler for the GPIO chardev
*/
struct gpio_device *gdev = cdev->gdev;
struct gpio_chip *gc = gdev->chip;
void __user *ip = (void __user *)arg;
- struct gpio_desc *desc;
__u32 offset;
/* We fail any subsequent ioctl():s when the chip is gone */
memset(&chipinfo, 0, sizeof(chipinfo));
- strncpy(chipinfo.name, dev_name(&gdev->dev),
+ strscpy(chipinfo.name, dev_name(&gdev->dev),
sizeof(chipinfo.name));
- chipinfo.name[sizeof(chipinfo.name)-1] = '\0';
- strncpy(chipinfo.label, gdev->label,
+ strscpy(chipinfo.label, gdev->label,
sizeof(chipinfo.label));
- chipinfo.label[sizeof(chipinfo.label)-1] = '\0';
chipinfo.lines = gdev->ngpio;
if (copy_to_user(ip, &chipinfo, sizeof(chipinfo)))
return -EFAULT;
return 0;
+#ifdef CONFIG_GPIO_CDEV_V1
} else if (cmd == GPIO_GET_LINEINFO_IOCTL) {
+ struct gpio_desc *desc;
struct gpioline_info lineinfo;
+ struct gpio_v2_line_info lineinfo_v2;
if (copy_from_user(&lineinfo, ip, sizeof(lineinfo)))
return -EFAULT;
if (IS_ERR(desc))
return PTR_ERR(desc);
- gpio_desc_to_lineinfo(desc, &lineinfo);
+ gpio_desc_to_lineinfo(desc, &lineinfo_v2);
+ gpio_v2_line_info_to_v1(&lineinfo_v2, &lineinfo);
if (copy_to_user(ip, &lineinfo, sizeof(lineinfo)))
return -EFAULT;
} else if (cmd == GPIO_GET_LINEEVENT_IOCTL) {
return lineevent_create(gdev, ip);
} else if (cmd == GPIO_GET_LINEINFO_WATCH_IOCTL) {
+ struct gpio_desc *desc;
struct gpioline_info lineinfo;
+ struct gpio_v2_line_info lineinfo_v2;
if (copy_from_user(&lineinfo, ip, sizeof(lineinfo)))
return -EFAULT;
if (IS_ERR(desc))
return PTR_ERR(desc);
+ if (lineinfo_ensure_abi_version(cdev, 1))
+ return -EPERM;
+
if (test_and_set_bit(lineinfo.line_offset, cdev->watched_lines))
return -EBUSY;
- gpio_desc_to_lineinfo(desc, &lineinfo);
+ gpio_desc_to_lineinfo(desc, &lineinfo_v2);
+ gpio_v2_line_info_to_v1(&lineinfo_v2, &lineinfo);
if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) {
clear_bit(lineinfo.line_offset, cdev->watched_lines);
}
return 0;
+#endif /* CONFIG_GPIO_CDEV_V1 */
+ } else if (cmd == GPIO_V2_GET_LINEINFO_IOCTL ||
+ cmd == GPIO_V2_GET_LINEINFO_WATCH_IOCTL) {
+ return lineinfo_get(cdev, ip,
+ cmd == GPIO_V2_GET_LINEINFO_WATCH_IOCTL);
+ } else if (cmd == GPIO_V2_GET_LINE_IOCTL) {
+ return linereq_create(gdev, ip);
} else if (cmd == GPIO_GET_LINEINFO_UNWATCH_IOCTL) {
if (copy_from_user(&offset, ip, sizeof(offset)))
return -EFAULT;
unsigned long action, void *data)
{
struct gpio_chardev_data *cdev = to_gpio_chardev_data(nb);
- struct gpioline_info_changed chg;
+ struct gpio_v2_line_info_changed chg;
struct gpio_desc *desc = data;
int ret;
return NOTIFY_DONE;
memset(&chg, 0, sizeof(chg));
- chg.info.line_offset = gpio_chip_hwgpio(desc);
chg.event_type = action;
- chg.timestamp = ktime_get_ns();
+ chg.timestamp_ns = ktime_get_ns();
gpio_desc_to_lineinfo(desc, &chg.info);
ret = kfifo_in_spinlocked(&cdev->events, &chg, 1, &cdev->wait.lock);
size_t count, loff_t *off)
{
struct gpio_chardev_data *cdev = file->private_data;
- struct gpioline_info_changed event;
+ struct gpio_v2_line_info_changed event;
ssize_t bytes_read = 0;
int ret;
+ size_t event_size;
- if (count < sizeof(event))
+#ifndef CONFIG_GPIO_CDEV_V1
+ event_size = sizeof(struct gpio_v2_line_info_changed);
+ if (count < event_size)
return -EINVAL;
+#endif
do {
spin_lock(&cdev->wait.lock);
return ret;
}
}
-
+#ifdef CONFIG_GPIO_CDEV_V1
+ /* must be after kfifo check so watch_abi_version is set */
+ if (atomic_read(&cdev->watch_abi_version) == 2)
+ event_size = sizeof(struct gpio_v2_line_info_changed);
+ else
+ event_size = sizeof(struct gpioline_info_changed);
+ if (count < event_size) {
+ spin_unlock(&cdev->wait.lock);
+ return -EINVAL;
+ }
+#endif
ret = kfifo_out(&cdev->events, &event, 1);
spin_unlock(&cdev->wait.lock);
if (ret != 1) {
/* We should never get here. See lineevent_read(). */
}
- if (copy_to_user(buf + bytes_read, &event, sizeof(event)))
+#ifdef CONFIG_GPIO_CDEV_V1
+ if (event_size == sizeof(struct gpio_v2_line_info_changed)) {
+ if (copy_to_user(buf + bytes_read, &event, event_size))
+ return -EFAULT;
+ } else {
+ struct gpioline_info_changed event_v1;
+
+ gpio_v2_line_info_changed_to_v1(&event, &event_v1);
+ if (copy_to_user(buf + bytes_read, &event_v1,
+ event_size))
+ return -EFAULT;
+ }
+#else
+ if (copy_to_user(buf + bytes_read, &event, event_size))
return -EFAULT;
- bytes_read += sizeof(event);
+#endif
+ bytes_read += event_size;
} while (count >= bytes_read + sizeof(event));
return bytes_read;
#ifndef GPIOLIB_CDEV_H
#define GPIOLIB_CDEV_H
-#include <linux/device.h>
+#include <linux/types.h>
+
+struct gpio_device;
+
+#ifdef CONFIG_GPIO_CDEV
int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt);
void gpiolib_cdev_unregister(struct gpio_device *gdev);
+#else
+
+static inline int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt)
+{
+ return 0;
+}
+
+static inline void gpiolib_cdev_unregister(struct gpio_device *gdev)
+{
+}
+
+#endif /* CONFIG_GPIO_CDEV */
+
#endif /* GPIOLIB_CDEV_H */
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Device property helpers for GPIO chips.
- *
- * Copyright (C) 2016, Intel Corporation
- * Author: Mika Westerberg <mika.westerberg@linux.intel.com>
- */
-
-#include <linux/property.h>
-#include <linux/slab.h>
-#include <linux/gpio/consumer.h>
-#include <linux/gpio/driver.h>
-#include <linux/export.h>
-
-#include "gpiolib.h"
-
-/**
- * devprop_gpiochip_set_names - Set GPIO line names using device properties
- * @chip: GPIO chip whose lines should be named, if possible
- * @fwnode: Property Node containing the gpio-line-names property
- *
- * Looks for device property "gpio-line-names" and if it exists assigns
- * GPIO line names for the chip. The memory allocated for the assigned
- * names belong to the underlying firmware node and should not be released
- * by the caller.
- */
-void devprop_gpiochip_set_names(struct gpio_chip *chip,
- const struct fwnode_handle *fwnode)
-{
- struct gpio_device *gdev = chip->gpiodev;
- const char **names;
- int ret, i;
- int count;
-
- count = fwnode_property_read_string_array(fwnode, "gpio-line-names",
- NULL, 0);
- if (count < 0)
- return;
-
- if (count > gdev->ngpio) {
- dev_warn(&gdev->dev, "gpio-line-names is length %d but should be at most length %d",
- count, gdev->ngpio);
- count = gdev->ngpio;
- }
-
- names = kcalloc(count, sizeof(*names), GFP_KERNEL);
- if (!names)
- return;
-
- ret = fwnode_property_read_string_array(fwnode, "gpio-line-names",
- names, count);
- if (ret < 0) {
- dev_warn(&gdev->dev, "failed to read GPIO line names\n");
- kfree(names);
- return;
- }
-
- for (i = 0; i < count; i++)
- gdev->descs[i].name = names[i];
-
- kfree(names);
-}
-EXPORT_SYMBOL_GPL(devprop_gpiochip_set_names);
if (ret)
return ret;
- /* If the chip defines names itself, these take precedence */
- if (!chip->names)
- devprop_gpiochip_set_names(chip,
- of_fwnode_handle(chip->of_node));
-
of_node_get(chip->of_node);
ret = of_gpiochip_scan_gpios(chip);
struct gpio_device *gdev = gc->gpiodev;
int i;
- if (!gc->names)
- return 0;
-
/* First check all names if they are unique */
for (i = 0; i != gc->ngpio; ++i) {
struct gpio_desc *gpio;
return 0;
}
+/*
+ * devprop_gpiochip_set_names - Set GPIO line names using device properties
+ * @chip: GPIO chip whose lines should be named, if possible
+ *
+ * Looks for device property "gpio-line-names" and if it exists assigns
+ * GPIO line names for the chip. The memory allocated for the assigned
+ * names belong to the underlying software node and should not be released
+ * by the caller.
+ */
+static int devprop_gpiochip_set_names(struct gpio_chip *chip)
+{
+ struct gpio_device *gdev = chip->gpiodev;
+ struct device *dev = chip->parent;
+ const char **names;
+ int ret, i;
+ int count;
+
+ /* GPIO chip may not have a parent device whose properties we inspect. */
+ if (!dev)
+ return 0;
+
+ count = device_property_string_array_count(dev, "gpio-line-names");
+ if (count < 0)
+ return 0;
+
+ if (count > gdev->ngpio) {
+ dev_warn(&gdev->dev, "gpio-line-names is length %d but should be at most length %d",
+ count, gdev->ngpio);
+ count = gdev->ngpio;
+ }
+
+ names = kcalloc(count, sizeof(*names), GFP_KERNEL);
+ if (!names)
+ return -ENOMEM;
+
+ ret = device_property_read_string_array(dev, "gpio-line-names",
+ names, count);
+ if (ret < 0) {
+ dev_warn(&gdev->dev, "failed to read GPIO line names\n");
+ kfree(names);
+ return ret;
+ }
+
+ for (i = 0; i < count; i++)
+ gdev->descs[i].name = names[i];
+
+ kfree(names);
+
+ return 0;
+}
+
static unsigned long *gpiochip_allocate_mask(struct gpio_chip *gc)
{
unsigned long *p;
struct gpio_device *gdev = dev_get_drvdata(dev);
list_del(&gdev->list);
- ida_simple_remove(&gpio_ida, gdev->id);
+ ida_free(&gpio_ida, gdev->id);
kfree_const(gdev->label);
kfree(gdev->descs);
kfree(gdev);
gc->of_node = gdev->dev.of_node;
#endif
- gdev->id = ida_simple_get(&gpio_ida, 0, 0, GFP_KERNEL);
+ gdev->id = ida_alloc(&gpio_ida, GFP_KERNEL);
if (gdev->id < 0) {
ret = gdev->id;
goto err_free_gdev;
INIT_LIST_HEAD(&gdev->pin_ranges);
#endif
- ret = gpiochip_set_desc_names(gc);
+ if (gc->names)
+ ret = gpiochip_set_desc_names(gc);
+ else
+ ret = devprop_gpiochip_set_names(gc);
if (ret)
goto err_remove_from_list;
err_free_descs:
kfree(gdev->descs);
err_free_ida:
- ida_simple_remove(&gpio_ida, gdev->id);
+ ida_free(&gpio_ida, gdev->id);
err_free_gdev:
/* failures here can mean systems won't boot... */
pr_err("%s: GPIOs %d..%d (%s) failed to register, %d\n", __func__,
clear_bit(FLAG_PULL_UP, &desc->flags);
clear_bit(FLAG_PULL_DOWN, &desc->flags);
clear_bit(FLAG_BIAS_DISABLE, &desc->flags);
+ clear_bit(FLAG_EDGE_RISING, &desc->flags);
+ clear_bit(FLAG_EDGE_FALLING, &desc->flags);
clear_bit(FLAG_IS_HOGGED, &desc->flags);
#ifdef CONFIG_OF_DYNAMIC
desc->hog = NULL;
+#endif
+#ifdef CONFIG_GPIO_CDEV
+ WRITE_ONCE(desc->debounce_period_us, 0);
#endif
ret = true;
}
return 0;
}
-static const struct seq_operations gpiolib_seq_ops = {
+static const struct seq_operations gpiolib_sops = {
.start = gpiolib_seq_start,
.next = gpiolib_seq_next,
.stop = gpiolib_seq_stop,
.show = gpiolib_seq_show,
};
-
-static int gpiolib_open(struct inode *inode, struct file *file)
-{
- return seq_open(file, &gpiolib_seq_ops);
-}
-
-static const struct file_operations gpiolib_operations = {
- .owner = THIS_MODULE,
- .open = gpiolib_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = seq_release,
-};
+DEFINE_SEQ_ATTRIBUTE(gpiolib);
static int __init gpiolib_debugfs_init(void)
{
/* /sys/kernel/debug/gpio */
- debugfs_create_file("gpio", S_IFREG | S_IRUGO, NULL, NULL,
- &gpiolib_operations);
+ debugfs_create_file("gpio", 0444, NULL, NULL, &gpiolib_fops);
return 0;
}
subsys_initcall(gpiolib_debugfs_init);
#define FLAG_PULL_UP 13 /* GPIO has pull up enabled */
#define FLAG_PULL_DOWN 14 /* GPIO has pull down enabled */
#define FLAG_BIAS_DISABLE 15 /* GPIO has pull disabled */
+#define FLAG_EDGE_RISING 16 /* GPIO CDEV detects rising edge events */
+#define FLAG_EDGE_FALLING 17 /* GPIO CDEV detects falling edge events */
/* Connection label */
const char *label;
#ifdef CONFIG_OF_DYNAMIC
struct device_node *hog;
#endif
+#ifdef CONFIG_GPIO_CDEV
+ /* debounce period in microseconds */
+ unsigned int debounce_period_us;
+#endif
};
int gpiod_request(struct gpio_desc *desc, const char *label);
})
/* GPUVM API */
-int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, unsigned int pasid,
+int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, u32 pasid,
void **vm, void **process_info,
struct dma_fence **ef);
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
- struct file *filp, unsigned int pasid,
+ struct file *filp, u32 pasid,
void **vm, void **process_info,
struct dma_fence **ef);
void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
unlock_srbm(kgd);
}
-static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
unsigned int vmid)
{
struct amdgpu_device *adev = get_amdgpu_device(kgd);
unlock_srbm(kgd);
}
-static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
unsigned int vmid)
{
struct amdgpu_device *adev = get_amdgpu_device(kgd);
unlock_srbm(kgd);
}
-static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
unsigned int vmid)
{
struct amdgpu_device *adev = get_amdgpu_device(kgd);
unlock_srbm(kgd);
}
-int kgd_gfx_v9_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+int kgd_gfx_v9_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
unsigned int vmid)
{
struct amdgpu_device *adev = get_amdgpu_device(kgd);
uint32_t sh_mem_config,
uint32_t sh_mem_ape1_base, uint32_t sh_mem_ape1_limit,
uint32_t sh_mem_bases);
-int kgd_gfx_v9_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid,
+int kgd_gfx_v9_set_pasid_vmid_mapping(struct kgd_dev *kgd, u32 pasid,
unsigned int vmid);
int kgd_gfx_v9_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id);
int kgd_gfx_v9_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
return ret;
}
-int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, unsigned int pasid,
+int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, u32 pasid,
void **vm, void **process_info,
struct dma_fence **ef)
{
}
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
- struct file *filp, unsigned int pasid,
+ struct file *filp, u32 pasid,
void **vm, void **process_info,
struct dma_fence **ef)
{
MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
-MODULE_FIRMWARE("amdgpu/sienna_cichlid_gpu_info.bin");
-MODULE_FIRMWARE("amdgpu/navy_flounder_gpu_info.bin");
#define AMDGPU_RESUME_MS 2000
case CHIP_CARRIZO:
case CHIP_STONEY:
case CHIP_VEGA20:
+ case CHIP_SIENNA_CICHLID:
+ case CHIP_NAVY_FLOUNDER:
default:
return 0;
case CHIP_VEGA10:
case CHIP_NAVI12:
chip_name = "navi12";
break;
- case CHIP_SIENNA_CICHLID:
- chip_name = "sienna_cichlid";
- break;
- case CHIP_NAVY_FLOUNDER:
- chip_name = "navy_flounder";
- break;
}
snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
take the current one */
if (active && !adev->have_disp_power_ref) {
adev->have_disp_power_ref = true;
- goto out;
+ return ret;
}
/* if we have no active crtcs, then drop the power ref
we got before */
{0x1002, 0x1636, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_RENOIR|AMD_IS_APU},
/* Navi12 */
- {0x1002, 0x7360, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVI12|AMD_EXP_HW_SUPPORT},
- {0x1002, 0x7362, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVI12|AMD_EXP_HW_SUPPORT},
+ {0x1002, 0x7360, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVI12},
+ {0x1002, 0x7362, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_NAVI12},
+
+ /* Sienna_Cichlid */
+ {0x1002, 0x73A0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID},
+ {0x1002, 0x73A2, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID},
+ {0x1002, 0x73A3, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID},
+ {0x1002, 0x73AB, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID},
+ {0x1002, 0x73AE, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID},
+ {0x1002, 0x73BF, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID},
{0, 0, 0}
};
/* Helper to free pasid from a fence callback */
struct amdgpu_pasid_cb {
struct dma_fence_cb cb;
- unsigned int pasid;
+ u32 pasid;
};
/**
* amdgpu_pasid_free - Free a PASID
* @pasid: PASID to free
*/
-void amdgpu_pasid_free(unsigned int pasid)
+void amdgpu_pasid_free(u32 pasid)
{
trace_amdgpu_pasid_freed(pasid);
ida_simple_remove(&amdgpu_pasid_ida, pasid);
* Free the pasid only after all the fences in resv are signaled.
*/
void amdgpu_pasid_free_delayed(struct dma_resv *resv,
- unsigned int pasid)
+ u32 pasid)
{
struct dma_fence *fence, **fences;
struct amdgpu_pasid_cb *cb;
};
int amdgpu_pasid_alloc(unsigned int bits);
-void amdgpu_pasid_free(unsigned int pasid);
+void amdgpu_pasid_free(u32 pasid);
void amdgpu_pasid_free_delayed(struct dma_resv *resv,
- unsigned int pasid);
+ u32 pasid);
bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev,
struct amdgpu_vmid *id);
struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
struct amdgpu_bo_list *list;
struct amdgpu_bo *pd;
- unsigned int pasid;
+ u32 pasid;
int handle;
if (!fpriv)
release_sg:
kfree(ttm->sg);
+ ttm->sg = NULL;
return r;
}
* 0 for success, error for failure.
*/
int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
- int vm_context, unsigned int pasid)
+ int vm_context, u32 pasid)
{
struct amdgpu_bo_param bp;
struct amdgpu_bo *root;
* 0 for success, -errno for errors.
*/
int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm,
- unsigned int pasid)
+ u32 pasid)
{
bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
int r;
* @pasid: PASID identifier for VM
* @task_info: task_info to fill.
*/
-void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid,
+void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid,
struct amdgpu_task_info *task_info)
{
struct amdgpu_vm *vm;
* Try to gracefully handle a VM fault. Return true if the fault was handled and
* shouldn't be reported any more.
*/
-bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, unsigned int pasid,
+bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
uint64_t addr)
{
struct amdgpu_bo *root;
long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout);
int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
- int vm_context, unsigned int pasid);
-int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned int pasid);
+ int vm_context, u32 pasid);
+int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, u32 pasid);
void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
struct amdgpu_job *job);
void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev);
-void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid,
+void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid,
struct amdgpu_task_info *task_info);
-bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, unsigned int pasid,
+bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
uint64_t addr);
void amdgpu_vm_set_task_info(struct amdgpu_vm *vm);
if (!gfx_v10_0_navi10_gfxoff_should_enable(adev))
adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
break;
+ case CHIP_NAVY_FLOUNDER:
+ adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
+ break;
default:
break;
}
* it doesn't support SRIOV. */
if (amdgpu_discovery) {
r = amdgpu_discovery_reg_base_init(adev);
- if (r) {
- DRM_WARN("failed to init reg base from ip discovery table, "
- "fallback to legacy init method\n");
- vega10_reg_base_init(adev);
- }
+ if (r == 0)
+ break;
+ DRM_WARN("failed to init reg base from ip discovery table, "
+ "fallback to legacy init method\n");
}
+ vega10_reg_base_init(adev);
break;
case CHIP_VEGA20:
vega20_reg_base_init(adev);
| UVD_SUVD_CGC_GATE__IME_HEVC_MASK
| UVD_SUVD_CGC_GATE__EFC_MASK
| UVD_SUVD_CGC_GATE__SAOE_MASK
- | 0x08000000
+ | UVD_SUVD_CGC_GATE__SRE_AV1_MASK
| UVD_SUVD_CGC_GATE__FBC_PCLK_MASK
| UVD_SUVD_CGC_GATE__FBC_CCLK_MASK
- | 0x40000000
+ | UVD_SUVD_CGC_GATE__SCM_AV1_MASK
| UVD_SUVD_CGC_GATE__SMPA_MASK);
WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE, data);
data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2);
data |= (UVD_SUVD_CGC_GATE2__MPBE0_MASK
| UVD_SUVD_CGC_GATE2__MPBE1_MASK
- | 0x00000004
- | 0x00000008
+ | UVD_SUVD_CGC_GATE2__SIT_AV1_MASK
+ | UVD_SUVD_CGC_GATE2__SDB_AV1_MASK
| UVD_SUVD_CGC_GATE2__MPC1_MASK);
WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2, data);
| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
- | 0x00008000
- | 0x00010000
+ | UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
| UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
| UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
| UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
- | 0x00008000
- | 0x00010000
+ | UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
+ | UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
| UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
| UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
| UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
(const struct cik_ih_ring_entry *)ih_ring_entry;
uint32_t context_id = ihre->data & 0xfffffff;
unsigned int vmid = (ihre->ring_id & 0x0000ff00) >> 8;
- unsigned int pasid = (ihre->ring_id & 0xffff0000) >> 16;
+ u32 pasid = (ihre->ring_id & 0xffff0000) >> 16;
if (pasid == 0)
return;
}
static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
- unsigned int pasid, uint64_t vmid0_address,
+ u32 pasid, uint64_t vmid0_address,
uint32_t *packet_buff, size_t size_in_bytes)
{
struct pm4__release_mem *rm_packet;
};
struct kfd_dbgmgr {
- unsigned int pasid;
+ u32 pasid;
struct kfd_dev *dev;
struct kfd_dbgdev *dbgdev;
};
#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
- unsigned int pasid, unsigned int vmid);
+ u32 pasid, unsigned int vmid);
static int execute_queues_cpsch(struct device_queue_manager *dqm,
enum kfd_unmap_queues_filter filter,
}
static int
-set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
+set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,
unsigned int vmid)
{
return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
kfree(dqm);
}
-int kfd_process_vm_fault(struct device_queue_manager *dqm,
- unsigned int pasid)
+int kfd_process_vm_fault(struct device_queue_manager *dqm, u32 pasid)
{
struct kfd_process_device *pdd;
struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
}
}
-void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
+void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
uint32_t valid_id_bits)
{
struct kfd_event *ev = NULL;
}
#ifdef KFD_SUPPORT_IOMMU_V2
-void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
+void kfd_signal_iommu_event(struct kfd_dev *dev, u32 pasid,
unsigned long address, bool is_write_requested,
bool is_execute_requested)
{
}
#endif /* KFD_SUPPORT_IOMMU_V2 */
-void kfd_signal_hw_exception_event(unsigned int pasid)
+void kfd_signal_hw_exception_event(u32 pasid)
{
/*
* Because we are called from arbitrary context (workqueue) as opposed
kfd_unref_process(p);
}
-void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
+void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,
struct kfd_vm_fault_info *info)
{
struct kfd_event *ev;
#define KFD_EVENT_TYPE_DEBUG 5
#define KFD_EVENT_TYPE_MEMORY 8
-extern void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
- uint32_t valid_id_bits);
+extern void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
+ uint32_t valid_id_bits);
#endif
}
/* Callback for process shutdown invoked by the IOMMU driver */
-static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid)
+static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, u32 pasid)
{
struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
struct kfd_process *p;
}
/* This function called by IOMMU driver on PPR failure */
-static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid,
- unsigned long address, u16 flags)
+static int iommu_invalid_ppr_cb(struct pci_dev *pdev, u32 pasid,
+ unsigned long address, u16 flags)
{
struct kfd_dev *dev;
return 1U << pasid_bits;
}
-unsigned int kfd_pasid_alloc(void)
+u32 kfd_pasid_alloc(void)
{
int r = amdgpu_pasid_alloc(pasid_bits);
return 0;
}
-void kfd_pasid_free(unsigned int pasid)
+void kfd_pasid_free(u32 pasid)
{
amdgpu_pasid_free(pasid);
}
/* We want to receive a notification when the mm_struct is destroyed */
struct mmu_notifier mmu_notifier;
- uint16_t pasid;
+ u32 pasid;
unsigned int doorbell_index;
/*
void kfd_process_destroy_wq(void);
struct kfd_process *kfd_create_process(struct file *filep);
struct kfd_process *kfd_get_process(const struct task_struct *);
-struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
+struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid);
struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
void kfd_unref_process(struct kfd_process *p);
int kfd_process_evict_queues(struct kfd_process *p);
void kfd_pasid_exit(void);
bool kfd_set_pasid_limit(unsigned int new_limit);
unsigned int kfd_get_pasid_limit(void);
-unsigned int kfd_pasid_alloc(void);
-void kfd_pasid_free(unsigned int pasid);
+u32 kfd_pasid_alloc(void);
+void kfd_pasid_free(u32 pasid);
/* Doorbells */
size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
enum kfd_queue_type type);
void kernel_queue_uninit(struct kernel_queue *kq, bool hanging);
-int kfd_process_vm_fault(struct device_queue_manager *dqm, unsigned int pasid);
+int kfd_process_vm_fault(struct device_queue_manager *dqm, u32 pasid);
/* Process Queue Manager */
struct process_queue_node {
uint32_t num_events, void __user *data,
bool all, uint32_t user_timeout_ms,
uint32_t *wait_result);
-void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
+void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
uint32_t valid_id_bits);
void kfd_signal_iommu_event(struct kfd_dev *dev,
- unsigned int pasid, unsigned long address,
- bool is_write_requested, bool is_execute_requested);
-void kfd_signal_hw_exception_event(unsigned int pasid);
+ u32 pasid, unsigned long address,
+ bool is_write_requested, bool is_execute_requested);
+void kfd_signal_hw_exception_event(u32 pasid);
int kfd_set_event(struct kfd_process *p, uint32_t event_id);
int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
uint64_t *event_page_offset, uint32_t *event_slot_index);
int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
-void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
+void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,
struct kfd_vm_fault_info *info);
void kfd_signal_reset_event(struct kfd_dev *dev);
}
/* This increments the process->ref counter. */
-struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
+struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
{
struct kfd_process *p, *ret_p = NULL;
unsigned int temp;
if (dmcu)
ret = dmcu_load_iram(dmcu, params);
else if (adev->dm.dc->ctx->dmub_srv)
- ret = dmub_init_abm_config(adev->dm.dc->res_pool->abm, params);
+ ret = dmub_init_abm_config(adev->dm.dc->res_pool, params);
if (!ret)
return -EINVAL;
int i = 0;
hdcp_work = kcalloc(max_caps, sizeof(*hdcp_work), GFP_KERNEL);
- if (hdcp_work == NULL)
+ if (ZERO_OR_NULL_PTR(hdcp_work))
return NULL;
hdcp_work->srm = kcalloc(PSP_HDCP_SRM_FIRST_GEN_MAX_SIZE, sizeof(*hdcp_work->srm), GFP_KERNEL);
} else {
struct clk_log_info log_info = {0};
- clk_mgr->smu_ver = rn_vbios_smu_get_smu_version(clk_mgr);
clk_mgr->periodic_retraining_disabled = rn_vbios_smu_is_periodic_retraining_disabled(clk_mgr);
/* SMU Version 55.51.0 and up no longer have an issue
dcn30_dio_link_encoder.o dcn30_resource.o
-CFLAGS_$(AMDDALPATH)/dc/dcn30/dcn30_optc.o := -mhard-float -msse -mpreferred-stack-boundary=4
-
+ifdef CONFIG_X86
CFLAGS_$(AMDDALPATH)/dc/dcn30/dcn30_resource.o := -mhard-float -msse
+CFLAGS_$(AMDDALPATH)/dc/dcn30/dcn30_optc.o := -mhard-float -msse
+endif
+
+ifdef CONFIG_PPC64
+CFLAGS_$(AMDDALPATH)/dc/dcn30/dcn30_resource.o := -mhard-float -maltivec
+CFLAGS_$(AMDDALPATH)/dc/dcn30/dcn30_optc.o := -mhard-float -maltivec
+endif
+
+ifdef CONFIG_ARM64
+CFLAGS_REMOVE_$(AMDDALPATH)/dc/dcn30/dcn30_resource.o := -mgeneral-regs-only
+CFLAGS_REMOVE_$(AMDDALPATH)/dc/dcn30/dcn30_optc.o := -mgeneral-regs-only
+endif
+
ifdef CONFIG_CC_IS_GCC
ifeq ($(call cc-ifversion, -lt, 0701, y), y)
IS_OLD_GCC = 1
# GCC < 7.1 cannot compile code using `double` and -mpreferred-stack-boundary=3
# (8B stack alignment).
CFLAGS_$(AMDDALPATH)/dc/dcn30/dcn30_resource.o += -mpreferred-stack-boundary=4
+CFLAGS_$(AMDDALPATH)/dc/dcn30/dcn30_optc.o += -mpreferred-stack-boundary=4
else
CFLAGS_$(AMDDALPATH)/dc/dcn30/dcn30_resource.o += -msse2
+CFLAGS_$(AMDDALPATH)/dc/dcn30/dcn30_optc.o += -msse2
endif
AMD_DAL_DCN30 = $(addprefix $(AMDDALPATH)/dc/dcn30/,$(DCN30))
params, ram_table, big_endian);
}
-bool dmub_init_abm_config(struct abm *abm,
+bool dmub_init_abm_config(struct resource_pool *res_pool,
struct dmcu_iram_parameters params)
{
struct iram_table_v_2_2 ram_table;
bool result = false;
uint32_t i, j = 0;
- if (abm == NULL)
+#if defined(CONFIG_DRM_AMD_DC_DCN3_0)
+ if (res_pool->abm == NULL && res_pool->multiple_abms[0] == NULL)
return false;
+#else
+ if (res_pool->abm == NULL)
+ return false;
+#endif
memset(&ram_table, 0, sizeof(ram_table));
memset(&config, 0, sizeof(config));
config.min_abm_backlight = ram_table.min_abm_backlight;
- result = abm->funcs->init_abm_config(
- abm, (char *)(&config), sizeof(struct abm_config_table));
+#if defined(CONFIG_DRM_AMD_DC_DCN3_0)
+ if (res_pool->multiple_abms[0]) {
+ result = res_pool->multiple_abms[0]->funcs->init_abm_config(
+ res_pool->multiple_abms[0], (char *)(&config), sizeof(struct abm_config_table));
+ } else
+#endif
+ result = res_pool->abm->funcs->init_abm_config(
+ res_pool->abm, (char *)(&config), sizeof(struct abm_config_table));
return result;
}
#include "dc/inc/hw/dmcu.h"
#include "dc/inc/hw/abm.h"
+struct resource_pool;
+
enum abm_defines {
abm_defines_max_level = 4,
bool dmcu_load_iram(struct dmcu *dmcu,
struct dmcu_iram_parameters params);
-bool dmub_init_abm_config(struct abm *abm,
+bool dmub_init_abm_config(struct resource_pool *res_pool,
struct dmcu_iram_parameters params);
#endif /* MODULES_POWER_POWER_HELPERS_H_ */
#define mmDB_STENCIL_WRITE_BASE_DEFAULT 0x00000000
#define mmDB_RESERVED_REG_1_DEFAULT 0x00000000
#define mmDB_RESERVED_REG_3_DEFAULT 0x00000000
+#define mmDB_VRS_OVERRIDE_CNTL_DEFAULT 0x00000000
#define mmDB_Z_READ_BASE_HI_DEFAULT 0x00000000
#define mmDB_STENCIL_READ_BASE_HI_DEFAULT 0x00000000
#define mmDB_Z_WRITE_BASE_HI_DEFAULT 0x00000000
#define mmPA_SU_OVER_RASTERIZATION_CNTL_DEFAULT 0x00000000
#define mmPA_STEREO_CNTL_DEFAULT 0x00000000
#define mmPA_STATE_STEREO_X_DEFAULT 0x00000000
+#define mmPA_CL_VRS_CNTL_DEFAULT 0x00000000
#define mmPA_SU_POINT_SIZE_DEFAULT 0x00000000
#define mmPA_SU_POINT_MINMAX_DEFAULT 0x00000000
#define mmPA_SU_LINE_CNTL_DEFAULT 0x00000000
#define mmDB_RESERVED_REG_1_BASE_IDX 1
#define mmDB_RESERVED_REG_3 0x0017
#define mmDB_RESERVED_REG_3_BASE_IDX 1
+#define mmDB_VRS_OVERRIDE_CNTL 0x0019
+#define mmDB_VRS_OVERRIDE_CNTL_BASE_IDX 1
#define mmDB_Z_READ_BASE_HI 0x001a
#define mmDB_Z_READ_BASE_HI_BASE_IDX 1
#define mmDB_STENCIL_READ_BASE_HI 0x001b
#define mmPA_STEREO_CNTL_BASE_IDX 1
#define mmPA_STATE_STEREO_X 0x0211
#define mmPA_STATE_STEREO_X_BASE_IDX 1
+#define mmPA_CL_VRS_CNTL 0x0212
+#define mmPA_CL_VRS_CNTL_BASE_IDX 1
#define mmPA_SU_POINT_SIZE 0x0280
#define mmPA_SU_POINT_SIZE_BASE_IDX 1
#define mmPA_SU_POINT_MINMAX 0x0281
#define DB_EXCEPTION_CONTROL__AUTO_FLUSH_HTILE__SHIFT 0x3
#define DB_EXCEPTION_CONTROL__AUTO_FLUSH_QUAD__SHIFT 0x4
#define DB_EXCEPTION_CONTROL__FORCE_SUMMARIZE__SHIFT 0x8
+#define DB_EXCEPTION_CONTROL__FORCE_VRS_RATE_FINE__SHIFT 0x10
#define DB_EXCEPTION_CONTROL__DTAG_WATERMARK__SHIFT 0x18
#define DB_EXCEPTION_CONTROL__EARLY_Z_PANIC_DISABLE_MASK 0x00000001L
#define DB_EXCEPTION_CONTROL__LATE_Z_PANIC_DISABLE_MASK 0x00000002L
#define DB_EXCEPTION_CONTROL__AUTO_FLUSH_HTILE_MASK 0x00000008L
#define DB_EXCEPTION_CONTROL__AUTO_FLUSH_QUAD_MASK 0x00000010L
#define DB_EXCEPTION_CONTROL__FORCE_SUMMARIZE_MASK 0x00000F00L
+#define DB_EXCEPTION_CONTROL__FORCE_VRS_RATE_FINE_MASK 0x00FF0000L
#define DB_EXCEPTION_CONTROL__DTAG_WATERMARK_MASK 0x7F000000L
//DB_DFSM_CONFIG
#define DB_DFSM_CONFIG__BYPASS_DFSM__SHIFT 0x0
#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CM__SHIFT 0x18
#define CB_HW_CONTROL_3__DISABLE_NACK_COLOR_RD_WR_OPT__SHIFT 0x19
#define CB_HW_CONTROL_3__DISABLE_BLENDER_CLOCK_GATING__SHIFT 0x1a
+#define CB_HW_CONTROL_3__DISABLE_DCC_VRS_OPT__SHIFT 0x1c
#define CB_HW_CONTROL_3__DISABLE_FMASK_NOFETCH_OPT__SHIFT 0x1e
#define CB_HW_CONTROL_3__DISABLE_FMASK_NOFETCH_OPT_BC__SHIFT 0x1f
#define CB_HW_CONTROL_3__DISABLE_SLOW_MODE_EMPTY_HALF_QUAD_KILL_MASK 0x00000001L
#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CM_MASK 0x01000000L
#define CB_HW_CONTROL_3__DISABLE_NACK_COLOR_RD_WR_OPT_MASK 0x02000000L
#define CB_HW_CONTROL_3__DISABLE_BLENDER_CLOCK_GATING_MASK 0x04000000L
+#define CB_HW_CONTROL_3__DISABLE_DCC_VRS_OPT_MASK 0x10000000L
#define CB_HW_CONTROL_3__DISABLE_FMASK_NOFETCH_OPT_MASK 0x40000000L
#define CB_HW_CONTROL_3__DISABLE_FMASK_NOFETCH_OPT_BC_MASK 0x80000000L
//CB_HW_CONTROL
#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE__SHIFT 0x0
+#define CB_HW_CONTROL__DISABLE_VRS_FILLRATE_OPTIMIZATION__SHIFT 0x1
#define CB_HW_CONTROL__DISABLE_FILLRATE_OPT_FIX_WITH_CFC__SHIFT 0x3
#define CB_HW_CONTROL__DISABLE_POST_DCC_WITH_CFC_FIX__SHIFT 0x4
+#define CB_HW_CONTROL__DISABLE_COMPRESS_1FRAG_WHEN_VRS_RATE_HINT_EN__SHIFT 0x5
#define CB_HW_CONTROL__RMI_CREDITS__SHIFT 0x6
#define CB_HW_CONTROL__CHICKEN_BITS__SHIFT 0xc
#define CB_HW_CONTROL__DISABLE_FMASK_MULTI_MGCG_DOMAINS__SHIFT 0xf
#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT__SHIFT 0x1e
#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE__SHIFT 0x1f
#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE_MASK 0x00000001L
+#define CB_HW_CONTROL__DISABLE_VRS_FILLRATE_OPTIMIZATION_MASK 0x00000002L
#define CB_HW_CONTROL__DISABLE_FILLRATE_OPT_FIX_WITH_CFC_MASK 0x00000008L
#define CB_HW_CONTROL__DISABLE_POST_DCC_WITH_CFC_FIX_MASK 0x00000010L
+#define CB_HW_CONTROL__DISABLE_COMPRESS_1FRAG_WHEN_VRS_RATE_HINT_EN_MASK 0x00000020L
#define CB_HW_CONTROL__RMI_CREDITS_MASK 0x00000FC0L
#define CB_HW_CONTROL__CHICKEN_BITS_MASK 0x00007000L
#define CB_HW_CONTROL__DISABLE_FMASK_MULTI_MGCG_DOMAINS_MASK 0x00008000L
#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS__SHIFT 0x16
#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS__SHIFT 0x17
#define DB_RENDER_OVERRIDE2__ALLOW_PARTIAL_RES_HIER_KILL__SHIFT 0x19
+#define DB_RENDER_OVERRIDE2__FORCE_VRS_RATE_FINE__SHIFT 0x1a
#define DB_RENDER_OVERRIDE2__CENTROID_COMPUTATION_MODE__SHIFT 0x1b
#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL_MASK 0x00000003L
#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN_MASK 0x0000001CL
#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS_MASK 0x00400000L
#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS_MASK 0x00800000L
#define DB_RENDER_OVERRIDE2__ALLOW_PARTIAL_RES_HIER_KILL_MASK 0x02000000L
+#define DB_RENDER_OVERRIDE2__FORCE_VRS_RATE_FINE_MASK 0x04000000L
#define DB_RENDER_OVERRIDE2__CENTROID_COMPUTATION_MODE_MASK 0x18000000L
//DB_HTILE_DATA_BASE
#define DB_HTILE_DATA_BASE__BASE_256B__SHIFT 0x0
//DB_RESERVED_REG_3
#define DB_RESERVED_REG_3__FIELD_1__SHIFT 0x0
#define DB_RESERVED_REG_3__FIELD_1_MASK 0x003FFFFFL
+//DB_VRS_OVERRIDE_CNTL
+#define DB_VRS_OVERRIDE_CNTL__VRS_OVERRIDE_RATE_COMBINER_MODE__SHIFT 0x0
+#define DB_VRS_OVERRIDE_CNTL__VRS_OVERRIDE_RATE_X__SHIFT 0x4
+#define DB_VRS_OVERRIDE_CNTL__VRS_OVERRIDE_RATE_Y__SHIFT 0x6
+#define DB_VRS_OVERRIDE_CNTL__VRS_OVERRIDE_RATE_COMBINER_MODE_MASK 0x00000007L
+#define DB_VRS_OVERRIDE_CNTL__VRS_OVERRIDE_RATE_X_MASK 0x00000030L
+#define DB_VRS_OVERRIDE_CNTL__VRS_OVERRIDE_RATE_Y_MASK 0x000000C0L
//DB_Z_READ_BASE_HI
#define DB_Z_READ_BASE_HI__BASE_HI__SHIFT 0x0
#define DB_Z_READ_BASE_HI__BASE_HI_MASK 0x000000FFL
#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA__SHIFT 0x18
#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG__SHIFT 0x19
#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH__SHIFT 0x1b
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VRS_RATE__SHIFT 0x1c
#define PA_CL_VS_OUT_CNTL__BYPASS_VTX_RATE_COMBINER__SHIFT 0x1d
#define PA_CL_VS_OUT_CNTL__BYPASS_PRIM_RATE_COMBINER__SHIFT 0x1e
#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0_MASK 0x00000001L
#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA_MASK 0x01000000L
#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG_MASK 0x02000000L
#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH_MASK 0x08000000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VRS_RATE_MASK 0x10000000L
#define PA_CL_VS_OUT_CNTL__BYPASS_VTX_RATE_COMBINER_MASK 0x20000000L
#define PA_CL_VS_OUT_CNTL__BYPASS_PRIM_RATE_COMBINER_MASK 0x40000000L
//PA_CL_NANINF_CNTL
//PA_STATE_STEREO_X
#define PA_STATE_STEREO_X__STEREO_X_OFFSET__SHIFT 0x0
#define PA_STATE_STEREO_X__STEREO_X_OFFSET_MASK 0xFFFFFFFFL
+//PA_CL_VRS_CNTL
+#define PA_CL_VRS_CNTL__VERTEX_RATE_COMBINER_MODE__SHIFT 0x0
+#define PA_CL_VRS_CNTL__PRIMITIVE_RATE_COMBINER_MODE__SHIFT 0x3
+#define PA_CL_VRS_CNTL__HTILE_RATE_COMBINER_MODE__SHIFT 0x6
+#define PA_CL_VRS_CNTL__SAMPLE_ITER_COMBINER_MODE__SHIFT 0x9
+#define PA_CL_VRS_CNTL__EXPOSE_VRS_PIXELS_MASK__SHIFT 0xd
+#define PA_CL_VRS_CNTL__CMASK_RATE_HINT_FORCE_ZERO__SHIFT 0xe
+#define PA_CL_VRS_CNTL__VERTEX_RATE_COMBINER_MODE_MASK 0x00000007L
+#define PA_CL_VRS_CNTL__PRIMITIVE_RATE_COMBINER_MODE_MASK 0x00000038L
+#define PA_CL_VRS_CNTL__HTILE_RATE_COMBINER_MODE_MASK 0x000001C0L
+#define PA_CL_VRS_CNTL__SAMPLE_ITER_COMBINER_MODE_MASK 0x00000E00L
+#define PA_CL_VRS_CNTL__EXPOSE_VRS_PIXELS_MASK_MASK 0x00002000L
+#define PA_CL_VRS_CNTL__CMASK_RATE_HINT_FORCE_ZERO_MASK 0x00004000L
//PA_SU_POINT_SIZE
#define PA_SU_POINT_SIZE__HEIGHT__SHIFT 0x0
#define PA_SU_POINT_SIZE__WIDTH__SHIFT 0x10
#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE__SHIFT 0x10
#define DB_HTILE_SURFACE__RESERVED_FIELD_6__SHIFT 0x11
#define DB_HTILE_SURFACE__PIPE_ALIGNED__SHIFT 0x12
+#define DB_HTILE_SURFACE__VRS_HTILE_ENCODING__SHIFT 0x13
#define DB_HTILE_SURFACE__RESERVED_FIELD_1_MASK 0x00000001L
#define DB_HTILE_SURFACE__FULL_CACHE_MASK 0x00000002L
#define DB_HTILE_SURFACE__RESERVED_FIELD_2_MASK 0x00000004L
#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE_MASK 0x00010000L
#define DB_HTILE_SURFACE__RESERVED_FIELD_6_MASK 0x00020000L
#define DB_HTILE_SURFACE__PIPE_ALIGNED_MASK 0x00040000L
+#define DB_HTILE_SURFACE__VRS_HTILE_ENCODING_MASK 0x00180000L
//DB_SRESULTS_COMPARE_STATE0
#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0__SHIFT 0x0
#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0__SHIFT 0x4
#define CB_COLOR0_ATTRIB3__CMASK_PIPE_ALIGNED__SHIFT 0x1a
#define CB_COLOR0_ATTRIB3__RESOURCE_LEVEL__SHIFT 0x1b
#define CB_COLOR0_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT 0x1e
+#define CB_COLOR0_ATTRIB3__VRS_RATE_HINT_ENABLE__SHIFT 0x1f
#define CB_COLOR0_ATTRIB3__MIP0_DEPTH_MASK 0x00001FFFL
#define CB_COLOR0_ATTRIB3__META_LINEAR_MASK 0x00002000L
#define CB_COLOR0_ATTRIB3__COLOR_SW_MODE_MASK 0x0007C000L
#define CB_COLOR0_ATTRIB3__CMASK_PIPE_ALIGNED_MASK 0x04000000L
#define CB_COLOR0_ATTRIB3__RESOURCE_LEVEL_MASK 0x38000000L
#define CB_COLOR0_ATTRIB3__DCC_PIPE_ALIGNED_MASK 0x40000000L
+#define CB_COLOR0_ATTRIB3__VRS_RATE_HINT_ENABLE_MASK 0x80000000L
//CB_COLOR1_ATTRIB3
#define CB_COLOR1_ATTRIB3__MIP0_DEPTH__SHIFT 0x0
#define CB_COLOR1_ATTRIB3__META_LINEAR__SHIFT 0xd
#define CB_COLOR1_ATTRIB3__CMASK_PIPE_ALIGNED__SHIFT 0x1a
#define CB_COLOR1_ATTRIB3__RESOURCE_LEVEL__SHIFT 0x1b
#define CB_COLOR1_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT 0x1e
+#define CB_COLOR1_ATTRIB3__VRS_RATE_HINT_ENABLE__SHIFT 0x1f
#define CB_COLOR1_ATTRIB3__MIP0_DEPTH_MASK 0x00001FFFL
#define CB_COLOR1_ATTRIB3__META_LINEAR_MASK 0x00002000L
#define CB_COLOR1_ATTRIB3__COLOR_SW_MODE_MASK 0x0007C000L
#define CB_COLOR1_ATTRIB3__CMASK_PIPE_ALIGNED_MASK 0x04000000L
#define CB_COLOR1_ATTRIB3__RESOURCE_LEVEL_MASK 0x38000000L
#define CB_COLOR1_ATTRIB3__DCC_PIPE_ALIGNED_MASK 0x40000000L
+#define CB_COLOR1_ATTRIB3__VRS_RATE_HINT_ENABLE_MASK 0x80000000L
//CB_COLOR2_ATTRIB3
#define CB_COLOR2_ATTRIB3__MIP0_DEPTH__SHIFT 0x0
#define CB_COLOR2_ATTRIB3__META_LINEAR__SHIFT 0xd
#define CB_COLOR2_ATTRIB3__CMASK_PIPE_ALIGNED__SHIFT 0x1a
#define CB_COLOR2_ATTRIB3__RESOURCE_LEVEL__SHIFT 0x1b
#define CB_COLOR2_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT 0x1e
+#define CB_COLOR2_ATTRIB3__VRS_RATE_HINT_ENABLE__SHIFT 0x1f
#define CB_COLOR2_ATTRIB3__MIP0_DEPTH_MASK 0x00001FFFL
#define CB_COLOR2_ATTRIB3__META_LINEAR_MASK 0x00002000L
#define CB_COLOR2_ATTRIB3__COLOR_SW_MODE_MASK 0x0007C000L
#define CB_COLOR2_ATTRIB3__CMASK_PIPE_ALIGNED_MASK 0x04000000L
#define CB_COLOR2_ATTRIB3__RESOURCE_LEVEL_MASK 0x38000000L
#define CB_COLOR2_ATTRIB3__DCC_PIPE_ALIGNED_MASK 0x40000000L
+#define CB_COLOR2_ATTRIB3__VRS_RATE_HINT_ENABLE_MASK 0x80000000L
//CB_COLOR3_ATTRIB3
#define CB_COLOR3_ATTRIB3__MIP0_DEPTH__SHIFT 0x0
#define CB_COLOR3_ATTRIB3__META_LINEAR__SHIFT 0xd
#define CB_COLOR3_ATTRIB3__CMASK_PIPE_ALIGNED__SHIFT 0x1a
#define CB_COLOR3_ATTRIB3__RESOURCE_LEVEL__SHIFT 0x1b
#define CB_COLOR3_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT 0x1e
+#define CB_COLOR3_ATTRIB3__VRS_RATE_HINT_ENABLE__SHIFT 0x1f
#define CB_COLOR3_ATTRIB3__MIP0_DEPTH_MASK 0x00001FFFL
#define CB_COLOR3_ATTRIB3__META_LINEAR_MASK 0x00002000L
#define CB_COLOR3_ATTRIB3__COLOR_SW_MODE_MASK 0x0007C000L
#define CB_COLOR3_ATTRIB3__CMASK_PIPE_ALIGNED_MASK 0x04000000L
#define CB_COLOR3_ATTRIB3__RESOURCE_LEVEL_MASK 0x38000000L
#define CB_COLOR3_ATTRIB3__DCC_PIPE_ALIGNED_MASK 0x40000000L
+#define CB_COLOR3_ATTRIB3__VRS_RATE_HINT_ENABLE_MASK 0x80000000L
//CB_COLOR4_ATTRIB3
#define CB_COLOR4_ATTRIB3__MIP0_DEPTH__SHIFT 0x0
#define CB_COLOR4_ATTRIB3__META_LINEAR__SHIFT 0xd
#define CB_COLOR4_ATTRIB3__CMASK_PIPE_ALIGNED__SHIFT 0x1a
#define CB_COLOR4_ATTRIB3__RESOURCE_LEVEL__SHIFT 0x1b
#define CB_COLOR4_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT 0x1e
+#define CB_COLOR4_ATTRIB3__VRS_RATE_HINT_ENABLE__SHIFT 0x1f
#define CB_COLOR4_ATTRIB3__MIP0_DEPTH_MASK 0x00001FFFL
#define CB_COLOR4_ATTRIB3__META_LINEAR_MASK 0x00002000L
#define CB_COLOR4_ATTRIB3__COLOR_SW_MODE_MASK 0x0007C000L
#define CB_COLOR4_ATTRIB3__CMASK_PIPE_ALIGNED_MASK 0x04000000L
#define CB_COLOR4_ATTRIB3__RESOURCE_LEVEL_MASK 0x38000000L
#define CB_COLOR4_ATTRIB3__DCC_PIPE_ALIGNED_MASK 0x40000000L
+#define CB_COLOR4_ATTRIB3__VRS_RATE_HINT_ENABLE_MASK 0x80000000L
//CB_COLOR5_ATTRIB3
#define CB_COLOR5_ATTRIB3__MIP0_DEPTH__SHIFT 0x0
#define CB_COLOR5_ATTRIB3__META_LINEAR__SHIFT 0xd
#define CB_COLOR5_ATTRIB3__CMASK_PIPE_ALIGNED__SHIFT 0x1a
#define CB_COLOR5_ATTRIB3__RESOURCE_LEVEL__SHIFT 0x1b
#define CB_COLOR5_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT 0x1e
+#define CB_COLOR5_ATTRIB3__VRS_RATE_HINT_ENABLE__SHIFT 0x1f
#define CB_COLOR5_ATTRIB3__MIP0_DEPTH_MASK 0x00001FFFL
#define CB_COLOR5_ATTRIB3__META_LINEAR_MASK 0x00002000L
#define CB_COLOR5_ATTRIB3__COLOR_SW_MODE_MASK 0x0007C000L
#define CB_COLOR5_ATTRIB3__CMASK_PIPE_ALIGNED_MASK 0x04000000L
#define CB_COLOR5_ATTRIB3__RESOURCE_LEVEL_MASK 0x38000000L
#define CB_COLOR5_ATTRIB3__DCC_PIPE_ALIGNED_MASK 0x40000000L
+#define CB_COLOR5_ATTRIB3__VRS_RATE_HINT_ENABLE_MASK 0x80000000L
//CB_COLOR6_ATTRIB3
#define CB_COLOR6_ATTRIB3__MIP0_DEPTH__SHIFT 0x0
#define CB_COLOR6_ATTRIB3__META_LINEAR__SHIFT 0xd
#define CB_COLOR6_ATTRIB3__CMASK_PIPE_ALIGNED__SHIFT 0x1a
#define CB_COLOR6_ATTRIB3__RESOURCE_LEVEL__SHIFT 0x1b
#define CB_COLOR6_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT 0x1e
+#define CB_COLOR6_ATTRIB3__VRS_RATE_HINT_ENABLE__SHIFT 0x1f
#define CB_COLOR6_ATTRIB3__MIP0_DEPTH_MASK 0x00001FFFL
#define CB_COLOR6_ATTRIB3__META_LINEAR_MASK 0x00002000L
#define CB_COLOR6_ATTRIB3__COLOR_SW_MODE_MASK 0x0007C000L
#define CB_COLOR6_ATTRIB3__CMASK_PIPE_ALIGNED_MASK 0x04000000L
#define CB_COLOR6_ATTRIB3__RESOURCE_LEVEL_MASK 0x38000000L
#define CB_COLOR6_ATTRIB3__DCC_PIPE_ALIGNED_MASK 0x40000000L
+#define CB_COLOR6_ATTRIB3__VRS_RATE_HINT_ENABLE_MASK 0x80000000L
//CB_COLOR7_ATTRIB3
#define CB_COLOR7_ATTRIB3__MIP0_DEPTH__SHIFT 0x0
#define CB_COLOR7_ATTRIB3__META_LINEAR__SHIFT 0xd
#define CB_COLOR7_ATTRIB3__CMASK_PIPE_ALIGNED__SHIFT 0x1a
#define CB_COLOR7_ATTRIB3__RESOURCE_LEVEL__SHIFT 0x1b
#define CB_COLOR7_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT 0x1e
+#define CB_COLOR7_ATTRIB3__VRS_RATE_HINT_ENABLE__SHIFT 0x1f
#define CB_COLOR7_ATTRIB3__MIP0_DEPTH_MASK 0x00001FFFL
#define CB_COLOR7_ATTRIB3__META_LINEAR_MASK 0x00002000L
#define CB_COLOR7_ATTRIB3__COLOR_SW_MODE_MASK 0x0007C000L
#define CB_COLOR7_ATTRIB3__CMASK_PIPE_ALIGNED_MASK 0x04000000L
#define CB_COLOR7_ATTRIB3__RESOURCE_LEVEL_MASK 0x38000000L
#define CB_COLOR7_ATTRIB3__DCC_PIPE_ALIGNED_MASK 0x40000000L
+#define CB_COLOR7_ATTRIB3__VRS_RATE_HINT_ENABLE_MASK 0x80000000L
// addressBlock: gc_gfxudec
#define VCN_FEATURES__HAS_MJPEG2_IDCT_DEC__SHIFT 0x7
#define VCN_FEATURES__HAS_SCLR_DEC__SHIFT 0x8
#define VCN_FEATURES__HAS_VP9_DEC__SHIFT 0x9
+#define VCN_FEATURES__HAS_AV1_DEC__SHIFT 0xa
#define VCN_FEATURES__HAS_EFC_ENC__SHIFT 0xb
#define VCN_FEATURES__HAS_EFC_HDR2SDR_ENC__SHIFT 0xc
#define VCN_FEATURES__HAS_DUAL_MJPEG_DEC__SHIFT 0xd
#define VCN_FEATURES__HAS_MJPEG2_IDCT_DEC_MASK 0x00000080L
#define VCN_FEATURES__HAS_SCLR_DEC_MASK 0x00000100L
#define VCN_FEATURES__HAS_VP9_DEC_MASK 0x00000200L
+#define VCN_FEATURES__HAS_AV1_DEC_MASK 0x00000400L
#define VCN_FEATURES__HAS_EFC_ENC_MASK 0x00000800L
#define VCN_FEATURES__HAS_EFC_HDR2SDR_ENC_MASK 0x00001000L
#define VCN_FEATURES__HAS_DUAL_MJPEG_DEC_MASK 0x00002000L
#define UVD_SUVD_CGC_GATE__IME_HEVC__SHIFT 0x18
#define UVD_SUVD_CGC_GATE__EFC__SHIFT 0x19
#define UVD_SUVD_CGC_GATE__SAOE__SHIFT 0x1a
+#define UVD_SUVD_CGC_GATE__SRE_AV1__SHIFT 0x1b
#define UVD_SUVD_CGC_GATE__FBC_PCLK__SHIFT 0x1c
#define UVD_SUVD_CGC_GATE__FBC_CCLK__SHIFT 0x1d
+#define UVD_SUVD_CGC_GATE__SCM_AV1__SHIFT 0x1e
#define UVD_SUVD_CGC_GATE__SMPA__SHIFT 0x1f
#define UVD_SUVD_CGC_GATE__SRE_MASK 0x00000001L
#define UVD_SUVD_CGC_GATE__SIT_MASK 0x00000002L
#define UVD_SUVD_CGC_GATE__IME_HEVC_MASK 0x01000000L
#define UVD_SUVD_CGC_GATE__EFC_MASK 0x02000000L
#define UVD_SUVD_CGC_GATE__SAOE_MASK 0x04000000L
+#define UVD_SUVD_CGC_GATE__SRE_AV1_MASK 0x08000000L
#define UVD_SUVD_CGC_GATE__FBC_PCLK_MASK 0x10000000L
#define UVD_SUVD_CGC_GATE__FBC_CCLK_MASK 0x20000000L
+#define UVD_SUVD_CGC_GATE__SCM_AV1_MASK 0x40000000L
#define UVD_SUVD_CGC_GATE__SMPA_MASK 0x80000000L
//UVD_SUVD_CGC_STATUS
#define UVD_SUVD_CGC_STATUS__SRE_VCLK__SHIFT 0x0
#define UVD_SUVD_CGC_STATUS__IME_HEVC_DCLK__SHIFT 0x1b
#define UVD_SUVD_CGC_STATUS__EFC_DCLK__SHIFT 0x1c
#define UVD_SUVD_CGC_STATUS__SAOE_DCLK__SHIFT 0x1d
+#define UVD_SUVD_CGC_STATUS__SRE_AV1_VCLK__SHIFT 0x1e
+#define UVD_SUVD_CGC_STATUS__SCM_AV1_DCLK__SHIFT 0x1f
#define UVD_SUVD_CGC_STATUS__SRE_VCLK_MASK 0x00000001L
#define UVD_SUVD_CGC_STATUS__SRE_DCLK_MASK 0x00000002L
#define UVD_SUVD_CGC_STATUS__SIT_DCLK_MASK 0x00000004L
#define UVD_SUVD_CGC_STATUS__IME_HEVC_DCLK_MASK 0x08000000L
#define UVD_SUVD_CGC_STATUS__EFC_DCLK_MASK 0x10000000L
#define UVD_SUVD_CGC_STATUS__SAOE_DCLK_MASK 0x20000000L
+#define UVD_SUVD_CGC_STATUS__SRE_AV1_VCLK_MASK 0x40000000L
+#define UVD_SUVD_CGC_STATUS__SCM_AV1_DCLK_MASK 0x80000000L
//UVD_SUVD_CGC_CTRL
#define UVD_SUVD_CGC_CTRL__SRE_MODE__SHIFT 0x0
#define UVD_SUVD_CGC_CTRL__SIT_MODE__SHIFT 0x1
#define UVD_SUVD_CGC_CTRL__SMPA_MODE__SHIFT 0xc
#define UVD_SUVD_CGC_CTRL__MPBE0_MODE__SHIFT 0xd
#define UVD_SUVD_CGC_CTRL__MPBE1_MODE__SHIFT 0xe
+#define UVD_SUVD_CGC_CTRL__SIT_AV1_MODE__SHIFT 0xf
+#define UVD_SUVD_CGC_CTRL__SDB_AV1_MODE__SHIFT 0x10
#define UVD_SUVD_CGC_CTRL__MPC1_MODE__SHIFT 0x11
#define UVD_SUVD_CGC_CTRL__FBC_PCLK__SHIFT 0x1c
#define UVD_SUVD_CGC_CTRL__FBC_CCLK__SHIFT 0x1d
#define UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK 0x00001000L
#define UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK 0x00002000L
#define UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK 0x00004000L
+#define UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK 0x00008000L
+#define UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK 0x00010000L
#define UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK 0x00020000L
#define UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK 0x10000000L
#define UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK 0x20000000L
#define UVD_SUVD_CGC_STATUS2__SMPA_VCLK__SHIFT 0x0
#define UVD_SUVD_CGC_STATUS2__SMPA_DCLK__SHIFT 0x1
#define UVD_SUVD_CGC_STATUS2__MPBE1_DCLK__SHIFT 0x3
+#define UVD_SUVD_CGC_STATUS2__SIT_AV1_DCLK__SHIFT 0x4
+#define UVD_SUVD_CGC_STATUS2__SDB_AV1_DCLK__SHIFT 0x5
#define UVD_SUVD_CGC_STATUS2__MPC1_DCLK__SHIFT 0x6
#define UVD_SUVD_CGC_STATUS2__MPC1_SCLK__SHIFT 0x7
#define UVD_SUVD_CGC_STATUS2__MPC1_VCLK__SHIFT 0x8
#define UVD_SUVD_CGC_STATUS2__SMPA_VCLK_MASK 0x00000001L
#define UVD_SUVD_CGC_STATUS2__SMPA_DCLK_MASK 0x00000002L
#define UVD_SUVD_CGC_STATUS2__MPBE1_DCLK_MASK 0x00000008L
+#define UVD_SUVD_CGC_STATUS2__SIT_AV1_DCLK_MASK 0x00000010L
+#define UVD_SUVD_CGC_STATUS2__SDB_AV1_DCLK_MASK 0x00000020L
#define UVD_SUVD_CGC_STATUS2__MPC1_DCLK_MASK 0x00000040L
#define UVD_SUVD_CGC_STATUS2__MPC1_SCLK_MASK 0x00000080L
#define UVD_SUVD_CGC_STATUS2__MPC1_VCLK_MASK 0x00000100L
//UVD_SUVD_CGC_GATE2
#define UVD_SUVD_CGC_GATE2__MPBE0__SHIFT 0x0
#define UVD_SUVD_CGC_GATE2__MPBE1__SHIFT 0x1
+#define UVD_SUVD_CGC_GATE2__SIT_AV1__SHIFT 0x2
+#define UVD_SUVD_CGC_GATE2__SDB_AV1__SHIFT 0x3
#define UVD_SUVD_CGC_GATE2__MPC1__SHIFT 0x4
#define UVD_SUVD_CGC_GATE2__MPBE0_MASK 0x00000001L
#define UVD_SUVD_CGC_GATE2__MPBE1_MASK 0x00000002L
+#define UVD_SUVD_CGC_GATE2__SIT_AV1_MASK 0x00000004L
+#define UVD_SUVD_CGC_GATE2__SDB_AV1_MASK 0x00000008L
#define UVD_SUVD_CGC_GATE2__MPC1_MASK 0x00000010L
//UVD_SUVD_INT_STATUS2
#define UVD_SUVD_INT_STATUS2__SMPA_FUNC_INT__SHIFT 0x0
#define UVD_SUVD_INT_STATUS2__SMPA_ERR_INT__SHIFT 0x5
+#define UVD_SUVD_INT_STATUS2__SDB_AV1_FUNC_INT__SHIFT 0x6
+#define UVD_SUVD_INT_STATUS2__SDB_AV1_ERR_INT__SHIFT 0xb
#define UVD_SUVD_INT_STATUS2__SMPA_FUNC_INT_MASK 0x0000001FL
#define UVD_SUVD_INT_STATUS2__SMPA_ERR_INT_MASK 0x00000020L
+#define UVD_SUVD_INT_STATUS2__SDB_AV1_FUNC_INT_MASK 0x000007C0L
+#define UVD_SUVD_INT_STATUS2__SDB_AV1_ERR_INT_MASK 0x00000800L
//UVD_SUVD_INT_EN2
#define UVD_SUVD_INT_EN2__SMPA_FUNC_INT_EN__SHIFT 0x0
#define UVD_SUVD_INT_EN2__SMPA_ERR_INT_EN__SHIFT 0x5
+#define UVD_SUVD_INT_EN2__SDB_AV1_FUNC_INT_EN__SHIFT 0x6
+#define UVD_SUVD_INT_EN2__SDB_AV1_ERR_INT_EN__SHIFT 0xb
#define UVD_SUVD_INT_EN2__SMPA_FUNC_INT_EN_MASK 0x0000001FL
#define UVD_SUVD_INT_EN2__SMPA_ERR_INT_EN_MASK 0x00000020L
+#define UVD_SUVD_INT_EN2__SDB_AV1_FUNC_INT_EN_MASK 0x000007C0L
+#define UVD_SUVD_INT_EN2__SDB_AV1_ERR_INT_EN_MASK 0x00000800L
//UVD_SUVD_INT_ACK2
#define UVD_SUVD_INT_ACK2__SMPA_FUNC_INT_ACK__SHIFT 0x0
#define UVD_SUVD_INT_ACK2__SMPA_ERR_INT_ACK__SHIFT 0x5
+#define UVD_SUVD_INT_ACK2__SDB_AV1_FUNC_INT_ACK__SHIFT 0x6
+#define UVD_SUVD_INT_ACK2__SDB_AV1_ERR_INT_ACK__SHIFT 0xb
#define UVD_SUVD_INT_ACK2__SMPA_FUNC_INT_ACK_MASK 0x0000001FL
#define UVD_SUVD_INT_ACK2__SMPA_ERR_INT_ACK_MASK 0x00000020L
+#define UVD_SUVD_INT_ACK2__SDB_AV1_FUNC_INT_ACK_MASK 0x000007C0L
+#define UVD_SUVD_INT_ACK2__SDB_AV1_ERR_INT_ACK_MASK 0x00000800L
// addressBlock: uvd0_ecpudec
uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases);
- int (*set_pasid_vmid_mapping)(struct kgd_dev *kgd, unsigned int pasid,
+ int (*set_pasid_vmid_mapping)(struct kgd_dev *kgd, u32 pasid,
unsigned int vmid);
int (*init_interrupts)(struct kgd_dev *kgd, uint32_t pipe_id);
return ret;
}
- /*
- * Set initialized values (get from vbios) to dpm tables context such as
- * gfxclk, memclk, dcefclk, and etc. And enable the DPM feature for each
- * type of clks.
- */
- ret = smu_set_default_dpm_table(smu);
- if (ret) {
- dev_err(adev->dev, "Failed to setup default dpm clock tables!\n");
- return ret;
- }
-
ret = smu_populate_umd_state_clk(smu);
if (ret) {
dev_err(adev->dev, "Failed to populate UMD state clocks!\n");
return ret;
}
+ /*
+ * Set initialized values (get from vbios) to dpm tables context such as
+ * gfxclk, memclk, dcefclk, and etc. And enable the DPM feature for each
+ * type of clks.
+ */
+ ret = smu_set_default_dpm_table(smu);
+ if (ret) {
+ dev_err(adev->dev, "Failed to setup default dpm clock tables!\n");
+ return ret;
+ }
+
ret = smu_notify_display_change(smu);
if (ret)
return ret;
struct smu10_hwmgr *data = hwmgr->backend;
uint32_t min_sclk = hwmgr->display_config->min_core_set_clock;
uint32_t min_mclk = hwmgr->display_config->min_mem_set_clock/100;
+ uint32_t index_fclk = data->clock_vol_info.vdd_dep_on_fclk->count - 1;
+ uint32_t index_socclk = data->clock_vol_info.vdd_dep_on_socclk->count - 1;
if (hwmgr->smu_version < 0x1E3700) {
pr_info("smu firmware version too old, can not set dpm level\n");
smum_send_msg_to_smc_with_parameter(hwmgr,
PPSMC_MSG_SetHardMinFclkByFreq,
hwmgr->display_config->num_display > 3 ?
- SMU10_UMD_PSTATE_PEAK_FCLK :
+ data->clock_vol_info.vdd_dep_on_fclk->entries[0].clk :
min_mclk,
NULL);
smum_send_msg_to_smc_with_parameter(hwmgr,
PPSMC_MSG_SetHardMinSocclkByFreq,
- SMU10_UMD_PSTATE_MIN_SOCCLK,
+ data->clock_vol_info.vdd_dep_on_socclk->entries[0].clk,
NULL);
smum_send_msg_to_smc_with_parameter(hwmgr,
PPSMC_MSG_SetHardMinVcn,
NULL);
smum_send_msg_to_smc_with_parameter(hwmgr,
PPSMC_MSG_SetSoftMaxFclkByFreq,
- SMU10_UMD_PSTATE_PEAK_FCLK,
+ data->clock_vol_info.vdd_dep_on_fclk->entries[index_fclk].clk,
NULL);
smum_send_msg_to_smc_with_parameter(hwmgr,
PPSMC_MSG_SetSoftMaxSocclkByFreq,
- SMU10_UMD_PSTATE_PEAK_SOCCLK,
+ data->clock_vol_info.vdd_dep_on_socclk->entries[index_socclk].clk,
NULL);
smum_send_msg_to_smc_with_parameter(hwmgr,
PPSMC_MSG_SetSoftMaxVcn,
{
int i;
- BUG_ON(numbytes > MAX_SW_I2C_COMMANDS);
-
req->I2CcontrollerPort = 0;
req->I2CSpeed = 2;
req->SlaveAddress = address;
struct smu_table_context *smu_table = &adev->smu.smu_table;
struct smu_table *table = &smu_table->driver_table;
+ if (numbytes > MAX_SW_I2C_COMMANDS) {
+ dev_err(adev->dev, "numbytes requested %d is over max allowed %d\n",
+ numbytes, MAX_SW_I2C_COMMANDS);
+ return -EINVAL;
+ }
+
memset(&req, 0, sizeof(req));
navi10_fill_i2c_req(&req, false, address, numbytes, data);
SwI2cRequest_t req;
struct amdgpu_device *adev = to_amdgpu_device(control);
+ if (numbytes > MAX_SW_I2C_COMMANDS) {
+ dev_err(adev->dev, "numbytes requested %d is over max allowed %d\n",
+ numbytes, MAX_SW_I2C_COMMANDS);
+ return -EINVAL;
+ }
+
memset(&req, 0, sizeof(req));
navi10_fill_i2c_req(&req, true, address, numbytes, data);
*sclk_mask = 0;
} else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK) {
if (mclk_mask)
- *mclk_mask = 0;
+ /* mclk levels are in reverse order */
+ *mclk_mask = NUM_MEMCLK_DPM_LEVELS - 1;
} else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
if(sclk_mask)
/* The sclk as gfxclk and has three level about max/min/current */
*sclk_mask = 3 - 1;
if(mclk_mask)
- *mclk_mask = NUM_MEMCLK_DPM_LEVELS - 1;
+ /* mclk levels are in reverse order */
+ *mclk_mask = 0;
if(soc_mask)
*soc_mask = NUM_SOCCLK_DPM_LEVELS - 1;
case SMU_UCLK:
case SMU_FCLK:
case SMU_MCLK:
- ret = renoir_get_dpm_clk_limited(smu, clk_type, 0, min);
+ ret = renoir_get_dpm_clk_limited(smu, clk_type, NUM_MEMCLK_DPM_LEVELS - 1, min);
if (ret)
goto failed;
break;
{
int i;
- BUG_ON(numbytes > MAX_SW_I2C_COMMANDS);
-
req->I2CcontrollerPort = 0;
req->I2CSpeed = 2;
req->SlaveAddress = address;
struct smu_table_context *smu_table = &adev->smu.smu_table;
struct smu_table *table = &smu_table->driver_table;
+ if (numbytes > MAX_SW_I2C_COMMANDS) {
+ dev_err(adev->dev, "numbytes requested %d is over max allowed %d\n",
+ numbytes, MAX_SW_I2C_COMMANDS);
+ return -EINVAL;
+ }
+
memset(&req, 0, sizeof(req));
sienna_cichlid_fill_i2c_req(&req, false, address, numbytes, data);
SwI2cRequest_t req;
struct amdgpu_device *adev = to_amdgpu_device(control);
+ if (numbytes > MAX_SW_I2C_COMMANDS) {
+ dev_err(adev->dev, "numbytes requested %d is over max allowed %d\n",
+ numbytes, MAX_SW_I2C_COMMANDS);
+ return -EINVAL;
+ }
+
memset(&req, 0, sizeof(req));
sienna_cichlid_fill_i2c_req(&req, true, address, numbytes, data);
static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt,
struct intel_vgpu_creation_params *param)
{
+ struct drm_i915_private *dev_priv = gvt->gt->i915;
struct intel_vgpu *vgpu;
int ret;
if (ret)
goto out_clean_sched_policy;
- ret = intel_gvt_hypervisor_set_edid(vgpu, PORT_D);
+ if (IS_BROADWELL(dev_priv))
+ ret = intel_gvt_hypervisor_set_edid(vgpu, PORT_B);
+ else
+ ret = intel_gvt_hypervisor_set_edid(vgpu, PORT_D);
if (ret)
goto out_clean_sched_policy;
return (void *)ref->active ?: (void *)ref->retire ?: (void *)ref;
}
-static struct debug_obj_descr active_debug_desc = {
+static const struct debug_obj_descr active_debug_desc = {
.name = "i915_active",
.debug_hint = active_debug_hint,
};
#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
-static struct debug_obj_descr i915_sw_fence_debug_descr = {
+static const struct debug_obj_descr i915_sw_fence_debug_descr = {
.name = "i915_sw_fence",
.debug_hint = i915_sw_fence_debug_hint,
};
struct drm_i915_private *mock_gem_device(void)
{
- struct drm_i915_private *i915;
- struct pci_dev *pdev;
#if IS_ENABLED(CONFIG_IOMMU_API) && defined(CONFIG_INTEL_IOMMU)
- struct dev_iommu iommu;
+ static struct dev_iommu fake_iommu = { .priv = (void *)-1 };
#endif
+ struct drm_i915_private *i915;
+ struct pci_dev *pdev;
int err;
pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
#if IS_ENABLED(CONFIG_IOMMU_API) && defined(CONFIG_INTEL_IOMMU)
- /* HACK HACK HACK to disable iommu for the fake device; force identity mapping */
- memset(&iommu, 0, sizeof(iommu));
- iommu.priv = (void *)-1;
- pdev->dev.iommu = &iommu;
+ /* HACK to disable iommu for the fake device; force identity mapping */
+ pdev->dev.iommu = &fake_iommu;
#endif
pci_set_drvdata(pdev, i915);
nouveau_mem_del(struct ttm_mem_reg *reg)
{
struct nouveau_mem *mem = nouveau_mem(reg);
+ if (!mem)
+ return;
nouveau_mem_fini(mem);
kfree(reg->mm_node);
reg->mm_node = NULL;
case 0x168: device->chip = &nv168_chipset; break;
default:
nvdev_error(device, "unknown chipset (%08x)\n", boot0);
+ ret = -ENODEV;
goto done;
}
/* VI channel CSC units offsets */
#define CCSC00_OFFSET 0xAA050
-#define CCSC01_OFFSET 0xFA000
+#define CCSC01_OFFSET 0xFA050
#define CCSC10_OFFSET 0xA0000
#define CCSC11_OFFSET 0xF0000
.reg_bits = 32,
.val_bits = 32,
.reg_stride = 4,
- .max_register = 0xbfffc, /* guessed */
+ .max_register = 0xffffc, /* guessed */
};
static int sun8i_mixer_of_get_id(struct device_node *node)
DSI_PAD_PREEMP_PD(0x03) | DSI_PAD_PREEMP_PU(0x3);
tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_3);
- err = tegra_mipi_calibrate(dsi->mipi);
+ err = tegra_mipi_start_calibration(dsi->mipi);
if (err < 0)
return err;
- return tegra_mipi_wait(dsi->mipi);
+ return tegra_mipi_finish_calibration(dsi->mipi);
}
static void tegra_dsi_set_timeout(struct tegra_dsi *dsi, unsigned long bclk,
card->num_links = 1;
card->name = "vc4-hdmi";
card->dev = dev;
+ card->owner = THIS_MODULE;
/*
* Be careful, snd_soc_register_card() calls dev_set_drvdata() and
id = ida_alloc_max(&gman->gmr_ida, gman->max_gmr_ids - 1, GFP_KERNEL);
if (id < 0)
- return (id != -ENOMEM ? 0 : id);
+ return id;
spin_lock(&gman->lock);
mem->start = node->start;
}
- return 0;
+ return ret;
}
}
EXPORT_SYMBOL(tegra_mipi_disable);
-int tegra_mipi_wait(struct tegra_mipi_device *device)
+int tegra_mipi_finish_calibration(struct tegra_mipi_device *device)
{
struct tegra_mipi *mipi = device->mipi;
void __iomem *status_reg = mipi->regs + (MIPI_CAL_STATUS << 2);
u32 value;
int err;
- err = clk_enable(device->mipi->clk);
- if (err < 0)
- return err;
-
- mutex_lock(&device->mipi->lock);
-
err = readl_relaxed_poll_timeout(status_reg, value,
!(value & MIPI_CAL_STATUS_ACTIVE) &&
(value & MIPI_CAL_STATUS_DONE), 50,
return err;
}
-EXPORT_SYMBOL(tegra_mipi_wait);
+EXPORT_SYMBOL(tegra_mipi_finish_calibration);
-int tegra_mipi_calibrate(struct tegra_mipi_device *device)
+int tegra_mipi_start_calibration(struct tegra_mipi_device *device)
{
const struct tegra_mipi_soc *soc = device->mipi->soc;
unsigned int i;
value |= MIPI_CAL_CTRL_START;
tegra_mipi_writel(device->mipi, value, MIPI_CAL_CTRL);
- mutex_unlock(&device->mipi->lock);
- clk_disable(device->mipi->clk);
+ /*
+ * Wait for min 72uS to let calibration logic finish calibration
+ * sequence codes before waiting for pads idle state to apply the
+ * results.
+ */
+ usleep_range(75, 80);
return 0;
}
-EXPORT_SYMBOL(tegra_mipi_calibrate);
+EXPORT_SYMBOL(tegra_mipi_start_calibration);
static const struct tegra_mipi_pad tegra114_mipi_pads[] = {
{ .data = MIPI_CAL_CONFIG_CSIA },
for (i = 0; i+1 < sz; i += 2) {
w = (raw_data[i] << 8) | (raw_data[i+1]);
rawir.pulse = !!(w & 0x8000);
- rawir.duration = US_TO_NS(rawir.pulse ? (65536 - w) : w);
+ rawir.duration = rawir.pulse ? (65536 - w) : w;
/* Quirk!! - see above */
- if (i == 0 && rawir.duration > 15000000)
- rawir.duration -= 15000000;
+ if (i == 0 && rawir.duration > 15000)
+ rawir.duration -= 15000;
ir_raw_event_store(data->rc_dev, &rawir);
}
ir_raw_event_handle(data->rc_dev);
rdev->dev.parent = &data->hdev->dev;
rdev->driver_name = PICOLCD_NAME;
rdev->map_name = RC_MAP_RC6_MCE;
- rdev->timeout = MS_TO_NS(100);
- rdev->rx_resolution = US_TO_NS(1);
+ rdev->timeout = MS_TO_US(100);
+ rdev->rx_resolution = 1;
ret = rc_register_device(rdev);
if (ret)
will be called mcp3021.
config SENSORS_MLXREG_FAN
- tristate "Mellanox Mellanox FAN driver"
+ tristate "Mellanox FAN driver"
depends on MELLANOX_PLATFORM
imply THERMAL
select REGMAP
This driver can also be built as a module. If so the module
will be called menf21bmc_hwmon.
+config SENSORS_MR75203
+ tristate "Moortec Semiconductor MR75203 PVT Controller"
+ select REGMAP_MMIO
+ help
+ If you say yes here you get support for Moortec MR75203
+ PVT controller.
+
+ This driver can also be built as a module. If so, the module
+ will be called mr75203.
+
config SENSORS_ADCXX
tristate "National Semiconductor ADCxxxSxxx"
depends on SPI_MASTER
If you say yes here you get support for the temperature
and power sensors for APM X-Gene SoC.
+config SENSORS_INTEL_M10_BMC_HWMON
+ tristate "Intel MAX10 BMC Hardware Monitoring"
+ depends on MFD_INTEL_M10_BMC
+ help
+ This driver provides support for the hardware monitoring functionality
+ on Intel MAX10 BMC chip.
+
+ This BMC Chip is used on Intel FPGA PCIe Acceleration Cards (PAC). Its
+ sensors monitor various telemetry data of different components on the
+ card, e.g. board temperature, FPGA core temperature/voltage/current.
+
if ACPI
comment "ACPI drivers"
obj-$(CONFIG_SENSORS_INA209) += ina209.o
obj-$(CONFIG_SENSORS_INA2XX) += ina2xx.o
obj-$(CONFIG_SENSORS_INA3221) += ina3221.o
+obj-$(CONFIG_SENSORS_INTEL_M10_BMC_HWMON) += intel-m10-bmc-hwmon.o
obj-$(CONFIG_SENSORS_IT87) += it87.o
obj-$(CONFIG_SENSORS_JC42) += jc42.o
obj-$(CONFIG_SENSORS_K8TEMP) += k8temp.o
obj-$(CONFIG_SENSORS_TC654) += tc654.o
obj-$(CONFIG_SENSORS_MLXREG_FAN) += mlxreg-fan.o
obj-$(CONFIG_SENSORS_MENF21BMC_HWMON) += menf21bmc_hwmon.o
+obj-$(CONFIG_SENSORS_MR75203) += mr75203.o
obj-$(CONFIG_SENSORS_NCT6683) += nct6683.o
obj-$(CONFIG_SENSORS_NCT6775) += nct6775.o
obj-$(CONFIG_SENSORS_NCT7802) += nct7802.o
ATTRIBUTE_GROUPS(ad7414);
-static int ad7414_probe(struct i2c_client *client,
- const struct i2c_device_id *dev_id)
+static int ad7414_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct ad7414_data *data;
.name = "ad7414",
.of_match_table = of_match_ptr(ad7414_of_match),
},
- .probe = ad7414_probe,
+ .probe_new = ad7414_probe,
.id_table = ad7414_id,
};
}
}
-static int ad7418_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id ad7418_id[];
+
+static int ad7418_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct i2c_adapter *adapter = client->adapter;
if (dev->of_node)
data->type = (enum chips)of_device_get_match_data(dev);
else
- data->type = id->driver_data;
+ data->type = i2c_match_id(ad7418_id, client)->driver_data;
switch (data->type) {
case ad7416:
.name = "ad7418",
.of_match_table = ad7418_dt_ids,
},
- .probe = ad7418_probe,
+ .probe_new = ad7418_probe,
.id_table = ad7418_id,
};
return 0;
}
-static int adc128_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int adc128_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct regulator *regulator;
.name = "adc128d818",
.of_match_table = of_match_ptr(adc128_of_match),
},
- .probe = adc128_probe,
+ .probe_new = adc128_probe,
.remove = adc128_remove,
.id_table = adc128_id,
.detect = adc128_detect,
i2c_smbus_write_byte_data(client, ADM1021_REG_CONV_RATE_W, 0x04);
}
-static int adm1021_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id adm1021_id[];
+
+static int adm1021_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct adm1021_data *data;
return -ENOMEM;
data->client = client;
- data->type = id->driver_data;
+ data->type = i2c_match_id(adm1021_id, client)->driver_data;
mutex_init(&data->update_lock);
/* Initialize the ADM1021 chip */
.driver = {
.name = "adm1021",
},
- .probe = adm1021_probe,
+ .probe_new = adm1021_probe,
.id_table = adm1021_id,
.detect = adm1021_detect,
.address_list = normal_i2c,
(reg&0x7E)|0x01);
}
-static int adm1025_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int adm1025_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver = {
.name = "adm1025",
},
- .probe = adm1025_probe,
+ .probe_new = adm1025_probe,
.id_table = adm1025_id,
.detect = adm1025_detect,
.address_list = normal_i2c,
}
}
-static int adm1026_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int adm1026_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver = {
.name = "adm1026",
},
- .probe = adm1026_probe,
+ .probe_new = adm1026_probe,
.id_table = adm1026_id,
.detect = adm1026_detect,
.address_list = normal_i2c,
return 1;
}
-static int adm1029_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int adm1029_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct adm1029_data *data;
.driver = {
.name = "adm1029",
},
- .probe = adm1029_probe,
+ .probe_new = adm1029_probe,
.id_table = adm1029_id,
.detect = adm1029_detect,
.address_list = normal_i2c,
data->update_interval = update_intervals[i];
}
-static int adm1031_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id adm1031_id[];
+
+static int adm1031_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
i2c_set_clientdata(client, data);
data->client = client;
- data->chip_type = id->driver_data;
+ data->chip_type = i2c_match_id(adm1031_id, client)->driver_data;
mutex_init(&data->update_lock);
if (data->chip_type == adm1030)
.driver = {
.name = "adm1031",
},
- .probe = adm1031_probe,
+ .probe_new = adm1031_probe,
.id_table = adm1031_id,
.detect = adm1031_detect,
.address_list = normal_i2c,
regulator_disable(st->reg);
}
-static int adm1177_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int adm1177_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.name = "adm1177",
.of_match_table = adm1177_dt_ids,
},
- .probe = adm1177_probe,
+ .probe_new = adm1177_probe,
.id_table = adm1177_id,
};
module_i2c_driver(adm1177_driver);
#include <linux/err.h>
#include <linux/mutex.h>
#include <linux/jiffies.h>
+#include <linux/regmap.h>
/* Addresses to scan */
static const unsigned short normal_i2c[] = { 0x2c, 0x2d, 0x2e, 0x2f,
/* per client data */
struct adm9240_data {
struct i2c_client *client;
+ struct regmap *regmap;
struct mutex update_lock;
char valid;
unsigned long last_updated_measure;
};
/* write new fan div, callers must hold data->update_lock */
-static void adm9240_write_fan_div(struct i2c_client *client, int nr,
+static int adm9240_write_fan_div(struct adm9240_data *data, int nr,
u8 fan_div)
{
- u8 reg, old, shift = (nr + 2) * 2;
+ unsigned int reg, old, shift = (nr + 2) * 2;
+ int err;
- reg = i2c_smbus_read_byte_data(client, ADM9240_REG_VID_FAN_DIV);
+ err = regmap_read(data->regmap, ADM9240_REG_VID_FAN_DIV, ®);
+ if (err < 0)
+ return err;
old = (reg >> shift) & 3;
reg &= ~(3 << shift);
reg |= (fan_div << shift);
- i2c_smbus_write_byte_data(client, ADM9240_REG_VID_FAN_DIV, reg);
- dev_dbg(&client->dev,
+ err = regmap_write(data->regmap, ADM9240_REG_VID_FAN_DIV, reg);
+ if (err < 0)
+ return err;
+ dev_dbg(&data->client->dev,
"fan%d clock divider changed from %u to %u\n",
nr + 1, 1 << old, 1 << fan_div);
+
+ return 0;
+}
+
+static int adm9240_update_measure(struct adm9240_data *data)
+{
+ unsigned int val;
+ u8 regs[2];
+ int err;
+ int i;
+
+ err = regmap_bulk_read(data->regmap, ADM9240_REG_IN(0), &data->in[0], 6);
+ if (err < 0)
+ return err;
+ err = regmap_bulk_read(data->regmap, ADM9240_REG_INT(0), ®s, 2);
+ if (err < 0)
+ return err;
+
+ data->alarms = regs[0] | regs[1] << 8;
+
+ /*
+ * read temperature: assume temperature changes less than
+ * 0.5'C per two measurement cycles thus ignore possible
+ * but unlikely aliasing error on lsb reading. --Grant
+ */
+ err = regmap_read(data->regmap, ADM9240_REG_TEMP, &val);
+ if (err < 0)
+ return err;
+ data->temp = val << 8;
+ err = regmap_read(data->regmap, ADM9240_REG_TEMP_CONF, &val);
+ if (err < 0)
+ return err;
+ data->temp |= val;
+
+ err = regmap_bulk_read(data->regmap, ADM9240_REG_FAN(0),
+ &data->fan[0], 2);
+ if (err < 0)
+ return err;
+
+ for (i = 0; i < 2; i++) { /* read fans */
+ /* adjust fan clock divider on overflow */
+ if (data->valid && data->fan[i] == 255 &&
+ data->fan_div[i] < 3) {
+
+ err = adm9240_write_fan_div(data, i,
+ ++data->fan_div[i]);
+ if (err < 0)
+ return err;
+
+ /* adjust fan_min if active, but not to 0 */
+ if (data->fan_min[i] < 255 &&
+ data->fan_min[i] >= 2)
+ data->fan_min[i] /= 2;
+ }
+ }
+
+ return 0;
+}
+
+static int adm9240_update_config(struct adm9240_data *data)
+{
+ unsigned int val;
+ int i;
+ int err;
+
+ for (i = 0; i < 6; i++) {
+ err = regmap_raw_read(data->regmap, ADM9240_REG_IN_MIN(i),
+ &data->in_min[i], 1);
+ if (err < 0)
+ return err;
+ err = regmap_raw_read(data->regmap, ADM9240_REG_IN_MAX(i),
+ &data->in_max[i], 1);
+ if (err < 0)
+ return err;
+ }
+ err = regmap_bulk_read(data->regmap, ADM9240_REG_FAN_MIN(0),
+ &data->fan_min[0], 2);
+ if (err < 0)
+ return err;
+ err = regmap_bulk_read(data->regmap, ADM9240_REG_TEMP_MAX(0),
+ &data->temp_max[0], 2);
+ if (err < 0)
+ return err;
+
+ /* read fan divs and 5-bit VID */
+ err = regmap_read(data->regmap, ADM9240_REG_VID_FAN_DIV, &val);
+ if (err < 0)
+ return err;
+ data->fan_div[0] = (val >> 4) & 3;
+ data->fan_div[1] = (val >> 6) & 3;
+ data->vid = val & 0x0f;
+ err = regmap_read(data->regmap, ADM9240_REG_VID4, &val);
+ if (err < 0)
+ return err;
+ data->vid |= (val & 1) << 4;
+ /* read analog out */
+ err = regmap_raw_read(data->regmap, ADM9240_REG_ANALOG_OUT,
+ &data->aout, 1);
+
+ return err;
}
static struct adm9240_data *adm9240_update_device(struct device *dev)
{
struct adm9240_data *data = dev_get_drvdata(dev);
- struct i2c_client *client = data->client;
- int i;
+ int err;
mutex_lock(&data->update_lock);
/* minimum measurement cycle: 1.75 seconds */
if (time_after(jiffies, data->last_updated_measure + (HZ * 7 / 4))
|| !data->valid) {
-
- for (i = 0; i < 6; i++) { /* read voltages */
- data->in[i] = i2c_smbus_read_byte_data(client,
- ADM9240_REG_IN(i));
- }
- data->alarms = i2c_smbus_read_byte_data(client,
- ADM9240_REG_INT(0)) |
- i2c_smbus_read_byte_data(client,
- ADM9240_REG_INT(1)) << 8;
-
- /*
- * read temperature: assume temperature changes less than
- * 0.5'C per two measurement cycles thus ignore possible
- * but unlikely aliasing error on lsb reading. --Grant
- */
- data->temp = (i2c_smbus_read_byte_data(client,
- ADM9240_REG_TEMP) << 8) |
- i2c_smbus_read_byte_data(client,
- ADM9240_REG_TEMP_CONF);
-
- for (i = 0; i < 2; i++) { /* read fans */
- data->fan[i] = i2c_smbus_read_byte_data(client,
- ADM9240_REG_FAN(i));
-
- /* adjust fan clock divider on overflow */
- if (data->valid && data->fan[i] == 255 &&
- data->fan_div[i] < 3) {
-
- adm9240_write_fan_div(client, i,
- ++data->fan_div[i]);
-
- /* adjust fan_min if active, but not to 0 */
- if (data->fan_min[i] < 255 &&
- data->fan_min[i] >= 2)
- data->fan_min[i] /= 2;
- }
+ err = adm9240_update_measure(data);
+ if (err < 0) {
+ data->valid = 0;
+ mutex_unlock(&data->update_lock);
+ return ERR_PTR(err);
}
data->last_updated_measure = jiffies;
}
/* minimum config reading cycle: 300 seconds */
if (time_after(jiffies, data->last_updated_config + (HZ * 300))
|| !data->valid) {
-
- for (i = 0; i < 6; i++) {
- data->in_min[i] = i2c_smbus_read_byte_data(client,
- ADM9240_REG_IN_MIN(i));
- data->in_max[i] = i2c_smbus_read_byte_data(client,
- ADM9240_REG_IN_MAX(i));
- }
- for (i = 0; i < 2; i++) {
- data->fan_min[i] = i2c_smbus_read_byte_data(client,
- ADM9240_REG_FAN_MIN(i));
+ err = adm9240_update_config(data);
+ if (err < 0) {
+ data->valid = 0;
+ mutex_unlock(&data->update_lock);
+ return ERR_PTR(err);
}
- data->temp_max[0] = i2c_smbus_read_byte_data(client,
- ADM9240_REG_TEMP_MAX(0));
- data->temp_max[1] = i2c_smbus_read_byte_data(client,
- ADM9240_REG_TEMP_MAX(1));
-
- /* read fan divs and 5-bit VID */
- i = i2c_smbus_read_byte_data(client, ADM9240_REG_VID_FAN_DIV);
- data->fan_div[0] = (i >> 4) & 3;
- data->fan_div[1] = (i >> 6) & 3;
- data->vid = i & 0x0f;
- data->vid |= (i2c_smbus_read_byte_data(client,
- ADM9240_REG_VID4) & 1) << 4;
- /* read analog out */
- data->aout = i2c_smbus_read_byte_data(client,
- ADM9240_REG_ANALOG_OUT);
-
data->last_updated_config = jiffies;
data->valid = 1;
}
struct device_attribute *dummy, char *buf)
{
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%d\n", data->temp / 128 * 500); /* 9-bit value */
}
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%d\n", data->temp_max[attr->index] * 1000);
}
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct adm9240_data *data = dev_get_drvdata(dev);
- struct i2c_client *client = data->client;
long val;
int err;
mutex_lock(&data->update_lock);
data->temp_max[attr->index] = TEMP_TO_REG(val);
- i2c_smbus_write_byte_data(client, ADM9240_REG_TEMP_MAX(attr->index),
- data->temp_max[attr->index]);
+ err = regmap_write(data->regmap, ADM9240_REG_TEMP_MAX(attr->index),
+ data->temp_max[attr->index]);
mutex_unlock(&data->update_lock);
- return count;
+ return err < 0 ? err : count;
}
static DEVICE_ATTR_RO(temp1_input);
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%d\n", IN_FROM_REG(data->in[attr->index],
attr->index));
}
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%d\n", IN_FROM_REG(data->in_min[attr->index],
attr->index));
}
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%d\n", IN_FROM_REG(data->in_max[attr->index],
attr->index));
}
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct adm9240_data *data = dev_get_drvdata(dev);
- struct i2c_client *client = data->client;
unsigned long val;
int err;
mutex_lock(&data->update_lock);
data->in_min[attr->index] = IN_TO_REG(val, attr->index);
- i2c_smbus_write_byte_data(client, ADM9240_REG_IN_MIN(attr->index),
- data->in_min[attr->index]);
+ err = regmap_write(data->regmap, ADM9240_REG_IN_MIN(attr->index),
+ data->in_min[attr->index]);
mutex_unlock(&data->update_lock);
- return count;
+ return err < 0 ? err : count;
}
static ssize_t in_max_store(struct device *dev,
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct adm9240_data *data = dev_get_drvdata(dev);
- struct i2c_client *client = data->client;
unsigned long val;
int err;
mutex_lock(&data->update_lock);
data->in_max[attr->index] = IN_TO_REG(val, attr->index);
- i2c_smbus_write_byte_data(client, ADM9240_REG_IN_MAX(attr->index),
- data->in_max[attr->index]);
+ err = regmap_write(data->regmap, ADM9240_REG_IN_MAX(attr->index),
+ data->in_max[attr->index]);
mutex_unlock(&data->update_lock);
- return count;
+ return err < 0 ? err : count;
}
static SENSOR_DEVICE_ATTR_RO(in0_input, in, 0);
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%d\n", FAN_FROM_REG(data->fan[attr->index],
1 << data->fan_div[attr->index]));
}
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%d\n", FAN_FROM_REG(data->fan_min[attr->index],
1 << data->fan_div[attr->index]));
}
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%d\n", 1 << data->fan_div[attr->index]);
}
if (new_div != data->fan_div[nr]) {
data->fan_div[nr] = new_div;
- adm9240_write_fan_div(client, nr, new_div);
+ adm9240_write_fan_div(data, nr, new_div);
}
- i2c_smbus_write_byte_data(client, ADM9240_REG_FAN_MIN(nr),
- data->fan_min[nr]);
+ err = regmap_write(data->regmap, ADM9240_REG_FAN_MIN(nr),
+ data->fan_min[nr]);
mutex_unlock(&data->update_lock);
- return count;
+ return err < 0 ? err : count;
}
static SENSOR_DEVICE_ATTR_RO(fan1_input, fan, 0);
struct device_attribute *attr, char *buf)
{
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%u\n", data->alarms);
}
static DEVICE_ATTR_RO(alarms);
{
int bitnr = to_sensor_dev_attr(attr)->index;
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%u\n", (data->alarms >> bitnr) & 1);
}
static SENSOR_DEVICE_ATTR_RO(in0_alarm, alarm, 0);
struct device_attribute *attr, char *buf)
{
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%d\n", vid_from_reg(data->vid, data->vrm));
}
static DEVICE_ATTR_RO(cpu0_vid);
struct device_attribute *attr, char *buf)
{
struct adm9240_data *data = adm9240_update_device(dev);
+
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
return sprintf(buf, "%d\n", AOUT_FROM_REG(data->aout));
}
const char *buf, size_t count)
{
struct adm9240_data *data = dev_get_drvdata(dev);
- struct i2c_client *client = data->client;
long val;
int err;
mutex_lock(&data->update_lock);
data->aout = AOUT_TO_REG(val);
- i2c_smbus_write_byte_data(client, ADM9240_REG_ANALOG_OUT, data->aout);
+ err = regmap_write(data->regmap, ADM9240_REG_ANALOG_OUT, data->aout);
mutex_unlock(&data->update_lock);
- return count;
+ return err < 0 ? err : count;
}
static DEVICE_ATTR_RW(aout_output);
const char *buf, size_t count)
{
struct adm9240_data *data = dev_get_drvdata(dev);
- struct i2c_client *client = data->client;
unsigned long val;
+ int err;
if (kstrtoul(buf, 10, &val) || val != 0)
return -EINVAL;
mutex_lock(&data->update_lock);
- i2c_smbus_write_byte_data(client, ADM9240_REG_CHASSIS_CLEAR, 0x80);
+ err = regmap_write(data->regmap, ADM9240_REG_CHASSIS_CLEAR, 0x80);
data->valid = 0; /* Force cache refresh */
mutex_unlock(&data->update_lock);
- dev_dbg(&client->dev, "chassis intrusion latch cleared\n");
+ if (err < 0)
+ return err;
+ dev_dbg(&data->client->dev, "chassis intrusion latch cleared\n");
return count;
}
return 0;
}
-static void adm9240_init_client(struct i2c_client *client)
+static int adm9240_init_client(struct i2c_client *client, struct adm9240_data *data)
{
- struct adm9240_data *data = i2c_get_clientdata(client);
- u8 conf = i2c_smbus_read_byte_data(client, ADM9240_REG_CONFIG);
- u8 mode = i2c_smbus_read_byte_data(client, ADM9240_REG_TEMP_CONF) & 3;
+ u8 conf, mode;
+ int err;
+
+ err = regmap_raw_read(data->regmap, ADM9240_REG_CONFIG, &conf, 1);
+ if (err < 0)
+ return err;
+ err = regmap_raw_read(data->regmap, ADM9240_REG_TEMP_CONF, &mode, 1);
+ if (err < 0)
+ return err;
+ mode &= 3;
data->vrm = vid_which_vrm(); /* need this to report vid as mV */
int i;
for (i = 0; i < 6; i++) {
- i2c_smbus_write_byte_data(client,
- ADM9240_REG_IN_MIN(i), 0);
- i2c_smbus_write_byte_data(client,
- ADM9240_REG_IN_MAX(i), 255);
+ err = regmap_write(data->regmap,
+ ADM9240_REG_IN_MIN(i), 0);
+ if (err < 0)
+ return err;
+ err = regmap_write(data->regmap,
+ ADM9240_REG_IN_MAX(i), 255);
+ if (err < 0)
+ return err;
+ }
+ for (i = 0; i < 2; i++) {
+ err = regmap_write(data->regmap,
+ ADM9240_REG_FAN_MIN(i), 255);
+ if (err < 0)
+ return err;
+ }
+ for (i = 0; i < 2; i++) {
+ err = regmap_write(data->regmap,
+ ADM9240_REG_TEMP_MAX(i), 127);
+ if (err < 0)
+ return err;
}
- i2c_smbus_write_byte_data(client,
- ADM9240_REG_FAN_MIN(0), 255);
- i2c_smbus_write_byte_data(client,
- ADM9240_REG_FAN_MIN(1), 255);
- i2c_smbus_write_byte_data(client,
- ADM9240_REG_TEMP_MAX(0), 127);
- i2c_smbus_write_byte_data(client,
- ADM9240_REG_TEMP_MAX(1), 127);
/* start measurement cycle */
- i2c_smbus_write_byte_data(client, ADM9240_REG_CONFIG, 1);
+ err = regmap_write(data->regmap, ADM9240_REG_CONFIG, 1);
+ if (err < 0)
+ return err;
dev_info(&client->dev,
"cold start: config was 0x%02x mode %u\n", conf, mode);
}
+
+ return 0;
}
-static int adm9240_probe(struct i2c_client *new_client,
- const struct i2c_device_id *id)
+static const struct regmap_config adm9240_regmap_config = {
+ .reg_bits = 8,
+ .val_bits = 8,
+ .use_single_read = true,
+ .use_single_write = true,
+};
+
+static int adm9240_probe(struct i2c_client *new_client)
{
struct device *dev = &new_client->dev;
struct device *hwmon_dev;
struct adm9240_data *data;
+ int err;
data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
if (!data)
return -ENOMEM;
- i2c_set_clientdata(new_client, data);
data->client = new_client;
mutex_init(&data->update_lock);
+ data->regmap = devm_regmap_init_i2c(new_client, &adm9240_regmap_config);
+ if (IS_ERR(data->regmap))
+ return PTR_ERR(data->regmap);
- adm9240_init_client(new_client);
+ err = adm9240_init_client(new_client, data);
+ if (err < 0)
+ return err;
hwmon_dev = devm_hwmon_device_register_with_groups(dev,
new_client->name,
.driver = {
.name = "adm9240",
},
- .probe = adm9240_probe,
+ .probe_new = adm9240_probe,
.id_table = adm9240_id,
.detect = adm9240_detect,
.address_list = normal_i2c,
.val_bits = 8,
};
-static int ads7828_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id ads7828_device_ids[];
+
+static int ads7828_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct ads7828_platform_data *pdata = dev_get_platdata(dev);
chip = (enum ads7828_chips)
of_device_get_match_data(&client->dev);
else
- chip = id->driver_data;
+ chip = i2c_match_id(ads7828_device_ids, client)->driver_data;
/* Bound Vref with min/max values */
vref_mv = clamp_val(vref_mv, ADS7828_EXT_VREF_MV_MIN,
},
.id_table = ads7828_device_ids,
- .probe = ads7828_probe,
+ .probe_new = ads7828_probe,
};
module_i2c_driver(ads7828_driver);
.write_byte = adt7410_i2c_write_byte,
};
-static int adt7410_i2c_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int adt7410_i2c_probe(struct i2c_client *client)
{
if (!i2c_check_functionality(client->adapter,
I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA))
.name = "adt7410",
.pm = ADT7X10_DEV_PM_OPS,
},
- .probe = adt7410_i2c_probe,
+ .probe_new = adt7410_i2c_probe,
.remove = adt7410_i2c_remove,
.id_table = adt7410_ids,
.address_list = I2C_ADDRS(0x48, 0x49, 0x4a, 0x4b),
.info = adt7411_info,
};
-static int adt7411_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int adt7411_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct adt7411_data *data;
.driver = {
.name = "adt7411",
},
- .probe = adt7411_probe,
+ .probe_new = adt7411_probe,
.id_table = adt7411_id,
.detect = adt7411_detect,
.address_list = normal_i2c,
return 0;
}
-static int adt7462_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int adt7462_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct adt7462_data *data;
.driver = {
.name = "adt7462",
},
- .probe = adt7462_probe,
+ .probe_new = adt7462_probe,
.id_table = adt7462_id,
.detect = adt7462_detect,
.address_list = normal_i2c,
}
}
-static int adt7470_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int adt7470_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct adt7470_data *data;
.driver = {
.name = "adt7470",
},
- .probe = adt7470_probe,
+ .probe_new = adt7470_probe,
.remove = adt7470_remove,
.id_table = adt7470_id,
.detect = adt7470_detect,
return 0;
}
-static int adt7475_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int adt7475_probe(struct i2c_client *client)
{
enum chips chip;
static const char * const names[] = {
struct device *hwmon_dev;
int i, ret = 0, revision, group_num = 0;
u8 config3;
+ const struct i2c_device_id *id = i2c_match_id(adt7475_id, client);
data = devm_kzalloc(&client->dev, sizeof(*data), GFP_KERNEL);
if (data == NULL)
.name = "adt7475",
.of_match_table = of_match_ptr(adt7475_of_match),
},
- .probe = adt7475_probe,
+ .probe_new = adt7475_probe,
.id_table = adt7475_id,
.detect = adt7475_detect,
.address_list = normal_i2c,
return 0;
}
-static int amc6821_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int amc6821_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct amc6821_data *data;
.driver = {
.name = "amc6821",
},
- .probe = amc6821_probe,
+ .probe_new = amc6821_probe,
.id_table = amc6821_id,
.detect = amc6821_detect,
.address_list = normal_i2c,
struct sensor_accumulator {
u64 energy_ctr;
u64 prev_value;
- char label[10];
};
struct amd_energy_data {
struct mutex lock;
/* An accumulator for each core and socket */
struct sensor_accumulator *accums;
+ unsigned int timeout_ms;
/* Energy Status Units */
- u64 energy_units;
+ int energy_units;
int nr_cpus;
int nr_socks;
int core_id;
+ char (*label)[10];
};
static int amd_energy_read_labels(struct device *dev,
{
struct amd_energy_data *data = dev_get_drvdata(dev);
- *str = data->accums[channel].label;
+ *str = data->label[channel];
return 0;
}
data->energy_units = (rapl_units & AMD_ENERGY_UNIT_MASK) >> 8;
}
-static void accumulate_socket_delta(struct amd_energy_data *data,
- int sock, int cpu)
+static void accumulate_delta(struct amd_energy_data *data,
+ int channel, int cpu, u32 reg)
{
- struct sensor_accumulator *s_accum;
+ struct sensor_accumulator *accum;
u64 input;
mutex_lock(&data->lock);
- rdmsrl_safe_on_cpu(cpu, ENERGY_PKG_MSR, &input);
+ rdmsrl_safe_on_cpu(cpu, reg, &input);
input &= AMD_ENERGY_MASK;
- s_accum = &data->accums[data->nr_cpus + sock];
- if (input >= s_accum->prev_value)
- s_accum->energy_ctr +=
- input - s_accum->prev_value;
+ accum = &data->accums[channel];
+ if (input >= accum->prev_value)
+ accum->energy_ctr +=
+ input - accum->prev_value;
else
- s_accum->energy_ctr += UINT_MAX -
- s_accum->prev_value + input;
+ accum->energy_ctr += UINT_MAX -
+ accum->prev_value + input;
- s_accum->prev_value = input;
+ accum->prev_value = input;
mutex_unlock(&data->lock);
}
-static void accumulate_core_delta(struct amd_energy_data *data)
+static void read_accumulate(struct amd_energy_data *data)
{
- struct sensor_accumulator *c_accum;
- u64 input;
- int cpu;
+ int sock, scpu, cpu;
+
+ for (sock = 0; sock < data->nr_socks; sock++) {
+ scpu = cpumask_first_and(cpu_online_mask,
+ cpumask_of_node(sock));
+
+ accumulate_delta(data, data->nr_cpus + sock,
+ scpu, ENERGY_PKG_MSR);
+ }
- mutex_lock(&data->lock);
if (data->core_id >= data->nr_cpus)
data->core_id = 0;
cpu = data->core_id;
+ if (cpu_online(cpu))
+ accumulate_delta(data, cpu, cpu, ENERGY_CORE_MSR);
- if (!cpu_online(cpu))
- goto out;
-
- rdmsrl_safe_on_cpu(cpu, ENERGY_CORE_MSR, &input);
- input &= AMD_ENERGY_MASK;
-
- c_accum = &data->accums[cpu];
-
- if (input >= c_accum->prev_value)
- c_accum->energy_ctr +=
- input - c_accum->prev_value;
- else
- c_accum->energy_ctr += UINT_MAX -
- c_accum->prev_value + input;
-
- c_accum->prev_value = input;
-
-out:
data->core_id++;
- mutex_unlock(&data->lock);
-}
-
-static void read_accumulate(struct amd_energy_data *data)
-{
- int sock;
-
- for (sock = 0; sock < data->nr_socks; sock++) {
- int cpu;
-
- cpu = cpumask_first_and(cpu_online_mask,
- cpumask_of_node(sock));
-
- accumulate_socket_delta(data, sock, cpu);
- }
-
- accumulate_core_delta(data);
}
static void amd_add_delta(struct amd_energy_data *data, int ch,
- int cpu, long *val, bool is_core)
+ int cpu, long *val, u32 reg)
{
- struct sensor_accumulator *s_accum, *c_accum;
+ struct sensor_accumulator *accum;
u64 input;
mutex_lock(&data->lock);
- if (!is_core) {
- rdmsrl_safe_on_cpu(cpu, ENERGY_PKG_MSR, &input);
- input &= AMD_ENERGY_MASK;
-
- s_accum = &data->accums[ch];
- if (input >= s_accum->prev_value)
- input += s_accum->energy_ctr -
- s_accum->prev_value;
- else
- input += UINT_MAX - s_accum->prev_value +
- s_accum->energy_ctr;
- } else {
- rdmsrl_safe_on_cpu(cpu, ENERGY_CORE_MSR, &input);
- input &= AMD_ENERGY_MASK;
+ rdmsrl_safe_on_cpu(cpu, reg, &input);
+ input &= AMD_ENERGY_MASK;
- c_accum = &data->accums[ch];
- if (input >= c_accum->prev_value)
- input += c_accum->energy_ctr -
- c_accum->prev_value;
- else
- input += UINT_MAX - c_accum->prev_value +
- c_accum->energy_ctr;
- }
+ accum = &data->accums[ch];
+ if (input >= accum->prev_value)
+ input += accum->energy_ctr -
+ accum->prev_value;
+ else
+ input += UINT_MAX - accum->prev_value +
+ accum->energy_ctr;
/* Energy consumed = (1/(2^ESU) * RAW * 1000000UL) μJoules */
*val = div64_ul(input * 1000000UL, BIT(data->energy_units));
u32 attr, int channel, long *val)
{
struct amd_energy_data *data = dev_get_drvdata(dev);
+ u32 reg;
int cpu;
if (channel >= data->nr_cpus) {
cpu = cpumask_first_and(cpu_online_mask,
cpumask_of_node
(channel - data->nr_cpus));
- amd_add_delta(data, channel, cpu, val, false);
+ reg = ENERGY_PKG_MSR;
} else {
cpu = channel;
if (!cpu_online(cpu))
return -ENODEV;
- amd_add_delta(data, channel, cpu, val, true);
+ reg = ENERGY_CORE_MSR;
}
+ amd_add_delta(data, channel, cpu, val, reg);
return 0;
}
static int energy_accumulator(void *p)
{
struct amd_energy_data *data = (struct amd_energy_data *)p;
+ unsigned int timeout = data->timeout_ms;
while (!kthread_should_stop()) {
/*
if (kthread_should_stop())
break;
- /*
- * On a 240W system, with default resolution the
- * Socket Energy status register may wrap around in
- * 2^32*15.3 e-6/240 = 273.8041 secs (~4.5 mins)
- *
- * let us accumulate for every 100secs
- */
- schedule_timeout(msecs_to_jiffies(100000));
+ schedule_timeout(msecs_to_jiffies(timeout));
}
return 0;
}
static int amd_create_sensor(struct device *dev,
struct amd_energy_data *data,
- u8 type, u32 config)
+ enum hwmon_sensor_types type, u32 config)
{
struct hwmon_channel_info *info = &data->energy_info;
struct sensor_accumulator *accums;
int i, num_siblings, cpus, sockets;
u32 *s_config;
+ char (*label_l)[10];
/* Identify the number of siblings per core */
num_siblings = ((cpuid_ebx(0x8000001e) >> 8) & 0xff) + 1;
if (!accums)
return -ENOMEM;
+ label_l = devm_kcalloc(dev, cpus + sockets,
+ sizeof(*label_l), GFP_KERNEL);
+ if (!label_l)
+ return -ENOMEM;
+
info->type = type;
info->config = s_config;
data->nr_cpus = cpus;
data->nr_socks = sockets;
data->accums = accums;
+ data->label = label_l;
for (i = 0; i < cpus + sockets; i++) {
s_config[i] = config;
if (i < cpus)
- scnprintf(accums[i].label, 10,
- "Ecore%03u", i);
+ scnprintf(label_l[i], 10, "Ecore%03u", i);
else
- scnprintf(accums[i].label, 10,
- "Esocket%u", (i - cpus));
+ scnprintf(label_l[i], 10, "Esocket%u", (i - cpus));
}
return 0;
struct device *hwmon_dev;
struct amd_energy_data *data;
struct device *dev = &pdev->dev;
+ int ret;
data = devm_kzalloc(dev,
sizeof(struct amd_energy_data), GFP_KERNEL);
dev_set_drvdata(dev, data);
/* Populate per-core energy reporting */
data->info[0] = &data->energy_info;
- amd_create_sensor(dev, data, hwmon_energy,
- HWMON_E_INPUT | HWMON_E_LABEL);
+ ret = amd_create_sensor(dev, data, hwmon_energy,
+ HWMON_E_INPUT | HWMON_E_LABEL);
+ if (ret)
+ return ret;
mutex_init(&data->lock);
get_energy_units(data);
if (IS_ERR(hwmon_dev))
return PTR_ERR(hwmon_dev);
+ /*
+ * On a system with peak wattage of 250W
+ * timeout = 2 ^ 32 / 2 ^ energy_units / 250 secs
+ */
+ data->timeout_ms = 1000 *
+ BIT(min(28, 31 - data->energy_units)) / 250;
+
data->wrap_accumulate = kthread_run(energy_accumulator, data,
"%s", dev_name(hwmon_dev));
- if (IS_ERR(data->wrap_accumulate))
- return PTR_ERR(data->wrap_accumulate);
-
return PTR_ERR_OR_ZERO(data->wrap_accumulate);
}
static int asb100_read_value(struct i2c_client *client, u16 reg);
static void asb100_write_value(struct i2c_client *client, u16 reg, u16 val);
-static int asb100_probe(struct i2c_client *client,
- const struct i2c_device_id *id);
+static int asb100_probe(struct i2c_client *client);
static int asb100_detect(struct i2c_client *client,
struct i2c_board_info *info);
static int asb100_remove(struct i2c_client *client);
.driver = {
.name = "asb100",
},
- .probe = asb100_probe,
+ .probe_new = asb100_probe,
.remove = asb100_remove,
.id_table = asb100_id,
.detect = asb100_detect,
return 0;
}
-static int asb100_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int asb100_probe(struct i2c_client *client)
{
int err;
struct asb100_data *data;
}
static int
-asc7621_probe(struct i2c_client *client, const struct i2c_device_id *id)
+asc7621_probe(struct i2c_client *client)
{
struct asc7621_data *data;
int i, err;
.driver = {
.name = "asc7621",
},
- .probe = asc7621_probe,
+ .probe_new = asc7621_probe,
.remove = asc7621_remove,
.id_table = asc7621_id,
.detect = asc7621_detect,
};
ATTRIBUTE_GROUPS(atxp1);
-static int atxp1_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int atxp1_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct atxp1_data *data;
.driver = {
.name = "atxp1",
},
- .probe = atxp1_probe,
+ .probe_new = atxp1_probe,
.id_table = atxp1_id,
};
#include <linux/bitops.h>
#include <linux/clk.h>
#include <linux/completion.h>
+#include <linux/delay.h>
#include <linux/device.h>
#include <linux/hwmon-sysfs.h>
#include <linux/hwmon.h>
long *val)
{
struct pvt_cache *cache = &pvt->cache[type];
+ unsigned long timeout;
u32 data;
int ret;
pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_DVALID, 0);
pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, PVT_CTRL_EN);
- wait_for_completion(&cache->conversion);
+ /*
+ * Wait with timeout since in case if the sensor is suddenly powered
+ * down the request won't be completed and the caller will hang up on
+ * this procedure until the power is back up again. Multiply the
+ * timeout by the factor of two to prevent a false timeout.
+ */
+ timeout = 2 * usecs_to_jiffies(ktime_to_us(pvt->timeout));
+ ret = wait_for_completion_timeout(&cache->conversion, timeout);
pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, 0);
pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_DVALID,
mutex_unlock(&pvt->iface_mtx);
+ if (!ret)
+ return -ETIMEDOUT;
+
if (type == PVT_TEMP)
*val = pvt_calc_poly(&poly_N_to_temp, data);
else
static int pvt_read_timeout(struct pvt_hwmon *pvt, long *val)
{
- unsigned long rate;
- ktime_t kt;
- u32 data;
-
- rate = clk_get_rate(pvt->clks[PVT_CLOCK_REF].clk);
- if (!rate)
- return -ENODEV;
-
- /*
- * Don't bother with mutex here, since we just read data from MMIO.
- * We also have to scale the ticks timeout up to compensate the
- * ms-ns-data translations.
- */
- data = readl(pvt->regs + PVT_TTIMEOUT) + 1;
+ int ret;
- /*
- * Calculate ref-clock based delay (Ttotal) between two consecutive
- * data samples of the same sensor. So we first must calculate the
- * delay introduced by the internal ref-clock timer (Tref * Fclk).
- * Then add the constant timeout cuased by each conversion latency
- * (Tmin). The basic formulae for each conversion is following:
- * Ttotal = Tref * Fclk + Tmin
- * Note if alarms are enabled the sensors are polled one after
- * another, so in order to have the delay being applicable for each
- * sensor the requested value must be equally redistirbuted.
- */
-#if defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
- kt = ktime_set(PVT_SENSORS_NUM * (u64)data, 0);
- kt = ktime_divns(kt, rate);
- kt = ktime_add_ns(kt, PVT_SENSORS_NUM * PVT_TOUT_MIN);
-#else
- kt = ktime_set(data, 0);
- kt = ktime_divns(kt, rate);
- kt = ktime_add_ns(kt, PVT_TOUT_MIN);
-#endif
+ ret = mutex_lock_interruptible(&pvt->iface_mtx);
+ if (ret)
+ return ret;
/* Return the result in msec as hwmon sysfs interface requires. */
- *val = ktime_to_ms(kt);
+ *val = ktime_to_ms(pvt->timeout);
+
+ mutex_unlock(&pvt->iface_mtx);
return 0;
}
static int pvt_write_timeout(struct pvt_hwmon *pvt, long val)
{
unsigned long rate;
- ktime_t kt;
+ ktime_t kt, cache;
u32 data;
int ret;
* between all available sensors to have the requested delay
* applicable to each individual sensor.
*/
- kt = ms_to_ktime(val);
+ cache = kt = ms_to_ktime(val);
#if defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
kt = ktime_divns(kt, PVT_SENSORS_NUM);
#endif
return ret;
pvt_set_tout(pvt, data);
+ pvt->timeout = cache;
mutex_unlock(&pvt->iface_mtx);
return 0;
}
-static void pvt_init_iface(struct pvt_hwmon *pvt)
+static int pvt_check_pwr(struct pvt_hwmon *pvt)
{
+ unsigned long tout;
+ int ret = 0;
+ u32 data;
+
+ /*
+ * Test out the sensor conversion functionality. If it is not done on
+ * time then the domain must have been unpowered and we won't be able
+ * to use the device later in this driver.
+ * Note If the power source is lost during the normal driver work the
+ * data read procedure will either return -ETIMEDOUT (for the
+ * alarm-less driver configuration) or just stop the repeated
+ * conversion. In the later case alas we won't be able to detect the
+ * problem.
+ */
+ pvt_update(pvt->regs + PVT_INTR_MASK, PVT_INTR_ALL, PVT_INTR_ALL);
+ pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, PVT_CTRL_EN);
+ pvt_set_tout(pvt, 0);
+ readl(pvt->regs + PVT_DATA);
+
+ tout = PVT_TOUT_MIN / NSEC_PER_USEC;
+ usleep_range(tout, 2 * tout);
+
+ data = readl(pvt->regs + PVT_DATA);
+ if (!(data & PVT_DATA_VALID)) {
+ ret = -ENODEV;
+ dev_err(pvt->dev, "Sensor is powered down\n");
+ }
+
+ pvt_update(pvt->regs + PVT_CTRL, PVT_CTRL_EN, 0);
+
+ return ret;
+}
+
+static int pvt_init_iface(struct pvt_hwmon *pvt)
+{
+ unsigned long rate;
u32 trim, temp;
+ rate = clk_get_rate(pvt->clks[PVT_CLOCK_REF].clk);
+ if (!rate) {
+ dev_err(pvt->dev, "Invalid reference clock rate\n");
+ return -ENODEV;
+ }
+
/*
* Make sure all interrupts and controller are disabled so not to
* accidentally have ISR executed before the driver data is fully
pvt_set_mode(pvt, pvt_info[pvt->sensor].mode);
pvt_set_tout(pvt, PVT_TOUT_DEF);
+ /*
+ * Preserve the current ref-clock based delay (Ttotal) between the
+ * sensors data samples in the driver data so not to recalculate it
+ * each time on the data requests and timeout reads. It consists of the
+ * delay introduced by the internal ref-clock timer (N / Fclk) and the
+ * constant timeout caused by each conversion latency (Tmin):
+ * Ttotal = N / Fclk + Tmin
+ * If alarms are enabled the sensors are polled one after another and
+ * in order to get the next measurement of a particular sensor the
+ * caller will have to wait for at most until all the others are
+ * polled. In that case the formulae will look a bit different:
+ * Ttotal = 5 * (N / Fclk + Tmin)
+ */
+#if defined(CONFIG_SENSORS_BT1_PVT_ALARMS)
+ pvt->timeout = ktime_set(PVT_SENSORS_NUM * PVT_TOUT_DEF, 0);
+ pvt->timeout = ktime_divns(pvt->timeout, rate);
+ pvt->timeout = ktime_add_ns(pvt->timeout, PVT_SENSORS_NUM * PVT_TOUT_MIN);
+#else
+ pvt->timeout = ktime_set(PVT_TOUT_DEF, 0);
+ pvt->timeout = ktime_divns(pvt->timeout, rate);
+ pvt->timeout = ktime_add_ns(pvt->timeout, PVT_TOUT_MIN);
+#endif
+
trim = PVT_TRIM_DEF;
if (!of_property_read_u32(pvt->dev->of_node,
"baikal,pvt-temp-offset-millicelsius", &temp))
trim = pvt_calc_trim(temp);
pvt_set_trim(pvt, trim);
+
+ return 0;
}
static int pvt_request_irq(struct pvt_hwmon *pvt)
if (ret)
return ret;
- pvt_init_iface(pvt);
+ ret = pvt_check_pwr(pvt);
+ if (ret)
+ return ret;
+
+ ret = pvt_init_iface(pvt);
+ if (ret)
+ return ret;
ret = pvt_request_irq(pvt);
if (ret)
#include <linux/completion.h>
#include <linux/hwmon.h>
#include <linux/kernel.h>
+#include <linux/ktime.h>
#include <linux/mutex.h>
#include <linux/seqlock.h>
* if alarms are disabled).
* @sensor: current PVT sensor the data conversion is being performed for.
* @cache: data cache descriptor.
+ * @timeout: conversion timeout cache.
*/
struct pvt_hwmon {
struct device *dev;
struct mutex iface_mtx;
enum pvt_sensor_type sensor;
struct pvt_cache cache[PVT_SENSORS_NUM];
+ ktime_t timeout;
};
/*
return 0;
}
-static int dme1737_i2c_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id dme1737_id[];
+
+static int dme1737_i2c_probe(struct i2c_client *client)
{
struct dme1737_data *data;
struct device *dev = &client->dev;
return -ENOMEM;
i2c_set_clientdata(client, data);
- data->type = id->driver_data;
+ data->type = i2c_match_id(dme1737_id, client)->driver_data;
data->client = client;
data->name = client->name;
mutex_init(&data->update_lock);
.driver = {
.name = "dme1737",
},
- .probe = dme1737_i2c_probe,
+ .probe_new = dme1737_i2c_probe,
.remove = dme1737_i2c_remove,
.id_table = dme1737_id,
.detect = dme1737_i2c_detect,
};
__ATTRIBUTE_GROUPS(ds1621);
-static int ds1621_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id ds1621_id[];
+
+static int ds1621_probe(struct i2c_client *client)
{
struct ds1621_data *data;
struct device *hwmon_dev;
mutex_init(&data->update_lock);
- data->kind = id->driver_data;
+ data->kind = i2c_match_id(ds1621_id, client)->driver_data;
data->client = client;
/* Initialize the DS1621 chip */
.driver = {
.name = "ds1621",
},
- .probe = ds1621_probe,
+ .probe_new = ds1621_probe,
.id_table = ds1621_id,
};
ATTRIBUTE_GROUPS(ds620);
-static int ds620_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ds620_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver = {
.name = "ds620",
},
- .probe = ds620_probe,
+ .probe_new = ds620_probe,
.id_table = ds620_id,
};
.volatile_reg = emc1403_regmap_is_volatile,
};
-static int emc1403_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id emc1403_idtable[];
+
+static int emc1403_probe(struct i2c_client *client)
{
struct thermal_data *data;
struct device *hwmon_dev;
+ const struct i2c_device_id *id = i2c_match_id(emc1403_idtable, client);
data = devm_kzalloc(&client->dev, sizeof(struct thermal_data),
GFP_KERNEL);
.name = "emc1403",
},
.detect = emc1403_detect,
- .probe = emc1403_probe,
+ .probe_new = emc1403_probe,
.id_table = emc1403_idtable,
.address_list = emc1403_address_list,
};
};
static int
-emc2103_probe(struct i2c_client *client, const struct i2c_device_id *id)
+emc2103_probe(struct i2c_client *client)
{
struct emc2103_data *data;
struct device *hwmon_dev;
.driver = {
.name = "emc2103",
},
- .probe = emc2103_probe,
+ .probe_new = emc2103_probe,
.id_table = emc2103_ids,
.detect = emc2103_detect,
.address_list = normal_i2c,
return 0;
}
-static int emc6w201_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int emc6w201_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct emc6w201_data *data;
.driver = {
.name = "emc6w201",
},
- .probe = emc6w201_probe,
+ .probe_new = emc6w201_probe,
.id_table = emc6w201_id,
.detect = emc6w201_detect,
.address_list = normal_i2c,
static int f75375_detect(struct i2c_client *client,
struct i2c_board_info *info);
-static int f75375_probe(struct i2c_client *client,
- const struct i2c_device_id *id);
+static int f75375_probe(struct i2c_client *client);
static int f75375_remove(struct i2c_client *client);
static const struct i2c_device_id f75375_id[] = {
.driver = {
.name = "f75375",
},
- .probe = f75375_probe,
+ .probe_new = f75375_probe,
.remove = f75375_remove,
.id_table = f75375_id,
.detect = f75375_detect,
}
-static int f75375_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int f75375_probe(struct i2c_client *client)
{
struct f75375_data *data;
struct f75375s_platform_data *f75375s_pdata =
i2c_set_clientdata(client, data);
mutex_init(&data->update_lock);
- data->kind = id->driver_data;
+ data->kind = i2c_match_id(f75375_id, client)->driver_data;
err = sysfs_create_group(&client->dev.kobj, &f75375_group);
if (err)
* Functions declarations
*/
-static int fschmd_probe(struct i2c_client *client,
- const struct i2c_device_id *id);
+static int fschmd_probe(struct i2c_client *client);
static int fschmd_detect(struct i2c_client *client,
struct i2c_board_info *info);
static int fschmd_remove(struct i2c_client *client);
.driver = {
.name = "fschmd",
},
- .probe = fschmd_probe,
+ .probe_new = fschmd_probe,
.remove = fschmd_remove,
.id_table = fschmd_id,
.detect = fschmd_detect,
return 0;
}
-static int fschmd_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int fschmd_probe(struct i2c_client *client)
{
struct fschmd_data *data;
const char * const names[7] = { "Poseidon", "Hermes", "Scylla",
"Heracles", "Heimdall", "Hades", "Syleus" };
const int watchdog_minors[] = { WATCHDOG_MINOR, 212, 213, 214, 215 };
int i, err;
- enum chips kind = id->driver_data;
+ enum chips kind = i2c_match_id(fschmd_id, client)->driver_data;
data = kzalloc(sizeof(struct fschmd_data), GFP_KERNEL);
if (!data)
return 0;
}
-static int fts_probe(struct i2c_client *client, const struct i2c_device_id *id)
+static int fts_probe(struct i2c_client *client)
{
u8 revision;
struct fts_data *data;
.name = "ftsteutates",
},
.id_table = fts_id,
- .probe = fts_probe,
+ .probe_new = fts_probe,
.remove = fts_remove,
.detect = fts_detect,
.address_list = normal_i2c,
* new-style driver model code
*/
-static int g760a_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int g760a_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct g760a_data *data;
.driver = {
.name = "g760a",
},
- .probe = g760a_probe,
+ .probe_new = g760a_probe,
.id_table = g760a_id,
};
data->fan_cmd1);
}
-static int g762_probe(struct i2c_client *client, const struct i2c_device_id *id)
+static int g762_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.name = DRVNAME,
.of_match_table = of_match_ptr(g762_dt_match),
},
- .probe = g762_probe,
+ .probe_new = g762_probe,
.id_table = g762_id,
};
gl518_write_value(client, GL518_REG_CONF, 0x40 | regvalue);
}
-static int gl518_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int gl518_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver = {
.name = "gl518sm",
},
- .probe = gl518_probe,
+ .probe_new = gl518_probe,
.id_table = gl518_id,
.detect = gl518_detect,
.address_list = normal_i2c,
gl520_write_value(client, GL520_REG_BEEP_MASK, data->beep_mask);
}
-static int gl520_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int gl520_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver = {
.name = "gl520sm",
},
- .probe = gl520_probe,
+ .probe_new = gl520_probe,
.id_table = gl520_id,
.detect = gl520_detect,
.address_list = normal_i2c,
#define GSC_HWMON_MAX_TEMP_CH 16
#define GSC_HWMON_MAX_IN_CH 16
+#define GSC_HWMON_MAX_FAN_CH 16
#define GSC_HWMON_RESOLUTION 12
#define GSC_HWMON_VREF 2500
struct regmap *regmap;
const struct gsc_hwmon_channel *temp_ch[GSC_HWMON_MAX_TEMP_CH];
const struct gsc_hwmon_channel *in_ch[GSC_HWMON_MAX_IN_CH];
+ const struct gsc_hwmon_channel *fan_ch[GSC_HWMON_MAX_FAN_CH];
u32 temp_config[GSC_HWMON_MAX_TEMP_CH + 1];
u32 in_config[GSC_HWMON_MAX_IN_CH + 1];
+ u32 fan_config[GSC_HWMON_MAX_FAN_CH + 1];
struct hwmon_channel_info temp_info;
struct hwmon_channel_info in_info;
- const struct hwmon_channel_info *info[3];
+ struct hwmon_channel_info fan_info;
+ const struct hwmon_channel_info *info[4];
struct hwmon_chip_info chip;
};
case hwmon_temp:
ch = hwmon->temp_ch[channel];
break;
+ case hwmon_fan:
+ ch = hwmon->fan_ch[channel];
+ break;
default:
return -EOPNOTSUPP;
}
/* adjust by uV offset */
tmp += ch->mvoffset;
break;
+ case mode_fan:
+ tmp *= 30; /* convert to revolutions per minute */
+ break;
case mode_voltage_24bit:
case mode_voltage_16bit:
/* no adjustment needed */
case hwmon_temp:
*buf = hwmon->temp_ch[channel]->name;
break;
+ case hwmon_fan:
+ *buf = hwmon->fan_ch[channel]->name;
+ break;
default:
return -ENOTSUPP;
}
struct gsc_hwmon_platform_data *pdata = dev_get_platdata(dev);
struct gsc_hwmon_data *hwmon;
const struct attribute_group **groups;
- int i, i_in, i_temp;
+ int i, i_in, i_temp, i_fan;
if (!pdata) {
pdata = gsc_hwmon_get_devtree_pdata(dev);
if (IS_ERR(hwmon->regmap))
return PTR_ERR(hwmon->regmap);
- for (i = 0, i_in = 0, i_temp = 0; i < hwmon->pdata->nchannels; i++) {
+ for (i = 0, i_in = 0, i_temp = 0, i_fan = 0; i < hwmon->pdata->nchannels; i++) {
const struct gsc_hwmon_channel *ch = &pdata->channels[i];
switch (ch->mode) {
HWMON_T_LABEL;
i_temp++;
break;
+ case mode_fan:
+ if (i_fan == GSC_HWMON_MAX_FAN_CH) {
+ dev_err(gsc->dev, "too many fan channels\n");
+ return -EINVAL;
+ }
+ hwmon->fan_ch[i_fan] = ch;
+ hwmon->fan_config[i_fan] = HWMON_F_INPUT |
+ HWMON_F_LABEL;
+ i_fan++;
+ break;
case mode_voltage_24bit:
case mode_voltage_16bit:
case mode_voltage_raw:
hwmon->chip.info = hwmon->info;
hwmon->info[0] = &hwmon->temp_info;
hwmon->info[1] = &hwmon->in_info;
+ hwmon->info[2] = &hwmon->fan_info;
hwmon->temp_info.type = hwmon_temp;
hwmon->temp_info.config = hwmon->temp_config;
hwmon->in_info.type = hwmon_in;
hwmon->in_info.config = hwmon->in_config;
+ hwmon->fan_info.type = hwmon_fan;
+ hwmon->fan_info.config = hwmon->fan_config;
groups = pdata->fan_base ? gsc_hwmon_groups : NULL;
hwmon_dev = devm_hwmon_device_register_with_info(dev,
ATTRIBUTE_GROUPS(hih6130);
-static int hih6130_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int hih6130_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct hih6130 *hih6130;
.name = "hih6130",
.of_match_table = of_match_ptr(hih6130_of_match),
},
- .probe = hih6130_probe,
+ .probe_new = hih6130_probe,
.id_table = hih6130_id,
};
[hwmon_temp_lowest] = "temp%d_lowest",
[hwmon_temp_highest] = "temp%d_highest",
[hwmon_temp_reset_history] = "temp%d_reset_history",
+ [hwmon_temp_rated_min] = "temp%d_rated_min",
+ [hwmon_temp_rated_max] = "temp%d_rated_max",
};
static const char * const hwmon_in_attr_templates[] = {
[hwmon_in_max_alarm] = "in%d_max_alarm",
[hwmon_in_lcrit_alarm] = "in%d_lcrit_alarm",
[hwmon_in_crit_alarm] = "in%d_crit_alarm",
+ [hwmon_in_rated_min] = "in%d_rated_min",
+ [hwmon_in_rated_max] = "in%d_rated_max",
};
static const char * const hwmon_curr_attr_templates[] = {
[hwmon_curr_max_alarm] = "curr%d_max_alarm",
[hwmon_curr_lcrit_alarm] = "curr%d_lcrit_alarm",
[hwmon_curr_crit_alarm] = "curr%d_crit_alarm",
+ [hwmon_curr_rated_min] = "curr%d_rated_min",
+ [hwmon_curr_rated_max] = "curr%d_rated_max",
};
static const char * const hwmon_power_attr_templates[] = {
[hwmon_power_max_alarm] = "power%d_max_alarm",
[hwmon_power_lcrit_alarm] = "power%d_lcrit_alarm",
[hwmon_power_crit_alarm] = "power%d_crit_alarm",
+ [hwmon_power_rated_min] = "power%d_rated_min",
+ [hwmon_power_rated_max] = "power%d_rated_max",
};
static const char * const hwmon_energy_attr_templates[] = {
[hwmon_humidity_max_hyst] = "humidity%d_max_hyst",
[hwmon_humidity_alarm] = "humidity%d_alarm",
[hwmon_humidity_fault] = "humidity%d_fault",
+ [hwmon_humidity_rated_min] = "humidity%d_rated_min",
+ [hwmon_humidity_rated_max] = "humidity%d_rated_max",
};
static const char * const hwmon_fan_attr_templates[] = {
return 0;
}
-static int ina209_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ina209_probe(struct i2c_client *client)
{
struct i2c_adapter *adapter = client->adapter;
struct ina209_data *data;
.name = "ina209",
.of_match_table = of_match_ptr(ina209_of_match),
},
- .probe = ina209_probe,
+ .probe_new = ina209_probe,
.remove = ina209_remove,
.id_table = ina209_id,
};
.attrs = ina226_attrs,
};
-static int ina2xx_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id ina2xx_id[];
+
+static int ina2xx_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct ina2xx_data *data;
if (client->dev.of_node)
chip = (enum ina2xx_ids)of_device_get_match_data(&client->dev);
else
- chip = id->driver_data;
+ chip = i2c_match_id(ina2xx_id, client)->driver_data;
data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
if (!data)
.name = "ina2xx",
.of_match_table = of_match_ptr(ina2xx_of_match),
},
- .probe = ina2xx_probe,
+ .probe_new = ina2xx_probe,
.id_table = ina2xx_id,
};
return 0;
}
-static int ina3221_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ina3221_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct ina3221_data *ina;
MODULE_DEVICE_TABLE(i2c, ina3221_ids);
static struct i2c_driver ina3221_i2c_driver = {
- .probe = ina3221_probe,
+ .probe_new = ina3221_probe,
.remove = ina3221_remove,
.driver = {
.name = INA3221_DRIVER_NAME,
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Intel MAX 10 BMC HWMON Driver
+ *
+ * Copyright (C) 2018-2020 Intel Corporation. All rights reserved.
+ *
+ */
+#include <linux/device.h>
+#include <linux/hwmon.h>
+#include <linux/mfd/intel-m10-bmc.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/platform_device.h>
+
+struct m10bmc_sdata {
+ unsigned int reg_input;
+ unsigned int reg_max;
+ unsigned int reg_crit;
+ unsigned int reg_hyst;
+ unsigned int reg_min;
+ unsigned int multiplier;
+ const char *label;
+};
+
+struct m10bmc_hwmon_board_data {
+ const struct m10bmc_sdata *tables[hwmon_max];
+ const struct hwmon_channel_info **hinfo;
+};
+
+struct m10bmc_hwmon {
+ struct device *dev;
+ struct hwmon_chip_info chip;
+ char *hw_name;
+ struct intel_m10bmc *m10bmc;
+ const struct m10bmc_hwmon_board_data *bdata;
+};
+
+static const struct m10bmc_sdata n3000bmc_temp_tbl[] = {
+ { 0x100, 0x104, 0x108, 0x10c, 0x0, 500, "Board Temperature" },
+ { 0x110, 0x114, 0x118, 0x0, 0x0, 500, "FPGA Die Temperature" },
+ { 0x11c, 0x124, 0x120, 0x0, 0x0, 500, "QSFP0 Temperature" },
+ { 0x12c, 0x134, 0x130, 0x0, 0x0, 500, "QSFP1 Temperature" },
+ { 0x168, 0x0, 0x0, 0x0, 0x0, 500, "Retimer A Temperature" },
+ { 0x16c, 0x0, 0x0, 0x0, 0x0, 500, "Retimer A SerDes Temperature" },
+ { 0x170, 0x0, 0x0, 0x0, 0x0, 500, "Retimer B Temperature" },
+ { 0x174, 0x0, 0x0, 0x0, 0x0, 500, "Retimer B SerDes Temperature" },
+};
+
+static const struct m10bmc_sdata n3000bmc_in_tbl[] = {
+ { 0x128, 0x0, 0x0, 0x0, 0x0, 1, "QSFP0 Supply Voltage" },
+ { 0x138, 0x0, 0x0, 0x0, 0x0, 1, "QSFP1 Supply Voltage" },
+ { 0x13c, 0x0, 0x0, 0x0, 0x0, 1, "FPGA Core Voltage" },
+ { 0x144, 0x0, 0x0, 0x0, 0x0, 1, "12V Backplane Voltage" },
+ { 0x14c, 0x0, 0x0, 0x0, 0x0, 1, "1.2V Voltage" },
+ { 0x150, 0x0, 0x0, 0x0, 0x0, 1, "12V AUX Voltage" },
+ { 0x158, 0x0, 0x0, 0x0, 0x0, 1, "1.8V Voltage" },
+ { 0x15c, 0x0, 0x0, 0x0, 0x0, 1, "3.3V Voltage" },
+};
+
+static const struct m10bmc_sdata n3000bmc_curr_tbl[] = {
+ { 0x140, 0x0, 0x0, 0x0, 0x0, 1, "FPGA Core Current" },
+ { 0x148, 0x0, 0x0, 0x0, 0x0, 1, "12V Backplane Current" },
+ { 0x154, 0x0, 0x0, 0x0, 0x0, 1, "12V AUX Current" },
+};
+
+static const struct m10bmc_sdata n3000bmc_power_tbl[] = {
+ { 0x160, 0x0, 0x0, 0x0, 0x0, 1000, "Board Power" },
+};
+
+static const struct hwmon_channel_info *n3000bmc_hinfo[] = {
+ HWMON_CHANNEL_INFO(temp,
+ HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MAX_HYST |
+ HWMON_T_CRIT | HWMON_T_CRIT_HYST | HWMON_T_LABEL,
+ HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT |
+ HWMON_T_LABEL,
+ HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT |
+ HWMON_T_LABEL,
+ HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT |
+ HWMON_T_LABEL,
+ HWMON_T_INPUT | HWMON_T_LABEL,
+ HWMON_T_INPUT | HWMON_T_LABEL,
+ HWMON_T_INPUT | HWMON_T_LABEL,
+ HWMON_T_INPUT | HWMON_T_LABEL),
+ HWMON_CHANNEL_INFO(in,
+ HWMON_I_INPUT | HWMON_I_LABEL,
+ HWMON_I_INPUT | HWMON_I_LABEL,
+ HWMON_I_INPUT | HWMON_I_LABEL,
+ HWMON_I_INPUT | HWMON_I_LABEL,
+ HWMON_I_INPUT | HWMON_I_LABEL,
+ HWMON_I_INPUT | HWMON_I_LABEL,
+ HWMON_I_INPUT | HWMON_I_LABEL,
+ HWMON_I_INPUT | HWMON_I_LABEL),
+ HWMON_CHANNEL_INFO(curr,
+ HWMON_C_INPUT | HWMON_C_LABEL,
+ HWMON_C_INPUT | HWMON_C_LABEL,
+ HWMON_C_INPUT | HWMON_C_LABEL),
+ HWMON_CHANNEL_INFO(power,
+ HWMON_P_INPUT | HWMON_P_LABEL),
+ NULL
+};
+
+static const struct m10bmc_hwmon_board_data n3000bmc_hwmon_bdata = {
+ .tables = {
+ [hwmon_temp] = n3000bmc_temp_tbl,
+ [hwmon_in] = n3000bmc_in_tbl,
+ [hwmon_curr] = n3000bmc_curr_tbl,
+ [hwmon_power] = n3000bmc_power_tbl,
+ },
+
+ .hinfo = n3000bmc_hinfo,
+};
+
+static umode_t
+m10bmc_hwmon_is_visible(const void *data, enum hwmon_sensor_types type,
+ u32 attr, int channel)
+{
+ return 0444;
+}
+
+static const struct m10bmc_sdata *
+find_sensor_data(struct m10bmc_hwmon *hw, enum hwmon_sensor_types type,
+ int channel)
+{
+ const struct m10bmc_sdata *tbl;
+
+ tbl = hw->bdata->tables[type];
+ if (!tbl)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ return &tbl[channel];
+}
+
+static int do_sensor_read(struct m10bmc_hwmon *hw,
+ const struct m10bmc_sdata *data,
+ unsigned int regoff, long *val)
+{
+ unsigned int regval;
+ int ret;
+
+ ret = m10bmc_sys_read(hw->m10bmc, regoff, ®val);
+ if (ret)
+ return ret;
+
+ /*
+ * BMC Firmware will return 0xdeadbeef if the sensor value is invalid
+ * at that time. This usually happens on sensor channels which connect
+ * to external pluggable modules, e.g. QSFP temperature and voltage.
+ * When the QSFP is unplugged from cage, driver will get 0xdeadbeef
+ * from their registers.
+ */
+ if (regval == 0xdeadbeef)
+ return -ENODATA;
+
+ *val = regval * data->multiplier;
+
+ return 0;
+}
+
+static int m10bmc_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
+ u32 attr, int channel, long *val)
+{
+ struct m10bmc_hwmon *hw = dev_get_drvdata(dev);
+ unsigned int reg = 0, reg_hyst = 0;
+ const struct m10bmc_sdata *data;
+ long hyst, value;
+ int ret;
+
+ data = find_sensor_data(hw, type, channel);
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
+ switch (type) {
+ case hwmon_temp:
+ switch (attr) {
+ case hwmon_temp_input:
+ reg = data->reg_input;
+ break;
+ case hwmon_temp_max_hyst:
+ reg_hyst = data->reg_hyst;
+ fallthrough;
+ case hwmon_temp_max:
+ reg = data->reg_max;
+ break;
+ case hwmon_temp_crit_hyst:
+ reg_hyst = data->reg_hyst;
+ fallthrough;
+ case hwmon_temp_crit:
+ reg = data->reg_crit;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+ break;
+ case hwmon_in:
+ switch (attr) {
+ case hwmon_in_input:
+ reg = data->reg_input;
+ break;
+ case hwmon_in_max:
+ reg = data->reg_max;
+ break;
+ case hwmon_in_crit:
+ reg = data->reg_crit;
+ break;
+ case hwmon_in_min:
+ reg = data->reg_min;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+ break;
+ case hwmon_curr:
+ switch (attr) {
+ case hwmon_curr_input:
+ reg = data->reg_input;
+ break;
+ case hwmon_curr_max:
+ reg = data->reg_max;
+ break;
+ case hwmon_curr_crit:
+ reg = data->reg_crit;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+ break;
+ case hwmon_power:
+ switch (attr) {
+ case hwmon_power_input:
+ reg = data->reg_input;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ if (!reg)
+ return -EOPNOTSUPP;
+
+ ret = do_sensor_read(hw, data, reg, &value);
+ if (ret)
+ return ret;
+
+ if (reg_hyst) {
+ ret = do_sensor_read(hw, data, reg_hyst, &hyst);
+ if (ret)
+ return ret;
+
+ value -= hyst;
+ }
+
+ *val = value;
+
+ return 0;
+}
+
+static int m10bmc_hwmon_read_string(struct device *dev,
+ enum hwmon_sensor_types type,
+ u32 attr, int channel, const char **str)
+{
+ struct m10bmc_hwmon *hw = dev_get_drvdata(dev);
+ const struct m10bmc_sdata *data;
+
+ data = find_sensor_data(hw, type, channel);
+ if (IS_ERR(data))
+ return PTR_ERR(data);
+
+ *str = data->label;
+
+ return 0;
+}
+
+static const struct hwmon_ops m10bmc_hwmon_ops = {
+ .is_visible = m10bmc_hwmon_is_visible,
+ .read = m10bmc_hwmon_read,
+ .read_string = m10bmc_hwmon_read_string,
+};
+
+static int m10bmc_hwmon_probe(struct platform_device *pdev)
+{
+ const struct platform_device_id *id = platform_get_device_id(pdev);
+ struct intel_m10bmc *m10bmc = dev_get_drvdata(pdev->dev.parent);
+ struct device *hwmon_dev, *dev = &pdev->dev;
+ struct m10bmc_hwmon *hw;
+ int i;
+
+ hw = devm_kzalloc(dev, sizeof(*hw), GFP_KERNEL);
+ if (!hw)
+ return -ENOMEM;
+
+ hw->dev = dev;
+ hw->m10bmc = m10bmc;
+ hw->bdata = (const struct m10bmc_hwmon_board_data *)id->driver_data;
+
+ hw->chip.info = hw->bdata->hinfo;
+ hw->chip.ops = &m10bmc_hwmon_ops;
+
+ hw->hw_name = devm_kstrdup(dev, id->name, GFP_KERNEL);
+ if (!hw->hw_name)
+ return -ENOMEM;
+
+ for (i = 0; hw->hw_name[i]; i++)
+ if (hwmon_is_bad_char(hw->hw_name[i]))
+ hw->hw_name[i] = '_';
+
+ hwmon_dev = devm_hwmon_device_register_with_info(dev, hw->hw_name,
+ hw, &hw->chip, NULL);
+ return PTR_ERR_OR_ZERO(hwmon_dev);
+}
+
+static const struct platform_device_id intel_m10bmc_hwmon_ids[] = {
+ {
+ .name = "n3000bmc-hwmon",
+ .driver_data = (unsigned long)&n3000bmc_hwmon_bdata,
+ },
+ { }
+};
+
+static struct platform_driver intel_m10bmc_hwmon_driver = {
+ .probe = m10bmc_hwmon_probe,
+ .driver = {
+ .name = "intel-m10-bmc-hwmon",
+ },
+ .id_table = intel_m10bmc_hwmon_ids,
+};
+module_platform_driver(intel_m10bmc_hwmon_driver);
+
+MODULE_DEVICE_TABLE(platform, intel_m10bmc_hwmon_ids);
+MODULE_AUTHOR("Intel Corporation");
+MODULE_DESCRIPTION("Intel MAX 10 BMC hardware monitor");
+MODULE_LICENSE("GPL");
.info = jc42_info,
};
-static int jc42_probe(struct i2c_client *client, const struct i2c_device_id *id)
+static int jc42_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.pm = JC42_DEV_PM_OPS,
.of_match_table = of_match_ptr(jc42_of_ids),
},
- .probe = jc42_probe,
+ .probe_new = jc42_probe,
.remove = jc42_remove,
.id_table = jc42_id,
.detect = jc42_detect,
*/
#include <linux/bitops.h>
-#include <linux/debugfs.h>
#include <linux/err.h>
#include <linux/hwmon.h>
#include <linux/init.h>
#define F15H_M60H_HARDWARE_TEMP_CTRL_OFFSET 0xd8200c64
#define F15H_M60H_REPORTED_TEMP_CTRL_OFFSET 0xd8200ca4
-/* F17h M01h Access througn SMN */
-#define F17H_M01H_REPORTED_TEMP_CTRL_OFFSET 0x00059800
+/* Common for Zen CPU families (Family 17h and 18h) */
+#define ZEN_REPORTED_TEMP_CTRL_OFFSET 0x00059800
-#define F17H_M70H_CCD_TEMP(x) (0x00059954 + ((x) * 4))
-#define F17H_M70H_CCD_TEMP_VALID BIT(11)
-#define F17H_M70H_CCD_TEMP_MASK GENMASK(10, 0)
+#define ZEN_CCD_TEMP(x) (0x00059954 + ((x) * 4))
+#define ZEN_CCD_TEMP_VALID BIT(11)
+#define ZEN_CCD_TEMP_MASK GENMASK(10, 0)
-#define F17H_M01H_SVI 0x0005A000
-#define F17H_M01H_SVI_TEL_PLANE0 (F17H_M01H_SVI + 0xc)
-#define F17H_M01H_SVI_TEL_PLANE1 (F17H_M01H_SVI + 0x10)
+#define ZEN_CUR_TEMP_SHIFT 21
+#define ZEN_CUR_TEMP_RANGE_SEL_MASK BIT(19)
-#define CUR_TEMP_SHIFT 21
-#define CUR_TEMP_RANGE_SEL_MASK BIT(19)
+#define ZEN_SVI_BASE 0x0005A000
-#define CFACTOR_ICORE 1000000 /* 1A / LSB */
-#define CFACTOR_ISOC 250000 /* 0.25A / LSB */
+/* F17h thermal registers through SMN */
+#define F17H_M01H_SVI_TEL_PLANE0 (ZEN_SVI_BASE + 0xc)
+#define F17H_M01H_SVI_TEL_PLANE1 (ZEN_SVI_BASE + 0x10)
+#define F17H_M31H_SVI_TEL_PLANE0 (ZEN_SVI_BASE + 0x14)
+#define F17H_M31H_SVI_TEL_PLANE1 (ZEN_SVI_BASE + 0x10)
+
+#define F17H_M01H_CFACTOR_ICORE 1000000 /* 1A / LSB */
+#define F17H_M01H_CFACTOR_ISOC 250000 /* 0.25A / LSB */
+#define F17H_M31H_CFACTOR_ICORE 1000000 /* 1A / LSB */
+#define F17H_M31H_CFACTOR_ISOC 310000 /* 0.31A / LSB */
+
+/* F19h thermal registers through SMN */
+#define F19H_M01_SVI_TEL_PLANE0 (ZEN_SVI_BASE + 0x14)
+#define F19H_M01_SVI_TEL_PLANE1 (ZEN_SVI_BASE + 0x10)
+
+#define F19H_M01H_CFACTOR_ICORE 1000000 /* 1A / LSB */
+#define F19H_M01H_CFACTOR_ISOC 310000 /* 0.31A / LSB */
struct k10temp_data {
struct pci_dev *pdev;
F15H_M60H_REPORTED_TEMP_CTRL_OFFSET, regval);
}
-static void read_tempreg_nb_f17(struct pci_dev *pdev, u32 *regval)
+static void read_tempreg_nb_zen(struct pci_dev *pdev, u32 *regval)
{
amd_smn_read(amd_pci_dev_to_node_id(pdev),
- F17H_M01H_REPORTED_TEMP_CTRL_OFFSET, regval);
+ ZEN_REPORTED_TEMP_CTRL_OFFSET, regval);
}
static long get_raw_temp(struct k10temp_data *data)
long temp;
data->read_tempreg(data->pdev, ®val);
- temp = (regval >> CUR_TEMP_SHIFT) * 125;
+ temp = (regval >> ZEN_CUR_TEMP_SHIFT) * 125;
if (regval & data->temp_adjust_mask)
temp -= 49000;
return temp;
break;
case 2 ... 9: /* Tccd{1-8} */
amd_smn_read(amd_pci_dev_to_node_id(data->pdev),
- F17H_M70H_CCD_TEMP(channel - 2), ®val);
- *val = (regval & F17H_M70H_CCD_TEMP_MASK) * 125 - 49000;
+ ZEN_CCD_TEMP(channel - 2), ®val);
+ *val = (regval & ZEN_CCD_TEMP_MASK) * 125 - 49000;
break;
default:
return -EOPNOTSUPP;
(boot_cpu_data.x86_model == 4 && boot_cpu_data.x86_stepping <= 2);
}
-#ifdef CONFIG_DEBUG_FS
-
-static void k10temp_smn_regs_show(struct seq_file *s, struct pci_dev *pdev,
- u32 addr, int count)
-{
- u32 reg;
- int i;
-
- for (i = 0; i < count; i++) {
- if (!(i & 3))
- seq_printf(s, "0x%06x: ", addr + i * 4);
- amd_smn_read(amd_pci_dev_to_node_id(pdev), addr + i * 4, ®);
- seq_printf(s, "%08x ", reg);
- if ((i & 3) == 3)
- seq_puts(s, "\n");
- }
-}
-
-static int svi_show(struct seq_file *s, void *unused)
-{
- struct k10temp_data *data = s->private;
-
- k10temp_smn_regs_show(s, data->pdev, F17H_M01H_SVI, 32);
- return 0;
-}
-DEFINE_SHOW_ATTRIBUTE(svi);
-
-static int thm_show(struct seq_file *s, void *unused)
-{
- struct k10temp_data *data = s->private;
-
- k10temp_smn_regs_show(s, data->pdev,
- F17H_M01H_REPORTED_TEMP_CTRL_OFFSET, 256);
- return 0;
-}
-DEFINE_SHOW_ATTRIBUTE(thm);
-
-static void k10temp_debugfs_cleanup(void *ddir)
-{
- debugfs_remove_recursive(ddir);
-}
-
-static void k10temp_init_debugfs(struct k10temp_data *data)
-{
- struct dentry *debugfs;
- char name[32];
-
- /* Only show debugfs data for Family 17h/18h CPUs */
- if (!data->is_zen)
- return;
-
- scnprintf(name, sizeof(name), "k10temp-%s", pci_name(data->pdev));
-
- debugfs = debugfs_create_dir(name, NULL);
- if (debugfs) {
- debugfs_create_file("svi", 0444, debugfs, data, &svi_fops);
- debugfs_create_file("thm", 0444, debugfs, data, &thm_fops);
- devm_add_action_or_reset(&data->pdev->dev,
- k10temp_debugfs_cleanup, debugfs);
- }
-}
-
-#else
-
-static void k10temp_init_debugfs(struct k10temp_data *data)
-{
-}
-
-#endif
-
static const struct hwmon_channel_info *k10temp_info[] = {
HWMON_CHANNEL_INFO(temp,
HWMON_T_INPUT | HWMON_T_MAX |
for (i = 0; i < limit; i++) {
amd_smn_read(amd_pci_dev_to_node_id(pdev),
- F17H_M70H_CCD_TEMP(i), ®val);
- if (regval & F17H_M70H_CCD_TEMP_VALID)
+ ZEN_CCD_TEMP(i), ®val);
+ if (regval & ZEN_CCD_TEMP_VALID)
data->show_temp |= BIT(TCCD_BIT(i));
}
}
data->read_htcreg = read_htcreg_nb_f15;
data->read_tempreg = read_tempreg_nb_f15;
} else if (boot_cpu_data.x86 == 0x17 || boot_cpu_data.x86 == 0x18) {
- data->temp_adjust_mask = CUR_TEMP_RANGE_SEL_MASK;
- data->read_tempreg = read_tempreg_nb_f17;
+ data->temp_adjust_mask = ZEN_CUR_TEMP_RANGE_SEL_MASK;
+ data->read_tempreg = read_tempreg_nb_zen;
data->show_temp |= BIT(TDIE_BIT); /* show Tdie */
data->is_zen = true;
data->show_current = !is_threadripper() && !is_epyc();
data->svi_addr[0] = F17H_M01H_SVI_TEL_PLANE0;
data->svi_addr[1] = F17H_M01H_SVI_TEL_PLANE1;
- data->cfactor[0] = CFACTOR_ICORE;
- data->cfactor[1] = CFACTOR_ISOC;
+ data->cfactor[0] = F17H_M01H_CFACTOR_ICORE;
+ data->cfactor[1] = F17H_M01H_CFACTOR_ISOC;
k10temp_get_ccd_support(pdev, data, 4);
break;
case 0x31: /* Zen2 Threadripper */
case 0x71: /* Zen2 */
data->show_current = !is_threadripper() && !is_epyc();
- data->cfactor[0] = CFACTOR_ICORE;
- data->cfactor[1] = CFACTOR_ISOC;
- data->svi_addr[0] = F17H_M01H_SVI_TEL_PLANE1;
- data->svi_addr[1] = F17H_M01H_SVI_TEL_PLANE0;
+ data->cfactor[0] = F17H_M31H_CFACTOR_ICORE;
+ data->cfactor[1] = F17H_M31H_CFACTOR_ISOC;
+ data->svi_addr[0] = F17H_M31H_SVI_TEL_PLANE0;
+ data->svi_addr[1] = F17H_M31H_SVI_TEL_PLANE1;
+ k10temp_get_ccd_support(pdev, data, 8);
+ break;
+ }
+ } else if (boot_cpu_data.x86 == 0x19) {
+ data->temp_adjust_mask = ZEN_CUR_TEMP_RANGE_SEL_MASK;
+ data->read_tempreg = read_tempreg_nb_zen;
+ data->show_temp |= BIT(TDIE_BIT);
+ data->is_zen = true;
+
+ switch (boot_cpu_data.x86_model) {
+ case 0x0 ... 0x1: /* Zen3 */
+ data->show_current = true;
+ data->svi_addr[0] = F19H_M01_SVI_TEL_PLANE0;
+ data->svi_addr[1] = F19H_M01_SVI_TEL_PLANE1;
+ data->cfactor[0] = F19H_M01H_CFACTOR_ICORE;
+ data->cfactor[1] = F19H_M01H_CFACTOR_ISOC;
k10temp_get_ccd_support(pdev, data, 8);
break;
}
hwmon_dev = devm_hwmon_device_register_with_info(dev, "k10temp", data,
&k10temp_chip_info,
NULL);
- if (IS_ERR(hwmon_dev))
- return PTR_ERR(hwmon_dev);
-
- k10temp_init_debugfs(data);
-
- return 0;
+ return PTR_ERR_OR_ZERO(hwmon_dev);
}
static const struct pci_device_id k10temp_id_table[] = {
{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_M30H_DF_F3) },
{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_M60H_DF_F3) },
{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F3) },
+ { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_19H_DF_F3) },
{ PCI_VDEVICE(HYGON, PCI_DEVICE_ID_AMD_17H_DF_F3) },
{}
};
.attrs = pem_fan_attributes,
};
-static int pem_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int pem_probe(struct i2c_client *client)
{
struct i2c_adapter *adapter = client->adapter;
struct device *dev = &client->dev;
.driver = {
.name = "lineage_pem",
},
- .probe = pem_probe,
+ .probe_new = pem_probe,
.id_table = pem_id,
};
(data->config_fan & 0x20) ? "manual" : "auto");
}
-static int lm63_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id lm63_id[];
+
+static int lm63_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
if (client->dev.of_node)
data->kind = (enum chips)of_device_get_match_data(&client->dev);
else
- data->kind = id->driver_data;
+ data->kind = i2c_match_id(lm63_id, client)->driver_data;
if (data->kind == lm64)
data->temp2_offset = 16000;
.name = "lm63",
.of_match_table = of_match_ptr(lm63_of_match),
},
- .probe = lm63_probe,
+ .probe_new = lm63_probe,
.id_table = lm63_id,
.detect = lm63_detect,
.address_list = normal_i2c,
/* device probe and removal */
static int
-lm73_probe(struct i2c_client *client, const struct i2c_device_id *id)
+lm73_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.name = "lm73",
.of_match_table = lm73_of_match,
},
- .probe = lm73_probe,
+ .probe_new = lm73_probe,
.id_table = lm73_ids,
.detect = lm73_detect,
.address_list = normal_i2c,
#include <linux/of.h>
#include <linux/regmap.h>
#include <linux/util_macros.h>
+#include <linux/regulator/consumer.h>
#include "lm75.h"
/*
struct lm75_data {
struct i2c_client *client;
struct regmap *regmap;
+ struct regulator *vs;
u8 orig_conf;
u8 current_conf;
u8 resolution; /* In bits, 9 to 16 */
.use_single_write = true,
};
+static void lm75_disable_regulator(void *data)
+{
+ struct lm75_data *lm75 = data;
+
+ regulator_disable(lm75->vs);
+}
+
static void lm75_remove(void *data)
{
struct lm75_data *lm75 = data;
i2c_smbus_write_byte_data(client, LM75_REG_CONF, lm75->orig_conf);
}
-static int
-lm75_probe(struct i2c_client *client, const struct i2c_device_id *id)
+static const struct i2c_device_id lm75_ids[];
+
+static int lm75_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
if (client->dev.of_node)
kind = (enum lm75_type)of_device_get_match_data(&client->dev);
else
- kind = id->driver_data;
+ kind = i2c_match_id(lm75_ids, client)->driver_data;
if (!i2c_check_functionality(client->adapter,
I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA))
data->client = client;
data->kind = kind;
+ data->vs = devm_regulator_get(dev, "vs");
+ if (IS_ERR(data->vs))
+ return PTR_ERR(data->vs);
+
data->regmap = devm_regmap_init_i2c(client, &lm75_regmap_config);
if (IS_ERR(data->regmap))
return PTR_ERR(data->regmap);
data->sample_time = data->params->default_sample_time;
data->resolution = data->params->default_resolution;
+ /* Enable the power */
+ err = regulator_enable(data->vs);
+ if (err) {
+ dev_err(dev, "failed to enable regulator: %d\n", err);
+ return err;
+ }
+
+ err = devm_add_action_or_reset(dev, lm75_disable_regulator, data);
+ if (err)
+ return err;
+
/* Cache original configuration */
status = i2c_smbus_read_byte_data(client, LM75_REG_CONF);
if (status < 0) {
.of_match_table = of_match_ptr(lm75_of_match),
.pm = LM75_DEV_PM_OPS,
},
- .probe = lm75_probe,
+ .probe_new = lm75_probe,
.id_table = lm75_ids,
.detect = lm75_detect,
.address_list = normal_i2c,
lm77_write_value(client, LM77_REG_CONF, conf & 0xfe);
}
-static int lm77_probe(struct i2c_client *client, const struct i2c_device_id *id)
+static int lm77_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver = {
.name = "lm77",
},
- .probe = lm77_probe,
+ .probe_new = lm77_probe,
.id_table = lm77_id,
.detect = lm77_detect,
.address_list = normal_i2c,
return -ENODEV;
}
-static int lm78_i2c_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id lm78_i2c_id[];
+
+static int lm78_i2c_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
return -ENOMEM;
data->client = client;
- data->type = id->driver_data;
+ data->type = i2c_match_id(lm78_i2c_id, client)->driver_data;
/* Initialize the LM78 chip */
lm78_init_device(data);
.driver = {
.name = "lm78",
},
- .probe = lm78_i2c_probe,
+ .probe_new = lm78_i2c_probe,
.id_table = lm78_i2c_id,
.detect = lm78_i2c_detect,
.address_list = normal_i2c,
return 0;
}
-static int lm80_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int lm80_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver = {
.name = "lm80",
},
- .probe = lm80_probe,
+ .probe_new = lm80_probe,
.id_table = lm80_id,
.detect = lm80_detect,
.address_list = normal_i2c,
return 0;
}
-static int lm83_probe(struct i2c_client *new_client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id lm83_id[];
+
+static int lm83_probe(struct i2c_client *new_client)
{
struct device *hwmon_dev;
struct lm83_data *data;
* declare 1 and 3 common, and then 2 and 4 only for the LM83.
*/
data->groups[0] = &lm83_group;
- if (id->driver_data == lm83)
+ if (i2c_match_id(lm83_id, new_client)->driver_data == lm83)
data->groups[1] = &lm83_group_opt;
hwmon_dev = devm_hwmon_device_register_with_groups(&new_client->dev,
.driver = {
.name = "lm83",
},
- .probe = lm83_probe,
+ .probe_new = lm83_probe,
.id_table = lm83_id,
.detect = lm83_detect,
.address_list = normal_i2c,
return 0;
}
-static int lm85_probe(struct i2c_client *client, const struct i2c_device_id *id)
+static const struct i2c_device_id lm85_id[];
+
+static int lm85_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
if (client->dev.of_node)
data->type = (enum chips)of_device_get_match_data(&client->dev);
else
- data->type = id->driver_data;
+ data->type = i2c_match_id(lm85_id, client)->driver_data;
mutex_init(&data->update_lock);
/* Fill in the chip specific driver values */
.name = "lm85",
.of_match_table = of_match_ptr(lm85_of_match),
},
- .probe = lm85_probe,
+ .probe_new = lm85_probe,
.id_table = lm85_id,
.detect = lm85_detect,
.address_list = normal_i2c,
return 0;
}
-static int lm87_probe(struct i2c_client *client, const struct i2c_device_id *id)
+static int lm87_probe(struct i2c_client *client)
{
struct lm87_data *data;
struct device *hwmon_dev;
.name = "lm87",
.of_match_table = lm87_of_match,
},
- .probe = lm87_probe,
+ .probe_new = lm87_probe,
.id_table = lm87_id,
.detect = lm87_detect,
.address_list = normal_i2c,
.write = lm90_write,
};
-static int lm90_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int lm90_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct i2c_adapter *adapter = client->adapter;
if (client->dev.of_node)
data->kind = (enum chips)of_device_get_match_data(&client->dev);
else
- data->kind = id->driver_data;
+ data->kind = i2c_match_id(lm90_id, client)->driver_data;
if (data->kind == adm1032) {
if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE))
client->flags &= ~I2C_CLIENT_PEC;
.name = "lm90",
.of_match_table = of_match_ptr(lm90_of_match),
},
- .probe = lm90_probe,
+ .probe_new = lm90_probe,
.alert = lm90_alert,
.id_table = lm90_id,
.detect = lm90_detect,
return 0;
}
-static int lm92_probe(struct i2c_client *new_client,
- const struct i2c_device_id *id)
+static int lm92_probe(struct i2c_client *new_client)
{
struct device *hwmon_dev;
struct lm92_data *data;
.driver = {
.name = "lm92",
},
- .probe = lm92_probe,
+ .probe_new = lm92_probe,
.id_table = lm92_id,
.detect = lm92_detect,
.address_list = normal_i2c,
return 0;
}
-static int lm93_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int lm93_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct lm93_data *data;
.driver = {
.name = "lm93",
},
- .probe = lm93_probe,
+ .probe_new = lm93_probe,
.id_table = lm93_id,
.detect = lm93_detect,
.address_list = normal_i2c,
return 0;
}
-static int lm95234_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id lm95234_id[];
+
+static int lm95234_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct lm95234_data *data;
return err;
data->groups[0] = &lm95234_common_group;
- if (id->driver_data == lm95234)
+ if (i2c_match_id(lm95234_id, client)->driver_data == lm95234)
data->groups[1] = &lm95234_group;
hwmon_dev = devm_hwmon_device_register_with_groups(dev, client->name,
.driver = {
.name = DRVNAME,
},
- .probe = lm95234_probe,
+ .probe_new = lm95234_probe,
.id_table = lm95234_id,
.detect = lm95234_detect,
.address_list = normal_i2c,
.info = lm95241_info,
};
-static int lm95241_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int lm95241_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct lm95241_data *data;
.driver = {
.name = DEVNAME,
},
- .probe = lm95241_probe,
+ .probe_new = lm95241_probe,
.id_table = lm95241_id,
.detect = lm95241_detect,
.address_list = normal_i2c,
.info = lm95245_info,
};
-static int lm95245_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int lm95245_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct lm95245_data *data;
.name = "lm95245",
.of_match_table = of_match_ptr(lm95245_of_match),
},
- .probe = lm95245_probe,
+ .probe_new = lm95245_probe,
.id_table = lm95245_id,
.detect = lm95245_detect,
.address_list = normal_i2c,
.max_register = LTC2945_MIN_ADIN_THRES_L,
};
-static int ltc2945_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ltc2945_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver = {
.name = "ltc2945",
},
- .probe = ltc2945_probe,
+ .probe_new = ltc2945_probe,
.id_table = ltc2945_id,
};
.val_bits = 8,
};
-static int ltc2947_probe(struct i2c_client *i2c,
- const struct i2c_device_id *id)
+static int ltc2947_probe(struct i2c_client *i2c)
{
struct regmap *map;
.of_match_table = ltc2947_of_match,
.pm = <c2947_pm_ops,
},
- .probe = ltc2947_probe,
+ .probe_new = ltc2947_probe,
.id_table = ltc2947_id,
};
module_i2c_driver(ltc2947_driver);
};
__ATTRIBUTE_GROUPS(ltc2990);
-static int ltc2990_i2c_probe(struct i2c_client *i2c,
- const struct i2c_device_id *id)
+static int ltc2990_i2c_probe(struct i2c_client *i2c)
{
int ret;
struct device *hwmon_dev;
.driver = {
.name = "ltc2990",
},
- .probe = ltc2990_i2c_probe,
+ .probe_new = ltc2990_i2c_probe,
.id_table = ltc2990_i2c_id,
};
};
ATTRIBUTE_GROUPS(ltc4151);
-static int ltc4151_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ltc4151_probe(struct i2c_client *client)
{
struct i2c_adapter *adapter = client->adapter;
struct device *dev = &client->dev;
.name = "ltc4151",
.of_match_table = of_match_ptr(ltc4151_match),
},
- .probe = ltc4151_probe,
+ .probe_new = ltc4151_probe,
.id_table = ltc4151_id,
};
};
ATTRIBUTE_GROUPS(ltc4215);
-static int ltc4215_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ltc4215_probe(struct i2c_client *client)
{
struct i2c_adapter *adapter = client->adapter;
struct device *dev = &client->dev;
.driver = {
.name = "ltc4215",
},
- .probe = ltc4215_probe,
+ .probe_new = ltc4215_probe,
.id_table = ltc4215_id,
};
.max_register = LTC4222_ADC_CONTROL,
};
-static int ltc4222_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ltc4222_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver = {
.name = "ltc4222",
},
- .probe = ltc4222_probe,
+ .probe_new = ltc4222_probe,
.id_table = ltc4222_id,
};
return false;
}
-static int ltc4245_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ltc4245_probe(struct i2c_client *client)
{
struct i2c_adapter *adapter = client->adapter;
struct ltc4245_data *data;
.driver = {
.name = "ltc4245",
},
- .probe = ltc4245_probe,
+ .probe_new = ltc4245_probe,
.id_table = ltc4245_id,
};
.max_register = LTC4260_ADIN,
};
-static int ltc4260_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ltc4260_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver = {
.name = "ltc4260",
},
- .probe = ltc4260_probe,
+ .probe_new = ltc4260_probe,
.id_table = ltc4260_id,
};
};
ATTRIBUTE_GROUPS(ltc4261);
-static int ltc4261_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ltc4261_probe(struct i2c_client *client)
{
struct i2c_adapter *adapter = client->adapter;
struct device *dev = &client->dev;
.driver = {
.name = "ltc4261",
},
- .probe = ltc4261_probe,
+ .probe_new = ltc4261_probe,
.id_table = ltc4261_id,
};
.is_visible = max16065_secondary_is_visible,
};
-static int max16065_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id max16065_id[];
+
+static int max16065_probe(struct i2c_client *client)
{
struct i2c_adapter *adapter = client->adapter;
struct max16065_data *data;
bool have_secondary; /* true if chip has secondary limits */
bool secondary_is_max = false; /* secondary limits reflect max */
int groups = 0;
+ const struct i2c_device_id *id = i2c_match_id(max16065_id, client);
if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA
| I2C_FUNC_SMBUS_READ_WORD_DATA))
.driver = {
.name = "max16065",
},
- .probe = max16065_probe,
+ .probe_new = max16065_probe,
.id_table = max16065_id,
};
config & 0xBF); /* run */
}
-static int max1619_probe(struct i2c_client *new_client,
- const struct i2c_device_id *id)
+static int max1619_probe(struct i2c_client *new_client)
{
struct max1619_data *data;
struct device *hwmon_dev;
.name = "max1619",
.of_match_table = of_match_ptr(max1619_of_match),
},
- .probe = max1619_probe,
+ .probe_new = max1619_probe,
.id_table = max1619_id,
.detect = max1619_detect,
.address_list = normal_i2c,
return 0;
}
-static int max1668_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id max1668_id[];
+
+static int max1668_probe(struct i2c_client *client)
{
struct i2c_adapter *adapter = client->adapter;
struct device *dev = &client->dev;
return -ENOMEM;
data->client = client;
- data->type = id->driver_data;
+ data->type = i2c_match_id(max1668_id, client)->driver_data;
mutex_init(&data->update_lock);
/* sysfs hooks */
.driver = {
.name = "max1668",
},
- .probe = max1668_probe,
+ .probe_new = max1668_probe,
.id_table = max1668_id,
.detect = max1668_detect,
.address_list = max1668_addr_list,
}
static int
-max31730_probe(struct i2c_client *client, const struct i2c_device_id *id)
+max31730_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.of_match_table = of_match_ptr(max31730_of_match),
.pm = &max31730_pm_ops,
},
- .probe = max31730_probe,
+ .probe_new = max31730_probe,
.id_table = max31730_ids,
.detect = max31730_detect,
.address_list = normal_i2c,
return 0;
}
-static int max31790_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int max31790_probe(struct i2c_client *client)
{
struct i2c_adapter *adapter = client->adapter;
struct device *dev = &client->dev;
static struct i2c_driver max31790_driver = {
.class = I2C_CLASS_HWMON,
- .probe = max31790_probe,
+ .probe_new = max31790_probe,
.driver = {
.name = "max31790",
},
.info = max6621_info,
};
-static int max6621_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int max6621_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct max6621_data *data;
.name = MAX6621_DRV_NAME,
.of_match_table = of_match_ptr(max6621_of_match),
},
- .probe = max6621_probe,
+ .probe_new = max6621_probe,
.id_table = max6621_id,
};
return 0;
}
-static int max6639_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int max6639_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct max6639_data *data;
.name = "max6639",
.pm = &max6639_pm_ops,
},
- .probe = max6639_probe,
+ .probe_new = max6639_probe,
.id_table = max6639_id,
.detect = max6639_detect,
.address_list = normal_i2c,
};
ATTRIBUTE_GROUPS(max6642);
-static int max6642_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int max6642_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct max6642_data *data;
.driver = {
.name = "max6642",
},
- .probe = max6642_probe,
+ .probe_new = max6642_probe,
.id_table = max6642_id,
.detect = max6642_detect,
.address_list = normal_i2c,
.info = max6650_info,
};
-static int max6650_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id max6650_id[];
+
+static int max6650_probe(struct i2c_client *client)
{
struct thermal_cooling_device *cooling_dev;
struct device *dev = &client->dev;
data->client = client;
i2c_set_clientdata(client, data);
mutex_init(&data->update_lock);
- data->nr_fans = of_id ? (int)(uintptr_t)of_id->data : id->driver_data;
+ data->nr_fans = of_id ? (int)(uintptr_t)of_id->data :
+ i2c_match_id(max6650_id, client)->driver_data;
/*
* Initialize the max6650 chip
.name = "max6650",
.of_match_table = of_match_ptr(max6650_dt_match),
},
- .probe = max6650_probe,
+ .probe_new = max6650_probe,
.id_table = max6650_id,
};
return 0;
}
-static int max6697_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id max6697_id[];
+
+static int max6697_probe(struct i2c_client *client)
{
struct i2c_adapter *adapter = client->adapter;
struct device *dev = &client->dev;
if (client->dev.of_node)
data->type = (enum chips)of_device_get_match_data(&client->dev);
else
- data->type = id->driver_data;
+ data->type = i2c_match_id(max6697_id, client)->driver_data;
data->chip = &max6697_chip_data[data->type];
data->client = client;
mutex_init(&data->update_lock);
.name = "max6697",
.of_match_table = of_match_ptr(max6697_of_match),
},
- .probe = max6697_probe,
+ .probe_new = max6697_probe,
.id_table = max6697_id,
};
static DEVICE_ATTR_RO(in0_input);
-static int mcp3021_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id mcp3021_id[];
+
+static int mcp3021_probe(struct i2c_client *client)
{
int err;
struct mcp3021_data *data = NULL;
data->vdd = MCP3021_VDD_REF_DEFAULT;
}
- switch (id->driver_data) {
+ switch (i2c_match_id(mcp3021_id, client)->driver_data) {
case mcp3021:
data->sar_shift = MCP3021_SAR_SHIFT;
data->sar_mask = MCP3021_SAR_MASK;
.name = "mcp3021",
.of_match_table = of_match_ptr(of_mcp3021_match),
},
- .probe = mcp3021_probe,
+ .probe_new = mcp3021_probe,
.remove = mcp3021_remove,
.id_table = mcp3021_id,
};
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 MaxLinear, Inc.
+ *
+ * This driver is a hardware monitoring driver for PVT controller
+ * (MR75203) which is used to configure & control Moortec embedded
+ * analog IP to enable multiple embedded temperature sensor(TS),
+ * voltage monitor(VM) & process detector(PD) modules.
+ */
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/hwmon.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+/* PVT Common register */
+#define PVT_IP_CONFIG 0x04
+#define TS_NUM_MSK GENMASK(4, 0)
+#define TS_NUM_SFT 0
+#define PD_NUM_MSK GENMASK(12, 8)
+#define PD_NUM_SFT 8
+#define VM_NUM_MSK GENMASK(20, 16)
+#define VM_NUM_SFT 16
+#define CH_NUM_MSK GENMASK(31, 24)
+#define CH_NUM_SFT 24
+
+/* Macro Common Register */
+#define CLK_SYNTH 0x00
+#define CLK_SYNTH_LO_SFT 0
+#define CLK_SYNTH_HI_SFT 8
+#define CLK_SYNTH_HOLD_SFT 16
+#define CLK_SYNTH_EN BIT(24)
+#define CLK_SYS_CYCLES_MAX 514
+#define CLK_SYS_CYCLES_MIN 2
+#define HZ_PER_MHZ 1000000L
+
+#define SDIF_DISABLE 0x04
+
+#define SDIF_STAT 0x08
+#define SDIF_BUSY BIT(0)
+#define SDIF_LOCK BIT(1)
+
+#define SDIF_W 0x0c
+#define SDIF_PROG BIT(31)
+#define SDIF_WRN_W BIT(27)
+#define SDIF_WRN_R 0x00
+#define SDIF_ADDR_SFT 24
+
+#define SDIF_HALT 0x10
+#define SDIF_CTRL 0x14
+#define SDIF_SMPL_CTRL 0x20
+
+/* TS & PD Individual Macro Register */
+#define COM_REG_SIZE 0x40
+
+#define SDIF_DONE(n) (COM_REG_SIZE + 0x14 + 0x40 * (n))
+#define SDIF_SMPL_DONE BIT(0)
+
+#define SDIF_DATA(n) (COM_REG_SIZE + 0x18 + 0x40 * (n))
+#define SAMPLE_DATA_MSK GENMASK(15, 0)
+
+#define HILO_RESET(n) (COM_REG_SIZE + 0x2c + 0x40 * (n))
+
+/* VM Individual Macro Register */
+#define VM_COM_REG_SIZE 0x200
+#define VM_SDIF_DONE(n) (VM_COM_REG_SIZE + 0x34 + 0x200 * (n))
+#define VM_SDIF_DATA(n) (VM_COM_REG_SIZE + 0x40 + 0x200 * (n))
+
+/* SDA Slave Register */
+#define IP_CTRL 0x00
+#define IP_RST_REL BIT(1)
+#define IP_RUN_CONT BIT(3)
+#define IP_AUTO BIT(8)
+#define IP_VM_MODE BIT(10)
+
+#define IP_CFG 0x01
+#define CFG0_MODE_2 BIT(0)
+#define CFG0_PARALLEL_OUT 0
+#define CFG0_12_BIT 0
+#define CFG1_VOL_MEAS_MODE 0
+#define CFG1_PARALLEL_OUT 0
+#define CFG1_14_BIT 0
+
+#define IP_DATA 0x03
+
+#define IP_POLL 0x04
+#define VM_CH_INIT BIT(20)
+#define VM_CH_REQ BIT(21)
+
+#define IP_TMR 0x05
+#define POWER_DELAY_CYCLE_256 0x80
+#define POWER_DELAY_CYCLE_64 0x40
+
+#define PVT_POLL_DELAY_US 20
+#define PVT_POLL_TIMEOUT_US 20000
+#define PVT_H_CONST 100000
+#define PVT_CAL5_CONST 2047
+#define PVT_G_CONST 40000
+#define PVT_CONV_BITS 10
+#define PVT_N_CONST 90
+#define PVT_R_CONST 245805
+
+struct pvt_device {
+ struct regmap *c_map;
+ struct regmap *t_map;
+ struct regmap *p_map;
+ struct regmap *v_map;
+ struct clk *clk;
+ struct reset_control *rst;
+ u32 t_num;
+ u32 p_num;
+ u32 v_num;
+ u32 ip_freq;
+ u8 *vm_idx;
+};
+
+static umode_t pvt_is_visible(const void *data, enum hwmon_sensor_types type,
+ u32 attr, int channel)
+{
+ switch (type) {
+ case hwmon_temp:
+ if (attr == hwmon_temp_input)
+ return 0444;
+ break;
+ case hwmon_in:
+ if (attr == hwmon_in_input)
+ return 0444;
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+static int pvt_read_temp(struct device *dev, u32 attr, int channel, long *val)
+{
+ struct pvt_device *pvt = dev_get_drvdata(dev);
+ struct regmap *t_map = pvt->t_map;
+ u32 stat, nbs;
+ int ret;
+ u64 tmp;
+
+ switch (attr) {
+ case hwmon_temp_input:
+ ret = regmap_read_poll_timeout(t_map, SDIF_DONE(channel),
+ stat, stat & SDIF_SMPL_DONE,
+ PVT_POLL_DELAY_US,
+ PVT_POLL_TIMEOUT_US);
+ if (ret)
+ return ret;
+
+ ret = regmap_read(t_map, SDIF_DATA(channel), &nbs);
+ if(ret < 0)
+ return ret;
+
+ nbs &= SAMPLE_DATA_MSK;
+
+ /*
+ * Convert the register value to
+ * degrees centigrade temperature
+ */
+ tmp = nbs * PVT_H_CONST;
+ do_div(tmp, PVT_CAL5_CONST);
+ *val = tmp - PVT_G_CONST - pvt->ip_freq;
+
+ return 0;
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static int pvt_read_in(struct device *dev, u32 attr, int channel, long *val)
+{
+ struct pvt_device *pvt = dev_get_drvdata(dev);
+ struct regmap *v_map = pvt->v_map;
+ u32 n, stat;
+ u8 vm_idx;
+ int ret;
+
+ if (channel >= pvt->v_num)
+ return -EINVAL;
+
+ vm_idx = pvt->vm_idx[channel];
+
+ switch (attr) {
+ case hwmon_in_input:
+ ret = regmap_read_poll_timeout(v_map, VM_SDIF_DONE(vm_idx),
+ stat, stat & SDIF_SMPL_DONE,
+ PVT_POLL_DELAY_US,
+ PVT_POLL_TIMEOUT_US);
+ if (ret)
+ return ret;
+
+ ret = regmap_read(v_map, VM_SDIF_DATA(vm_idx), &n);
+ if(ret < 0)
+ return ret;
+
+ n &= SAMPLE_DATA_MSK;
+ /* Convert the N bitstream count into voltage */
+ *val = (PVT_N_CONST * n - PVT_R_CONST) >> PVT_CONV_BITS;
+
+ return 0;
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static int pvt_read(struct device *dev, enum hwmon_sensor_types type,
+ u32 attr, int channel, long *val)
+{
+ switch (type) {
+ case hwmon_temp:
+ return pvt_read_temp(dev, attr, channel, val);
+ case hwmon_in:
+ return pvt_read_in(dev, attr, channel, val);
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static const u32 pvt_chip_config[] = {
+ HWMON_C_REGISTER_TZ,
+ 0
+};
+
+static const struct hwmon_channel_info pvt_chip = {
+ .type = hwmon_chip,
+ .config = pvt_chip_config,
+};
+
+static struct hwmon_channel_info pvt_temp = {
+ .type = hwmon_temp,
+};
+
+static struct hwmon_channel_info pvt_in = {
+ .type = hwmon_in,
+};
+
+static const struct hwmon_ops pvt_hwmon_ops = {
+ .is_visible = pvt_is_visible,
+ .read = pvt_read,
+};
+
+static struct hwmon_chip_info pvt_chip_info = {
+ .ops = &pvt_hwmon_ops,
+};
+
+static int pvt_init(struct pvt_device *pvt)
+{
+ u16 sys_freq, key, middle, low = 4, high = 8;
+ struct regmap *t_map = pvt->t_map;
+ struct regmap *p_map = pvt->p_map;
+ struct regmap *v_map = pvt->v_map;
+ u32 t_num = pvt->t_num;
+ u32 p_num = pvt->p_num;
+ u32 v_num = pvt->v_num;
+ u32 clk_synth, val;
+ int ret;
+
+ sys_freq = clk_get_rate(pvt->clk) / HZ_PER_MHZ;
+ while (high >= low) {
+ middle = (low + high + 1) / 2;
+ key = DIV_ROUND_CLOSEST(sys_freq, middle);
+ if (key > CLK_SYS_CYCLES_MAX) {
+ low = middle + 1;
+ continue;
+ } else if (key < CLK_SYS_CYCLES_MIN) {
+ high = middle - 1;
+ continue;
+ } else {
+ break;
+ }
+ }
+
+ /*
+ * The system supports 'clk_sys' to 'clk_ip' frequency ratios
+ * from 2:1 to 512:1
+ */
+ key = clamp_val(key, CLK_SYS_CYCLES_MIN, CLK_SYS_CYCLES_MAX) - 2;
+
+ clk_synth = ((key + 1) >> 1) << CLK_SYNTH_LO_SFT |
+ (key >> 1) << CLK_SYNTH_HI_SFT |
+ (key >> 1) << CLK_SYNTH_HOLD_SFT | CLK_SYNTH_EN;
+
+ pvt->ip_freq = sys_freq * 100 / (key + 2);
+
+ if (t_num) {
+ ret = regmap_write(t_map, SDIF_SMPL_CTRL, 0x0);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_write(t_map, SDIF_HALT, 0x0);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_write(t_map, CLK_SYNTH, clk_synth);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_write(t_map, SDIF_DISABLE, 0x0);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_read_poll_timeout(t_map, SDIF_STAT,
+ val, !(val & SDIF_BUSY),
+ PVT_POLL_DELAY_US,
+ PVT_POLL_TIMEOUT_US);
+ if (ret)
+ return ret;
+
+ val = CFG0_MODE_2 | CFG0_PARALLEL_OUT | CFG0_12_BIT |
+ IP_CFG << SDIF_ADDR_SFT | SDIF_WRN_W | SDIF_PROG;
+ ret = regmap_write(t_map, SDIF_W, val);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_read_poll_timeout(t_map, SDIF_STAT,
+ val, !(val & SDIF_BUSY),
+ PVT_POLL_DELAY_US,
+ PVT_POLL_TIMEOUT_US);
+ if (ret)
+ return ret;
+
+ val = POWER_DELAY_CYCLE_256 | IP_TMR << SDIF_ADDR_SFT |
+ SDIF_WRN_W | SDIF_PROG;
+ ret = regmap_write(t_map, SDIF_W, val);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_read_poll_timeout(t_map, SDIF_STAT,
+ val, !(val & SDIF_BUSY),
+ PVT_POLL_DELAY_US,
+ PVT_POLL_TIMEOUT_US);
+ if (ret)
+ return ret;
+
+ val = IP_RST_REL | IP_RUN_CONT | IP_AUTO |
+ IP_CTRL << SDIF_ADDR_SFT |
+ SDIF_WRN_W | SDIF_PROG;
+ ret = regmap_write(t_map, SDIF_W, val);
+ if(ret < 0)
+ return ret;
+ }
+
+ if (p_num) {
+ ret = regmap_write(p_map, SDIF_HALT, 0x0);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_write(p_map, SDIF_DISABLE, BIT(p_num) - 1);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_write(p_map, CLK_SYNTH, clk_synth);
+ if(ret < 0)
+ return ret;
+ }
+
+ if (v_num) {
+ ret = regmap_write(v_map, SDIF_SMPL_CTRL, 0x0);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_write(v_map, SDIF_HALT, 0x0);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_write(v_map, CLK_SYNTH, clk_synth);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_write(v_map, SDIF_DISABLE, 0x0);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_read_poll_timeout(v_map, SDIF_STAT,
+ val, !(val & SDIF_BUSY),
+ PVT_POLL_DELAY_US,
+ PVT_POLL_TIMEOUT_US);
+ if (ret)
+ return ret;
+
+ val = CFG1_VOL_MEAS_MODE | CFG1_PARALLEL_OUT |
+ CFG1_14_BIT | IP_CFG << SDIF_ADDR_SFT |
+ SDIF_WRN_W | SDIF_PROG;
+ ret = regmap_write(v_map, SDIF_W, val);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_read_poll_timeout(v_map, SDIF_STAT,
+ val, !(val & SDIF_BUSY),
+ PVT_POLL_DELAY_US,
+ PVT_POLL_TIMEOUT_US);
+ if (ret)
+ return ret;
+
+ val = POWER_DELAY_CYCLE_64 | IP_TMR << SDIF_ADDR_SFT |
+ SDIF_WRN_W | SDIF_PROG;
+ ret = regmap_write(v_map, SDIF_W, val);
+ if(ret < 0)
+ return ret;
+
+ ret = regmap_read_poll_timeout(v_map, SDIF_STAT,
+ val, !(val & SDIF_BUSY),
+ PVT_POLL_DELAY_US,
+ PVT_POLL_TIMEOUT_US);
+ if (ret)
+ return ret;
+
+ val = IP_RST_REL | IP_RUN_CONT | IP_AUTO | IP_VM_MODE |
+ IP_CTRL << SDIF_ADDR_SFT |
+ SDIF_WRN_W | SDIF_PROG;
+ ret = regmap_write(v_map, SDIF_W, val);
+ if(ret < 0)
+ return ret;
+ }
+
+ return 0;
+}
+
+static struct regmap_config pvt_regmap_config = {
+ .reg_bits = 32,
+ .reg_stride = 4,
+ .val_bits = 32,
+};
+
+static int pvt_get_regmap(struct platform_device *pdev, char *reg_name,
+ struct pvt_device *pvt)
+{
+ struct device *dev = &pdev->dev;
+ struct regmap **reg_map;
+ void __iomem *io_base;
+
+ if (!strcmp(reg_name, "common"))
+ reg_map = &pvt->c_map;
+ else if (!strcmp(reg_name, "ts"))
+ reg_map = &pvt->t_map;
+ else if (!strcmp(reg_name, "pd"))
+ reg_map = &pvt->p_map;
+ else if (!strcmp(reg_name, "vm"))
+ reg_map = &pvt->v_map;
+ else
+ return -EINVAL;
+
+ io_base = devm_platform_ioremap_resource_byname(pdev, reg_name);
+ if (IS_ERR(io_base))
+ return PTR_ERR(io_base);
+
+ pvt_regmap_config.name = reg_name;
+ *reg_map = devm_regmap_init_mmio(dev, io_base, &pvt_regmap_config);
+ if (IS_ERR(*reg_map)) {
+ dev_err(dev, "failed to init register map\n");
+ return PTR_ERR(*reg_map);
+ }
+
+ return 0;
+}
+
+static void pvt_clk_disable(void *data)
+{
+ struct pvt_device *pvt = data;
+
+ clk_disable_unprepare(pvt->clk);
+}
+
+static int pvt_clk_enable(struct device *dev, struct pvt_device *pvt)
+{
+ int ret;
+
+ ret = clk_prepare_enable(pvt->clk);
+ if (ret)
+ return ret;
+
+ return devm_add_action_or_reset(dev, pvt_clk_disable, pvt);
+}
+
+static void pvt_reset_control_assert(void *data)
+{
+ struct pvt_device *pvt = data;
+
+ reset_control_assert(pvt->rst);
+}
+
+static int pvt_reset_control_deassert(struct device *dev, struct pvt_device *pvt)
+{
+ int ret;
+
+ ret = reset_control_deassert(pvt->rst);
+ if (ret)
+ return ret;
+
+ return devm_add_action_or_reset(dev, pvt_reset_control_assert, pvt);
+}
+
+static int mr75203_probe(struct platform_device *pdev)
+{
+ const struct hwmon_channel_info **pvt_info;
+ u32 ts_num, vm_num, pd_num, val, index, i;
+ struct device *dev = &pdev->dev;
+ u32 *temp_config, *in_config;
+ struct device *hwmon_dev;
+ struct pvt_device *pvt;
+ int ret;
+
+ pvt = devm_kzalloc(dev, sizeof(*pvt), GFP_KERNEL);
+ if (!pvt)
+ return -ENOMEM;
+
+ ret = pvt_get_regmap(pdev, "common", pvt);
+ if (ret)
+ return ret;
+
+ pvt->clk = devm_clk_get(dev, NULL);
+ if (IS_ERR(pvt->clk))
+ return dev_err_probe(dev, PTR_ERR(pvt->clk), "failed to get clock\n");
+
+ ret = pvt_clk_enable(dev, pvt);
+ if (ret) {
+ dev_err(dev, "failed to enable clock\n");
+ return ret;
+ }
+
+ pvt->rst = devm_reset_control_get_exclusive(dev, NULL);
+ if (IS_ERR(pvt->rst))
+ return dev_err_probe(dev, PTR_ERR(pvt->rst),
+ "failed to get reset control\n");
+
+ ret = pvt_reset_control_deassert(dev, pvt);
+ if (ret)
+ return dev_err_probe(dev, ret, "cannot deassert reset control\n");
+
+ ret = regmap_read(pvt->c_map, PVT_IP_CONFIG, &val);
+ if(ret < 0)
+ return ret;
+
+ ts_num = (val & TS_NUM_MSK) >> TS_NUM_SFT;
+ pd_num = (val & PD_NUM_MSK) >> PD_NUM_SFT;
+ vm_num = (val & VM_NUM_MSK) >> VM_NUM_SFT;
+ pvt->t_num = ts_num;
+ pvt->p_num = pd_num;
+ pvt->v_num = vm_num;
+ val = 0;
+ if (ts_num)
+ val++;
+ if (vm_num)
+ val++;
+ if (!val)
+ return -ENODEV;
+
+ pvt_info = devm_kcalloc(dev, val + 2, sizeof(*pvt_info), GFP_KERNEL);
+ if (!pvt_info)
+ return -ENOMEM;
+ pvt_info[0] = &pvt_chip;
+ index = 1;
+
+ if (ts_num) {
+ ret = pvt_get_regmap(pdev, "ts", pvt);
+ if (ret)
+ return ret;
+
+ temp_config = devm_kcalloc(dev, ts_num + 1,
+ sizeof(*temp_config), GFP_KERNEL);
+ if (!temp_config)
+ return -ENOMEM;
+
+ memset32(temp_config, HWMON_T_INPUT, ts_num);
+ pvt_temp.config = temp_config;
+ pvt_info[index++] = &pvt_temp;
+ }
+
+ if (pd_num) {
+ ret = pvt_get_regmap(pdev, "pd", pvt);
+ if (ret)
+ return ret;
+ }
+
+ if (vm_num) {
+ u32 num = vm_num;
+
+ ret = pvt_get_regmap(pdev, "vm", pvt);
+ if (ret)
+ return ret;
+
+ pvt->vm_idx = devm_kcalloc(dev, vm_num, sizeof(*pvt->vm_idx),
+ GFP_KERNEL);
+ if (!pvt->vm_idx)
+ return -ENOMEM;
+
+ ret = device_property_read_u8_array(dev, "intel,vm-map",
+ pvt->vm_idx, vm_num);
+ if (ret) {
+ num = 0;
+ } else {
+ for (i = 0; i < vm_num; i++)
+ if (pvt->vm_idx[i] >= vm_num ||
+ pvt->vm_idx[i] == 0xff) {
+ num = i;
+ break;
+ }
+ }
+
+ /*
+ * Incase intel,vm-map property is not defined, we assume
+ * incremental channel numbers.
+ */
+ for (i = num; i < vm_num; i++)
+ pvt->vm_idx[i] = i;
+
+ in_config = devm_kcalloc(dev, num + 1,
+ sizeof(*in_config), GFP_KERNEL);
+ if (!in_config)
+ return -ENOMEM;
+
+ memset32(in_config, HWMON_I_INPUT, num);
+ in_config[num] = 0;
+ pvt_in.config = in_config;
+
+ pvt_info[index++] = &pvt_in;
+ }
+
+ ret = pvt_init(pvt);
+ if (ret) {
+ dev_err(dev, "failed to init pvt: %d\n", ret);
+ return ret;
+ }
+
+ pvt_chip_info.info = pvt_info;
+ hwmon_dev = devm_hwmon_device_register_with_info(dev, "pvt",
+ pvt,
+ &pvt_chip_info,
+ NULL);
+
+ return PTR_ERR_OR_ZERO(hwmon_dev);
+}
+
+static const struct of_device_id moortec_pvt_of_match[] = {
+ { .compatible = "moortec,mr75203" },
+ { }
+};
+MODULE_DEVICE_TABLE(of, moortec_pvt_of_match);
+
+static struct platform_driver moortec_pvt_driver = {
+ .driver = {
+ .name = "moortec-pvt",
+ .of_match_table = moortec_pvt_of_match,
+ },
+ .probe = mr75203_probe,
+};
+module_platform_driver(moortec_pvt_driver);
+
+MODULE_LICENSE("GPL v2");
return regmap_update_bits(data->regmap, REG_VMON_ENABLE, 0x03, 0x03);
}
-static int nct7802_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int nct7802_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct nct7802_data *data;
.name = DRVNAME,
},
.detect = nct7802_detect,
- .probe = nct7802_probe,
+ .probe_new = nct7802_probe,
.id_table = nct7802_idtable,
.address_list = nct7802_address_list,
};
.get_timeleft = nct7904_wdt_get_timeleft,
};
-static int nct7904_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int nct7904_probe(struct i2c_client *client)
{
struct nct7904_data *data;
struct device *hwmon_dev;
.driver = {
.name = "nct7904",
},
- .probe = nct7904_probe,
+ .probe_new = nct7904_probe,
.id_table = nct7904_id,
.detect = nct7904_detect,
.address_list = normal_i2c,
return 0;
}
-static int p8_i2c_occ_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int p8_i2c_occ_probe(struct i2c_client *client)
{
struct occ *occ;
struct p8_i2c_occ *ctx = devm_kzalloc(&client->dev, sizeof(*ctx),
.name = "occ-hwmon",
.of_match_table = p8_i2c_occ_of_match,
},
- .probe = p8_i2c_occ_probe,
+ .probe_new = p8_i2c_occ_probe,
.remove = p8_i2c_occ_remove,
};
* Real code
*/
-static int pcf8591_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int pcf8591_probe(struct i2c_client *client)
{
struct pcf8591_data *data;
int err;
.driver = {
.name = "pcf8591",
},
- .probe = pcf8591_probe,
+ .probe_new = pcf8591_probe,
.remove = pcf8591_remove,
.id_table = pcf8591_id,
};
This driver can also be built as a module. If so, the module will
be called pmbus.
+config SENSORS_ADM1266
+ tristate "Analog Devices ADM1266 Sequencer"
+ select CRC8
+ depends on GPIOLIB
+ help
+ If you say yes here you get hardware monitoring support for Analog
+ Devices ADM1266 Cascadable Super Sequencer.
+
+ This driver can also be built as a module. If so, the module will
+ be called adm1266.
+
config SENSORS_ADM1275
tristate "Analog Devices ADM1275 and compatibles"
help
This driver can also be built as a module. If so, the module will
be called max8688.
+config SENSORS_MP2975
+ tristate "MPS MP2975"
+ help
+ If you say yes here you get hardware monitoring support for MPS
+ MP2975 Dual Loop Digital Multi-Phase Controller.
+
+ This driver can also be built as a module. If so, the module will
+ be called mp2975.
+
config SENSORS_PXE1610
tristate "Infineon PXE1610"
help
obj-$(CONFIG_PMBUS) += pmbus_core.o
obj-$(CONFIG_SENSORS_PMBUS) += pmbus.o
+obj-$(CONFIG_SENSORS_ADM1266) += adm1266.o
obj-$(CONFIG_SENSORS_ADM1275) += adm1275.o
obj-$(CONFIG_SENSORS_BEL_PFE) += bel-pfe.o
obj-$(CONFIG_SENSORS_IBM_CFFPS) += ibm-cffps.o
obj-$(CONFIG_SENSORS_MAX31785) += max31785.o
obj-$(CONFIG_SENSORS_MAX34440) += max34440.o
obj-$(CONFIG_SENSORS_MAX8688) += max8688.o
+obj-$(CONFIG_SENSORS_MP2975) += mp2975.o
obj-$(CONFIG_SENSORS_PXE1610) += pxe1610.o
obj-$(CONFIG_SENSORS_TPS40422) += tps40422.o
obj-$(CONFIG_SENSORS_TPS53679) += tps53679.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ADM1266 - Cascadable Super Sequencer with Margin
+ * Control and Fault Recording
+ *
+ * Copyright 2020 Analog Devices Inc.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/crc8.h>
+#include <linux/debugfs.h>
+#include <linux/gpio/driver.h>
+#include <linux/i2c.h>
+#include <linux/i2c-smbus.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/nvmem-provider.h>
+#include "pmbus.h"
+#include <linux/slab.h>
+#include <linux/timekeeping.h>
+
+#define ADM1266_BLACKBOX_CONFIG 0xD3
+#define ADM1266_PDIO_CONFIG 0xD4
+#define ADM1266_READ_STATE 0xD9
+#define ADM1266_READ_BLACKBOX 0xDE
+#define ADM1266_SET_RTC 0xDF
+#define ADM1266_GPIO_CONFIG 0xE1
+#define ADM1266_BLACKBOX_INFO 0xE6
+#define ADM1266_PDIO_STATUS 0xE9
+#define ADM1266_GPIO_STATUS 0xEA
+
+/* ADM1266 GPIO defines */
+#define ADM1266_GPIO_NR 9
+#define ADM1266_GPIO_FUNCTIONS(x) FIELD_GET(BIT(0), x)
+#define ADM1266_GPIO_INPUT_EN(x) FIELD_GET(BIT(2), x)
+#define ADM1266_GPIO_OUTPUT_EN(x) FIELD_GET(BIT(3), x)
+#define ADM1266_GPIO_OPEN_DRAIN(x) FIELD_GET(BIT(4), x)
+
+/* ADM1266 PDIO defines */
+#define ADM1266_PDIO_NR 16
+#define ADM1266_PDIO_PIN_CFG(x) FIELD_GET(GENMASK(15, 13), x)
+#define ADM1266_PDIO_GLITCH_FILT(x) FIELD_GET(GENMASK(12, 9), x)
+#define ADM1266_PDIO_OUT_CFG(x) FIELD_GET(GENMASK(2, 0), x)
+
+#define ADM1266_BLACKBOX_OFFSET 0
+#define ADM1266_BLACKBOX_SIZE 64
+
+#define ADM1266_PMBUS_BLOCK_MAX 255
+
+struct adm1266_data {
+ struct pmbus_driver_info info;
+ struct gpio_chip gc;
+ const char *gpio_names[ADM1266_GPIO_NR + ADM1266_PDIO_NR];
+ struct i2c_client *client;
+ struct dentry *debugfs_dir;
+ struct nvmem_config nvmem_config;
+ struct nvmem_device *nvmem;
+ u8 *dev_mem;
+ struct mutex buf_mutex;
+ u8 write_buf[ADM1266_PMBUS_BLOCK_MAX + 1] ____cacheline_aligned;
+ u8 read_buf[ADM1266_PMBUS_BLOCK_MAX + 1] ____cacheline_aligned;
+};
+
+static const struct nvmem_cell_info adm1266_nvmem_cells[] = {
+ {
+ .name = "blackbox",
+ .offset = ADM1266_BLACKBOX_OFFSET,
+ .bytes = 2048,
+ },
+};
+
+DECLARE_CRC8_TABLE(pmbus_crc_table);
+
+/*
+ * Different from Block Read as it sends data and waits for the slave to
+ * return a value dependent on that data. The protocol is simply a Write Block
+ * followed by a Read Block without the Read-Block command field and the
+ * Write-Block STOP bit.
+ */
+static int adm1266_pmbus_block_xfer(struct adm1266_data *data, u8 cmd, u8 w_len, u8 *data_w,
+ u8 *data_r)
+{
+ struct i2c_client *client = data->client;
+ struct i2c_msg msgs[2] = {
+ {
+ .addr = client->addr,
+ .flags = I2C_M_DMA_SAFE,
+ .buf = data->write_buf,
+ .len = w_len + 2,
+ },
+ {
+ .addr = client->addr,
+ .flags = I2C_M_RD | I2C_M_DMA_SAFE,
+ .buf = data->read_buf,
+ .len = ADM1266_PMBUS_BLOCK_MAX + 2,
+ }
+ };
+ u8 addr;
+ u8 crc;
+ int ret;
+
+ mutex_lock(&data->buf_mutex);
+
+ msgs[0].buf[0] = cmd;
+ msgs[0].buf[1] = w_len;
+ memcpy(&msgs[0].buf[2], data_w, w_len);
+
+ ret = i2c_transfer(client->adapter, msgs, 2);
+ if (ret != 2) {
+ if (ret >= 0)
+ ret = -EPROTO;
+
+ mutex_unlock(&data->buf_mutex);
+
+ return ret;
+ }
+
+ if (client->flags & I2C_CLIENT_PEC) {
+ addr = i2c_8bit_addr_from_msg(&msgs[0]);
+ crc = crc8(pmbus_crc_table, &addr, 1, 0);
+ crc = crc8(pmbus_crc_table, msgs[0].buf, msgs[0].len, crc);
+
+ addr = i2c_8bit_addr_from_msg(&msgs[1]);
+ crc = crc8(pmbus_crc_table, &addr, 1, crc);
+ crc = crc8(pmbus_crc_table, msgs[1].buf, msgs[1].buf[0] + 1, crc);
+
+ if (crc != msgs[1].buf[msgs[1].buf[0] + 1]) {
+ mutex_unlock(&data->buf_mutex);
+ return -EBADMSG;
+ }
+ }
+
+ memcpy(data_r, &msgs[1].buf[1], msgs[1].buf[0]);
+
+ ret = msgs[1].buf[0];
+ mutex_unlock(&data->buf_mutex);
+
+ return ret;
+}
+
+static const unsigned int adm1266_gpio_mapping[ADM1266_GPIO_NR][2] = {
+ {1, 0},
+ {2, 1},
+ {3, 2},
+ {4, 8},
+ {5, 9},
+ {6, 10},
+ {7, 11},
+ {8, 6},
+ {9, 7},
+};
+
+static const char *adm1266_names[ADM1266_GPIO_NR + ADM1266_PDIO_NR] = {
+ "GPIO1", "GPIO2", "GPIO3", "GPIO4", "GPIO5", "GPIO6", "GPIO7", "GPIO8",
+ "GPIO9", "PDIO1", "PDIO2", "PDIO3", "PDIO4", "PDIO5", "PDIO6",
+ "PDIO7", "PDIO8", "PDIO9", "PDIO10", "PDIO11", "PDIO12", "PDIO13",
+ "PDIO14", "PDIO15", "PDIO16",
+};
+
+static int adm1266_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+ struct adm1266_data *data = gpiochip_get_data(chip);
+ u8 read_buf[I2C_SMBUS_BLOCK_MAX + 1];
+ unsigned long pins_status;
+ unsigned int pmbus_cmd;
+ int ret;
+
+ if (offset < ADM1266_GPIO_NR)
+ pmbus_cmd = ADM1266_GPIO_STATUS;
+ else
+ pmbus_cmd = ADM1266_PDIO_STATUS;
+
+ ret = i2c_smbus_read_block_data(data->client, pmbus_cmd, read_buf);
+ if (ret < 0)
+ return ret;
+
+ pins_status = read_buf[0] + (read_buf[1] << 8);
+ if (offset < ADM1266_GPIO_NR)
+ return test_bit(adm1266_gpio_mapping[offset][1], &pins_status);
+
+ return test_bit(offset - ADM1266_GPIO_NR, &pins_status);
+}
+
+static int adm1266_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask,
+ unsigned long *bits)
+{
+ struct adm1266_data *data = gpiochip_get_data(chip);
+ u8 read_buf[ADM1266_PMBUS_BLOCK_MAX + 1];
+ unsigned long status;
+ unsigned int gpio_nr;
+ int ret;
+
+ ret = i2c_smbus_read_block_data(data->client, ADM1266_GPIO_STATUS, read_buf);
+ if (ret < 0)
+ return ret;
+
+ status = read_buf[0] + (read_buf[1] << 8);
+
+ *bits = 0;
+ for_each_set_bit(gpio_nr, mask, ADM1266_GPIO_NR) {
+ if (test_bit(adm1266_gpio_mapping[gpio_nr][1], &status))
+ set_bit(gpio_nr, bits);
+ }
+
+ ret = i2c_smbus_read_block_data(data->client, ADM1266_PDIO_STATUS, read_buf);
+ if (ret < 0)
+ return ret;
+
+ status = read_buf[0] + (read_buf[1] << 8);
+
+ *bits = 0;
+ for_each_set_bit_from(gpio_nr, mask, ADM1266_GPIO_NR + ADM1266_PDIO_STATUS) {
+ if (test_bit(gpio_nr - ADM1266_GPIO_NR, &status))
+ set_bit(gpio_nr, bits);
+ }
+
+ return 0;
+}
+
+static void adm1266_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
+{
+ struct adm1266_data *data = gpiochip_get_data(chip);
+ u8 read_buf[ADM1266_PMBUS_BLOCK_MAX + 1];
+ unsigned long gpio_config;
+ unsigned long pdio_config;
+ unsigned long pin_cfg;
+ u8 write_cmd;
+ int ret;
+ int i;
+
+ for (i = 0; i < ADM1266_GPIO_NR; i++) {
+ write_cmd = adm1266_gpio_mapping[i][1];
+ ret = adm1266_pmbus_block_xfer(data, ADM1266_GPIO_CONFIG, 1, &write_cmd, read_buf);
+ if (ret != 2)
+ return;
+
+ gpio_config = read_buf[0];
+ seq_puts(s, adm1266_names[i]);
+
+ seq_puts(s, " ( ");
+ if (!ADM1266_GPIO_FUNCTIONS(gpio_config)) {
+ seq_puts(s, "high-Z )\n");
+ continue;
+ }
+ if (ADM1266_GPIO_INPUT_EN(gpio_config))
+ seq_puts(s, "input ");
+ if (ADM1266_GPIO_OUTPUT_EN(gpio_config))
+ seq_puts(s, "output ");
+ if (ADM1266_GPIO_OPEN_DRAIN(gpio_config))
+ seq_puts(s, "open-drain )\n");
+ else
+ seq_puts(s, "push-pull )\n");
+ }
+
+ write_cmd = 0xFF;
+ ret = adm1266_pmbus_block_xfer(data, ADM1266_PDIO_CONFIG, 1, &write_cmd, read_buf);
+ if (ret != 32)
+ return;
+
+ for (i = 0; i < ADM1266_PDIO_NR; i++) {
+ seq_puts(s, adm1266_names[ADM1266_GPIO_NR + i]);
+
+ pdio_config = read_buf[2 * i];
+ pdio_config += (read_buf[2 * i + 1] << 8);
+ pin_cfg = ADM1266_PDIO_PIN_CFG(pdio_config);
+
+ seq_puts(s, " ( ");
+ if (!pin_cfg || pin_cfg > 5) {
+ seq_puts(s, "high-Z )\n");
+ continue;
+ }
+
+ if (pin_cfg & BIT(0))
+ seq_puts(s, "output ");
+
+ if (pin_cfg & BIT(1))
+ seq_puts(s, "input ");
+
+ seq_puts(s, ")\n");
+ }
+}
+
+static int adm1266_config_gpio(struct adm1266_data *data)
+{
+ const char *name = dev_name(&data->client->dev);
+ char *gpio_name;
+ int ret;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(data->gpio_names); i++) {
+ gpio_name = devm_kasprintf(&data->client->dev, GFP_KERNEL, "adm1266-%x-%s",
+ data->client->addr, adm1266_names[i]);
+ if (!gpio_name)
+ return -ENOMEM;
+
+ data->gpio_names[i] = gpio_name;
+ }
+
+ data->gc.label = name;
+ data->gc.parent = &data->client->dev;
+ data->gc.owner = THIS_MODULE;
+ data->gc.base = -1;
+ data->gc.names = data->gpio_names;
+ data->gc.ngpio = ARRAY_SIZE(data->gpio_names);
+ data->gc.get = adm1266_gpio_get;
+ data->gc.get_multiple = adm1266_gpio_get_multiple;
+ data->gc.dbg_show = adm1266_gpio_dbg_show;
+
+ ret = devm_gpiochip_add_data(&data->client->dev, &data->gc, data);
+ if (ret)
+ dev_err(&data->client->dev, "GPIO registering failed (%d)\n", ret);
+
+ return ret;
+}
+
+static int adm1266_state_read(struct seq_file *s, void *pdata)
+{
+ struct device *dev = s->private;
+ struct i2c_client *client = to_i2c_client(dev);
+ int ret;
+
+ ret = i2c_smbus_read_word_data(client, ADM1266_READ_STATE);
+ if (ret < 0)
+ return ret;
+
+ seq_printf(s, "%d\n", ret);
+
+ return 0;
+}
+
+static void adm1266_init_debugfs(struct adm1266_data *data)
+{
+ struct dentry *root;
+
+ root = pmbus_get_debugfs_dir(data->client);
+ if (!root)
+ return;
+
+ data->debugfs_dir = debugfs_create_dir(data->client->name, root);
+ if (!data->debugfs_dir)
+ return;
+
+ debugfs_create_devm_seqfile(&data->client->dev, "sequencer_state", data->debugfs_dir,
+ adm1266_state_read);
+}
+
+static int adm1266_nvmem_read_blackbox(struct adm1266_data *data, u8 *read_buff)
+{
+ int record_count;
+ char index;
+ u8 buf[5];
+ int ret;
+
+ ret = i2c_smbus_read_block_data(data->client, ADM1266_BLACKBOX_INFO, buf);
+ if (ret < 0)
+ return ret;
+
+ if (ret != 4)
+ return -EIO;
+
+ record_count = buf[3];
+
+ for (index = 0; index < record_count; index++) {
+ ret = adm1266_pmbus_block_xfer(data, ADM1266_READ_BLACKBOX, 1, &index, read_buff);
+ if (ret < 0)
+ return ret;
+
+ if (ret != ADM1266_BLACKBOX_SIZE)
+ return -EIO;
+
+ read_buff += ADM1266_BLACKBOX_SIZE;
+ }
+
+ return 0;
+}
+
+static int adm1266_nvmem_read(void *priv, unsigned int offset, void *val, size_t bytes)
+{
+ struct adm1266_data *data = priv;
+ int ret;
+
+ if (offset + bytes > data->nvmem_config.size)
+ return -EINVAL;
+
+ if (offset == 0) {
+ memset(data->dev_mem, 0, data->nvmem_config.size);
+
+ ret = adm1266_nvmem_read_blackbox(data, data->dev_mem);
+ if (ret) {
+ dev_err(&data->client->dev, "Could not read blackbox!");
+ return ret;
+ }
+ }
+
+ memcpy(val, data->dev_mem + offset, bytes);
+
+ return 0;
+}
+
+static int adm1266_config_nvmem(struct adm1266_data *data)
+{
+ data->nvmem_config.name = dev_name(&data->client->dev);
+ data->nvmem_config.dev = &data->client->dev;
+ data->nvmem_config.root_only = true;
+ data->nvmem_config.read_only = true;
+ data->nvmem_config.owner = THIS_MODULE;
+ data->nvmem_config.reg_read = adm1266_nvmem_read;
+ data->nvmem_config.cells = adm1266_nvmem_cells;
+ data->nvmem_config.ncells = ARRAY_SIZE(adm1266_nvmem_cells);
+ data->nvmem_config.priv = data;
+ data->nvmem_config.stride = 1;
+ data->nvmem_config.word_size = 1;
+ data->nvmem_config.size = adm1266_nvmem_cells[0].bytes;
+
+ data->dev_mem = devm_kzalloc(&data->client->dev, data->nvmem_config.size, GFP_KERNEL);
+ if (!data->dev_mem)
+ return -ENOMEM;
+
+ data->nvmem = devm_nvmem_register(&data->client->dev, &data->nvmem_config);
+ if (IS_ERR(data->nvmem)) {
+ dev_err(&data->client->dev, "Could not register nvmem!");
+ return PTR_ERR(data->nvmem);
+ }
+
+ return 0;
+}
+
+static int adm1266_set_rtc(struct adm1266_data *data)
+{
+ time64_t kt;
+ char write_buf[6];
+ int i;
+
+ kt = ktime_get_seconds();
+
+ memset(write_buf, 0, sizeof(write_buf));
+
+ for (i = 0; i < 4; i++)
+ write_buf[2 + i] = (kt >> (i * 8)) & 0xFF;
+
+ return i2c_smbus_write_block_data(data->client, ADM1266_SET_RTC, sizeof(write_buf),
+ write_buf);
+}
+
+static int adm1266_probe(struct i2c_client *client)
+{
+ struct adm1266_data *data;
+ int ret;
+ int i;
+
+ data = devm_kzalloc(&client->dev, sizeof(struct adm1266_data), GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+
+ data->client = client;
+ data->info.pages = 17;
+ data->info.format[PSC_VOLTAGE_OUT] = linear;
+ for (i = 0; i < data->info.pages; i++)
+ data->info.func[i] = PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT;
+
+ crc8_populate_msb(pmbus_crc_table, 0x7);
+ mutex_init(&data->buf_mutex);
+
+ ret = adm1266_config_gpio(data);
+ if (ret < 0)
+ return ret;
+
+ ret = adm1266_set_rtc(data);
+ if (ret < 0)
+ return ret;
+
+ ret = adm1266_config_nvmem(data);
+ if (ret < 0)
+ return ret;
+
+ ret = pmbus_do_probe(client, &data->info);
+ if (ret)
+ return ret;
+
+ adm1266_init_debugfs(data);
+
+ return 0;
+}
+
+static const struct of_device_id adm1266_of_match[] = {
+ { .compatible = "adi,adm1266" },
+ { }
+};
+MODULE_DEVICE_TABLE(of, adm1266_of_match);
+
+static const struct i2c_device_id adm1266_id[] = {
+ { "adm1266", 0 },
+ { }
+};
+MODULE_DEVICE_TABLE(i2c, adm1266_id);
+
+static struct i2c_driver adm1266_driver = {
+ .driver = {
+ .name = "adm1266",
+ .of_match_table = adm1266_of_match,
+ },
+ .probe_new = adm1266_probe,
+ .remove = pmbus_do_remove,
+ .id_table = adm1266_id,
+};
+
+module_i2c_driver(adm1266_driver);
+
+MODULE_AUTHOR("Alexandru Tachici <alexandru.tachici@analog.com>");
+MODULE_DESCRIPTION("PMBus driver for Analog Devices ADM1266");
+MODULE_LICENSE("GPL v2");
};
MODULE_DEVICE_TABLE(i2c, adm1275_id);
-static int adm1275_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int adm1275_probe(struct i2c_client *client)
{
s32 (*config_read_fn)(const struct i2c_client *client, u8 reg);
u8 block_buffer[I2C_SMBUS_BLOCK_MAX + 1];
return -ENODEV;
}
- if (id->driver_data != mid->driver_data)
+ if (strcmp(client->name, mid->name) != 0)
dev_notice(&client->dev,
"Device mismatch: Configured %s, detected %s\n",
- id->name, mid->name);
+ client->name, mid->name);
if (mid->driver_data == adm1272 || mid->driver_data == adm1278 ||
mid->driver_data == adm1293 || mid->driver_data == adm1294)
info->R[PSC_TEMPERATURE] = coefficients[tindex].R;
}
- return pmbus_do_probe(client, id, info);
+ return pmbus_do_probe(client, info);
}
static struct i2c_driver adm1275_driver = {
.driver = {
.name = "adm1275",
},
- .probe = adm1275_probe,
+ .probe_new = adm1275_probe,
.remove = pmbus_do_remove,
.id_table = adm1275_id,
};
},
};
-static int pfe_pmbus_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id pfe_device_id[];
+
+static int pfe_pmbus_probe(struct i2c_client *client)
{
int model;
- model = (int)id->driver_data;
+ model = (int)i2c_match_id(pfe_device_id, client)->driver_data;
/*
* PFE3000-12-069RA devices may not stay in page 0 during device
i2c_smbus_write_byte_data(client, PMBUS_PAGE, 0);
}
- return pmbus_do_probe(client, id, &pfe_driver_info[model]);
+ return pmbus_do_probe(client, &pfe_driver_info[model]);
}
static const struct i2c_device_id pfe_device_id[] = {
.driver = {
.name = "bel-pfe",
},
- .probe = pfe_pmbus_probe,
+ .probe_new = pfe_pmbus_probe,
.remove = pmbus_do_remove,
.id_table = pfe_device_id,
};
struct led_classdev led;
};
+static const struct i2c_device_id ibm_cffps_id[];
+
#define to_psu(x, y) container_of((x), struct ibm_cffps, debugfs_entries[(y)])
static ssize_t ibm_cffps_read_input_history(struct ibm_cffps *psu,
.flags = PMBUS_SKIP_STATUS_CHECK,
};
-static int ibm_cffps_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ibm_cffps_probe(struct i2c_client *client)
{
int i, rc;
enum versions vs = cffps_unknown;
struct dentry *ibm_cffps_dir;
struct ibm_cffps *psu;
const void *md = of_device_get_match_data(&client->dev);
+ const struct i2c_device_id *id;
- if (md)
+ if (md) {
vs = (enum versions)md;
- else if (id)
- vs = (enum versions)id->driver_data;
+ } else {
+ id = i2c_match_id(ibm_cffps_id, client);
+ if (id)
+ vs = (enum versions)id->driver_data;
+ }
if (vs == cffps_unknown) {
u16 ccin_revision = 0;
}
client->dev.platform_data = &ibm_cffps_pdata;
- rc = pmbus_do_probe(client, id, &ibm_cffps_info[vs]);
+ rc = pmbus_do_probe(client, &ibm_cffps_info[vs]);
if (rc)
return rc;
.name = "ibm-cffps",
.of_match_table = ibm_cffps_of_match,
},
- .probe = ibm_cffps_probe,
+ .probe_new = ibm_cffps_probe,
.remove = pmbus_do_remove,
.id_table = ibm_cffps_id,
};
.flags = PMBUS_SKIP_STATUS_CHECK,
};
-static int ipsps_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ipsps_probe(struct i2c_client *client)
{
client->dev.platform_data = &ipsps_pdata;
- return pmbus_do_probe(client, id, &ipsps_info);
+ return pmbus_do_probe(client, &ipsps_info);
}
static const struct i2c_device_id ipsps_id[] = {
.name = "inspur-ipsps",
.of_match_table = of_match_ptr(ipsps_of_match),
},
- .probe = ipsps_probe,
+ .probe_new = ipsps_probe,
.remove = pmbus_do_remove,
.id_table = ipsps_id,
};
return ret;
}
-static int ir35221_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ir35221_probe(struct i2c_client *client)
{
struct pmbus_driver_info *info;
u8 buf[I2C_SMBUS_BLOCK_MAX];
| PMBUS_HAVE_STATUS_INPUT | PMBUS_HAVE_STATUS_TEMP;
info->func[1] = info->func[0];
- return pmbus_do_probe(client, id, info);
+ return pmbus_do_probe(client, info);
}
static const struct i2c_device_id ir35221_id[] = {
.driver = {
.name = "ir35221",
},
- .probe = ir35221_probe,
+ .probe_new = ir35221_probe,
.remove = pmbus_do_remove,
.id_table = ir35221_id,
};
| PMBUS_HAVE_POUT,
};
-static int ir38064_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ir38064_probe(struct i2c_client *client)
{
- return pmbus_do_probe(client, id, &ir38064_info);
+ return pmbus_do_probe(client, &ir38064_info);
}
static const struct i2c_device_id ir38064_id[] = {
.driver = {
.name = "ir38064",
},
- .probe = ir38064_probe,
+ .probe_new = ir38064_probe,
.remove = pmbus_do_remove,
.id_table = ir38064_id,
};
.func[4] = IRPS5401_LDO_FUNC,
};
-static int irps5401_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int irps5401_probe(struct i2c_client *client)
{
- return pmbus_do_probe(client, id, &irps5401_info);
+ return pmbus_do_probe(client, &irps5401_info);
}
static const struct i2c_device_id irps5401_id[] = {
.driver = {
.name = "irps5401",
},
- .probe = irps5401_probe,
+ .probe_new = irps5401_probe,
.remove = pmbus_do_remove,
.id_table = irps5401_id,
};
raa_dmpvr2_hv,
};
+static const struct i2c_device_id raa_dmpvr_id[];
+
static ssize_t isl68137_avs_enable_show_page(struct i2c_client *client,
int page,
char *buf)
| PMBUS_HAVE_STATUS_IOUT | PMBUS_HAVE_POUT,
};
-static int isl68137_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int isl68137_probe(struct i2c_client *client)
{
struct pmbus_driver_info *info;
return -ENOMEM;
memcpy(info, &raa_dmpvr_info, sizeof(*info));
- switch (id->driver_data) {
+ switch (i2c_match_id(raa_dmpvr_id, client)->driver_data) {
case raa_dmpvr1_2rail:
info->pages = 2;
info->R[PSC_VOLTAGE_IN] = 3;
return -ENODEV;
}
- return pmbus_do_probe(client, id, info);
+ return pmbus_do_probe(client, info);
}
static const struct i2c_device_id raa_dmpvr_id[] = {
.driver = {
.name = "isl68137",
},
- .probe = isl68137_probe,
+ .probe_new = isl68137_probe,
.remove = pmbus_do_remove,
.id_table = raa_dmpvr_id,
};
#define to_lm25066_data(x) container_of(x, struct lm25066_data, info)
+static const struct i2c_device_id lm25066_id[];
+
static int lm25066_read_word_data(struct i2c_client *client, int page,
int phase, int reg)
{
return ret;
}
-static int lm25066_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int lm25066_probe(struct i2c_client *client)
{
int config;
struct lm25066_data *data;
if (config < 0)
return config;
- data->id = id->driver_data;
+ data->id = i2c_match_id(lm25066_id, client)->driver_data;
info = &data->info;
info->pages = 1;
info->b[PSC_POWER] = coeff[PSC_POWER].b;
}
- return pmbus_do_probe(client, id, info);
+ return pmbus_do_probe(client, info);
}
static const struct i2c_device_id lm25066_id[] = {
.driver = {
.name = "lm25066",
},
- .probe = lm25066_probe,
+ .probe_new = lm25066_probe,
.remove = pmbus_do_remove,
.id_table = lm25066_id,
};
return -ENODEV;
}
-static int ltc2978_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ltc2978_probe(struct i2c_client *client)
{
int i, chip_id;
struct ltc2978_data *data;
struct pmbus_driver_info *info;
+ const struct i2c_device_id *id;
if (!i2c_check_functionality(client->adapter,
I2C_FUNC_SMBUS_READ_WORD_DATA))
return chip_id;
data->id = chip_id;
+ id = i2c_match_id(ltc2978_id, client);
if (data->id != id->driver_data)
dev_warn(&client->dev,
- "Device mismatch: Configured %s, detected %s\n",
+ "Device mismatch: Configured %s (%d), detected %d\n",
id->name,
- ltc2978_id[data->id].name);
+ (int) id->driver_data,
+ chip_id);
info = &data->info;
info->write_word_data = ltc2978_write_word_data;
}
#endif
- return pmbus_do_probe(client, id, info);
+ return pmbus_do_probe(client, info);
}
.name = "ltc2978",
.of_match_table = of_match_ptr(ltc2978_of_match),
},
- .probe = ltc2978_probe,
+ .probe_new = ltc2978_probe,
.remove = pmbus_do_remove,
.id_table = ltc2978_id,
};
.write_word_data = ltc3815_write_word_data,
};
-static int ltc3815_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ltc3815_probe(struct i2c_client *client)
{
int chip_id;
if ((chip_id & LTC3815_ID_MASK) != LTC3815_ID)
return -ENODEV;
- return pmbus_do_probe(client, id, <c3815_info);
+ return pmbus_do_probe(client, <c3815_info);
}
static struct i2c_driver ltc3815_driver = {
.driver = {
.name = "ltc3815",
},
- .probe = ltc3815_probe,
+ .probe_new = ltc3815_probe,
.remove = pmbus_do_remove,
.id_table = ltc3815_id,
};
.write_word_data = max16064_write_word_data,
};
-static int max16064_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int max16064_probe(struct i2c_client *client)
{
- return pmbus_do_probe(client, id, &max16064_info);
+ return pmbus_do_probe(client, &max16064_info);
}
static const struct i2c_device_id max16064_id[] = {
.driver = {
.name = "max16064",
},
- .probe = max16064_probe,
+ .probe_new = max16064_probe,
.remove = pmbus_do_remove,
.id_table = max16064_id,
};
i2c_unregister_device(data->vsa);
}
-static int max16601_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int max16601_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
u8 buf[I2C_SMBUS_BLOCK_MAX + 1];
data->info = max16601_info;
- return pmbus_do_probe(client, id, &data->info);
+ return pmbus_do_probe(client, &data->info);
}
static const struct i2c_device_id max16601_id[] = {
.driver = {
.name = "max16601",
},
- .probe = max16601_probe,
+ .probe_new = max16601_probe,
.remove = pmbus_do_remove,
.id_table = max16601_id,
};
*/
#include <linux/bits.h>
+#include <linux/debugfs.h>
#include <linux/err.h>
#include <linux/i2c.h>
#include <linux/init.h>
max20743
};
+enum {
+ MAX20730_DEBUGFS_VOUT_MIN = 0,
+ MAX20730_DEBUGFS_FREQUENCY,
+ MAX20730_DEBUGFS_PG_DELAY,
+ MAX20730_DEBUGFS_INTERNAL_GAIN,
+ MAX20730_DEBUGFS_BOOT_VOLTAGE,
+ MAX20730_DEBUGFS_OUT_V_RAMP_RATE,
+ MAX20730_DEBUGFS_OC_PROTECT_MODE,
+ MAX20730_DEBUGFS_SS_TIMING,
+ MAX20730_DEBUGFS_IMAX,
+ MAX20730_DEBUGFS_OPERATION,
+ MAX20730_DEBUGFS_ON_OFF_CONFIG,
+ MAX20730_DEBUGFS_SMBALERT_MASK,
+ MAX20730_DEBUGFS_VOUT_MODE,
+ MAX20730_DEBUGFS_VOUT_COMMAND,
+ MAX20730_DEBUGFS_VOUT_MAX,
+ MAX20730_DEBUGFS_NUM_ENTRIES
+};
+
struct max20730_data {
enum chips id;
struct pmbus_driver_info info;
struct mutex lock; /* Used to protect against parallel writes */
u16 mfr_devset1;
+ u16 mfr_devset2;
+ u16 mfr_voutmin;
+ u32 vout_voltage_divider[2];
};
#define to_max20730_data(x) container_of(x, struct max20730_data, info)
+#define VOLT_FROM_REG(val) DIV_ROUND_CLOSEST((val), 1 << 9)
+
+#define PMBUS_SMB_ALERT_MASK 0x1B
+
+#define MAX20730_MFR_VOUT_MIN 0xd1
#define MAX20730_MFR_DEVSET1 0xd2
+#define MAX20730_MFR_DEVSET2 0xd3
+
+#define MAX20730_MFR_VOUT_MIN_MASK GENMASK(9, 0)
+#define MAX20730_MFR_VOUT_MIN_BIT_POS 0
+
+#define MAX20730_MFR_DEVSET1_RGAIN_MASK (BIT(13) | BIT(14))
+#define MAX20730_MFR_DEVSET1_OTP_MASK (BIT(11) | BIT(12))
+#define MAX20730_MFR_DEVSET1_VBOOT_MASK (BIT(8) | BIT(9))
+#define MAX20730_MFR_DEVSET1_OCP_MASK (BIT(5) | BIT(6))
+#define MAX20730_MFR_DEVSET1_FSW_MASK GENMASK(4, 2)
+#define MAX20730_MFR_DEVSET1_TSTAT_MASK (BIT(0) | BIT(1))
+
+#define MAX20730_MFR_DEVSET1_RGAIN_BIT_POS 13
+#define MAX20730_MFR_DEVSET1_OTP_BIT_POS 11
+#define MAX20730_MFR_DEVSET1_VBOOT_BIT_POS 8
+#define MAX20730_MFR_DEVSET1_OCP_BIT_POS 5
+#define MAX20730_MFR_DEVSET1_FSW_BIT_POS 2
+#define MAX20730_MFR_DEVSET1_TSTAT_BIT_POS 0
+
+#define MAX20730_MFR_DEVSET2_IMAX_MASK GENMASK(10, 8)
+#define MAX20730_MFR_DEVSET2_VRATE (BIT(6) | BIT(7))
+#define MAX20730_MFR_DEVSET2_OCPM_MASK BIT(5)
+#define MAX20730_MFR_DEVSET2_SS_MASK (BIT(0) | BIT(1))
+
+#define MAX20730_MFR_DEVSET2_IMAX_BIT_POS 8
+#define MAX20730_MFR_DEVSET2_VRATE_BIT_POS 6
+#define MAX20730_MFR_DEVSET2_OCPM_BIT_POS 5
+#define MAX20730_MFR_DEVSET2_SS_BIT_POS 0
+
+#define DEBUG_FS_DATA_MAX 16
+
+struct max20730_debugfs_data {
+ struct i2c_client *client;
+ int debugfs_entries[MAX20730_DEBUGFS_NUM_ENTRIES];
+};
+
+#define to_psu(x, y) container_of((x), \
+ struct max20730_debugfs_data, debugfs_entries[(y)])
+
+#ifdef CONFIG_DEBUG_FS
+static ssize_t max20730_debugfs_read(struct file *file, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ int ret, len;
+ int *idxp = file->private_data;
+ int idx = *idxp;
+ struct max20730_debugfs_data *psu = to_psu(idxp, idx);
+ const struct pmbus_driver_info *info;
+ const struct max20730_data *data;
+ char tbuf[DEBUG_FS_DATA_MAX] = { 0 };
+ u16 val;
+
+ info = pmbus_get_driver_info(psu->client);
+ data = to_max20730_data(info);
+
+ switch (idx) {
+ case MAX20730_DEBUGFS_VOUT_MIN:
+ ret = VOLT_FROM_REG(data->mfr_voutmin * 10000);
+ len = snprintf(tbuf, DEBUG_FS_DATA_MAX, "%d.%d\n",
+ ret / 10000, ret % 10000);
+ break;
+ case MAX20730_DEBUGFS_FREQUENCY:
+ val = (data->mfr_devset1 & MAX20730_MFR_DEVSET1_FSW_MASK)
+ >> MAX20730_MFR_DEVSET1_FSW_BIT_POS;
+
+ if (val == 0)
+ ret = 400;
+ else if (val == 1)
+ ret = 500;
+ else if (val == 2 || val == 3)
+ ret = 600;
+ else if (val == 4)
+ ret = 700;
+ else if (val == 5)
+ ret = 800;
+ else
+ ret = 900;
+ len = snprintf(tbuf, DEBUG_FS_DATA_MAX, "%d\n", ret);
+ break;
+ case MAX20730_DEBUGFS_PG_DELAY:
+ val = (data->mfr_devset1 & MAX20730_MFR_DEVSET1_TSTAT_MASK)
+ >> MAX20730_MFR_DEVSET1_TSTAT_BIT_POS;
+
+ if (val == 0)
+ len = strlcpy(tbuf, "2000\n", DEBUG_FS_DATA_MAX);
+ else if (val == 1)
+ len = strlcpy(tbuf, "125\n", DEBUG_FS_DATA_MAX);
+ else if (val == 2)
+ len = strlcpy(tbuf, "62.5\n", DEBUG_FS_DATA_MAX);
+ else
+ len = strlcpy(tbuf, "32\n", DEBUG_FS_DATA_MAX);
+ break;
+ case MAX20730_DEBUGFS_INTERNAL_GAIN:
+ val = (data->mfr_devset1 & MAX20730_MFR_DEVSET1_RGAIN_MASK)
+ >> MAX20730_MFR_DEVSET1_RGAIN_BIT_POS;
+
+ if (data->id == max20734) {
+ /* AN6209 */
+ if (val == 0)
+ len = strlcpy(tbuf, "0.8\n", DEBUG_FS_DATA_MAX);
+ else if (val == 1)
+ len = strlcpy(tbuf, "3.2\n", DEBUG_FS_DATA_MAX);
+ else if (val == 2)
+ len = strlcpy(tbuf, "1.6\n", DEBUG_FS_DATA_MAX);
+ else
+ len = strlcpy(tbuf, "6.4\n", DEBUG_FS_DATA_MAX);
+ } else if (data->id == max20730 || data->id == max20710) {
+ /* AN6042 or AN6140 */
+ if (val == 0)
+ len = strlcpy(tbuf, "0.9\n", DEBUG_FS_DATA_MAX);
+ else if (val == 1)
+ len = strlcpy(tbuf, "3.6\n", DEBUG_FS_DATA_MAX);
+ else if (val == 2)
+ len = strlcpy(tbuf, "1.8\n", DEBUG_FS_DATA_MAX);
+ else
+ len = strlcpy(tbuf, "7.2\n", DEBUG_FS_DATA_MAX);
+ } else if (data->id == max20743) {
+ /* AN6042 */
+ if (val == 0)
+ len = strlcpy(tbuf, "0.45\n", DEBUG_FS_DATA_MAX);
+ else if (val == 1)
+ len = strlcpy(tbuf, "1.8\n", DEBUG_FS_DATA_MAX);
+ else if (val == 2)
+ len = strlcpy(tbuf, "0.9\n", DEBUG_FS_DATA_MAX);
+ else
+ len = strlcpy(tbuf, "3.6\n", DEBUG_FS_DATA_MAX);
+ } else {
+ len = strlcpy(tbuf, "Not supported\n", DEBUG_FS_DATA_MAX);
+ }
+ break;
+ case MAX20730_DEBUGFS_BOOT_VOLTAGE:
+ val = (data->mfr_devset1 & MAX20730_MFR_DEVSET1_VBOOT_MASK)
+ >> MAX20730_MFR_DEVSET1_VBOOT_BIT_POS;
+
+ if (val == 0)
+ len = strlcpy(tbuf, "0.6484\n", DEBUG_FS_DATA_MAX);
+ else if (val == 1)
+ len = strlcpy(tbuf, "0.8984\n", DEBUG_FS_DATA_MAX);
+ else if (val == 2)
+ len = strlcpy(tbuf, "1.0\n", DEBUG_FS_DATA_MAX);
+ else
+ len = strlcpy(tbuf, "Invalid\n", DEBUG_FS_DATA_MAX);
+ break;
+ case MAX20730_DEBUGFS_OUT_V_RAMP_RATE:
+ val = (data->mfr_devset2 & MAX20730_MFR_DEVSET2_VRATE)
+ >> MAX20730_MFR_DEVSET2_VRATE_BIT_POS;
+
+ if (val == 0)
+ len = strlcpy(tbuf, "4\n", DEBUG_FS_DATA_MAX);
+ else if (val == 1)
+ len = strlcpy(tbuf, "2\n", DEBUG_FS_DATA_MAX);
+ else if (val == 2)
+ len = strlcpy(tbuf, "1\n", DEBUG_FS_DATA_MAX);
+ else
+ len = strlcpy(tbuf, "Invalid\n", DEBUG_FS_DATA_MAX);
+ break;
+ case MAX20730_DEBUGFS_OC_PROTECT_MODE:
+ ret = (data->mfr_devset2 & MAX20730_MFR_DEVSET2_OCPM_MASK)
+ >> MAX20730_MFR_DEVSET2_OCPM_BIT_POS;
+ len = snprintf(tbuf, DEBUG_FS_DATA_MAX, "%d\n", ret);
+ break;
+ case MAX20730_DEBUGFS_SS_TIMING:
+ val = (data->mfr_devset2 & MAX20730_MFR_DEVSET2_SS_MASK)
+ >> MAX20730_MFR_DEVSET2_SS_BIT_POS;
+
+ if (val == 0)
+ len = strlcpy(tbuf, "0.75\n", DEBUG_FS_DATA_MAX);
+ else if (val == 1)
+ len = strlcpy(tbuf, "1.5\n", DEBUG_FS_DATA_MAX);
+ else if (val == 2)
+ len = strlcpy(tbuf, "3\n", DEBUG_FS_DATA_MAX);
+ else
+ len = strlcpy(tbuf, "6\n", DEBUG_FS_DATA_MAX);
+ break;
+ case MAX20730_DEBUGFS_IMAX:
+ ret = (data->mfr_devset2 & MAX20730_MFR_DEVSET2_IMAX_MASK)
+ >> MAX20730_MFR_DEVSET2_IMAX_BIT_POS;
+ len = snprintf(tbuf, DEBUG_FS_DATA_MAX, "%d\n", ret);
+ break;
+ case MAX20730_DEBUGFS_OPERATION:
+ ret = i2c_smbus_read_byte_data(psu->client, PMBUS_OPERATION);
+ if (ret < 0)
+ return ret;
+ len = snprintf(tbuf, DEBUG_FS_DATA_MAX, "%d\n", ret);
+ break;
+ case MAX20730_DEBUGFS_ON_OFF_CONFIG:
+ ret = i2c_smbus_read_byte_data(psu->client, PMBUS_ON_OFF_CONFIG);
+ if (ret < 0)
+ return ret;
+ len = snprintf(tbuf, DEBUG_FS_DATA_MAX, "%d\n", ret);
+ break;
+ case MAX20730_DEBUGFS_SMBALERT_MASK:
+ ret = i2c_smbus_read_word_data(psu->client,
+ PMBUS_SMB_ALERT_MASK);
+ if (ret < 0)
+ return ret;
+ len = snprintf(tbuf, DEBUG_FS_DATA_MAX, "%d\n", ret);
+ break;
+ case MAX20730_DEBUGFS_VOUT_MODE:
+ ret = i2c_smbus_read_byte_data(psu->client, PMBUS_VOUT_MODE);
+ if (ret < 0)
+ return ret;
+ len = snprintf(tbuf, DEBUG_FS_DATA_MAX, "%d\n", ret);
+ break;
+ case MAX20730_DEBUGFS_VOUT_COMMAND:
+ ret = i2c_smbus_read_word_data(psu->client, PMBUS_VOUT_COMMAND);
+ if (ret < 0)
+ return ret;
+
+ ret = VOLT_FROM_REG(ret * 10000);
+ len = snprintf(tbuf, DEBUG_FS_DATA_MAX,
+ "%d.%d\n", ret / 10000, ret % 10000);
+ break;
+ case MAX20730_DEBUGFS_VOUT_MAX:
+ ret = i2c_smbus_read_word_data(psu->client, PMBUS_VOUT_MAX);
+ if (ret < 0)
+ return ret;
+
+ ret = VOLT_FROM_REG(ret * 10000);
+ len = snprintf(tbuf, DEBUG_FS_DATA_MAX,
+ "%d.%d\n", ret / 10000, ret % 10000);
+ break;
+ default:
+ len = strlcpy(tbuf, "Invalid\n", DEBUG_FS_DATA_MAX);
+ }
+
+ return simple_read_from_buffer(buf, count, ppos, tbuf, len);
+}
+
+static const struct file_operations max20730_fops = {
+ .llseek = noop_llseek,
+ .read = max20730_debugfs_read,
+ .write = NULL,
+ .open = simple_open,
+};
+
+static int max20730_init_debugfs(struct i2c_client *client,
+ struct max20730_data *data)
+{
+ int ret, i;
+ struct dentry *debugfs;
+ struct dentry *max20730_dir;
+ struct max20730_debugfs_data *psu;
+
+ ret = i2c_smbus_read_word_data(client, MAX20730_MFR_DEVSET2);
+ if (ret < 0)
+ return ret;
+ data->mfr_devset2 = ret;
+
+ ret = i2c_smbus_read_word_data(client, MAX20730_MFR_VOUT_MIN);
+ if (ret < 0)
+ return ret;
+ data->mfr_voutmin = ret;
+
+ psu = devm_kzalloc(&client->dev, sizeof(*psu), GFP_KERNEL);
+ if (!psu)
+ return -ENOMEM;
+ psu->client = client;
+
+ debugfs = pmbus_get_debugfs_dir(client);
+ if (!debugfs)
+ return -ENOENT;
+
+ max20730_dir = debugfs_create_dir(client->name, debugfs);
+ if (!max20730_dir)
+ return -ENOENT;
+
+ for (i = 0; i < MAX20730_DEBUGFS_NUM_ENTRIES; ++i)
+ psu->debugfs_entries[i] = i;
+
+ debugfs_create_file("vout_min", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_VOUT_MIN],
+ &max20730_fops);
+ debugfs_create_file("frequency", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_FREQUENCY],
+ &max20730_fops);
+ debugfs_create_file("power_good_delay", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_PG_DELAY],
+ &max20730_fops);
+ debugfs_create_file("internal_gain", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_INTERNAL_GAIN],
+ &max20730_fops);
+ debugfs_create_file("boot_voltage", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_BOOT_VOLTAGE],
+ &max20730_fops);
+ debugfs_create_file("out_voltage_ramp_rate", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_OUT_V_RAMP_RATE],
+ &max20730_fops);
+ debugfs_create_file("oc_protection_mode", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_OC_PROTECT_MODE],
+ &max20730_fops);
+ debugfs_create_file("soft_start_timing", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_SS_TIMING],
+ &max20730_fops);
+ debugfs_create_file("imax", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_IMAX],
+ &max20730_fops);
+ debugfs_create_file("operation", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_OPERATION],
+ &max20730_fops);
+ debugfs_create_file("on_off_config", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_ON_OFF_CONFIG],
+ &max20730_fops);
+ debugfs_create_file("smbalert_mask", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_SMBALERT_MASK],
+ &max20730_fops);
+ debugfs_create_file("vout_mode", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_VOUT_MODE],
+ &max20730_fops);
+ debugfs_create_file("vout_command", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_VOUT_COMMAND],
+ &max20730_fops);
+ debugfs_create_file("vout_max", 0444, max20730_dir,
+ &psu->debugfs_entries[MAX20730_DEBUGFS_VOUT_MAX],
+ &max20730_fops);
+
+ return 0;
+}
+#else
+static int max20730_init_debugfs(struct i2c_client *client,
+ struct max20730_data *data)
+{
+ return 0;
+}
+#endif /* CONFIG_DEBUG_FS */
+
+static const struct i2c_device_id max20730_id[];
/*
* Convert discreet value to direct data format. Strictly speaking, all passed
max_c = max_current[data->id][(data->mfr_devset1 >> 5) & 0x3];
ret = val_to_direct(max_c, PSC_CURRENT_OUT, info);
break;
+ case PMBUS_READ_VOUT:
+ ret = pmbus_read_word_data(client, page, phase, reg);
+ if (ret > 0 && data->vout_voltage_divider[0] && data->vout_voltage_divider[1]) {
+ u64 temp = DIV_ROUND_CLOSEST_ULL((u64)ret * data->vout_voltage_divider[1],
+ data->vout_voltage_divider[0]);
+ ret = clamp_val(temp, 0, 0xffff);
+ }
+ break;
default:
ret = -ENODATA;
break;
},
};
-static int max20730_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int max20730_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
u8 buf[I2C_SMBUS_BLOCK_MAX + 1];
if (client->dev.of_node)
chip_id = (enum chips)of_device_get_match_data(dev);
else
- chip_id = id->driver_data;
+ chip_id = i2c_match_id(max20730_id, client)->driver_data;
data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
if (!data)
data->id = chip_id;
mutex_init(&data->lock);
memcpy(&data->info, &max20730_info[chip_id], sizeof(data->info));
+ if (of_property_read_u32_array(client->dev.of_node, "vout-voltage-divider",
+ data->vout_voltage_divider,
+ ARRAY_SIZE(data->vout_voltage_divider)) != 0)
+ memset(data->vout_voltage_divider, 0, sizeof(data->vout_voltage_divider));
+ if (data->vout_voltage_divider[1] < data->vout_voltage_divider[0]) {
+ dev_err(dev,
+ "The total resistance of voltage divider is less than output resistance\n");
+ return -EINVAL;
+ }
ret = i2c_smbus_read_word_data(client, MAX20730_MFR_DEVSET1);
if (ret < 0)
return ret;
data->mfr_devset1 = ret;
- return pmbus_do_probe(client, id, &data->info);
+ ret = pmbus_do_probe(client, &data->info);
+ if (ret < 0)
+ return ret;
+
+ ret = max20730_init_debugfs(client, data);
+ if (ret)
+ dev_warn(dev, "Failed to register debugfs: %d\n",
+ ret);
+
+ return 0;
}
static const struct i2c_device_id max20730_id[] = {
.name = "max20730",
.of_match_table = max20730_of_match,
},
- .probe = max20730_probe,
+ .probe_new = max20730_probe,
.remove = pmbus_do_remove,
.id_table = max20730_id,
};
PMBUS_HAVE_POUT,
};
-static int max20751_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int max20751_probe(struct i2c_client *client)
{
- return pmbus_do_probe(client, id, &max20751_info);
+ return pmbus_do_probe(client, &max20751_info);
}
static const struct i2c_device_id max20751_id[] = {
.driver = {
.name = "max20751",
},
- .probe = max20751_probe,
+ .probe_new = max20751_probe,
.remove = pmbus_do_remove,
.id_table = max20751_id,
};
return 0;
}
-static int max31785_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int max31785_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct pmbus_driver_info *info;
if (ret == MAX31785A) {
dual_tach = true;
} else if (ret == MAX31785) {
- if (!strcmp("max31785a", id->name))
+ if (!strcmp("max31785a", client->name))
dev_warn(dev, "Expected max3175a, found max31785: cannot provide secondary tachometer readings\n");
} else {
return -ENODEV;
return ret;
}
- return pmbus_do_probe(client, id, info);
+ return pmbus_do_probe(client, info);
}
static const struct i2c_device_id max31785_id[] = {
.name = "max31785",
.of_match_table = max31785_of_match,
},
- .probe = max31785_probe,
+ .probe_new = max31785_probe,
.remove = pmbus_do_remove,
.id_table = max31785_id,
};
#define MAX34440_STATUS_OT_FAULT BIT(5)
#define MAX34440_STATUS_OT_WARN BIT(6)
+/*
+ * The whole max344* family have IOUT_OC_WARN_LIMIT and IOUT_OC_FAULT_LIMIT
+ * swapped from the standard pmbus spec addresses.
+ */
+#define MAX34440_IOUT_OC_WARN_LIMIT 0x46
+#define MAX34440_IOUT_OC_FAULT_LIMIT 0x4A
+
#define MAX34451_MFR_CHANNEL_CONFIG 0xe4
#define MAX34451_MFR_CHANNEL_CONFIG_SEL_MASK 0x3f
#define to_max34440_data(x) container_of(x, struct max34440_data, info)
+static const struct i2c_device_id max34440_id[];
+
static int max34440_read_word_data(struct i2c_client *client, int page,
int phase, int reg)
{
const struct max34440_data *data = to_max34440_data(info);
switch (reg) {
+ case PMBUS_IOUT_OC_FAULT_LIMIT:
+ ret = pmbus_read_word_data(client, page, phase,
+ MAX34440_IOUT_OC_FAULT_LIMIT);
+ break;
+ case PMBUS_IOUT_OC_WARN_LIMIT:
+ ret = pmbus_read_word_data(client, page, phase,
+ MAX34440_IOUT_OC_WARN_LIMIT);
+ break;
case PMBUS_VIRT_READ_VOUT_MIN:
ret = pmbus_read_word_data(client, page, phase,
MAX34440_MFR_VOUT_MIN);
int ret;
switch (reg) {
+ case PMBUS_IOUT_OC_FAULT_LIMIT:
+ ret = pmbus_write_word_data(client, page, MAX34440_IOUT_OC_FAULT_LIMIT,
+ word);
+ break;
+ case PMBUS_IOUT_OC_WARN_LIMIT:
+ ret = pmbus_write_word_data(client, page, MAX34440_IOUT_OC_WARN_LIMIT,
+ word);
+ break;
case PMBUS_VIRT_RESET_POUT_HISTORY:
ret = pmbus_write_word_data(client, page,
MAX34446_MFR_POUT_PEAK, 0);
.func[18] = PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP,
.func[19] = PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP,
.func[20] = PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP,
- .read_byte_data = max34440_read_byte_data,
.read_word_data = max34440_read_word_data,
.write_word_data = max34440_write_word_data,
},
.func[15] = PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP,
.func[16] = PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP,
.func[17] = PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP,
- .read_byte_data = max34440_read_byte_data,
.read_word_data = max34440_read_word_data,
.write_word_data = max34440_write_word_data,
},
.func[19] = PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP,
.func[20] = PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP,
.func[21] = PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP,
- .read_byte_data = max34440_read_byte_data,
.read_word_data = max34440_read_word_data,
.write_word_data = max34440_write_word_data,
},
};
-static int max34440_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int max34440_probe(struct i2c_client *client)
{
struct max34440_data *data;
int rv;
GFP_KERNEL);
if (!data)
return -ENOMEM;
- data->id = id->driver_data;
- data->info = max34440_info[id->driver_data];
+ data->id = i2c_match_id(max34440_id, client)->driver_data;
+ data->info = max34440_info[data->id];
if (data->id == max34451) {
rv = max34451_set_supported_funcs(client, data);
return rv;
}
- return pmbus_do_probe(client, id, &data->info);
+ return pmbus_do_probe(client, &data->info);
}
static const struct i2c_device_id max34440_id[] = {
.driver = {
.name = "max34440",
},
- .probe = max34440_probe,
+ .probe_new = max34440_probe,
.remove = pmbus_do_remove,
.id_table = max34440_id,
};
.write_word_data = max8688_write_word_data,
};
-static int max8688_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int max8688_probe(struct i2c_client *client)
{
- return pmbus_do_probe(client, id, &max8688_info);
+ return pmbus_do_probe(client, &max8688_info);
}
static const struct i2c_device_id max8688_id[] = {
.driver = {
.name = "max8688",
},
- .probe = max8688_probe,
+ .probe_new = max8688_probe,
.remove = pmbus_do_remove,
.id_table = max8688_id,
};
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Hardware monitoring driver for MPS Multi-phase Digital VR Controllers
+ *
+ * Copyright (C) 2020 Nvidia Technologies Ltd.
+ */
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include "pmbus.h"
+
+/* Vendor specific registers. */
+#define MP2975_MFR_APS_HYS_R2 0x0d
+#define MP2975_MFR_SLOPE_TRIM3 0x1d
+#define MP2975_MFR_VR_MULTI_CONFIG_R1 0x0d
+#define MP2975_MFR_VR_MULTI_CONFIG_R2 0x1d
+#define MP2975_MFR_APS_DECAY_ADV 0x56
+#define MP2975_MFR_DC_LOOP_CTRL 0x59
+#define MP2975_MFR_OCP_UCP_PHASE_SET 0x65
+#define MP2975_MFR_VR_CONFIG1 0x68
+#define MP2975_MFR_READ_CS1_2 0x82
+#define MP2975_MFR_READ_CS3_4 0x83
+#define MP2975_MFR_READ_CS5_6 0x84
+#define MP2975_MFR_READ_CS7_8 0x85
+#define MP2975_MFR_READ_CS9_10 0x86
+#define MP2975_MFR_READ_CS11_12 0x87
+#define MP2975_MFR_READ_IOUT_PK 0x90
+#define MP2975_MFR_READ_POUT_PK 0x91
+#define MP2975_MFR_READ_VREF_R1 0xa1
+#define MP2975_MFR_READ_VREF_R2 0xa3
+#define MP2975_MFR_OVP_TH_SET 0xe5
+#define MP2975_MFR_UVP_SET 0xe6
+
+#define MP2975_VOUT_FORMAT BIT(15)
+#define MP2975_VID_STEP_SEL_R1 BIT(4)
+#define MP2975_IMVP9_EN_R1 BIT(13)
+#define MP2975_VID_STEP_SEL_R2 BIT(3)
+#define MP2975_IMVP9_EN_R2 BIT(12)
+#define MP2975_PRT_THRES_DIV_OV_EN BIT(14)
+#define MP2975_DRMOS_KCS GENMASK(13, 12)
+#define MP2975_PROT_DEV_OV_OFF 10
+#define MP2975_PROT_DEV_OV_ON 5
+#define MP2975_SENSE_AMPL BIT(11)
+#define MP2975_SENSE_AMPL_UNIT 1
+#define MP2975_SENSE_AMPL_HALF 2
+#define MP2975_VIN_UV_LIMIT_UNIT 8
+
+#define MP2975_MAX_PHASE_RAIL1 8
+#define MP2975_MAX_PHASE_RAIL2 4
+#define MP2975_PAGE_NUM 2
+
+#define MP2975_RAIL2_FUNC (PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT | \
+ PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT | \
+ PMBUS_PHASE_VIRTUAL)
+
+struct mp2975_data {
+ struct pmbus_driver_info info;
+ int vout_scale;
+ int vid_step[MP2975_PAGE_NUM];
+ int vref[MP2975_PAGE_NUM];
+ int vref_off[MP2975_PAGE_NUM];
+ int vout_max[MP2975_PAGE_NUM];
+ int vout_ov_fixed[MP2975_PAGE_NUM];
+ int vout_format[MP2975_PAGE_NUM];
+ int curr_sense_gain[MP2975_PAGE_NUM];
+};
+
+#define to_mp2975_data(x) container_of(x, struct mp2975_data, info)
+
+static int mp2975_read_byte_data(struct i2c_client *client, int page, int reg)
+{
+ switch (reg) {
+ case PMBUS_VOUT_MODE:
+ /*
+ * Enforce VOUT direct format, since device allows to set the
+ * different formats for the different rails. Conversion from
+ * VID to direct provided by driver internally, in case it is
+ * necessary.
+ */
+ return PB_VOUT_MODE_DIRECT;
+ default:
+ return -ENODATA;
+ }
+}
+
+static int
+mp2975_read_word_helper(struct i2c_client *client, int page, int phase, u8 reg,
+ u16 mask)
+{
+ int ret = pmbus_read_word_data(client, page, phase, reg);
+
+ return (ret > 0) ? ret & mask : ret;
+}
+
+static int
+mp2975_vid2direct(int vrf, int val)
+{
+ switch (vrf) {
+ case vr12:
+ if (val >= 0x01)
+ return 250 + (val - 1) * 5;
+ break;
+ case vr13:
+ if (val >= 0x01)
+ return 500 + (val - 1) * 10;
+ break;
+ case imvp9:
+ if (val >= 0x01)
+ return 200 + (val - 1) * 10;
+ break;
+ default:
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static int
+mp2975_read_phase(struct i2c_client *client, struct mp2975_data *data,
+ int page, int phase, u8 reg)
+{
+ int ph_curr, ret;
+
+ ret = pmbus_read_word_data(client, page, phase, reg);
+ if (ret < 0)
+ return ret;
+
+ if (!((phase + 1) % MP2975_PAGE_NUM))
+ ret >>= 8;
+ ret &= 0xff;
+
+ /*
+ * Output value is calculated as: (READ_CSx / 80 – 1.23) / (Kcs * Rcs)
+ * where:
+ * - Kcs is the DrMOS current sense gain of power stage, which is
+ * obtained from the register MP2975_MFR_VR_CONFIG1, bits 13-12 with
+ * the following selection of DrMOS (data->curr_sense_gain[page]):
+ * 00b - 5µA/A, 01b - 8.5µA/A, 10b - 9.7µA/A, 11b - 10µA/A.
+ * - Rcs is the internal phase current sense resistor which is constant
+ * value 1kΩ.
+ */
+ ph_curr = ret * 100 - 9800;
+
+ /*
+ * Current phase sensing, providing by the device is not accurate
+ * for the light load. This because sampling of current occurrence of
+ * bit weight has a big deviation for light load. For handling such
+ * case phase current is represented as the maximum between the value
+ * calculated above and total rail current divided by number phases.
+ */
+ ret = pmbus_read_word_data(client, page, phase, PMBUS_READ_IOUT);
+ if (ret < 0)
+ return ret;
+
+ return max_t(int, DIV_ROUND_CLOSEST(ret, data->info.phases[page]),
+ DIV_ROUND_CLOSEST(ph_curr, data->curr_sense_gain[page]));
+}
+
+static int
+mp2975_read_phases(struct i2c_client *client, struct mp2975_data *data,
+ int page, int phase)
+{
+ int ret;
+
+ if (page) {
+ switch (phase) {
+ case 0 ... 1:
+ ret = mp2975_read_phase(client, data, page, phase,
+ MP2975_MFR_READ_CS7_8);
+ break;
+ case 2 ... 3:
+ ret = mp2975_read_phase(client, data, page, phase,
+ MP2975_MFR_READ_CS9_10);
+ break;
+ case 4 ... 5:
+ ret = mp2975_read_phase(client, data, page, phase,
+ MP2975_MFR_READ_CS11_12);
+ break;
+ default:
+ return -ENODATA;
+ }
+ } else {
+ switch (phase) {
+ case 0 ... 1:
+ ret = mp2975_read_phase(client, data, page, phase,
+ MP2975_MFR_READ_CS1_2);
+ break;
+ case 2 ... 3:
+ ret = mp2975_read_phase(client, data, page, phase,
+ MP2975_MFR_READ_CS3_4);
+ break;
+ case 4 ... 5:
+ ret = mp2975_read_phase(client, data, page, phase,
+ MP2975_MFR_READ_CS5_6);
+ break;
+ case 6 ... 7:
+ ret = mp2975_read_phase(client, data, page, phase,
+ MP2975_MFR_READ_CS7_8);
+ break;
+ case 8 ... 9:
+ ret = mp2975_read_phase(client, data, page, phase,
+ MP2975_MFR_READ_CS9_10);
+ break;
+ case 10 ... 11:
+ ret = mp2975_read_phase(client, data, page, phase,
+ MP2975_MFR_READ_CS11_12);
+ break;
+ default:
+ return -ENODATA;
+ }
+ }
+ return ret;
+}
+
+static int mp2975_read_word_data(struct i2c_client *client, int page,
+ int phase, int reg)
+{
+ const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+ struct mp2975_data *data = to_mp2975_data(info);
+ int ret;
+
+ switch (reg) {
+ case PMBUS_OT_FAULT_LIMIT:
+ ret = mp2975_read_word_helper(client, page, phase, reg,
+ GENMASK(7, 0));
+ break;
+ case PMBUS_VIN_OV_FAULT_LIMIT:
+ ret = mp2975_read_word_helper(client, page, phase, reg,
+ GENMASK(7, 0));
+ if (ret < 0)
+ return ret;
+
+ ret = DIV_ROUND_CLOSEST(ret, MP2975_VIN_UV_LIMIT_UNIT);
+ break;
+ case PMBUS_VOUT_OV_FAULT_LIMIT:
+ /*
+ * Register provides two values for over-voltage protection
+ * threshold for fixed (ovp2) and tracking (ovp1) modes. The
+ * minimum of these two values is provided as over-voltage
+ * fault alarm.
+ */
+ ret = mp2975_read_word_helper(client, page, phase,
+ MP2975_MFR_OVP_TH_SET,
+ GENMASK(2, 0));
+ if (ret < 0)
+ return ret;
+
+ ret = min_t(int, data->vout_max[page] + 50 * (ret + 1),
+ data->vout_ov_fixed[page]);
+ break;
+ case PMBUS_VOUT_UV_FAULT_LIMIT:
+ ret = mp2975_read_word_helper(client, page, phase,
+ MP2975_MFR_UVP_SET,
+ GENMASK(2, 0));
+ if (ret < 0)
+ return ret;
+
+ ret = DIV_ROUND_CLOSEST(data->vref[page] * 10 - 50 *
+ (ret + 1) * data->vout_scale, 10);
+ break;
+ case PMBUS_READ_VOUT:
+ ret = mp2975_read_word_helper(client, page, phase, reg,
+ GENMASK(11, 0));
+ if (ret < 0)
+ return ret;
+
+ /*
+ * READ_VOUT can be provided in VID or direct format. The
+ * format type is specified by bit 15 of the register
+ * MP2975_MFR_DC_LOOP_CTRL. The driver enforces VOUT direct
+ * format, since device allows to set the different formats for
+ * the different rails and also all VOUT limits registers are
+ * provided in a direct format. In case format is VID - convert
+ * to direct.
+ */
+ if (data->vout_format[page] == vid)
+ ret = mp2975_vid2direct(info->vrm_version[page], ret);
+ break;
+ case PMBUS_VIRT_READ_POUT_MAX:
+ ret = mp2975_read_word_helper(client, page, phase,
+ MP2975_MFR_READ_POUT_PK,
+ GENMASK(12, 0));
+ if (ret < 0)
+ return ret;
+
+ ret = DIV_ROUND_CLOSEST(ret, 4);
+ break;
+ case PMBUS_VIRT_READ_IOUT_MAX:
+ ret = mp2975_read_word_helper(client, page, phase,
+ MP2975_MFR_READ_IOUT_PK,
+ GENMASK(12, 0));
+ if (ret < 0)
+ return ret;
+
+ ret = DIV_ROUND_CLOSEST(ret, 4);
+ break;
+ case PMBUS_READ_IOUT:
+ ret = mp2975_read_phases(client, data, page, phase);
+ if (ret < 0)
+ return ret;
+
+ break;
+ case PMBUS_UT_WARN_LIMIT:
+ case PMBUS_UT_FAULT_LIMIT:
+ case PMBUS_VIN_UV_WARN_LIMIT:
+ case PMBUS_VIN_UV_FAULT_LIMIT:
+ case PMBUS_VOUT_UV_WARN_LIMIT:
+ case PMBUS_VOUT_OV_WARN_LIMIT:
+ case PMBUS_VIN_OV_WARN_LIMIT:
+ case PMBUS_IIN_OC_FAULT_LIMIT:
+ case PMBUS_IOUT_OC_LV_FAULT_LIMIT:
+ case PMBUS_IIN_OC_WARN_LIMIT:
+ case PMBUS_IOUT_OC_WARN_LIMIT:
+ case PMBUS_IOUT_OC_FAULT_LIMIT:
+ case PMBUS_IOUT_UC_FAULT_LIMIT:
+ case PMBUS_POUT_OP_FAULT_LIMIT:
+ case PMBUS_POUT_OP_WARN_LIMIT:
+ case PMBUS_PIN_OP_WARN_LIMIT:
+ return -ENXIO;
+ default:
+ return -ENODATA;
+ }
+
+ return ret;
+}
+
+static int mp2975_identify_multiphase_rail2(struct i2c_client *client)
+{
+ int ret;
+
+ /*
+ * Identify multiphase for rail 2 - could be from 0 to 4.
+ * In case phase number is zero – only page zero is supported
+ */
+ ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, 2);
+ if (ret < 0)
+ return ret;
+
+ /* Identify multiphase for rail 2 - could be from 0 to 4. */
+ ret = i2c_smbus_read_word_data(client, MP2975_MFR_VR_MULTI_CONFIG_R2);
+ if (ret < 0)
+ return ret;
+
+ ret &= GENMASK(2, 0);
+ return (ret >= 4) ? 4 : ret;
+}
+
+static void mp2975_set_phase_rail1(struct pmbus_driver_info *info)
+{
+ int i;
+
+ for (i = 0 ; i < info->phases[0]; i++)
+ info->pfunc[i] = PMBUS_HAVE_IOUT;
+}
+
+static void
+mp2975_set_phase_rail2(struct pmbus_driver_info *info, int num_phases)
+{
+ int i;
+
+ /* Set phases for rail 2 from upper to lower. */
+ for (i = 1; i <= num_phases; i++)
+ info->pfunc[MP2975_MAX_PHASE_RAIL1 - i] = PMBUS_HAVE_IOUT;
+}
+
+static int
+mp2975_identify_multiphase(struct i2c_client *client, struct mp2975_data *data,
+ struct pmbus_driver_info *info)
+{
+ int num_phases2, ret;
+
+ ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, 2);
+ if (ret < 0)
+ return ret;
+
+ /* Identify multiphase for rail 1 - could be from 1 to 8. */
+ ret = i2c_smbus_read_word_data(client, MP2975_MFR_VR_MULTI_CONFIG_R1);
+ if (ret <= 0)
+ return ret;
+
+ info->phases[0] = ret & GENMASK(3, 0);
+
+ /*
+ * The device provides a total of 8 PWM pins, and can be configured
+ * to different phase count applications for rail 1 and rail 2.
+ * Rail 1 can be set to 8 phases, while rail 2 can only be set to 4
+ * phases at most. When rail 1’s phase count is configured as 0, rail
+ * 1 operates with 1-phase DCM. When rail 2 phase count is configured
+ * as 0, rail 2 is disabled.
+ */
+ if (info->phases[0] > MP2975_MAX_PHASE_RAIL1)
+ return -EINVAL;
+
+ mp2975_set_phase_rail1(info);
+ num_phases2 = min(MP2975_MAX_PHASE_RAIL1 - info->phases[0],
+ MP2975_MAX_PHASE_RAIL2);
+ if (info->phases[1] && info->phases[1] <= num_phases2)
+ mp2975_set_phase_rail2(info, num_phases2);
+
+ return 0;
+}
+
+static int
+mp2975_identify_vid(struct i2c_client *client, struct mp2975_data *data,
+ struct pmbus_driver_info *info, u32 reg, int page,
+ u32 imvp_bit, u32 vr_bit)
+{
+ int ret;
+
+ /* Identify VID mode and step selection. */
+ ret = i2c_smbus_read_word_data(client, reg);
+ if (ret < 0)
+ return ret;
+
+ if (ret & imvp_bit) {
+ info->vrm_version[page] = imvp9;
+ data->vid_step[page] = MP2975_PROT_DEV_OV_OFF;
+ } else if (ret & vr_bit) {
+ info->vrm_version[page] = vr12;
+ data->vid_step[page] = MP2975_PROT_DEV_OV_ON;
+ } else {
+ info->vrm_version[page] = vr13;
+ data->vid_step[page] = MP2975_PROT_DEV_OV_OFF;
+ }
+
+ return 0;
+}
+
+static int
+mp2975_identify_rails_vid(struct i2c_client *client, struct mp2975_data *data,
+ struct pmbus_driver_info *info)
+{
+ int ret;
+
+ ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, 2);
+ if (ret < 0)
+ return ret;
+
+ /* Identify VID mode for rail 1. */
+ ret = mp2975_identify_vid(client, data, info,
+ MP2975_MFR_VR_MULTI_CONFIG_R1, 0,
+ MP2975_IMVP9_EN_R1, MP2975_VID_STEP_SEL_R1);
+ if (ret < 0)
+ return ret;
+
+ /* Identify VID mode for rail 2, if connected. */
+ if (info->phases[1])
+ ret = mp2975_identify_vid(client, data, info,
+ MP2975_MFR_VR_MULTI_CONFIG_R2, 1,
+ MP2975_IMVP9_EN_R2,
+ MP2975_VID_STEP_SEL_R2);
+ return ret;
+}
+
+static int
+mp2975_current_sense_gain_get(struct i2c_client *client,
+ struct mp2975_data *data)
+{
+ int i, ret;
+
+ /*
+ * Obtain DrMOS current sense gain of power stage from the register
+ * MP2975_MFR_VR_CONFIG1, bits 13-12. The value is selected as below:
+ * 00b - 5µA/A, 01b - 8.5µA/A, 10b - 9.7µA/A, 11b - 10µA/A. Other
+ * values are invalid.
+ */
+ for (i = 0 ; i < data->info.pages; i++) {
+ ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, i);
+ if (ret < 0)
+ return ret;
+ ret = i2c_smbus_read_word_data(client,
+ MP2975_MFR_VR_CONFIG1);
+ if (ret < 0)
+ return ret;
+
+ switch ((ret & MP2975_DRMOS_KCS) >> 12) {
+ case 0:
+ data->curr_sense_gain[i] = 50;
+ break;
+ case 1:
+ data->curr_sense_gain[i] = 85;
+ break;
+ case 2:
+ data->curr_sense_gain[i] = 97;
+ break;
+ default:
+ data->curr_sense_gain[i] = 100;
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static int
+mp2975_vref_get(struct i2c_client *client, struct mp2975_data *data,
+ struct pmbus_driver_info *info)
+{
+ int ret;
+
+ ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, 3);
+ if (ret < 0)
+ return ret;
+
+ /* Get voltage reference value for rail 1. */
+ ret = i2c_smbus_read_word_data(client, MP2975_MFR_READ_VREF_R1);
+ if (ret < 0)
+ return ret;
+
+ data->vref[0] = ret * data->vid_step[0];
+
+ /* Get voltage reference value for rail 2, if connected. */
+ if (data->info.pages == MP2975_PAGE_NUM) {
+ ret = i2c_smbus_read_word_data(client, MP2975_MFR_READ_VREF_R2);
+ if (ret < 0)
+ return ret;
+
+ data->vref[1] = ret * data->vid_step[1];
+ }
+ return 0;
+}
+
+static int
+mp2975_vref_offset_get(struct i2c_client *client, struct mp2975_data *data,
+ int page)
+{
+ int ret;
+
+ ret = i2c_smbus_read_word_data(client, MP2975_MFR_OVP_TH_SET);
+ if (ret < 0)
+ return ret;
+
+ switch ((ret & GENMASK(5, 3)) >> 3) {
+ case 1:
+ data->vref_off[page] = 140;
+ break;
+ case 2:
+ data->vref_off[page] = 220;
+ break;
+ case 4:
+ data->vref_off[page] = 400;
+ break;
+ default:
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static int
+mp2975_vout_max_get(struct i2c_client *client, struct mp2975_data *data,
+ struct pmbus_driver_info *info, int page)
+{
+ int ret;
+
+ /* Get maximum reference voltage of VID-DAC in VID format. */
+ ret = i2c_smbus_read_word_data(client, PMBUS_VOUT_MAX);
+ if (ret < 0)
+ return ret;
+
+ data->vout_max[page] = mp2975_vid2direct(info->vrm_version[page], ret &
+ GENMASK(8, 0));
+ return 0;
+}
+
+static int
+mp2975_identify_vout_format(struct i2c_client *client,
+ struct mp2975_data *data, int page)
+{
+ int ret;
+
+ ret = i2c_smbus_read_word_data(client, MP2975_MFR_DC_LOOP_CTRL);
+ if (ret < 0)
+ return ret;
+
+ if (ret & MP2975_VOUT_FORMAT)
+ data->vout_format[page] = vid;
+ else
+ data->vout_format[page] = direct;
+ return 0;
+}
+
+static int
+mp2975_vout_ov_scale_get(struct i2c_client *client, struct mp2975_data *data,
+ struct pmbus_driver_info *info)
+{
+ int thres_dev, sense_ampl, ret;
+
+ ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, 0);
+ if (ret < 0)
+ return ret;
+
+ /*
+ * Get divider for over- and under-voltage protection thresholds
+ * configuration from the Advanced Options of Auto Phase Shedding and
+ * decay register.
+ */
+ ret = i2c_smbus_read_word_data(client, MP2975_MFR_APS_DECAY_ADV);
+ if (ret < 0)
+ return ret;
+ thres_dev = ret & MP2975_PRT_THRES_DIV_OV_EN ? MP2975_PROT_DEV_OV_ON :
+ MP2975_PROT_DEV_OV_OFF;
+
+ /* Select the gain of remote sense amplifier. */
+ ret = i2c_smbus_read_word_data(client, PMBUS_VOUT_SCALE_LOOP);
+ if (ret < 0)
+ return ret;
+ sense_ampl = ret & MP2975_SENSE_AMPL ? MP2975_SENSE_AMPL_HALF :
+ MP2975_SENSE_AMPL_UNIT;
+
+ data->vout_scale = sense_ampl * thres_dev;
+
+ return 0;
+}
+
+static int
+mp2975_vout_per_rail_config_get(struct i2c_client *client,
+ struct mp2975_data *data,
+ struct pmbus_driver_info *info)
+{
+ int i, ret;
+
+ for (i = 0; i < data->info.pages; i++) {
+ ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, i);
+ if (ret < 0)
+ return ret;
+
+ /* Obtain voltage reference offsets. */
+ ret = mp2975_vref_offset_get(client, data, i);
+ if (ret < 0)
+ return ret;
+
+ /* Obtain maximum voltage values. */
+ ret = mp2975_vout_max_get(client, data, info, i);
+ if (ret < 0)
+ return ret;
+
+ /*
+ * Get VOUT format for READ_VOUT command : VID or direct.
+ * Pages on same device can be configured with different
+ * formats.
+ */
+ ret = mp2975_identify_vout_format(client, data, i);
+ if (ret < 0)
+ return ret;
+
+ /*
+ * Set over-voltage fixed value. Thresholds are provided as
+ * fixed value, and tracking value. The minimum of them are
+ * exposed as over-voltage critical threshold.
+ */
+ data->vout_ov_fixed[i] = data->vref[i] +
+ DIV_ROUND_CLOSEST(data->vref_off[i] *
+ data->vout_scale,
+ 10);
+ }
+
+ return 0;
+}
+
+static struct pmbus_driver_info mp2975_info = {
+ .pages = 1,
+ .format[PSC_VOLTAGE_IN] = linear,
+ .format[PSC_VOLTAGE_OUT] = direct,
+ .format[PSC_TEMPERATURE] = direct,
+ .format[PSC_CURRENT_IN] = linear,
+ .format[PSC_CURRENT_OUT] = direct,
+ .format[PSC_POWER] = direct,
+ .m[PSC_TEMPERATURE] = 1,
+ .m[PSC_VOLTAGE_OUT] = 1,
+ .R[PSC_VOLTAGE_OUT] = 3,
+ .m[PSC_CURRENT_OUT] = 1,
+ .m[PSC_POWER] = 1,
+ .func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
+ PMBUS_HAVE_IIN | PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT |
+ PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP | PMBUS_HAVE_POUT |
+ PMBUS_HAVE_PIN | PMBUS_HAVE_STATUS_INPUT | PMBUS_PHASE_VIRTUAL,
+ .read_byte_data = mp2975_read_byte_data,
+ .read_word_data = mp2975_read_word_data,
+};
+
+static int mp2975_probe(struct i2c_client *client)
+{
+ struct pmbus_driver_info *info;
+ struct mp2975_data *data;
+ int ret;
+
+ data = devm_kzalloc(&client->dev, sizeof(struct mp2975_data),
+ GFP_KERNEL);
+ if (!data)
+ return -ENOMEM;
+
+ memcpy(&data->info, &mp2975_info, sizeof(*info));
+ info = &data->info;
+
+ /* Identify multiphase configuration for rail 2. */
+ ret = mp2975_identify_multiphase_rail2(client);
+ if (ret < 0)
+ return ret;
+
+ if (ret) {
+ /* Two rails are connected. */
+ data->info.pages = MP2975_PAGE_NUM;
+ data->info.phases[1] = ret;
+ data->info.func[1] = MP2975_RAIL2_FUNC;
+ }
+
+ /* Identify multiphase configuration. */
+ ret = mp2975_identify_multiphase(client, data, info);
+ if (ret)
+ return ret;
+
+ /* Identify VID setting per rail. */
+ ret = mp2975_identify_rails_vid(client, data, info);
+ if (ret < 0)
+ return ret;
+
+ /* Obtain current sense gain of power stage. */
+ ret = mp2975_current_sense_gain_get(client, data);
+ if (ret)
+ return ret;
+
+ /* Obtain voltage reference values. */
+ ret = mp2975_vref_get(client, data, info);
+ if (ret)
+ return ret;
+
+ /* Obtain vout over-voltage scales. */
+ ret = mp2975_vout_ov_scale_get(client, data, info);
+ if (ret < 0)
+ return ret;
+
+ /* Obtain offsets, maximum and format for vout. */
+ ret = mp2975_vout_per_rail_config_get(client, data, info);
+ if (ret)
+ return ret;
+
+ return pmbus_do_probe(client, info);
+}
+
+static const struct i2c_device_id mp2975_id[] = {
+ {"mp2975", 0},
+ {}
+};
+
+MODULE_DEVICE_TABLE(i2c, mp2975_id);
+
+static const struct of_device_id __maybe_unused mp2975_of_match[] = {
+ {.compatible = "mps,mp2975"},
+ {}
+};
+MODULE_DEVICE_TABLE(of, mp2975_of_match);
+
+static struct i2c_driver mp2975_driver = {
+ .driver = {
+ .name = "mp2975",
+ .of_match_table = of_match_ptr(mp2975_of_match),
+ },
+ .probe_new = mp2975_probe,
+ .remove = pmbus_do_remove,
+ .id_table = mp2975_id,
+};
+
+module_i2c_driver(mp2975_driver);
+
+MODULE_AUTHOR("Vadim Pasternak <vadimp@nvidia.com>");
+MODULE_DESCRIPTION("PMBus driver for MPS MP2975 device");
+MODULE_LICENSE("GPL");
u32 flags;
};
+static const struct i2c_device_id pmbus_id[];
+
/*
* Find sensor groups and status registers on each page.
*/
return ret;
}
-static int pmbus_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int pmbus_probe(struct i2c_client *client)
{
struct pmbus_driver_info *info;
struct pmbus_platform_data *pdata = NULL;
if (!info)
return -ENOMEM;
- device_info = (struct pmbus_device_info *)id->driver_data;
+ device_info = (struct pmbus_device_info *)i2c_match_id(pmbus_id, client)->driver_data;
if (device_info->flags & PMBUS_SKIP_STATUS_CHECK) {
pdata = devm_kzalloc(dev, sizeof(struct pmbus_platform_data),
GFP_KERNEL);
info->identify = pmbus_identify;
dev->platform_data = pdata;
- return pmbus_do_probe(client, id, info);
+ return pmbus_do_probe(client, info);
}
static const struct pmbus_device_info pmbus_info_one = {
.driver = {
.name = "pmbus",
},
- .probe = pmbus_probe,
+ .probe_new = pmbus_probe,
.remove = pmbus_do_remove,
.id_table = pmbus_id,
};
PMBUS_MFR_DATE = 0x9D,
PMBUS_MFR_SERIAL = 0x9E,
+ PMBUS_MFR_VIN_MIN = 0xA0,
+ PMBUS_MFR_VIN_MAX = 0xA1,
+ PMBUS_MFR_IIN_MAX = 0xA2,
+ PMBUS_MFR_PIN_MAX = 0xA3,
+ PMBUS_MFR_VOUT_MIN = 0xA4,
+ PMBUS_MFR_VOUT_MAX = 0xA5,
+ PMBUS_MFR_IOUT_MAX = 0xA6,
+ PMBUS_MFR_POUT_MAX = 0xA7,
+
PMBUS_IC_DEVICE_ID = 0xAD,
PMBUS_IC_DEVICE_REV = 0xAE,
+ PMBUS_MFR_MAX_TEMP_1 = 0xC0,
+ PMBUS_MFR_MAX_TEMP_2 = 0xC1,
+ PMBUS_MFR_MAX_TEMP_3 = 0xC2,
+
/*
* Virtual registers.
* Useful to support attributes which are not supported by standard PMBus
void pmbus_clear_faults(struct i2c_client *client);
bool pmbus_check_byte_register(struct i2c_client *client, int page, int reg);
bool pmbus_check_word_register(struct i2c_client *client, int page, int reg);
-int pmbus_do_probe(struct i2c_client *client, const struct i2c_device_id *id,
- struct pmbus_driver_info *info);
+int pmbus_do_probe(struct i2c_client *client, struct pmbus_driver_info *info);
int pmbus_do_remove(struct i2c_client *client);
const struct pmbus_driver_info *pmbus_get_driver_info(struct i2c_client
*client);
#include <linux/i2c.h>
#include <linux/hwmon.h>
#include <linux/hwmon-sysfs.h>
-#include <linux/jiffies.h>
#include <linux/pmbus.h>
#include <linux/regulator/driver.h>
#include <linux/regulator/machine.h>
* with each call to krealloc
*/
#define PMBUS_ATTR_ALLOC_SIZE 32
-
-/*
- * Index into status register array, per status register group
- */
-#define PB_STATUS_BASE 0
-#define PB_STATUS_VOUT_BASE (PB_STATUS_BASE + PMBUS_PAGES)
-#define PB_STATUS_IOUT_BASE (PB_STATUS_VOUT_BASE + PMBUS_PAGES)
-#define PB_STATUS_FAN_BASE (PB_STATUS_IOUT_BASE + PMBUS_PAGES)
-#define PB_STATUS_FAN34_BASE (PB_STATUS_FAN_BASE + PMBUS_PAGES)
-#define PB_STATUS_TEMP_BASE (PB_STATUS_FAN34_BASE + PMBUS_PAGES)
-#define PB_STATUS_INPUT_BASE (PB_STATUS_TEMP_BASE + PMBUS_PAGES)
-#define PB_STATUS_VMON_BASE (PB_STATUS_INPUT_BASE + 1)
-
-#define PB_NUM_STATUS_REG (PB_STATUS_VMON_BASE + 1)
-
#define PMBUS_NAME_SIZE 24
struct pmbus_sensor {
#define to_pmbus_label(_attr) \
container_of(_attr, struct pmbus_label, attribute)
+/* Macros for converting between sensor index and register/page/status mask */
+
+#define PB_STATUS_MASK 0xffff
+#define PB_REG_SHIFT 16
+#define PB_REG_MASK 0x3ff
+#define PB_PAGE_SHIFT 26
+#define PB_PAGE_MASK 0x3f
+
+#define pb_reg_to_index(page, reg, mask) (((page) << PB_PAGE_SHIFT) | \
+ ((reg) << PB_REG_SHIFT) | (mask))
+
+#define pb_index_to_page(index) (((index) >> PB_PAGE_SHIFT) & PB_PAGE_MASK)
+#define pb_index_to_reg(index) (((index) >> PB_REG_SHIFT) & PB_REG_MASK)
+#define pb_index_to_mask(index) ((index) & PB_STATUS_MASK)
+
struct pmbus_data {
struct device *dev;
struct device *hwmon_dev;
struct pmbus_sensor *sensors;
struct mutex update_lock;
- bool valid;
- unsigned long last_updated; /* in jiffies */
-
- /*
- * A single status register covers multiple attributes,
- * so we keep them all together.
- */
- u16 status[PB_NUM_STATUS_REG];
bool has_status_word; /* device uses STATUS_WORD register */
int (*read_status)(struct i2c_client *client, int page);
void pmbus_clear_cache(struct i2c_client *client)
{
struct pmbus_data *data = i2c_get_clientdata(client);
+ struct pmbus_sensor *sensor;
- data->valid = false;
+ for (sensor = data->sensors; sensor; sensor = sensor->next)
+ sensor->data = -ENODATA;
}
EXPORT_SYMBOL_GPL(pmbus_clear_cache);
}
EXPORT_SYMBOL_GPL(pmbus_get_driver_info);
-static struct _pmbus_status {
- u32 func;
- u16 base;
- u16 reg;
-} pmbus_status[] = {
- { PMBUS_HAVE_STATUS_VOUT, PB_STATUS_VOUT_BASE, PMBUS_STATUS_VOUT },
- { PMBUS_HAVE_STATUS_IOUT, PB_STATUS_IOUT_BASE, PMBUS_STATUS_IOUT },
- { PMBUS_HAVE_STATUS_TEMP, PB_STATUS_TEMP_BASE,
- PMBUS_STATUS_TEMPERATURE },
- { PMBUS_HAVE_STATUS_FAN12, PB_STATUS_FAN_BASE, PMBUS_STATUS_FAN_12 },
- { PMBUS_HAVE_STATUS_FAN34, PB_STATUS_FAN34_BASE, PMBUS_STATUS_FAN_34 },
-};
-
-static struct pmbus_data *pmbus_update_device(struct device *dev)
+static int pmbus_get_status(struct i2c_client *client, int page, int reg)
{
- struct i2c_client *client = to_i2c_client(dev->parent);
struct pmbus_data *data = i2c_get_clientdata(client);
- const struct pmbus_driver_info *info = data->info;
- struct pmbus_sensor *sensor;
-
- mutex_lock(&data->update_lock);
- if (time_after(jiffies, data->last_updated + HZ) || !data->valid) {
- int i, j;
-
- for (i = 0; i < info->pages; i++) {
- data->status[PB_STATUS_BASE + i]
- = data->read_status(client, i);
- for (j = 0; j < ARRAY_SIZE(pmbus_status); j++) {
- struct _pmbus_status *s = &pmbus_status[j];
-
- if (!(info->func[i] & s->func))
- continue;
- data->status[s->base + i]
- = _pmbus_read_byte_data(client, i,
- s->reg);
- }
- }
+ int status;
- if (info->func[0] & PMBUS_HAVE_STATUS_INPUT)
- data->status[PB_STATUS_INPUT_BASE]
- = _pmbus_read_byte_data(client, 0,
- PMBUS_STATUS_INPUT);
-
- if (info->func[0] & PMBUS_HAVE_STATUS_VMON)
- data->status[PB_STATUS_VMON_BASE]
- = _pmbus_read_byte_data(client, 0,
- PMBUS_VIRT_STATUS_VMON);
-
- for (sensor = data->sensors; sensor; sensor = sensor->next) {
- if (!data->valid || sensor->update)
- sensor->data
- = _pmbus_read_word_data(client,
- sensor->page,
- sensor->phase,
- sensor->reg);
- }
- pmbus_clear_faults(client);
- data->last_updated = jiffies;
- data->valid = 1;
+ switch (reg) {
+ case PMBUS_STATUS_WORD:
+ status = data->read_status(client, page);
+ break;
+ default:
+ status = _pmbus_read_byte_data(client, page, reg);
+ break;
}
- mutex_unlock(&data->update_lock);
- return data;
+ if (status < 0)
+ pmbus_clear_faults(client);
+ return status;
+}
+
+static void pmbus_update_sensor_data(struct i2c_client *client, struct pmbus_sensor *sensor)
+{
+ if (sensor->data < 0 || sensor->update)
+ sensor->data = _pmbus_read_word_data(client, sensor->page,
+ sensor->phase, sensor->reg);
}
/*
* If a negative value is stored in any of the referenced registers, this value
* reflects an error code which will be returned.
*/
-static int pmbus_get_boolean(struct pmbus_data *data, struct pmbus_boolean *b,
+static int pmbus_get_boolean(struct i2c_client *client, struct pmbus_boolean *b,
int index)
{
+ struct pmbus_data *data = i2c_get_clientdata(client);
struct pmbus_sensor *s1 = b->s1;
struct pmbus_sensor *s2 = b->s2;
- u16 reg = (index >> 16) & 0xffff;
- u16 mask = index & 0xffff;
+ u16 mask = pb_index_to_mask(index);
+ u8 page = pb_index_to_page(index);
+ u16 reg = pb_index_to_reg(index);
int ret, status;
u16 regval;
- status = data->status[reg];
- if (status < 0)
- return status;
+ mutex_lock(&data->update_lock);
+ status = pmbus_get_status(client, page, reg);
+ if (status < 0) {
+ ret = status;
+ goto unlock;
+ }
+
+ if (s1)
+ pmbus_update_sensor_data(client, s1);
+ if (s2)
+ pmbus_update_sensor_data(client, s2);
regval = status & mask;
- if (!s1 && !s2) {
- ret = !!regval;
- } else if (!s1 || !s2) {
- WARN(1, "Bad boolean descriptor %p: s1=%p, s2=%p\n", b, s1, s2);
- return 0;
- } else {
+ if (s1 && s2) {
s64 v1, v2;
- if (s1->data < 0)
- return s1->data;
- if (s2->data < 0)
- return s2->data;
+ if (s1->data < 0) {
+ ret = s1->data;
+ goto unlock;
+ }
+ if (s2->data < 0) {
+ ret = s2->data;
+ goto unlock;
+ }
v1 = pmbus_reg2data(data, s1);
v2 = pmbus_reg2data(data, s2);
ret = !!(regval && v1 >= v2);
+ } else {
+ ret = !!regval;
}
+unlock:
+ mutex_unlock(&data->update_lock);
return ret;
}
{
struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
struct pmbus_boolean *boolean = to_pmbus_boolean(attr);
- struct pmbus_data *data = pmbus_update_device(dev);
+ struct i2c_client *client = to_i2c_client(dev->parent);
int val;
- val = pmbus_get_boolean(data, boolean, attr->index);
+ val = pmbus_get_boolean(client, boolean, attr->index);
if (val < 0)
return val;
return snprintf(buf, PAGE_SIZE, "%d\n", val);
static ssize_t pmbus_show_sensor(struct device *dev,
struct device_attribute *devattr, char *buf)
{
- struct pmbus_data *data = pmbus_update_device(dev);
+ struct i2c_client *client = to_i2c_client(dev->parent);
struct pmbus_sensor *sensor = to_pmbus_sensor(devattr);
+ struct pmbus_data *data = i2c_get_clientdata(client);
+ pmbus_update_sensor_data(client, sensor);
if (sensor->data < 0)
return sensor->data;
const char *name, const char *type, int seq,
struct pmbus_sensor *s1,
struct pmbus_sensor *s2,
- u16 reg, u16 mask)
+ u8 page, u16 reg, u16 mask)
{
struct pmbus_boolean *boolean;
struct sensor_device_attribute *a;
+ if (WARN((s1 && !s2) || (!s1 && s2), "Bad s1/s2 parameters\n"))
+ return -EINVAL;
+
boolean = devm_kzalloc(data->dev, sizeof(*boolean), GFP_KERNEL);
if (!boolean)
return -ENOMEM;
boolean->s1 = s1;
boolean->s2 = s2;
pmbus_attr_init(a, boolean->name, 0444, pmbus_show_boolean, NULL,
- (reg << 16) | mask);
+ pb_reg_to_index(page, reg, mask));
return pmbus_add_attribute(data, &a->dev_attr.attr);
}
sensor->class = class;
sensor->update = update;
sensor->convert = convert;
+ sensor->data = -ENODATA;
pmbus_dev_attr_init(a, sensor->name,
readonly ? 0444 : 0644,
pmbus_show_sensor, pmbus_set_sensor);
bool compare; /* true if compare function needed */
u32 func; /* sensor mask */
u32 sfunc; /* sensor status mask */
- int sbase; /* status base register */
+ int sreg; /* status register */
const struct pmbus_limit_attr *limit;/* limit registers */
};
: NULL,
attr->compare ? l->low ? base : curr
: NULL,
- attr->sbase + page, l->sbit);
+ page, attr->sreg, l->sbit);
if (ret)
return ret;
have_alarm = 1;
pmbus_check_status_register(client, page)) {
ret = pmbus_add_boolean(data, name, "alarm", index,
NULL, NULL,
- PB_STATUS_BASE + page,
+ page, PMBUS_STATUS_WORD,
attr->gbit);
if (ret)
return ret;
}, {
.reg = PMBUS_VIRT_RESET_VIN_HISTORY,
.attr = "reset_history",
+ }, {
+ .reg = PMBUS_MFR_VIN_MIN,
+ .attr = "rated_min",
+ }, {
+ .reg = PMBUS_MFR_VIN_MAX,
+ .attr = "rated_max",
},
};
}, {
.reg = PMBUS_VIRT_RESET_VOUT_HISTORY,
.attr = "reset_history",
- }
+ }, {
+ .reg = PMBUS_MFR_VOUT_MIN,
+ .attr = "rated_min",
+ }, {
+ .reg = PMBUS_MFR_VOUT_MAX,
+ .attr = "rated_max",
+ },
};
static const struct pmbus_sensor_attr voltage_attributes[] = {
.label = "vin",
.func = PMBUS_HAVE_VIN,
.sfunc = PMBUS_HAVE_STATUS_INPUT,
- .sbase = PB_STATUS_INPUT_BASE,
+ .sreg = PMBUS_STATUS_INPUT,
.gbit = PB_STATUS_VIN_UV,
.limit = vin_limit_attrs,
.nlimit = ARRAY_SIZE(vin_limit_attrs),
.label = "vmon",
.func = PMBUS_HAVE_VMON,
.sfunc = PMBUS_HAVE_STATUS_VMON,
- .sbase = PB_STATUS_VMON_BASE,
+ .sreg = PMBUS_VIRT_STATUS_VMON,
.limit = vmon_limit_attrs,
.nlimit = ARRAY_SIZE(vmon_limit_attrs),
}, {
.paged = true,
.func = PMBUS_HAVE_VOUT,
.sfunc = PMBUS_HAVE_STATUS_VOUT,
- .sbase = PB_STATUS_VOUT_BASE,
+ .sreg = PMBUS_STATUS_VOUT,
.gbit = PB_STATUS_VOUT_OV,
.limit = vout_limit_attrs,
.nlimit = ARRAY_SIZE(vout_limit_attrs),
}, {
.reg = PMBUS_VIRT_RESET_IIN_HISTORY,
.attr = "reset_history",
- }
+ }, {
+ .reg = PMBUS_MFR_IIN_MAX,
+ .attr = "rated_max",
+ },
};
static const struct pmbus_limit_attr iout_limit_attrs[] = {
}, {
.reg = PMBUS_VIRT_RESET_IOUT_HISTORY,
.attr = "reset_history",
- }
+ }, {
+ .reg = PMBUS_MFR_IOUT_MAX,
+ .attr = "rated_max",
+ },
};
static const struct pmbus_sensor_attr current_attributes[] = {
.label = "iin",
.func = PMBUS_HAVE_IIN,
.sfunc = PMBUS_HAVE_STATUS_INPUT,
- .sbase = PB_STATUS_INPUT_BASE,
+ .sreg = PMBUS_STATUS_INPUT,
.gbit = PB_STATUS_INPUT,
.limit = iin_limit_attrs,
.nlimit = ARRAY_SIZE(iin_limit_attrs),
.paged = true,
.func = PMBUS_HAVE_IOUT,
.sfunc = PMBUS_HAVE_STATUS_IOUT,
- .sbase = PB_STATUS_IOUT_BASE,
+ .sreg = PMBUS_STATUS_IOUT,
.gbit = PB_STATUS_IOUT_OC,
.limit = iout_limit_attrs,
.nlimit = ARRAY_SIZE(iout_limit_attrs),
}, {
.reg = PMBUS_VIRT_RESET_PIN_HISTORY,
.attr = "reset_history",
- }
+ }, {
+ .reg = PMBUS_MFR_PIN_MAX,
+ .attr = "rated_max",
+ },
};
static const struct pmbus_limit_attr pout_limit_attrs[] = {
}, {
.reg = PMBUS_VIRT_RESET_POUT_HISTORY,
.attr = "reset_history",
- }
+ }, {
+ .reg = PMBUS_MFR_POUT_MAX,
+ .attr = "rated_max",
+ },
};
static const struct pmbus_sensor_attr power_attributes[] = {
.label = "pin",
.func = PMBUS_HAVE_PIN,
.sfunc = PMBUS_HAVE_STATUS_INPUT,
- .sbase = PB_STATUS_INPUT_BASE,
+ .sreg = PMBUS_STATUS_INPUT,
.gbit = PB_STATUS_INPUT,
.limit = pin_limit_attrs,
.nlimit = ARRAY_SIZE(pin_limit_attrs),
.paged = true,
.func = PMBUS_HAVE_POUT,
.sfunc = PMBUS_HAVE_STATUS_IOUT,
- .sbase = PB_STATUS_IOUT_BASE,
+ .sreg = PMBUS_STATUS_IOUT,
.limit = pout_limit_attrs,
.nlimit = ARRAY_SIZE(pout_limit_attrs),
}
}, {
.reg = PMBUS_VIRT_RESET_TEMP_HISTORY,
.attr = "reset_history",
- }
+ }, {
+ .reg = PMBUS_MFR_MAX_TEMP_1,
+ .attr = "rated_max",
+ },
};
static const struct pmbus_limit_attr temp_limit_attrs2[] = {
}, {
.reg = PMBUS_VIRT_RESET_TEMP2_HISTORY,
.attr = "reset_history",
- }
+ }, {
+ .reg = PMBUS_MFR_MAX_TEMP_2,
+ .attr = "rated_max",
+ },
};
static const struct pmbus_limit_attr temp_limit_attrs3[] = {
.attr = "crit",
.alarm = "crit_alarm",
.sbit = PB_TEMP_OT_FAULT,
- }
+ }, {
+ .reg = PMBUS_MFR_MAX_TEMP_3,
+ .attr = "rated_max",
+ },
};
static const struct pmbus_sensor_attr temp_attributes[] = {
.compare = true,
.func = PMBUS_HAVE_TEMP,
.sfunc = PMBUS_HAVE_STATUS_TEMP,
- .sbase = PB_STATUS_TEMP_BASE,
+ .sreg = PMBUS_STATUS_TEMPERATURE,
.gbit = PB_STATUS_TEMPERATURE,
.limit = temp_limit_attrs,
.nlimit = ARRAY_SIZE(temp_limit_attrs),
.compare = true,
.func = PMBUS_HAVE_TEMP2,
.sfunc = PMBUS_HAVE_STATUS_TEMP,
- .sbase = PB_STATUS_TEMP_BASE,
+ .sreg = PMBUS_STATUS_TEMPERATURE,
.gbit = PB_STATUS_TEMPERATURE,
.limit = temp_limit_attrs2,
.nlimit = ARRAY_SIZE(temp_limit_attrs2),
.compare = true,
.func = PMBUS_HAVE_TEMP3,
.sfunc = PMBUS_HAVE_STATUS_TEMP,
- .sbase = PB_STATUS_TEMP_BASE,
+ .sreg = PMBUS_STATUS_TEMPERATURE,
.gbit = PB_STATUS_TEMPERATURE,
.limit = temp_limit_attrs3,
.nlimit = ARRAY_SIZE(temp_limit_attrs3),
if ((info->func[page] & pmbus_fan_status_flags[f]) &&
pmbus_check_byte_register(client,
page, pmbus_fan_status_registers[f])) {
- int base;
+ int reg;
if (f > 1) /* fan 3, 4 */
- base = PB_STATUS_FAN34_BASE + page;
+ reg = PMBUS_STATUS_FAN_34;
else
- base = PB_STATUS_FAN_BASE + page;
+ reg = PMBUS_STATUS_FAN_12;
ret = pmbus_add_boolean(data, "fan",
- "alarm", index, NULL, NULL, base,
+ "alarm", index, NULL, NULL, page, reg,
PB_FAN_FAN1_WARNING >> (f & 1));
if (ret)
return ret;
ret = pmbus_add_boolean(data, "fan",
- "fault", index, NULL, NULL, base,
+ "fault", index, NULL, NULL, page, reg,
PB_FAN_FAN1_FAULT >> (f & 1));
if (ret)
return ret;
DEFINE_DEBUGFS_ATTRIBUTE(pmbus_debugfs_ops_status, pmbus_debugfs_get_status,
NULL, "0x%04llx\n");
+static int pmbus_debugfs_get_pec(void *data, u64 *val)
+{
+ struct i2c_client *client = data;
+
+ *val = !!(client->flags & I2C_CLIENT_PEC);
+
+ return 0;
+}
+
+static int pmbus_debugfs_set_pec(void *data, u64 val)
+{
+ int rc;
+ struct i2c_client *client = data;
+
+ if (!val) {
+ client->flags &= ~I2C_CLIENT_PEC;
+ return 0;
+ }
+
+ if (val != 1)
+ return -EINVAL;
+
+ rc = i2c_smbus_read_byte_data(client, PMBUS_CAPABILITY);
+ if (rc < 0)
+ return rc;
+
+ if (!(rc & PB_CAPABILITY_ERROR_CHECK))
+ return -EOPNOTSUPP;
+
+ client->flags |= I2C_CLIENT_PEC;
+
+ return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(pmbus_debugfs_ops_pec, pmbus_debugfs_get_pec,
+ pmbus_debugfs_set_pec, "%llu\n");
+
static int pmbus_init_debugfs(struct i2c_client *client,
struct pmbus_data *data)
{
if (!entries)
return -ENOMEM;
+ debugfs_create_file("pec", 0664, data->debugfs, client,
+ &pmbus_debugfs_ops_pec);
+
for (i = 0; i < data->info->pages; ++i) {
/* Check accessibility of status register if it's not page 0 */
if (!i || pmbus_check_status_register(client, i)) {
}
#endif /* IS_ENABLED(CONFIG_DEBUG_FS) */
-int pmbus_do_probe(struct i2c_client *client, const struct i2c_device_id *id,
- struct pmbus_driver_info *info)
+int pmbus_do_probe(struct i2c_client *client, struct pmbus_driver_info *info)
{
struct device *dev = &client->dev;
const struct pmbus_platform_data *pdata = dev_get_platdata(dev);
.identify = pxe1610_identify,
};
-static int pxe1610_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int pxe1610_probe(struct i2c_client *client)
{
struct pmbus_driver_info *info;
u8 buf[I2C_SMBUS_BLOCK_MAX];
if (!info)
return -ENOMEM;
- return pmbus_do_probe(client, id, info);
+ return pmbus_do_probe(client, info);
}
static const struct i2c_device_id pxe1610_id[] = {
.driver = {
.name = "pxe1610",
},
- .probe = pxe1610_probe,
+ .probe_new = pxe1610_probe,
.remove = pmbus_do_remove,
.id_table = pxe1610_id,
};
| PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT,
};
-static int tps40422_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int tps40422_probe(struct i2c_client *client)
{
- return pmbus_do_probe(client, id, &tps40422_info);
+ return pmbus_do_probe(client, &tps40422_info);
}
static const struct i2c_device_id tps40422_id[] = {
.driver = {
.name = "tps40422",
},
- .probe = tps40422_probe,
+ .probe_new = tps40422_probe,
.remove = pmbus_do_remove,
.id_table = tps40422_id,
};
#define TPS53681_MFR_SPECIFIC_20 0xe4 /* Number of phases, per page */
+static const struct i2c_device_id tps53679_id[];
+
static int tps53679_identify_mode(struct i2c_client *client,
struct pmbus_driver_info *info)
{
.pfunc[5] = PMBUS_HAVE_IOUT,
};
-static int tps53679_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int tps53679_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct pmbus_driver_info *info;
if (dev->of_node)
chip_id = (enum chips)of_device_get_match_data(dev);
else
- chip_id = id->driver_data;
+ chip_id = i2c_match_id(tps53679_id, client)->driver_data;
info = devm_kmemdup(dev, &tps53679_info, sizeof(*info), GFP_KERNEL);
if (!info)
return -ENODEV;
}
- return pmbus_do_probe(client, id, info);
+ return pmbus_do_probe(client, info);
}
static const struct i2c_device_id tps53679_id[] = {
.name = "tps53679",
.of_match_table = of_match_ptr(tps53679_of_match),
},
- .probe = tps53679_probe,
+ .probe_new = tps53679_probe,
.remove = pmbus_do_remove,
.id_table = tps53679_id,
};
}
#endif /* CONFIG_DEBUG_FS */
-static int ucd9000_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ucd9000_probe(struct i2c_client *client)
{
u8 block_buffer[I2C_SMBUS_BLOCK_MAX + 1];
struct ucd9000_data *data;
if (client->dev.of_node)
chip = (enum chips)of_device_get_match_data(&client->dev);
else
- chip = id->driver_data;
+ chip = mid->driver_data;
- if (chip != ucd9000 && chip != mid->driver_data)
+ if (chip != ucd9000 && strcmp(client->name, mid->name) != 0)
dev_notice(&client->dev,
"Device mismatch: Configured %s, detected %s\n",
- id->name, mid->name);
+ client->name, mid->name);
data = devm_kzalloc(&client->dev, sizeof(struct ucd9000_data),
GFP_KERNEL);
ucd9000_probe_gpio(client, mid, data);
- ret = pmbus_do_probe(client, mid, info);
+ ret = pmbus_do_probe(client, info);
if (ret)
return ret;
.name = "ucd9000",
.of_match_table = of_match_ptr(ucd9000_of_match),
},
- .probe = ucd9000_probe,
+ .probe_new = ucd9000_probe,
.remove = pmbus_do_remove,
.id_table = ucd9000_id,
};
};
MODULE_DEVICE_TABLE(of, ucd9200_of_match);
-static int ucd9200_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int ucd9200_probe(struct i2c_client *client)
{
u8 block_buffer[I2C_SMBUS_BLOCK_MAX + 1];
struct pmbus_driver_info *info;
if (client->dev.of_node)
chip = (enum chips)of_device_get_match_data(&client->dev);
else
- chip = id->driver_data;
+ chip = mid->driver_data;
- if (chip != ucd9200 && chip != mid->driver_data)
+ if (chip != ucd9200 && strcmp(client->name, mid->name) != 0)
dev_notice(&client->dev,
"Device mismatch: Configured %s, detected %s\n",
- id->name, mid->name);
+ client->name, mid->name);
info = devm_kzalloc(&client->dev, sizeof(struct pmbus_driver_info),
GFP_KERNEL);
if (mid->driver_data == ucd9240)
info->func[0] |= PMBUS_HAVE_FAN12 | PMBUS_HAVE_STATUS_FAN12;
- return pmbus_do_probe(client, mid, info);
+ return pmbus_do_probe(client, info);
}
/* This is the driver that will be inserted */
.name = "ucd9200",
.of_match_table = of_match_ptr(ucd9200_of_match),
},
- .probe = ucd9200_probe,
+ .probe_new = ucd9200_probe,
.remove = pmbus_do_remove,
.id_table = ucd9200_id,
};
.read_word_data = xdpe122_read_word_data,
};
-static int xdpe122_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int xdpe122_probe(struct i2c_client *client)
{
struct pmbus_driver_info *info;
if (!info)
return -ENOMEM;
- return pmbus_do_probe(client, id, info);
+ return pmbus_do_probe(client, info);
}
static const struct i2c_device_id xdpe122_id[] = {
.name = "xdpe12284",
.of_match_table = of_match_ptr(xdpe122_of_match),
},
- .probe = xdpe122_probe,
+ .probe_new = xdpe122_probe,
.remove = pmbus_do_remove,
.id_table = xdpe122_id,
};
};
MODULE_DEVICE_TABLE(i2c, zl6100_id);
-static int zl6100_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int zl6100_probe(struct i2c_client *client)
{
int ret;
struct zl6100_data *data;
dev_err(&client->dev, "Unsupported device\n");
return -ENODEV;
}
- if (id->driver_data != mid->driver_data)
+ if (strcmp(client->name, mid->name) != 0)
dev_notice(&client->dev,
"Device mismatch: Configured %s, detected %s\n",
- id->name, mid->name);
+ client->name, mid->name);
data = devm_kzalloc(&client->dev, sizeof(struct zl6100_data),
GFP_KERNEL);
info->write_word_data = zl6100_write_word_data;
info->write_byte = zl6100_write_byte;
- return pmbus_do_probe(client, mid, info);
+ return pmbus_do_probe(client, info);
}
static struct i2c_driver zl6100_driver = {
.driver = {
.name = "zl6100",
},
- .probe = zl6100_probe,
+ .probe_new = zl6100_probe,
.remove = pmbus_do_remove,
.id_table = zl6100_id,
};
ATTRIBUTE_GROUPS(powr1220);
-static int powr1220_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int powr1220_probe(struct i2c_client *client)
{
struct powr1220_data *data;
struct device *hwmon_dev;
.driver = {
.name = "powr1220",
},
- .probe = powr1220_probe,
+ .probe_new = powr1220_probe,
.id_table = powr1220_ids,
};
mutex_init(&ctx->lock);
ctx->pwm = devm_of_pwm_get(dev, dev->of_node, NULL);
- if (IS_ERR(ctx->pwm)) {
- ret = PTR_ERR(ctx->pwm);
-
- if (ret != -EPROBE_DEFER)
- dev_err(dev, "Could not get PWM: %d\n", ret);
-
- return ret;
- }
+ if (IS_ERR(ctx->pwm))
+ return dev_err_probe(dev, PTR_ERR(ctx->pwm), "Could not get PWM\n");
platform_set_drvdata(pdev, ctx);
}
}
- if (nr_count[hwmon_temp])
- nr_count[hwmon_chip]++, nr_types++;
+ if (nr_count[hwmon_temp]) {
+ nr_count[hwmon_chip]++;
+ nr_types++;
+ }
scmi_hwmon_chan = devm_kcalloc(dev, nr_types, sizeof(*scmi_hwmon_chan),
GFP_KERNEL);
ATTRIBUTE_GROUPS(sht21);
-static int sht21_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int sht21_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
static struct i2c_driver sht21_driver = {
.driver.name = "sht21",
- .probe = sht21_probe,
+ .probe_new = sht21_probe,
.id_table = sht21_id,
};
ATTRIBUTE_GROUPS(sht3x);
ATTRIBUTE_GROUPS(sts3x);
-static int sht3x_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id sht3x_ids[];
+
+static int sht3x_probe(struct i2c_client *client)
{
int ret;
struct sht3x_data *data;
if (ret)
return ret;
- if (id->driver_data == sts3x)
+ if (i2c_match_id(sht3x_ids, client)->driver_data == sts3x)
attribute_groups = sts3x_groups;
else
attribute_groups = sht3x_groups;
static struct i2c_driver sht3x_i2c_driver = {
.driver.name = "sht3x",
- .probe = sht3x_probe,
+ .probe_new = sht3x_probe,
.id_table = sht3x_ids,
};
#include <linux/err.h>
#include <linux/delay.h>
#include <linux/platform_data/shtc1.h>
+#include <linux/of.h>
/* commands (high precision mode) */
static const unsigned char shtc1_cmd_measure_blocking_hpm[] = { 0x7C, 0xA2 };
}
}
-static int shtc1_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id shtc1_id[];
+
+static int shtc1_probe(struct i2c_client *client)
{
int ret;
u16 id_reg;
char id_reg_buf[2];
struct shtc1_data *data;
struct device *hwmon_dev;
- enum shtcx_chips chip = id->driver_data;
+ enum shtcx_chips chip = i2c_match_id(shtc1_id, client)->driver_data;
struct i2c_adapter *adap = client->adapter;
struct device *dev = &client->dev;
+ struct device_node *np = dev->of_node;
if (!i2c_check_functionality(adap, I2C_FUNC_I2C)) {
dev_err(dev, "plain i2c transactions not supported\n");
data->client = client;
data->chip = chip;
- if (client->dev.platform_data)
- data->setup = *(struct shtc1_platform_data *)dev->platform_data;
+ if (np) {
+ data->setup.blocking_io = of_property_read_bool(np, "sensirion,blocking-io");
+ data->setup.high_precision = !of_property_read_bool(np, "sensicon,low-precision");
+ } else {
+ if (client->dev.platform_data)
+ data->setup = *(struct shtc1_platform_data *)dev->platform_data;
+ }
+
shtc1_select_command(data);
mutex_init(&data->update_lock);
};
MODULE_DEVICE_TABLE(i2c, shtc1_id);
+static const struct of_device_id shtc1_of_match[] = {
+ { .compatible = "sensirion,shtc1" },
+ { .compatible = "sensirion,shtw1" },
+ { .compatible = "sensirion,shtc3" },
+ { }
+};
+MODULE_DEVICE_TABLE(of, shtc1_of_match);
+
static struct i2c_driver shtc1_i2c_driver = {
- .driver.name = "shtc1",
- .probe = shtc1_probe,
+ .driver = {
+ .name = "shtc1",
+ .of_match_table = shtc1_of_match,
+ },
+ .probe_new = shtc1_probe,
.id_table = shtc1_id,
};
ATTRIBUTE_GROUPS(smm665);
-static int smm665_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id smm665_id[];
+
+static int smm665_probe(struct i2c_client *client)
{
struct i2c_adapter *adapter = client->adapter;
struct smm665_data *data;
mutex_init(&data->update_lock);
data->client = client;
- data->type = id->driver_data;
+ data->type = i2c_match_id(smm665_id, client)->driver_data;
data->cmdreg = i2c_new_dummy_device(adapter, (client->addr & ~SMM665_REGMASK)
| SMM665_CMDREG_BASE);
if (IS_ERR(data->cmdreg))
.driver = {
.name = "smm665",
},
- .probe = smm665_probe,
+ .probe_new = smm665_probe,
.remove = smm665_remove,
.id_table = smm665_id,
};
return 0;
}
-static int smsc47m192_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int smsc47m192_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver = {
.name = "smsc47m192",
},
- .probe = smsc47m192_probe,
+ .probe_new = smsc47m192_probe,
.id_table = smsc47m192_id,
.detect = smsc47m192_detect,
.address_list = normal_i2c,
case hwmon_temp_input:
stat = readl_relaxed(hwmon->base + TEMP_STAT);
if (!(stat & TEMP_STAT_VALID))
- return -EIO;
+ return -EAGAIN;
value = stat & TEMP_STAT_TEMP;
/*
* From register documentation:
};
ATTRIBUTE_GROUPS(stts751);
-static int stts751_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int stts751_probe(struct i2c_client *client)
{
struct stts751_priv *priv;
int ret;
.name = DEVNAME,
.of_match_table = of_match_ptr(stts751_of_match),
},
- .probe = stts751_probe,
+ .probe_new = stts751_probe,
.id_table = stts751_id,
.detect = stts751_detect,
.alert = stts751_alert,
* device probe and removal
*/
-static int tc654_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int tc654_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct tc654_data *data;
.driver = {
.name = "tc654",
},
- .probe = tc654_probe,
+ .probe_new = tc654_probe,
.id_table = tc654_id,
};
ATTRIBUTE_GROUPS(tc74);
-static int tc74_probe(struct i2c_client *client,
- const struct i2c_device_id *dev_id)
+static int tc74_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct tc74_data *data;
.driver = {
.name = "tc74",
},
- .probe = tc74_probe,
+ .probe_new = tc74_probe,
.id_table = tc74_id,
};
i2c_smbus_write_byte_data(client, THMC50_REG_CONF, config);
}
-static int thmc50_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id thmc50_id[];
+
+static int thmc50_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct thmc50_data *data;
return -ENOMEM;
data->client = client;
- data->type = id->driver_data;
+ data->type = i2c_match_id(thmc50_id, client)->driver_data;
mutex_init(&data->update_lock);
thmc50_init_client(data);
.driver = {
.name = "thmc50",
},
- .probe = thmc50_probe,
+ .probe_new = thmc50_probe,
.id_table = thmc50_id,
.detect = thmc50_detect,
.address_list = normal_i2c,
.use_single_write = true,
};
-static int tmp102_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int tmp102_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.driver.name = DRIVER_NAME,
.driver.of_match_table = of_match_ptr(tmp102_of_match),
.driver.pm = &tmp102_dev_pm_ops,
- .probe = tmp102_probe,
+ .probe_new = tmp102_probe,
.id_table = tmp102_id,
};
.volatile_reg = tmp103_regmap_is_volatile,
};
-static int tmp103_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int tmp103_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.of_match_table = of_match_ptr(tmp103_of_match),
.pm = &tmp103_dev_pm_ops,
},
- .probe = tmp103_probe,
+ .probe_new = tmp103_probe,
.id_table = tmp103_id,
};
.use_single_write = true,
};
-static int tmp108_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int tmp108_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.pm = &tmp108_dev_pm_ops,
.of_match_table = of_match_ptr(tmp108_of_ids),
},
- .probe = tmp108_probe,
+ .probe_new = tmp108_probe,
.id_table = tmp108_i2c_ids,
};
return 0;
}
-static int tmp401_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int tmp401_probe(struct i2c_client *client)
{
static const char * const names[] = {
"TMP401", "TMP411", "TMP431", "TMP432", "TMP435", "TMP461"
data->client = client;
mutex_init(&data->update_lock);
- data->kind = id->driver_data;
+ data->kind = i2c_match_id(tmp401_id, client)->driver_data;
/* Initialize the TMP401 chip */
status = tmp401_init_client(data, client);
.driver = {
.name = "tmp401",
},
- .probe = tmp401_probe,
+ .probe_new = tmp401_probe,
.id_table = tmp401_id,
.detect = tmp401_detect,
.address_list = normal_i2c,
.read = tmp421_read,
};
-static int tmp421_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int tmp421_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
data->channels = (unsigned long)
of_device_get_match_data(&client->dev);
else
- data->channels = id->driver_data;
+ data->channels = i2c_match_id(tmp421_id, client)->driver_data;
data->client = client;
err = tmp421_init_client(client);
.name = "tmp421",
.of_match_table = of_match_ptr(tmp421_of_match),
},
- .probe = tmp421_probe,
+ .probe_new = tmp421_probe,
.id_table = tmp421_id,
.detect = tmp421_detect,
.address_list = normal_i2c,
/*
* The valus is read in voltage in the chip but reported as
* current to the user.
- * 2's compliment number shifted by one to four depending
+ * 2's complement number shifted by one to four depending
* on the pga gain setting. 1lsb = 10uV
*/
*val = sign_extend32(regval, 17 - tmp51x_get_pga_shift(data));
return 0;
}
-static int tmp51x_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int tmp51x_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct tmp51x_data *data;
if (client->dev.of_node)
data->id = (enum tmp51x_ids)device_get_match_data(&client->dev);
else
- data->id = id->driver_data;
+ data->id = i2c_match_id(tmp51x_id, client)->driver_data;
ret = tmp51x_configure(dev, data);
if (ret < 0) {
if (IS_ERR(hwmon_dev))
return PTR_ERR(hwmon_dev);
- dev_dbg(dev, "power monitor %s\n", id->name);
+ dev_dbg(dev, "power monitor %s\n", client->name);
return 0;
}
.name = "tmp51x",
.of_match_table = of_match_ptr(tmp51x_of_match),
},
- .probe = tmp51x_probe,
+ .probe_new = tmp51x_probe,
.id_table = tmp51x_id,
};
data,
&w83627ehf_chip_info,
w83627ehf_groups);
+ if (IS_ERR(hwmon_dev)) {
+ err = PTR_ERR(hwmon_dev);
+ goto exit_release;
+ }
- return PTR_ERR_OR_ZERO(hwmon_dev);
+ return 0;
exit_release:
release_region(res->start, IOREGION_LENGTH);
.val_bits = 8,
};
-static int w83773_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int w83773_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct device *hwmon_dev;
.name = "w83773g",
.of_match_table = of_match_ptr(w83773_of_match),
},
- .probe = w83773_probe,
+ .probe_new = w83773_probe,
.id_table = w83773_id,
};
sysfs_remove_group(&dev->kobj, &w83781d_group_other);
}
-static int
-w83781d_probe(struct i2c_client *client, const struct i2c_device_id *id)
+static const struct i2c_device_id w83781d_ids[];
+
+static int w83781d_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct w83781d_data *data;
mutex_init(&data->lock);
mutex_init(&data->update_lock);
- data->type = id->driver_data;
+ data->type = i2c_match_id(w83781d_ids, client)->driver_data;
data->client = client;
/* attach secondary i2c lm75-like clients */
.driver = {
.name = "w83781d",
},
- .probe = w83781d_probe,
+ .probe_new = w83781d_probe,
.remove = w83781d_remove,
.id_table = w83781d_ids,
.detect = w83781d_detect,
u8 vrm; /* hwmon-vid */
};
-static int w83791d_probe(struct i2c_client *client,
- const struct i2c_device_id *id);
+static int w83791d_probe(struct i2c_client *client);
static int w83791d_detect(struct i2c_client *client,
struct i2c_board_info *info);
static int w83791d_remove(struct i2c_client *client);
.driver = {
.name = "w83791d",
},
- .probe = w83791d_probe,
+ .probe_new = w83791d_probe,
.remove = w83791d_remove,
.id_table = w83791d_id,
.detect = w83791d_detect,
return 0;
}
-static int w83791d_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int w83791d_probe(struct i2c_client *client)
{
struct w83791d_data *data;
struct device *dev = &client->dev;
u8 sf2_levels[3][4]; /* Smart FanII: Fan1,2,3 duty cycle levels */
};
-static int w83792d_probe(struct i2c_client *client,
- const struct i2c_device_id *id);
+static int w83792d_probe(struct i2c_client *client);
static int w83792d_detect(struct i2c_client *client,
struct i2c_board_info *info);
static int w83792d_remove(struct i2c_client *client);
.driver = {
.name = "w83792d",
},
- .probe = w83792d_probe,
+ .probe_new = w83792d_probe,
.remove = w83792d_remove,
.id_table = w83792d_id,
.detect = w83792d_detect,
}
static int
-w83792d_probe(struct i2c_client *client, const struct i2c_device_id *id)
+w83792d_probe(struct i2c_client *client)
{
struct w83792d_data *data;
struct device *dev = &client->dev;
static u8 w83793_read_value(struct i2c_client *client, u16 reg);
static int w83793_write_value(struct i2c_client *client, u16 reg, u8 value);
-static int w83793_probe(struct i2c_client *client,
- const struct i2c_device_id *id);
+static int w83793_probe(struct i2c_client *client);
static int w83793_detect(struct i2c_client *client,
struct i2c_board_info *info);
static int w83793_remove(struct i2c_client *client);
.driver = {
.name = "w83793",
},
- .probe = w83793_probe,
+ .probe_new = w83793_probe,
.remove = w83793_remove,
.id_table = w83793_id,
.detect = w83793_detect,
return 0;
}
-static int w83793_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int w83793_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
static const int watchdog_minors[] = {
}
}
-static int w83795_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static const struct i2c_device_id w83795_id[];
+
+static int w83795_probe(struct i2c_client *client)
{
int i;
u8 tmp;
return -ENOMEM;
i2c_set_clientdata(client, data);
- data->chip_type = id->driver_data;
+ data->chip_type = i2c_match_id(w83795_id, client)->driver_data;
data->bank = i2c_smbus_read_byte_data(client, W83795_REG_BANKSEL);
mutex_init(&data->update_lock);
.driver = {
.name = "w83795",
},
- .probe = w83795_probe,
+ .probe_new = w83795_probe,
.remove = w83795_remove,
.id_table = w83795_id,
* Functions declaration
*/
-static int w83l785ts_probe(struct i2c_client *client,
- const struct i2c_device_id *id);
+static int w83l785ts_probe(struct i2c_client *client);
static int w83l785ts_detect(struct i2c_client *client,
struct i2c_board_info *info);
static int w83l785ts_remove(struct i2c_client *client);
.driver = {
.name = "w83l785ts",
},
- .probe = w83l785ts_probe,
+ .probe_new = w83l785ts_probe,
.remove = w83l785ts_remove,
.id_table = w83l785ts_id,
.detect = w83l785ts_detect,
return 0;
}
-static int w83l785ts_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+static int w83l785ts_probe(struct i2c_client *client)
{
struct w83l785ts_data *data;
struct device *dev = &client->dev;
}
static int
-w83l786ng_probe(struct i2c_client *client, const struct i2c_device_id *id)
+w83l786ng_probe(struct i2c_client *client)
{
struct device *dev = &client->dev;
struct w83l786ng_data *data;
.driver = {
.name = "w83l786ng",
},
- .probe = w83l786ng_probe,
+ .probe_new = w83l786ng_probe,
.id_table = w83l786ng_id,
.detect = w83l786ng_detect,
.address_list = normal_i2c,
char res1[4]; /* Reserved */
ushort rpbase; /* Relocation pointer */
char res2[2]; /* Reserved */
+ /* The following elements are only for CPM2 */
+ char res3[4]; /* Reserved */
+ uint sdmatmp; /* Internal */
};
#define I2COM_START 0x80
pci_set_drvdata(dev, priv);
+ dev_pm_set_driver_flags(&dev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
pm_runtime_set_autosuspend_delay(&dev->dev, 1000);
pm_runtime_use_autosuspend(&dev->dev);
pm_runtime_put_autosuspend(&dev->dev);
* Copyright (C) 2014 Beniamino Galvani <b.galvani@gmail.com>
*/
+#include <linux/bitfield.h>
#include <linux/clk.h>
#include <linux/completion.h>
#include <linux/i2c.h>
#define REG_CTRL_ACK_IGNORE BIT(1)
#define REG_CTRL_STATUS BIT(2)
#define REG_CTRL_ERROR BIT(3)
-#define REG_CTRL_CLKDIV_SHIFT 12
-#define REG_CTRL_CLKDIV_MASK GENMASK(21, 12)
-#define REG_CTRL_CLKDIVEXT_SHIFT 28
-#define REG_CTRL_CLKDIVEXT_MASK GENMASK(29, 28)
+#define REG_CTRL_CLKDIV GENMASK(21, 12)
+#define REG_CTRL_CLKDIVEXT GENMASK(29, 28)
+
+#define REG_SLV_ADDR GENMASK(7, 0)
+#define REG_SLV_SDA_FILTER GENMASK(10, 8)
+#define REG_SLV_SCL_FILTER GENMASK(13, 11)
+#define REG_SLV_SCL_LOW GENMASK(27, 16)
+#define REG_SLV_SCL_LOW_EN BIT(28)
#define I2C_TIMEOUT_MS 500
+#define FILTER_DELAY 15
enum {
TOKEN_END = 0,
unsigned long clk_rate = clk_get_rate(i2c->clk);
unsigned int div;
- div = DIV_ROUND_UP(clk_rate, freq * i2c->data->div_factor);
+ div = DIV_ROUND_UP(clk_rate, freq);
+ div -= FILTER_DELAY;
+ div = DIV_ROUND_UP(div, i2c->data->div_factor);
/* clock divider has 12 bits */
- if (div >= (1 << 12)) {
+ if (div > GENMASK(11, 0)) {
dev_err(i2c->dev, "requested bus frequency too low\n");
- div = (1 << 12) - 1;
+ div = GENMASK(11, 0);
}
- meson_i2c_set_mask(i2c, REG_CTRL, REG_CTRL_CLKDIV_MASK,
- (div & GENMASK(9, 0)) << REG_CTRL_CLKDIV_SHIFT);
+ meson_i2c_set_mask(i2c, REG_CTRL, REG_CTRL_CLKDIV,
+ FIELD_PREP(REG_CTRL_CLKDIV, div & GENMASK(9, 0)));
+
+ meson_i2c_set_mask(i2c, REG_CTRL, REG_CTRL_CLKDIVEXT,
+ FIELD_PREP(REG_CTRL_CLKDIVEXT, div >> 10));
- meson_i2c_set_mask(i2c, REG_CTRL, REG_CTRL_CLKDIVEXT_MASK,
- (div >> 10) << REG_CTRL_CLKDIVEXT_SHIFT);
+ /* Disable HIGH/LOW mode */
+ meson_i2c_set_mask(i2c, REG_SLAVE_ADDR, REG_SLV_SCL_LOW_EN, 0);
dev_dbg(i2c->dev, "%s: clk %lu, freq %u, div %u\n", __func__,
clk_rate, freq, div);
token = (msg->flags & I2C_M_RD) ? TOKEN_SLAVE_ADDR_READ :
TOKEN_SLAVE_ADDR_WRITE;
- writel(msg->addr << 1, i2c->regs + REG_SLAVE_ADDR);
+
+ meson_i2c_set_mask(i2c, REG_SLAVE_ADDR, REG_SLV_ADDR,
+ FIELD_PREP(REG_SLV_ADDR, msg->addr << 1));
+
meson_i2c_add_token(i2c, TOKEN_START);
meson_i2c_add_token(i2c, token);
}
struct meson_i2c *i2c = adap->algo_data;
int i, ret = 0;
- clk_enable(i2c->clk);
-
for (i = 0; i < num; i++) {
ret = meson_i2c_xfer_msg(i2c, msgs + i, i == num - 1, atomic);
if (ret)
break;
}
- clk_disable(i2c->clk);
-
return ret ?: i;
}
return ret;
}
- ret = clk_prepare(i2c->clk);
+ ret = clk_prepare_enable(i2c->clk);
if (ret < 0) {
dev_err(&pdev->dev, "can't prepare clock\n");
return ret;
ret = i2c_add_adapter(&i2c->adap);
if (ret < 0) {
- clk_unprepare(i2c->clk);
+ clk_disable_unprepare(i2c->clk);
return ret;
}
+ /* Disable filtering */
+ meson_i2c_set_mask(i2c, REG_SLAVE_ADDR,
+ REG_SLV_SDA_FILTER | REG_SLV_SCL_FILTER, 0);
+
meson_i2c_set_clk_div(i2c, timings.bus_freq_hz);
return 0;
struct meson_i2c *i2c = platform_get_drvdata(pdev);
i2c_del_adapter(&i2c->adap);
- clk_unprepare(i2c->clk);
+ clk_disable_unprepare(i2c->clk);
return 0;
}
if (bus->cmd_err == -EAGAIN)
ret = i2c_recover_bus(adap);
+ /*
+ * After any type of error, check if LAST bit is still set,
+ * due to a HW issue.
+ * It cannot be cleared without resetting the module.
+ */
+ if (bus->cmd_err &&
+ (NPCM_I2CRXF_CTL_LAST_PEC & ioread8(bus->reg + NPCM_I2CRXF_CTL)))
+ npcm_i2c_reset(bus);
+
#if IS_ENABLED(CONFIG_I2C_SLAVE)
/* reenable slave if it was enabled */
if (bus->slave)
fifostat = readl(i2c_dev->base + OWL_I2C_REG_FIFOSTAT);
if (fifostat & OWL_I2C_FIFOSTAT_RNB) {
i2c_dev->err = -ENXIO;
+ /* Clear NACK error bit by writing "1" */
+ owl_i2c_update_reg(i2c_dev->base + OWL_I2C_REG_FIFOSTAT,
+ OWL_I2C_FIFOSTAT_RNB, true);
goto stop;
}
stat = readl(i2c_dev->base + OWL_I2C_REG_STAT);
if (stat & OWL_I2C_STAT_BEB) {
i2c_dev->err = -EIO;
+ /* Clear BUS error bit by writing "1" */
+ owl_i2c_update_reg(i2c_dev->base + OWL_I2C_REG_STAT,
+ OWL_I2C_STAT_BEB, true);
goto stop;
}
depends on MAC
help
This is the IDE driver for the on-board IDE interface on some m68k
- Macintosh models. It supports both the `Quadra style' (used in
- Quadra/ Centris 630 and Performa 588 models) and `Powerbook style'
- (used in the Powerbook 150 and 190 models) IDE interface.
+ Macintosh models, namely Quadra/Centris 630, Performa 588 and
+ Powerbook 150. The IDE interface on the Powerbook 190 is not
+ supported by this driver and requires BLK_DEV_PLATFORM or
+ PATA_PLATFORM.
Say Y if you have such an Macintosh model and want to use IDE
devices (hard disks, CD-ROM drives, etc.) that are connected to the
return err >= 0 ? put_user_long(err, arg) : err;
set_val:
- if (bdev != bdev->bd_contains)
+ if (bdev_is_partition(bdev))
err = -EINVAL;
else {
if (!capable(CAP_SYS_ADMIN))
switch (cmd) {
case HDIO_OBSOLETE_IDENTITY:
case HDIO_GET_IDENTITY:
- if (bdev != bdev->bd_contains)
+ if (bdev_is_partition(bdev))
return -EINVAL;
return ide_get_identity_ioctl(drive, cmd, argp);
case HDIO_GET_NICE:
#include <linux/delay.h>
#include <linux/ide.h>
#include <linux/module.h>
+#include <linux/platform_device.h>
#include <asm/macintosh.h>
-#include <asm/macints.h>
-#include <asm/mac_baboon.h>
+
+#define DRV_NAME "mac_ide"
#define IDE_BASE 0x50F1A000 /* Base address of IDE controller */
* Probe for a Macintosh IDE interface
*/
-static int __init macide_init(void)
+static int mac_ide_probe(struct platform_device *pdev)
{
- unsigned long base;
- int irq;
+ struct resource *mem, *irq;
struct ide_hw hw, *hws[] = { &hw };
struct ide_port_info d = macide_port_info;
+ struct ide_host *host;
+ int rc;
if (!MACH_IS_MAC)
return -ENODEV;
- switch (macintosh_config->ide_type) {
- case MAC_IDE_QUADRA:
- base = IDE_BASE;
- irq = IRQ_NUBUS_F;
- break;
- case MAC_IDE_PB:
- base = IDE_BASE;
- irq = IRQ_NUBUS_C;
- break;
- case MAC_IDE_BABOON:
- base = BABOON_BASE;
- d.port_ops = NULL;
- irq = IRQ_BABOON_1;
- break;
- default:
+ mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!mem)
+ return -ENODEV;
+
+ irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+ if (!irq)
return -ENODEV;
+
+ if (!devm_request_mem_region(&pdev->dev, mem->start,
+ resource_size(mem), DRV_NAME)) {
+ dev_err(&pdev->dev, "resources busy\n");
+ return -EBUSY;
}
printk(KERN_INFO "ide: Macintosh %s IDE controller\n",
mac_ide_name[macintosh_config->ide_type - 1]);
- macide_setup_ports(&hw, base, irq);
+ macide_setup_ports(&hw, mem->start, irq->start);
- return ide_host_add(&d, hws, 1, NULL);
+ rc = ide_host_add(&d, hws, 1, &host);
+ if (rc)
+ return rc;
+
+ platform_set_drvdata(pdev, host);
+ return 0;
}
-module_init(macide_init);
+static int mac_ide_remove(struct platform_device *pdev)
+{
+ struct ide_host *host = platform_get_drvdata(pdev);
+
+ ide_host_remove(host);
+ return 0;
+}
+
+static struct platform_driver mac_ide_driver = {
+ .driver = {
+ .name = DRV_NAME,
+ },
+ .probe = mac_ide_probe,
+ .remove = mac_ide_remove,
+};
+
+module_platform_driver(mac_ide_driver);
+MODULE_ALIAS("platform:" DRV_NAME);
MODULE_LICENSE("GPL");
static struct ad7124_chip_info ad7124_chip_info_tbl[] = {
[ID_AD7124_4] = {
- .name = "ad7127-4",
+ .name = "ad7124-4",
.chip_id = CHIPID_AD7124_4,
.num_inputs = 8,
},
[ID_AD7124_8] = {
- .name = "ad7127-8",
+ .name = "ad7124-8",
.chip_id = CHIPID_AD7124_8,
.num_inputs = 16,
},
static struct platform_driver adc5_driver = {
.driver = {
- .name = "qcom-spmi-adc5.c",
+ .name = "qcom-spmi-adc5",
.of_match_table = adc5_match_table,
},
.probe = adc5_probe,
}
EXPORT_SYMBOL(rdma_read_gid_attr_ndev_rcu);
-static int get_lower_dev_vlan(struct net_device *lower_dev, void *data)
+static int get_lower_dev_vlan(struct net_device *lower_dev,
+ struct netdev_nested_priv *priv)
{
- u16 *vlan_id = data;
+ u16 *vlan_id = (u16 *)priv->data;
if (is_vlan_dev(lower_dev))
*vlan_id = vlan_dev_vlan_id(lower_dev);
int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr,
u16 *vlan_id, u8 *smac)
{
+ struct netdev_nested_priv priv = {
+ .data = (void *)vlan_id,
+ };
struct net_device *ndev;
rcu_read_lock();
* the lower vlan device for this gid entry.
*/
netdev_walk_all_lower_dev_rcu(attr->ndev,
- get_lower_dev_vlan, vlan_id);
+ get_lower_dev_vlan, &priv);
}
}
rcu_read_unlock();
bool found;
};
-static int get_lower_vlan_dev_tc(struct net_device *dev, void *data)
+static int get_lower_vlan_dev_tc(struct net_device *dev,
+ struct netdev_nested_priv *priv)
{
- struct iboe_prio_tc_map *map = data;
+ struct iboe_prio_tc_map *map = (struct iboe_prio_tc_map *)priv->data;
if (is_vlan_dev(dev))
map->output_tc = get_vlan_ndev_tc(dev, map->input_prio);
{
struct iboe_prio_tc_map prio_tc_map = {};
int prio = rt_tos2priority(tos);
+ struct netdev_nested_priv priv;
/* If VLAN device, get it directly from the VLAN netdev */
if (is_vlan_dev(ndev))
return get_vlan_ndev_tc(ndev, prio);
prio_tc_map.input_prio = prio;
+ priv.data = (void *)&prio_tc_map;
rcu_read_lock();
netdev_walk_all_lower_dev_rcu(ndev,
get_lower_vlan_dev_tc,
- &prio_tc_map);
+ &priv);
rcu_read_unlock();
/* If map is found from lower device, use it; Otherwise
* continue with the current netdevice to get priority to tc map.
remove_client_context(device, cid);
}
+ ib_cq_pool_destroy(device);
+
/* Pairs with refcount_set in enable_device */
ib_device_put(device);
wait_for_completion(&device->unreg_completion);
goto out;
}
+ ib_cq_pool_init(device);
+
down_read(&clients_rwsem);
xa_for_each_marked (&clients, index, client, CLIENT_REGISTERED) {
ret = add_client_context(device, client);
goto dev_cleanup;
}
- ib_cq_pool_init(device);
ret = enable_device_and_get(device);
dev_set_uevent_suppress(&device->dev, false);
/* Mark for userspace that device is ready */
goto out;
disable_device(ib_dev);
- ib_cq_pool_destroy(ib_dev);
/* Expedite removing unregistered pointers from the hash table */
free_netdevs(ib_dev);
struct net_device *upper;
};
-static int netdev_upper_walk(struct net_device *upper, void *data)
+static int netdev_upper_walk(struct net_device *upper,
+ struct netdev_nested_priv *priv)
{
struct upper_list *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
- struct list_head *upper_list = data;
+ struct list_head *upper_list = (struct list_head *)priv->data;
if (!entry)
return 0;
struct net_device *ndev))
{
struct net_device *ndev = cookie;
+ struct netdev_nested_priv priv;
struct upper_list *upper_iter;
struct upper_list *upper_temp;
LIST_HEAD(upper_list);
+ priv.data = &upper_list;
rcu_read_lock();
- netdev_walk_all_upper_dev_rcu(ndev, netdev_upper_walk, &upper_list);
+ netdev_walk_all_upper_dev_rcu(ndev, netdev_upper_walk, &priv);
rcu_read_unlock();
handle_netdev(ib_dev, port, ndev);
rvt_pr_err(rdi,
"%s timeout mr %p pd %p lkey %x refcount %ld\n",
t, mr, mr->pd, mr->lkey,
- atomic_long_read(&mr->refcount.count));
+ atomic_long_read(&mr->refcount.data->count));
rvt_get_mr(mr);
return -EBUSY;
}
struct net_device *result;
};
-static int ipoib_upper_walk(struct net_device *upper, void *_data)
+static int ipoib_upper_walk(struct net_device *upper,
+ struct netdev_nested_priv *priv)
{
- struct ipoib_walk_data *data = _data;
+ struct ipoib_walk_data *data = (struct ipoib_walk_data *)priv->data;
int ret = 0;
if (ipoib_is_dev_match_addr_rcu(data->addr, upper)) {
static struct net_device *ipoib_get_net_dev_match_addr(
const struct sockaddr *addr, struct net_device *dev)
{
+ struct netdev_nested_priv priv;
struct ipoib_walk_data data = {
.addr = addr,
};
+ priv.data = (void *)&data;
rcu_read_lock();
if (ipoib_is_dev_match_addr_rcu(addr, dev)) {
dev_hold(dev);
goto out;
}
- netdev_walk_all_upper_dev_rcu(dev, ipoib_upper_walk, &data);
+ netdev_walk_all_upper_dev_rcu(dev, ipoib_upper_walk, &priv);
out:
rcu_read_unlock();
return data.result;
case TP_VARIANT_ALPS:
case TP_VARIANT_ELAN:
case TP_VARIANT_NXP:
+ case TP_VARIANT_JYT_SYNAPTICS:
+ case TP_VARIANT_SYNAPTICS:
if (variant_id)
*variant_id = param[0];
if (firmware_id)
DMI_MATCH(DMI_BOARD_VENDOR, "MICRO-STAR INTERNATIONAL CO., LTD"),
},
},
+ {
+ /* Acer Aspire 5 A515 */
+ .matches = {
+ DMI_MATCH(DMI_BOARD_NAME, "Grumpy_PK"),
+ DMI_MATCH(DMI_BOARD_VENDOR, "PK"),
+ },
+ },
{ }
};
extern int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb);
extern void amd_iommu_domain_direct_map(struct iommu_domain *dom);
extern int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids);
-extern int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
+extern int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid,
u64 address);
-extern int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid);
-extern int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
+extern int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid);
+extern int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid,
unsigned long cr3);
-extern int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid);
+extern int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid);
extern struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev);
#ifdef CONFIG_IRQ_REMAP
#define PPR_INVALID 0x1
#define PPR_FAILURE 0xf
-extern int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
+extern int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid,
int status, int tag);
static inline bool is_rd890_iommu(struct pci_dev *pdev)
return 0;
}
-/*
- * Reads the device exclusion range from ACPI and initializes the IOMMU with
- * it
- */
-static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
-{
- if (!(m->flags & IVMD_FLAG_EXCL_RANGE))
- return;
-
- /*
- * Treat per-device exclusion ranges as r/w unity-mapped regions
- * since some buggy BIOSes might lead to the overwritten exclusion
- * range (exclusion_start and exclusion_length members). This
- * happens when there are multiple exclusion ranges (IVMD entries)
- * defined in ACPI table.
- */
- m->flags = (IVMD_FLAG_IW | IVMD_FLAG_IR | IVMD_FLAG_UNITY_MAP);
-}
-
/*
* Takes a pointer to an AMD IOMMU entry in the ACPI table and
* initializes the hardware and our data structures with it.
}
}
-/* called when we find an exclusion range definition in ACPI */
-static int __init init_exclusion_range(struct ivmd_header *m)
-{
- int i;
-
- switch (m->type) {
- case ACPI_IVMD_TYPE:
- set_device_exclusion_range(m->devid, m);
- break;
- case ACPI_IVMD_TYPE_ALL:
- for (i = 0; i <= amd_iommu_last_bdf; ++i)
- set_device_exclusion_range(i, m);
- break;
- case ACPI_IVMD_TYPE_RANGE:
- for (i = m->devid; i <= m->aux; ++i)
- set_device_exclusion_range(i, m);
- break;
- default:
- break;
- }
-
- return 0;
-}
-
/* called for unity map ACPI definition */
static int __init init_unity_map_range(struct ivmd_header *m)
{
if (e == NULL)
return -ENOMEM;
- if (m->flags & IVMD_FLAG_EXCL_RANGE)
- init_exclusion_range(m);
-
switch (m->type) {
default:
kfree(e);
e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
e->prot = m->flags >> 1;
+ /*
+ * Treat per-device exclusion ranges as r/w unity-mapped regions
+ * since some buggy BIOSes might lead to the overwritten exclusion
+ * range (exclusion_start and exclusion_length members). This
+ * happens when there are multiple exclusion ranges (IVMD entries)
+ * defined in ACPI table.
+ */
+ if (m->flags & IVMD_FLAG_EXCL_RANGE)
+ e->prot = (IVMD_FLAG_IW | IVMD_FLAG_IR) >> 1;
+
DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x"
" range_start: %016llx range_end: %016llx flags: %x\n", s,
PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start),
static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
{
struct device *dev = iommu->iommu.dev;
- int type, devid, pasid, flags, tag;
+ int type, devid, flags, tag;
volatile u32 *event = __evt;
int count = 0;
u64 address;
+ u32 pasid;
retry:
type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
}
}
}
-#endif /* CONFIG_IRQ_REMAP */
+
+static void
+amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu)
+{
+ if (!irq_remapping_enabled || !dev_is_pci(dev) ||
+ pci_dev_has_special_msi_domain(to_pci_dev(dev)))
+ return;
+
+ dev_set_msi_domain(dev, iommu->msi_domain);
+}
+
+#else /* CONFIG_IRQ_REMAP */
+static inline void
+amd_iommu_set_pci_msi_domain(struct device *dev, struct amd_iommu *iommu) { }
+#endif /* !CONFIG_IRQ_REMAP */
#define AMD_IOMMU_INT_MASK \
(MMIO_STATUS_EVT_INT_MASK | \
cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
}
-static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
+static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, u32 pasid,
u64 address, bool size)
{
memset(cmd, 0, sizeof(*cmd));
CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
}
-static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
+static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, u32 pasid,
int qdep, u64 address, bool size)
{
memset(cmd, 0, sizeof(*cmd));
CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
}
-static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
+static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid,
int status, int tag, bool gn)
{
memset(cmd, 0, sizeof(*cmd));
iommu_dev = ERR_PTR(ret);
iommu_ignore_device(dev);
} else {
+ amd_iommu_set_pci_msi_domain(dev, iommu);
iommu_dev = &iommu->iommu;
}
}
EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
-static int __flush_pasid(struct protection_domain *domain, int pasid,
+static int __flush_pasid(struct protection_domain *domain, u32 pasid,
u64 address, bool size)
{
struct iommu_dev_data *dev_data;
return ret;
}
-static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
+static int __amd_iommu_flush_page(struct protection_domain *domain, u32 pasid,
u64 address)
{
return __flush_pasid(domain, pasid, address, false);
}
-int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
+int amd_iommu_flush_page(struct iommu_domain *dom, u32 pasid,
u64 address)
{
struct protection_domain *domain = to_pdomain(dom);
}
EXPORT_SYMBOL(amd_iommu_flush_page);
-static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
+static int __amd_iommu_flush_tlb(struct protection_domain *domain, u32 pasid)
{
return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
true);
}
-int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
+int amd_iommu_flush_tlb(struct iommu_domain *dom, u32 pasid)
{
struct protection_domain *domain = to_pdomain(dom);
unsigned long flags;
}
EXPORT_SYMBOL(amd_iommu_flush_tlb);
-static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
+static u64 *__get_gcr3_pte(u64 *root, int level, u32 pasid, bool alloc)
{
int index;
u64 *pte;
return pte;
}
-static int __set_gcr3(struct protection_domain *domain, int pasid,
+static int __set_gcr3(struct protection_domain *domain, u32 pasid,
unsigned long cr3)
{
struct domain_pgtable pgtable;
return __amd_iommu_flush_tlb(domain, pasid);
}
-static int __clear_gcr3(struct protection_domain *domain, int pasid)
+static int __clear_gcr3(struct protection_domain *domain, u32 pasid)
{
struct domain_pgtable pgtable;
u64 *pte;
return __amd_iommu_flush_tlb(domain, pasid);
}
-int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
+int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, u32 pasid,
unsigned long cr3)
{
struct protection_domain *domain = to_pdomain(dom);
}
EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);
-int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
+int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, u32 pasid)
{
struct protection_domain *domain = to_pdomain(dom);
unsigned long flags;
}
EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
-int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
+int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid,
int status, int tag)
{
struct iommu_dev_data *dev_data;
static int get_devid(struct irq_alloc_info *info)
{
- int devid = -1;
-
switch (info->type) {
case X86_IRQ_ALLOC_TYPE_IOAPIC:
- devid = get_ioapic_devid(info->ioapic_id);
- break;
+ case X86_IRQ_ALLOC_TYPE_IOAPIC_GET_PARENT:
+ return get_ioapic_devid(info->devid);
case X86_IRQ_ALLOC_TYPE_HPET:
- devid = get_hpet_devid(info->hpet_id);
- break;
- case X86_IRQ_ALLOC_TYPE_MSI:
- case X86_IRQ_ALLOC_TYPE_MSIX:
- devid = get_device_id(&info->msi_dev->dev);
- break;
+ case X86_IRQ_ALLOC_TYPE_HPET_GET_PARENT:
+ return get_hpet_devid(info->devid);
+ case X86_IRQ_ALLOC_TYPE_PCI_MSI:
+ case X86_IRQ_ALLOC_TYPE_PCI_MSIX:
+ return get_device_id(msi_desc_to_dev(info->desc));
default:
- BUG_ON(1);
- break;
+ WARN_ON_ONCE(1);
+ return -1;
}
-
- return devid;
}
-static struct irq_domain *get_ir_irq_domain(struct irq_alloc_info *info)
+static struct irq_domain *get_irq_domain_for_devid(struct irq_alloc_info *info,
+ int devid)
{
- struct amd_iommu *iommu;
- int devid;
+ struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
- if (!info)
+ if (!iommu)
return NULL;
- devid = get_devid(info);
- if (devid >= 0) {
- iommu = amd_iommu_rlookup_table[devid];
- if (iommu)
- return iommu->ir_domain;
+ switch (info->type) {
+ case X86_IRQ_ALLOC_TYPE_IOAPIC_GET_PARENT:
+ case X86_IRQ_ALLOC_TYPE_HPET_GET_PARENT:
+ return iommu->ir_domain;
+ default:
+ WARN_ON_ONCE(1);
+ return NULL;
}
-
- return NULL;
}
static struct irq_domain *get_irq_domain(struct irq_alloc_info *info)
{
- struct amd_iommu *iommu;
int devid;
if (!info)
return NULL;
- switch (info->type) {
- case X86_IRQ_ALLOC_TYPE_MSI:
- case X86_IRQ_ALLOC_TYPE_MSIX:
- devid = get_device_id(&info->msi_dev->dev);
- if (devid < 0)
- return NULL;
-
- iommu = amd_iommu_rlookup_table[devid];
- if (iommu)
- return iommu->msi_domain;
- break;
- default:
- break;
- }
-
- return NULL;
+ devid = get_devid(info);
+ if (devid < 0)
+ return NULL;
+ return get_irq_domain_for_devid(info, devid);
}
struct irq_remap_ops amd_iommu_irq_ops = {
.disable = amd_iommu_disable,
.reenable = amd_iommu_reenable,
.enable_faulting = amd_iommu_enable_faulting,
- .get_ir_irq_domain = get_ir_irq_domain,
.get_irq_domain = get_irq_domain,
};
switch (info->type) {
case X86_IRQ_ALLOC_TYPE_IOAPIC:
/* Setup IOAPIC entry */
- entry = info->ioapic_entry;
- info->ioapic_entry = NULL;
+ entry = info->ioapic.entry;
+ info->ioapic.entry = NULL;
memset(entry, 0, sizeof(*entry));
entry->vector = index;
entry->mask = 0;
- entry->trigger = info->ioapic_trigger;
- entry->polarity = info->ioapic_polarity;
+ entry->trigger = info->ioapic.trigger;
+ entry->polarity = info->ioapic.polarity;
/* Mask level triggered irqs. */
- if (info->ioapic_trigger)
+ if (info->ioapic.trigger)
entry->mask = 1;
break;
case X86_IRQ_ALLOC_TYPE_HPET:
- case X86_IRQ_ALLOC_TYPE_MSI:
- case X86_IRQ_ALLOC_TYPE_MSIX:
+ case X86_IRQ_ALLOC_TYPE_PCI_MSI:
+ case X86_IRQ_ALLOC_TYPE_PCI_MSIX:
msg->address_hi = MSI_ADDR_BASE_HI;
msg->address_lo = MSI_ADDR_BASE_LO;
msg->data = irte_info->index;
if (!info)
return -EINVAL;
- if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_MSI &&
- info->type != X86_IRQ_ALLOC_TYPE_MSIX)
+ if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_PCI_MSI &&
+ info->type != X86_IRQ_ALLOC_TYPE_PCI_MSIX)
return -EINVAL;
/*
* With IRQ remapping enabled, don't need contiguous CPU vectors
* to support multiple MSI interrupts.
*/
- if (info->type == X86_IRQ_ALLOC_TYPE_MSI)
+ if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI)
info->flags &= ~X86_IRQ_ALLOC_CONTIGUOUS_VECTORS;
devid = get_devid(info);
iommu->irte_ops->set_allocated(table, i);
}
WARN_ON(table->min_index != 32);
- index = info->ioapic_pin;
+ index = info->ioapic.pin;
} else {
index = -ENOMEM;
}
- } else if (info->type == X86_IRQ_ALLOC_TYPE_MSI ||
- info->type == X86_IRQ_ALLOC_TYPE_MSIX) {
- bool align = (info->type == X86_IRQ_ALLOC_TYPE_MSI);
+ } else if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI ||
+ info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX) {
+ bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI);
- index = alloc_irq_index(devid, nr_irqs, align, info->msi_dev);
+ index = alloc_irq_index(devid, nr_irqs, align,
+ msi_desc_to_pci_dev(info->desc));
} else {
index = alloc_irq_index(devid, nr_irqs, false, NULL);
}
for (i = 0; i < nr_irqs; i++) {
irq_data = irq_domain_get_irq_data(domain, virq + i);
- cfg = irqd_cfg(irq_data);
- if (!irq_data || !cfg) {
+ cfg = irq_data ? irqd_cfg(irq_data) : NULL;
+ if (!cfg) {
ret = -EINVAL;
goto out_free_data;
}
struct mmu_notifier mn; /* mmu_notifier handle */
struct pri_queue pri[PRI_QUEUE_SIZE]; /* PRI tag states */
struct device_state *device_state; /* Link to our device_state */
- int pasid; /* PASID index */
+ u32 pasid; /* PASID index */
bool invalid; /* Used during setup and
teardown of the pasid */
spinlock_t lock; /* Protect pri_queues and
struct mm_struct *mm;
u64 address;
u16 devid;
- u16 pasid;
+ u32 pasid;
u16 tag;
u16 finish;
u16 flags;
/* Must be called under dev_state->lock */
static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,
- int pasid, bool alloc)
+ u32 pasid, bool alloc)
{
struct pasid_state **root, **ptr;
int level, index;
static int set_pasid_state(struct device_state *dev_state,
struct pasid_state *pasid_state,
- int pasid)
+ u32 pasid)
{
struct pasid_state **ptr;
unsigned long flags;
return ret;
}
-static void clear_pasid_state(struct device_state *dev_state, int pasid)
+static void clear_pasid_state(struct device_state *dev_state, u32 pasid)
{
struct pasid_state **ptr;
unsigned long flags;
}
static struct pasid_state *get_pasid_state(struct device_state *dev_state,
- int pasid)
+ u32 pasid)
{
struct pasid_state **ptr, *ret = NULL;
unsigned long flags;
.notifier_call = ppr_notifier,
};
-int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,
+int amd_iommu_bind_pasid(struct pci_dev *pdev, u32 pasid,
struct task_struct *task)
{
struct pasid_state *pasid_state;
return -EINVAL;
ret = -EINVAL;
- if (pasid < 0 || pasid >= dev_state->max_pasids)
+ if (pasid >= dev_state->max_pasids)
goto out;
ret = -ENOMEM;
}
EXPORT_SYMBOL(amd_iommu_bind_pasid);
-void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid)
+void amd_iommu_unbind_pasid(struct pci_dev *pdev, u32 pasid)
{
struct pasid_state *pasid_state;
struct device_state *dev_state;
if (dev_state == NULL)
return;
- if (pasid < 0 || pasid >= dev_state->max_pasids)
+ if (pasid >= dev_state->max_pasids)
goto out;
pasid_state = get_pasid_state(dev_state, pasid);
return -ENODEV;
data = platform_get_drvdata(sysmmu);
- if (!data)
+ if (!data) {
+ put_device(&sysmmu->dev);
return -ENODEV;
+ }
if (!owner) {
owner = kzalloc(sizeof(*owner), GFP_KERNEL);
- if (!owner)
+ if (!owner) {
+ put_device(&sysmmu->dev);
return -ENOMEM;
+ }
INIT_LIST_HEAD(&owner->controllers);
mutex_init(&owner->rpm_lock);
* in the chip_data and hyperv_irq_remapping_activate()/hyperv_ir_set_
* affinity() set vector and dest_apicid directly into IO-APIC entry.
*/
- irq_data->chip_data = info->ioapic_entry;
+ irq_data->chip_data = info->ioapic.entry;
/*
* Hypver-V IO APIC irq affinity should be in the scope of
return IRQ_REMAP_X2APIC_MODE;
}
-static struct irq_domain *hyperv_get_ir_irq_domain(struct irq_alloc_info *info)
+static struct irq_domain *hyperv_get_irq_domain(struct irq_alloc_info *info)
{
- if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC)
+ if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC_GET_PARENT)
return ioapic_ir_domain;
else
return NULL;
struct irq_remap_ops hyperv_irq_remap_ops = {
.prepare = hyperv_prepare_irq_remapping,
.enable = hyperv_enable_irq_remapping,
- .get_ir_irq_domain = hyperv_get_ir_irq_domain,
+ .get_irq_domain = hyperv_get_irq_domain,
};
#endif
if (ret < 0 && dmar_dev_scope_status == 0)
dmar_dev_scope_status = ret;
+ if (ret >= 0)
+ intel_irq_remap_add_device(info);
+
return ret;
}
}
void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,
- u64 granu, int pasid)
+ u64 granu, u32 pasid)
{
struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
}
static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
- u8 fault_reason, int pasid, u16 source_id,
+ u8 fault_reason, u32 pasid, u16 source_id,
unsigned long long addr)
{
const char *reason;
u8 fault_reason;
u16 source_id;
u64 guest_addr;
- int type, pasid;
+ u32 pasid;
+ int type;
u32 data;
bool pasid_present;
static int domain_setup_first_level(struct intel_iommu *iommu,
struct dmar_domain *domain,
struct device *dev,
- int pasid)
+ u32 pasid)
{
int flags = PASID_FLAG_SUPERVISOR_MODE;
struct dma_pte *pgd = domain->pgd;
}
/* Setup the PASID entry for requests without PASID: */
- spin_lock(&iommu->lock);
+ spin_lock_irqsave(&iommu->lock, flags);
if (hw_pass_through && domain_type_is_si(domain))
ret = intel_pasid_setup_pass_through(iommu, domain,
dev, PASID_RID2PASID);
else
ret = intel_pasid_setup_second_level(iommu, domain,
dev, PASID_RID2PASID);
- spin_unlock(&iommu->lock);
+ spin_unlock_irqrestore(&iommu->lock, flags);
if (ret) {
dev_err(dev, "Setup RID2PASID failed\n");
dmar_remove_one_dev_info(dev);
return -ENODEV;
if (domain->default_pasid <= 0) {
- int pasid;
+ u32 pasid;
/* No private data needed for the default pasid */
pasid = ioasid_alloc(NULL, PASID_MIN,
return rc;
}
-static struct intel_iommu *map_hpet_to_ir(u8 hpet_id)
+static struct irq_domain *map_hpet_to_ir(u8 hpet_id)
{
int i;
- for (i = 0; i < MAX_HPET_TBS; i++)
+ for (i = 0; i < MAX_HPET_TBS; i++) {
if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu)
- return ir_hpet[i].iommu;
+ return ir_hpet[i].iommu->ir_domain;
+ }
return NULL;
}
-static struct intel_iommu *map_ioapic_to_ir(int apic)
+static struct intel_iommu *map_ioapic_to_iommu(int apic)
{
int i;
- for (i = 0; i < MAX_IO_APICS; i++)
+ for (i = 0; i < MAX_IO_APICS; i++) {
if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu)
return ir_ioapic[i].iommu;
+ }
return NULL;
}
-static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev)
+static struct irq_domain *map_ioapic_to_ir(int apic)
{
- struct dmar_drhd_unit *drhd;
+ struct intel_iommu *iommu = map_ioapic_to_iommu(apic);
- drhd = dmar_find_matched_drhd_unit(dev);
- if (!drhd)
- return NULL;
+ return iommu ? iommu->ir_domain : NULL;
+}
+
+static struct irq_domain *map_dev_to_ir(struct pci_dev *dev)
+{
+ struct dmar_drhd_unit *drhd = dmar_find_matched_drhd_unit(dev);
- return drhd->iommu;
+ return drhd ? drhd->iommu->ir_msi_domain : NULL;
}
static int clear_entries(struct irq_2_iommu *irq_iommu)
for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) {
int ioapic_id = mpc_ioapic_id(ioapic_idx);
- if (!map_ioapic_to_ir(ioapic_id)) {
+ if (!map_ioapic_to_iommu(ioapic_id)) {
pr_err(FW_BUG "ioapic %d has no mapping iommu, "
"interrupt remapping will be disabled\n",
ioapic_id);
return -1;
}
+/*
+ * Store the MSI remapping domain pointer in the device if enabled.
+ *
+ * This is called from dmar_pci_bus_add_dev() so it works even when DMA
+ * remapping is disabled. Only update the pointer if the device is not
+ * already handled by a non default PCI/MSI interrupt domain. This protects
+ * e.g. VMD devices.
+ */
+void intel_irq_remap_add_device(struct dmar_pci_notify_info *info)
+{
+ if (!irq_remapping_enabled || pci_dev_has_special_msi_domain(info->dev))
+ return;
+
+ dev_set_msi_domain(&info->dev->dev, map_dev_to_ir(info->dev));
+}
+
static void prepare_irte(struct irte *irte, int vector, unsigned int dest)
{
memset(irte, 0, sizeof(*irte));
irte->redir_hint = 1;
}
-static struct irq_domain *intel_get_ir_irq_domain(struct irq_alloc_info *info)
-{
- struct intel_iommu *iommu = NULL;
-
- if (!info)
- return NULL;
-
- switch (info->type) {
- case X86_IRQ_ALLOC_TYPE_IOAPIC:
- iommu = map_ioapic_to_ir(info->ioapic_id);
- break;
- case X86_IRQ_ALLOC_TYPE_HPET:
- iommu = map_hpet_to_ir(info->hpet_id);
- break;
- case X86_IRQ_ALLOC_TYPE_MSI:
- case X86_IRQ_ALLOC_TYPE_MSIX:
- iommu = map_dev_to_ir(info->msi_dev);
- break;
- default:
- BUG_ON(1);
- break;
- }
-
- return iommu ? iommu->ir_domain : NULL;
-}
-
static struct irq_domain *intel_get_irq_domain(struct irq_alloc_info *info)
{
- struct intel_iommu *iommu;
-
if (!info)
return NULL;
switch (info->type) {
- case X86_IRQ_ALLOC_TYPE_MSI:
- case X86_IRQ_ALLOC_TYPE_MSIX:
- iommu = map_dev_to_ir(info->msi_dev);
- if (iommu)
- return iommu->ir_msi_domain;
- break;
+ case X86_IRQ_ALLOC_TYPE_IOAPIC_GET_PARENT:
+ return map_ioapic_to_ir(info->devid);
+ case X86_IRQ_ALLOC_TYPE_HPET_GET_PARENT:
+ return map_hpet_to_ir(info->devid);
default:
- break;
+ WARN_ON_ONCE(1);
+ return NULL;
}
-
- return NULL;
}
struct irq_remap_ops intel_irq_remap_ops = {
.disable = disable_irq_remapping,
.reenable = reenable_irq_remapping,
.enable_faulting = enable_drhd_fault_handling,
- .get_ir_irq_domain = intel_get_ir_irq_domain,
.get_irq_domain = intel_get_irq_domain,
};
switch (info->type) {
case X86_IRQ_ALLOC_TYPE_IOAPIC:
/* Set source-id of interrupt request */
- set_ioapic_sid(irte, info->ioapic_id);
+ set_ioapic_sid(irte, info->devid);
apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: Set IRTE entry (P:%d FPD:%d Dst_Mode:%d Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X Avail:%X Vector:%02X Dest:%08X SID:%04X SQ:%X SVT:%X)\n",
- info->ioapic_id, irte->present, irte->fpd,
+ info->devid, irte->present, irte->fpd,
irte->dst_mode, irte->redir_hint,
irte->trigger_mode, irte->dlvry_mode,
irte->avail, irte->vector, irte->dest_id,
irte->sid, irte->sq, irte->svt);
- entry = (struct IR_IO_APIC_route_entry *)info->ioapic_entry;
- info->ioapic_entry = NULL;
+ entry = (struct IR_IO_APIC_route_entry *)info->ioapic.entry;
+ info->ioapic.entry = NULL;
memset(entry, 0, sizeof(*entry));
entry->index2 = (index >> 15) & 0x1;
entry->zero = 0;
* IO-APIC RTE will be configured with virtual vector.
* irq handler will do the explicit EOI to the io-apic.
*/
- entry->vector = info->ioapic_pin;
+ entry->vector = info->ioapic.pin;
entry->mask = 0; /* enable IRQ */
- entry->trigger = info->ioapic_trigger;
- entry->polarity = info->ioapic_polarity;
- if (info->ioapic_trigger)
+ entry->trigger = info->ioapic.trigger;
+ entry->polarity = info->ioapic.polarity;
+ if (info->ioapic.trigger)
entry->mask = 1; /* Mask level triggered irqs. */
break;
case X86_IRQ_ALLOC_TYPE_HPET:
- case X86_IRQ_ALLOC_TYPE_MSI:
- case X86_IRQ_ALLOC_TYPE_MSIX:
+ case X86_IRQ_ALLOC_TYPE_PCI_MSI:
+ case X86_IRQ_ALLOC_TYPE_PCI_MSIX:
if (info->type == X86_IRQ_ALLOC_TYPE_HPET)
- set_hpet_sid(irte, info->hpet_id);
+ set_hpet_sid(irte, info->devid);
else
- set_msi_sid(irte, info->msi_dev);
+ set_msi_sid(irte, msi_desc_to_pci_dev(info->desc));
msg->address_hi = MSI_ADDR_BASE_HI;
msg->data = sub_handle;
if (!info || !iommu)
return -EINVAL;
- if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_MSI &&
- info->type != X86_IRQ_ALLOC_TYPE_MSIX)
+ if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_PCI_MSI &&
+ info->type != X86_IRQ_ALLOC_TYPE_PCI_MSIX)
return -EINVAL;
/*
* With IRQ remapping enabled, don't need contiguous CPU vectors
* to support multiple MSI interrupts.
*/
- if (info->type == X86_IRQ_ALLOC_TYPE_MSI)
+ if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI)
info->flags &= ~X86_IRQ_ALLOC_CONTIGUOUS_VECTORS;
ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
static DEFINE_SPINLOCK(pasid_lock);
u32 intel_pasid_max_id = PASID_MAX;
-int vcmd_alloc_pasid(struct intel_iommu *iommu, unsigned int *pasid)
+int vcmd_alloc_pasid(struct intel_iommu *iommu, u32 *pasid)
{
unsigned long flags;
u8 status_code;
return ret;
}
-void vcmd_free_pasid(struct intel_iommu *iommu, unsigned int pasid)
+void vcmd_free_pasid(struct intel_iommu *iommu, u32 pasid)
{
unsigned long flags;
u8 status_code;
struct pasid_table *pasid_table;
struct pasid_table_opaque data;
struct page *pages;
- int max_pasid = 0;
+ u32 max_pasid = 0;
int ret, order;
int size;
INIT_LIST_HEAD(&pasid_table->dev);
if (info->pasid_supported)
- max_pasid = min_t(int, pci_max_pasids(to_pci_dev(dev)),
+ max_pasid = min_t(u32, pci_max_pasids(to_pci_dev(dev)),
intel_pasid_max_id);
size = max_pasid >> (PASID_PDE_SHIFT - 3);
return info->pasid_table->max_pasid;
}
-struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid)
+struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid)
{
struct device_domain_info *info;
struct pasid_table *pasid_table;
int dir_index, index;
pasid_table = intel_pasid_get_table(dev);
- if (WARN_ON(!pasid_table || pasid < 0 ||
- pasid >= intel_pasid_get_dev_max_id(dev)))
+ if (WARN_ON(!pasid_table || pasid >= intel_pasid_get_dev_max_id(dev)))
return NULL;
dir = pasid_table->table;
}
static void
-intel_pasid_clear_entry(struct device *dev, int pasid, bool fault_ignore)
+intel_pasid_clear_entry(struct device *dev, u32 pasid, bool fault_ignore)
{
struct pasid_entry *pe;
static void
pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu,
- u16 did, int pasid)
+ u16 did, u32 pasid)
{
struct qi_desc desc;
static void
devtlb_invalidation_with_pasid(struct intel_iommu *iommu,
- struct device *dev, int pasid)
+ struct device *dev, u32 pasid)
{
struct device_domain_info *info;
u16 sid, qdep, pfsid;
}
void intel_pasid_tear_down_entry(struct intel_iommu *iommu, struct device *dev,
- int pasid, bool fault_ignore)
+ u32 pasid, bool fault_ignore)
{
struct pasid_entry *pte;
u16 did;
static void pasid_flush_caches(struct intel_iommu *iommu,
struct pasid_entry *pte,
- int pasid, u16 did)
+ u32 pasid, u16 did)
{
if (!ecap_coherent(iommu->ecap))
clflush_cache_range(pte, sizeof(*pte));
*/
int intel_pasid_setup_first_level(struct intel_iommu *iommu,
struct device *dev, pgd_t *pgd,
- int pasid, u16 did, int flags)
+ u32 pasid, u16 did, int flags)
{
struct pasid_entry *pte;
*/
int intel_pasid_setup_second_level(struct intel_iommu *iommu,
struct dmar_domain *domain,
- struct device *dev, int pasid)
+ struct device *dev, u32 pasid)
{
struct pasid_entry *pte;
struct dma_pte *pgd;
*/
int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
struct dmar_domain *domain,
- struct device *dev, int pasid)
+ struct device *dev, u32 pasid)
{
u16 did = FLPT_DEFAULT_DID;
struct pasid_entry *pte;
* @addr_width: Address width of the first level (guest)
*/
int intel_pasid_setup_nested(struct intel_iommu *iommu, struct device *dev,
- pgd_t *gpgd, int pasid,
+ pgd_t *gpgd, u32 pasid,
struct iommu_gpasid_bind_data_vtd *pasid_data,
struct dmar_domain *domain, int addr_width)
{
struct pasid_table {
void *table; /* pasid table pointer */
int order; /* page order of pasid table */
- int max_pasid; /* max pasid */
+ u32 max_pasid; /* max pasid */
struct list_head dev; /* device list */
};
return READ_ONCE(pte->val[0]) & PASID_PTE_PRESENT;
}
-extern u32 intel_pasid_max_id;
+extern unsigned int intel_pasid_max_id;
int intel_pasid_alloc_id(void *ptr, int start, int end, gfp_t gfp);
-void intel_pasid_free_id(int pasid);
-void *intel_pasid_lookup_id(int pasid);
+void intel_pasid_free_id(u32 pasid);
+void *intel_pasid_lookup_id(u32 pasid);
int intel_pasid_alloc_table(struct device *dev);
void intel_pasid_free_table(struct device *dev);
struct pasid_table *intel_pasid_get_table(struct device *dev);
int intel_pasid_get_dev_max_id(struct device *dev);
-struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid);
+struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid);
int intel_pasid_setup_first_level(struct intel_iommu *iommu,
struct device *dev, pgd_t *pgd,
- int pasid, u16 did, int flags);
+ u32 pasid, u16 did, int flags);
int intel_pasid_setup_second_level(struct intel_iommu *iommu,
struct dmar_domain *domain,
- struct device *dev, int pasid);
+ struct device *dev, u32 pasid);
int intel_pasid_setup_pass_through(struct intel_iommu *iommu,
struct dmar_domain *domain,
- struct device *dev, int pasid);
+ struct device *dev, u32 pasid);
int intel_pasid_setup_nested(struct intel_iommu *iommu,
- struct device *dev, pgd_t *pgd, int pasid,
+ struct device *dev, pgd_t *pgd, u32 pasid,
struct iommu_gpasid_bind_data_vtd *pasid_data,
struct dmar_domain *domain, int addr_width);
void intel_pasid_tear_down_entry(struct intel_iommu *iommu,
- struct device *dev, int pasid,
+ struct device *dev, u32 pasid,
bool fault_ignore);
-int vcmd_alloc_pasid(struct intel_iommu *iommu, unsigned int *pasid);
-void vcmd_free_pasid(struct intel_iommu *iommu, unsigned int pasid);
+int vcmd_alloc_pasid(struct intel_iommu *iommu, u32 *pasid);
+void vcmd_free_pasid(struct intel_iommu *iommu, u32 pasid);
#endif /* __INTEL_PASID_H */
#include <linux/mm_types.h>
#include <linux/ioasid.h>
#include <asm/page.h>
+#include <asm/fpu/api.h>
#include "pasid.h"
static irqreturn_t prq_event_thread(int irq, void *d);
-static void intel_svm_drain_prq(struct device *dev, int pasid);
+static void intel_svm_drain_prq(struct device *dev, u32 pasid);
#define PRQ_ORDER 0
return ret;
}
-int intel_svm_unbind_gpasid(struct device *dev, int pasid)
+int intel_svm_unbind_gpasid(struct device *dev, u32 pasid)
{
struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
struct intel_svm_dev *sdev;
return ret;
}
+static void _load_pasid(void *unused)
+{
+ update_pasid();
+}
+
+static void load_pasid(struct mm_struct *mm, u32 pasid)
+{
+ mutex_lock(&mm->context.lock);
+
+ /* Synchronize with READ_ONCE in update_pasid(). */
+ smp_store_release(&mm->pasid, pasid);
+
+ /* Update PASID MSR on all CPUs running the mm's tasks. */
+ on_each_cpu_mask(mm_cpumask(mm), _load_pasid, NULL, true);
+
+ mutex_unlock(&mm->context.lock);
+}
+
/* Caller must hold pasid_mutex, mm reference */
static int
-intel_svm_bind_mm(struct device *dev, int flags, struct svm_dev_ops *ops,
+intel_svm_bind_mm(struct device *dev, unsigned int flags,
+ struct svm_dev_ops *ops,
struct mm_struct *mm, struct intel_svm_dev **sd)
{
struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
}
list_add_tail(&svm->list, &global_svm_list);
+ if (mm) {
+ /* The newly allocated pasid is loaded to the mm. */
+ load_pasid(mm, svm->pasid);
+ }
} else {
/*
* Binding a new device with existing PASID, need to setup
}
/* Caller must hold pasid_mutex */
-static int intel_svm_unbind_mm(struct device *dev, int pasid)
+static int intel_svm_unbind_mm(struct device *dev, u32 pasid)
{
struct intel_svm_dev *sdev;
struct intel_iommu *iommu;
if (list_empty(&svm->devs)) {
ioasid_free(svm->pasid);
- if (svm->mm)
+ if (svm->mm) {
mmu_notifier_unregister(&svm->notifier, svm->mm);
+ /* Clear mm's pasid. */
+ load_pasid(svm->mm, PASID_DISABLED);
+ }
list_del(&svm->list);
/* We mandate that no page faults may be outstanding
* for the PASID when intel_svm_unbind_mm() is called.
* described in VT-d spec CH7.10 to drain all page requests and page
* responses pending in the hardware.
*/
-static void intel_svm_drain_prq(struct device *dev, int pasid)
+static void intel_svm_drain_prq(struct device *dev, u32 pasid)
{
struct device_domain_info *info;
struct dmar_domain *domain;
{
struct iommu_sva *sva = ERR_PTR(-EINVAL);
struct intel_svm_dev *sdev = NULL;
- int flags = 0;
+ unsigned int flags = 0;
int ret;
/*
* and intel_svm etc.
*/
if (drvdata)
- flags = *(int *)drvdata;
+ flags = *(unsigned int *)drvdata;
mutex_lock(&pasid_mutex);
ret = intel_svm_bind_mm(dev, flags, NULL, mm, &sdev);
if (ret)
mutex_unlock(&pasid_mutex);
}
-int intel_svm_get_pasid(struct iommu_sva *sva)
+u32 intel_svm_get_pasid(struct iommu_sva *sva)
{
struct intel_svm_dev *sdev;
- int pasid;
+ u32 pasid;
mutex_lock(&pasid_mutex);
sdev = to_intel_svm_dev(sva);
}
EXPORT_SYMBOL_GPL(iommu_sva_unbind_device);
-int iommu_sva_get_pasid(struct iommu_sva *handle)
+u32 iommu_sva_get_pasid(struct iommu_sva *handle)
{
const struct iommu_ops *ops = handle->dev->bus->iommu_ops;
panic(msg);
}
-/**
- * irq_remapping_get_ir_irq_domain - Get the irqdomain associated with the IOMMU
- * device serving request @info
- * @info: interrupt allocation information, used to identify the IOMMU device
- *
- * It's used to get parent irqdomain for HPET and IOAPIC irqdomains.
- * Returns pointer to IRQ domain, or NULL on failure.
- */
-struct irq_domain *
-irq_remapping_get_ir_irq_domain(struct irq_alloc_info *info)
-{
- if (!remap_ops || !remap_ops->get_ir_irq_domain)
- return NULL;
-
- return remap_ops->get_ir_irq_domain(info);
-}
-
/**
* irq_remapping_get_irq_domain - Get the irqdomain serving the request @info
* @info: interrupt allocation information, used to identify the IOMMU device
*
- * There will be one PCI MSI/MSIX irqdomain associated with each interrupt
- * remapping device, so this interface is used to retrieve the PCI MSI/MSIX
- * irqdomain serving request @info.
* Returns pointer to IRQ domain, or NULL on failure.
*/
-struct irq_domain *
-irq_remapping_get_irq_domain(struct irq_alloc_info *info)
+struct irq_domain *irq_remapping_get_irq_domain(struct irq_alloc_info *info)
{
if (!remap_ops || !remap_ops->get_irq_domain)
return NULL;
/* Enable fault handling */
int (*enable_faulting)(void);
- /* Get the irqdomain associated the IOMMU device */
- struct irq_domain *(*get_ir_irq_domain)(struct irq_alloc_info *);
-
- /* Get the MSI irqdomain associated with the IOMMU device */
+ /* Get the irqdomain associated to IOMMU device */
struct irq_domain *(*get_irq_domain)(struct irq_alloc_info *);
};
config DW_APB_ICTL
bool
select GENERIC_IRQ_CHIP
- select IRQ_DOMAIN
+ select IRQ_DOMAIN_HIERARCHY
config FARADAY_FTINTC010
bool
interrupt pins, as found on SH/R-Mobile and R-Car Gen1 SoCs.
config RENESAS_IRQC
- bool "Renesas R-Mobile APE6 and R-Car IRQC support" if COMPILE_TEST
+ bool "Renesas R-Mobile APE6, R-Car Gen{2,3} and RZ/G{1,2} IRQC support" if COMPILE_TEST
select GENERIC_IRQ_CHIP
select IRQ_DOMAIN
help
Enable support for the Renesas Interrupt Controller for external
- devices, as found on R-Mobile APE6, R-Car Gen2, and R-Car Gen3 SoCs.
+ devices, as found on R-Mobile APE6, R-Car Gen{2,3} and RZ/G{1,2} SoCs.
config RENESAS_RZA1_IRQC
bool "Renesas RZ/A1 IRQC support" if COMPILE_TEST
If you wish to use interrupt aggregator irq resources managed by the
TI System Controller, say Y here. Otherwise, say N.
+config TI_PRUSS_INTC
+ tristate "TI PRU-ICSS Interrupt Controller"
+ depends on ARCH_DAVINCI || SOC_AM33XX || SOC_AM43XX || SOC_DRA7XX || ARCH_KEYSTONE || ARCH_K3
+ select IRQ_DOMAIN
+ help
+ This enables support for the PRU-ICSS Local Interrupt Controller
+ present within a PRU-ICSS subsystem present on various TI SoCs.
+ The PRUSS INTC enables various interrupts to be routed to multiple
+ different processors within the SoC.
+
config RISCV_INTC
bool "RISC-V Local Interrupt Controller"
depends on RISCV
help
Support for the Loongson PCH MSI Controller.
+config MST_IRQ
+ bool "MStar Interrupt Controller"
+ default ARCH_MEDIATEK
+ select IRQ_DOMAIN
+ select IRQ_DOMAIN_HIERARCHY
+ help
+ Support MStar Interrupt Controller.
+
endmenu
obj-$(CONFIG_ATH79) += irq-ath79-misc.o
obj-$(CONFIG_ARCH_BCM2835) += irq-bcm2835.o
obj-$(CONFIG_ARCH_BCM2835) += irq-bcm2836.o
+obj-$(CONFIG_ARCH_ACTIONS) += irq-owl-sirq.o
obj-$(CONFIG_DAVINCI_AINTC) += irq-davinci-aintc.o
obj-$(CONFIG_DAVINCI_CP_INTC) += irq-davinci-cp-intc.o
obj-$(CONFIG_EXYNOS_IRQ_COMBINER) += exynos-combiner.o
obj-$(CONFIG_LS1X_IRQ) += irq-ls1x.o
obj-$(CONFIG_TI_SCI_INTR_IRQCHIP) += irq-ti-sci-intr.o
obj-$(CONFIG_TI_SCI_INTA_IRQCHIP) += irq-ti-sci-inta.o
+obj-$(CONFIG_TI_PRUSS_INTC) += irq-pruss-intc.o
obj-$(CONFIG_LOONGSON_LIOINTC) += irq-loongson-liointc.o
obj-$(CONFIG_LOONGSON_HTPIC) += irq-loongson-htpic.o
obj-$(CONFIG_LOONGSON_HTVEC) += irq-loongson-htvec.o
obj-$(CONFIG_LOONGSON_PCH_PIC) += irq-loongson-pch-pic.o
obj-$(CONFIG_LOONGSON_PCH_MSI) += irq-loongson-pch-msi.o
+obj-$(CONFIG_MST_IRQ) += irq-mst-intc.o
}
#endif
+static void armada_xp_mpic_perf_init(void)
+{
+ unsigned long cpuid = cpu_logical_map(smp_processor_id());
+
+ /* Enable Performance Counter Overflow interrupts */
+ writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid),
+ per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS);
+}
+
#ifdef CONFIG_SMP
+static struct irq_domain *ipi_domain;
+
+static void armada_370_xp_ipi_mask(struct irq_data *d)
+{
+ u32 reg;
+ reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
+ reg &= ~BIT(d->hwirq);
+ writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
+}
+
+static void armada_370_xp_ipi_unmask(struct irq_data *d)
+{
+ u32 reg;
+ reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
+ reg |= BIT(d->hwirq);
+ writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
+}
+
+static void armada_370_xp_ipi_send_mask(struct irq_data *d,
+ const struct cpumask *mask)
+{
+ unsigned long map = 0;
+ int cpu;
+
+ /* Convert our logical CPU mask into a physical one. */
+ for_each_cpu(cpu, mask)
+ map |= 1 << cpu_logical_map(cpu);
+
+ /*
+ * Ensure that stores to Normal memory are visible to the
+ * other CPUs before issuing the IPI.
+ */
+ dsb();
+
+ /* submit softirq */
+ writel((map << 8) | d->hwirq, main_int_base +
+ ARMADA_370_XP_SW_TRIG_INT_OFFS);
+}
+
+static void armada_370_xp_ipi_eoi(struct irq_data *d)
+{
+ writel(~BIT(d->hwirq), per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
+}
+
+static struct irq_chip ipi_irqchip = {
+ .name = "IPI",
+ .irq_mask = armada_370_xp_ipi_mask,
+ .irq_unmask = armada_370_xp_ipi_unmask,
+ .irq_eoi = armada_370_xp_ipi_eoi,
+ .ipi_send_mask = armada_370_xp_ipi_send_mask,
+};
+
+static int armada_370_xp_ipi_alloc(struct irq_domain *d,
+ unsigned int virq,
+ unsigned int nr_irqs, void *args)
+{
+ int i;
+
+ for (i = 0; i < nr_irqs; i++) {
+ irq_set_percpu_devid(virq + i);
+ irq_domain_set_info(d, virq + i, i, &ipi_irqchip,
+ d->host_data,
+ handle_percpu_devid_fasteoi_ipi,
+ NULL, NULL);
+ }
+
+ return 0;
+}
+
+static void armada_370_xp_ipi_free(struct irq_domain *d,
+ unsigned int virq,
+ unsigned int nr_irqs)
+{
+ /* Not freeing IPIs */
+}
+
+static const struct irq_domain_ops ipi_domain_ops = {
+ .alloc = armada_370_xp_ipi_alloc,
+ .free = armada_370_xp_ipi_free,
+};
+
+static void ipi_resume(void)
+{
+ int i;
+
+ for (i = 0; i < IPI_DOORBELL_END; i++) {
+ int irq;
+
+ irq = irq_find_mapping(ipi_domain, i);
+ if (irq <= 0)
+ continue;
+ if (irq_percpu_is_enabled(irq)) {
+ struct irq_data *d;
+ d = irq_domain_get_irq_data(ipi_domain, irq);
+ armada_370_xp_ipi_unmask(d);
+ }
+ }
+}
+
+static __init void armada_xp_ipi_init(struct device_node *node)
+{
+ int base_ipi;
+
+ ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node),
+ IPI_DOORBELL_END,
+ &ipi_domain_ops, NULL);
+ if (WARN_ON(!ipi_domain))
+ return;
+
+ irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI);
+ base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, IPI_DOORBELL_END,
+ NUMA_NO_NODE, NULL, false, NULL);
+ if (WARN_ON(!base_ipi))
+ return;
+
+ set_smp_ipi_range(base_ipi, IPI_DOORBELL_END);
+}
+
static DEFINE_RAW_SPINLOCK(irq_controller_lock);
static int armada_xp_set_affinity(struct irq_data *d,
return IRQ_SET_MASK_OK;
}
-#endif
-
-static struct irq_chip armada_370_xp_irq_chip = {
- .name = "MPIC",
- .irq_mask = armada_370_xp_irq_mask,
- .irq_mask_ack = armada_370_xp_irq_mask,
- .irq_unmask = armada_370_xp_irq_unmask,
-#ifdef CONFIG_SMP
- .irq_set_affinity = armada_xp_set_affinity,
-#endif
- .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
-};
-
-static int armada_370_xp_mpic_irq_map(struct irq_domain *h,
- unsigned int virq, irq_hw_number_t hw)
-{
- armada_370_xp_irq_mask(irq_get_irq_data(virq));
- if (!is_percpu_irq(hw))
- writel(hw, per_cpu_int_base +
- ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
- else
- writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS);
- irq_set_status_flags(virq, IRQ_LEVEL);
-
- if (is_percpu_irq(hw)) {
- irq_set_percpu_devid(virq);
- irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
- handle_percpu_devid_irq);
- } else {
- irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
- handle_level_irq);
- irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq)));
- }
- irq_set_probe(virq);
-
- return 0;
-}
static void armada_xp_mpic_smp_cpu_init(void)
{
for (i = 0; i < nr_irqs; i++)
writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS);
+ /* Disable all IPIs */
+ writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
+
/* Clear pending IPIs */
writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
- /* Enable first 8 IPIs */
- writel(IPI_DOORBELL_MASK, per_cpu_int_base +
- ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
-
/* Unmask IPI interrupt */
writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
}
-static void armada_xp_mpic_perf_init(void)
-{
- unsigned long cpuid = cpu_logical_map(smp_processor_id());
-
- /* Enable Performance Counter Overflow interrupts */
- writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid),
- per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS);
-}
-
-#ifdef CONFIG_SMP
-static void armada_mpic_send_doorbell(const struct cpumask *mask,
- unsigned int irq)
-{
- int cpu;
- unsigned long map = 0;
-
- /* Convert our logical CPU mask into a physical one. */
- for_each_cpu(cpu, mask)
- map |= 1 << cpu_logical_map(cpu);
-
- /*
- * Ensure that stores to Normal memory are visible to the
- * other CPUs before issuing the IPI.
- */
- dsb();
-
- /* submit softirq */
- writel((map << 8) | irq, main_int_base +
- ARMADA_370_XP_SW_TRIG_INT_OFFS);
-}
-
static void armada_xp_mpic_reenable_percpu(void)
{
unsigned int irq;
armada_370_xp_irq_unmask(data);
}
+
+ ipi_resume();
}
static int armada_xp_mpic_starting_cpu(unsigned int cpu)
enable_percpu_irq(parent_irq, IRQ_TYPE_NONE);
return 0;
}
+#else
+static void armada_xp_mpic_smp_cpu_init(void) {}
+static void ipi_resume(void) {}
+#endif
+
+static struct irq_chip armada_370_xp_irq_chip = {
+ .name = "MPIC",
+ .irq_mask = armada_370_xp_irq_mask,
+ .irq_mask_ack = armada_370_xp_irq_mask,
+ .irq_unmask = armada_370_xp_irq_unmask,
+#ifdef CONFIG_SMP
+ .irq_set_affinity = armada_xp_set_affinity,
#endif
+ .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
+};
+
+static int armada_370_xp_mpic_irq_map(struct irq_domain *h,
+ unsigned int virq, irq_hw_number_t hw)
+{
+ armada_370_xp_irq_mask(irq_get_irq_data(virq));
+ if (!is_percpu_irq(hw))
+ writel(hw, per_cpu_int_base +
+ ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
+ else
+ writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS);
+ irq_set_status_flags(virq, IRQ_LEVEL);
+
+ if (is_percpu_irq(hw)) {
+ irq_set_percpu_devid(virq);
+ irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
+ handle_percpu_devid_irq);
+ } else {
+ irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
+ handle_level_irq);
+ irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq)));
+ }
+ irq_set_probe(virq);
+
+ return 0;
+}
static const struct irq_domain_ops armada_370_xp_mpic_irq_ops = {
.map = armada_370_xp_mpic_irq_map,
#ifdef CONFIG_SMP
/* IPI Handling */
if (irqnr == 0) {
- u32 ipimask, ipinr;
+ unsigned long ipimask;
+ int ipi;
ipimask = readl_relaxed(per_cpu_int_base +
ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS)
& IPI_DOORBELL_MASK;
- writel(~ipimask, per_cpu_int_base +
- ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
-
- /* Handle all pending doorbells */
- for (ipinr = IPI_DOORBELL_START;
- ipinr < IPI_DOORBELL_END; ipinr++) {
- if (ipimask & (0x1 << ipinr))
- handle_IPI(ipinr, regs);
- }
- continue;
+ for_each_set_bit(ipi, &ipimask, IPI_DOORBELL_END)
+ handle_domain_irq(ipi_domain, ipi, regs);
}
#endif
writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
if (doorbell_mask_reg & PCI_MSI_DOORBELL_MASK)
writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
+
+ ipi_resume();
}
static struct syscore_ops armada_370_xp_mpic_syscore_ops = {
irq_set_default_host(armada_370_xp_mpic_domain);
set_handle_irq(armada_370_xp_handle_irq);
#ifdef CONFIG_SMP
- set_smp_cross_call(armada_mpic_send_doorbell);
+ armada_xp_ipi_init(node);
cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING,
"irqchip/armada/ipi:starting",
armada_xp_mpic_starting_cpu, NULL);
#include <linux/of_irq.h>
#include <linux/irqchip.h>
#include <linux/irqdomain.h>
+#include <linux/irqchip/chained_irq.h>
#include <linux/irqchip/irq-bcm2836.h>
#include <asm/exception.h>
.irq_unmask = bcm2836_arm_irqchip_unmask_gpu_irq,
};
+static void bcm2836_arm_irqchip_dummy_op(struct irq_data *d)
+{
+}
+
+static struct irq_chip bcm2836_arm_irqchip_dummy = {
+ .name = "bcm2836-dummy",
+ .irq_eoi = bcm2836_arm_irqchip_dummy_op,
+};
+
static int bcm2836_map(struct irq_domain *d, unsigned int irq,
irq_hw_number_t hw)
{
struct irq_chip *chip;
switch (hw) {
+ case LOCAL_IRQ_MAILBOX0:
+ chip = &bcm2836_arm_irqchip_dummy;
+ break;
case LOCAL_IRQ_CNTPSIRQ:
case LOCAL_IRQ_CNTPNSIRQ:
case LOCAL_IRQ_CNTHPIRQ:
u32 stat;
stat = readl_relaxed(intc.base + LOCAL_IRQ_PENDING0 + 4 * cpu);
- if (stat & BIT(LOCAL_IRQ_MAILBOX0)) {
-#ifdef CONFIG_SMP
- void __iomem *mailbox0 = (intc.base +
- LOCAL_MAILBOX0_CLR0 + 16 * cpu);
- u32 mbox_val = readl(mailbox0);
- u32 ipi = ffs(mbox_val) - 1;
-
- writel(1 << ipi, mailbox0);
- handle_IPI(ipi, regs);
-#endif
- } else if (stat) {
+ if (stat) {
u32 hwirq = ffs(stat) - 1;
handle_domain_irq(intc.domain, hwirq, regs);
}
#ifdef CONFIG_SMP
-static void bcm2836_arm_irqchip_send_ipi(const struct cpumask *mask,
- unsigned int ipi)
+static struct irq_domain *ipi_domain;
+
+static void bcm2836_arm_irqchip_handle_ipi(struct irq_desc *desc)
+{
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ int cpu = smp_processor_id();
+ u32 mbox_val;
+
+ chained_irq_enter(chip, desc);
+
+ mbox_val = readl_relaxed(intc.base + LOCAL_MAILBOX0_CLR0 + 16 * cpu);
+ if (mbox_val) {
+ int hwirq = ffs(mbox_val) - 1;
+ generic_handle_irq(irq_find_mapping(ipi_domain, hwirq));
+ }
+
+ chained_irq_exit(chip, desc);
+}
+
+static void bcm2836_arm_irqchip_ipi_eoi(struct irq_data *d)
+{
+ int cpu = smp_processor_id();
+
+ writel_relaxed(BIT(d->hwirq),
+ intc.base + LOCAL_MAILBOX0_CLR0 + 16 * cpu);
+}
+
+static void bcm2836_arm_irqchip_ipi_send_mask(struct irq_data *d,
+ const struct cpumask *mask)
{
int cpu;
void __iomem *mailbox0_base = intc.base + LOCAL_MAILBOX0_SET0;
*/
smp_wmb();
- for_each_cpu(cpu, mask) {
- writel(1 << ipi, mailbox0_base + 16 * cpu);
+ for_each_cpu(cpu, mask)
+ writel_relaxed(BIT(d->hwirq), mailbox0_base + 16 * cpu);
+}
+
+static struct irq_chip bcm2836_arm_irqchip_ipi = {
+ .name = "IPI",
+ .irq_mask = bcm2836_arm_irqchip_dummy_op,
+ .irq_unmask = bcm2836_arm_irqchip_dummy_op,
+ .irq_eoi = bcm2836_arm_irqchip_ipi_eoi,
+ .ipi_send_mask = bcm2836_arm_irqchip_ipi_send_mask,
+};
+
+static int bcm2836_arm_irqchip_ipi_alloc(struct irq_domain *d,
+ unsigned int virq,
+ unsigned int nr_irqs, void *args)
+{
+ int i;
+
+ for (i = 0; i < nr_irqs; i++) {
+ irq_set_percpu_devid(virq + i);
+ irq_domain_set_info(d, virq + i, i, &bcm2836_arm_irqchip_ipi,
+ d->host_data,
+ handle_percpu_devid_fasteoi_ipi,
+ NULL, NULL);
}
+
+ return 0;
}
+static void bcm2836_arm_irqchip_ipi_free(struct irq_domain *d,
+ unsigned int virq,
+ unsigned int nr_irqs)
+{
+ /* Not freeing IPIs */
+}
+
+static const struct irq_domain_ops ipi_domain_ops = {
+ .alloc = bcm2836_arm_irqchip_ipi_alloc,
+ .free = bcm2836_arm_irqchip_ipi_free,
+};
+
static int bcm2836_cpu_starting(unsigned int cpu)
{
bcm2836_arm_irqchip_unmask_per_cpu_irq(LOCAL_MAILBOX_INT_CONTROL0, 0,
cpu);
return 0;
}
-#endif
-static const struct irq_domain_ops bcm2836_arm_irqchip_intc_ops = {
- .xlate = irq_domain_xlate_onetwocell,
- .map = bcm2836_map,
-};
+#define BITS_PER_MBOX 32
-static void
-bcm2836_arm_irqchip_smp_init(void)
+static void bcm2836_arm_irqchip_smp_init(void)
{
-#ifdef CONFIG_SMP
+ struct irq_fwspec ipi_fwspec = {
+ .fwnode = intc.domain->fwnode,
+ .param_count = 1,
+ .param = {
+ [0] = LOCAL_IRQ_MAILBOX0,
+ },
+ };
+ int base_ipi, mux_irq;
+
+ mux_irq = irq_create_fwspec_mapping(&ipi_fwspec);
+ if (WARN_ON(mux_irq <= 0))
+ return;
+
+ ipi_domain = irq_domain_create_linear(intc.domain->fwnode,
+ BITS_PER_MBOX, &ipi_domain_ops,
+ NULL);
+ if (WARN_ON(!ipi_domain))
+ return;
+
+ ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE;
+ irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI);
+
+ base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, BITS_PER_MBOX,
+ NUMA_NO_NODE, NULL,
+ false, NULL);
+
+ if (WARN_ON(!base_ipi))
+ return;
+
+ set_smp_ipi_range(base_ipi, BITS_PER_MBOX);
+
+ irq_set_chained_handler_and_data(mux_irq,
+ bcm2836_arm_irqchip_handle_ipi, NULL);
+
/* Unmask IPIs to the boot CPU. */
cpuhp_setup_state(CPUHP_AP_IRQ_BCM2836_STARTING,
"irqchip/bcm2836:starting", bcm2836_cpu_starting,
bcm2836_cpu_dying);
-
- set_smp_cross_call(bcm2836_arm_irqchip_send_ipi);
-#endif
}
+#else
+#define bcm2836_arm_irqchip_smp_init() do { } while(0)
+#endif
+
+static const struct irq_domain_ops bcm2836_arm_irqchip_intc_ops = {
+ .xlate = irq_domain_xlate_onetwocell,
+ .map = bcm2836_map,
+};
/*
* The LOCAL_IRQ_CNT* timer firings are based off of the external
if (!intc.domain)
panic("%pOF: unable to create IRQ domain\n", node);
+ irq_domain_update_bus_token(intc.domain, DOMAIN_BUS_WIRED);
+
bcm2836_arm_irqchip_smp_init();
set_handle_irq(bcm2836_arm_irqchip_handle_irq);
#include <linux/irqchip/chained_irq.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
+#include <linux/interrupt.h>
#define APB_INT_ENABLE_L 0x00
#define APB_INT_ENABLE_H 0x04
#define APB_INT_FINALSTATUS_H 0x34
#define APB_INT_BASE_OFFSET 0x04
-static void dw_apb_ictl_handler(struct irq_desc *desc)
+/* irq domain of the primary interrupt controller. */
+static struct irq_domain *dw_apb_ictl_irq_domain;
+
+static void __irq_entry dw_apb_ictl_handle_irq(struct pt_regs *regs)
+{
+ struct irq_domain *d = dw_apb_ictl_irq_domain;
+ int n;
+
+ for (n = 0; n < d->revmap_size; n += 32) {
+ struct irq_chip_generic *gc = irq_get_domain_generic_chip(d, n);
+ u32 stat = readl_relaxed(gc->reg_base + APB_INT_FINALSTATUS_L);
+
+ while (stat) {
+ u32 hwirq = ffs(stat) - 1;
+
+ handle_domain_irq(d, hwirq, regs);
+ stat &= ~BIT(hwirq);
+ }
+ }
+}
+
+static void dw_apb_ictl_handle_irq_cascaded(struct irq_desc *desc)
{
struct irq_domain *d = irq_desc_get_handler_data(desc);
struct irq_chip *chip = irq_desc_get_chip(desc);
u32 virq = irq_find_mapping(d, gc->irq_base + hwirq);
generic_handle_irq(virq);
- stat &= ~(1 << hwirq);
+ stat &= ~BIT(hwirq);
}
}
chained_irq_exit(chip, desc);
}
+static int dw_apb_ictl_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs, void *arg)
+{
+ int i, ret;
+ irq_hw_number_t hwirq;
+ unsigned int type = IRQ_TYPE_NONE;
+ struct irq_fwspec *fwspec = arg;
+
+ ret = irq_domain_translate_onecell(domain, fwspec, &hwirq, &type);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < nr_irqs; i++)
+ irq_map_generic_chip(domain, virq + i, hwirq + i);
+
+ return 0;
+}
+
+static const struct irq_domain_ops dw_apb_ictl_irq_domain_ops = {
+ .translate = irq_domain_translate_onecell,
+ .alloc = dw_apb_ictl_irq_domain_alloc,
+ .free = irq_domain_free_irqs_top,
+};
+
#ifdef CONFIG_PM
static void dw_apb_ictl_resume(struct irq_data *d)
{
static int __init dw_apb_ictl_init(struct device_node *np,
struct device_node *parent)
{
+ const struct irq_domain_ops *domain_ops;
unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN;
struct resource r;
struct irq_domain *domain;
struct irq_chip_generic *gc;
void __iomem *iobase;
- int ret, nrirqs, irq, i;
+ int ret, nrirqs, parent_irq, i;
u32 reg;
- /* Map the parent interrupt for the chained handler */
- irq = irq_of_parse_and_map(np, 0);
- if (irq <= 0) {
- pr_err("%pOF: unable to parse irq\n", np);
- return -EINVAL;
+ if (!parent) {
+ /* Used as the primary interrupt controller */
+ parent_irq = 0;
+ domain_ops = &dw_apb_ictl_irq_domain_ops;
+ } else {
+ /* Map the parent interrupt for the chained handler */
+ parent_irq = irq_of_parse_and_map(np, 0);
+ if (parent_irq <= 0) {
+ pr_err("%pOF: unable to parse irq\n", np);
+ return -EINVAL;
+ }
+ domain_ops = &irq_generic_chip_ops;
}
ret = of_address_to_resource(np, 0, &r);
else
nrirqs = fls(readl_relaxed(iobase + APB_INT_ENABLE_L));
- domain = irq_domain_add_linear(np, nrirqs,
- &irq_generic_chip_ops, NULL);
+ domain = irq_domain_add_linear(np, nrirqs, domain_ops, NULL);
if (!domain) {
pr_err("%pOF: unable to add irq domain\n", np);
ret = -ENOMEM;
gc->chip_types[0].chip.irq_resume = dw_apb_ictl_resume;
}
- irq_set_chained_handler_and_data(irq, dw_apb_ictl_handler, domain);
+ if (parent_irq) {
+ irq_set_chained_handler_and_data(parent_irq,
+ dw_apb_ictl_handle_irq_cascaded, domain);
+ } else {
+ dw_apb_ictl_irq_domain = domain;
+ set_handle_irq(dw_apb_ictl_handle_irq);
+ }
return 0;
writel_relaxed(GICD_INT_DEF_PRI_X4,
base + GIC_DIST_PRI + i * 4 / 4);
- /* Ensure all SGI interrupts are now enabled */
- writel_relaxed(GICD_INT_EN_SET_SGI, base + GIC_DIST_ENABLE_SET);
-
if (sync_access)
sync_access();
}
return 0;
}
+static int its_irq_retrigger(struct irq_data *d)
+{
+ return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
+}
+
/*
* Two favourable cases:
*
.irq_set_affinity = its_set_affinity,
.irq_compose_msi_msg = its_irq_compose_msi_msg,
.irq_set_irqchip_state = its_irq_set_irqchip_state,
+ .irq_retrigger = its_irq_retrigger,
.irq_set_vcpu_affinity = its_irq_set_vcpu_affinity,
};
#define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0)
#define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1)
+#define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1)
+
struct redist_region {
void __iomem *redist_base;
phys_addr_t phys_base;
*
* If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure
* EL1 are subject to a similar operation thus matching the priorities presented
- * from the (re)distributor when security is enabled.
+ * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0,
+ * these values are unchanched by the GIC.
*
* see GICv3/GICv4 Architecture Specification (IHI0069D):
* - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
* priorities.
* - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
* interrupt.
- *
- * For now, we only support pseudo-NMIs if we have non-secure view of
- * priorities.
*/
static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
DEFINE_STATIC_KEY_FALSE(gic_pmr_sync);
EXPORT_SYMBOL(gic_pmr_sync);
+DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities);
+EXPORT_SYMBOL(gic_nonsecure_priorities);
+
/* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */
static refcount_t *ppi_nmi_refs;
#define DEFAULT_PMR_VALUE 0xf0
enum gic_intid_range {
+ SGI_RANGE,
PPI_RANGE,
SPI_RANGE,
EPPI_RANGE,
static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
{
switch (hwirq) {
+ case 0 ... 15:
+ return SGI_RANGE;
case 16 ... 31:
return PPI_RANGE;
case 32 ... 1019:
return d->hwirq;
}
-static inline int gic_irq_in_rdist(struct irq_data *d)
+static inline bool gic_irq_in_rdist(struct irq_data *d)
{
- enum gic_intid_range range = get_intid_range(d);
- return range == PPI_RANGE || range == EPPI_RANGE;
+ switch (get_intid_range(d)) {
+ case SGI_RANGE:
+ case PPI_RANGE:
+ case EPPI_RANGE:
+ return true;
+ default:
+ return false;
+ }
}
static inline void __iomem *gic_dist_base(struct irq_data *d)
{
switch (get_intid_range(d)) {
+ case SGI_RANGE:
case PPI_RANGE:
case EPPI_RANGE:
/* SGI+PPI -> SGI_base for this CPU */
static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index)
{
switch (get_intid_range(d)) {
+ case SGI_RANGE:
case PPI_RANGE:
case SPI_RANGE:
*index = d->hwirq;
{
u32 reg;
- if (d->hwirq >= 8192) /* PPI/SPI only */
+ if (d->hwirq >= 8192) /* SGI/PPI/SPI only */
return -EINVAL;
switch (which) {
u32 offset, index;
int ret;
- /* Interrupt configuration for SGIs can't be changed */
- if (irq < 16)
- return -EINVAL;
-
range = get_intid_range(d);
+ /* Interrupt configuration for SGIs can't be changed */
+ if (range == SGI_RANGE)
+ return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0;
+
/* SPIs have restrictions on the supported types */
if ((range == SPI_RANGE || range == ESPI_RANGE) &&
type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
{
+ if (get_intid_range(d) == SGI_RANGE)
+ return -EINVAL;
+
if (vcpu)
irqd_set_forwarded_to_vcpu(d);
else
if ((irqnr >= 1020 && irqnr <= 1023))
return;
- /* Treat anything but SGIs in a uniform way */
- if (likely(irqnr > 15)) {
- int err;
-
- if (static_branch_likely(&supports_deactivate_key))
- gic_write_eoir(irqnr);
- else
- isb();
-
- err = handle_domain_irq(gic_data.domain, irqnr, regs);
- if (err) {
- WARN_ONCE(true, "Unexpected interrupt received!\n");
- gic_deactivate_unhandled(irqnr);
- }
- return;
- }
- if (irqnr < 16) {
+ if (static_branch_likely(&supports_deactivate_key))
gic_write_eoir(irqnr);
- if (static_branch_likely(&supports_deactivate_key))
- gic_write_dir(irqnr);
-#ifdef CONFIG_SMP
- /*
- * Unlike GICv2, we don't need an smp_rmb() here.
- * The control dependency from gic_read_iar to
- * the ISB in gic_write_eoir is enough to ensure
- * that any shared data read by handle_IPI will
- * be read after the ACK.
- */
- handle_IPI(irqnr, regs);
-#else
- WARN_ONCE(true, "Unexpected SGI received!\n");
-#endif
+ else
+ isb();
+
+ if (handle_domain_irq(gic_data.domain, irqnr, regs)) {
+ WARN_ONCE(true, "Unexpected interrupt received!\n");
+ gic_deactivate_unhandled(irqnr);
}
}
/* Set priority mask register */
if (!gic_prio_masking_enabled()) {
write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
- } else {
+ } else if (gic_supports_nmi()) {
/*
* Mismatch configuration with boot CPU, the system is likely
* to die as interrupt masking will not work properly on all
* CPUs
+ *
+ * The boot CPU calls this function before enabling NMI support,
+ * and as a result we'll never see this warning in the boot path
+ * for that CPU.
*/
- WARN_ON(gic_supports_nmi() && group0 &&
- !gic_dist_security_disabled());
+ if (static_branch_unlikely(&gic_nonsecure_priorities))
+ WARN_ON(!group0 || gic_dist_security_disabled());
+ else
+ WARN_ON(group0 && !gic_dist_security_disabled());
}
/*
gic_write_sgi1r(val);
}
-static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
+static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask)
{
int cpu;
- if (WARN_ON(irq >= 16))
+ if (WARN_ON(d->hwirq >= 16))
return;
/*
u16 tlist;
tlist = gic_compute_target_list(&cpu, mask, cluster_id);
- gic_send_sgi(cluster_id, tlist, irq);
+ gic_send_sgi(cluster_id, tlist, d->hwirq);
}
/* Force the above writes to ICC_SGI1R_EL1 to be executed */
static void __init gic_smp_init(void)
{
- set_smp_cross_call(gic_raise_softirq);
+ struct irq_fwspec sgi_fwspec = {
+ .fwnode = gic_data.fwnode,
+ .param_count = 1,
+ };
+ int base_sgi;
+
cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
"irqchip/arm/gicv3:starting",
gic_starting_cpu, NULL);
+
+ /* Register all 8 non-secure SGIs */
+ base_sgi = __irq_domain_alloc_irqs(gic_data.domain, -1, 8,
+ NUMA_NO_NODE, &sgi_fwspec,
+ false, NULL);
+ if (WARN_ON(base_sgi <= 0))
+ return;
+
+ set_smp_ipi_range(base_sgi, 8);
}
static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
}
#else
#define gic_set_affinity NULL
+#define gic_ipi_send_mask NULL
#define gic_smp_init() do { } while(0)
#endif
+static int gic_retrigger(struct irq_data *data)
+{
+ return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true);
+}
+
#ifdef CONFIG_CPU_PM
static int gic_cpu_pm_notifier(struct notifier_block *self,
unsigned long cmd, void *v)
.irq_eoi = gic_eoi_irq,
.irq_set_type = gic_set_type,
.irq_set_affinity = gic_set_affinity,
+ .irq_retrigger = gic_retrigger,
.irq_get_irqchip_state = gic_irq_get_irqchip_state,
.irq_set_irqchip_state = gic_irq_set_irqchip_state,
.irq_nmi_setup = gic_irq_nmi_setup,
.irq_nmi_teardown = gic_irq_nmi_teardown,
+ .ipi_send_mask = gic_ipi_send_mask,
.flags = IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE |
IRQCHIP_MASK_ON_SUSPEND,
.irq_eoi = gic_eoimode1_eoi_irq,
.irq_set_type = gic_set_type,
.irq_set_affinity = gic_set_affinity,
+ .irq_retrigger = gic_retrigger,
.irq_get_irqchip_state = gic_irq_get_irqchip_state,
.irq_set_irqchip_state = gic_irq_set_irqchip_state,
.irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity,
.irq_nmi_setup = gic_irq_nmi_setup,
.irq_nmi_teardown = gic_irq_nmi_teardown,
+ .ipi_send_mask = gic_ipi_send_mask,
.flags = IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE |
IRQCHIP_MASK_ON_SUSPEND,
irq_hw_number_t hw)
{
struct irq_chip *chip = &gic_chip;
+ struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq));
if (static_branch_likely(&supports_deactivate_key))
chip = &gic_eoimode1_chip;
switch (__get_intid_range(hw)) {
+ case SGI_RANGE:
+ irq_set_percpu_devid(irq);
+ irq_domain_set_info(d, irq, hw, chip, d->host_data,
+ handle_percpu_devid_fasteoi_ipi,
+ NULL, NULL);
+ break;
+
case PPI_RANGE:
case EPPI_RANGE:
irq_set_percpu_devid(irq);
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_fasteoi_irq, NULL, NULL);
irq_set_probe(irq);
- irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
+ irqd_set_single_target(irqd);
break;
case LPI_RANGE:
return -EPERM;
}
+ /* Prevents SW retriggers which mess up the ACK/EOI ordering */
+ irqd_set_handle_enforce_irqctx(irqd);
return 0;
}
-#define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1)
-
static int gic_irq_domain_translate(struct irq_domain *d,
struct irq_fwspec *fwspec,
unsigned long *hwirq,
unsigned int *type)
{
+ if (fwspec->param_count == 1 && fwspec->param[0] < 16) {
+ *hwirq = fwspec->param[0];
+ *type = IRQ_TYPE_EDGE_RISING;
+ return 0;
+ }
+
if (is_of_node(fwspec->fwnode)) {
if (fwspec->param_count < 3)
return -EINVAL;
if (!gic_prio_masking_enabled())
return;
- if (gic_has_group0() && !gic_dist_security_disabled()) {
- pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n");
- return;
- }
-
ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL);
if (!ppi_nmi_refs)
return;
if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK)
static_branch_enable(&gic_pmr_sync);
- pr_info("%s ICC_PMR_EL1 synchronisation\n",
- static_branch_unlikely(&gic_pmr_sync) ? "Forcing" : "Relaxing");
+ pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n",
+ static_branch_unlikely(&gic_pmr_sync) ? "forced" : "relaxed");
+
+ /*
+ * How priority values are used by the GIC depends on two things:
+ * the security state of the GIC (controlled by the GICD_CTRL.DS bit)
+ * and if Group 0 interrupts can be delivered to Linux in the non-secure
+ * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the
+ * the ICC_PMR_EL1 register and the priority that software assigns to
+ * interrupts:
+ *
+ * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority
+ * -----------------------------------------------------------
+ * 1 | - | unchanged | unchanged
+ * -----------------------------------------------------------
+ * 0 | 1 | non-secure | non-secure
+ * -----------------------------------------------------------
+ * 0 | 0 | unchanged | non-secure
+ *
+ * where non-secure means that the value is right-shifted by one and the
+ * MSB bit set, to make it fit in the non-secure priority range.
+ *
+ * In the first two cases, where ICC_PMR_EL1 and the interrupt priority
+ * are both either modified or unchanged, we can use the same set of
+ * priorities.
+ *
+ * In the last case, where only the interrupt priorities are modified to
+ * be in the non-secure range, we use a different PMR value to mask IRQs
+ * and the rest of the values that we use remain unchanged.
+ */
+ if (gic_has_group0() && !gic_dist_security_disabled())
+ static_branch_enable(&gic_nonsecure_priorities);
static_branch_enable(&supports_pseudo_nmis);
gic_update_rdist_properties();
- gic_smp_init();
gic_dist_init();
gic_cpu_init();
+ gic_smp_init();
gic_cpu_pm_init();
if (gic_dist_supports_lpis()) {
#endif
struct irq_domain *domain;
unsigned int gic_irqs;
-#ifdef CONFIG_GIC_NON_BANKED
- void __iomem *(*get_base)(union gic_base *);
-#endif
};
#ifdef CONFIG_BL_SWITCHER
static struct gic_kvm_info gic_v2_kvm_info;
+static DEFINE_PER_CPU(u32, sgi_intid);
+
#ifdef CONFIG_GIC_NON_BANKED
-static void __iomem *gic_get_percpu_base(union gic_base *base)
-{
- return raw_cpu_read(*base->percpu_base);
-}
+static DEFINE_STATIC_KEY_FALSE(frankengic_key);
-static void __iomem *gic_get_common_base(union gic_base *base)
+static void enable_frankengic(void)
{
- return base->common_base;
+ static_branch_enable(&frankengic_key);
}
-static inline void __iomem *gic_data_dist_base(struct gic_chip_data *data)
+static inline void __iomem *__get_base(union gic_base *base)
{
- return data->get_base(&data->dist_base);
-}
+ if (static_branch_unlikely(&frankengic_key))
+ return raw_cpu_read(*base->percpu_base);
-static inline void __iomem *gic_data_cpu_base(struct gic_chip_data *data)
-{
- return data->get_base(&data->cpu_base);
+ return base->common_base;
}
-static inline void gic_set_base_accessor(struct gic_chip_data *data,
- void __iomem *(*f)(union gic_base *))
-{
- data->get_base = f;
-}
+#define gic_data_dist_base(d) __get_base(&(d)->dist_base)
+#define gic_data_cpu_base(d) __get_base(&(d)->cpu_base)
#else
#define gic_data_dist_base(d) ((d)->dist_base.common_base)
#define gic_data_cpu_base(d) ((d)->cpu_base.common_base)
-#define gic_set_base_accessor(d, f)
+#define enable_frankengic() do { } while(0)
#endif
static inline void __iomem *gic_dist_base(struct irq_data *d)
static void gic_eoi_irq(struct irq_data *d)
{
- writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI);
+ u32 hwirq = gic_irq(d);
+
+ if (hwirq < 16)
+ hwirq = this_cpu_read(sgi_intid);
+
+ writel_relaxed(hwirq, gic_cpu_base(d) + GIC_CPU_EOI);
}
static void gic_eoimode1_eoi_irq(struct irq_data *d)
{
+ u32 hwirq = gic_irq(d);
+
/* Do not deactivate an IRQ forwarded to a vcpu. */
if (irqd_is_forwarded_to_vcpu(d))
return;
- writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_DEACTIVATE);
+ if (hwirq < 16)
+ hwirq = this_cpu_read(sgi_intid);
+
+ writel_relaxed(hwirq, gic_cpu_base(d) + GIC_CPU_DEACTIVATE);
}
static int gic_irq_set_irqchip_state(struct irq_data *d,
/* Interrupt configuration for SGIs can't be changed */
if (gicirq < 16)
- return -EINVAL;
+ return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0;
/* SPIs have restrictions on the supported types */
if (gicirq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
{
/* Only interrupts on the primary GIC can be forwarded to a vcpu. */
- if (cascading_gic_irq(d))
+ if (cascading_gic_irq(d) || gic_irq(d) < 16)
return -EINVAL;
if (vcpu)
return 0;
}
-#ifdef CONFIG_SMP
-static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
- bool force)
+static int gic_retrigger(struct irq_data *data)
{
- void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + gic_irq(d);
- unsigned int cpu;
-
- if (!force)
- cpu = cpumask_any_and(mask_val, cpu_online_mask);
- else
- cpu = cpumask_first(mask_val);
-
- if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids)
- return -EINVAL;
-
- writeb_relaxed(gic_cpu_map[cpu], reg);
- irq_data_update_effective_affinity(d, cpumask_of(cpu));
-
- return IRQ_SET_MASK_OK_DONE;
+ return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true);
}
-#endif
static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
{
irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK);
irqnr = irqstat & GICC_IAR_INT_ID_MASK;
- if (likely(irqnr > 15 && irqnr < 1020)) {
- if (static_branch_likely(&supports_deactivate_key))
- writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
- isb();
- handle_domain_irq(gic->domain, irqnr, regs);
- continue;
- }
- if (irqnr < 16) {
+ if (unlikely(irqnr >= 1020))
+ break;
+
+ if (static_branch_likely(&supports_deactivate_key))
writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
- if (static_branch_likely(&supports_deactivate_key))
- writel_relaxed(irqstat, cpu_base + GIC_CPU_DEACTIVATE);
-#ifdef CONFIG_SMP
+ isb();
+
+ /*
+ * Ensure any shared data written by the CPU sending the IPI
+ * is read after we've read the ACK register on the GIC.
+ *
+ * Pairs with the write barrier in gic_ipi_send_mask
+ */
+ if (irqnr <= 15) {
+ smp_rmb();
+
/*
- * Ensure any shared data written by the CPU sending
- * the IPI is read after we've read the ACK register
- * on the GIC.
- *
- * Pairs with the write barrier in gic_raise_softirq
+ * The GIC encodes the source CPU in GICC_IAR,
+ * leading to the deactivation to fail if not
+ * written back as is to GICC_EOI. Stash the INTID
+ * away for gic_eoi_irq() to write back. This only
+ * works because we don't nest SGIs...
*/
- smp_rmb();
- handle_IPI(irqnr, regs);
-#endif
- continue;
+ this_cpu_write(sgi_intid, irqstat);
}
- break;
+
+ handle_domain_irq(gic->domain, irqnr, regs);
} while (1);
}
.irq_unmask = gic_unmask_irq,
.irq_eoi = gic_eoi_irq,
.irq_set_type = gic_set_type,
+ .irq_retrigger = gic_retrigger,
.irq_get_irqchip_state = gic_irq_get_irqchip_state,
.irq_set_irqchip_state = gic_irq_set_irqchip_state,
.flags = IRQCHIP_SET_TYPE_MASKED |
int i;
for (i = 0; i < CONFIG_ARM_GIC_MAX_NR; i++) {
-#ifdef CONFIG_GIC_NON_BANKED
- /* Skip over unused GICs */
- if (!gic_data[i].get_base)
- continue;
-#endif
switch (cmd) {
case CPU_PM_ENTER:
gic_cpu_save(&gic_data[i]);
#endif
#ifdef CONFIG_SMP
-static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
+static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
+ bool force)
+{
+ void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + gic_irq(d);
+ unsigned int cpu;
+
+ if (!force)
+ cpu = cpumask_any_and(mask_val, cpu_online_mask);
+ else
+ cpu = cpumask_first(mask_val);
+
+ if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids)
+ return -EINVAL;
+
+ writeb_relaxed(gic_cpu_map[cpu], reg);
+ irq_data_update_effective_affinity(d, cpumask_of(cpu));
+
+ return IRQ_SET_MASK_OK_DONE;
+}
+
+static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask)
{
int cpu;
unsigned long flags, map = 0;
if (unlikely(nr_cpu_ids == 1)) {
/* Only one CPU? let's do a self-IPI... */
- writel_relaxed(2 << 24 | irq,
+ writel_relaxed(2 << 24 | d->hwirq,
gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
return;
}
dmb(ishst);
/* this always happens on GIC0 */
- writel_relaxed(map << 16 | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
+ writel_relaxed(map << 16 | d->hwirq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
gic_unlock_irqrestore(flags);
}
+
+static int gic_starting_cpu(unsigned int cpu)
+{
+ gic_cpu_init(&gic_data[0]);
+ return 0;
+}
+
+static __init void gic_smp_init(void)
+{
+ struct irq_fwspec sgi_fwspec = {
+ .fwnode = gic_data[0].domain->fwnode,
+ .param_count = 1,
+ };
+ int base_sgi;
+
+ cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
+ "irqchip/arm/gic:starting",
+ gic_starting_cpu, NULL);
+
+ base_sgi = __irq_domain_alloc_irqs(gic_data[0].domain, -1, 8,
+ NUMA_NO_NODE, &sgi_fwspec,
+ false, NULL);
+ if (WARN_ON(base_sgi <= 0))
+ return;
+
+ set_smp_ipi_range(base_sgi, 8);
+}
+#else
+#define gic_smp_init() do { } while(0)
+#define gic_set_affinity NULL
+#define gic_ipi_send_mask NULL
#endif
#ifdef CONFIG_BL_SWITCHER
irq_hw_number_t hw)
{
struct gic_chip_data *gic = d->host_data;
+ struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq));
- if (hw < 32) {
+ switch (hw) {
+ case 0 ... 15:
+ irq_set_percpu_devid(irq);
+ irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
+ handle_percpu_devid_fasteoi_ipi,
+ NULL, NULL);
+ break;
+ case 16 ... 31:
irq_set_percpu_devid(irq);
irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
handle_percpu_devid_irq, NULL, NULL);
- } else {
+ break;
+ default:
irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data,
handle_fasteoi_irq, NULL, NULL);
irq_set_probe(irq);
- irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
+ irqd_set_single_target(irqd);
+ break;
}
+
+ /* Prevents SW retriggers which mess up the ACK/EOI ordering */
+ irqd_set_handle_enforce_irqctx(irqd);
return 0;
}
unsigned long *hwirq,
unsigned int *type)
{
+ if (fwspec->param_count == 1 && fwspec->param[0] < 16) {
+ *hwirq = fwspec->param[0];
+ *type = IRQ_TYPE_EDGE_RISING;
+ return 0;
+ }
+
if (is_of_node(fwspec->fwnode)) {
if (fwspec->param_count < 3)
return -EINVAL;
- /* Get the interrupt number and add 16 to skip over SGIs */
- *hwirq = fwspec->param[1] + 16;
-
- /*
- * For SPIs, we need to add 16 more to get the GIC irq
- * ID number
- */
- if (!fwspec->param[0])
- *hwirq += 16;
+ switch (fwspec->param[0]) {
+ case 0: /* SPI */
+ *hwirq = fwspec->param[1] + 32;
+ break;
+ case 1: /* PPI */
+ *hwirq = fwspec->param[1] + 16;
+ break;
+ default:
+ return -EINVAL;
+ }
*type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
return -EINVAL;
}
-static int gic_starting_cpu(unsigned int cpu)
-{
- gic_cpu_init(&gic_data[0]);
- return 0;
-}
-
static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
unsigned int nr_irqs, void *arg)
{
gic->chip.irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity;
}
-#ifdef CONFIG_SMP
- if (gic == &gic_data[0])
+ if (gic == &gic_data[0]) {
gic->chip.irq_set_affinity = gic_set_affinity;
-#endif
+ gic->chip.ipi_send_mask = gic_ipi_send_mask;
+ }
}
static int gic_init_bases(struct gic_chip_data *gic,
gic->raw_cpu_base + offset;
}
- gic_set_base_accessor(gic, gic_get_percpu_base);
+ enable_frankengic();
} else {
/* Normal, sane GIC... */
WARN(gic->percpu_offset,
gic->percpu_offset);
gic->dist_base.common_base = gic->raw_dist_base;
gic->cpu_base.common_base = gic->raw_cpu_base;
- gic_set_base_accessor(gic, gic_get_common_base);
}
/*
*/
for (i = 0; i < NR_GIC_CPU_IF; i++)
gic_cpu_map[i] = 0xff;
-#ifdef CONFIG_SMP
- set_smp_cross_call(gic_raise_softirq);
-#endif
- cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
- "irqchip/arm/gic:starting",
- gic_starting_cpu, NULL);
+
set_handle_irq(gic_handle_irq);
if (static_branch_likely(&supports_deactivate_key))
pr_info("GIC: Using split EOI/Deactivate mode\n");
ret = gic_init_bases(gic, handle);
if (ret)
kfree(name);
+ else if (gic == &gic_data[0])
+ gic_smp_init();
return ret;
}
return IRQ_SET_MASK_OK;
}
+
+static void hip04_ipi_send_mask(struct irq_data *d, const struct cpumask *mask)
+{
+ int cpu;
+ unsigned long flags, map = 0;
+
+ raw_spin_lock_irqsave(&irq_controller_lock, flags);
+
+ /* Convert our logical CPU mask into a physical one. */
+ for_each_cpu(cpu, mask)
+ map |= hip04_cpu_map[cpu];
+
+ /*
+ * Ensure that stores to Normal memory are visible to the
+ * other CPUs before they observe us issuing the IPI.
+ */
+ dmb(ishst);
+
+ /* this always happens on GIC0 */
+ writel_relaxed(map << 8 | d->hwirq, hip04_data.dist_base + GIC_DIST_SOFTINT);
+
+ raw_spin_unlock_irqrestore(&irq_controller_lock, flags);
+}
#endif
static void __exception_irq_entry hip04_handle_irq(struct pt_regs *regs)
irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK);
irqnr = irqstat & GICC_IAR_INT_ID_MASK;
- if (likely(irqnr > 15 && irqnr <= HIP04_MAX_IRQS)) {
+ if (irqnr <= HIP04_MAX_IRQS)
handle_domain_irq(hip04_data.domain, irqnr, regs);
- continue;
- }
- if (irqnr < 16) {
- writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
-#ifdef CONFIG_SMP
- handle_IPI(irqnr, regs);
-#endif
- continue;
- }
- break;
- } while (1);
+ } while (irqnr > HIP04_MAX_IRQS);
}
static struct irq_chip hip04_irq_chip = {
.irq_set_type = hip04_irq_set_type,
#ifdef CONFIG_SMP
.irq_set_affinity = hip04_irq_set_affinity,
+ .ipi_send_mask = hip04_ipi_send_mask,
#endif
.flags = IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE |
writel_relaxed(1, base + GIC_CPU_CTRL);
}
-#ifdef CONFIG_SMP
-static void hip04_raise_softirq(const struct cpumask *mask, unsigned int irq)
-{
- int cpu;
- unsigned long flags, map = 0;
-
- raw_spin_lock_irqsave(&irq_controller_lock, flags);
-
- /* Convert our logical CPU mask into a physical one. */
- for_each_cpu(cpu, mask)
- map |= hip04_cpu_map[cpu];
-
- /*
- * Ensure that stores to Normal memory are visible to the
- * other CPUs before they observe us issuing the IPI.
- */
- dmb(ishst);
-
- /* this always happens on GIC0 */
- writel_relaxed(map << 8 | irq, hip04_data.dist_base + GIC_DIST_SOFTINT);
-
- raw_spin_unlock_irqrestore(&irq_controller_lock, flags);
-}
-#endif
-
static int hip04_irq_domain_map(struct irq_domain *d, unsigned int irq,
irq_hw_number_t hw)
{
- if (hw < 32) {
+ if (hw < 16) {
+ irq_set_percpu_devid(irq);
+ irq_set_chip_and_handler(irq, &hip04_irq_chip,
+ handle_percpu_devid_fasteoi_ipi);
+ } else if (hw < 32) {
irq_set_percpu_devid(irq);
irq_set_chip_and_handler(irq, &hip04_irq_chip,
handle_percpu_devid_irq);
- irq_set_status_flags(irq, IRQ_NOAUTOEN);
} else {
irq_set_chip_and_handler(irq, &hip04_irq_chip,
handle_fasteoi_irq);
unsigned long *out_hwirq,
unsigned int *out_type)
{
- unsigned long ret = 0;
-
if (irq_domain_get_of_node(d) != controller)
return -EINVAL;
+ if (intsize == 1 && intspec[0] < 16) {
+ *out_hwirq = intspec[0];
+ *out_type = IRQ_TYPE_EDGE_RISING;
+ return 0;
+ }
if (intsize < 3)
return -EINVAL;
*out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
- return ret;
+ return 0;
}
static int hip04_irq_starting_cpu(unsigned int cpu)
static int __init
hip04_of_init(struct device_node *node, struct device_node *parent)
{
- irq_hw_number_t hwirq_base = 16;
int nr_irqs, irq_base, i;
if (WARN_ON(!node))
nr_irqs = HIP04_MAX_IRQS;
hip04_data.nr_irqs = nr_irqs;
- nr_irqs -= hwirq_base; /* calculate # of irqs to allocate */
-
- irq_base = irq_alloc_descs(-1, hwirq_base, nr_irqs, numa_node_id());
+ irq_base = irq_alloc_descs(-1, 0, nr_irqs, numa_node_id());
if (irq_base < 0) {
pr_err("failed to allocate IRQ numbers\n");
return -EINVAL;
}
hip04_data.domain = irq_domain_add_legacy(node, nr_irqs, irq_base,
- hwirq_base,
+ 0,
&hip04_irq_domain_ops,
&hip04_data);
-
if (WARN_ON(!hip04_data.domain))
return -EINVAL;
#ifdef CONFIG_SMP
- set_smp_cross_call(hip04_raise_softirq);
+ set_smp_ipi_range(irq_base, 16);
#endif
set_handle_irq(hip04_handle_irq);
}
data->ipg_clk = devm_clk_get(&pdev->dev, "ipg");
- if (IS_ERR(data->ipg_clk)) {
- ret = PTR_ERR(data->ipg_clk);
- if (ret != -EPROBE_DEFER)
- dev_err(&pdev->dev, "failed to get ipg clk: %d\n", ret);
- return ret;
- }
+ if (IS_ERR(data->ipg_clk))
+ return dev_err_probe(&pdev->dev, PTR_ERR(data->ipg_clk),
+ "failed to get ipg clk\n");
data->channum = channum;
raw_spin_lock_init(&data->lock);
}
data->ipg_clk = devm_clk_get(&pdev->dev, "ipg");
- if (IS_ERR(data->ipg_clk)) {
- ret = PTR_ERR(data->ipg_clk);
- if (ret != -EPROBE_DEFER)
- dev_err(&pdev->dev, "failed to get ipg clk: %d\n", ret);
- return ret;
- }
+ if (IS_ERR(data->ipg_clk))
+ return dev_err_probe(&pdev->dev, PTR_ERR(data->ipg_clk),
+ "failed to get ipg clk\n");
raw_spin_lock_init(&data->lock);
/* Clear IRQ cause registers, mask all interrupts */
for (idx = 0; idx < priv->num_parents; idx++) {
writel_relaxed(0x0, priv->base + HTVEC_EN_OFF + 4 * idx);
- writel_relaxed(0xFFFFFFFF, priv->base);
+ writel_relaxed(0xFFFFFFFF, priv->base + 4 * idx);
}
}
goto free_priv;
}
- /* Interrupt may come from any of the 4 interrupt line */
+ /* Interrupt may come from any of the 8 interrupt lines */
for (i = 0; i < HTVEC_MAX_PARENT_IRQ; i++) {
parent_irq[i] = irq_of_parse_and_map(node, i);
if (parent_irq[i] <= 0)
--- /dev/null
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+/*
+ * Copyright (c) 2020 MediaTek Inc.
+ * Author Mark-PK Tsai <mark-pk.tsai@mediatek.com>
+ */
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqdomain.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#define INTC_MASK 0x0
+#define INTC_EOI 0x20
+
+struct mst_intc_chip_data {
+ raw_spinlock_t lock;
+ unsigned int irq_start, nr_irqs;
+ void __iomem *base;
+ bool no_eoi;
+};
+
+static void mst_set_irq(struct irq_data *d, u32 offset)
+{
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+ struct mst_intc_chip_data *cd = irq_data_get_irq_chip_data(d);
+ u16 val, mask;
+ unsigned long flags;
+
+ mask = 1 << (hwirq % 16);
+ offset += (hwirq / 16) * 4;
+
+ raw_spin_lock_irqsave(&cd->lock, flags);
+ val = readw_relaxed(cd->base + offset) | mask;
+ writew_relaxed(val, cd->base + offset);
+ raw_spin_unlock_irqrestore(&cd->lock, flags);
+}
+
+static void mst_clear_irq(struct irq_data *d, u32 offset)
+{
+ irq_hw_number_t hwirq = irqd_to_hwirq(d);
+ struct mst_intc_chip_data *cd = irq_data_get_irq_chip_data(d);
+ u16 val, mask;
+ unsigned long flags;
+
+ mask = 1 << (hwirq % 16);
+ offset += (hwirq / 16) * 4;
+
+ raw_spin_lock_irqsave(&cd->lock, flags);
+ val = readw_relaxed(cd->base + offset) & ~mask;
+ writew_relaxed(val, cd->base + offset);
+ raw_spin_unlock_irqrestore(&cd->lock, flags);
+}
+
+static void mst_intc_mask_irq(struct irq_data *d)
+{
+ mst_set_irq(d, INTC_MASK);
+ irq_chip_mask_parent(d);
+}
+
+static void mst_intc_unmask_irq(struct irq_data *d)
+{
+ mst_clear_irq(d, INTC_MASK);
+ irq_chip_unmask_parent(d);
+}
+
+static void mst_intc_eoi_irq(struct irq_data *d)
+{
+ struct mst_intc_chip_data *cd = irq_data_get_irq_chip_data(d);
+
+ if (!cd->no_eoi)
+ mst_set_irq(d, INTC_EOI);
+
+ irq_chip_eoi_parent(d);
+}
+
+static struct irq_chip mst_intc_chip = {
+ .name = "mst-intc",
+ .irq_mask = mst_intc_mask_irq,
+ .irq_unmask = mst_intc_unmask_irq,
+ .irq_eoi = mst_intc_eoi_irq,
+ .irq_get_irqchip_state = irq_chip_get_parent_state,
+ .irq_set_irqchip_state = irq_chip_set_parent_state,
+ .irq_set_affinity = irq_chip_set_affinity_parent,
+ .irq_set_vcpu_affinity = irq_chip_set_vcpu_affinity_parent,
+ .irq_set_type = irq_chip_set_type_parent,
+ .irq_retrigger = irq_chip_retrigger_hierarchy,
+ .flags = IRQCHIP_SET_TYPE_MASKED |
+ IRQCHIP_SKIP_SET_WAKE |
+ IRQCHIP_MASK_ON_SUSPEND,
+};
+
+static int mst_intc_domain_translate(struct irq_domain *d,
+ struct irq_fwspec *fwspec,
+ unsigned long *hwirq,
+ unsigned int *type)
+{
+ struct mst_intc_chip_data *cd = d->host_data;
+
+ if (is_of_node(fwspec->fwnode)) {
+ if (fwspec->param_count != 3)
+ return -EINVAL;
+
+ /* No PPI should point to this domain */
+ if (fwspec->param[0] != 0)
+ return -EINVAL;
+
+ if (fwspec->param[1] >= cd->nr_irqs)
+ return -EINVAL;
+
+ *hwirq = fwspec->param[1];
+ *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+static int mst_intc_domain_alloc(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs, void *data)
+{
+ int i;
+ irq_hw_number_t hwirq;
+ struct irq_fwspec parent_fwspec, *fwspec = data;
+ struct mst_intc_chip_data *cd = domain->host_data;
+
+ /* Not GIC compliant */
+ if (fwspec->param_count != 3)
+ return -EINVAL;
+
+ /* No PPI should point to this domain */
+ if (fwspec->param[0])
+ return -EINVAL;
+
+ hwirq = fwspec->param[1];
+ for (i = 0; i < nr_irqs; i++)
+ irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
+ &mst_intc_chip,
+ domain->host_data);
+
+ parent_fwspec = *fwspec;
+ parent_fwspec.fwnode = domain->parent->fwnode;
+ parent_fwspec.param[1] = cd->irq_start + hwirq;
+ return irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &parent_fwspec);
+}
+
+static const struct irq_domain_ops mst_intc_domain_ops = {
+ .translate = mst_intc_domain_translate,
+ .alloc = mst_intc_domain_alloc,
+ .free = irq_domain_free_irqs_common,
+};
+
+int __init
+mst_intc_of_init(struct device_node *dn, struct device_node *parent)
+{
+ struct irq_domain *domain, *domain_parent;
+ struct mst_intc_chip_data *cd;
+ u32 irq_start, irq_end;
+
+ domain_parent = irq_find_host(parent);
+ if (!domain_parent) {
+ pr_err("mst-intc: interrupt-parent not found\n");
+ return -EINVAL;
+ }
+
+ if (of_property_read_u32_index(dn, "mstar,irqs-map-range", 0, &irq_start) ||
+ of_property_read_u32_index(dn, "mstar,irqs-map-range", 1, &irq_end))
+ return -EINVAL;
+
+ cd = kzalloc(sizeof(*cd), GFP_KERNEL);
+ if (!cd)
+ return -ENOMEM;
+
+ cd->base = of_iomap(dn, 0);
+ if (!cd->base) {
+ kfree(cd);
+ return -ENOMEM;
+ }
+
+ cd->no_eoi = of_property_read_bool(dn, "mstar,intc-no-eoi");
+ raw_spin_lock_init(&cd->lock);
+ cd->irq_start = irq_start;
+ cd->nr_irqs = irq_end - irq_start + 1;
+ domain = irq_domain_add_hierarchy(domain_parent, 0, cd->nr_irqs, dn,
+ &mst_intc_domain_ops, cd);
+ if (!domain) {
+ iounmap(cd->base);
+ kfree(cd);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+IRQCHIP_DECLARE(mst_intc, "mstar,mst-intc", mst_intc_of_init);
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Actions Semi Owl SoCs SIRQ interrupt controller driver
+ *
+ * Copyright (C) 2014 Actions Semi Inc.
+ * David Liu <liuwei@actions-semi.com>
+ *
+ * Author: Parthiban Nallathambi <pn@denx.de>
+ * Author: Saravanan Sekar <sravanhome@gmail.com>
+ * Author: Cristian Ciocaltea <cristian.ciocaltea@gmail.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/interrupt.h>
+#include <linux/irqchip.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+#define NUM_SIRQ 3
+
+#define INTC_EXTCTL_PENDING BIT(0)
+#define INTC_EXTCTL_CLK_SEL BIT(4)
+#define INTC_EXTCTL_EN BIT(5)
+#define INTC_EXTCTL_TYPE_MASK GENMASK(7, 6)
+#define INTC_EXTCTL_TYPE_HIGH 0
+#define INTC_EXTCTL_TYPE_LOW BIT(6)
+#define INTC_EXTCTL_TYPE_RISING BIT(7)
+#define INTC_EXTCTL_TYPE_FALLING (BIT(6) | BIT(7))
+
+/* S500 & S700 SIRQ control register masks */
+#define INTC_EXTCTL_SIRQ0_MASK GENMASK(23, 16)
+#define INTC_EXTCTL_SIRQ1_MASK GENMASK(15, 8)
+#define INTC_EXTCTL_SIRQ2_MASK GENMASK(7, 0)
+
+/* S900 SIRQ control register offsets, relative to controller base address */
+#define INTC_EXTCTL0 0x0000
+#define INTC_EXTCTL1 0x0328
+#define INTC_EXTCTL2 0x032c
+
+struct owl_sirq_params {
+ /* INTC_EXTCTL reg shared for all three SIRQ lines */
+ bool reg_shared;
+ /* INTC_EXTCTL reg offsets relative to controller base address */
+ u16 reg_offset[NUM_SIRQ];
+};
+
+struct owl_sirq_chip_data {
+ const struct owl_sirq_params *params;
+ void __iomem *base;
+ raw_spinlock_t lock;
+ u32 ext_irqs[NUM_SIRQ];
+};
+
+/* S500 & S700 SoCs */
+static const struct owl_sirq_params owl_sirq_s500_params = {
+ .reg_shared = true,
+ .reg_offset = { 0, 0, 0 },
+};
+
+/* S900 SoC */
+static const struct owl_sirq_params owl_sirq_s900_params = {
+ .reg_shared = false,
+ .reg_offset = { INTC_EXTCTL0, INTC_EXTCTL1, INTC_EXTCTL2 },
+};
+
+static u32 owl_field_get(u32 val, u32 index)
+{
+ switch (index) {
+ case 0:
+ return FIELD_GET(INTC_EXTCTL_SIRQ0_MASK, val);
+ case 1:
+ return FIELD_GET(INTC_EXTCTL_SIRQ1_MASK, val);
+ case 2:
+ default:
+ return FIELD_GET(INTC_EXTCTL_SIRQ2_MASK, val);
+ }
+}
+
+static u32 owl_field_prep(u32 val, u32 index)
+{
+ switch (index) {
+ case 0:
+ return FIELD_PREP(INTC_EXTCTL_SIRQ0_MASK, val);
+ case 1:
+ return FIELD_PREP(INTC_EXTCTL_SIRQ1_MASK, val);
+ case 2:
+ default:
+ return FIELD_PREP(INTC_EXTCTL_SIRQ2_MASK, val);
+ }
+}
+
+static u32 owl_sirq_read_extctl(struct owl_sirq_chip_data *data, u32 index)
+{
+ u32 val;
+
+ val = readl_relaxed(data->base + data->params->reg_offset[index]);
+ if (data->params->reg_shared)
+ val = owl_field_get(val, index);
+
+ return val;
+}
+
+static void owl_sirq_write_extctl(struct owl_sirq_chip_data *data,
+ u32 extctl, u32 index)
+{
+ u32 val;
+
+ if (data->params->reg_shared) {
+ val = readl_relaxed(data->base + data->params->reg_offset[index]);
+ val &= ~owl_field_prep(0xff, index);
+ extctl = owl_field_prep(extctl, index) | val;
+ }
+
+ writel_relaxed(extctl, data->base + data->params->reg_offset[index]);
+}
+
+static void owl_sirq_clear_set_extctl(struct owl_sirq_chip_data *d,
+ u32 clear, u32 set, u32 index)
+{
+ unsigned long flags;
+ u32 val;
+
+ raw_spin_lock_irqsave(&d->lock, flags);
+ val = owl_sirq_read_extctl(d, index);
+ val &= ~clear;
+ val |= set;
+ owl_sirq_write_extctl(d, val, index);
+ raw_spin_unlock_irqrestore(&d->lock, flags);
+}
+
+static void owl_sirq_eoi(struct irq_data *data)
+{
+ struct owl_sirq_chip_data *chip_data = irq_data_get_irq_chip_data(data);
+
+ /*
+ * Software must clear external interrupt pending, when interrupt type
+ * is edge triggered, so we need per SIRQ based clearing.
+ */
+ if (!irqd_is_level_type(data))
+ owl_sirq_clear_set_extctl(chip_data, 0, INTC_EXTCTL_PENDING,
+ data->hwirq);
+
+ irq_chip_eoi_parent(data);
+}
+
+static void owl_sirq_mask(struct irq_data *data)
+{
+ struct owl_sirq_chip_data *chip_data = irq_data_get_irq_chip_data(data);
+
+ owl_sirq_clear_set_extctl(chip_data, INTC_EXTCTL_EN, 0, data->hwirq);
+ irq_chip_mask_parent(data);
+}
+
+static void owl_sirq_unmask(struct irq_data *data)
+{
+ struct owl_sirq_chip_data *chip_data = irq_data_get_irq_chip_data(data);
+
+ owl_sirq_clear_set_extctl(chip_data, 0, INTC_EXTCTL_EN, data->hwirq);
+ irq_chip_unmask_parent(data);
+}
+
+/*
+ * GIC does not handle falling edge or active low, hence SIRQ shall be
+ * programmed to convert falling edge to rising edge signal and active
+ * low to active high signal.
+ */
+static int owl_sirq_set_type(struct irq_data *data, unsigned int type)
+{
+ struct owl_sirq_chip_data *chip_data = irq_data_get_irq_chip_data(data);
+ u32 sirq_type;
+
+ switch (type) {
+ case IRQ_TYPE_LEVEL_LOW:
+ sirq_type = INTC_EXTCTL_TYPE_LOW;
+ type = IRQ_TYPE_LEVEL_HIGH;
+ break;
+ case IRQ_TYPE_LEVEL_HIGH:
+ sirq_type = INTC_EXTCTL_TYPE_HIGH;
+ break;
+ case IRQ_TYPE_EDGE_FALLING:
+ sirq_type = INTC_EXTCTL_TYPE_FALLING;
+ type = IRQ_TYPE_EDGE_RISING;
+ break;
+ case IRQ_TYPE_EDGE_RISING:
+ sirq_type = INTC_EXTCTL_TYPE_RISING;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ owl_sirq_clear_set_extctl(chip_data, INTC_EXTCTL_TYPE_MASK, sirq_type,
+ data->hwirq);
+
+ return irq_chip_set_type_parent(data, type);
+}
+
+static struct irq_chip owl_sirq_chip = {
+ .name = "owl-sirq",
+ .irq_mask = owl_sirq_mask,
+ .irq_unmask = owl_sirq_unmask,
+ .irq_eoi = owl_sirq_eoi,
+ .irq_set_type = owl_sirq_set_type,
+ .irq_retrigger = irq_chip_retrigger_hierarchy,
+#ifdef CONFIG_SMP
+ .irq_set_affinity = irq_chip_set_affinity_parent,
+#endif
+};
+
+static int owl_sirq_domain_translate(struct irq_domain *d,
+ struct irq_fwspec *fwspec,
+ unsigned long *hwirq,
+ unsigned int *type)
+{
+ if (!is_of_node(fwspec->fwnode))
+ return -EINVAL;
+
+ if (fwspec->param_count != 2 || fwspec->param[0] >= NUM_SIRQ)
+ return -EINVAL;
+
+ *hwirq = fwspec->param[0];
+ *type = fwspec->param[1];
+
+ return 0;
+}
+
+static int owl_sirq_domain_alloc(struct irq_domain *domain, unsigned int virq,
+ unsigned int nr_irqs, void *data)
+{
+ struct owl_sirq_chip_data *chip_data = domain->host_data;
+ struct irq_fwspec *fwspec = data;
+ struct irq_fwspec parent_fwspec;
+ irq_hw_number_t hwirq;
+ unsigned int type;
+ int ret;
+
+ if (WARN_ON(nr_irqs != 1))
+ return -EINVAL;
+
+ ret = owl_sirq_domain_translate(domain, fwspec, &hwirq, &type);
+ if (ret)
+ return ret;
+
+ switch (type) {
+ case IRQ_TYPE_EDGE_RISING:
+ case IRQ_TYPE_LEVEL_HIGH:
+ break;
+ case IRQ_TYPE_EDGE_FALLING:
+ type = IRQ_TYPE_EDGE_RISING;
+ break;
+ case IRQ_TYPE_LEVEL_LOW:
+ type = IRQ_TYPE_LEVEL_HIGH;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ irq_domain_set_hwirq_and_chip(domain, virq, hwirq, &owl_sirq_chip,
+ chip_data);
+
+ parent_fwspec.fwnode = domain->parent->fwnode;
+ parent_fwspec.param_count = 3;
+ parent_fwspec.param[0] = GIC_SPI;
+ parent_fwspec.param[1] = chip_data->ext_irqs[hwirq];
+ parent_fwspec.param[2] = type;
+
+ return irq_domain_alloc_irqs_parent(domain, virq, 1, &parent_fwspec);
+}
+
+static const struct irq_domain_ops owl_sirq_domain_ops = {
+ .translate = owl_sirq_domain_translate,
+ .alloc = owl_sirq_domain_alloc,
+ .free = irq_domain_free_irqs_common,
+};
+
+static int __init owl_sirq_init(const struct owl_sirq_params *params,
+ struct device_node *node,
+ struct device_node *parent)
+{
+ struct irq_domain *domain, *parent_domain;
+ struct owl_sirq_chip_data *chip_data;
+ int ret, i;
+
+ parent_domain = irq_find_host(parent);
+ if (!parent_domain) {
+ pr_err("%pOF: failed to find sirq parent domain\n", node);
+ return -ENXIO;
+ }
+
+ chip_data = kzalloc(sizeof(*chip_data), GFP_KERNEL);
+ if (!chip_data)
+ return -ENOMEM;
+
+ raw_spin_lock_init(&chip_data->lock);
+
+ chip_data->params = params;
+
+ chip_data->base = of_iomap(node, 0);
+ if (!chip_data->base) {
+ pr_err("%pOF: failed to map sirq registers\n", node);
+ ret = -ENXIO;
+ goto out_free;
+ }
+
+ for (i = 0; i < NUM_SIRQ; i++) {
+ struct of_phandle_args irq;
+
+ ret = of_irq_parse_one(node, i, &irq);
+ if (ret) {
+ pr_err("%pOF: failed to parse interrupt %d\n", node, i);
+ goto out_unmap;
+ }
+
+ if (WARN_ON(irq.args_count != 3)) {
+ ret = -EINVAL;
+ goto out_unmap;
+ }
+
+ chip_data->ext_irqs[i] = irq.args[1];
+
+ /* Set 24MHz external interrupt clock freq */
+ owl_sirq_clear_set_extctl(chip_data, 0, INTC_EXTCTL_CLK_SEL, i);
+ }
+
+ domain = irq_domain_add_hierarchy(parent_domain, 0, NUM_SIRQ, node,
+ &owl_sirq_domain_ops, chip_data);
+ if (!domain) {
+ pr_err("%pOF: failed to add domain\n", node);
+ ret = -ENOMEM;
+ goto out_unmap;
+ }
+
+ return 0;
+
+out_unmap:
+ iounmap(chip_data->base);
+out_free:
+ kfree(chip_data);
+
+ return ret;
+}
+
+static int __init owl_sirq_s500_of_init(struct device_node *node,
+ struct device_node *parent)
+{
+ return owl_sirq_init(&owl_sirq_s500_params, node, parent);
+}
+
+IRQCHIP_DECLARE(owl_sirq_s500, "actions,s500-sirq", owl_sirq_s500_of_init);
+IRQCHIP_DECLARE(owl_sirq_s700, "actions,s700-sirq", owl_sirq_s500_of_init);
+
+static int __init owl_sirq_s900_of_init(struct device_node *node,
+ struct device_node *parent)
+{
+ return owl_sirq_init(&owl_sirq_s900_params, node, parent);
+}
+
+IRQCHIP_DECLARE(owl_sirq_s900, "actions,s900-sirq", owl_sirq_s900_of_init);
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * PRU-ICSS INTC IRQChip driver for various TI SoCs
+ *
+ * Copyright (C) 2016-2020 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Author(s):
+ * Andrew F. Davis <afd@ti.com>
+ * Suman Anna <s-anna@ti.com>
+ * Grzegorz Jaszczyk <grzegorz.jaszczyk@linaro.org> for Texas Instruments
+ *
+ * Copyright (C) 2019 David Lechner <david@lechnology.com>
+ */
+
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+/*
+ * Number of host interrupts reaching the main MPU sub-system. Note that this
+ * is not the same as the total number of host interrupts supported by the PRUSS
+ * INTC instance
+ */
+#define MAX_NUM_HOST_IRQS 8
+
+/* minimum starting host interrupt number for MPU */
+#define FIRST_PRU_HOST_INT 2
+
+/* PRU_ICSS_INTC registers */
+#define PRU_INTC_REVID 0x0000
+#define PRU_INTC_CR 0x0004
+#define PRU_INTC_GER 0x0010
+#define PRU_INTC_GNLR 0x001c
+#define PRU_INTC_SISR 0x0020
+#define PRU_INTC_SICR 0x0024
+#define PRU_INTC_EISR 0x0028
+#define PRU_INTC_EICR 0x002c
+#define PRU_INTC_HIEISR 0x0034
+#define PRU_INTC_HIDISR 0x0038
+#define PRU_INTC_GPIR 0x0080
+#define PRU_INTC_SRSR(x) (0x0200 + (x) * 4)
+#define PRU_INTC_SECR(x) (0x0280 + (x) * 4)
+#define PRU_INTC_ESR(x) (0x0300 + (x) * 4)
+#define PRU_INTC_ECR(x) (0x0380 + (x) * 4)
+#define PRU_INTC_CMR(x) (0x0400 + (x) * 4)
+#define PRU_INTC_HMR(x) (0x0800 + (x) * 4)
+#define PRU_INTC_HIPIR(x) (0x0900 + (x) * 4)
+#define PRU_INTC_SIPR(x) (0x0d00 + (x) * 4)
+#define PRU_INTC_SITR(x) (0x0d80 + (x) * 4)
+#define PRU_INTC_HINLR(x) (0x1100 + (x) * 4)
+#define PRU_INTC_HIER 0x1500
+
+/* CMR register bit-field macros */
+#define CMR_EVT_MAP_MASK 0xf
+#define CMR_EVT_MAP_BITS 8
+#define CMR_EVT_PER_REG 4
+
+/* HMR register bit-field macros */
+#define HMR_CH_MAP_MASK 0xf
+#define HMR_CH_MAP_BITS 8
+#define HMR_CH_PER_REG 4
+
+/* HIPIR register bit-fields */
+#define INTC_HIPIR_NONE_HINT 0x80000000
+
+#define MAX_PRU_SYS_EVENTS 160
+#define MAX_PRU_CHANNELS 20
+
+/**
+ * struct pruss_intc_map_record - keeps track of actual mapping state
+ * @value: The currently mapped value (channel or host)
+ * @ref_count: Keeps track of number of current users of this resource
+ */
+struct pruss_intc_map_record {
+ u8 value;
+ u8 ref_count;
+};
+
+/**
+ * struct pruss_intc_match_data - match data to handle SoC variations
+ * @num_system_events: number of input system events handled by the PRUSS INTC
+ * @num_host_events: number of host events (which is equal to number of
+ * channels) supported by the PRUSS INTC
+ */
+struct pruss_intc_match_data {
+ u8 num_system_events;
+ u8 num_host_events;
+};
+
+/**
+ * struct pruss_intc - PRUSS interrupt controller structure
+ * @event_channel: current state of system event to channel mappings
+ * @channel_host: current state of channel to host mappings
+ * @irqs: kernel irq numbers corresponding to PRUSS host interrupts
+ * @base: base virtual address of INTC register space
+ * @domain: irq domain for this interrupt controller
+ * @soc_config: cached PRUSS INTC IP configuration data
+ * @dev: PRUSS INTC device pointer
+ * @lock: mutex to serialize interrupts mapping
+ */
+struct pruss_intc {
+ struct pruss_intc_map_record event_channel[MAX_PRU_SYS_EVENTS];
+ struct pruss_intc_map_record channel_host[MAX_PRU_CHANNELS];
+ unsigned int irqs[MAX_NUM_HOST_IRQS];
+ void __iomem *base;
+ struct irq_domain *domain;
+ const struct pruss_intc_match_data *soc_config;
+ struct device *dev;
+ struct mutex lock; /* PRUSS INTC lock */
+};
+
+/**
+ * struct pruss_host_irq_data - PRUSS host irq data structure
+ * @intc: PRUSS interrupt controller pointer
+ * @host_irq: host irq number
+ */
+struct pruss_host_irq_data {
+ struct pruss_intc *intc;
+ u8 host_irq;
+};
+
+static inline u32 pruss_intc_read_reg(struct pruss_intc *intc, unsigned int reg)
+{
+ return readl_relaxed(intc->base + reg);
+}
+
+static inline void pruss_intc_write_reg(struct pruss_intc *intc,
+ unsigned int reg, u32 val)
+{
+ writel_relaxed(val, intc->base + reg);
+}
+
+static void pruss_intc_update_cmr(struct pruss_intc *intc, unsigned int evt,
+ u8 ch)
+{
+ u32 idx, offset, val;
+
+ idx = evt / CMR_EVT_PER_REG;
+ offset = (evt % CMR_EVT_PER_REG) * CMR_EVT_MAP_BITS;
+
+ val = pruss_intc_read_reg(intc, PRU_INTC_CMR(idx));
+ val &= ~(CMR_EVT_MAP_MASK << offset);
+ val |= ch << offset;
+ pruss_intc_write_reg(intc, PRU_INTC_CMR(idx), val);
+
+ dev_dbg(intc->dev, "SYSEV%u -> CH%d (CMR%d 0x%08x)\n", evt, ch,
+ idx, pruss_intc_read_reg(intc, PRU_INTC_CMR(idx)));
+}
+
+static void pruss_intc_update_hmr(struct pruss_intc *intc, u8 ch, u8 host)
+{
+ u32 idx, offset, val;
+
+ idx = ch / HMR_CH_PER_REG;
+ offset = (ch % HMR_CH_PER_REG) * HMR_CH_MAP_BITS;
+
+ val = pruss_intc_read_reg(intc, PRU_INTC_HMR(idx));
+ val &= ~(HMR_CH_MAP_MASK << offset);
+ val |= host << offset;
+ pruss_intc_write_reg(intc, PRU_INTC_HMR(idx), val);
+
+ dev_dbg(intc->dev, "CH%d -> HOST%d (HMR%d 0x%08x)\n", ch, host, idx,
+ pruss_intc_read_reg(intc, PRU_INTC_HMR(idx)));
+}
+
+/**
+ * pruss_intc_map() - configure the PRUSS INTC
+ * @intc: PRUSS interrupt controller pointer
+ * @hwirq: the system event number
+ *
+ * Configures the PRUSS INTC with the provided configuration from the one parsed
+ * in the xlate function.
+ */
+static void pruss_intc_map(struct pruss_intc *intc, unsigned long hwirq)
+{
+ struct device *dev = intc->dev;
+ u8 ch, host, reg_idx;
+ u32 val;
+
+ mutex_lock(&intc->lock);
+
+ intc->event_channel[hwirq].ref_count++;
+
+ ch = intc->event_channel[hwirq].value;
+ host = intc->channel_host[ch].value;
+
+ pruss_intc_update_cmr(intc, hwirq, ch);
+
+ reg_idx = hwirq / 32;
+ val = BIT(hwirq % 32);
+
+ /* clear and enable system event */
+ pruss_intc_write_reg(intc, PRU_INTC_ESR(reg_idx), val);
+ pruss_intc_write_reg(intc, PRU_INTC_SECR(reg_idx), val);
+
+ if (++intc->channel_host[ch].ref_count == 1) {
+ pruss_intc_update_hmr(intc, ch, host);
+
+ /* enable host interrupts */
+ pruss_intc_write_reg(intc, PRU_INTC_HIEISR, host);
+ }
+
+ dev_dbg(dev, "mapped system_event = %lu channel = %d host = %d",
+ hwirq, ch, host);
+
+ mutex_unlock(&intc->lock);
+}
+
+/**
+ * pruss_intc_unmap() - unconfigure the PRUSS INTC
+ * @intc: PRUSS interrupt controller pointer
+ * @hwirq: the system event number
+ *
+ * Undo whatever was done in pruss_intc_map() for a PRU core.
+ * Mappings are reference counted, so resources are only disabled when there
+ * are no longer any users.
+ */
+static void pruss_intc_unmap(struct pruss_intc *intc, unsigned long hwirq)
+{
+ u8 ch, host, reg_idx;
+ u32 val;
+
+ mutex_lock(&intc->lock);
+
+ ch = intc->event_channel[hwirq].value;
+ host = intc->channel_host[ch].value;
+
+ if (--intc->channel_host[ch].ref_count == 0) {
+ /* disable host interrupts */
+ pruss_intc_write_reg(intc, PRU_INTC_HIDISR, host);
+
+ /* clear the map using reset value 0 */
+ pruss_intc_update_hmr(intc, ch, 0);
+ }
+
+ intc->event_channel[hwirq].ref_count--;
+ reg_idx = hwirq / 32;
+ val = BIT(hwirq % 32);
+
+ /* disable system events */
+ pruss_intc_write_reg(intc, PRU_INTC_ECR(reg_idx), val);
+ /* clear any pending status */
+ pruss_intc_write_reg(intc, PRU_INTC_SECR(reg_idx), val);
+
+ /* clear the map using reset value 0 */
+ pruss_intc_update_cmr(intc, hwirq, 0);
+
+ dev_dbg(intc->dev, "unmapped system_event = %lu channel = %d host = %d\n",
+ hwirq, ch, host);
+
+ mutex_unlock(&intc->lock);
+}
+
+static void pruss_intc_init(struct pruss_intc *intc)
+{
+ const struct pruss_intc_match_data *soc_config = intc->soc_config;
+ int num_chnl_map_regs, num_host_intr_regs, num_event_type_regs, i;
+
+ num_chnl_map_regs = DIV_ROUND_UP(soc_config->num_system_events,
+ CMR_EVT_PER_REG);
+ num_host_intr_regs = DIV_ROUND_UP(soc_config->num_host_events,
+ HMR_CH_PER_REG);
+ num_event_type_regs = DIV_ROUND_UP(soc_config->num_system_events, 32);
+
+ /*
+ * configure polarity (SIPR register) to active high and
+ * type (SITR register) to level interrupt for all system events
+ */
+ for (i = 0; i < num_event_type_regs; i++) {
+ pruss_intc_write_reg(intc, PRU_INTC_SIPR(i), 0xffffffff);
+ pruss_intc_write_reg(intc, PRU_INTC_SITR(i), 0);
+ }
+
+ /* clear all interrupt channel map registers, 4 events per register */
+ for (i = 0; i < num_chnl_map_regs; i++)
+ pruss_intc_write_reg(intc, PRU_INTC_CMR(i), 0);
+
+ /* clear all host interrupt map registers, 4 channels per register */
+ for (i = 0; i < num_host_intr_regs; i++)
+ pruss_intc_write_reg(intc, PRU_INTC_HMR(i), 0);
+
+ /* global interrupt enable */
+ pruss_intc_write_reg(intc, PRU_INTC_GER, 1);
+}
+
+static void pruss_intc_irq_ack(struct irq_data *data)
+{
+ struct pruss_intc *intc = irq_data_get_irq_chip_data(data);
+ unsigned int hwirq = data->hwirq;
+
+ pruss_intc_write_reg(intc, PRU_INTC_SICR, hwirq);
+}
+
+static void pruss_intc_irq_mask(struct irq_data *data)
+{
+ struct pruss_intc *intc = irq_data_get_irq_chip_data(data);
+ unsigned int hwirq = data->hwirq;
+
+ pruss_intc_write_reg(intc, PRU_INTC_EICR, hwirq);
+}
+
+static void pruss_intc_irq_unmask(struct irq_data *data)
+{
+ struct pruss_intc *intc = irq_data_get_irq_chip_data(data);
+ unsigned int hwirq = data->hwirq;
+
+ pruss_intc_write_reg(intc, PRU_INTC_EISR, hwirq);
+}
+
+static int pruss_intc_irq_reqres(struct irq_data *data)
+{
+ if (!try_module_get(THIS_MODULE))
+ return -ENODEV;
+
+ return 0;
+}
+
+static void pruss_intc_irq_relres(struct irq_data *data)
+{
+ module_put(THIS_MODULE);
+}
+
+static int pruss_intc_irq_get_irqchip_state(struct irq_data *data,
+ enum irqchip_irq_state which,
+ bool *state)
+{
+ struct pruss_intc *intc = irq_data_get_irq_chip_data(data);
+ u32 reg, mask, srsr;
+
+ if (which != IRQCHIP_STATE_PENDING)
+ return -EINVAL;
+
+ reg = PRU_INTC_SRSR(data->hwirq / 32);
+ mask = BIT(data->hwirq % 32);
+
+ srsr = pruss_intc_read_reg(intc, reg);
+
+ *state = !!(srsr & mask);
+
+ return 0;
+}
+
+static int pruss_intc_irq_set_irqchip_state(struct irq_data *data,
+ enum irqchip_irq_state which,
+ bool state)
+{
+ struct pruss_intc *intc = irq_data_get_irq_chip_data(data);
+
+ if (which != IRQCHIP_STATE_PENDING)
+ return -EINVAL;
+
+ if (state)
+ pruss_intc_write_reg(intc, PRU_INTC_SISR, data->hwirq);
+ else
+ pruss_intc_write_reg(intc, PRU_INTC_SICR, data->hwirq);
+
+ return 0;
+}
+
+static struct irq_chip pruss_irqchip = {
+ .name = "pruss-intc",
+ .irq_ack = pruss_intc_irq_ack,
+ .irq_mask = pruss_intc_irq_mask,
+ .irq_unmask = pruss_intc_irq_unmask,
+ .irq_request_resources = pruss_intc_irq_reqres,
+ .irq_release_resources = pruss_intc_irq_relres,
+ .irq_get_irqchip_state = pruss_intc_irq_get_irqchip_state,
+ .irq_set_irqchip_state = pruss_intc_irq_set_irqchip_state,
+};
+
+static int pruss_intc_validate_mapping(struct pruss_intc *intc, int event,
+ int channel, int host)
+{
+ struct device *dev = intc->dev;
+ int ret = 0;
+
+ mutex_lock(&intc->lock);
+
+ /* check if sysevent already assigned */
+ if (intc->event_channel[event].ref_count > 0 &&
+ intc->event_channel[event].value != channel) {
+ dev_err(dev, "event %d (req. ch %d) already assigned to channel %d\n",
+ event, channel, intc->event_channel[event].value);
+ ret = -EBUSY;
+ goto unlock;
+ }
+
+ /* check if channel already assigned */
+ if (intc->channel_host[channel].ref_count > 0 &&
+ intc->channel_host[channel].value != host) {
+ dev_err(dev, "channel %d (req. host %d) already assigned to host %d\n",
+ channel, host, intc->channel_host[channel].value);
+ ret = -EBUSY;
+ goto unlock;
+ }
+
+ intc->event_channel[event].value = channel;
+ intc->channel_host[channel].value = host;
+
+unlock:
+ mutex_unlock(&intc->lock);
+ return ret;
+}
+
+static int
+pruss_intc_irq_domain_xlate(struct irq_domain *d, struct device_node *node,
+ const u32 *intspec, unsigned int intsize,
+ unsigned long *out_hwirq, unsigned int *out_type)
+{
+ struct pruss_intc *intc = d->host_data;
+ struct device *dev = intc->dev;
+ int ret, sys_event, channel, host;
+
+ if (intsize < 3)
+ return -EINVAL;
+
+ sys_event = intspec[0];
+ if (sys_event < 0 || sys_event >= intc->soc_config->num_system_events) {
+ dev_err(dev, "%d is not valid event number\n", sys_event);
+ return -EINVAL;
+ }
+
+ channel = intspec[1];
+ if (channel < 0 || channel >= intc->soc_config->num_host_events) {
+ dev_err(dev, "%d is not valid channel number", channel);
+ return -EINVAL;
+ }
+
+ host = intspec[2];
+ if (host < 0 || host >= intc->soc_config->num_host_events) {
+ dev_err(dev, "%d is not valid host irq number\n", host);
+ return -EINVAL;
+ }
+
+ /* check if requested sys_event was already mapped, if so validate it */
+ ret = pruss_intc_validate_mapping(intc, sys_event, channel, host);
+ if (ret)
+ return ret;
+
+ *out_hwirq = sys_event;
+ *out_type = IRQ_TYPE_LEVEL_HIGH;
+
+ return 0;
+}
+
+static int pruss_intc_irq_domain_map(struct irq_domain *d, unsigned int virq,
+ irq_hw_number_t hw)
+{
+ struct pruss_intc *intc = d->host_data;
+
+ pruss_intc_map(intc, hw);
+
+ irq_set_chip_data(virq, intc);
+ irq_set_chip_and_handler(virq, &pruss_irqchip, handle_level_irq);
+
+ return 0;
+}
+
+static void pruss_intc_irq_domain_unmap(struct irq_domain *d, unsigned int virq)
+{
+ struct pruss_intc *intc = d->host_data;
+ unsigned long hwirq = irqd_to_hwirq(irq_get_irq_data(virq));
+
+ irq_set_chip_and_handler(virq, NULL, NULL);
+ irq_set_chip_data(virq, NULL);
+ pruss_intc_unmap(intc, hwirq);
+}
+
+static const struct irq_domain_ops pruss_intc_irq_domain_ops = {
+ .xlate = pruss_intc_irq_domain_xlate,
+ .map = pruss_intc_irq_domain_map,
+ .unmap = pruss_intc_irq_domain_unmap,
+};
+
+static void pruss_intc_irq_handler(struct irq_desc *desc)
+{
+ unsigned int irq = irq_desc_get_irq(desc);
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct pruss_host_irq_data *host_irq_data = irq_get_handler_data(irq);
+ struct pruss_intc *intc = host_irq_data->intc;
+ u8 host_irq = host_irq_data->host_irq + FIRST_PRU_HOST_INT;
+
+ chained_irq_enter(chip, desc);
+
+ while (true) {
+ u32 hipir;
+ unsigned int virq;
+ int hwirq;
+
+ /* get highest priority pending PRUSS system event */
+ hipir = pruss_intc_read_reg(intc, PRU_INTC_HIPIR(host_irq));
+ if (hipir & INTC_HIPIR_NONE_HINT)
+ break;
+
+ hwirq = hipir & GENMASK(9, 0);
+ virq = irq_find_mapping(intc->domain, hwirq);
+
+ /*
+ * NOTE: manually ACK any system events that do not have a
+ * handler mapped yet
+ */
+ if (WARN_ON_ONCE(!virq))
+ pruss_intc_write_reg(intc, PRU_INTC_SICR, hwirq);
+ else
+ generic_handle_irq(virq);
+ }
+
+ chained_irq_exit(chip, desc);
+}
+
+static const char * const irq_names[MAX_NUM_HOST_IRQS] = {
+ "host_intr0", "host_intr1", "host_intr2", "host_intr3",
+ "host_intr4", "host_intr5", "host_intr6", "host_intr7",
+};
+
+static int pruss_intc_probe(struct platform_device *pdev)
+{
+ const struct pruss_intc_match_data *data;
+ struct device *dev = &pdev->dev;
+ struct pruss_intc *intc;
+ struct pruss_host_irq_data *host_data;
+ int i, irq, ret;
+ u8 max_system_events, irqs_reserved = 0;
+
+ data = of_device_get_match_data(dev);
+ if (!data)
+ return -ENODEV;
+
+ max_system_events = data->num_system_events;
+
+ intc = devm_kzalloc(dev, sizeof(*intc), GFP_KERNEL);
+ if (!intc)
+ return -ENOMEM;
+
+ intc->soc_config = data;
+ intc->dev = dev;
+ platform_set_drvdata(pdev, intc);
+
+ intc->base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(intc->base))
+ return PTR_ERR(intc->base);
+
+ ret = of_property_read_u8(dev->of_node, "ti,irqs-reserved",
+ &irqs_reserved);
+
+ /*
+ * The irqs-reserved is used only for some SoC's therefore not having
+ * this property is still valid
+ */
+ if (ret < 0 && ret != -EINVAL)
+ return ret;
+
+ pruss_intc_init(intc);
+
+ mutex_init(&intc->lock);
+
+ intc->domain = irq_domain_add_linear(dev->of_node, max_system_events,
+ &pruss_intc_irq_domain_ops, intc);
+ if (!intc->domain)
+ return -ENOMEM;
+
+ for (i = 0; i < MAX_NUM_HOST_IRQS; i++) {
+ if (irqs_reserved & BIT(i))
+ continue;
+
+ irq = platform_get_irq_byname(pdev, irq_names[i]);
+ if (irq <= 0) {
+ ret = (irq == 0) ? -EINVAL : irq;
+ goto fail_irq;
+ }
+
+ intc->irqs[i] = irq;
+
+ host_data = devm_kzalloc(dev, sizeof(*host_data), GFP_KERNEL);
+ if (!host_data) {
+ ret = -ENOMEM;
+ goto fail_irq;
+ }
+
+ host_data->intc = intc;
+ host_data->host_irq = i;
+
+ irq_set_handler_data(irq, host_data);
+ irq_set_chained_handler(irq, pruss_intc_irq_handler);
+ }
+
+ return 0;
+
+fail_irq:
+ while (--i >= 0) {
+ if (intc->irqs[i])
+ irq_set_chained_handler_and_data(intc->irqs[i], NULL,
+ NULL);
+ }
+
+ irq_domain_remove(intc->domain);
+
+ return ret;
+}
+
+static int pruss_intc_remove(struct platform_device *pdev)
+{
+ struct pruss_intc *intc = platform_get_drvdata(pdev);
+ u8 max_system_events = intc->soc_config->num_system_events;
+ unsigned int hwirq;
+ int i;
+
+ for (i = 0; i < MAX_NUM_HOST_IRQS; i++) {
+ if (intc->irqs[i])
+ irq_set_chained_handler_and_data(intc->irqs[i], NULL,
+ NULL);
+ }
+
+ for (hwirq = 0; hwirq < max_system_events; hwirq++)
+ irq_dispose_mapping(irq_find_mapping(intc->domain, hwirq));
+
+ irq_domain_remove(intc->domain);
+
+ return 0;
+}
+
+static const struct pruss_intc_match_data pruss_intc_data = {
+ .num_system_events = 64,
+ .num_host_events = 10,
+};
+
+static const struct pruss_intc_match_data icssg_intc_data = {
+ .num_system_events = 160,
+ .num_host_events = 20,
+};
+
+static const struct of_device_id pruss_intc_of_match[] = {
+ {
+ .compatible = "ti,pruss-intc",
+ .data = &pruss_intc_data,
+ },
+ {
+ .compatible = "ti,icssg-intc",
+ .data = &icssg_intc_data,
+ },
+ { /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, pruss_intc_of_match);
+
+static struct platform_driver pruss_intc_driver = {
+ .driver = {
+ .name = "pruss-intc",
+ .of_match_table = pruss_intc_of_match,
+ .suppress_bind_attrs = true,
+ },
+ .probe = pruss_intc_probe,
+ .remove = pruss_intc_remove,
+};
+module_platform_driver(pruss_intc_driver);
+
+MODULE_AUTHOR("Andrew F. Davis <afd@ti.com>");
+MODULE_AUTHOR("Suman Anna <s-anna@ti.com>");
+MODULE_AUTHOR("Grzegorz Jaszczyk <grzegorz.jaszczyk@linaro.org>");
+MODULE_DESCRIPTION("TI PRU-ICSS INTC Driver");
+MODULE_LICENSE("GPL v2");
struct irq_fwspec parent_fwspec;
struct device_node *parent_node;
unsigned int parent_virq;
- u16 vint_id, p_hwirq;
- int ret;
+ int p_hwirq, ret;
+ u16 vint_id;
vint_id = ti_sci_get_free_resource(inta->vint);
if (vint_id == TI_SCI_RESOURCE_NULL)
inta->pdev = pdev;
inta->sci = devm_ti_sci_get_by_phandle(dev, "ti,sci");
- if (IS_ERR(inta->sci)) {
- ret = PTR_ERR(inta->sci);
- if (ret != -EPROBE_DEFER)
- dev_err(dev, "ti,sci read fail %d\n", ret);
- inta->sci = NULL;
- return ret;
- }
+ if (IS_ERR(inta->sci))
+ return dev_err_probe(dev, PTR_ERR(inta->sci),
+ "ti,sci read fail\n");
ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id", &inta->ti_sci_id);
if (ret) {
struct ti_sci_intr_irq_domain *intr = domain->host_data;
struct device_node *parent_node;
struct irq_fwspec fwspec;
- u16 out_irq, p_hwirq;
- int err = 0;
+ int p_hwirq, err = 0;
+ u16 out_irq;
out_irq = ti_sci_get_free_resource(intr->out_irqs);
if (out_irq == TI_SCI_RESOURCE_NULL)
}
intr->sci = devm_ti_sci_get_by_phandle(dev, "ti,sci");
- if (IS_ERR(intr->sci)) {
- ret = PTR_ERR(intr->sci);
- if (ret != -EPROBE_DEFER)
- dev_err(dev, "ti,sci read fail %d\n", ret);
- intr->sci = NULL;
- return ret;
- }
+ if (IS_ERR(intr->sci))
+ return dev_err_probe(dev, PTR_ERR(intr->sci),
+ "ti,sci read fail\n");
ret = of_property_read_u32(dev_of_node(dev), "ti,sci-dev-id",
&intr->ti_sci_id);
.irq_set_type = qcom_pdc_gic_set_type,
.flags = IRQCHIP_MASK_ON_SUSPEND |
IRQCHIP_SET_TYPE_MASKED |
- IRQCHIP_SKIP_SET_WAKE,
+ IRQCHIP_SKIP_SET_WAKE |
+ IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND,
.irq_set_vcpu_affinity = irq_chip_set_vcpu_affinity_parent,
.irq_set_affinity = irq_chip_set_affinity_parent,
};
static int pdc_setup_pin_mapping(struct device_node *np)
{
- int ret, n;
+ int ret, n, i;
+ u32 irq_index, reg_index, val;
n = of_property_count_elems_of_size(np, "qcom,pdc-ranges", sizeof(u32));
if (n <= 0 || n % 3)
&pdc_region[n].cnt);
if (ret)
return ret;
+
+ for (i = 0; i < pdc_region[n].cnt; i++) {
+ reg_index = (i + pdc_region[n].pin_base) >> 5;
+ irq_index = (i + pdc_region[n].pin_base) & 0x1f;
+ val = pdc_reg_read(IRQ_ENABLE_BANK, reg_index);
+ val &= ~BIT(irq_index);
+ pdc_reg_write(IRQ_ENABLE_BANK, reg_index, val);
+ }
}
return 0;
.name = "linear",
.version = {1, 4, 0},
#ifdef CONFIG_BLK_DEV_ZONED
- .features = DM_TARGET_PASSES_INTEGRITY | DM_TARGET_ZONED_HM,
+ .features = DM_TARGET_PASSES_INTEGRITY | DM_TARGET_NOWAIT |
+ DM_TARGET_ZONED_HM,
.report_zones = linear_report_zones,
#else
- .features = DM_TARGET_PASSES_INTEGRITY,
+ .features = DM_TARGET_PASSES_INTEGRITY | DM_TARGET_NOWAIT,
#endif
.module = THIS_MODULE,
.ctr = linear_ctr,
struct request_queue *q = bdev_get_queue(bdev);
/* request-based cannot stack on partitions! */
- if (bdev != bdev->bd_contains)
+ if (bdev_is_partition(bdev))
return false;
return queue_is_mq(q);
return true;
}
+static int device_not_nowait_capable(struct dm_target *ti, struct dm_dev *dev,
+ sector_t start, sector_t len, void *data)
+{
+ struct request_queue *q = bdev_get_queue(dev->bdev);
+
+ return q && !blk_queue_nowait(q);
+}
+
+static bool dm_table_supports_nowait(struct dm_table *t)
+{
+ struct dm_target *ti;
+ unsigned i = 0;
+
+ while (i < dm_table_get_num_targets(t)) {
+ ti = dm_table_get_target(t, i++);
+
+ if (!dm_target_supports_nowait(ti->type))
+ return false;
+
+ if (!ti->type->iterate_devices ||
+ ti->type->iterate_devices(ti, device_not_nowait_capable, NULL))
+ return false;
+ }
+
+ return true;
+}
+
static int device_not_discard_capable(struct dm_target *ti, struct dm_dev *dev,
sector_t start, sector_t len, void *data)
{
*/
q->limits = *limits;
+ if (dm_table_supports_nowait(t))
+ blk_queue_flag_set(QUEUE_FLAG_NOWAIT, q);
+ else
+ blk_queue_flag_clear(QUEUE_FLAG_NOWAIT, q);
+
if (!dm_table_supports_discards(t)) {
blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q);
/* Must also clear discard limits... */
#define pmem_assign(dest, src) ((dest) = (src))
#endif
-#if defined(__HAVE_ARCH_MEMCPY_MCSAFE) && defined(DM_WRITECACHE_HAS_PMEM)
+#if IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC) && defined(DM_WRITECACHE_HAS_PMEM)
#define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
#endif
}
wc->freelist_size = 0;
- r = memcpy_mcsafe(&sb_seq_count, &sb(wc)->seq_count, sizeof(uint64_t));
+ r = copy_mc_to_kernel(&sb_seq_count, &sb(wc)->seq_count,
+ sizeof(uint64_t));
if (r) {
writecache_error(wc, r, "hardware memory error when reading superblock: %d", r);
sb_seq_count = cpu_to_le64(0);
e->seq_count = -1;
continue;
}
- r = memcpy_mcsafe(&wme, memory_entry(wc, e), sizeof(struct wc_memory_entry));
+ r = copy_mc_to_kernel(&wme, memory_entry(wc, e),
+ sizeof(struct wc_memory_entry));
if (r) {
writecache_error(wc, r, "hardware memory error when reading metadata entry %lu: %d",
(unsigned long)b, r);
if (rw == READ) {
int r;
- r = memcpy_mcsafe(buf, data, size);
+ r = copy_mc_to_kernel(buf, data, size);
flush_dcache_page(bio_page(bio));
if (unlikely(r)) {
writecache_error(wc, r, "hardware memory error when reading data: %d", r);
}
}
- r = memcpy_mcsafe(&s, sb(wc), sizeof(struct wc_memory_superblock));
+ r = copy_mc_to_kernel(&s, sb(wc), sizeof(struct wc_memory_superblock));
if (r) {
ti->error = "Hardware memory error when reading superblock";
goto bad;
ti->error = "Unable to initialize device";
goto bad;
}
- r = memcpy_mcsafe(&s, sb(wc), sizeof(struct wc_memory_superblock));
+ r = copy_mc_to_kernel(&s, sb(wc),
+ sizeof(struct wc_memory_superblock));
if (r) {
ti->error = "Hardware memory error when reading superblock";
goto bad;
sector_t sector, unsigned len)
{
struct bio *clone = &tio->clone;
+ int r;
__bio_clone_fast(clone, bio);
- bio_crypt_clone(clone, bio, GFP_NOIO);
+ r = bio_crypt_clone(clone, bio, GFP_NOIO);
+ if (r < 0)
+ return r;
if (bio_integrity(bio)) {
- int r;
-
if (unlikely(!dm_target_has_integrity(tio->ti->type) &&
!dm_target_passes_integrity(tio->ti->type))) {
DMWARN("%s: the target %s doesn't support integrity data.",
return ret;
}
-static void dm_queue_split(struct mapped_device *md, struct dm_target *ti, struct bio **bio)
-{
- unsigned len, sector_count;
-
- sector_count = bio_sectors(*bio);
- len = min_t(sector_t, max_io_len((*bio)->bi_iter.bi_sector, ti), sector_count);
-
- if (sector_count > len) {
- struct bio *split = bio_split(*bio, len, GFP_NOIO, &md->queue->bio_split);
-
- bio_chain(split, *bio);
- trace_block_split(md->queue, split, (*bio)->bi_iter.bi_sector);
- submit_bio_noacct(*bio);
- *bio = split;
- }
-}
-
static blk_qc_t dm_process_bio(struct mapped_device *md,
struct dm_table *map, struct bio *bio)
{
}
/*
- * If in ->queue_bio we need to use blk_queue_split(), otherwise
+ * If in ->submit_bio we need to use blk_queue_split(), otherwise
* queue_limits for abnormal requests (e.g. discard, writesame, etc)
* won't be imposed.
+ * If called from dm_wq_work() for deferred bio processing, bio
+ * was already handled by following code with previous ->submit_bio.
*/
if (current->bio_list) {
if (is_abnormal_io(bio))
blk_queue_split(&bio);
- else
- dm_queue_split(md, ti, &bio);
+ /* regular IO is split by __split_and_process_bio */
}
if (dm_get_md_type(md) == DM_TYPE_NVME_BIO_BASED)
return __process_bio(md, map, bio, ti);
- else
- return __split_and_process_bio(md, map, bio);
+ return __split_and_process_bio(md, map, bio);
}
static blk_qc_t dm_submit_bio(struct bio *bio)
if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
dm_put_live_table(md, srcu_idx);
- if (!(bio->bi_opf & REQ_RAHEAD))
+ if (bio->bi_opf & REQ_NOWAIT)
+ bio_wouldblock_error(bio);
+ else if (!(bio->bi_opf & REQ_RAHEAD))
queue_io(md, bio);
else
bio_io_error(bio);
test_bit(Journal, &rdev2->flags) ||
rdev2->raid_disk == -1)
continue;
- if (rdev->bdev->bd_contains ==
- rdev2->bdev->bd_contains) {
+ if (rdev->bdev->bd_disk == rdev2->bdev->bd_disk) {
rcu_read_unlock();
return 1;
}
rdev_for_each(rdev, mddev)
rdev_for_each(rdev2, mddev) {
if (rdev < rdev2 &&
- rdev->bdev->bd_contains ==
- rdev2->bdev->bd_contains) {
+ rdev->bdev->bd_disk ==
+ rdev2->bdev->bd_disk) {
pr_warn("%s: WARNING: %s appears to be on the same physical disk as %s.\n",
mdname(mddev),
bdevname(rdev->bdev,b),
idle = 1;
rcu_read_lock();
rdev_for_each_rcu(rdev, mddev) {
- struct gendisk *disk = rdev->bdev->bd_contains->bd_disk;
+ struct gendisk *disk = rdev->bdev->bd_disk;
curr_events = (int)part_stat_read_accum(&disk->part0, sectors) -
atomic_read(&disk->sync_io);
/* sync IO will cause sync_io to increase before the disk_stats
static inline void md_sync_acct(struct block_device *bdev, unsigned long nr_sectors)
{
- atomic_add(nr_sectors, &bdev->bd_contains->bd_disk->sync_io);
+ atomic_add(nr_sectors, &bdev->bd_disk->sync_io);
}
static inline void md_sync_acct_bio(struct bio *bio, unsigned long nr_sectors)
struct cec_data *data;
bool is_raw = msg_is_raw(msg);
+ if (adap->devnode.unregistered)
+ return -ENODEV;
+
msg->rx_ts = 0;
msg->tx_ts = 0;
msg->rx_status = 0;
if (WARN_ON(!msg->len || msg->len > CEC_MAX_MSG_SIZE))
return;
+ if (adap->devnode.unregistered)
+ return;
+
/*
* Some CEC adapters will receive the messages that they transmitted.
* This test filters out those messages by checking if we are the
/* Cancel the pending timeout work */
if (!cancel_delayed_work(&data->work)) {
mutex_unlock(&adap->lock);
- flush_scheduled_work();
+ cancel_delayed_work_sync(&data->work);
mutex_lock(&adap->lock);
}
/*
*/
if (!adap->passthrough && from_unregistered)
return 0;
- /* Fall through */
+ fallthrough;
case CEC_MSG_GIVE_DEVICE_VENDOR_ID:
case CEC_MSG_GIVE_FEATURES:
case CEC_MSG_GIVE_PHYSICAL_ADDR:
adap->rc->allowed_protocols = RC_PROTO_BIT_CEC;
adap->rc->priv = adap;
adap->rc->map_name = RC_MAP_CEC;
- adap->rc->timeout = MS_TO_NS(550);
+ adap->rc->timeout = MS_TO_US(550);
#endif
return adap;
}
if (!top_cec_dir)
return 0;
- adap->cec_dir = debugfs_create_dir(dev_name(&adap->devnode.dev), top_cec_dir);
- if (IS_ERR_OR_NULL(adap->cec_dir)) {
- pr_warn("cec-%s: Failed to create debugfs dir\n", adap->name);
- return 0;
- }
- adap->status_file = debugfs_create_devm_seqfile(&adap->devnode.dev,
- "status", adap->cec_dir, cec_adap_status);
- if (IS_ERR_OR_NULL(adap->status_file)) {
- pr_warn("cec-%s: Failed to create status file\n", adap->name);
- debugfs_remove_recursive(adap->cec_dir);
- adap->cec_dir = NULL;
- return 0;
- }
+ adap->cec_dir = debugfs_create_dir(dev_name(&adap->devnode.dev),
+ top_cec_dir);
+
+ debugfs_create_devm_seqfile(&adap->devnode.dev, "status", adap->cec_dir,
+ cec_adap_status);
+
if (!adap->ops->error_inj_show || !adap->ops->error_inj_parse_line)
return 0;
- adap->error_inj_file = debugfs_create_file("error-inj", 0644,
- adap->cec_dir, adap,
- &cec_error_inj_fops);
- if (IS_ERR_OR_NULL(adap->error_inj_file))
- pr_warn("cec-%s: Failed to create error-inj file\n",
- adap->name);
+ debugfs_create_file("error-inj", 0644, adap->cec_dir, adap,
+ &cec_error_inj_fops);
#endif
return 0;
}
{
if (IS_ERR_OR_NULL(adap))
return;
- kthread_stop(adap->kthread);
if (adap->kthread_config)
kthread_stop(adap->kthread_config);
+ kthread_stop(adap->kthread);
if (adap->ops->adap_free)
adap->ops->adap_free(adap);
#ifdef CONFIG_MEDIA_CEC_RC
wake_up_interruptible(&pin->kthread_waitq);
break;
}
- /* fall through */
+ fallthrough;
case CEC_ST_TX_DATA_BIT_0_HIGH:
case CEC_ST_TX_DATA_BIT_0_HIGH_SHORT:
case CEC_ST_TX_DATA_BIT_0_HIGH_LONG:
wake_up_interruptible(&pin->kthread_waitq);
break;
}
- /* fall through */
+ fallthrough;
case CEC_ST_TX_DATA_BIT_HIGH_CUSTOM:
if (tx_last_bit(pin)) {
/* Error Injection: just stop sending after this bit */
break;
}
pin->tx_bit++;
- /* fall through */
+ fallthrough;
case CEC_ST_TX_START_BIT_HIGH:
case CEC_ST_TX_START_BIT_HIGH_SHORT:
case CEC_ST_TX_START_BIT_HIGH_LONG:
cec->ir->allowed_protocols = RC_PROTO_BIT_RC5;
cec->ir->priv = cec;
cec->ir->map_name = RC_MAP_HAUPPAUGE;
- cec->ir->timeout = MS_TO_NS(100);
+ cec->ir->timeout = MS_TO_US(100);
/* Clear the status register */
status = smb_rd16(SECOCEC_STATUS_REG_1, &val);
pulse8->new_rx_msg[0] = pulse8->buf[1];
break;
}
- /* fall through */
+ fallthrough;
case MSGCODE_FRAME_DATA:
if (pulse8->new_rx_msg_len < CEC_MAX_MSG_SIZE)
pulse8->new_rx_msg[pulse8->new_rx_msg_len++] =
struct page *pg;
int i;
- sglist = kcalloc(nr_pages, sizeof(struct scatterlist), GFP_KERNEL);
+ sglist = kmalloc_array(nr_pages, sizeof(struct scatterlist), GFP_KERNEL);
if (NULL == sglist)
return NULL;
sg_init_table(sglist, nr_pages);
.board_cfg.rf_switch_uhf = 17,
},
[SMS1XXX_BOARD_HAUPPAUGE_TIGER_MINICARD_R2] = {
- .name = "Hauppauge WinTV MiniCard",
+ .name = "Hauppauge WinTV MiniCard Rev 2",
.type = SMS_NOVA_B0,
.fw[DEVICE_MODE_DVBT_BDA] = SMS_FW_DVBT_HCW_55XXX,
.default_mode = DEVICE_MODE_DVBT_BDA,
for (i = 0; i < len >> 2; i++) {
struct ir_raw_event ev = {
- .duration = abs(samples[i]) * 1000, /* Convert to ns */
+ .duration = abs(samples[i]),
.pulse = (samples[i] > 0) ? false : true
};
return -ENOMEM;
coredev->ir.controller = 0; /* Todo: vega/nova SPI number */
- coredev->ir.timeout = IR_DEFAULT_TIMEOUT;
+ coredev->ir.timeout = US_TO_NS(IR_DEFAULT_TIMEOUT);
pr_debug("IR port %d, timeout %d ms\n",
coredev->ir.controller, coredev->ir.timeout);
}
EXPORT_SYMBOL(vb2_verify_memory_type);
-static void set_queue_consistency(struct vb2_queue *q, bool consistent_mem)
-{
- q->dma_attrs &= ~DMA_ATTR_NON_CONSISTENT;
-
- if (!vb2_queue_allows_cache_hints(q))
- return;
- if (!consistent_mem)
- q->dma_attrs |= DMA_ATTR_NON_CONSISTENT;
-}
-
-static bool verify_consistency_attr(struct vb2_queue *q, bool consistent_mem)
-{
- bool queue_is_consistent = !(q->dma_attrs & DMA_ATTR_NON_CONSISTENT);
-
- if (consistent_mem != queue_is_consistent) {
- dprintk(q, 1, "memory consistency model mismatch\n");
- return false;
- }
- return true;
-}
-
int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory,
- unsigned int flags, unsigned int *count)
+ unsigned int *count)
{
unsigned int num_buffers, allocated_buffers, num_planes = 0;
unsigned plane_sizes[VB2_MAX_PLANES] = { };
- bool consistent_mem = true;
unsigned int i;
int ret;
- if (flags & V4L2_FLAG_MEMORY_NON_CONSISTENT)
- consistent_mem = false;
-
if (q->streaming) {
dprintk(q, 1, "streaming active\n");
return -EBUSY;
}
if (*count == 0 || q->num_buffers != 0 ||
- (q->memory != VB2_MEMORY_UNKNOWN && q->memory != memory) ||
- !verify_consistency_attr(q, consistent_mem)) {
+ (q->memory != VB2_MEMORY_UNKNOWN && q->memory != memory)) {
/*
* We already have buffers allocated, so first check if they
* are not in use and can be freed.
num_buffers = min_t(unsigned int, num_buffers, VB2_MAX_FRAME);
memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
q->memory = memory;
- set_queue_consistency(q, consistent_mem);
/*
* Ask the driver how many buffers and planes per buffer it requires.
EXPORT_SYMBOL_GPL(vb2_core_reqbufs);
int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory,
- unsigned int flags, unsigned int *count,
+ unsigned int *count,
unsigned int requested_planes,
const unsigned int requested_sizes[])
{
unsigned int num_planes = 0, num_buffers, allocated_buffers;
unsigned plane_sizes[VB2_MAX_PLANES] = { };
- bool consistent_mem = true;
int ret;
- if (flags & V4L2_FLAG_MEMORY_NON_CONSISTENT)
- consistent_mem = false;
-
if (q->num_buffers == VB2_MAX_FRAME) {
dprintk(q, 1, "maximum number of buffers already allocated\n");
return -ENOBUFS;
}
memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
q->memory = memory;
- set_queue_consistency(q, consistent_mem);
q->waiting_for_buffers = !q->is_output;
} else {
if (q->memory != memory) {
dprintk(q, 1, "memory model mismatch\n");
return -EINVAL;
}
- if (!verify_consistency_attr(q, consistent_mem))
- return -EINVAL;
}
num_buffers = min(*count, VB2_MAX_FRAME - q->num_buffers);
fileio->memory = VB2_MEMORY_MMAP;
fileio->type = q->type;
q->fileio = fileio;
- ret = vb2_core_reqbufs(q, fileio->memory, 0, &fileio->count);
+ ret = vb2_core_reqbufs(q, fileio->memory, &fileio->count);
if (ret)
goto err_kfree;
err_reqbufs:
fileio->count = 0;
- vb2_core_reqbufs(q, fileio->memory, 0, &fileio->count);
+ vb2_core_reqbufs(q, fileio->memory, &fileio->count);
err_kfree:
q->fileio = NULL;
vb2_core_streamoff(q, q->type);
q->fileio = NULL;
fileio->count = 0;
- vb2_core_reqbufs(q, fileio->memory, 0, &fileio->count);
+ vb2_core_reqbufs(q, fileio->memory, &fileio->count);
kfree(fileio);
dprintk(q, 3, "file io emulator closed\n");
}
struct dma_buf_attachment *db_attach;
};
-static inline bool vb2_dc_buffer_consistent(unsigned long attr)
-{
- return !(attr & DMA_ATTR_NON_CONSISTENT);
-}
-
/*********************************************/
/* scatterlist table functions */
/*********************************************/
unsigned int i;
unsigned long size = 0;
- for_each_sg(sgt->sgl, s, sgt->nents, i) {
+ for_each_sgtable_dma_sg(sgt, s, i) {
if (sg_dma_address(s) != expected)
break;
- expected = sg_dma_address(s) + sg_dma_len(s);
+ expected += sg_dma_len(s);
size += sg_dma_len(s);
}
return size;
if (!sgt)
return;
- dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->orig_nents,
- buf->dma_dir);
+ dma_sync_sgtable_for_device(buf->dev, sgt, buf->dma_dir);
}
static void vb2_dc_finish(void *buf_priv)
if (!sgt)
return;
- dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir);
+ dma_sync_sgtable_for_cpu(buf->dev, sgt, buf->dma_dir);
}
/*********************************************/
* memory locations do not require any explicit cache
* maintenance prior or after being used by the device.
*/
- dma_unmap_sg_attrs(db_attach->dev, sgt->sgl, sgt->orig_nents,
- attach->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
+ dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir,
+ DMA_ATTR_SKIP_CPU_SYNC);
sg_free_table(sgt);
kfree(attach);
db_attach->priv = NULL;
/* release any previous cache */
if (attach->dma_dir != DMA_NONE) {
- dma_unmap_sg_attrs(db_attach->dev, sgt->sgl, sgt->orig_nents,
- attach->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
+ dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir,
+ DMA_ATTR_SKIP_CPU_SYNC);
attach->dma_dir = DMA_NONE;
}
* mapping to the client with new direction, no cache sync
* required see comment in vb2_dc_dmabuf_ops_detach()
*/
- sgt->nents = dma_map_sg_attrs(db_attach->dev, sgt->sgl, sgt->orig_nents,
- dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
- if (!sgt->nents) {
+ if (dma_map_sgtable(db_attach->dev, sgt, dma_dir,
+ DMA_ATTR_SKIP_CPU_SYNC)) {
pr_err("failed to map scatterlist\n");
mutex_unlock(lock);
return ERR_PTR(-EIO);
vb2_dc_dmabuf_ops_begin_cpu_access(struct dma_buf *dbuf,
enum dma_data_direction direction)
{
- struct vb2_dc_buf *buf = dbuf->priv;
- struct sg_table *sgt = buf->dma_sgt;
-
- if (vb2_dc_buffer_consistent(buf->attrs))
- return 0;
-
- dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);
return 0;
}
vb2_dc_dmabuf_ops_end_cpu_access(struct dma_buf *dbuf,
enum dma_data_direction direction)
{
- struct vb2_dc_buf *buf = dbuf->priv;
- struct sg_table *sgt = buf->dma_sgt;
-
- if (vb2_dc_buffer_consistent(buf->attrs))
- return 0;
-
- dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->nents, buf->dma_dir);
return 0;
}
* No need to sync to CPU, it's already synced to the CPU
* since the finish() memop will have been called before this.
*/
- dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
- buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
+ dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir,
+ DMA_ATTR_SKIP_CPU_SYNC);
pages = frame_vector_pages(buf->vec);
/* sgt should exist only if vector contains pages... */
BUG_ON(IS_ERR(pages));
* No need to sync to the device, this will happen later when the
* prepare() memop is called.
*/
- sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
- buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
- if (sgt->nents <= 0) {
+ if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir,
+ DMA_ATTR_SKIP_CPU_SYNC)) {
pr_err("failed to map scatterlist\n");
ret = -EIO;
goto fail_sgt_init;
return buf;
fail_map_sg:
- dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
- buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
+ dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
fail_sgt_init:
sg_free_table(sgt);
/*
* NOTE: dma-sg allocates memory using the page allocator directly, so
* there is no memory consistency guarantee, hence dma-sg ignores DMA
- * attributes passed from the upper layer. That means that
- * V4L2_FLAG_MEMORY_NON_CONSISTENT has no effect on dma-sg buffers.
+ * attributes passed from the upper layer.
*/
buf->pages = kvmalloc_array(buf->num_pages, sizeof(struct page *),
GFP_KERNEL | __GFP_ZERO);
* No need to sync to the device, this will happen later when the
* prepare() memop is called.
*/
- sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
- buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
- if (!sgt->nents)
+ if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir,
+ DMA_ATTR_SKIP_CPU_SYNC))
goto fail_map;
buf->handler.refcount = &buf->refcount;
if (refcount_dec_and_test(&buf->refcount)) {
dprintk(1, "%s: Freeing buffer of %d pages\n", __func__,
buf->num_pages);
- dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
- buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
+ dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir,
+ DMA_ATTR_SKIP_CPU_SYNC);
if (buf->vaddr)
vm_unmap_ram(buf->vaddr, buf->num_pages);
sg_free_table(buf->dma_sgt);
struct vb2_dma_sg_buf *buf = buf_priv;
struct sg_table *sgt = buf->dma_sgt;
- dma_sync_sg_for_device(buf->dev, sgt->sgl, sgt->orig_nents,
- buf->dma_dir);
+ dma_sync_sgtable_for_device(buf->dev, sgt, buf->dma_dir);
}
static void vb2_dma_sg_finish(void *buf_priv)
struct vb2_dma_sg_buf *buf = buf_priv;
struct sg_table *sgt = buf->dma_sgt;
- dma_sync_sg_for_cpu(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir);
+ dma_sync_sgtable_for_cpu(buf->dev, sgt, buf->dma_dir);
}
static void *vb2_dma_sg_get_userptr(struct device *dev, unsigned long vaddr,
* No need to sync to the device, this will happen later when the
* prepare() memop is called.
*/
- sgt->nents = dma_map_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents,
- buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
- if (!sgt->nents)
+ if (dma_map_sgtable(buf->dev, sgt, buf->dma_dir,
+ DMA_ATTR_SKIP_CPU_SYNC))
goto userptr_fail_map;
return buf;
dprintk(1, "%s: Releasing userspace buffer of %d pages\n",
__func__, buf->num_pages);
- dma_unmap_sg_attrs(buf->dev, sgt->sgl, sgt->orig_nents, buf->dma_dir,
- DMA_ATTR_SKIP_CPU_SYNC);
+ dma_unmap_sgtable(buf->dev, sgt, buf->dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
if (buf->vaddr)
vm_unmap_ram(buf->vaddr, buf->num_pages);
sg_free_table(buf->dma_sgt);
/* release the scatterlist cache */
if (attach->dma_dir != DMA_NONE)
- dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
- attach->dma_dir);
+ dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir, 0);
sg_free_table(sgt);
kfree(attach);
db_attach->priv = NULL;
/* release any previous cache */
if (attach->dma_dir != DMA_NONE) {
- dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
- attach->dma_dir);
+ dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir, 0);
attach->dma_dir = DMA_NONE;
}
/* mapping to the client with new direction */
- sgt->nents = dma_map_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
- dma_dir);
- if (!sgt->nents) {
+ if (dma_map_sgtable(db_attach->dev, sgt, dma_dir, 0)) {
pr_err("failed to map scatterlist\n");
mutex_unlock(lock);
return ERR_PTR(-EIO);
* the Free Software Foundation.
*/
+#include <linux/device.h>
#include <linux/err.h>
+#include <linux/freezer.h>
#include <linux/kernel.h>
-#include <linux/module.h>
+#include <linux/kthread.h>
#include <linux/mm.h>
+#include <linux/module.h>
#include <linux/poll.h>
-#include <linux/slab.h>
#include <linux/sched.h>
-#include <linux/freezer.h>
-#include <linux/kthread.h>
+#include <linux/slab.h>
+#include <media/v4l2-common.h>
#include <media/v4l2-dev.h>
#include <media/v4l2-device.h>
-#include <media/v4l2-fh.h>
#include <media/v4l2-event.h>
-#include <media/v4l2-common.h>
+#include <media/v4l2-fh.h>
#include <media/videobuf2-v4l2.h>
break;
case VB2_BUF_STATE_ERROR:
b->flags |= V4L2_BUF_FLAG_ERROR;
- /* fall through */
+ fallthrough;
case VB2_BUF_STATE_DONE:
b->flags |= V4L2_BUF_FLAG_DONE;
break;
#endif
}
-static void clear_consistency_attr(struct vb2_queue *q,
- int memory,
- unsigned int *flags)
-{
- if (!q->allow_cache_hints || memory != V4L2_MEMORY_MMAP)
- *flags &= ~V4L2_FLAG_MEMORY_NON_CONSISTENT;
-}
-
int vb2_reqbufs(struct vb2_queue *q, struct v4l2_requestbuffers *req)
{
int ret = vb2_verify_memory_type(q, req->memory, req->type);
fill_buf_caps(q, &req->capabilities);
- clear_consistency_attr(q, req->memory, &req->flags);
- return ret ? ret : vb2_core_reqbufs(q, req->memory,
- req->flags, &req->count);
+ return ret ? ret : vb2_core_reqbufs(q, req->memory, &req->count);
}
EXPORT_SYMBOL_GPL(vb2_reqbufs);
unsigned i;
fill_buf_caps(q, &create->capabilities);
- clear_consistency_attr(q, create->memory, &create->flags);
create->index = q->num_buffers;
if (create->count == 0)
return ret != -EBUSY ? ret : 0;
if (requested_sizes[i] == 0)
return -EINVAL;
return ret ? ret : vb2_core_create_bufs(q, create->memory,
- create->flags,
&create->count,
requested_planes,
requested_sizes);
int res = vb2_verify_memory_type(vdev->queue, p->memory, p->type);
fill_buf_caps(vdev->queue, &p->capabilities);
- clear_consistency_attr(vdev->queue, p->memory, &p->flags);
if (res)
return res;
if (vb2_queue_is_busy(vdev, file))
return -EBUSY;
- res = vb2_core_reqbufs(vdev->queue, p->memory, p->flags, &p->count);
+ res = vb2_core_reqbufs(vdev->queue, p->memory, &p->count);
/* If count == 0, then the owner has released all buffers and he
is no longer owner of the queue. Otherwise we have a new owner. */
if (res == 0)
p->index = vdev->queue->num_buffers;
fill_buf_caps(vdev->queue, &p->capabilities);
- clear_consistency_attr(vdev->queue, p->memory, &p->flags);
/*
* If count == 0, then just check if memory and type are valid.
* Any -EBUSY result from vb2_verify_memory_type can be mapped to 0.
EXPORT_SYMBOL_GPL(vb2_fop_get_unmapped_area);
#endif
+void vb2_video_unregister_device(struct video_device *vdev)
+{
+ /* Check if vdev was ever registered at all */
+ if (!vdev || !video_is_registered(vdev))
+ return;
+
+ /*
+ * Calling this function only makes sense if vdev->queue is set.
+ * If it is NULL, then just call video_unregister_device() instead.
+ */
+ WARN_ON(!vdev->queue);
+
+ /*
+ * Take a reference to the device since video_unregister_device()
+ * calls device_unregister(), but we don't want that to release
+ * the device since we want to clean up the queue first.
+ */
+ get_device(&vdev->dev);
+ video_unregister_device(vdev);
+ if (vdev->queue && vdev->queue->owner) {
+ struct mutex *lock = vdev->queue->lock ?
+ vdev->queue->lock : vdev->lock;
+
+ if (lock)
+ mutex_lock(lock);
+ vb2_queue_release(vdev->queue);
+ vdev->queue->owner = NULL;
+ if (lock)
+ mutex_unlock(lock);
+ }
+ /*
+ * Now we put the device, and in most cases this will release
+ * everything.
+ */
+ put_device(&vdev->dev);
+}
+EXPORT_SYMBOL_GPL(vb2_video_unregister_device);
+
/* vb2_ops helpers. Only use if vq->lock is non-NULL. */
void vb2_ops_wait_prepare(struct vb2_queue *vq)
kfree(attach);
return ret;
}
- for_each_sg(sgt->sgl, sg, sgt->nents, i) {
+ for_each_sgtable_sg(sgt, sg, i) {
struct page *page = vmalloc_to_page(vaddr);
if (!page) {
/* release the scatterlist cache */
if (attach->dma_dir != DMA_NONE)
- dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
- attach->dma_dir);
+ dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir, 0);
sg_free_table(sgt);
kfree(attach);
db_attach->priv = NULL;
/* release any previous cache */
if (attach->dma_dir != DMA_NONE) {
- dma_unmap_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
- attach->dma_dir);
+ dma_unmap_sgtable(db_attach->dev, sgt, attach->dma_dir, 0);
attach->dma_dir = DMA_NONE;
}
/* mapping to the client with new direction */
- sgt->nents = dma_map_sg(db_attach->dev, sgt->sgl, sgt->orig_nents,
- dma_dir);
- if (!sgt->nents) {
+ if (dma_map_sgtable(db_attach->dev, sgt, dma_dir, 0)) {
pr_err("failed to map scatterlist\n");
mutex_unlock(lock);
return ERR_PTR(-EIO);
ctx->buf_siz = req->size;
ctx->buf_cnt = req->count;
- ret = vb2_core_reqbufs(&ctx->vb_q, VB2_MEMORY_MMAP, 0, &req->count);
+ ret = vb2_core_reqbufs(&ctx->vb_q, VB2_MEMORY_MMAP, &req->count);
if (ret) {
ctx->state = DVB_VB2_STATE_NONE;
dprintk(1, "[%s] count=%d size=%d errno=%d\n", ctx->name,
state->strength_en = 2;
break;
}
- /* Fall through */
+ fallthrough;
case 1:
if (time_is_after_jiffies(state->strength_jiffies + msecs_to_jiffies(2000)))
break;
retry_count += 1;
status = read16(state, SIO_HI_RA_RAM_CMD__A,
&wait_cmd);
- } while ((status < 0) && (retry_count < DRXK_MAX_RETRIES)
- && (wait_cmd != 0));
+ } while ((status < 0 || wait_cmd) && (retry_count < DRXK_MAX_RETRIES));
if (status < 0)
goto error;
status = read16(state, SIO_HI_RA_RAM_RES__A, p_result);
switch (config->lg_chip) {
default:
lg_warn("invalid chip requested, defaulting to LG2160");
- /* fall-thru */
+ fallthrough;
case LG2160:
memcpy(&state->frontend.ops, &lg2160_ops,
sizeof(struct dvb_frontend_ops));
if (ret)
goto err;
}
- /* fall through */
+ fallthrough;
default:
u16tmp = DIV_ROUND_UP(target_mclk, dev->cfg->ts_clk);
u8tmp1 = u16tmp / 2 - 1;
wait_t = (786432 + state->srate / 2) / state->srate;
else
wait_t = (1572864 + state->srate / 2) / state->srate;
- if (state->srate < 5000)
- /* FIXME ! , should be a long wait ! */
- msleep_interruptible(wait_t);
- else
- msleep_interruptible(wait_t);
+
+ msleep_interruptible(wait_t);
if (sync_chk(state, &junk) == 0) {
iq_vt_set(state, 1);
#include <linux/delay.h>
#include <linux/firmware.h>
#include <linux/i2c.h>
-#include <linux/version.h>
#include <linux/mutex.h>
#include <linux/vmalloc.h>
#include <asm/div64.h>
default:
v4l2_ctrl_handler_init(&dev->hdl, 0);
dev_err(&pdev->dev, "Unsupported tuner\n");
+ ret = -ENODEV;
goto err_v4l2_ctrl_handler_free;
}
if (dev->hdl.error) {
{
s32 BDR;
s32 BDRI;
- s16 SFIL=0;
+ s16 SFIL = 0;
u16 NDEC = 0;
u32 tmp, ratio;
- if (symbolrate > XIN/2)
- symbolrate = XIN/2;
- if (symbolrate < 500000)
+ if (symbolrate > XIN / 2)
+ symbolrate = XIN / 2;
+ else if (symbolrate < 500000)
symbolrate = 500000;
- if (symbolrate < XIN/16) NDEC = 1;
- if (symbolrate < XIN/32) NDEC = 2;
- if (symbolrate < XIN/64) NDEC = 3;
-
- if (symbolrate < (u32)(XIN/12.3)) SFIL = 1;
- if (symbolrate < (u32)(XIN/16)) SFIL = 0;
- if (symbolrate < (u32)(XIN/24.6)) SFIL = 1;
- if (symbolrate < (u32)(XIN/32)) SFIL = 0;
- if (symbolrate < (u32)(XIN/49.2)) SFIL = 1;
- if (symbolrate < (u32)(XIN/64)) SFIL = 0;
- if (symbolrate < (u32)(XIN/98.4)) SFIL = 1;
+ if (symbolrate < XIN / 16)
+ NDEC = 1;
+ if (symbolrate < XIN / 32)
+ NDEC = 2;
+ if (symbolrate < XIN / 64)
+ NDEC = 3;
+
+ if (symbolrate < XIN * 10 / 123)
+ SFIL = 1;
+ if (symbolrate < XIN * 10 / 160)
+ SFIL = 0;
+ if (symbolrate < XIN * 10 / 246)
+ SFIL = 1;
+ if (symbolrate < XIN * 10 / 320)
+ SFIL = 0;
+ if (symbolrate < XIN * 10 / 492)
+ SFIL = 1;
+ if (symbolrate < XIN * 10 / 640)
+ SFIL = 0;
+ if (symbolrate < XIN * 10 / 984)
+ SFIL = 1;
symbolrate <<= NDEC;
ratio = (symbolrate << 4) / FIN;
#include <media/dvb_frontend.h>
#include "tda10086.h"
-#define SACLK 96000000
+#define SACLK 96000000U
struct tda10086_state {
struct i2c_adapter* i2c;
dprintk ("%s %i\n", __func__, symbol_rate);
/* setup the decimation and anti-aliasing filters.. */
- if (symbol_rate < (u32) (SACLK * 0.0137)) {
+ if (symbol_rate < SACLK / 10000 * 137) {
dfn=4;
afs=1;
- } else if (symbol_rate < (u32) (SACLK * 0.0208)) {
+ } else if (symbol_rate < SACLK / 10000 * 208) {
dfn=4;
afs=0;
- } else if (symbol_rate < (u32) (SACLK * 0.0270)) {
+ } else if (symbol_rate < SACLK / 10000 * 270) {
dfn=3;
afs=1;
- } else if (symbol_rate < (u32) (SACLK * 0.0416)) {
+ } else if (symbol_rate < SACLK / 10000 * 416) {
dfn=3;
afs=0;
- } else if (symbol_rate < (u32) (SACLK * 0.0550)) {
+ } else if (symbol_rate < SACLK / 10000 * 550) {
dfn=2;
afs=1;
- } else if (symbol_rate < (u32) (SACLK * 0.0833)) {
+ } else if (symbol_rate < SACLK / 10000 * 833) {
dfn=2;
afs=0;
- } else if (symbol_rate < (u32) (SACLK * 0.1100)) {
+ } else if (symbol_rate < SACLK / 10000 * 1100) {
dfn=1;
afs=1;
- } else if (symbol_rate < (u32) (SACLK * 0.1666)) {
+ } else if (symbol_rate < SACLK / 10000 * 1666) {
dfn=1;
afs=0;
- } else if (symbol_rate < (u32) (SACLK * 0.2200)) {
+ } else if (symbol_rate < SACLK / 10000 * 2200) {
dfn=0;
afs=1;
- } else if (symbol_rate < (u32) (SACLK * 0.3333)) {
+ } else if (symbol_rate < SACLK / 10000 * 3333) {
dfn=0;
afs=0;
} else {
state->m_bFMInput = (ulFMInput == 2);
}
-static bool SearchMap1(struct SMap Map[],
- u32 Frequency, u8 *pParam)
+static bool SearchMap1(const struct SMap map[], u32 frequency, u8 *param)
{
int i = 0;
- while ((Map[i].m_Frequency != 0) && (Frequency > Map[i].m_Frequency))
+ while ((map[i].m_Frequency != 0) && (frequency > map[i].m_Frequency))
i += 1;
- if (Map[i].m_Frequency == 0)
+ if (map[i].m_Frequency == 0)
return false;
- *pParam = Map[i].m_Param;
+ *param = map[i].m_Param;
return true;
}
-static bool SearchMap2(struct SMapI Map[],
- u32 Frequency, s32 *pParam)
+static bool SearchMap2(const struct SMapI map[], u32 frequency, s32 *param)
{
int i = 0;
- while ((Map[i].m_Frequency != 0) &&
- (Frequency > Map[i].m_Frequency))
+ while ((map[i].m_Frequency != 0) &&
+ (frequency > map[i].m_Frequency))
i += 1;
- if (Map[i].m_Frequency == 0)
+ if (map[i].m_Frequency == 0)
return false;
- *pParam = Map[i].m_Param;
+ *param = map[i].m_Param;
return true;
}
-static bool SearchMap3(struct SMap2 Map[], u32 Frequency,
- u8 *pParam1, u8 *pParam2)
+static bool SearchMap3(const struct SMap2 map[], u32 frequency, u8 *param1,
+ u8 *param2)
{
int i = 0;
- while ((Map[i].m_Frequency != 0) &&
- (Frequency > Map[i].m_Frequency))
+ while ((map[i].m_Frequency != 0) &&
+ (frequency > map[i].m_Frequency))
i += 1;
- if (Map[i].m_Frequency == 0)
+ if (map[i].m_Frequency == 0)
return false;
- *pParam1 = Map[i].m_Param1;
- *pParam2 = Map[i].m_Param2;
+ *param1 = map[i].m_Param1;
+ *param2 = map[i].m_Param2;
return true;
}
-static bool SearchMap4(struct SRFBandMap Map[],
- u32 Frequency, u8 *pRFBand)
+static bool SearchMap4(const struct SRFBandMap map[], u32 frequency, u8 *rfband)
{
int i = 0;
- while (i < 7 && (Frequency > Map[i].m_RF_max))
+ while (i < 7 && (frequency > map[i].m_RF_max))
i += 1;
if (i == 7)
return false;
- *pRFBand = i;
+ *rfband = i;
return true;
}
HF_DVBC_8MHZ, HF_DVBC
};
-static struct SStandardParam m_StandardTable[] = {
+static const struct SStandardParam m_StandardTable[] = {
{ 0, 0, 0x00, 0x00 }, /* HF_None */
{ 6000000, 7000000, 0x1D, 0x2C }, /* HF_B, */
{ 6900000, 8000000, 0x1E, 0x2C }, /* HF_DK, */
{ 0, 0, 0x00, 0x00 }, /* HF_DVBC (Unused) */
};
-static struct SMap m_BP_Filter_Map[] = {
+static const struct SMap m_BP_Filter_Map[] = {
{ 62000000, 0x00 },
{ 84000000, 0x01 },
{ 100000000, 0x02 },
{ 0, 0x00 }, /* Table End */
};
-static struct SMapI m_RF_Cal_Map[] = {
+static const struct SMapI m_RF_Cal_Map[] = {
{ 41000000, 0x0F },
{ 43000000, 0x1C },
{ 45000000, 0x2F },
};
-static struct SMap2 m_KM_Map[] = {
+static const struct SMap2 m_KM_Map[] = {
{ 47900000, 3, 2 },
{ 61100000, 3, 1 },
{ 350000000, 3, 0 },
{ 0, 0x00 }, /* Table End */
};
-static struct SMap2 m_Main_PLL_Map[] = {
+static const struct SMap2 m_Main_PLL_Map[] = {
{ 33125000, 0x57, 0xF0 },
{ 35500000, 0x56, 0xE0 },
{ 38188000, 0x55, 0xD0 },
{ 0, 0x00, 0x00 }, /* Table End */
};
-static struct SMap2 m_Cal_PLL_Map[] = {
+static const struct SMap2 m_Cal_PLL_Map[] = {
{ 33813000, 0xDD, 0xD0 },
{ 36625000, 0xDC, 0xC0 },
{ 39938000, 0xDB, 0xB0 },
{ 0, 0x00, 0x00 }, /* Table End */
};
-static struct SMap m_GainTaper_Map[] = {
+static const struct SMap m_GainTaper_Map[] = {
{ 45400000, 0x1F },
{ 45800000, 0x1E },
{ 46200000, 0x1D },
{ 0, 0x00 }, /* Table End */
};
-static struct SMap m_RF_Cal_DC_Over_DT_Map[] = {
+static const struct SMap m_RF_Cal_DC_Over_DT_Map[] = {
{ 47900000, 0x00 },
{ 55000000, 0x00 },
{ 61100000, 0x0A },
};
-static struct SMap m_IR_Meas_Map[] = {
+static const struct SMap m_IR_Meas_Map[] = {
{ 200000000, 0x05 },
{ 400000000, 0x06 },
{ 865000000, 0x07 },
{ 0, 0x00 }, /* Table End */
};
-static struct SMap2 m_CID_Target_Map[] = {
+static const struct SMap2 m_CID_Target_Map[] = {
{ 46000000, 0x04, 18 },
{ 52200000, 0x0A, 15 },
{ 70100000, 0x01, 40 },
{ 0, 0x00, 0 }, /* Table End */
};
-static struct SRFBandMap m_RF_Band_Map[7] = {
+static const struct SRFBandMap m_RF_Band_Map[7] = {
{ 47900000, 46000000, 0, 0},
{ 61100000, 52200000, 0, 0},
{ 152600000, 70100000, 136800000, 0},
#else
-/**
- * zd1301_demod_get_dvb_frontend() - Attach a zd1301 frontend
- * @dev: Pointer to platform device
- *
- * Return: Pointer to %struct dvb_frontend or NULL if attach fails.
- */
static inline struct dvb_frontend *zd1301_demod_get_dvb_frontend(struct platform_device *dev)
{
printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__);
name_len = fw_csr_string(unit->directory, CSR_MODEL,
name, sizeof(name));
- if (name_len < 0)
- return name_len;
+ if (name_len < 0) {
+ err = name_len;
+ goto fail_free;
+ }
for (i = ARRAY_SIZE(model_names); --i; )
if (strlen(model_names[i]) <= name_len &&
strncmp(name, model_names[i], name_len) == 0)
config VIDEO_OV7740
tristate "OmniVision OV7740 sensor support"
depends on I2C && VIDEO_V4L2
- select REGMAP_I2C
+ select REGMAP_SCCB
help
This is a Video4Linux2 sensor driver for the OmniVision
OV7740 VGA camera sensor.
case V4L2_FIELD_NONE:
if (state->chip_info->flags & ADV7180_FLAG_I2P)
break;
- /* fall through */
+ fallthrough;
default:
format->format.field = V4L2_FIELD_ALTERNATE;
break;
return adv7180_set_pad_format(sd, cfg, &fmt);
}
-static int adv7180_g_mbus_config(struct v4l2_subdev *sd,
- struct v4l2_mbus_config *cfg)
+static int adv7180_get_mbus_config(struct v4l2_subdev *sd,
+ unsigned int pad,
+ struct v4l2_mbus_config *cfg)
{
struct adv7180_state *state = to_state(sd);
.querystd = adv7180_querystd,
.g_input_status = adv7180_g_input_status,
.s_routing = adv7180_s_routing,
- .g_mbus_config = adv7180_g_mbus_config,
.g_pixelaspect = adv7180_g_pixelaspect,
.g_tvnorms = adv7180_g_tvnorms,
.s_stream = adv7180_s_stream,
.enum_mbus_code = adv7180_enum_mbus_code,
.set_fmt = adv7180_set_pad_format,
.get_fmt = adv7180_get_pad_format,
+ .get_mbus_config = adv7180_get_mbus_config,
};
static const struct v4l2_subdev_sensor_ops adv7180_sensor_ops = {
int ret = 0;
/* Enable n-lane MIPI */
- adv748x_write_check(state, page, 0x00, 0x80 | tx->num_lanes, &ret);
+ adv748x_write_check(state, page, 0x00, 0x80 | tx->active_lanes, &ret);
/* Set Auto DPHY Timing */
- adv748x_write_check(state, page, 0x00, 0xa0 | tx->num_lanes, &ret);
+ adv748x_write_check(state, page, 0x00, 0xa0 | tx->active_lanes, &ret);
/* ADI Required Write */
if (tx->src == &state->hdmi.sd) {
usleep_range(2000, 2500);
/* Power-up CSI-TX */
- adv748x_write_check(state, page, 0x00, 0x20 | tx->num_lanes, &ret);
+ adv748x_write_check(state, page, 0x00, 0x20 | tx->active_lanes, &ret);
usleep_range(1000, 1500);
/* ADI Required Writes */
adv748x_write_check(state, page, 0x1e, 0x00, &ret);
/* Enable n-lane MIPI */
- adv748x_write_check(state, page, 0x00, 0x80 | tx->num_lanes, &ret);
+ adv748x_write_check(state, page, 0x00, 0x80 | tx->active_lanes, &ret);
/* i2c_mipi_pll_en - 1'b1 */
adv748x_write_check(state, page, 0xda, 0x01, &ret);
if (state->afe.tx) {
/* AFE Requires TXA enabled, even when output to TXB */
io10 |= ADV748X_IO_10_CSI4_EN;
- if (is_txa(tx))
+ if (is_txa(tx)) {
+ /*
+ * Output from the SD-core (480i and 576i) from the TXA
+ * interface requires reducing the number of enabled
+ * data lanes in order to guarantee a valid link
+ * frequency.
+ */
+ tx->active_lanes = min(tx->num_lanes, 2U);
io10 |= ADV748X_IO_10_CSI4_IN_SEL_AFE;
- else
+ } else {
+ /* TXB has a single data lane, no need to adjust. */
io10 |= ADV748X_IO_10_CSI1_EN;
+ }
}
- if (state->hdmi.tx)
+ if (state->hdmi.tx) {
+ /*
+ * Restore the number of active lanes, in case we have gone
+ * through an AFE->TXA streaming sessions.
+ */
+ tx->active_lanes = tx->num_lanes;
io10 |= ADV748X_IO_10_CSI4_EN;
+ }
return io_clrset(state, ADV748X_IO_10, io10_mask, io10);
}
}
state->txa.num_lanes = num_lanes;
+ state->txa.active_lanes = num_lanes;
adv_dbg(state, "TXA: using %u lanes\n", state->txa.num_lanes);
}
}
state->txb.num_lanes = num_lanes;
+ state->txb.active_lanes = num_lanes;
adv_dbg(state, "TXB: using %u lanes\n", state->txb.num_lanes);
}
return ret;
}
+static int adv748x_csi2_get_mbus_config(struct v4l2_subdev *sd, unsigned int pad,
+ struct v4l2_mbus_config *config)
+{
+ struct adv748x_csi2 *tx = adv748x_sd_to_csi2(sd);
+
+ if (pad != ADV748X_CSI2_SOURCE)
+ return -EINVAL;
+
+ config->type = V4L2_MBUS_CSI2_DPHY;
+ switch (tx->active_lanes) {
+ case 1:
+ config->flags = V4L2_MBUS_CSI2_1_LANE;
+ break;
+
+ case 2:
+ config->flags = V4L2_MBUS_CSI2_2_LANE;
+ break;
+
+ case 3:
+ config->flags = V4L2_MBUS_CSI2_3_LANE;
+ break;
+
+ case 4:
+ config->flags = V4L2_MBUS_CSI2_4_LANE;
+ break;
+ }
+
+ return 0;
+}
+
static const struct v4l2_subdev_pad_ops adv748x_csi2_pad_ops = {
.get_fmt = adv748x_csi2_get_format,
.set_fmt = adv748x_csi2_set_format,
+ .get_mbus_config = adv748x_csi2_get_mbus_config,
};
/* -----------------------------------------------------------------------------
unsigned int page;
unsigned int port;
unsigned int num_lanes;
+ unsigned int active_lanes;
struct media_pad pads[ADV748X_CSI2_NR_PADS];
struct v4l2_ctrl_handler ctrl_hdl;
reg->val = adv7511_cec_read(sd, reg->reg & 0xff);
break;
}
- /* fall through */
+ fallthrough;
default:
v4l2_info(sd, "Register %03llx not supported\n", reg->reg);
adv7511_inv_register(sd);
adv7511_cec_write(sd, reg->reg & 0xff, reg->val & 0xff);
break;
}
- /* fall through */
+ fallthrough;
default:
v4l2_info(sd, "Register %03llx not supported\n", reg->reg);
adv7511_inv_register(sd);
}
v = (unsigned) pulse_width_count_to_ns(
- (u16) (p->hw_fifo_data & FIFO_RXTX), divider);
+ (u16)(p->hw_fifo_data & FIFO_RXTX), divider) / 1000;
if (v > IR_MAX_DURATION)
v = IR_MAX_DURATION;
module_i2c_driver(dw9807_i2c_driver);
-MODULE_AUTHOR("Chiang, Alan <alanx.chiang@intel.com>");
+MODULE_AUTHOR("Chiang, Alan");
MODULE_DESCRIPTION("DW9807 VCM driver");
MODULE_LICENSE("GPL v2");
error:
imx219_stop_streaming(imx219);
- imx219->streaming = 0;
+ imx219->streaming = false;
return ret;
}
module_i2c_driver(imx258_i2c_driver);
MODULE_AUTHOR("Yeh, Andy <andy.yeh@intel.com>");
-MODULE_AUTHOR("Chiang, Alan <alanx.chiang@intel.com>");
+MODULE_AUTHOR("Chiang, Alan");
MODULE_AUTHOR("Chen, Jason <jasonx.z.chen@intel.com>");
MODULE_DESCRIPTION("Sony IMX258 sensor driver");
MODULE_LICENSE("GPL v2");
ret = imx274_set_frame_interval(imx274, fi->interval);
if (!ret) {
+ fi->interval = imx274->frame_interval;
+
/*
* exposure time range is decided by frame interval
* need to update it after frame interval changes
__func__, frame_interval.numerator,
frame_interval.denominator);
- if (frame_interval.numerator == 0) {
- err = -EINVAL;
- goto fail;
+ if (frame_interval.numerator == 0 || frame_interval.denominator == 0) {
+ frame_interval.denominator = IMX274_DEF_FRAME_RATE;
+ frame_interval.numerator = 1;
}
req_frame_rate = (u32)(frame_interval.denominator
ret = regulator_bulk_enable(ARRAY_SIZE(supplies), supplies);
if (ret) {
- info->set_power(&client->dev, 0);
+ if (info->set_power)
+ info->set_power(&client->dev, 0);
return ret;
}
#define MAX9286_SRC_PAD 4
struct max9286_source {
- struct v4l2_async_subdev asd;
struct v4l2_subdev *sd;
struct fwnode_handle *fwnode;
};
-#define asd_to_max9286_source(_asd) \
- container_of(_asd, struct max9286_source, asd)
+struct max9286_asd {
+ struct v4l2_async_subdev base;
+ struct max9286_source *source;
+};
+
+static inline struct max9286_asd *to_max9286_asd(struct v4l2_async_subdev *asd)
+{
+ return container_of(asd, struct max9286_asd, base);
+}
struct max9286_priv {
struct i2c_client *client;
* to 5 milliseconds.
*/
for (i = 0; i < 10; i++) {
- ret = max9286_read(priv, 0x49) & 0xf0;
+ ret = max9286_read(priv, 0x49);
if (ret < 0)
return -EIO;
+ ret &= 0xf0;
if (ret == conflink_mask)
break;
struct v4l2_async_subdev *asd)
{
struct max9286_priv *priv = sd_to_max9286(notifier->sd);
- struct max9286_source *source = asd_to_max9286_source(asd);
+ struct max9286_source *source = to_max9286_asd(asd)->source;
unsigned int index = to_index(priv, source);
unsigned int src_pad;
int ret;
struct v4l2_async_subdev *asd)
{
struct max9286_priv *priv = sd_to_max9286(notifier->sd);
- struct max9286_source *source = asd_to_max9286_source(asd);
+ struct max9286_source *source = to_max9286_asd(asd)->source;
unsigned int index = to_index(priv, source);
source->sd = NULL;
for_each_source(priv, source) {
unsigned int i = to_index(priv, source);
-
- source->asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
- source->asd.match.fwnode = source->fwnode;
-
- ret = v4l2_async_notifier_add_subdev(&priv->notifier,
- &source->asd);
- if (ret) {
- dev_err(dev, "Failed to add subdev for source %d", i);
+ struct v4l2_async_subdev *asd;
+
+ asd = v4l2_async_notifier_add_fwnode_subdev(&priv->notifier,
+ source->fwnode,
+ sizeof(*asd));
+ if (IS_ERR(asd)) {
+ dev_err(dev, "Failed to add subdev for source %u: %ld",
+ i, PTR_ERR(asd));
v4l2_async_notifier_cleanup(&priv->notifier);
- return ret;
+ return PTR_ERR(asd);
}
- /*
- * Balance the reference counting handled through
- * v4l2_async_notifier_cleanup()
- */
- fwnode_handle_get(source->fwnode);
+ to_max9286_asd(asd)->source = source;
}
priv->notifier.ops = &max9286_notify_ops;
return 0;
}
-static int ml86v7667_g_mbus_config(struct v4l2_subdev *sd,
- struct v4l2_mbus_config *cfg)
+static int ml86v7667_get_mbus_config(struct v4l2_subdev *sd,
+ unsigned int pad,
+ struct v4l2_mbus_config *cfg)
{
cfg->flags = V4L2_MBUS_MASTER | V4L2_MBUS_PCLK_SAMPLE_RISING |
V4L2_MBUS_DATA_ACTIVE_HIGH;
.s_std = ml86v7667_s_std,
.querystd = ml86v7667_querystd,
.g_input_status = ml86v7667_g_input_status,
- .g_mbus_config = ml86v7667_g_mbus_config,
};
static const struct v4l2_subdev_pad_ops ml86v7667_subdev_pad_ops = {
.enum_mbus_code = ml86v7667_enum_mbus_code,
.get_fmt = ml86v7667_fill_fmt,
.set_fmt = ml86v7667_fill_fmt,
+ .get_mbus_config = ml86v7667_get_mbus_config,
};
static const struct v4l2_subdev_core_ops ml86v7667_subdev_core_ops = {
break;
case 0: /* 4.5 */
state->detected_std = V4L2_STD_MN;
- /* fall-through */
+ fallthrough;
default:
no_second:
state->second = msp3400c_carrier_detect_main[max1].cdo;
return 0;
}
-static int mt9m001_g_mbus_config(struct v4l2_subdev *sd,
- struct v4l2_mbus_config *cfg)
+static int mt9m001_get_mbus_config(struct v4l2_subdev *sd,
+ unsigned int pad,
+ struct v4l2_mbus_config *cfg)
{
/* MT9M001 has all capture_format parameters fixed */
cfg->flags = V4L2_MBUS_PCLK_SAMPLE_FALLING |
static const struct v4l2_subdev_video_ops mt9m001_subdev_video_ops = {
.s_stream = mt9m001_s_stream,
- .g_mbus_config = mt9m001_g_mbus_config,
};
static const struct v4l2_subdev_sensor_ops mt9m001_subdev_sensor_ops = {
.set_selection = mt9m001_set_selection,
.get_fmt = mt9m001_get_fmt,
.set_fmt = mt9m001_set_fmt,
+ .get_mbus_config = mt9m001_get_mbus_config,
};
static const struct v4l2_subdev_ops mt9m001_subdev_ops = {
return 0;
}
-static int mt9m111_g_mbus_config(struct v4l2_subdev *sd,
- struct v4l2_mbus_config *cfg)
+static int mt9m111_get_mbus_config(struct v4l2_subdev *sd,
+ unsigned int pad,
+ struct v4l2_mbus_config *cfg)
{
struct mt9m111 *mt9m111 = container_of(sd, struct mt9m111, subdev);
}
static const struct v4l2_subdev_video_ops mt9m111_subdev_video_ops = {
- .g_mbus_config = mt9m111_g_mbus_config,
.s_stream = mt9m111_s_stream,
.g_frame_interval = mt9m111_g_frame_interval,
.s_frame_interval = mt9m111_s_frame_interval,
.set_selection = mt9m111_set_selection,
.get_fmt = mt9m111_get_fmt,
.set_fmt = mt9m111_set_fmt,
+ .get_mbus_config = mt9m111_get_mbus_config,
};
static const struct v4l2_subdev_ops mt9m111_subdev_ops = {
if (!nvm)
return -ENOMEM;
+ nvm->nvm_buffer = devm_kzalloc(dev, CUSTOMER_USE_OTP_SIZE, GFP_KERNEL);
+ if (!nvm->nvm_buffer)
+ return -ENOMEM;
+
regmap_config.val_bits = 8;
regmap_config.reg_bits = 16;
regmap_config.disable_locking = true;
nvm->regmap = regmap;
+ ret = ov2740_load_otp_data(client, nvm);
+ if (ret) {
+ dev_err(dev, "failed to load OTP data, ret %d\n", ret);
+ return ret;
+ }
+
nvmem_config.name = dev_name(dev);
nvmem_config.dev = dev;
nvmem_config.read_only = true;
nvmem_config.size = CUSTOMER_USE_OTP_SIZE;
nvm->nvmem = devm_nvmem_register(dev, &nvmem_config);
- if (IS_ERR(nvm->nvmem))
- return PTR_ERR(nvm->nvmem);
- nvm->nvm_buffer = devm_kzalloc(dev, CUSTOMER_USE_OTP_SIZE, GFP_KERNEL);
- if (!nvm->nvm_buffer)
- return -ENOMEM;
-
- ret = ov2740_load_otp_data(client, nvm);
- if (ret)
- dev_err(dev, "failed to load OTP data, ret %d\n", ret);
-
- return ret;
+ return PTR_ERR_OR_ZERO(nvm->nvmem);
}
static int ov2740_probe(struct i2c_client *client)
ret = ov2740_register_nvmem(client);
if (ret)
- dev_err(&client->dev, "register nvmem failed, ret %d\n", ret);
+ dev_warn(&client->dev, "register nvmem failed, ret %d\n", ret);
/*
* Device is already turned on by i2c-core with ACPI domain PM.
#define OV5640_REG_SYS_RESET02 0x3002
#define OV5640_REG_SYS_CLOCK_ENABLE02 0x3006
#define OV5640_REG_SYS_CTRL0 0x3008
+#define OV5640_REG_SYS_CTRL0_SW_PWDN 0x42
+#define OV5640_REG_SYS_CTRL0_SW_PWUP 0x02
#define OV5640_REG_CHIP_ID 0x300a
#define OV5640_REG_IO_MIPI_CTRL00 0x300e
#define OV5640_REG_PAD_OUTPUT_ENABLE01 0x3017
#define OV5640_REG_VFIFO_HSIZE 0x4602
#define OV5640_REG_VFIFO_VSIZE 0x4604
#define OV5640_REG_JPG_MODE_SELECT 0x4713
+#define OV5640_REG_CCIR656_CTRL00 0x4730
#define OV5640_REG_POLARITY_CTRL00 0x4740
#define OV5640_REG_MIPI_CTRL00 0x4800
#define OV5640_REG_DEBUG_MODE 0x4814
/* YUV422 UYVY VGA@30fps */
static const struct reg_value ov5640_init_setting_30fps_VGA[] = {
{0x3103, 0x11, 0, 0}, {0x3008, 0x82, 0, 5}, {0x3008, 0x42, 0, 0},
- {0x3103, 0x03, 0, 0}, {0x3017, 0x00, 0, 0}, {0x3018, 0x00, 0, 0},
- {0x3630, 0x36, 0, 0},
+ {0x3103, 0x03, 0, 0}, {0x3630, 0x36, 0, 0},
{0x3631, 0x0e, 0, 0}, {0x3632, 0xe2, 0, 0}, {0x3633, 0x12, 0, 0},
{0x3621, 0xe0, 0, 0}, {0x3704, 0xa0, 0, 0}, {0x3703, 0x5a, 0, 0},
{0x3715, 0x78, 0, 0}, {0x3717, 0x01, 0, 0}, {0x370b, 0x60, 0, 0},
* +->| PLL Root Div | - reg 0x3037, bit 4
* +-+------------+
* | +---------+
- * +->| Bit Div | - reg 0x3035, bits 0-3
+ * +->| Bit Div | - reg 0x3034, bits 0-3
* +-+-------+
* | +-------------+
* +->| SCLK Div | - reg 0x3108, bits 0-1
val = regs->val;
mask = regs->mask;
+ /* remain in power down mode for DVP */
+ if (regs->reg_addr == OV5640_REG_SYS_CTRL0 &&
+ val == OV5640_REG_SYS_CTRL0_SW_PWUP &&
+ sensor->ep.bus_type != V4L2_MBUS_CSI2_DPHY)
+ continue;
+
if (mask)
ret = ov5640_mod_reg(sensor, reg_addr, mask, val);
else
BIT(1), on ? 0 : BIT(1));
}
-static int ov5640_set_stream_dvp(struct ov5640_dev *sensor, bool on)
+static int ov5640_set_stream_bt656(struct ov5640_dev *sensor, bool on)
{
int ret;
- unsigned int flags = sensor->ep.bus.parallel.flags;
- u8 pclk_pol = 0;
- u8 hsync_pol = 0;
- u8 vsync_pol = 0;
-
- /*
- * Note about parallel port configuration.
- *
- * When configured in parallel mode, the OV5640 will
- * output 10 bits data on DVP data lines [9:0].
- * If only 8 bits data are wanted, the 8 bits data lines
- * of the camera interface must be physically connected
- * on the DVP data lines [9:2].
- *
- * Control lines polarity can be configured through
- * devicetree endpoint control lines properties.
- * If no endpoint control lines properties are set,
- * polarity will be as below:
- * - VSYNC: active high
- * - HREF: active low
- * - PCLK: active low
- */
- if (on) {
- /*
- * configure parallel port control lines polarity
- *
- * POLARITY CTRL0
- * - [5]: PCLK polarity (0: active low, 1: active high)
- * - [1]: HREF polarity (0: active low, 1: active high)
- * - [0]: VSYNC polarity (mismatch here between
- * datasheet and hardware, 0 is active high
- * and 1 is active low...)
- */
- if (flags & V4L2_MBUS_PCLK_SAMPLE_RISING)
- pclk_pol = 1;
- if (flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH)
- hsync_pol = 1;
- if (flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)
- vsync_pol = 1;
-
- ret = ov5640_write_reg(sensor,
- OV5640_REG_POLARITY_CTRL00,
- (pclk_pol << 5) |
- (hsync_pol << 1) |
- vsync_pol);
-
- if (ret)
- return ret;
- }
-
- /*
- * powerdown MIPI TX/RX PHY & disable MIPI
- *
- * MIPI CONTROL 00
- * 4: PWDN PHY TX
- * 3: PWDN PHY RX
- * 2: MIPI enable
- */
- ret = ov5640_write_reg(sensor,
- OV5640_REG_IO_MIPI_CTRL00, on ? 0x18 : 0);
+ ret = ov5640_write_reg(sensor, OV5640_REG_CCIR656_CTRL00,
+ on ? 0x1 : 0x00);
if (ret)
return ret;
- /*
- * enable VSYNC/HREF/PCLK DVP control lines
- * & D[9:6] DVP data lines
- *
- * PAD OUTPUT ENABLE 01
- * - 6: VSYNC output enable
- * - 5: HREF output enable
- * - 4: PCLK output enable
- * - [3:0]: D[9:6] output enable
- */
- ret = ov5640_write_reg(sensor,
- OV5640_REG_PAD_OUTPUT_ENABLE01,
- on ? 0x7f : 0);
- if (ret)
- return ret;
+ return ov5640_write_reg(sensor, OV5640_REG_SYS_CTRL0, on ?
+ OV5640_REG_SYS_CTRL0_SW_PWUP :
+ OV5640_REG_SYS_CTRL0_SW_PWDN);
+}
- /*
- * enable D[5:0] DVP data lines
- *
- * PAD OUTPUT ENABLE 02
- * - [7:2]: D[5:0] output enable
- */
- return ov5640_write_reg(sensor,
- OV5640_REG_PAD_OUTPUT_ENABLE02,
- on ? 0xfc : 0);
+static int ov5640_set_stream_dvp(struct ov5640_dev *sensor, bool on)
+{
+ return ov5640_write_reg(sensor, OV5640_REG_SYS_CTRL0, on ?
+ OV5640_REG_SYS_CTRL0_SW_PWUP :
+ OV5640_REG_SYS_CTRL0_SW_PWDN);
}
static int ov5640_set_stream_mipi(struct ov5640_dev *sensor, bool on)
clk_disable_unprepare(sensor->xclk);
}
-static int ov5640_set_power(struct ov5640_dev *sensor, bool on)
+static int ov5640_set_power_mipi(struct ov5640_dev *sensor, bool on)
{
- int ret = 0;
+ int ret;
+
+ if (!on) {
+ /* Reset MIPI bus settings to their default values. */
+ ov5640_write_reg(sensor, OV5640_REG_IO_MIPI_CTRL00, 0x58);
+ ov5640_write_reg(sensor, OV5640_REG_MIPI_CTRL00, 0x04);
+ ov5640_write_reg(sensor, OV5640_REG_PAD_OUTPUT00, 0x00);
+ return 0;
+ }
+
+ /*
+ * Power up MIPI HS Tx and LS Rx; 2 data lanes mode
+ *
+ * 0x300e = 0x40
+ * [7:5] = 010 : 2 data lanes mode (see FIXME note in
+ * "ov5640_set_stream_mipi()")
+ * [4] = 0 : Power up MIPI HS Tx
+ * [3] = 0 : Power up MIPI LS Rx
+ * [2] = 0 : MIPI interface disabled
+ */
+ ret = ov5640_write_reg(sensor, OV5640_REG_IO_MIPI_CTRL00, 0x40);
+ if (ret)
+ return ret;
+
+ /*
+ * Gate clock and set LP11 in 'no packets mode' (idle)
+ *
+ * 0x4800 = 0x24
+ * [5] = 1 : Gate clock when 'no packets'
+ * [2] = 1 : MIPI bus in LP11 when 'no packets'
+ */
+ ret = ov5640_write_reg(sensor, OV5640_REG_MIPI_CTRL00, 0x24);
+ if (ret)
+ return ret;
+
+ /*
+ * Set data lanes and clock in LP11 when 'sleeping'
+ *
+ * 0x3019 = 0x70
+ * [6] = 1 : MIPI data lane 2 in LP11 when 'sleeping'
+ * [5] = 1 : MIPI data lane 1 in LP11 when 'sleeping'
+ * [4] = 1 : MIPI clock lane in LP11 when 'sleeping'
+ */
+ ret = ov5640_write_reg(sensor, OV5640_REG_PAD_OUTPUT00, 0x70);
+ if (ret)
+ return ret;
+
+ /* Give lanes some time to coax into LP11 state. */
+ usleep_range(500, 1000);
+
+ return 0;
+}
+
+static int ov5640_set_power_dvp(struct ov5640_dev *sensor, bool on)
+{
+ unsigned int flags = sensor->ep.bus.parallel.flags;
+ u8 pclk_pol = 0;
+ u8 hsync_pol = 0;
+ u8 vsync_pol = 0;
+ int ret;
+
+ if (!on) {
+ /* Reset settings to their default values. */
+ ov5640_write_reg(sensor, OV5640_REG_IO_MIPI_CTRL00, 0x58);
+ ov5640_write_reg(sensor, OV5640_REG_POLARITY_CTRL00, 0x20);
+ ov5640_write_reg(sensor, OV5640_REG_PAD_OUTPUT_ENABLE01, 0x00);
+ ov5640_write_reg(sensor, OV5640_REG_PAD_OUTPUT_ENABLE02, 0x00);
+ return 0;
+ }
+
+ /*
+ * Note about parallel port configuration.
+ *
+ * When configured in parallel mode, the OV5640 will
+ * output 10 bits data on DVP data lines [9:0].
+ * If only 8 bits data are wanted, the 8 bits data lines
+ * of the camera interface must be physically connected
+ * on the DVP data lines [9:2].
+ *
+ * Control lines polarity can be configured through
+ * devicetree endpoint control lines properties.
+ * If no endpoint control lines properties are set,
+ * polarity will be as below:
+ * - VSYNC: active high
+ * - HREF: active low
+ * - PCLK: active low
+ */
+ /*
+ * configure parallel port control lines polarity
+ *
+ * POLARITY CTRL0
+ * - [5]: PCLK polarity (0: active low, 1: active high)
+ * - [1]: HREF polarity (0: active low, 1: active high)
+ * - [0]: VSYNC polarity (mismatch here between
+ * datasheet and hardware, 0 is active high
+ * and 1 is active low...)
+ */
+ if (sensor->ep.bus_type == V4L2_MBUS_PARALLEL) {
+ if (flags & V4L2_MBUS_PCLK_SAMPLE_RISING)
+ pclk_pol = 1;
+ if (flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH)
+ hsync_pol = 1;
+ if (flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)
+ vsync_pol = 1;
+
+ ret = ov5640_write_reg(sensor, OV5640_REG_POLARITY_CTRL00,
+ (pclk_pol << 5) | (hsync_pol << 1) |
+ vsync_pol);
- if (on) {
- ret = ov5640_set_power_on(sensor);
if (ret)
return ret;
+ }
- ret = ov5640_restore_mode(sensor);
- if (ret)
- goto power_off;
+ /*
+ * powerdown MIPI TX/RX PHY & disable MIPI
+ *
+ * MIPI CONTROL 00
+ * 4: PWDN PHY TX
+ * 3: PWDN PHY RX
+ * 2: MIPI enable
+ */
+ ret = ov5640_write_reg(sensor, OV5640_REG_IO_MIPI_CTRL00, 0x18);
+ if (ret)
+ return ret;
- /* We're done here for DVP bus, while CSI-2 needs setup. */
- if (sensor->ep.bus_type != V4L2_MBUS_CSI2_DPHY)
- return 0;
+ /*
+ * enable VSYNC/HREF/PCLK DVP control lines
+ * & D[9:6] DVP data lines
+ *
+ * PAD OUTPUT ENABLE 01
+ * - 6: VSYNC output enable
+ * - 5: HREF output enable
+ * - 4: PCLK output enable
+ * - [3:0]: D[9:6] output enable
+ */
+ ret = ov5640_write_reg(sensor, OV5640_REG_PAD_OUTPUT_ENABLE01,
+ sensor->ep.bus_type == V4L2_MBUS_PARALLEL ?
+ 0x7f : 0x1f);
+ if (ret)
+ return ret;
- /*
- * Power up MIPI HS Tx and LS Rx; 2 data lanes mode
- *
- * 0x300e = 0x40
- * [7:5] = 010 : 2 data lanes mode (see FIXME note in
- * "ov5640_set_stream_mipi()")
- * [4] = 0 : Power up MIPI HS Tx
- * [3] = 0 : Power up MIPI LS Rx
- * [2] = 0 : MIPI interface disabled
- */
- ret = ov5640_write_reg(sensor,
- OV5640_REG_IO_MIPI_CTRL00, 0x40);
- if (ret)
- goto power_off;
+ /*
+ * enable D[5:0] DVP data lines
+ *
+ * PAD OUTPUT ENABLE 02
+ * - [7:2]: D[5:0] output enable
+ */
+ return ov5640_write_reg(sensor, OV5640_REG_PAD_OUTPUT_ENABLE02, 0xfc);
+}
- /*
- * Gate clock and set LP11 in 'no packets mode' (idle)
- *
- * 0x4800 = 0x24
- * [5] = 1 : Gate clock when 'no packets'
- * [2] = 1 : MIPI bus in LP11 when 'no packets'
- */
- ret = ov5640_write_reg(sensor,
- OV5640_REG_MIPI_CTRL00, 0x24);
+static int ov5640_set_power(struct ov5640_dev *sensor, bool on)
+{
+ int ret = 0;
+
+ if (on) {
+ ret = ov5640_set_power_on(sensor);
if (ret)
- goto power_off;
+ return ret;
- /*
- * Set data lanes and clock in LP11 when 'sleeping'
- *
- * 0x3019 = 0x70
- * [6] = 1 : MIPI data lane 2 in LP11 when 'sleeping'
- * [5] = 1 : MIPI data lane 1 in LP11 when 'sleeping'
- * [4] = 1 : MIPI clock lane in LP11 when 'sleeping'
- */
- ret = ov5640_write_reg(sensor,
- OV5640_REG_PAD_OUTPUT00, 0x70);
+ ret = ov5640_restore_mode(sensor);
if (ret)
goto power_off;
+ }
- /* Give lanes some time to coax into LP11 state. */
- usleep_range(500, 1000);
-
- } else {
- if (sensor->ep.bus_type == V4L2_MBUS_CSI2_DPHY) {
- /* Reset MIPI bus settings to their default values. */
- ov5640_write_reg(sensor,
- OV5640_REG_IO_MIPI_CTRL00, 0x58);
- ov5640_write_reg(sensor,
- OV5640_REG_MIPI_CTRL00, 0x04);
- ov5640_write_reg(sensor,
- OV5640_REG_PAD_OUTPUT00, 0x00);
- }
+ if (sensor->ep.bus_type == V4L2_MBUS_CSI2_DPHY)
+ ret = ov5640_set_power_mipi(sensor, on);
+ else
+ ret = ov5640_set_power_dvp(sensor, on);
+ if (ret)
+ goto power_off;
+ if (!on)
ov5640_set_power_off(sensor);
- }
return 0;
if (sensor->ep.bus_type == V4L2_MBUS_CSI2_DPHY)
ret = ov5640_set_stream_mipi(sensor, enable);
+ else if (sensor->ep.bus_type == V4L2_MBUS_BT656)
+ ret = ov5640_set_stream_bt656(sensor, enable);
else
ret = ov5640_set_stream_dvp(sensor, enable);
switch (rotation) {
case 180:
sensor->upside_down = true;
- /* fall through */
+ fallthrough;
case 0:
break;
default:
return ret;
}
+ if (sensor->ep.bus_type != V4L2_MBUS_PARALLEL &&
+ sensor->ep.bus_type != V4L2_MBUS_CSI2_DPHY &&
+ sensor->ep.bus_type != V4L2_MBUS_BT656) {
+ dev_err(dev, "Unsupported bus type %d\n", sensor->ep.bus_type);
+ return -EINVAL;
+ }
+
/* get system clock (xclk) */
sensor->xclk = devm_clk_get(dev, "xclk");
if (IS_ERR(sensor->xclk)) {
/* Propagate change of current control to all related controls */
if (ctrl->id == V4L2_CID_VBLANK) {
/* Update max exposure while meeting expected vblanking */
- exposure_max = (ov5675->cur_mode->height + ctrl->val -
- OV5675_EXPOSURE_MAX_MARGIN) / 2;
+ exposure_max = ov5675->cur_mode->height + ctrl->val -
+ OV5675_EXPOSURE_MAX_MARGIN;
__v4l2_ctrl_modify_range(ov5675->exposure,
ov5675->exposure->minimum,
exposure_max, ov5675->exposure->step,
break;
case V4L2_CID_EXPOSURE:
- /* 3 least significant bits of expsoure are fractional part */
+ /* 4 least significant bits of expsoure are fractional part
+ * val = val << 4
+ * for ov5675, the unit of exposure is differnt from other
+ * OmniVision sensors, its exposure value is twice of the
+ * register value, the exposure should be divided by 2 before
+ * set register, e.g. val << 3.
+ */
ret = ov5675_write_reg(ov5675, OV5675_REG_EXPOSURE,
OV5675_REG_VALUE_24BIT, ctrl->val << 3);
break;
v4l2_ctrl_new_std(ctrl_hdlr, &ov5675_ctrl_ops, V4L2_CID_DIGITAL_GAIN,
OV5675_DGTL_GAIN_MIN, OV5675_DGTL_GAIN_MAX,
OV5675_DGTL_GAIN_STEP, OV5675_DGTL_GAIN_DEFAULT);
- exposure_max = (ov5675->cur_mode->vts_def -
- OV5675_EXPOSURE_MAX_MARGIN) / 2;
+ exposure_max = (ov5675->cur_mode->vts_def - OV5675_EXPOSURE_MAX_MARGIN);
ov5675->exposure = v4l2_ctrl_new_std(ctrl_hdlr, &ov5675_ctrl_ops,
V4L2_CID_EXPOSURE,
OV5675_EXPOSURE_MIN, exposure_max,
switch (mf->code) {
case MEDIA_BUS_FMT_Y10_1X10:
mf->code = MEDIA_BUS_FMT_Y8_1X8;
- /* fall through */
+ fallthrough;
case MEDIA_BUS_FMT_Y8_1X8:
case MEDIA_BUS_FMT_YVYU8_2X8:
case MEDIA_BUS_FMT_YUYV8_2X8:
break;
default:
mf->code = MEDIA_BUS_FMT_SBGGR8_1X8;
- /* fall through */
+ fallthrough;
case MEDIA_BUS_FMT_SBGGR8_1X8:
break;
}
};
/* Request bus settings on camera side */
-static int ov6650_g_mbus_config(struct v4l2_subdev *sd,
- struct v4l2_mbus_config *cfg)
+static int ov6650_get_mbus_config(struct v4l2_subdev *sd,
+ unsigned int pad,
+ struct v4l2_mbus_config *cfg)
{
+ struct i2c_client *client = v4l2_get_subdevdata(sd);
+ u8 comj, comf;
+ int ret;
+
+ ret = ov6650_reg_read(client, REG_COMJ, &comj);
+ if (ret)
+ return ret;
- cfg->flags = V4L2_MBUS_MASTER |
- V4L2_MBUS_PCLK_SAMPLE_RISING | V4L2_MBUS_PCLK_SAMPLE_FALLING |
- V4L2_MBUS_HSYNC_ACTIVE_HIGH | V4L2_MBUS_HSYNC_ACTIVE_LOW |
- V4L2_MBUS_VSYNC_ACTIVE_HIGH | V4L2_MBUS_VSYNC_ACTIVE_LOW |
- V4L2_MBUS_DATA_ACTIVE_HIGH;
+ ret = ov6650_reg_read(client, REG_COMF, &comf);
+ if (ret)
+ return ret;
+
+ cfg->flags = V4L2_MBUS_MASTER | V4L2_MBUS_DATA_ACTIVE_HIGH
+ | ((comj & COMJ_VSYNC_HIGH) ? V4L2_MBUS_VSYNC_ACTIVE_HIGH
+ : V4L2_MBUS_VSYNC_ACTIVE_LOW)
+ | ((comf & COMF_HREF_LOW) ? V4L2_MBUS_HSYNC_ACTIVE_LOW
+ : V4L2_MBUS_HSYNC_ACTIVE_HIGH)
+ | ((comj & COMJ_PCLK_RISING) ? V4L2_MBUS_PCLK_SAMPLE_RISING
+ : V4L2_MBUS_PCLK_SAMPLE_FALLING);
cfg->type = V4L2_MBUS_PARALLEL;
return 0;
}
/* Alter bus settings on camera side */
-static int ov6650_s_mbus_config(struct v4l2_subdev *sd,
- const struct v4l2_mbus_config *cfg)
+static int ov6650_set_mbus_config(struct v4l2_subdev *sd,
+ unsigned int pad,
+ struct v4l2_mbus_config *cfg)
{
struct i2c_client *client = v4l2_get_subdevdata(sd);
- int ret;
+ int ret = 0;
if (cfg->flags & V4L2_MBUS_PCLK_SAMPLE_RISING)
ret = ov6650_reg_rmw(client, REG_COMJ, COMJ_PCLK_RISING, 0);
- else
+ else if (cfg->flags & V4L2_MBUS_PCLK_SAMPLE_FALLING)
ret = ov6650_reg_rmw(client, REG_COMJ, 0, COMJ_PCLK_RISING);
if (ret)
return ret;
if (cfg->flags & V4L2_MBUS_HSYNC_ACTIVE_LOW)
ret = ov6650_reg_rmw(client, REG_COMF, COMF_HREF_LOW, 0);
- else
+ else if (cfg->flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH)
ret = ov6650_reg_rmw(client, REG_COMF, 0, COMF_HREF_LOW);
if (ret)
return ret;
if (cfg->flags & V4L2_MBUS_VSYNC_ACTIVE_HIGH)
ret = ov6650_reg_rmw(client, REG_COMJ, COMJ_VSYNC_HIGH, 0);
- else
+ else if (cfg->flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)
ret = ov6650_reg_rmw(client, REG_COMJ, 0, COMJ_VSYNC_HIGH);
+ if (ret)
+ return ret;
- return ret;
+ /*
+ * Update the configuration to report what is actually applied to
+ * the hardware.
+ */
+ return ov6650_get_mbus_config(sd, pad, cfg);
}
static const struct v4l2_subdev_video_ops ov6650_video_ops = {
.s_stream = ov6650_s_stream,
.g_frame_interval = ov6650_g_frame_interval,
.s_frame_interval = ov6650_s_frame_interval,
- .g_mbus_config = ov6650_g_mbus_config,
- .s_mbus_config = ov6650_s_mbus_config,
};
static const struct v4l2_subdev_pad_ops ov6650_pad_ops = {
.set_selection = ov6650_set_selection,
.get_fmt = ov6650_get_fmt,
.set_fmt = ov6650_set_fmt,
+ .get_mbus_config = ov6650_get_mbus_config,
+ .set_mbus_config = ov6650_set_mbus_config,
};
static const struct v4l2_subdev_ops ov6650_subdev_ops = {
struct v4l2_subdev *sd;
int ret;
- if (!i2c_check_functionality(client->adapter,
- I2C_FUNC_SMBUS_BYTE_DATA)) {
- dev_err(&client->dev,
- "OV7740: I2C-Adapter doesn't support SMBUS\n");
- return -EIO;
- }
-
ov7740 = devm_kzalloc(&client->dev, sizeof(*ov7740), GFP_KERNEL);
if (!ov7740)
return -ENOMEM;
if (ret)
return ret;
- ov7740->regmap = devm_regmap_init_i2c(client, &ov7740_regmap_config);
+ ov7740->regmap = devm_regmap_init_sccb(client, &ov7740_regmap_config);
if (IS_ERR(ov7740->regmap)) {
ret = PTR_ERR(ov7740->regmap);
dev_err(&client->dev, "Failed to allocate register map: %d\n",
}
sd = &ov7740->subdev;
- client->flags |= I2C_CLIENT_SCCB;
v4l2_i2c_subdev_init(sd, client, &ov7740_subdev_ops);
#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
{0x5e00, 0x00}
};
+static const struct ov8856_reg mode_3264x2448_regs[] = {
+ {0x0103, 0x01},
+ {0x0302, 0x3c},
+ {0x0303, 0x01},
+ {0x031e, 0x0c},
+ {0x3000, 0x20},
+ {0x3003, 0x08},
+ {0x300e, 0x20},
+ {0x3010, 0x00},
+ {0x3015, 0x84},
+ {0x3018, 0x72},
+ {0x3021, 0x23},
+ {0x3033, 0x24},
+ {0x3500, 0x00},
+ {0x3501, 0x9a},
+ {0x3502, 0x20},
+ {0x3503, 0x08},
+ {0x3505, 0x83},
+ {0x3508, 0x01},
+ {0x3509, 0x80},
+ {0x350c, 0x00},
+ {0x350d, 0x80},
+ {0x350e, 0x04},
+ {0x350f, 0x00},
+ {0x3510, 0x00},
+ {0x3511, 0x02},
+ {0x3512, 0x00},
+ {0x3600, 0x72},
+ {0x3601, 0x40},
+ {0x3602, 0x30},
+ {0x3610, 0xc5},
+ {0x3611, 0x58},
+ {0x3612, 0x5c},
+ {0x3613, 0xca},
+ {0x3614, 0x60},
+ {0x3628, 0xff},
+ {0x3629, 0xff},
+ {0x362a, 0xff},
+ {0x3633, 0x10},
+ {0x3634, 0x10},
+ {0x3635, 0x10},
+ {0x3636, 0x10},
+ {0x3663, 0x08},
+ {0x3669, 0x34},
+ {0x366d, 0x00},
+ {0x366e, 0x10},
+ {0x3706, 0x86},
+ {0x370b, 0x7e},
+ {0x3714, 0x23},
+ {0x3730, 0x12},
+ {0x3733, 0x10},
+ {0x3764, 0x00},
+ {0x3765, 0x00},
+ {0x3769, 0x62},
+ {0x376a, 0x2a},
+ {0x376b, 0x30},
+ {0x3780, 0x00},
+ {0x3781, 0x24},
+ {0x3782, 0x00},
+ {0x3783, 0x23},
+ {0x3798, 0x2f},
+ {0x37a1, 0x60},
+ {0x37a8, 0x6a},
+ {0x37ab, 0x3f},
+ {0x37c2, 0x04},
+ {0x37c3, 0xf1},
+ {0x37c9, 0x80},
+ {0x37cb, 0x16},
+ {0x37cc, 0x16},
+ {0x37cd, 0x16},
+ {0x37ce, 0x16},
+ {0x3800, 0x00},
+ {0x3801, 0x00},
+ {0x3802, 0x00},
+ {0x3803, 0x0c},
+ {0x3804, 0x0c},
+ {0x3805, 0xdf},
+ {0x3806, 0x09},
+ {0x3807, 0xa3},
+ {0x3808, 0x0c},
+ {0x3809, 0xc0},
+ {0x380a, 0x09},
+ {0x380b, 0x90},
+ {0x380c, 0x07},
+ {0x380d, 0x8c},
+ {0x380e, 0x09},
+ {0x380f, 0xb2},
+ {0x3810, 0x00},
+ {0x3811, 0x04},
+ {0x3812, 0x00},
+ {0x3813, 0x02},
+ {0x3814, 0x01},
+ {0x3815, 0x01},
+ {0x3816, 0x00},
+ {0x3817, 0x00},
+ {0x3818, 0x00},
+ {0x3819, 0x10},
+ {0x3820, 0x80},
+ {0x3821, 0x46},
+ {0x382a, 0x01},
+ {0x382b, 0x01},
+ {0x3830, 0x06},
+ {0x3836, 0x02},
+ {0x3862, 0x04},
+ {0x3863, 0x08},
+ {0x3cc0, 0x33},
+ {0x3d85, 0x17},
+ {0x3d8c, 0x73},
+ {0x3d8d, 0xde},
+ {0x4001, 0xe0},
+ {0x4003, 0x40},
+ {0x4008, 0x00},
+ {0x4009, 0x0b},
+ {0x400a, 0x00},
+ {0x400b, 0x84},
+ {0x400f, 0x80},
+ {0x4010, 0xf0},
+ {0x4011, 0xff},
+ {0x4012, 0x02},
+ {0x4013, 0x01},
+ {0x4014, 0x01},
+ {0x4015, 0x01},
+ {0x4042, 0x00},
+ {0x4043, 0x80},
+ {0x4044, 0x00},
+ {0x4045, 0x80},
+ {0x4046, 0x00},
+ {0x4047, 0x80},
+ {0x4048, 0x00},
+ {0x4049, 0x80},
+ {0x4041, 0x03},
+ {0x404c, 0x20},
+ {0x404d, 0x00},
+ {0x404e, 0x20},
+ {0x4203, 0x80},
+ {0x4307, 0x30},
+ {0x4317, 0x00},
+ {0x4502, 0x50},
+ {0x4503, 0x08},
+ {0x4601, 0x80},
+ {0x4800, 0x44},
+ {0x4816, 0x53},
+ {0x481b, 0x50},
+ {0x481f, 0x27},
+ {0x4823, 0x3c},
+ {0x482b, 0x00},
+ {0x4831, 0x66},
+ {0x4837, 0x16},
+ {0x483c, 0x0f},
+ {0x484b, 0x05},
+ {0x5000, 0x77},
+ {0x5001, 0x0a},
+ {0x5003, 0xc8},
+ {0x5004, 0x04},
+ {0x5006, 0x00},
+ {0x5007, 0x00},
+ {0x502e, 0x03},
+ {0x5030, 0x41},
+ {0x5780, 0x14},
+ {0x5781, 0x0f},
+ {0x5782, 0x44},
+ {0x5783, 0x02},
+ {0x5784, 0x01},
+ {0x5785, 0x01},
+ {0x5786, 0x00},
+ {0x5787, 0x04},
+ {0x5788, 0x02},
+ {0x5789, 0x0f},
+ {0x578a, 0xfd},
+ {0x578b, 0xf5},
+ {0x578c, 0xf5},
+ {0x578d, 0x03},
+ {0x578e, 0x08},
+ {0x578f, 0x0c},
+ {0x5790, 0x08},
+ {0x5791, 0x04},
+ {0x5792, 0x00},
+ {0x5793, 0x52},
+ {0x5794, 0xa3},
+ {0x5795, 0x02},
+ {0x5796, 0x20},
+ {0x5797, 0x20},
+ {0x5798, 0xd5},
+ {0x5799, 0xd5},
+ {0x579a, 0x00},
+ {0x579b, 0x50},
+ {0x579c, 0x00},
+ {0x579d, 0x2c},
+ {0x579e, 0x0c},
+ {0x579f, 0x40},
+ {0x57a0, 0x09},
+ {0x57a1, 0x40},
+ {0x59f8, 0x3d},
+ {0x5a08, 0x02},
+ {0x5b00, 0x02},
+ {0x5b01, 0x10},
+ {0x5b02, 0x03},
+ {0x5b03, 0xcf},
+ {0x5b05, 0x6c},
+ {0x5e00, 0x00},
+ {0x5e10, 0xfc}
+};
+
static const struct ov8856_reg mode_1640x1232_regs[] = {
{0x3000, 0x20},
{0x3003, 0x08},
{0x5e00, 0x00}
};
+static const struct ov8856_reg mode_1632x1224_regs[] = {
+ {0x0103, 0x01},
+ {0x0302, 0x3c},
+ {0x0303, 0x01},
+ {0x031e, 0x0c},
+ {0x3000, 0x20},
+ {0x3003, 0x08},
+ {0x300e, 0x20},
+ {0x3010, 0x00},
+ {0x3015, 0x84},
+ {0x3018, 0x72},
+ {0x3021, 0x23},
+ {0x3033, 0x24},
+ {0x3500, 0x00},
+ {0x3501, 0x4c},
+ {0x3502, 0xe0},
+ {0x3503, 0x08},
+ {0x3505, 0x83},
+ {0x3508, 0x01},
+ {0x3509, 0x80},
+ {0x350c, 0x00},
+ {0x350d, 0x80},
+ {0x350e, 0x04},
+ {0x350f, 0x00},
+ {0x3510, 0x00},
+ {0x3511, 0x02},
+ {0x3512, 0x00},
+ {0x3600, 0x72},
+ {0x3601, 0x40},
+ {0x3602, 0x30},
+ {0x3610, 0xc5},
+ {0x3611, 0x58},
+ {0x3612, 0x5c},
+ {0x3613, 0xca},
+ {0x3614, 0x60},
+ {0x3628, 0xff},
+ {0x3629, 0xff},
+ {0x362a, 0xff},
+ {0x3633, 0x10},
+ {0x3634, 0x10},
+ {0x3635, 0x10},
+ {0x3636, 0x10},
+ {0x3663, 0x08},
+ {0x3669, 0x34},
+ {0x366d, 0x00},
+ {0x366e, 0x08},
+ {0x3706, 0x86},
+ {0x370b, 0x7e},
+ {0x3714, 0x27},
+ {0x3730, 0x12},
+ {0x3733, 0x10},
+ {0x3764, 0x00},
+ {0x3765, 0x00},
+ {0x3769, 0x62},
+ {0x376a, 0x2a},
+ {0x376b, 0x30},
+ {0x3780, 0x00},
+ {0x3781, 0x24},
+ {0x3782, 0x00},
+ {0x3783, 0x23},
+ {0x3798, 0x2f},
+ {0x37a1, 0x60},
+ {0x37a8, 0x6a},
+ {0x37ab, 0x3f},
+ {0x37c2, 0x14},
+ {0x37c3, 0xf1},
+ {0x37c9, 0x80},
+ {0x37cb, 0x16},
+ {0x37cc, 0x16},
+ {0x37cd, 0x16},
+ {0x37ce, 0x16},
+ {0x3800, 0x00},
+ {0x3801, 0x00},
+ {0x3802, 0x00},
+ {0x3803, 0x0c},
+ {0x3804, 0x0c},
+ {0x3805, 0xdf},
+ {0x3806, 0x09},
+ {0x3807, 0xa3},
+ {0x3808, 0x06},
+ {0x3809, 0x60},
+ {0x380a, 0x04},
+ {0x380b, 0xc8},
+ {0x380c, 0x07},
+ {0x380d, 0x8c},
+ {0x380e, 0x09},
+ {0x380f, 0xb2},
+ {0x3810, 0x00},
+ {0x3811, 0x02},
+ {0x3812, 0x00},
+ {0x3813, 0x02},
+ {0x3814, 0x03},
+ {0x3815, 0x01},
+ {0x3816, 0x00},
+ {0x3817, 0x00},
+ {0x3818, 0x00},
+ {0x3819, 0x10},
+ {0x3820, 0x80},
+ {0x3821, 0x47},
+ {0x382a, 0x03},
+ {0x382b, 0x01},
+ {0x3830, 0x06},
+ {0x3836, 0x02},
+ {0x3862, 0x04},
+ {0x3863, 0x08},
+ {0x3cc0, 0x33},
+ {0x3d85, 0x17},
+ {0x3d8c, 0x73},
+ {0x3d8d, 0xde},
+ {0x4001, 0xe0},
+ {0x4003, 0x40},
+ {0x4008, 0x00},
+ {0x4009, 0x05},
+ {0x400a, 0x00},
+ {0x400b, 0x84},
+ {0x400f, 0x80},
+ {0x4010, 0xf0},
+ {0x4011, 0xff},
+ {0x4012, 0x02},
+ {0x4013, 0x01},
+ {0x4014, 0x01},
+ {0x4015, 0x01},
+ {0x4042, 0x00},
+ {0x4043, 0x80},
+ {0x4044, 0x00},
+ {0x4045, 0x80},
+ {0x4046, 0x00},
+ {0x4047, 0x80},
+ {0x4048, 0x00},
+ {0x4049, 0x80},
+ {0x4041, 0x03},
+ {0x404c, 0x20},
+ {0x404d, 0x00},
+ {0x404e, 0x20},
+ {0x4203, 0x80},
+ {0x4307, 0x30},
+ {0x4317, 0x00},
+ {0x4502, 0x50},
+ {0x4503, 0x08},
+ {0x4601, 0x80},
+ {0x4800, 0x44},
+ {0x4816, 0x53},
+ {0x481b, 0x50},
+ {0x481f, 0x27},
+ {0x4823, 0x3c},
+ {0x482b, 0x00},
+ {0x4831, 0x66},
+ {0x4837, 0x16},
+ {0x483c, 0x0f},
+ {0x484b, 0x05},
+ {0x5000, 0x77},
+ {0x5001, 0x0a},
+ {0x5003, 0xc8},
+ {0x5004, 0x04},
+ {0x5006, 0x00},
+ {0x5007, 0x00},
+ {0x502e, 0x03},
+ {0x5030, 0x41},
+ {0x5795, 0x00},
+ {0x5796, 0x10},
+ {0x5797, 0x10},
+ {0x5798, 0x73},
+ {0x5799, 0x73},
+ {0x579a, 0x00},
+ {0x579b, 0x28},
+ {0x579c, 0x00},
+ {0x579d, 0x16},
+ {0x579e, 0x06},
+ {0x579f, 0x20},
+ {0x57a0, 0x04},
+ {0x57a1, 0xa0},
+ {0x5780, 0x14},
+ {0x5781, 0x0f},
+ {0x5782, 0x44},
+ {0x5783, 0x02},
+ {0x5784, 0x01},
+ {0x5785, 0x01},
+ {0x5786, 0x00},
+ {0x5787, 0x04},
+ {0x5788, 0x02},
+ {0x5789, 0x0f},
+ {0x578a, 0xfd},
+ {0x578b, 0xf5},
+ {0x578c, 0xf5},
+ {0x578d, 0x03},
+ {0x578e, 0x08},
+ {0x578f, 0x0c},
+ {0x5790, 0x08},
+ {0x5791, 0x04},
+ {0x5792, 0x00},
+ {0x5793, 0x52},
+ {0x5794, 0xa3},
+ {0x59f8, 0x3d},
+ {0x5a08, 0x02},
+ {0x5b00, 0x02},
+ {0x5b01, 0x10},
+ {0x5b02, 0x03},
+ {0x5b03, 0xcf},
+ {0x5b05, 0x6c},
+ {0x5e00, 0x00},
+ {0x5e10, 0xfc}
+};
+
static const char * const ov8856_test_pattern_menu[] = {
"Disabled",
"Standard Color Bar",
},
.link_freq_index = OV8856_LINK_FREQ_720MBPS,
},
+ {
+ .width = 3264,
+ .height = 2448,
+ .hts = 1932,
+ .vts_def = 2482,
+ .vts_min = 2482,
+ .reg_list = {
+ .num_of_regs = ARRAY_SIZE(mode_3264x2448_regs),
+ .regs = mode_3264x2448_regs,
+ },
+ .link_freq_index = OV8856_LINK_FREQ_720MBPS,
+ },
{
.width = 1640,
.height = 1232,
.regs = mode_1640x1232_regs,
},
.link_freq_index = OV8856_LINK_FREQ_360MBPS,
+ },
+ {
+ .width = 1632,
+ .height = 1224,
+ .hts = 1932,
+ .vts_def = 2482,
+ .vts_min = 2482,
+ .reg_list = {
+ .num_of_regs = ARRAY_SIZE(mode_1632x1224_regs),
+ .regs = mode_1632x1224_regs,
+ },
+ .link_freq_index = OV8856_LINK_FREQ_360MBPS,
}
};
break;
default:
mf->code = MEDIA_BUS_FMT_UYVY8_2X8;
- /* fall through */
+ fallthrough;
case MEDIA_BUS_FMT_UYVY8_2X8:
mf->colorspace = V4L2_COLORSPACE_JPEG;
break;
};
/* Request bus settings on camera side */
-static int ov9640_g_mbus_config(struct v4l2_subdev *sd,
- struct v4l2_mbus_config *cfg)
+static int ov9640_get_mbus_config(struct v4l2_subdev *sd,
+ unsigned int pad,
+ struct v4l2_mbus_config *cfg)
{
cfg->flags = V4L2_MBUS_PCLK_SAMPLE_RISING | V4L2_MBUS_MASTER |
V4L2_MBUS_VSYNC_ACTIVE_HIGH | V4L2_MBUS_HSYNC_ACTIVE_HIGH |
static const struct v4l2_subdev_video_ops ov9640_video_ops = {
.s_stream = ov9640_s_stream,
- .g_mbus_config = ov9640_g_mbus_config,
};
static const struct v4l2_subdev_pad_ops ov9640_pad_ops = {
.enum_mbus_code = ov9640_enum_mbus_code,
.get_selection = ov9640_get_selection,
.set_fmt = ov9640_set_fmt,
+ .get_mbus_config = ov9640_get_mbus_config,
};
static const struct v4l2_subdev_ops ov9640_subdev_ops = {
break;
default:
v4l2_info(&state->sensor_sd, "Unknown AF status %#x\n", reg);
- /* Fall through */
+ fallthrough;
case REG_CAF_STATUS_UNFOCUSED:
case REG_AF_STATUS_UNFOCUSED:
case REG_AF_STATUS_INVALID:
struct {
u16 id;
u16 offset;
- } seq[0];
- u16 data[];
+ } seq[];
};
struct s5k5baf {
if (fw == NULL)
return NULL;
- data = fw->data + 2 * fw->count;
+ data = &fw->seq[0].id + 2 * fw->count;
for (i = 0; i < fw->count; ++i) {
if (fw->seq[i].id == seq_id)
sensor->binning_vertical = 1;
}
}
- /* Fall through */
+ fallthrough;
case V4L2_SEL_TGT_COMPOSE:
*crops[SMIAPP_PAD_SRC] = *comp;
break;
&& SMIA_LIM(sensor, SCALING_CAPABILITY)
!= SMIAPP_SCALING_CAPABILITY_NONE)
return 0;
- /* Fall through */
+ fallthrough;
default:
return -EINVAL;
}
case 180:
hwcfg->module_board_orient =
SMIAPP_MODULE_BOARD_ORIENT_180;
- /* Fall through */
+ fallthrough;
case 0:
break;
default:
.adap_monitor_all_enable = tc358743_cec_adap_monitor_all_enable,
};
-static void tc358743_cec_isr(struct v4l2_subdev *sd, u16 intstatus,
- bool *handled)
+static void tc358743_cec_handler(struct v4l2_subdev *sd, u16 intstatus,
+ bool *handled)
{
struct tc358743_state *state = to_state(sd);
unsigned int cec_rxint, cec_txint;
cec_transmit_attempt_done(state->cec_adap,
CEC_TX_STATUS_ERROR);
}
- *handled = true;
+ if (handled)
+ *handled = true;
}
if ((intstatus & MASK_CEC_RINT) &&
(cec_rxint & MASK_CECRIEND)) {
msg.msg[i] = v & 0xff;
}
cec_received_msg(state->cec_adap, &msg);
- *handled = true;
+ if (handled)
+ *handled = true;
}
i2c_wr16(sd, INTSTATUS,
intstatus & (MASK_CEC_RINT | MASK_CEC_TINT));
#ifdef CONFIG_VIDEO_TC358743_CEC
if (intstatus & (MASK_CEC_RINT | MASK_CEC_TINT)) {
- tc358743_cec_isr(sd, intstatus, handled);
+ tc358743_cec_handler(sd, intstatus, handled);
i2c_wr16(sd, INTSTATUS,
intstatus & (MASK_CEC_RINT | MASK_CEC_TINT));
intstatus &= ~(MASK_CEC_RINT | MASK_CEC_TINT);
static irqreturn_t tc358743_irq_handler(int irq, void *dev_id)
{
struct tc358743_state *state = dev_id;
- bool handled;
+ bool handled = false;
tc358743_isr(&state->sd, 0, &handled);
return 0;
}
-static int tc358743_g_mbus_config(struct v4l2_subdev *sd,
- struct v4l2_mbus_config *cfg)
+static int tc358743_get_mbus_config(struct v4l2_subdev *sd,
+ unsigned int pad,
+ struct v4l2_mbus_config *cfg)
{
struct tc358743_state *state = to_state(sd);
.s_dv_timings = tc358743_s_dv_timings,
.g_dv_timings = tc358743_g_dv_timings,
.query_dv_timings = tc358743_query_dv_timings,
- .g_mbus_config = tc358743_g_mbus_config,
.s_stream = tc358743_s_stream,
};
.set_edid = tc358743_s_edid,
.enum_dv_timings = tc358743_enum_dv_timings,
.dv_timings_cap = tc358743_dv_timings_cap,
+ .get_mbus_config = tc358743_get_mbus_config,
};
static const struct v4l2_subdev_ops tc358743_ops = {
{
struct tda1997x_state *state = to_state(sd);
struct tda1997x_platform_data *pdata = &state->pdata;
- bool sp_used_by_fifo = 1;
+ bool sp_used_by_fifo = true;
u8 reg;
if (!pdata->audout_format)
break;
case AUDCFG_TYPE_DST:
reg |= AUDCFG_TYPE_DST << AUDCFG_TYPE_SHIFT;
- sp_used_by_fifo = 0;
+ sp_used_by_fifo = false;
break;
case AUDCFG_TYPE_HBR:
reg |= AUDCFG_TYPE_HBR << AUDCFG_TYPE_SHIFT;
/* demuxed via AP0:AP3 */
reg |= AUDCFG_HBR_DEMUX << AUDCFG_HBR_SHIFT;
if (pdata->audout_format == AUDFMT_TYPE_SPDIF)
- sp_used_by_fifo = 0;
+ sp_used_by_fifo = false;
} else {
/* straight via AP0 */
reg |= AUDCFG_HBR_STRAIGHT << AUDCFG_HBR_SHIFT;
case 36:
mbus_codes[i++] = MEDIA_BUS_FMT_RGB121212_1X36;
mbus_codes[i++] = MEDIA_BUS_FMT_YUV12_1X36;
- /* fall-through */
+ fallthrough;
case 24:
mbus_codes[i++] = MEDIA_BUS_FMT_UYVY12_1X24;
break;
mbus_codes[i++] = MEDIA_BUS_FMT_RGB888_1X24;
mbus_codes[i++] = MEDIA_BUS_FMT_YUV8_1X24;
mbus_codes[i++] = MEDIA_BUS_FMT_UYVY12_1X24;
- /* fall through */
+ fallthrough;
case 20:
mbus_codes[i++] = MEDIA_BUS_FMT_UYVY10_1X20;
- /* fall through */
+ fallthrough;
case 16:
mbus_codes[i++] = MEDIA_BUS_FMT_UYVY8_1X16;
break;
case 16:
case 12:
mbus_codes[i++] = MEDIA_BUS_FMT_UYVY12_2X12;
- /* fall through */
+ fallthrough;
case 10:
mbus_codes[i++] = MEDIA_BUS_FMT_UYVY10_2X10;
- /* fall through */
+ fallthrough;
case 8:
mbus_codes[i++] = MEDIA_BUS_FMT_UYVY8_2X8;
break;
switch (decoder->input) {
case TVP5150_COMPOSITE1:
input |= 2;
- /* fall through */
+ fallthrough;
case TVP5150_COMPOSITE0:
break;
case TVP5150_SVIDEO:
}
}
-static int tvp5150_g_mbus_config(struct v4l2_subdev *sd,
- struct v4l2_mbus_config *cfg)
+static int tvp5150_get_mbus_config(struct v4l2_subdev *sd,
+ unsigned int pad,
+ struct v4l2_mbus_config *cfg)
{
struct tvp5150 *decoder = to_tvp5150(sd);
.querystd = tvp5150_querystd,
.s_stream = tvp5150_s_stream,
.s_routing = tvp5150_s_routing,
- .g_mbus_config = tvp5150_g_mbus_config,
};
static const struct v4l2_subdev_vbi_ops tvp5150_vbi_ops = {
.get_fmt = tvp5150_fill_fmt,
.get_selection = tvp5150_get_selection,
.set_selection = tvp5150_set_selection,
+ .get_mbus_config = tvp5150_get_mbus_config,
};
static const struct v4l2_subdev_ops tvp5150_ops = {
int ret;
ret = tvp7002_read(sd, reg->reg & 0xff, &val);
+ if (ret < 0)
+ return ret;
reg->val = val;
reg->size = 1;
- return ret;
+ return 0;
}
/*
return ret;
}
-static long media_device_request_alloc(struct media_device *mdev,
- int *alloc_fd)
+static long media_device_request_alloc(struct media_device *mdev, void *arg)
{
#ifdef CONFIG_MEDIA_CONTROLLER_REQUEST_API
+ int *alloc_fd = arg;
+
if (!mdev->ops || !mdev->ops->req_validate || !mdev->ops->req_queue)
return -ENOTTY;
#define MEDIA_IOC_ARG(__cmd, func, fl, from_user, to_user) \
[_IOC_NR(MEDIA_IOC_##__cmd)] = { \
.cmd = MEDIA_IOC_##__cmd, \
- .fn = (long (*)(struct media_device *, void *))func, \
+ .fn = func, \
.flags = fl, \
.arg_from_user = from_user, \
.arg_to_user = to_user, \
field = V4L2_FIELD_SEQ_TB;
break;
}
- /* fall through */
+ fallthrough;
default: /* FIELD_ANY case */
height2 = btv->crop[!!fh->do_crop].rect.height >> 1;
field = (f->fmt.pix.height > height2)
btv->id = dev->device;
if (pci_enable_device(dev)) {
pr_warn("%d: Can't enable device\n", btv->c.nr);
- return -EIO;
+ result = -EIO;
+ goto free_mem;
}
if (pci_set_dma_mask(dev, DMA_BIT_MASK(32))) {
pr_warn("%d: No suitable DMA available\n", btv->c.nr);
- return -EIO;
+ result = -EIO;
+ goto free_mem;
}
if (!request_mem_region(pci_resource_start(dev,0),
pci_resource_len(dev,0),
pr_warn("%d: can't request iomem (0x%llx)\n",
btv->c.nr,
(unsigned long long)pci_resource_start(dev, 0));
- return -EBUSY;
+ result = -EBUSY;
+ goto free_mem;
}
pci_set_master(dev);
pci_set_command(dev);
release_mem_region(pci_resource_start(btv->c.pci,0),
pci_resource_len(btv->c.pci,0));
pci_disable_device(btv->c.pci);
+
+free_mem:
+ bttvs[btv->c.nr] = NULL;
+ kfree(btv);
return result;
}
#define IF_FREQUENCYx6 217 /* 6 * 36.16666666667MHz */
-static void dvb_bt8xx_task(unsigned long data)
+static void dvb_bt8xx_task(struct tasklet_struct *t)
{
- struct dvb_bt8xx_card *card = (struct dvb_bt8xx_card *)data;
+ struct bt878 *bt = from_tasklet(bt, t, tasklet);
+ struct dvb_bt8xx_card *card = dev_get_drvdata(&bt->adapter->dev);
dprintk("%d\n", card->bt->finished_block);
goto err_disconnect_frontend;
}
- tasklet_init(&card->bt->tasklet, dvb_bt8xx_task, (unsigned long) card);
+ tasklet_setup(&card->bt->tasklet, dvb_bt8xx_task);
frontend_init(card, type);
iowrite8(data[i], ®s->txr_rxr);
/* Setup command */
- if (i == 0 && start != 0) {
+ if (i == 0 && start) {
/* Write + Start */
cmd = M00018_CR_BITMAP_WR_MSK |
M00018_CR_BITMAP_STA_MSK;
- } else if (i == len - 1 && stop != 0) {
+ } else if (i == len - 1 && stop) {
/* Write + Stop */
cmd = M00018_CR_BITMAP_WR_MSK |
M00018_CR_BITMAP_STO_MSK;
for (i = 0; i < len; i++) {
/* Setup command */
- if (i == 0 && start != 0) {
+ if (i == 0 && start) {
/* Read + Start */
cmd = M00018_CR_BITMAP_RD_MSK |
M00018_CR_BITMAP_STA_MSK;
- } else if (i == len - 1 && stop != 0) {
+ } else if (i == len - 1 && stop) {
/* Read + Stop */
cmd = M00018_CR_BITMAP_RD_MSK |
M00018_CR_BITMAP_STO_MSK;
{
struct cobalt *cobalt = s->cobalt;
- if (is_dma_done(s) == false)
+ if (!is_dma_done(s))
iowrite32(ABORT, CS_REG(s->dma_channel));
}
#define AUD_INT_MCHG_IRQ (1 << 21)
#define GP_COUNT_CONTROL_RESET 0x3
-static int cx23885_alsa_dma_init(struct cx23885_audio_dev *chip, int nr_pages)
+static int cx23885_alsa_dma_init(struct cx23885_audio_dev *chip,
+ unsigned long nr_pages)
{
struct cx23885_audio_buffer *buf = chip->buf;
struct page *pg;
buf->vaddr = vmalloc_32(nr_pages << PAGE_SHIFT);
if (NULL == buf->vaddr) {
- dprintk(1, "vmalloc_32(%d pages) failed\n", nr_pages);
+ dprintk(1, "vmalloc_32(%lu pages) failed\n", nr_pages);
return -ENOMEM;
}
- dprintk(1, "vmalloc is at addr %p, size=%d\n",
+ dprintk(1, "vmalloc is at addr %p, size=%lu\n",
buf->vaddr, nr_pages << PAGE_SHIFT);
memset(buf->vaddr, 0, nr_pages << PAGE_SHIFT);
struct cx23885_audio_buffer *buf = dev->buf;
buf->sglen = dma_map_sg(&dev->pci->dev, buf->sglist,
- buf->nr_pages, PCI_DMA_FROMDEVICE);
+ buf->nr_pages, DMA_FROM_DEVICE);
if (0 == buf->sglen) {
pr_warn("%s: cx23885_alsa_map_sg failed\n", __func__);
if (!buf->sglen)
return 0;
- dma_unmap_sg(&dev->pci->dev, buf->sglist, buf->sglen, PCI_DMA_FROMDEVICE);
+ dma_unmap_sg(&dev->pci->dev, buf->sglist, buf->nr_pages, DMA_FROM_DEVICE);
buf->sglen = 0;
return 0;
}
sprintf(cap->bus_info, "PCIe:%s", pci_name(dev->pci));
cap->capabilities = V4L2_CAP_READWRITE | V4L2_CAP_STREAMING |
V4L2_CAP_AUDIO | V4L2_CAP_VBI_CAPTURE |
- V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VBI_CAPTURE |
+ V4L2_CAP_VIDEO_CAPTURE |
V4L2_CAP_DEVICE_CAPS;
switch (dev->board) { /* i2c device tuners */
case CX23885_BOARD_HAUPPAUGE_HVR1265_K4:
struct cx23885_riscmem risc;
void *vaddr;
struct scatterlist *sglist;
- int sglen;
- int nr_pages;
+ int sglen;
+ unsigned long nr_pages;
};
struct cx23885_audio_dev {
}
v = (unsigned) pulse_width_count_to_ns(
- (u16) (p->hw_fifo_data & FIFO_RXTX), divider);
+ (u16)(p->hw_fifo_data & FIFO_RXTX), divider) / 1000;
if (v > IR_MAX_DURATION)
v = IR_MAX_DURATION;
struct cx25821_riscmem risc;
void *vaddr;
struct scatterlist *sglist;
- int sglen;
- int nr_pages;
+ int sglen;
+ unsigned long nr_pages;
};
struct cx25821_audio_dev {
#define PCI_MSK_AUD_EXT (1 << 4)
#define PCI_MSK_AUD_INT (1 << 3)
-static int cx25821_alsa_dma_init(struct cx25821_audio_dev *chip, int nr_pages)
+static int cx25821_alsa_dma_init(struct cx25821_audio_dev *chip,
+ unsigned long nr_pages)
{
struct cx25821_audio_buffer *buf = chip->buf;
struct page *pg;
buf->vaddr = vmalloc_32(nr_pages << PAGE_SHIFT);
if (NULL == buf->vaddr) {
- dprintk(1, "vmalloc_32(%d pages) failed\n", nr_pages);
+ dprintk(1, "vmalloc_32(%lu pages) failed\n", nr_pages);
return -ENOMEM;
}
- dprintk(1, "vmalloc is at addr 0x%p, size=%d\n",
+ dprintk(1, "vmalloc is at addr 0x%p, size=%lu\n",
buf->vaddr,
nr_pages << PAGE_SHIFT);
struct cx25821_audio_buffer *buf = dev->buf;
buf->sglen = dma_map_sg(&dev->pci->dev, buf->sglist,
- buf->nr_pages, PCI_DMA_FROMDEVICE);
+ buf->nr_pages, DMA_FROM_DEVICE);
if (0 == buf->sglen) {
pr_warn("%s: cx25821_alsa_map_sg failed\n", __func__);
if (!buf->sglen)
return 0;
- dma_unmap_sg(&dev->pci->dev, buf->sglist, buf->sglen, PCI_DMA_FROMDEVICE);
+ dma_unmap_sg(&dev->pci->dev, buf->sglist, buf->nr_pages, DMA_FROM_DEVICE);
buf->sglen = 0;
return 0;
}
*/
struct cx88_audio_buffer {
- unsigned int bpl;
- struct cx88_riscmem risc;
+ unsigned int bpl;
+ struct cx88_riscmem risc;
void *vaddr;
struct scatterlist *sglist;
int sglen;
- int nr_pages;
+ unsigned long nr_pages;
};
struct cx88_audio_dev {
return IRQ_RETVAL(handled);
}
-static int cx88_alsa_dma_init(struct cx88_audio_dev *chip, int nr_pages)
+static int cx88_alsa_dma_init(struct cx88_audio_dev *chip,
+ unsigned long nr_pages)
{
struct cx88_audio_buffer *buf = chip->buf;
struct page *pg;
buf->vaddr = vmalloc_32(nr_pages << PAGE_SHIFT);
if (!buf->vaddr) {
- dprintk(1, "vmalloc_32(%d pages) failed\n", nr_pages);
+ dprintk(1, "vmalloc_32(%lu pages) failed\n", nr_pages);
return -ENOMEM;
}
- dprintk(1, "vmalloc is at addr %p, size=%d\n",
+ dprintk(1, "vmalloc is at addr %p, size=%lu\n",
buf->vaddr, nr_pages << PAGE_SHIFT);
memset(buf->vaddr, 0, nr_pages << PAGE_SHIFT);
struct cx88_audio_buffer *buf = dev->buf;
buf->sglen = dma_map_sg(&dev->pci->dev, buf->sglist,
- buf->nr_pages, PCI_DMA_FROMDEVICE);
+ buf->nr_pages, DMA_FROM_DEVICE);
if (buf->sglen == 0) {
pr_warn("%s: cx88_alsa_map_sg failed\n", __func__);
if (!buf->sglen)
return 0;
- dma_unmap_sg(&dev->pci->dev, buf->sglist, buf->sglen,
- PCI_DMA_FROMDEVICE);
+ dma_unmap_sg(&dev->pci->dev, buf->sglist, buf->nr_pages,
+ DMA_FROM_DEVICE);
buf->sglen = 0;
return 0;
}
cx_clear(MO_GP0_IO, 0x00000040);
msleep(1000);
cx_set(MO_GP0_IO, 0x00004040);
- /* FALLTHROUGH */
+ fallthrough;
case CX88_BOARD_DVICO_FUSIONHDTV_DVB_T1:
case CX88_BOARD_DVICO_FUSIONHDTV_DVB_T_PLUS:
case CX88_BOARD_DVICO_FUSIONHDTV_DVB_T_HYBRID:
dev->scancode_mask = hardware_mask;
if (ir->sampling) {
- dev->timeout = 10 * 1000 * 1000; /* 10 ms */
+ dev->timeout = MS_TO_US(10); /* 10 ms */
} else {
dev->driver_type = RC_DRIVER_SCANCODE;
dev->allowed_protocols = rc_proto;
for (todo = 32; todo > 0; todo -= bits) {
ev.pulse = samples & 0x80000000 ? false : true;
bits = min(todo, 32U - fls(ev.pulse ? samples : ~samples));
- ev.duration = (bits * (NSEC_PER_SEC / 1000)) / ir_samplerate;
+ ev.duration = (bits * (USEC_PER_SEC / 1000)) / ir_samplerate;
ir_raw_event_store_with_filter(ir->dev, &ev);
samples <<= bits;
}
request_module("rtc-isl1208");
core->i2c_rtc = i2c_new_client_device(&core->i2c_adap, &rtc_info);
}
- /* fall-through */
+ fallthrough;
case CX88_BOARD_DVICO_FUSIONHDTV_5_PCI_NANO:
request_module("ir-kbd-i2c");
}
struct dt3155_priv *pd = container_of(v4l2_dev, struct dt3155_priv,
v4l2_dev);
- video_unregister_device(&pd->vdev);
+ vb2_video_unregister_device(&pd->vdev);
free_irq(pd->pdev->irq, pd);
- vb2_queue_release(&pd->vidq);
v4l2_device_unregister(&pd->v4l2_dev);
pci_iounmap(pdev, pd->regs);
pci_release_region(pdev, 0);
// SPDX-License-Identifier: GPL-2.0
/*
- * Copyright (C) 2017 Intel Corporation
+ * Copyright (C) 2017,2020 Intel Corporation
*
* Based partially on Intel IPU4 driver written by
* Sakari Ailus <sakari.ailus@linux.intel.com>
* Jouni Ukkonen <jouni.ukkonen@intel.com>
* Antti Laakso <antti.laakso@intel.com>
* et al.
- *
*/
#include <linux/delay.h>
#include <linux/interrupt.h>
+#include <linux/iopoll.h>
#include <linux/module.h>
#include <linux/pci.h>
+#include <linux/pfn.h>
#include <linux/pm_runtime.h>
#include <linux/property.h>
#include <linux/vmalloc.h>
static void cio2_fbpt_exit_dummy(struct cio2_device *cio2)
{
if (cio2->dummy_lop) {
- dma_free_coherent(&cio2->pci_dev->dev, CIO2_PAGE_SIZE,
+ dma_free_coherent(&cio2->pci_dev->dev, PAGE_SIZE,
cio2->dummy_lop, cio2->dummy_lop_bus_addr);
cio2->dummy_lop = NULL;
}
if (cio2->dummy_page) {
- dma_free_coherent(&cio2->pci_dev->dev, CIO2_PAGE_SIZE,
+ dma_free_coherent(&cio2->pci_dev->dev, PAGE_SIZE,
cio2->dummy_page, cio2->dummy_page_bus_addr);
cio2->dummy_page = NULL;
}
{
unsigned int i;
- cio2->dummy_page = dma_alloc_coherent(&cio2->pci_dev->dev,
- CIO2_PAGE_SIZE,
+ cio2->dummy_page = dma_alloc_coherent(&cio2->pci_dev->dev, PAGE_SIZE,
&cio2->dummy_page_bus_addr,
GFP_KERNEL);
- cio2->dummy_lop = dma_alloc_coherent(&cio2->pci_dev->dev,
- CIO2_PAGE_SIZE,
+ cio2->dummy_lop = dma_alloc_coherent(&cio2->pci_dev->dev, PAGE_SIZE,
&cio2->dummy_lop_bus_addr,
GFP_KERNEL);
if (!cio2->dummy_page || !cio2->dummy_lop) {
* List of Pointers(LOP) contains 1024x32b pointers to 4KB page each
* Initialize each entry to dummy_page bus base address.
*/
- for (i = 0; i < CIO2_PAGE_SIZE / sizeof(*cio2->dummy_lop); i++)
- cio2->dummy_lop[i] = cio2->dummy_page_bus_addr >> PAGE_SHIFT;
+ for (i = 0; i < CIO2_LOP_ENTRIES; i++)
+ cio2->dummy_lop[i] = PFN_DOWN(cio2->dummy_page_bus_addr);
return 0;
}
unsigned int i;
entry[0].first_entry.first_page_offset = 0;
- entry[1].second_entry.num_of_pages =
- CIO2_PAGE_SIZE / sizeof(u32) * CIO2_MAX_LOPS;
- entry[1].second_entry.last_page_available_bytes = CIO2_PAGE_SIZE - 1;
+ entry[1].second_entry.num_of_pages = CIO2_LOP_ENTRIES * CIO2_MAX_LOPS;
+ entry[1].second_entry.last_page_available_bytes = PAGE_SIZE - 1;
for (i = 0; i < CIO2_MAX_LOPS; i++)
- entry[i].lop_page_addr = cio2->dummy_lop_bus_addr >> PAGE_SHIFT;
+ entry[i].lop_page_addr = PFN_DOWN(cio2->dummy_lop_bus_addr);
cio2_fbpt_entry_enable(cio2, entry);
}
entry[0].first_entry.first_page_offset = b->offset;
remaining = length + entry[0].first_entry.first_page_offset;
- entry[1].second_entry.num_of_pages =
- DIV_ROUND_UP(remaining, CIO2_PAGE_SIZE);
+ entry[1].second_entry.num_of_pages = PFN_UP(remaining);
/*
* last_page_available_bytes has the offset of the last byte in the
* last page which is still accessible by DMA. DMA cannot access
* beyond this point. Valid range for this is from 0 to 4095.
* 0 indicates 1st byte in the page is DMA accessible.
- * 4095 (CIO2_PAGE_SIZE - 1) means every single byte in the last page
+ * 4095 (PAGE_SIZE - 1) means every single byte in the last page
* is available for DMA transfer.
*/
entry[1].second_entry.last_page_available_bytes =
(remaining & ~PAGE_MASK) ?
- (remaining & ~PAGE_MASK) - 1 :
- CIO2_PAGE_SIZE - 1;
+ (remaining & ~PAGE_MASK) - 1 : PAGE_SIZE - 1;
/* Fill FBPT */
remaining = length;
i = 0;
while (remaining > 0) {
- entry->lop_page_addr = b->lop_bus_addr[i] >> PAGE_SHIFT;
- remaining -= CIO2_PAGE_SIZE / sizeof(u32) * CIO2_PAGE_SIZE;
+ entry->lop_page_addr = PFN_DOWN(b->lop_bus_addr[i]);
+ remaining -= CIO2_LOP_ENTRIES * PAGE_SIZE;
entry++;
i++;
}
/*
* The first not meaningful FBPT entry should point to a valid LOP
*/
- entry->lop_page_addr = cio2->dummy_lop_bus_addr >> PAGE_SHIFT;
+ entry->lop_page_addr = PFN_DOWN(cio2->dummy_lop_bus_addr);
cio2_fbpt_entry_enable(cio2, entry);
}
struct cio2_csi2_timing *timing)
{
struct device *dev = &cio2->pci_dev->dev;
- struct v4l2_querymenu qm = {.id = V4L2_CID_LINK_FREQ, };
+ struct v4l2_querymenu qm = { .id = V4L2_CID_LINK_FREQ };
struct v4l2_ctrl *link_freq;
s64 freq;
int r;
}
/* Enable DMA */
- writel(q->fbpt_bus_addr >> PAGE_SHIFT,
- base + CIO2_REG_CDMABA(CIO2_DMA_CHAN));
+ writel(PFN_DOWN(q->fbpt_bus_addr), base + CIO2_REG_CDMABA(CIO2_DMA_CHAN));
writel(num_buffers1 << CIO2_CDMAC0_FBPT_LEN_SHIFT |
FBPT_WIDTH << CIO2_CDMAC0_FBPT_WIDTH_SHIFT |
static void cio2_hw_exit(struct cio2_device *cio2, struct cio2_queue *q)
{
- void __iomem *base = cio2->base;
- unsigned int i, maxloops = 1000;
+ void __iomem *const base = cio2->base;
+ unsigned int i;
+ u32 value;
+ int ret;
/* Disable CSI receiver and MIPI backend devices */
writel(0, q->csi_rx_base + CIO2_REG_IRQCTRL_MASK);
/* Halt DMA */
writel(0, base + CIO2_REG_CDMAC0(CIO2_DMA_CHAN));
- do {
- if (readl(base + CIO2_REG_CDMAC0(CIO2_DMA_CHAN)) &
- CIO2_CDMAC0_DMA_HALTED)
- break;
- usleep_range(1000, 2000);
- } while (--maxloops);
- if (!maxloops)
+ ret = readl_poll_timeout(base + CIO2_REG_CDMAC0(CIO2_DMA_CHAN),
+ value, value & CIO2_CDMAC0_DMA_HALTED,
+ 4000, 2000000);
+ if (ret)
dev_err(&cio2->pci_dev->dev,
"DMA %i can not be halted\n", CIO2_DMA_CHAN);
{
struct device *dev = &cio2->pci_dev->dev;
struct cio2_queue *q = cio2->cur_queue;
- int buffers_found = 0;
+ struct cio2_fbpt_entry *entry;
u64 ns = ktime_get_ns();
if (dma_chan >= CIO2_QUEUES) {
return;
}
+ entry = &q->fbpt[q->bufs_first * CIO2_MAX_LOPS];
+ if (entry->first_entry.ctrl & CIO2_FBPT_CTRL_VALID) {
+ dev_warn(&cio2->pci_dev->dev,
+ "no ready buffers found on DMA channel %u\n",
+ dma_chan);
+ return;
+ }
+
/* Find out which buffer(s) are ready */
do {
- struct cio2_fbpt_entry *const entry =
- &q->fbpt[q->bufs_first * CIO2_MAX_LOPS];
struct cio2_buffer *b;
- if (entry->first_entry.ctrl & CIO2_FBPT_CTRL_VALID)
- break;
-
b = q->bufs[q->bufs_first];
if (b) {
unsigned int bytes = entry[1].second_entry.num_of_bytes;
atomic_inc(&q->frame_sequence);
cio2_fbpt_entry_init_dummy(cio2, entry);
q->bufs_first = (q->bufs_first + 1) % CIO2_MAX_BUFFERS;
- buffers_found++;
- } while (1);
-
- if (buffers_found == 0)
- dev_warn(&cio2->pci_dev->dev,
- "no ready buffers found on DMA channel %u\n",
- dma_chan);
+ entry = &q->fbpt[q->bufs_first * CIO2_MAX_LOPS];
+ } while (!(entry->first_entry.ctrl & CIO2_FBPT_CTRL_VALID));
}
static void cio2_queue_event_sof(struct cio2_device *cio2, struct cio2_queue *q)
struct device *dev = &cio2->pci_dev->dev;
struct cio2_buffer *b =
container_of(vb, struct cio2_buffer, vbb.vb2_buf);
- static const unsigned int entries_per_page =
- CIO2_PAGE_SIZE / sizeof(u32);
- unsigned int pages = DIV_ROUND_UP(vb->planes[0].length, CIO2_PAGE_SIZE);
- unsigned int lops = DIV_ROUND_UP(pages + 1, entries_per_page);
+ unsigned int pages = PFN_UP(vb->planes[0].length);
+ unsigned int lops = DIV_ROUND_UP(pages + 1, CIO2_LOP_ENTRIES);
struct sg_table *sg;
struct sg_dma_page_iter sg_iter;
- int i, j;
+ unsigned int i, j;
if (lops <= 0 || lops > CIO2_MAX_LOPS) {
dev_err(dev, "%s: bad buffer size (%i)\n", __func__,
memset(b->lop, 0, sizeof(b->lop));
/* Allocate LOP table */
for (i = 0; i < lops; i++) {
- b->lop[i] = dma_alloc_coherent(dev, CIO2_PAGE_SIZE,
+ b->lop[i] = dma_alloc_coherent(dev, PAGE_SIZE,
&b->lop_bus_addr[i], GFP_KERNEL);
if (!b->lop[i])
goto fail;
b->offset = sg->sgl->offset;
i = j = 0;
- for_each_sg_dma_page (sg->sgl, &sg_iter, sg->nents, 0) {
+ for_each_sg_dma_page(sg->sgl, &sg_iter, sg->nents, 0) {
if (!pages--)
break;
- b->lop[i][j] = sg_page_iter_dma_address(&sg_iter) >> PAGE_SHIFT;
+ b->lop[i][j] = PFN_DOWN(sg_page_iter_dma_address(&sg_iter));
j++;
- if (j == entries_per_page) {
+ if (j == CIO2_LOP_ENTRIES) {
i++;
j = 0;
}
}
- b->lop[i][j] = cio2->dummy_page_bus_addr >> PAGE_SHIFT;
+ b->lop[i][j] = PFN_DOWN(cio2->dummy_page_bus_addr);
return 0;
fail:
- for (i--; i >= 0; i--)
- dma_free_coherent(dev, CIO2_PAGE_SIZE,
- b->lop[i], b->lop_bus_addr[i]);
+ while (i--)
+ dma_free_coherent(dev, PAGE_SIZE, b->lop[i], b->lop_bus_addr[i]);
return -ENOMEM;
}
/* Free LOP table */
for (i = 0; i < CIO2_MAX_LOPS; i++) {
if (b->lop[i])
- dma_free_coherent(&cio2->pci_dev->dev, CIO2_PAGE_SIZE,
+ dma_free_coherent(&cio2->pci_dev->dev, PAGE_SIZE,
b->lop[i], b->lop_bus_addr[i]);
}
}
if (r) {
dev_err(&cio2->pci_dev->dev,
"failed to initialize videobuf2 queue (%d)\n", r);
- goto fail_vbq;
+ goto fail_subdev;
}
/* Initialize vdev */
return 0;
fail_link:
- video_unregister_device(&q->vdev);
+ vb2_video_unregister_device(&q->vdev);
fail_vdev:
- vb2_queue_release(vbq);
-fail_vbq:
v4l2_device_unregister_subdev(subdev);
fail_subdev:
media_entity_cleanup(&vdev->entity);
static void cio2_queue_exit(struct cio2_device *cio2, struct cio2_queue *q)
{
- video_unregister_device(&q->vdev);
+ vb2_video_unregister_device(&q->vdev);
media_entity_cleanup(&q->vdev.entity);
- vb2_queue_release(&q->vbq);
v4l2_device_unregister_subdev(&q->subdev);
media_entity_cleanup(&q->subdev.entity);
cio2_fbpt_exit(q, &cio2->pci_dev->dev);
/**************** PCI interface ****************/
-static int cio2_pci_config_setup(struct pci_dev *dev)
-{
- u16 pci_command;
- int r = pci_enable_msi(dev);
-
- if (r) {
- dev_err(&dev->dev, "failed to enable MSI (%d)\n", r);
- return r;
- }
-
- pci_read_config_word(dev, PCI_COMMAND, &pci_command);
- pci_command |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
- PCI_COMMAND_INTX_DISABLE;
- pci_write_config_word(dev, PCI_COMMAND, pci_command);
-
- return 0;
-}
-
static int cio2_pci_probe(struct pci_dev *pci_dev,
const struct pci_device_id *id)
{
struct cio2_device *cio2;
- void __iomem *const *iomap;
int r;
cio2 = devm_kzalloc(&pci_dev->dev, sizeof(*cio2), GFP_KERNEL);
return -ENODEV;
}
- iomap = pcim_iomap_table(pci_dev);
- if (!iomap) {
- dev_err(&pci_dev->dev, "failed to iomap table\n");
- return -ENODEV;
- }
-
- cio2->base = iomap[CIO2_PCI_BAR];
+ cio2->base = pcim_iomap_table(pci_dev)[CIO2_PCI_BAR];
pci_set_drvdata(pci_dev, cio2);
return -ENODEV;
}
- r = cio2_pci_config_setup(pci_dev);
- if (r)
- return -ENODEV;
+ r = pci_enable_msi(pci_dev);
+ if (r) {
+ dev_err(&pci_dev->dev, "failed to enable MSI (%d)\n", r);
+ return r;
+ }
r = cio2_fbpt_init_dummy(cio2);
if (r)
static int __maybe_unused cio2_resume(struct device *dev)
{
struct cio2_device *cio2 = dev_get_drvdata(dev);
- int r = 0;
struct cio2_queue *q = cio2->cur_queue;
+ int r;
dev_dbg(dev, "cio2 resume\n");
if (!cio2->streaming)
static const struct pci_device_id cio2_pci_id_table[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, CIO2_PCI_ID) },
- { 0 }
+ { }
};
MODULE_DEVICE_TABLE(pci, cio2_pci_id_table);
#ifndef __IPU3_CIO2_H
#define __IPU3_CIO2_H
+#include <linux/types.h>
+
#define CIO2_NAME "ipu3-cio2"
#define CIO2_DEVICE_NAME "Intel IPU3 CIO2"
#define CIO2_ENTITY_NAME "ipu3-csi2"
/* 32MB = 8xFBPT_entry */
#define CIO2_MAX_LOPS 8
#define CIO2_MAX_BUFFERS (PAGE_SIZE / 16 / CIO2_MAX_LOPS)
+#define CIO2_LOP_ENTRIES (PAGE_SIZE / sizeof(u32))
#define CIO2_PAD_SINK 0
#define CIO2_PAD_SOURCE 1
sizeof(struct cio2_fbpt_entry))
#define CIO2_FBPT_SUBENTRY_UNIT 4
-#define CIO2_PAGE_SIZE 4096
/* cio2 fbpt first_entry ctrl status */
#define CIO2_FBPT_CTRL_VALID BIT(0)
}
-void mantis_dma_xfer(unsigned long data)
+void mantis_dma_xfer(struct tasklet_struct *t)
{
- struct mantis_pci *mantis = (struct mantis_pci *) data;
+ struct mantis_pci *mantis = from_tasklet(mantis, t, tasklet);
struct mantis_hwconfig *config = mantis->hwconfig;
while (mantis->last_block != mantis->busy_block) {
extern int mantis_dma_exit(struct mantis_pci *mantis);
extern void mantis_dma_start(struct mantis_pci *mantis);
extern void mantis_dma_stop(struct mantis_pci *mantis);
-extern void mantis_dma_xfer(unsigned long data);
+extern void mantis_dma_xfer(struct tasklet_struct *t);
#endif /* __MANTIS_DMA_H */
}
dvb_net_init(&mantis->dvb_adapter, &mantis->dvbnet, &mantis->demux.dmx);
- tasklet_init(&mantis->tasklet, mantis_dma_xfer, (unsigned long) mantis);
+ tasklet_setup(&mantis->tasklet, mantis_dma_xfer);
tasklet_disable(&mantis->tasklet);
if (mantis->hwconfig) {
result = config->frontend_init(mantis, mantis->fe);
.id_table = netup_unidvb_pci_tbl,
.probe = netup_unidvb_initdev,
.remove = netup_unidvb_finidev,
- .suspend = NULL,
- .resume = NULL,
};
module_pci_driver(netup_unidvb_pci_driver);
/* nGene interrupt handler **************************************************/
/****************************************************************************/
-static void event_tasklet(unsigned long data)
+static void event_tasklet(struct tasklet_struct *t)
{
- struct ngene *dev = (struct ngene *)data;
+ struct ngene *dev = from_tasklet(dev, t, event_tasklet);
while (dev->EventQueueReadIndex != dev->EventQueueWriteIndex) {
struct EVENT_BUFFER Event =
}
}
-static void demux_tasklet(unsigned long data)
+static void demux_tasklet(struct tasklet_struct *t)
{
- struct ngene_channel *chan = (struct ngene_channel *)data;
+ struct ngene_channel *chan = from_tasklet(chan, t, demux_tasklet);
struct device *pdev = &chan->dev->pci_dev->dev;
struct SBufferHeader *Cur = chan->nextBuffer;
struct device *pdev = &dev->pci_dev->dev;
int i;
- tasklet_init(&dev->event_tasklet, event_tasklet, (unsigned long)dev);
+ tasklet_setup(&dev->event_tasklet, event_tasklet);
memset_io(dev->iomem + 0xc000, 0x00, 0x220);
memset_io(dev->iomem + 0xc400, 0x00, 0x100);
struct ngene_info *ni = dev->card_info;
int io = ni->io_type[nr];
- tasklet_init(&chan->demux_tasklet, demux_tasklet, (unsigned long)chan);
+ tasklet_setup(&chan->demux_tasklet, demux_tasklet);
chan->users = 0;
chan->type = io;
chan->mode = chan->type; /* for now only one mode */
return err;
}
-static int saa7134_alsa_dma_init(struct saa7134_dev *dev, int nr_pages)
+static int saa7134_alsa_dma_init(struct saa7134_dev *dev,
+ unsigned long nr_pages)
{
struct saa7134_dmasound *dma = &dev->dmasound;
struct page *pg;
dma->vaddr = vmalloc_32(nr_pages << PAGE_SHIFT);
if (NULL == dma->vaddr) {
- pr_debug("vmalloc_32(%d pages) failed\n", nr_pages);
+ pr_debug("vmalloc_32(%lu pages) failed\n", nr_pages);
return -ENOMEM;
}
- pr_debug("vmalloc is at addr %p, size=%d\n",
+ pr_debug("vmalloc is at addr %p, size=%lu\n",
dma->vaddr, nr_pages << PAGE_SHIFT);
memset(dma->vaddr, 0, nr_pages << PAGE_SHIFT);
struct saa7134_dmasound *dma = &dev->dmasound;
dma->sglen = dma_map_sg(&dev->pci->dev, dma->sglist,
- dma->nr_pages, PCI_DMA_FROMDEVICE);
+ dma->nr_pages, DMA_FROM_DEVICE);
if (0 == dma->sglen) {
pr_warn("%s: saa7134_alsa_map_sg failed\n", __func__);
if (!dma->sglen)
return 0;
- dma_unmap_sg(&dev->pci->dev, dma->sglist, dma->sglen, PCI_DMA_FROMDEVICE);
+ dma_unmap_sg(&dev->pci->dev, dma->sglist, dma->nr_pages, DMA_FROM_DEVICE);
dma->sglen = 0;
return 0;
}
dev->name, saa7134_boards[dev->board].name);
break;
}
- /* fall-through */
+ fallthrough;
case SAA7134_BOARD_VIDEOMATE_DVBT_300:
case SAA7134_BOARD_ASUS_EUROPA2_HYBRID:
case SAA7134_BOARD_ASUS_EUROPA_HYBRID:
break;
case SAA7134_BOARD_HAUPPAUGE_HVR1110:
hauppauge_eeprom(dev, dev->eedata+0x80);
- /* fall-through */
+ fallthrough;
case SAA7134_BOARD_PINNACLE_PCTV_310i:
case SAA7134_BOARD_KWORLD_DVBT_210:
case SAA7134_BOARD_TEVION_DVBT_220RF:
struct saa7134_buf *tmp;
spin_lock_irqsave(&dev->slock, flags);
- if (!list_empty(&q->queue)) {
- list_for_each_safe(pos, n, &q->queue) {
- tmp = list_entry(pos, struct saa7134_buf, entry);
- vb2_buffer_done(&tmp->vb2.vb2_buf,
- VB2_BUF_STATE_ERROR);
- list_del(pos);
- tmp = NULL;
- }
+ list_for_each_safe(pos, n, &q->queue) {
+ tmp = list_entry(pos, struct saa7134_buf, entry);
+ vb2_buffer_done(&tmp->vb2.vb2_buf,
+ VB2_BUF_STATE_ERROR);
+ list_del(pos);
+ tmp = NULL;
}
spin_unlock_irqrestore(&dev->slock, flags);
saa7134_buffer_timeout(&q->timeout); /* also calls del_timer(&q->timeout) */
if (dev->video_dev) {
if (video_is_registered(dev->video_dev))
- video_unregister_device(dev->video_dev);
+ vb2_video_unregister_device(dev->video_dev);
else
video_device_release(dev->video_dev);
dev->video_dev = NULL;
}
if (dev->vbi_dev) {
if (video_is_registered(dev->vbi_dev))
- video_unregister_device(dev->vbi_dev);
+ vb2_video_unregister_device(dev->vbi_dev);
else
video_device_release(dev->vbi_dev);
dev->vbi_dev = NULL;
}
if (dev->radio_dev) {
if (video_is_registered(dev->radio_dev))
- video_unregister_device(dev->radio_dev);
+ vb2_video_unregister_device(dev->radio_dev);
else
video_device_release(dev->radio_dev);
dev->radio_dev = NULL;
kfree(dev);
}
-#ifdef CONFIG_PM
-
/* resends a current buffer in queue after resume */
-static int saa7134_buffer_requeue(struct saa7134_dev *dev,
- struct saa7134_dmaqueue *q)
+static int __maybe_unused saa7134_buffer_requeue(struct saa7134_dev *dev,
+ struct saa7134_dmaqueue *q)
{
struct saa7134_buf *buf, *next;
return 0;
}
-static int saa7134_suspend(struct pci_dev *pci_dev , pm_message_t state)
+static int __maybe_unused saa7134_suspend(struct device *dev_d)
{
+ struct pci_dev *pci_dev = to_pci_dev(dev_d);
struct v4l2_device *v4l2_dev = pci_get_drvdata(pci_dev);
struct saa7134_dev *dev = container_of(v4l2_dev, struct saa7134_dev, v4l2_dev);
if (dev->remote && dev->remote->dev->users)
saa7134_ir_close(dev->remote->dev);
- pci_save_state(pci_dev);
- pci_set_power_state(pci_dev, pci_choose_state(pci_dev, state));
-
return 0;
}
-static int saa7134_resume(struct pci_dev *pci_dev)
+static int __maybe_unused saa7134_resume(struct device *dev_d)
{
- struct v4l2_device *v4l2_dev = pci_get_drvdata(pci_dev);
+ struct v4l2_device *v4l2_dev = dev_get_drvdata(dev_d);
struct saa7134_dev *dev = container_of(v4l2_dev, struct saa7134_dev, v4l2_dev);
unsigned long flags;
- pci_set_power_state(pci_dev, PCI_D0);
- pci_restore_state(pci_dev);
-
/* Do things that are done in saa7134_initdev ,
except of initializing memory structures.*/
return 0;
}
-#endif
/* ----------------------------------------------------------- */
/* ----------------------------------------------------------- */
+static SIMPLE_DEV_PM_OPS(saa7134_pm_ops, saa7134_suspend, saa7134_resume);
+
static struct pci_driver saa7134_pci_driver = {
.name = "saa7134",
.id_table = saa7134_pci_tbl,
.probe = saa7134_initdev,
.remove = saa7134_finidev,
-#ifdef CONFIG_PM
- .suspend = saa7134_suspend,
- .resume = saa7134_resume
-#endif
+ .driver.pm = &saa7134_pm_ops,
};
static int __init saa7134_init(void)
if (NULL == dev->empress_dev)
return 0;
flush_work(&dev->empress_workqueue);
- video_unregister_device(dev->empress_dev);
- vb2_queue_release(&dev->empress_vbq);
+ vb2_video_unregister_device(dev->empress_dev);
v4l2_ctrl_handler_free(&dev->empress_ctrl_handler);
dev->empress_dev = NULL;
return 0;
free_page((unsigned long)saa->bottom);
v4l2_device_unregister_subdev(&saa->sd);
kfree(saa);
- video_unregister_device(&go->vdev);
+ vb2_video_unregister_device(&go->vdev);
v4l2_device_put(&go->v4l2_dev);
dev->empress_dev = NULL;
{
int err;
- audio_dbg(2, "dsp write reg 0x%x = 0x%06x\n", reg << 2, value);
+ audio_dbg(2, "dsp write reg 0x%x = 0x%06x\n",
+ (reg << 2) & 0xffffffff, value);
err = saa_dsp_wait_bit(dev,SAA7135_DSP_RWSTATE_WRR);
if (err < 0)
return err;
void saa7134_video_fini(struct saa7134_dev *dev)
{
/* free stuff */
- vb2_queue_release(&dev->video_vbq);
saa7134_pgtable_free(dev->pci, &dev->video_q.pt);
- vb2_queue_release(&dev->vbi_vbq);
saa7134_pgtable_free(dev->pci, &dev->vbi_q.pt);
v4l2_ctrl_handler_free(&dev->ctrl_handler);
if (card_has_radio(dev))
void *vaddr;
struct scatterlist *sglist;
int sglen;
- int nr_pages;
+ unsigned long nr_pages;
unsigned int dma_blk;
unsigned int read_offset;
unsigned int read_count;
list_for_each_safe(c, n, &port->dmaqueue.list) {
buf = list_entry(c, struct saa7164_buffer, list);
- if (buf->flags != SAA7164_BUFFER_FREE)
- BUG();
+ BUG_ON(buf->flags != SAA7164_BUFFER_FREE);
/* Place the buffer in the h/w queue */
saa7164_buffer_activate(buf, i);
/* Don't exceed the device maximum # bufs */
- if (i++ > port->hwcfg.buffercount)
- BUG();
+ BUG_ON(i > port->hwcfg.buffercount);
+ i++;
}
mutex_unlock(&port->dmaqueue_lock);
kfree(buf);
}
-
/* Find the current write point from the hardware */
wp = saa7164_readl(port->bufcounter);
- if (wp > (port->hwcfg.buffercount - 1))
- BUG();
+
+ BUG_ON(wp > (port->hwcfg.buffercount - 1));
/* Find the previous buffer to the current write point */
if (wp == 0)
/* TODO: turn this into a worker thread */
list_for_each_safe(c, n, &port->dmaqueue.list) {
buf = list_entry(c, struct saa7164_buffer, list);
- if (i++ > port->hwcfg.buffercount)
- BUG();
+ BUG_ON(i > port->hwcfg.buffercount);
+ i++;
if (buf->idx == rp) {
/* Found the buffer, deal with it */
{
struct saa7164_port *port = NULL;
- if ((portnr < 0) || (portnr >= SAA7164_MAX_PORTS))
- BUG();
+ BUG_ON((portnr < 0) || (portnr >= SAA7164_MAX_PORTS));
port = &dev->ports[portnr];
module_init(saa7164_init);
module_exit(saa7164_fini);
-
dprintk(DBGLVL_DVB, "%s(port=%d)\n", __func__, port->nr);
- if (port->type != SAA7164_MPEG_DVB)
- BUG();
+ BUG_ON(port->type != SAA7164_MPEG_DVB);
/* Sanity check that the PCI configuration space is active */
if (port->hwcfg.BARLocation == 0) {
dprintk(DBGLVL_DVB, "%s()\n", __func__);
- if (port->type != SAA7164_MPEG_DVB)
- BUG();
+ BUG_ON(port->type != SAA7164_MPEG_DVB);
/* Remove any allocated buffers */
mutex_lock(&port->dmaqueue_lock);
printk(KERN_ERR "%s() Frontend/I2C initialization failed\n", __func__);
return -1;
}
-
dprintk(DBGLVL_VBI, "%s()\n", __func__);
- if (port->type != SAA7164_MPEG_VBI)
- BUG();
+ BUG_ON(port->type != SAA7164_MPEG_VBI);
/* Sanity check that the PCI configuration space is active */
if (port->hwcfg.BARLocation == 0) {
dprintk(DBGLVL_VBI, "%s(port=%d)\n", __func__, port->nr);
- if (port->type != SAA7164_MPEG_VBI)
- BUG();
+ BUG_ON(port->type != SAA7164_MPEG_VBI);
if (port->v4l_device) {
if (port->v4l_device->minor != -1)
struct ir_raw_event rawir = {};
rawir.pulse = 0;
- rawir.duration = US_TO_NS(SMI_SAMPLE_PERIOD *
- SMI_SAMPLE_IDLEMIN);
+ rawir.duration = SMI_SAMPLE_PERIOD * SMI_SAMPLE_IDLEMIN;
ir_raw_event_store_with_filter(rc_dev, &rawir);
smi_set(IR_Init_Reg, rbIRhighidle);
}
rc_dev->dev.parent = &dev->pci_dev->dev;
rc_dev->map_name = dev->info->rc_map;
- rc_dev->timeout = MS_TO_NS(100);
- rc_dev->rx_resolution = US_TO_NS(SMI_SAMPLE_PERIOD);
+ rc_dev->timeout = MS_TO_US(100);
+ rc_dev->rx_resolution = SMI_SAMPLE_PERIOD;
ir->rc_dev = rc_dev;
ir->dev = dev;
}
/* tasklet handler: DMA data to dmx.*/
-static void smi_dma_xfer(unsigned long data)
+static void smi_dma_xfer(struct tasklet_struct *t)
{
- struct smi_port *port = (struct smi_port *) data;
+ struct smi_port *port = from_tasklet(port, t, tasklet);
struct smi_dev *dev = port->dev;
u32 intr_status, finishedData, dmaManagement;
u8 dmaChan0State, dmaChan1State;
}
smi_port_disableInterrupt(port);
- tasklet_init(&port->tasklet, smi_dma_xfer, (unsigned long)port);
+ tasklet_setup(&port->tasklet, smi_dma_xfer);
tasklet_disable(&port->tasklet);
port->enable = 1;
return 0;
default:
dev_warn(&pdev->dev, "Invalid chip_id 0x%02x, assuming 4 ch\n",
chip_id);
- /* fall through */
+ fallthrough;
case 5:
solo_dev->nr_chans = 4;
solo_dev->nr_ext = 1;
}
solo_dev->i2c_state = IIC_STATE_WRITE;
- /* fall through */
+ fallthrough;
case IIC_STATE_WRITE:
ret = solo_i2c_handle_write(solo_dev);
break;
vunreg:
video_set_drvdata(&vip->video_dev, NULL);
vrelease:
- video_unregister_device(&vip->video_dev);
+ vb2_video_unregister_device(&vip->video_dev);
free_irq(pdev->irq, vip);
release_buf:
pci_disable_msi(pdev);
unmap:
- vb2_queue_release(&vip->vb_vidq);
pci_iounmap(pdev, vip->iomem);
release:
pci_release_regions(pdev);
sta2x11_vip_clear_register(vip);
video_set_drvdata(&vip->video_dev, NULL);
- video_unregister_device(&vip->video_dev);
+ vb2_video_unregister_device(&vip->video_dev);
free_irq(pdev->irq, vip);
pci_disable_msi(pdev);
- vb2_queue_release(&vip->vb_vidq);
pci_iounmap(pdev, vip->iomem);
pci_release_regions(pdev);
irdebi(av7110, DEBISWAB, addr, 0, len);
}
-static void debiirq(unsigned long cookie)
+static void debiirq(struct tasklet_struct *t)
{
- struct av7110 *av7110 = (struct av7110 *)cookie;
+ struct av7110 *av7110 = from_tasklet(av7110, t, debi_tasklet);
int type = av7110->debitype;
int handle = (type >> 8) & 0x1f;
unsigned int xfer = 0;
}
/* irq from av7110 firmware writing the mailbox register in the DPRAM */
-static void gpioirq(unsigned long cookie)
+static void gpioirq(struct tasklet_struct *t)
{
- struct av7110 *av7110 = (struct av7110 *)cookie;
+ struct av7110 *av7110 = from_tasklet(av7110, t, gpio_tasklet);
u32 rxbuf, txbuf;
int len;
return status;
}
-static void vpeirq(unsigned long cookie)
+static void vpeirq(struct tasklet_struct *t)
{
- struct av7110 *budget = (struct av7110 *)cookie;
+ struct av7110 *budget = from_tasklet(budget, t, vpe_tasklet);
u8 *mem = (u8 *) (budget->grabbing);
u32 olddma = budget->ttbp;
u32 newdma = saa7146_read(budget->dev, PCI_VDP3);
saa7146_write(dev, NUM_LINE_BYTE3, (TS_HEIGHT << 16) | TS_WIDTH);
saa7146_write(dev, MC2, MASK_04 | MASK_20);
- tasklet_init(&av7110->vpe_tasklet, vpeirq, (unsigned long) av7110);
+ tasklet_setup(&av7110->vpe_tasklet, vpeirq);
} else if (budgetpatch) {
spin_lock_init(&av7110->feedlock1);
saa7146_write(dev, MC1, (MASK_13 | MASK_29));
/* end of budgetpatch register initialization */
- tasklet_init (&av7110->vpe_tasklet, vpeirq, (unsigned long) av7110);
+ tasklet_setup(&av7110->vpe_tasklet, vpeirq);
} else {
saa7146_write(dev, PCI_BT_V1, 0x1c00101f);
saa7146_write(dev, BCS_CTRL, 0x80400040);
saa7146_write(dev, GPIO_CTRL, 0x000000);
}
- tasklet_init (&av7110->debi_tasklet, debiirq, (unsigned long) av7110);
- tasklet_init (&av7110->gpio_tasklet, gpioirq, (unsigned long) av7110);
+ tasklet_setup(&av7110->debi_tasklet, debiirq);
+ tasklet_setup(&av7110->gpio_tasklet, gpioirq);
mutex_init(&av7110->pid_mutex);
buf[1] = div & 0xff;
buf[2] = 0x8e;
- if (freq < (u32) (16 * 168.25))
+ if (freq < 16U * 16825 / 100)
config = 0xa0;
- else if (freq < (u32) (16 * 447.25))
+ else if (freq < 16U * 44725 / 100)
config = 0x90;
else
config = 0x30;
u8 tuner_pll_address; /* used for philips_tdm1316l configs */
};
-static void msp430_ir_interrupt(unsigned long data)
+static void msp430_ir_interrupt(struct tasklet_struct *t)
{
- struct budget_ci *budget_ci = (struct budget_ci *) data;
+ struct budget_ci_ir *ir = from_tasklet(ir, t, msp430_irq_tasklet);
+ struct budget_ci *budget_ci = container_of(ir, typeof(*budget_ci), ir);
struct rc_dev *dev = budget_ci->ir.dev;
u32 command = ttpci_budget_debiread(&budget_ci->budget, DEBINOSWAP, DEBIADDR_IR, 2, 1, 0) >> 8;
budget_ci->ir.dev = dev;
- tasklet_init(&budget_ci->ir.msp430_irq_tasklet, msp430_ir_interrupt,
- (unsigned long) budget_ci);
+ tasklet_setup(&budget_ci->ir.msp430_irq_tasklet, msp430_ir_interrupt);
SAA7146_IER_ENABLE(saa, MASK_06);
saa7146_setgpio(saa, 3, SAA7146_GPIO_IRQHI);
return 0;
}
-static void ciintf_interrupt(unsigned long data)
+static void ciintf_interrupt(struct tasklet_struct *t)
{
- struct budget_ci *budget_ci = (struct budget_ci *) data;
+ struct budget_ci *budget_ci = from_tasklet(budget_ci, t,
+ ciintf_irq_tasklet);
struct saa7146_dev *saa = budget_ci->budget.dev;
unsigned int flags;
// Setup CI slot IRQ
if (budget_ci->ci_irq) {
- tasklet_init(&budget_ci->ciintf_irq_tasklet, ciintf_interrupt, (unsigned long) budget_ci);
+ tasklet_setup(&budget_ci->ciintf_irq_tasklet, ciintf_interrupt);
if (budget_ci->slot_status != SLOTSTATUS_NONE) {
saa7146_setgpio(saa, 0, SAA7146_GPIO_IRQLO);
} else {
return ret;
}
-static void vpeirq(unsigned long data)
+static void vpeirq(struct tasklet_struct *t)
{
- struct budget *budget = (struct budget *) data;
+ struct budget *budget = from_tasklet(budget, t, vpe_tasklet);
u8 *mem = (u8 *) (budget->grabbing);
u32 olddma = budget->ttbp;
u32 newdma = saa7146_read(budget->dev, PCI_VDP3);
/* upload all */
saa7146_write(dev, GPIO_CTRL, 0x000000);
- tasklet_init(&budget->vpe_tasklet, vpeirq, (unsigned long) budget);
+ tasklet_setup(&budget->vpe_tasklet, vpeirq);
/* frontend power on */
if (bi->type != BUDGET_FS_ACTIVY)
static v4l2_std_id tw5864_get_v4l2_std(enum tw5864_vid_std std);
static enum tw5864_vid_std tw5864_from_v4l2_std(v4l2_std_id v4l2_std);
-static void tw5864_handle_frame_task(unsigned long data);
+static void tw5864_handle_frame_task(struct tasklet_struct *t);
static void tw5864_handle_frame(struct tw5864_h264_frame *frame);
static void tw5864_frame_interval_set(struct tw5864_input *input);
fintv->type = V4L2_FRMIVAL_TYPE_STEPWISE;
ret = tw5864_frameinterval_get(input, &frameinterval);
+ if (ret)
+ return ret;
+
fintv->stepwise.step = frameinterval;
fintv->stepwise.min = frameinterval;
fintv->stepwise.max = frameinterval;
cp->capability = V4L2_CAP_TIMEPERFRAME;
ret = tw5864_frameinterval_get(input, &cp->timeperframe);
+ if (ret)
+ return ret;
+
cp->timeperframe.numerator *= input->frame_interval;
cp->capturemode = 0;
cp->readbuffers = 2;
dev->irqmask |= TW5864_INTR_VLC_DONE | TW5864_INTR_TIMER;
tw5864_irqmask_apply(dev);
- tasklet_init(&dev->tasklet, tw5864_handle_frame_task,
- (unsigned long)dev);
+ tasklet_setup(&dev->tasklet, tw5864_handle_frame_task);
for (i = 0; i < TW5864_INPUTS; i++) {
dev->inputs[i].root = dev;
free_v4l2_hdl:
v4l2_ctrl_handler_free(hdl);
- vb2_queue_release(&input->vidq);
free_mutex:
mutex_destroy(&input->lock);
static void tw5864_video_input_fini(struct tw5864_input *dev)
{
- video_unregister_device(&dev->vdev);
+ vb2_video_unregister_device(&dev->vdev);
v4l2_ctrl_handler_free(&dev->hdl);
- vb2_queue_release(&dev->vidq);
}
void tw5864_video_fini(struct tw5864_dev *dev)
return detected;
}
-static void tw5864_handle_frame_task(unsigned long data)
+static void tw5864_handle_frame_task(struct tasklet_struct *t)
{
- struct tw5864_dev *dev = (struct tw5864_dev *)data;
+ struct tw5864_dev *dev = from_tasklet(dev, t, tasklet);
unsigned long flags;
int batch_size = H264_BUF_CNT;
select VIDEOBUF2_DMA_CONTIG
select V4L2_MEM2MEM_DEV
select VIDEO_MEDIATEK_VPU
+ select MTK_SCP
help
Mediatek video codec driver provides HW capability to
encode and decode in a range of video formats
This driver rely on VPU driver to communicate with VPU.
- To compile this driver as a module, choose M here: the
- module will be called mtk-vcodec
+ To compile this driver as modules, choose M here: the
+ modules will be called mtk-vcodec-dec and mtk-vcodec-enc.
config VIDEO_MEM2MEM_DEINTERLACE
tristate "Deinterlace support"
help
This is a driver for the Renesas Frame Compression Processor (FCP).
The FCP is a companion module of video processing modules in the
- Renesas R-Car Gen3 SoCs. It handles memory access for the codec,
- VSP and FDP modules.
+ Renesas R-Car Gen3 and RZ/G2 SoCs. It handles memory access for
+ the codec, VSP and FDP modules.
To compile this driver as a module, choose M here: the module
will be called rcar-fcp.
video_set_drvdata(vdev, video);
rc = video_register_device(vdev, VFL_TYPE_VIDEO, 0);
if (rc) {
- vb2_queue_release(vbq);
v4l2_ctrl_handler_free(&video->ctrl_handler);
v4l2_device_unregister(v4l2_dev);
clk_unprepare(video->vclk);
clk_unprepare(video->eclk);
- video_unregister_device(&video->vdev);
-
- vb2_queue_release(&video->queue);
+ vb2_video_unregister_device(&video->vdev);
v4l2_ctrl_handler_free(&video->ctrl_handler);
break;
case CODA_960:
coda_write(dev, 0, CODA9_GDI_WPROT_RGN_EN);
- /* fallthrough */
+ fallthrough;
case CODA_HX4:
case CODA_7541:
coda_write(dev, CODA7_STREAM_BUF_DYNALLOC_EN |
CODA7_PICHEIGHT_MASK) << CODA_PICHEIGHT_OFFSET;
break;
}
- /* fallthrough */
+ fallthrough;
case CODA_960:
value = (q_data_src->rect.width & CODA7_PICWIDTH_MASK)
<< CODA7_PICWIDTH_OFFSET;
ctx->tiled_map_type = GDI_TILED_FRAME_MB_RASTER_MAP;
break;
}
- /* else fall through */
+ fallthrough;
case V4L2_PIX_FMT_YUV420:
case V4L2_PIX_FMT_YVU420:
case V4L2_PIX_FMT_YUV422P:
case V4L2_SEL_TGT_CROP_DEFAULT:
case V4L2_SEL_TGT_CROP_BOUNDS:
rsel = &r;
- /* fallthrough */
+ fallthrough;
case V4L2_SEL_TGT_CROP:
if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT ||
ctx->inst_type == CODA_INST_DECODER)
case V4L2_SEL_TGT_COMPOSE_BOUNDS:
case V4L2_SEL_TGT_COMPOSE_PADDED:
rsel = &r;
- /* fallthrough */
+ fallthrough;
case V4L2_SEL_TGT_COMPOSE:
case V4L2_SEL_TGT_COMPOSE_DEFAULT:
if (s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE ||
return 0;
}
- /* else fall through */
+ fallthrough;
case V4L2_SEL_TGT_NATIVE_SIZE:
case V4L2_SEL_TGT_COMPOSE:
return coda_g_selection(file, fh, s);
buf->blob.size = size;
buf->dentry = debugfs_create_blob(name, 0644, parent,
&buf->blob);
- if (!buf->dentry)
- dev_warn(dev->dev,
- "failed to create debugfs entry %s\n", name);
}
return 0;
*/
if (enable_bwb || ctx->inst_type == CODA_INST_ENCODER)
ctx->frame_mem_ctrl = CODA9_FRAME_ENABLE_BWB;
- /* fallthrough */
+ fallthrough;
case CODA_HX4:
case CODA_7541:
ctx->reg_idx = 0;
ida_init(&dev->ida);
dev->debugfs_root = debugfs_create_dir("coda", NULL);
- if (!dev->debugfs_root)
- dev_warn(&pdev->dev, "failed to create debugfs root\n");
/* allocate auxiliary per-device buffers for the BIT processor */
if (dev->devtype->product == CODA_DX6) {
return 0;
err_alloc_workqueue:
+ pm_runtime_disable(&pdev->dev);
+ pm_runtime_put_noidle(&pdev->dev);
destroy_workqueue(dev->workqueue);
err_v4l2_register:
v4l2_device_unregister(&dev->v4l2_dev);
ret = fimc_pipeline_call(ve, open, &ve->vdev.entity, true);
- if (ret == 0 && vc->user_subdev_api && vc->inh_sensor_ctrls) {
- /*
- * Recreate controls of the the video node to drop
- * any controls inherited from the sensor subdev.
- */
- fimc_ctrls_delete(vc->ctx);
-
- ret = fimc_ctrls_create(vc->ctx);
- if (ret == 0)
- vc->inh_sensor_ctrls = false;
- }
if (ret == 0)
ve->vdev.entity.use_count++;
if (ret < 0)
return ret;
- media_pipeline_stop(&vc->ve.vdev.entity);
- vc->streaming = false;
+ if (vc->streaming) {
+ media_pipeline_stop(&vc->ve.vdev.entity);
+ vc->streaming = false;
+ }
+
return 0;
}
case V4L2_SEL_TGT_COMPOSE_DEFAULT:
case V4L2_SEL_TGT_COMPOSE_BOUNDS:
f = &ctx->d_frame;
- /* fall through */
+ fallthrough;
case V4L2_SEL_TGT_CROP_BOUNDS:
case V4L2_SEL_TGT_CROP_DEFAULT:
s->r.left = 0;
case V4L2_SEL_TGT_COMPOSE:
f = &ctx->d_frame;
- /* fall through */
+ fallthrough;
case V4L2_SEL_TGT_CROP:
s->r.left = f->offs_h;
s->r.top = f->offs_v;
vc->input = sd->grp_id;
- if (vc->user_subdev_api || vc->inh_sensor_ctrls)
+ if (vc->user_subdev_api)
return 0;
/* Inherit V4L2 controls from the image sensor subdev. */
switch (sel->target) {
case V4L2_SEL_TGT_COMPOSE_BOUNDS:
f = &ctx->d_frame;
- /* fall through */
+ fallthrough;
case V4L2_SEL_TGT_CROP_BOUNDS:
r->width = f->o_width;
r->height = f->o_height;
return ret;
sd->entity.ops = &fimc_sd_media_ops;
+ sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_SCALER;
sd->internal_ops = &fimc_capture_sd_internal_ops;
v4l2_set_subdevdata(sd, fimc);
return 0;
spin_lock_init(&fimc->slock);
mutex_init(&fimc->lock);
- fimc->sysreg = fimc_get_sysreg_regmap(dev->of_node);
- if (IS_ERR(fimc->sysreg))
- return PTR_ERR(fimc->sysreg);
+ if (fimc->variant->has_isp_wb) {
+ fimc->sysreg = fimc_get_sysreg_regmap(dev->of_node);
+ if (IS_ERR(fimc->sysreg))
+ return PTR_ERR(fimc->sysreg);
+ }
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
fimc->regs = devm_ioremap_resource(dev, res);
return 0;
}
-/* Image pixel limits, similar across several FIMC HW revisions. */
-static const struct fimc_pix_limit s5p_pix_limit[4] = {
- [0] = {
- .scaler_en_w = 3264,
- .scaler_dis_w = 8192,
- .out_rot_en_w = 1920,
- .out_rot_dis_w = 4224,
- },
- [1] = {
- .scaler_en_w = 4224,
- .scaler_dis_w = 8192,
- .out_rot_en_w = 1920,
- .out_rot_dis_w = 4224,
- },
- [2] = {
- .scaler_en_w = 1920,
- .scaler_dis_w = 8192,
- .out_rot_en_w = 1280,
- .out_rot_dis_w = 1920,
- },
-};
-
-static const struct fimc_variant fimc0_variant_s5pv210 = {
- .has_inp_rot = 1,
- .has_out_rot = 1,
- .has_cam_if = 1,
- .min_inp_pixsize = 16,
- .min_out_pixsize = 16,
- .hor_offs_align = 8,
- .min_vsize_align = 16,
- .pix_limit = &s5p_pix_limit[1],
-};
-
-static const struct fimc_variant fimc1_variant_s5pv210 = {
- .has_inp_rot = 1,
- .has_out_rot = 1,
- .has_cam_if = 1,
- .has_mainscaler_ext = 1,
- .min_inp_pixsize = 16,
- .min_out_pixsize = 16,
- .hor_offs_align = 1,
- .min_vsize_align = 1,
- .pix_limit = &s5p_pix_limit[2],
-};
-
-static const struct fimc_variant fimc2_variant_s5pv210 = {
- .has_cam_if = 1,
- .min_inp_pixsize = 16,
- .min_out_pixsize = 16,
- .hor_offs_align = 8,
- .min_vsize_align = 16,
- .pix_limit = &s5p_pix_limit[2],
-};
-
/* S5PV210, S5PC110 */
static const struct fimc_drvdata fimc_drvdata_s5pv210 = {
- .variant = {
- [0] = &fimc0_variant_s5pv210,
- [1] = &fimc1_variant_s5pv210,
- [2] = &fimc2_variant_s5pv210,
- },
.num_entities = 3,
.lclk_frequency = 166000000UL,
.out_buf_count = 4,
* @buf_index: index for managing the output DMA buffers
* @frame_count: the frame counter for statistics
* @reqbufs_count: the number of buffers requested in REQBUFS ioctl
- * @input_index: input (camera sensor) index
* @input: capture input type, grp_id of the attached subdev
* @user_subdev_api: true if subdevs are not configured by the host driver
- * @inh_sensor_ctrls: a flag indicating v4l2 controls are inherited from
- * an image sensor subdev
*/
struct fimc_vid_cap {
struct fimc_ctx *ctx;
unsigned int frame_count;
unsigned int reqbufs_count;
bool streaming;
- int input_index;
u32 input;
bool user_subdev_api;
- bool inh_sensor_ctrls;
};
/**
is->debugfs_entry = NULL;
}
-static int fimc_is_debugfs_create(struct fimc_is *is)
+static void fimc_is_debugfs_create(struct fimc_is *is)
{
- struct dentry *dentry;
-
is->debugfs_entry = debugfs_create_dir("fimc_is", NULL);
- dentry = debugfs_create_file("fw_log", S_IRUGO, is->debugfs_entry,
- is, &fimc_is_fops);
- if (!dentry)
- fimc_is_debugfs_remove(is);
-
- return is->debugfs_entry == NULL ? -EIO : 0;
+ debugfs_create_file("fw_log", S_IRUGO, is->debugfs_entry, is,
+ &fimc_is_fops);
}
static int fimc_is_runtime_resume(struct device *dev);
if (ret < 0)
goto err_pm;
- ret = fimc_is_debugfs_create(is);
- if (ret < 0)
- goto err_sd;
+ fimc_is_debugfs_create(is);
ret = fimc_is_request_firmware(is, FIMC_IS_FW_FILENAME);
if (ret < 0)
err_dfs:
fimc_is_debugfs_remove(is);
-err_sd:
fimc_is_unregister_subdevs(is);
err_pm:
pm_runtime_put_noidle(dev);
if (on) {
ret = pm_runtime_get_sync(&is->pdev->dev);
- if (ret < 0)
+ if (ret < 0) {
+ pm_runtime_put(&is->pdev->dev);
return ret;
+ }
set_bit(IS_ST_PWR_ON, &is->state);
ret = fimc_is_start_firmware(is);
set_bit(ST_FLITE_IN_USE, &fimc->state);
ret = pm_runtime_get_sync(&fimc->pdev->dev);
if (ret < 0)
- goto unlock;
+ goto err_pm;
ret = v4l2_fh_open(file);
if (ret < 0)
switch (source->fimc_bus_type) {
case FIMC_BUS_TYPE_ITU_601:
case FIMC_BUS_TYPE_ITU_656:
+ if (fimc_fmt_is_user_defined(f->fmt->color)) {
+ cfg |= FIMC_REG_CISRCFMT_ITU601_8BIT;
+ break;
+ }
+
for (i = 0; i < ARRAY_SIZE(pix_desc); i++) {
if (vc->ci_fmt.code == pix_desc[i].pixelcode) {
cfg = pix_desc[i].cisrcfmt;
case FIMC_BUS_TYPE_ITU_601...FIMC_BUS_TYPE_ITU_656:
if (source->mux_id == 0) /* ITU-A, ITU-B: 0, 1 */
cfg |= FIMC_REG_CIGCTRL_SELCAM_ITU_A;
+ if (vid_cap->ci_fmt.code == MEDIA_BUS_FMT_JPEG_1X8)
+ cfg |= FIMC_REG_CIGCTRL_CAM_JPEG;
break;
case FIMC_BUS_TYPE_LCD_WRITEBACK_A:
cfg |= FIMC_REG_CIGCTRL_CAMIF_SELWB;
- /* fall through */
+ fallthrough;
case FIMC_BUS_TYPE_ISP_WRITEBACK:
if (fimc->variant->has_isp_wb)
cfg |= FIMC_REG_CIGCTRL_CAMIF_SELWB;
#include <linux/of_platform.h>
#include <linux/of_device.h>
#include <linux/of_graph.h>
+#include <linux/pinctrl/consumer.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/types.h>
switch (sd->grp_id) {
case GRP_ID_SENSOR:
sensor = sd;
- /* fall through */
+ fallthrough;
case GRP_ID_FIMC_IS_SENSOR:
p->subdevs[IDX_SENSOR] = sd;
break;
{ IDX_CSIS, IDX_FLITE, IDX_FIMC, IDX_SENSOR, IDX_IS_ISP },
};
struct fimc_pipeline *p = to_fimc_pipeline(ep);
- struct fimc_md *fmd = entity_to_fimc_mdev(&p->subdevs[IDX_CSIS]->entity);
enum fimc_subdev_index sd_id;
int i, ret = 0;
if (p->subdevs[IDX_SENSOR] == NULL) {
+ struct fimc_md *fmd;
+ struct v4l2_subdev *sd = p->subdevs[IDX_CSIS];
+
+ if (!sd)
+ sd = p->subdevs[IDX_FIMC];
+
+ if (!sd) {
+ /*
+ * If neither CSIS nor FIMC was set up,
+ * it's impossible to have any sensors
+ */
+ return -ENODEV;
+ }
+
+ fmd = entity_to_fimc_mdev(&sd->entity);
+
if (!fmd->user_subdev_api) {
/*
* Sensor must be already discovered if we
}
}
-/* Parse port node and register as a sub-device any sensor specified there. */
-static int fimc_md_parse_port_node(struct fimc_md *fmd,
- struct device_node *port,
- unsigned int index)
+static int fimc_md_parse_one_endpoint(struct fimc_md *fmd,
+ struct device_node *ep)
{
+ int index = fmd->num_sensors;
struct fimc_source_info *pd = &fmd->sensor[index].pdata;
- struct device_node *rem, *ep, *np;
+ struct device_node *rem, *np;
struct v4l2_fwnode_endpoint endpoint = { .bus_type = 0 };
int ret;
- /* Assume here a port node can have only one endpoint node. */
- ep = of_get_next_child(port, NULL);
- if (!ep)
- return 0;
-
ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep), &endpoint);
if (ret) {
of_node_put(ep);
return 0;
}
+/* Parse port node and register as a sub-device any sensor specified there. */
+static int fimc_md_parse_port_node(struct fimc_md *fmd,
+ struct device_node *port)
+{
+ struct device_node *ep;
+ int ret;
+
+ for_each_child_of_node(port, ep) {
+ ret = fimc_md_parse_one_endpoint(fmd, ep);
+ if (ret < 0)
+ return ret;
+ }
+
+ return 0;
+}
+
/* Register all SoC external sub-devices */
static int fimc_md_register_sensor_entities(struct fimc_md *fmd)
{
struct device_node *parent = fmd->pdev->dev.of_node;
struct device_node *ports = NULL;
struct device_node *node;
- int index = 0;
int ret;
/*
return -ENXIO;
ret = pm_runtime_get_sync(fmd->pmf);
- if (ret < 0)
+ if (ret < 0) {
+ pm_runtime_put(fmd->pmf);
return ret;
+ }
fmd->num_sensors = 0;
if (!port)
continue;
- ret = fimc_md_parse_port_node(fmd, port, index);
+ ret = fimc_md_parse_port_node(fmd, port);
of_node_put(port);
if (ret < 0) {
of_node_put(node);
goto cleanup;
}
- index++;
}
/* Attach sensors listed in the parallel-ports node */
goto rpm_put;
for_each_child_of_node(ports, node) {
- ret = fimc_md_parse_port_node(fmd, node, index);
+ ret = fimc_md_parse_port_node(fmd, node);
if (ret < 0) {
of_node_put(node);
goto cleanup;
}
- index++;
}
of_node_put(ports);
static DEVICE_ATTR(subdev_conf_mode, S_IWUSR | S_IRUGO,
fimc_md_sysfs_show, fimc_md_sysfs_store);
-static int fimc_md_get_pinctrl(struct fimc_md *fmd)
-{
- struct device *dev = &fmd->pdev->dev;
- struct fimc_pinctrl *pctl = &fmd->pinctl;
-
- pctl->pinctrl = devm_pinctrl_get(dev);
- if (IS_ERR(pctl->pinctrl))
- return PTR_ERR(pctl->pinctrl);
-
- pctl->state_default = pinctrl_lookup_state(pctl->pinctrl,
- PINCTRL_STATE_DEFAULT);
- if (IS_ERR(pctl->state_default))
- return PTR_ERR(pctl->state_default);
-
- pctl->state_idle = pinctrl_lookup_state(pctl->pinctrl,
- PINCTRL_STATE_IDLE);
- if (IS_ERR(pctl->state_idle))
- return PTR_ERR(pctl->state_idle);
-
- return 0;
-}
-
static int cam_clk_prepare(struct clk_hw *hw)
{
struct cam_clk *camclk = to_cam_clk(hw);
{
struct device *dev = &pdev->dev;
struct v4l2_device *v4l2_dev;
+ struct pinctrl *pinctrl;
struct fimc_md *fmd;
int ret;
if (ret)
goto err_v4l2dev;
- ret = fimc_md_get_pinctrl(fmd);
- if (ret < 0) {
+ pinctrl = devm_pinctrl_get(dev);
+ if (IS_ERR(pinctrl)) {
+ ret = PTR_ERR(pinctrl);
if (ret != EPROBE_DEFER)
dev_err(dev, "Failed to get pinctrl: %d\n", ret);
goto err_clk;
#define FIMC_IS_OF_NODE_NAME "fimc-is"
#define CSIS_OF_NODE_NAME "csis"
-#define PINCTRL_STATE_IDLE "idle"
-
#define FIMC_MAX_SENSORS 4
#define FIMC_MAX_CAMCLKS 2
#define DEFAULT_SENSOR_CLK_FREQ 24000000U
* @media_dev: top level media device
* @v4l2_dev: top level v4l2_device holding up the subdevs
* @pdev: platform device this media device is hooked up into
- * @pinctrl: camera port pinctrl handle
- * @state_default: pinctrl default state handle
- * @state_idle: pinctrl idle state handle
* @cam_clk_provider: CAMCLK clock provider structure
* @user_subdev_api: true if subdevs are not configured by the host driver
* @slock: spinlock protecting @sensor array
struct v4l2_device v4l2_dev;
struct platform_device *pdev;
- struct fimc_pinctrl {
- struct pinctrl *pinctrl;
- struct pinctrl_state *state_default;
- struct pinctrl_state *state_idle;
- } pinctl;
-
struct cam_clk_provider {
struct clk *clks[FIMC_MAX_CAMCLKS];
struct clk_onecell_data clk_data;
if (enable) {
s5pcsis_clear_counters(state);
ret = pm_runtime_get_sync(&state->pdev->dev);
- if (ret && ret != 1)
+ if (ret && ret != 1) {
+ pm_runtime_put_noidle(&state->pdev->dev);
return ret;
+ }
}
mutex_lock(&state->lock);
#define VIU_VERSION "0.5.1"
/* Allow building this driver with COMPILE_TEST */
-#if !defined(CONFIG_PPC) && !defined(CONFIG_MICROBLAZE)
+#if !defined(CONFIG_PPC) && !defined(CONFIG_MICROBLAZE) && !defined(CONFIG_M68K)
#define out_be32(v, a) iowrite32be(a, (void __iomem *)v)
#define in_be32(a) ioread32be((void __iomem *)a)
#endif
cam = kzalloc(sizeof(struct cafe_camera), GFP_KERNEL);
if (cam == NULL)
goto out;
+ pci_set_drvdata(pdev, cam);
cam->pdev = pdev;
mcam = &cam->mcam;
mcam->chip_id = MCAM_CAFE;
static void cafe_pci_remove(struct pci_dev *pdev)
{
- struct v4l2_device *v4l2_dev = dev_get_drvdata(&pdev->dev);
- struct cafe_camera *cam = to_cam(v4l2_dev);
+ struct cafe_camera *cam = pci_get_drvdata(pdev);
if (cam == NULL) {
printk(KERN_WARNING "pci_remove on unknown pdev %p\n", pdev);
*/
static int __maybe_unused cafe_pci_suspend(struct device *dev)
{
- struct v4l2_device *v4l2_dev = dev_get_drvdata(dev);
- struct cafe_camera *cam = to_cam(v4l2_dev);
+ struct cafe_camera *cam = dev_get_drvdata(dev);
mccic_suspend(&cam->mcam);
return 0;
static int __maybe_unused cafe_pci_resume(struct device *dev)
{
- struct v4l2_device *v4l2_dev = dev_get_drvdata(dev);
- struct cafe_camera *cam = to_cam(v4l2_dev);
+ struct cafe_camera *cam = dev_get_drvdata(dev);
cafe_ctlr_init(&cam->mcam);
return mccic_resume(&cam->mcam);
#include <linux/clk.h>
#include <linux/clk-provider.h>
#include <linux/videodev2.h>
+#include <linux/pm_runtime.h>
#include <media/v4l2-device.h>
#include <media/v4l2-ioctl.h>
#include <media/v4l2-ctrls.h>
dma_free_coherent(cam->dev, cam->dma_buf_size,
cam->dma_bufs[0], cam->dma_handles[0]);
cam->nbufs = 0;
- /* fall-through */
+ fallthrough;
case 0:
cam_err(cam, "Insufficient DMA buffers, cannot operate\n");
return -ENOMEM;
/*
* Copy data out to user space in the vmalloc case
*/
-static void mcam_frame_tasklet(unsigned long data)
+static void mcam_frame_tasklet(struct tasklet_struct *t)
{
- struct mcam_camera *cam = (struct mcam_camera *) data;
+ struct mcam_camera *cam = from_tasklet(cam, t, s_tasklet);
int i;
unsigned long flags;
struct mcam_vb_buffer *buf;
spin_unlock_irqrestore(&cam->dev_lock, flags);
}
-/* ---------------------------------------------------------------------- */
-/*
- * Controller clocks.
- */
-static void mcam_clk_enable(struct mcam_camera *mcam)
-{
- unsigned int i;
-
- for (i = 0; i < NR_MCAM_CLK; i++) {
- if (!IS_ERR(mcam->clk[i]))
- clk_prepare_enable(mcam->clk[i]);
- }
-}
-
-static void mcam_clk_disable(struct mcam_camera *mcam)
-{
- int i;
-
- for (i = NR_MCAM_CLK - 1; i >= 0; i--) {
- if (!IS_ERR(mcam->clk[i]))
- clk_disable_unprepare(mcam->clk[i]);
- }
-}
-
/* ---------------------------------------------------------------------- */
/*
* Master sensor clock.
break;
case B_vmalloc:
#ifdef MCAM_MODE_VMALLOC
- tasklet_init(&cam->s_tasklet, mcam_frame_tasklet,
- (unsigned long) cam);
+ tasklet_setup(&cam->s_tasklet, mcam_frame_tasklet);
vq->ops = &mcam_vb2_ops;
vq->mem_ops = &vb2_vmalloc_memops;
cam->dma_setup = mcam_ctlr_dma_vmalloc;
ret = sensor_call(cam, core, s_power, 1);
if (ret)
goto out;
- mcam_clk_enable(cam);
+ pm_runtime_get_sync(cam->dev);
__mcam_cam_reset(cam);
mcam_set_config_needed(cam, 1);
}
if (last_open) {
mcam_disable_mipi(cam);
sensor_call(cam, core, s_power, 0);
- mcam_clk_disable(cam);
+ pm_runtime_put(cam->dev);
if (cam->buffer_mode == B_vmalloc && alloc_bufs_at_read)
mcam_free_dma_bufs(cam);
}
mcam_ctlr_stop_dma(cam);
sensor_call(cam, core, s_power, 0);
- mcam_clk_disable(cam);
cam->state = cstate;
}
mutex_unlock(&cam->s_mutex);
mutex_lock(&cam->s_mutex);
if (!list_empty(&cam->vdev.fh_list)) {
- mcam_clk_enable(cam);
ret = sensor_call(cam, core, s_power, 1);
if (ret) {
mutex_unlock(&cam->s_mutex);
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
#include <linux/io.h>
#include <linux/list.h>
#include <linux/pm.h>
return container_of(mcam, struct mmp_camera, mcam);
}
-/*
- * A silly little infrastructure so we can keep track of our devices.
- * Chances are that we will never have more than one of them, but
- * the Armada 610 *does* have two controllers...
- */
-
-static LIST_HEAD(mmpcam_devices);
-static struct mutex mmpcam_devices_lock;
-
-static void mmpcam_add_device(struct mmp_camera *cam)
-{
- mutex_lock(&mmpcam_devices_lock);
- list_add(&cam->devlist, &mmpcam_devices);
- mutex_unlock(&mmpcam_devices_lock);
-}
-
-static void mmpcam_remove_device(struct mmp_camera *cam)
-{
- mutex_lock(&mmpcam_devices_lock);
- list_del(&cam->devlist);
- mutex_unlock(&mmpcam_devices_lock);
-}
-
-/*
- * Platform dev remove passes us a platform_device, and there's
- * no handy unused drvdata to stash a backpointer in. So just
- * dig it out of our list.
- */
-static struct mmp_camera *mmpcam_find_device(struct platform_device *pdev)
-{
- struct mmp_camera *cam;
-
- mutex_lock(&mmpcam_devices_lock);
- list_for_each_entry(cam, &mmpcam_devices, devlist) {
- if (cam->pdev == pdev) {
- mutex_unlock(&mmpcam_devices_lock);
- return cam;
- }
- }
- mutex_unlock(&mmpcam_devices_lock);
- return NULL;
-}
-
/*
* calc the dphy register values
* There are three dphy registers being used.
cam = devm_kzalloc(&pdev->dev, sizeof(*cam), GFP_KERNEL);
if (cam == NULL)
return -ENOMEM;
+ platform_set_drvdata(pdev, cam);
cam->pdev = pdev;
INIT_LIST_HEAD(&cam->devlist);
cam->irq = res->start;
ret = devm_request_irq(&pdev->dev, cam->irq, mmpcam_irq, IRQF_SHARED,
"mmp-camera", mcam);
- if (ret == 0) {
- mmpcam_add_device(cam);
- return 0;
- }
+ if (ret)
+ goto out;
+ pm_runtime_enable(&pdev->dev);
+ return 0;
out:
fwnode_handle_put(mcam->asd.match.fwnode);
mccic_shutdown(mcam);
{
struct mcam_camera *mcam = &cam->mcam;
- mmpcam_remove_device(cam);
mccic_shutdown(mcam);
+ pm_runtime_force_suspend(mcam->dev);
return 0;
}
static int mmpcam_platform_remove(struct platform_device *pdev)
{
- struct mmp_camera *cam = mmpcam_find_device(pdev);
+ struct mmp_camera *cam = platform_get_drvdata(pdev);
if (cam == NULL)
return -ENODEV;
/*
* Suspend/resume support.
*/
-#ifdef CONFIG_PM
-static int mmpcam_suspend(struct platform_device *pdev, pm_message_t state)
+static int mmpcam_runtime_resume(struct device *dev)
+{
+ struct mmp_camera *cam = dev_get_drvdata(dev);
+ struct mcam_camera *mcam = &cam->mcam;
+ unsigned int i;
+
+ for (i = 0; i < NR_MCAM_CLK; i++) {
+ if (!IS_ERR(mcam->clk[i]))
+ clk_prepare_enable(mcam->clk[i]);
+ }
+
+ return 0;
+}
+
+static int mmpcam_runtime_suspend(struct device *dev)
+{
+ struct mmp_camera *cam = dev_get_drvdata(dev);
+ struct mcam_camera *mcam = &cam->mcam;
+ int i;
+
+ for (i = NR_MCAM_CLK - 1; i >= 0; i--) {
+ if (!IS_ERR(mcam->clk[i]))
+ clk_disable_unprepare(mcam->clk[i]);
+ }
+
+ return 0;
+}
+
+static int __maybe_unused mmpcam_suspend(struct device *dev)
{
- struct mmp_camera *cam = mmpcam_find_device(pdev);
+ struct mmp_camera *cam = dev_get_drvdata(dev);
- if (state.event != PM_EVENT_SUSPEND)
- return 0;
- mccic_suspend(&cam->mcam);
+ if (!pm_runtime_suspended(dev))
+ mccic_suspend(&cam->mcam);
return 0;
}
-static int mmpcam_resume(struct platform_device *pdev)
+static int __maybe_unused mmpcam_resume(struct device *dev)
{
- struct mmp_camera *cam = mmpcam_find_device(pdev);
+ struct mmp_camera *cam = dev_get_drvdata(dev);
- return mccic_resume(&cam->mcam);
+ if (!pm_runtime_suspended(dev))
+ return mccic_resume(&cam->mcam);
+ return 0;
}
-#endif
+static const struct dev_pm_ops mmpcam_pm_ops = {
+ SET_RUNTIME_PM_OPS(mmpcam_runtime_suspend, mmpcam_runtime_resume, NULL)
+ SET_SYSTEM_SLEEP_PM_OPS(mmpcam_suspend, mmpcam_resume)
+};
static const struct of_device_id mmpcam_of_match[] = {
{ .compatible = "marvell,mmp2-ccic", },
static struct platform_driver mmpcam_driver = {
.probe = mmpcam_probe,
.remove = mmpcam_platform_remove,
-#ifdef CONFIG_PM
- .suspend = mmpcam_suspend,
- .resume = mmpcam_resume,
-#endif
.driver = {
.name = "mmp-camera",
.of_match_table = of_match_ptr(mmpcam_of_match),
+ .pm = &mmpcam_pm_ops,
}
};
-
-static int __init mmpcam_init_module(void)
-{
- mutex_init(&mmpcam_devices_lock);
- return platform_driver_register(&mmpcam_driver);
-}
-
-static void __exit mmpcam_exit_module(void)
-{
- platform_driver_unregister(&mmpcam_driver);
- /*
- * platform_driver_unregister() should have emptied the list
- */
- if (!list_empty(&mmpcam_devices))
- printk(KERN_ERR "mmp_camera leaving devices behind\n");
-}
-
-module_init(mmpcam_init_module);
-module_exit(mmpcam_exit_module);
+module_platform_driver(mmpcam_driver);
# SPDX-License-Identifier: GPL-2.0-only
-mtk_jpeg-objs := mtk_jpeg_core.o mtk_jpeg_hw.o mtk_jpeg_parse.o
+mtk_jpeg-objs := mtk_jpeg_core.o \
+ mtk_jpeg_dec_hw.o \
+ mtk_jpeg_dec_parse.o \
+ mtk_jpeg_enc_hw.o
obj-$(CONFIG_VIDEO_MEDIATEK_JPEG) += mtk_jpeg.o
* Copyright (c) 2016 MediaTek Inc.
* Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
* Rick Chang <rick.chang@mediatek.com>
+ * Xia Jiang <xia.jiang@mediatek.com>
*/
#include <linux/clk.h>
#include <media/videobuf2-dma-contig.h>
#include <soc/mediatek/smi.h>
-#include "mtk_jpeg_hw.h"
+#include "mtk_jpeg_enc_hw.h"
+#include "mtk_jpeg_dec_hw.h"
#include "mtk_jpeg_core.h"
-#include "mtk_jpeg_parse.h"
+#include "mtk_jpeg_dec_parse.h"
-static struct mtk_jpeg_fmt mtk_jpeg_formats[] = {
+static struct mtk_jpeg_fmt mtk_jpeg_enc_formats[] = {
{
.fourcc = V4L2_PIX_FMT_JPEG,
.colplanes = 1,
- .flags = MTK_JPEG_FMT_FLAG_DEC_OUTPUT,
+ .flags = MTK_JPEG_FMT_FLAG_CAPTURE,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_NV12M,
+ .hw_format = JPEG_ENC_YUV_FORMAT_NV12,
+ .h_sample = {4, 4},
+ .v_sample = {4, 2},
+ .colplanes = 2,
+ .h_align = 4,
+ .v_align = 4,
+ .flags = MTK_JPEG_FMT_FLAG_OUTPUT,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_NV21M,
+ .hw_format = JEPG_ENC_YUV_FORMAT_NV21,
+ .h_sample = {4, 4},
+ .v_sample = {4, 2},
+ .colplanes = 2,
+ .h_align = 4,
+ .v_align = 4,
+ .flags = MTK_JPEG_FMT_FLAG_OUTPUT,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_YUYV,
+ .hw_format = JPEG_ENC_YUV_FORMAT_YUYV,
+ .h_sample = {8},
+ .v_sample = {4},
+ .colplanes = 1,
+ .h_align = 5,
+ .v_align = 3,
+ .flags = MTK_JPEG_FMT_FLAG_OUTPUT,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_YVYU,
+ .hw_format = JPEG_ENC_YUV_FORMAT_YVYU,
+ .h_sample = {8},
+ .v_sample = {4},
+ .colplanes = 1,
+ .h_align = 5,
+ .v_align = 3,
+ .flags = MTK_JPEG_FMT_FLAG_OUTPUT,
+ },
+};
+
+static struct mtk_jpeg_fmt mtk_jpeg_dec_formats[] = {
+ {
+ .fourcc = V4L2_PIX_FMT_JPEG,
+ .colplanes = 1,
+ .flags = MTK_JPEG_FMT_FLAG_OUTPUT,
},
{
.fourcc = V4L2_PIX_FMT_YUV420M,
.colplanes = 3,
.h_align = 5,
.v_align = 4,
- .flags = MTK_JPEG_FMT_FLAG_DEC_CAPTURE,
+ .flags = MTK_JPEG_FMT_FLAG_CAPTURE,
},
{
.fourcc = V4L2_PIX_FMT_YUV422M,
.colplanes = 3,
.h_align = 5,
.v_align = 3,
- .flags = MTK_JPEG_FMT_FLAG_DEC_CAPTURE,
+ .flags = MTK_JPEG_FMT_FLAG_CAPTURE,
},
};
-#define MTK_JPEG_NUM_FORMATS ARRAY_SIZE(mtk_jpeg_formats)
-
-enum {
- MTK_JPEG_BUF_FLAGS_INIT = 0,
- MTK_JPEG_BUF_FLAGS_LAST_FRAME = 1,
-};
+#define MTK_JPEG_ENC_NUM_FORMATS ARRAY_SIZE(mtk_jpeg_enc_formats)
+#define MTK_JPEG_DEC_NUM_FORMATS ARRAY_SIZE(mtk_jpeg_dec_formats)
struct mtk_jpeg_src_buf {
struct vb2_v4l2_buffer b;
struct list_head list;
- int flags;
struct mtk_jpeg_dec_param dec_param;
};
static int debug;
module_param(debug, int, 0644);
+static inline struct mtk_jpeg_ctx *ctrl_to_ctx(struct v4l2_ctrl *ctrl)
+{
+ return container_of(ctrl->handler, struct mtk_jpeg_ctx, ctrl_hdl);
+}
+
static inline struct mtk_jpeg_ctx *mtk_jpeg_fh_to_ctx(struct v4l2_fh *fh)
{
return container_of(fh, struct mtk_jpeg_ctx, fh);
{
struct mtk_jpeg_dev *jpeg = video_drvdata(file);
- strscpy(cap->driver, MTK_JPEG_NAME " decoder", sizeof(cap->driver));
- strscpy(cap->card, MTK_JPEG_NAME " decoder", sizeof(cap->card));
+ strscpy(cap->driver, jpeg->variant->dev_name, sizeof(cap->driver));
+ strscpy(cap->card, jpeg->variant->dev_name, sizeof(cap->card));
snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
dev_name(jpeg->dev));
return 0;
}
+static int vidioc_jpeg_enc_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+ struct mtk_jpeg_ctx *ctx = ctrl_to_ctx(ctrl);
+
+ switch (ctrl->id) {
+ case V4L2_CID_JPEG_RESTART_INTERVAL:
+ ctx->restart_interval = ctrl->val;
+ break;
+ case V4L2_CID_JPEG_COMPRESSION_QUALITY:
+ ctx->enc_quality = ctrl->val;
+ break;
+ case V4L2_CID_JPEG_ACTIVE_MARKER:
+ ctx->enable_exif = ctrl->val & V4L2_JPEG_ACTIVE_MARKER_APP1;
+ break;
+ }
+
+ return 0;
+}
+
+static const struct v4l2_ctrl_ops mtk_jpeg_enc_ctrl_ops = {
+ .s_ctrl = vidioc_jpeg_enc_s_ctrl,
+};
+
+static int mtk_jpeg_enc_ctrls_setup(struct mtk_jpeg_ctx *ctx)
+{
+ const struct v4l2_ctrl_ops *ops = &mtk_jpeg_enc_ctrl_ops;
+ struct v4l2_ctrl_handler *handler = &ctx->ctrl_hdl;
+
+ v4l2_ctrl_handler_init(handler, 3);
+
+ v4l2_ctrl_new_std(handler, ops, V4L2_CID_JPEG_RESTART_INTERVAL, 0, 100,
+ 1, 0);
+ v4l2_ctrl_new_std(handler, ops, V4L2_CID_JPEG_COMPRESSION_QUALITY, 48,
+ 100, 1, 90);
+ v4l2_ctrl_new_std(handler, ops, V4L2_CID_JPEG_ACTIVE_MARKER, 0,
+ V4L2_JPEG_ACTIVE_MARKER_APP1, 0, 0);
+
+ if (handler->error) {
+ v4l2_ctrl_handler_free(&ctx->ctrl_hdl);
+ return handler->error;
+ }
+
+ v4l2_ctrl_handler_setup(&ctx->ctrl_hdl);
+
+ return 0;
+}
+
static int mtk_jpeg_enum_fmt(struct mtk_jpeg_fmt *mtk_jpeg_formats, int n,
struct v4l2_fmtdesc *f, u32 type)
{
static int mtk_jpeg_enum_fmt_vid_cap(struct file *file, void *priv,
struct v4l2_fmtdesc *f)
{
- return mtk_jpeg_enum_fmt(mtk_jpeg_formats, MTK_JPEG_NUM_FORMATS, f,
- MTK_JPEG_FMT_FLAG_DEC_CAPTURE);
+ struct mtk_jpeg_ctx *ctx = mtk_jpeg_fh_to_ctx(priv);
+ struct mtk_jpeg_dev *jpeg = ctx->jpeg;
+
+ return mtk_jpeg_enum_fmt(jpeg->variant->formats,
+ jpeg->variant->num_formats, f,
+ MTK_JPEG_FMT_FLAG_CAPTURE);
}
static int mtk_jpeg_enum_fmt_vid_out(struct file *file, void *priv,
struct v4l2_fmtdesc *f)
{
- return mtk_jpeg_enum_fmt(mtk_jpeg_formats, MTK_JPEG_NUM_FORMATS, f,
- MTK_JPEG_FMT_FLAG_DEC_OUTPUT);
+ struct mtk_jpeg_ctx *ctx = mtk_jpeg_fh_to_ctx(priv);
+ struct mtk_jpeg_dev *jpeg = ctx->jpeg;
+
+ return mtk_jpeg_enum_fmt(jpeg->variant->formats,
+ jpeg->variant->num_formats, f,
+ MTK_JPEG_FMT_FLAG_OUTPUT);
}
static struct mtk_jpeg_q_data *mtk_jpeg_get_q_data(struct mtk_jpeg_ctx *ctx,
return &ctx->cap_q;
}
-static struct mtk_jpeg_fmt *mtk_jpeg_find_format(struct mtk_jpeg_ctx *ctx,
- u32 pixelformat,
- unsigned int fmt_type)
+static struct mtk_jpeg_fmt *
+mtk_jpeg_find_format(struct mtk_jpeg_fmt *mtk_jpeg_formats, int num_formats,
+ u32 pixelformat, unsigned int fmt_type)
{
- unsigned int k, fmt_flag;
-
- fmt_flag = (fmt_type == MTK_JPEG_FMT_TYPE_OUTPUT) ?
- MTK_JPEG_FMT_FLAG_DEC_OUTPUT :
- MTK_JPEG_FMT_FLAG_DEC_CAPTURE;
+ unsigned int k;
+ struct mtk_jpeg_fmt *fmt;
- for (k = 0; k < MTK_JPEG_NUM_FORMATS; k++) {
- struct mtk_jpeg_fmt *fmt = &mtk_jpeg_formats[k];
+ for (k = 0; k < num_formats; k++) {
+ fmt = &mtk_jpeg_formats[k];
- if (fmt->fourcc == pixelformat && fmt->flags & fmt_flag)
+ if (fmt->fourcc == pixelformat && fmt->flags & fmt_type)
return fmt;
}
return NULL;
}
-static void mtk_jpeg_bound_align_image(u32 *w, unsigned int wmin,
- unsigned int wmax, unsigned int walign,
- u32 *h, unsigned int hmin,
- unsigned int hmax, unsigned int halign)
+static int mtk_jpeg_try_fmt_mplane(struct v4l2_pix_format_mplane *pix_mp,
+ struct mtk_jpeg_fmt *fmt)
{
- int width, height, w_step, h_step;
-
- width = *w;
- height = *h;
- w_step = 1 << walign;
- h_step = 1 << halign;
-
- v4l_bound_align_image(w, wmin, wmax, walign, h, hmin, hmax, halign, 0);
- if (*w < width && (*w + w_step) <= wmax)
- *w += w_step;
- if (*h < height && (*h + h_step) <= hmax)
- *h += h_step;
-}
-
-static void mtk_jpeg_adjust_fmt_mplane(struct mtk_jpeg_ctx *ctx,
- struct v4l2_format *f)
-{
- struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp;
- struct mtk_jpeg_q_data *q_data;
- int i;
-
- q_data = mtk_jpeg_get_q_data(ctx, f->type);
-
- pix_mp->width = q_data->w;
- pix_mp->height = q_data->h;
- pix_mp->pixelformat = q_data->fmt->fourcc;
- pix_mp->num_planes = q_data->fmt->colplanes;
-
- for (i = 0; i < pix_mp->num_planes; i++) {
- pix_mp->plane_fmt[i].bytesperline = q_data->bytesperline[i];
- pix_mp->plane_fmt[i].sizeimage = q_data->sizeimage[i];
- }
-}
-
-static int mtk_jpeg_try_fmt_mplane(struct v4l2_format *f,
- struct mtk_jpeg_fmt *fmt,
- struct mtk_jpeg_ctx *ctx, int q_type)
-{
- struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp;
- struct mtk_jpeg_dev *jpeg = ctx->jpeg;
int i;
- memset(pix_mp->reserved, 0, sizeof(pix_mp->reserved));
pix_mp->field = V4L2_FIELD_NONE;
- if (ctx->state != MTK_JPEG_INIT) {
- mtk_jpeg_adjust_fmt_mplane(ctx, f);
- goto end;
- }
-
pix_mp->num_planes = fmt->colplanes;
pix_mp->pixelformat = fmt->fourcc;
- if (q_type == MTK_JPEG_FMT_TYPE_OUTPUT) {
+ if (fmt->fourcc == V4L2_PIX_FMT_JPEG) {
struct v4l2_plane_pix_format *pfmt = &pix_mp->plane_fmt[0];
- mtk_jpeg_bound_align_image(&pix_mp->width, MTK_JPEG_MIN_WIDTH,
- MTK_JPEG_MAX_WIDTH, 0,
- &pix_mp->height, MTK_JPEG_MIN_HEIGHT,
- MTK_JPEG_MAX_HEIGHT, 0);
+ pix_mp->height = clamp(pix_mp->height, MTK_JPEG_MIN_HEIGHT,
+ MTK_JPEG_MAX_HEIGHT);
+ pix_mp->width = clamp(pix_mp->width, MTK_JPEG_MIN_WIDTH,
+ MTK_JPEG_MAX_WIDTH);
- memset(pfmt->reserved, 0, sizeof(pfmt->reserved));
pfmt->bytesperline = 0;
/* Source size must be aligned to 128 */
- pfmt->sizeimage = mtk_jpeg_align(pfmt->sizeimage, 128);
+ pfmt->sizeimage = round_up(pfmt->sizeimage, 128);
if (pfmt->sizeimage == 0)
pfmt->sizeimage = MTK_JPEG_DEFAULT_SIZEIMAGE;
- goto end;
+ return 0;
}
- /* type is MTK_JPEG_FMT_TYPE_CAPTURE */
- mtk_jpeg_bound_align_image(&pix_mp->width, MTK_JPEG_MIN_WIDTH,
- MTK_JPEG_MAX_WIDTH, fmt->h_align,
- &pix_mp->height, MTK_JPEG_MIN_HEIGHT,
- MTK_JPEG_MAX_HEIGHT, fmt->v_align);
+ /* other fourcc */
+ pix_mp->height = clamp(round_up(pix_mp->height, fmt->v_align),
+ MTK_JPEG_MIN_HEIGHT, MTK_JPEG_MAX_HEIGHT);
+ pix_mp->width = clamp(round_up(pix_mp->width, fmt->h_align),
+ MTK_JPEG_MIN_WIDTH, MTK_JPEG_MAX_WIDTH);
for (i = 0; i < fmt->colplanes; i++) {
struct v4l2_plane_pix_format *pfmt = &pix_mp->plane_fmt[i];
u32 stride = pix_mp->width * fmt->h_sample[i] / 4;
u32 h = pix_mp->height * fmt->v_sample[i] / 4;
- memset(pfmt->reserved, 0, sizeof(pfmt->reserved));
pfmt->bytesperline = stride;
pfmt->sizeimage = stride * h;
}
-end:
- v4l2_dbg(2, debug, &jpeg->v4l2_dev, "wxh:%ux%u\n",
- pix_mp->width, pix_mp->height);
- for (i = 0; i < pix_mp->num_planes; i++) {
- v4l2_dbg(2, debug, &jpeg->v4l2_dev,
- "plane[%d] bpl=%u, size=%u\n",
- i,
- pix_mp->plane_fmt[i].bytesperline,
- pix_mp->plane_fmt[i].sizeimage);
- }
return 0;
}
q_data = mtk_jpeg_get_q_data(ctx, f->type);
- memset(pix_mp->reserved, 0, sizeof(pix_mp->reserved));
- pix_mp->width = q_data->w;
- pix_mp->height = q_data->h;
+ pix_mp->width = q_data->pix_mp.width;
+ pix_mp->height = q_data->pix_mp.height;
pix_mp->field = V4L2_FIELD_NONE;
pix_mp->pixelformat = q_data->fmt->fourcc;
pix_mp->num_planes = q_data->fmt->colplanes;
- pix_mp->colorspace = ctx->colorspace;
- pix_mp->ycbcr_enc = ctx->ycbcr_enc;
- pix_mp->xfer_func = ctx->xfer_func;
- pix_mp->quantization = ctx->quantization;
+ pix_mp->colorspace = q_data->pix_mp.colorspace;
+ pix_mp->ycbcr_enc = q_data->pix_mp.ycbcr_enc;
+ pix_mp->xfer_func = q_data->pix_mp.xfer_func;
+ pix_mp->quantization = q_data->pix_mp.quantization;
v4l2_dbg(1, debug, &jpeg->v4l2_dev, "(%d) g_fmt:%c%c%c%c wxh:%ux%u\n",
f->type,
for (i = 0; i < pix_mp->num_planes; i++) {
struct v4l2_plane_pix_format *pfmt = &pix_mp->plane_fmt[i];
- pfmt->bytesperline = q_data->bytesperline[i];
- pfmt->sizeimage = q_data->sizeimage[i];
- memset(pfmt->reserved, 0, sizeof(pfmt->reserved));
+ pfmt->bytesperline = q_data->pix_mp.plane_fmt[i].bytesperline;
+ pfmt->sizeimage = q_data->pix_mp.plane_fmt[i].sizeimage;
v4l2_dbg(1, debug, &jpeg->v4l2_dev,
"plane[%d] bpl=%u, size=%u\n",
struct v4l2_format *f)
{
struct mtk_jpeg_ctx *ctx = mtk_jpeg_fh_to_ctx(priv);
+ struct mtk_jpeg_dev *jpeg = ctx->jpeg;
struct mtk_jpeg_fmt *fmt;
- fmt = mtk_jpeg_find_format(ctx, f->fmt.pix_mp.pixelformat,
- MTK_JPEG_FMT_TYPE_CAPTURE);
+ fmt = mtk_jpeg_find_format(jpeg->variant->formats,
+ jpeg->variant->num_formats,
+ f->fmt.pix_mp.pixelformat,
+ MTK_JPEG_FMT_FLAG_CAPTURE);
if (!fmt)
fmt = ctx->cap_q.fmt;
(fmt->fourcc >> 16 & 0xff),
(fmt->fourcc >> 24 & 0xff));
- return mtk_jpeg_try_fmt_mplane(f, fmt, ctx, MTK_JPEG_FMT_TYPE_CAPTURE);
+ if (ctx->state != MTK_JPEG_INIT) {
+ mtk_jpeg_g_fmt_vid_mplane(file, priv, f);
+ return 0;
+ }
+
+ return mtk_jpeg_try_fmt_mplane(&f->fmt.pix_mp, fmt);
}
static int mtk_jpeg_try_fmt_vid_out_mplane(struct file *file, void *priv,
struct v4l2_format *f)
{
struct mtk_jpeg_ctx *ctx = mtk_jpeg_fh_to_ctx(priv);
+ struct mtk_jpeg_dev *jpeg = ctx->jpeg;
struct mtk_jpeg_fmt *fmt;
- fmt = mtk_jpeg_find_format(ctx, f->fmt.pix_mp.pixelformat,
- MTK_JPEG_FMT_TYPE_OUTPUT);
+ fmt = mtk_jpeg_find_format(jpeg->variant->formats,
+ jpeg->variant->num_formats,
+ f->fmt.pix_mp.pixelformat,
+ MTK_JPEG_FMT_FLAG_OUTPUT);
if (!fmt)
fmt = ctx->out_q.fmt;
(fmt->fourcc >> 16 & 0xff),
(fmt->fourcc >> 24 & 0xff));
- return mtk_jpeg_try_fmt_mplane(f, fmt, ctx, MTK_JPEG_FMT_TYPE_OUTPUT);
+ if (ctx->state != MTK_JPEG_INIT) {
+ mtk_jpeg_g_fmt_vid_mplane(file, priv, f);
+ return 0;
+ }
+
+ return mtk_jpeg_try_fmt_mplane(&f->fmt.pix_mp, fmt);
}
static int mtk_jpeg_s_fmt_mplane(struct mtk_jpeg_ctx *ctx,
- struct v4l2_format *f)
+ struct v4l2_format *f, unsigned int fmt_type)
{
struct vb2_queue *vq;
struct mtk_jpeg_q_data *q_data = NULL;
struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp;
struct mtk_jpeg_dev *jpeg = ctx->jpeg;
- unsigned int f_type;
int i;
vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type);
return -EBUSY;
}
- f_type = V4L2_TYPE_IS_OUTPUT(f->type) ?
- MTK_JPEG_FMT_TYPE_OUTPUT : MTK_JPEG_FMT_TYPE_CAPTURE;
-
- q_data->fmt = mtk_jpeg_find_format(ctx, pix_mp->pixelformat, f_type);
- q_data->w = pix_mp->width;
- q_data->h = pix_mp->height;
- ctx->colorspace = pix_mp->colorspace;
- ctx->ycbcr_enc = pix_mp->ycbcr_enc;
- ctx->xfer_func = pix_mp->xfer_func;
- ctx->quantization = pix_mp->quantization;
+ q_data->fmt = mtk_jpeg_find_format(jpeg->variant->formats,
+ jpeg->variant->num_formats,
+ pix_mp->pixelformat, fmt_type);
+ q_data->pix_mp.width = pix_mp->width;
+ q_data->pix_mp.height = pix_mp->height;
+ q_data->enc_crop_rect.width = pix_mp->width;
+ q_data->enc_crop_rect.height = pix_mp->height;
+ q_data->pix_mp.colorspace = V4L2_COLORSPACE_SRGB;
+ q_data->pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_601;
+ q_data->pix_mp.xfer_func = V4L2_XFER_FUNC_SRGB;
+ q_data->pix_mp.quantization = V4L2_QUANTIZATION_FULL_RANGE;
v4l2_dbg(1, debug, &jpeg->v4l2_dev, "(%d) s_fmt:%c%c%c%c wxh:%ux%u\n",
f->type,
(q_data->fmt->fourcc >> 8 & 0xff),
(q_data->fmt->fourcc >> 16 & 0xff),
(q_data->fmt->fourcc >> 24 & 0xff),
- q_data->w, q_data->h);
+ q_data->pix_mp.width, q_data->pix_mp.height);
for (i = 0; i < q_data->fmt->colplanes; i++) {
- q_data->bytesperline[i] = pix_mp->plane_fmt[i].bytesperline;
- q_data->sizeimage[i] = pix_mp->plane_fmt[i].sizeimage;
+ q_data->pix_mp.plane_fmt[i].bytesperline =
+ pix_mp->plane_fmt[i].bytesperline;
+ q_data->pix_mp.plane_fmt[i].sizeimage =
+ pix_mp->plane_fmt[i].sizeimage;
v4l2_dbg(1, debug, &jpeg->v4l2_dev,
"plane[%d] bpl=%u, size=%u\n",
- i, q_data->bytesperline[i], q_data->sizeimage[i]);
+ i, q_data->pix_mp.plane_fmt[i].bytesperline,
+ q_data->pix_mp.plane_fmt[i].sizeimage);
}
return 0;
if (ret)
return ret;
- return mtk_jpeg_s_fmt_mplane(mtk_jpeg_fh_to_ctx(priv), f);
+ return mtk_jpeg_s_fmt_mplane(mtk_jpeg_fh_to_ctx(priv), f,
+ MTK_JPEG_FMT_FLAG_OUTPUT);
}
static int mtk_jpeg_s_fmt_vid_cap_mplane(struct file *file, void *priv,
if (ret)
return ret;
- return mtk_jpeg_s_fmt_mplane(mtk_jpeg_fh_to_ctx(priv), f);
+ return mtk_jpeg_s_fmt_mplane(mtk_jpeg_fh_to_ctx(priv), f,
+ MTK_JPEG_FMT_FLAG_CAPTURE);
}
static void mtk_jpeg_queue_src_chg_event(struct mtk_jpeg_ctx *ctx)
switch (sub->type) {
case V4L2_EVENT_SOURCE_CHANGE:
return v4l2_src_change_event_subscribe(fh, sub);
+ }
+
+ return v4l2_ctrl_subscribe_event(fh, sub);
+}
+
+static int mtk_jpeg_enc_g_selection(struct file *file, void *priv,
+ struct v4l2_selection *s)
+{
+ struct mtk_jpeg_ctx *ctx = mtk_jpeg_fh_to_ctx(priv);
+
+ if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
+ return -EINVAL;
+
+ switch (s->target) {
+ case V4L2_SEL_TGT_CROP:
+ s->r = ctx->out_q.enc_crop_rect;
+ break;
+ case V4L2_SEL_TGT_CROP_BOUNDS:
+ case V4L2_SEL_TGT_CROP_DEFAULT:
+ s->r.width = ctx->out_q.pix_mp.width;
+ s->r.height = ctx->out_q.pix_mp.height;
+ s->r.left = 0;
+ s->r.top = 0;
+ break;
default:
return -EINVAL;
}
+ return 0;
}
-static int mtk_jpeg_g_selection(struct file *file, void *priv,
- struct v4l2_selection *s)
+static int mtk_jpeg_dec_g_selection(struct file *file, void *priv,
+ struct v4l2_selection *s)
{
struct mtk_jpeg_ctx *ctx = mtk_jpeg_fh_to_ctx(priv);
switch (s->target) {
case V4L2_SEL_TGT_COMPOSE:
case V4L2_SEL_TGT_COMPOSE_DEFAULT:
- s->r.width = ctx->out_q.w;
- s->r.height = ctx->out_q.h;
+ s->r.width = ctx->out_q.pix_mp.width;
+ s->r.height = ctx->out_q.pix_mp.height;
s->r.left = 0;
s->r.top = 0;
break;
case V4L2_SEL_TGT_COMPOSE_BOUNDS:
case V4L2_SEL_TGT_COMPOSE_PADDED:
- s->r.width = ctx->cap_q.w;
- s->r.height = ctx->cap_q.h;
+ s->r.width = ctx->cap_q.pix_mp.width;
+ s->r.height = ctx->cap_q.pix_mp.height;
s->r.left = 0;
s->r.top = 0;
break;
return 0;
}
-static int mtk_jpeg_s_selection(struct file *file, void *priv,
- struct v4l2_selection *s)
+static int mtk_jpeg_enc_s_selection(struct file *file, void *priv,
+ struct v4l2_selection *s)
{
struct mtk_jpeg_ctx *ctx = mtk_jpeg_fh_to_ctx(priv);
- if (s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+ if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
return -EINVAL;
switch (s->target) {
- case V4L2_SEL_TGT_COMPOSE:
+ case V4L2_SEL_TGT_CROP:
s->r.left = 0;
s->r.top = 0;
- s->r.width = ctx->out_q.w;
- s->r.height = ctx->out_q.h;
+ s->r.width = min(s->r.width, ctx->out_q.pix_mp.width);
+ s->r.height = min(s->r.height, ctx->out_q.pix_mp.height);
+ ctx->out_q.enc_crop_rect = s->r;
break;
default:
return -EINVAL;
}
+
return 0;
}
-static int mtk_jpeg_qbuf(struct file *file, void *priv, struct v4l2_buffer *buf)
-{
- struct v4l2_fh *fh = file->private_data;
- struct mtk_jpeg_ctx *ctx = mtk_jpeg_fh_to_ctx(priv);
- struct vb2_queue *vq;
- struct vb2_buffer *vb;
- struct mtk_jpeg_src_buf *jpeg_src_buf;
-
- if (buf->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)
- goto end;
+static const struct v4l2_ioctl_ops mtk_jpeg_enc_ioctl_ops = {
+ .vidioc_querycap = mtk_jpeg_querycap,
+ .vidioc_enum_fmt_vid_cap = mtk_jpeg_enum_fmt_vid_cap,
+ .vidioc_enum_fmt_vid_out = mtk_jpeg_enum_fmt_vid_out,
+ .vidioc_try_fmt_vid_cap_mplane = mtk_jpeg_try_fmt_vid_cap_mplane,
+ .vidioc_try_fmt_vid_out_mplane = mtk_jpeg_try_fmt_vid_out_mplane,
+ .vidioc_g_fmt_vid_cap_mplane = mtk_jpeg_g_fmt_vid_mplane,
+ .vidioc_g_fmt_vid_out_mplane = mtk_jpeg_g_fmt_vid_mplane,
+ .vidioc_s_fmt_vid_cap_mplane = mtk_jpeg_s_fmt_vid_cap_mplane,
+ .vidioc_s_fmt_vid_out_mplane = mtk_jpeg_s_fmt_vid_out_mplane,
+ .vidioc_qbuf = v4l2_m2m_ioctl_qbuf,
+ .vidioc_subscribe_event = mtk_jpeg_subscribe_event,
+ .vidioc_g_selection = mtk_jpeg_enc_g_selection,
+ .vidioc_s_selection = mtk_jpeg_enc_s_selection,
- vq = v4l2_m2m_get_vq(fh->m2m_ctx, buf->type);
- if (buf->index >= vq->num_buffers) {
- dev_err(ctx->jpeg->dev, "buffer index out of range\n");
- return -EINVAL;
- }
+ .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs,
+ .vidioc_prepare_buf = v4l2_m2m_ioctl_prepare_buf,
+ .vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs,
+ .vidioc_querybuf = v4l2_m2m_ioctl_querybuf,
+ .vidioc_dqbuf = v4l2_m2m_ioctl_dqbuf,
+ .vidioc_expbuf = v4l2_m2m_ioctl_expbuf,
+ .vidioc_streamon = v4l2_m2m_ioctl_streamon,
+ .vidioc_streamoff = v4l2_m2m_ioctl_streamoff,
- vb = vb2_get_buffer(vq, buf->index);
- jpeg_src_buf = mtk_jpeg_vb2_to_srcbuf(vb);
- jpeg_src_buf->flags = (buf->m.planes[0].bytesused == 0) ?
- MTK_JPEG_BUF_FLAGS_LAST_FRAME : MTK_JPEG_BUF_FLAGS_INIT;
-end:
- return v4l2_m2m_qbuf(file, fh->m2m_ctx, buf);
-}
+ .vidioc_unsubscribe_event = v4l2_event_unsubscribe,
+};
-static const struct v4l2_ioctl_ops mtk_jpeg_ioctl_ops = {
+static const struct v4l2_ioctl_ops mtk_jpeg_dec_ioctl_ops = {
.vidioc_querycap = mtk_jpeg_querycap,
.vidioc_enum_fmt_vid_cap = mtk_jpeg_enum_fmt_vid_cap,
.vidioc_enum_fmt_vid_out = mtk_jpeg_enum_fmt_vid_out,
.vidioc_g_fmt_vid_out_mplane = mtk_jpeg_g_fmt_vid_mplane,
.vidioc_s_fmt_vid_cap_mplane = mtk_jpeg_s_fmt_vid_cap_mplane,
.vidioc_s_fmt_vid_out_mplane = mtk_jpeg_s_fmt_vid_out_mplane,
- .vidioc_qbuf = mtk_jpeg_qbuf,
+ .vidioc_qbuf = v4l2_m2m_ioctl_qbuf,
.vidioc_subscribe_event = mtk_jpeg_subscribe_event,
- .vidioc_g_selection = mtk_jpeg_g_selection,
- .vidioc_s_selection = mtk_jpeg_s_selection,
+ .vidioc_g_selection = mtk_jpeg_dec_g_selection,
.vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs,
.vidioc_prepare_buf = v4l2_m2m_ioctl_prepare_buf,
if (!q_data)
return -EINVAL;
+ if (*num_planes) {
+ for (i = 0; i < *num_planes; i++)
+ if (sizes[i] < q_data->pix_mp.plane_fmt[i].sizeimage)
+ return -EINVAL;
+ return 0;
+ }
+
*num_planes = q_data->fmt->colplanes;
for (i = 0; i < q_data->fmt->colplanes; i++) {
- sizes[i] = q_data->sizeimage[i];
+ sizes[i] = q_data->pix_mp.plane_fmt[i].sizeimage;
v4l2_dbg(1, debug, &jpeg->v4l2_dev, "sizeimage[%d]=%u\n",
i, sizes[i]);
}
{
struct mtk_jpeg_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
struct mtk_jpeg_q_data *q_data = NULL;
+ struct v4l2_plane_pix_format plane_fmt = {};
int i;
q_data = mtk_jpeg_get_q_data(ctx, vb->vb2_queue->type);
if (!q_data)
return -EINVAL;
- for (i = 0; i < q_data->fmt->colplanes; i++)
- vb2_set_plane_payload(vb, i, q_data->sizeimage[i]);
+ for (i = 0; i < q_data->fmt->colplanes; i++) {
+ plane_fmt = q_data->pix_mp.plane_fmt[i];
+ if (ctx->enable_exif &&
+ q_data->fmt->fourcc == V4L2_PIX_FMT_JPEG)
+ vb2_set_plane_payload(vb, i, plane_fmt.sizeimage +
+ MTK_JPEG_MAX_EXIF_SIZE);
+ else
+ vb2_set_plane_payload(vb, i, plane_fmt.sizeimage);
+ }
return 0;
}
struct mtk_jpeg_q_data *q_data;
q_data = &ctx->out_q;
- if (q_data->w != param->pic_w || q_data->h != param->pic_h) {
+ if (q_data->pix_mp.width != param->pic_w ||
+ q_data->pix_mp.height != param->pic_h) {
v4l2_dbg(1, debug, &jpeg->v4l2_dev, "Picture size change\n");
return true;
}
q_data = &ctx->cap_q;
- if (q_data->fmt != mtk_jpeg_find_format(ctx, param->dst_fourcc,
- MTK_JPEG_FMT_TYPE_CAPTURE)) {
+ if (q_data->fmt !=
+ mtk_jpeg_find_format(jpeg->variant->formats,
+ jpeg->variant->num_formats, param->dst_fourcc,
+ MTK_JPEG_FMT_FLAG_CAPTURE)) {
v4l2_dbg(1, debug, &jpeg->v4l2_dev, "format change\n");
return true;
}
int i;
q_data = &ctx->out_q;
- q_data->w = param->pic_w;
- q_data->h = param->pic_h;
+ q_data->pix_mp.width = param->pic_w;
+ q_data->pix_mp.height = param->pic_h;
q_data = &ctx->cap_q;
- q_data->w = param->dec_w;
- q_data->h = param->dec_h;
- q_data->fmt = mtk_jpeg_find_format(ctx,
+ q_data->pix_mp.width = param->dec_w;
+ q_data->pix_mp.height = param->dec_h;
+ q_data->fmt = mtk_jpeg_find_format(jpeg->variant->formats,
+ jpeg->variant->num_formats,
param->dst_fourcc,
- MTK_JPEG_FMT_TYPE_CAPTURE);
+ MTK_JPEG_FMT_FLAG_CAPTURE);
for (i = 0; i < q_data->fmt->colplanes; i++) {
- q_data->bytesperline[i] = param->mem_stride[i];
- q_data->sizeimage[i] = param->comp_size[i];
+ q_data->pix_mp.plane_fmt[i].bytesperline = param->mem_stride[i];
+ q_data->pix_mp.plane_fmt[i].sizeimage = param->comp_size[i];
}
v4l2_dbg(1, debug, &jpeg->v4l2_dev,
param->dec_w, param->dec_h);
}
-static void mtk_jpeg_buf_queue(struct vb2_buffer *vb)
+static void mtk_jpeg_enc_buf_queue(struct vb2_buffer *vb)
+{
+ struct mtk_jpeg_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+ struct mtk_jpeg_dev *jpeg = ctx->jpeg;
+
+ v4l2_dbg(2, debug, &jpeg->v4l2_dev, "(%d) buf_q id=%d, vb=%p\n",
+ vb->vb2_queue->type, vb->index, vb);
+
+ v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, to_vb2_v4l2_buffer(vb));
+}
+
+static void mtk_jpeg_dec_buf_queue(struct vb2_buffer *vb)
{
struct mtk_jpeg_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
struct mtk_jpeg_dec_param *param;
param = &jpeg_src_buf->dec_param;
memset(param, 0, sizeof(*param));
- if (jpeg_src_buf->flags & MTK_JPEG_BUF_FLAGS_LAST_FRAME) {
- v4l2_dbg(1, debug, &jpeg->v4l2_dev, "Got eos\n");
- goto end;
- }
header_valid = mtk_jpeg_parse(param, (u8 *)vb2_plane_vaddr(vb, 0),
vb2_get_plane_payload(vb, 0));
if (!header_valid) {
return v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
}
-static int mtk_jpeg_start_streaming(struct vb2_queue *q, unsigned int count)
+static void mtk_jpeg_enc_stop_streaming(struct vb2_queue *q)
{
struct mtk_jpeg_ctx *ctx = vb2_get_drv_priv(q);
struct vb2_v4l2_buffer *vb;
- int ret = 0;
- ret = pm_runtime_get_sync(ctx->jpeg->dev);
- if (ret < 0)
- goto err;
-
- return 0;
-err:
while ((vb = mtk_jpeg_buf_remove(ctx, q->type)))
- v4l2_m2m_buf_done(vb, VB2_BUF_STATE_QUEUED);
- return ret;
+ v4l2_m2m_buf_done(vb, VB2_BUF_STATE_ERROR);
}
-static void mtk_jpeg_stop_streaming(struct vb2_queue *q)
+static void mtk_jpeg_dec_stop_streaming(struct vb2_queue *q)
{
struct mtk_jpeg_ctx *ctx = vb2_get_drv_priv(q);
struct vb2_v4l2_buffer *vb;
while ((vb = mtk_jpeg_buf_remove(ctx, q->type)))
v4l2_m2m_buf_done(vb, VB2_BUF_STATE_ERROR);
-
- pm_runtime_put_sync(ctx->jpeg->dev);
}
-static const struct vb2_ops mtk_jpeg_qops = {
+static const struct vb2_ops mtk_jpeg_dec_qops = {
+ .queue_setup = mtk_jpeg_queue_setup,
+ .buf_prepare = mtk_jpeg_buf_prepare,
+ .buf_queue = mtk_jpeg_dec_buf_queue,
+ .wait_prepare = vb2_ops_wait_prepare,
+ .wait_finish = vb2_ops_wait_finish,
+ .stop_streaming = mtk_jpeg_dec_stop_streaming,
+};
+
+static const struct vb2_ops mtk_jpeg_enc_qops = {
.queue_setup = mtk_jpeg_queue_setup,
.buf_prepare = mtk_jpeg_buf_prepare,
- .buf_queue = mtk_jpeg_buf_queue,
+ .buf_queue = mtk_jpeg_enc_buf_queue,
.wait_prepare = vb2_ops_wait_prepare,
.wait_finish = vb2_ops_wait_finish,
- .start_streaming = mtk_jpeg_start_streaming,
- .stop_streaming = mtk_jpeg_stop_streaming,
+ .stop_streaming = mtk_jpeg_enc_stop_streaming,
};
static void mtk_jpeg_set_dec_src(struct mtk_jpeg_ctx *ctx,
{
bs->str_addr = vb2_dma_contig_plane_dma_addr(src_buf, 0);
bs->end_addr = bs->str_addr +
- mtk_jpeg_align(vb2_get_plane_payload(src_buf, 0), 16);
- bs->size = mtk_jpeg_align(vb2_plane_size(src_buf, 0), 128);
+ round_up(vb2_get_plane_payload(src_buf, 0), 16);
+ bs->size = round_up(vb2_plane_size(src_buf, 0), 128);
}
static int mtk_jpeg_set_dec_dst(struct mtk_jpeg_ctx *ctx,
return 0;
}
-static void mtk_jpeg_device_run(void *priv)
+static void mtk_jpeg_enc_device_run(void *priv)
+{
+ struct mtk_jpeg_ctx *ctx = priv;
+ struct mtk_jpeg_dev *jpeg = ctx->jpeg;
+ struct vb2_v4l2_buffer *src_buf, *dst_buf;
+ enum vb2_buffer_state buf_state = VB2_BUF_STATE_ERROR;
+ unsigned long flags;
+ int ret;
+
+ src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
+ dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
+
+ ret = pm_runtime_get_sync(jpeg->dev);
+ if (ret < 0)
+ goto enc_end;
+
+ schedule_delayed_work(&jpeg->job_timeout_work,
+ msecs_to_jiffies(MTK_JPEG_HW_TIMEOUT_MSEC));
+
+ spin_lock_irqsave(&jpeg->hw_lock, flags);
+
+ /*
+ * Resetting the hardware every frame is to ensure that all the
+ * registers are cleared. This is a hardware requirement.
+ */
+ mtk_jpeg_enc_reset(jpeg->reg_base);
+
+ mtk_jpeg_set_enc_src(ctx, jpeg->reg_base, &src_buf->vb2_buf);
+ mtk_jpeg_set_enc_dst(ctx, jpeg->reg_base, &dst_buf->vb2_buf);
+ mtk_jpeg_set_enc_params(ctx, jpeg->reg_base);
+ mtk_jpeg_enc_start(jpeg->reg_base);
+ spin_unlock_irqrestore(&jpeg->hw_lock, flags);
+ return;
+
+enc_end:
+ v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+ v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+ v4l2_m2m_buf_done(src_buf, buf_state);
+ v4l2_m2m_buf_done(dst_buf, buf_state);
+ v4l2_m2m_job_finish(jpeg->m2m_dev, ctx->fh.m2m_ctx);
+}
+
+static void mtk_jpeg_dec_device_run(void *priv)
{
struct mtk_jpeg_ctx *ctx = priv;
struct mtk_jpeg_dev *jpeg = ctx->jpeg;
struct mtk_jpeg_src_buf *jpeg_src_buf;
struct mtk_jpeg_bs bs;
struct mtk_jpeg_fb fb;
- int i;
+ int ret;
src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx);
jpeg_src_buf = mtk_jpeg_vb2_to_srcbuf(&src_buf->vb2_buf);
- if (jpeg_src_buf->flags & MTK_JPEG_BUF_FLAGS_LAST_FRAME) {
- for (i = 0; i < dst_buf->vb2_buf.num_planes; i++)
- vb2_set_plane_payload(&dst_buf->vb2_buf, i, 0);
- buf_state = VB2_BUF_STATE_DONE;
- goto dec_end;
- }
-
if (mtk_jpeg_check_resolution_change(ctx, &jpeg_src_buf->dec_param)) {
mtk_jpeg_queue_src_chg_event(ctx);
ctx->state = MTK_JPEG_SOURCE_CHANGE;
return;
}
+ ret = pm_runtime_get_sync(jpeg->dev);
+ if (ret < 0)
+ goto dec_end;
+
+ schedule_delayed_work(&jpeg->job_timeout_work,
+ msecs_to_jiffies(MTK_JPEG_HW_TIMEOUT_MSEC));
+
mtk_jpeg_set_dec_src(ctx, &src_buf->vb2_buf, &bs);
if (mtk_jpeg_set_dec_dst(ctx, &jpeg_src_buf->dec_param, &dst_buf->vb2_buf, &fb))
goto dec_end;
spin_lock_irqsave(&jpeg->hw_lock, flags);
- mtk_jpeg_dec_reset(jpeg->dec_reg_base);
- mtk_jpeg_dec_set_config(jpeg->dec_reg_base,
+ mtk_jpeg_dec_reset(jpeg->reg_base);
+ mtk_jpeg_dec_set_config(jpeg->reg_base,
&jpeg_src_buf->dec_param, &bs, &fb);
- mtk_jpeg_dec_start(jpeg->dec_reg_base);
+ mtk_jpeg_dec_start(jpeg->reg_base);
spin_unlock_irqrestore(&jpeg->hw_lock, flags);
return;
v4l2_m2m_job_finish(jpeg->m2m_dev, ctx->fh.m2m_ctx);
}
-static int mtk_jpeg_job_ready(void *priv)
+static int mtk_jpeg_dec_job_ready(void *priv)
{
struct mtk_jpeg_ctx *ctx = priv;
return (ctx->state == MTK_JPEG_RUNNING) ? 1 : 0;
}
-static const struct v4l2_m2m_ops mtk_jpeg_m2m_ops = {
- .device_run = mtk_jpeg_device_run,
- .job_ready = mtk_jpeg_job_ready,
+static const struct v4l2_m2m_ops mtk_jpeg_enc_m2m_ops = {
+ .device_run = mtk_jpeg_enc_device_run,
+};
+
+static const struct v4l2_m2m_ops mtk_jpeg_dec_m2m_ops = {
+ .device_run = mtk_jpeg_dec_device_run,
+ .job_ready = mtk_jpeg_dec_job_ready,
};
static int mtk_jpeg_queue_init(void *priv, struct vb2_queue *src_vq,
struct vb2_queue *dst_vq)
{
struct mtk_jpeg_ctx *ctx = priv;
+ struct mtk_jpeg_dev *jpeg = ctx->jpeg;
int ret;
src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
src_vq->io_modes = VB2_DMABUF | VB2_MMAP;
src_vq->drv_priv = ctx;
src_vq->buf_struct_size = sizeof(struct mtk_jpeg_src_buf);
- src_vq->ops = &mtk_jpeg_qops;
+ src_vq->ops = jpeg->variant->qops;
src_vq->mem_ops = &vb2_dma_contig_memops;
src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
src_vq->lock = &ctx->jpeg->lock;
dst_vq->io_modes = VB2_DMABUF | VB2_MMAP;
dst_vq->drv_priv = ctx;
dst_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer);
- dst_vq->ops = &mtk_jpeg_qops;
+ dst_vq->ops = jpeg->variant->qops;
dst_vq->mem_ops = &vb2_dma_contig_memops;
dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
dst_vq->lock = &ctx->jpeg->lock;
ret = mtk_smi_larb_get(jpeg->larb);
if (ret)
dev_err(jpeg->dev, "mtk_smi_larb_get larbvdec fail %d\n", ret);
- clk_prepare_enable(jpeg->clk_jdec_smi);
- clk_prepare_enable(jpeg->clk_jdec);
+
+ ret = clk_bulk_prepare_enable(jpeg->variant->num_clks,
+ jpeg->variant->clks);
+ if (ret)
+ dev_err(jpeg->dev, "Failed to open jpeg clk: %d\n", ret);
}
static void mtk_jpeg_clk_off(struct mtk_jpeg_dev *jpeg)
{
- clk_disable_unprepare(jpeg->clk_jdec);
- clk_disable_unprepare(jpeg->clk_jdec_smi);
+ clk_bulk_disable_unprepare(jpeg->variant->num_clks,
+ jpeg->variant->clks);
mtk_smi_larb_put(jpeg->larb);
}
+static irqreturn_t mtk_jpeg_enc_done(struct mtk_jpeg_dev *jpeg)
+{
+ struct mtk_jpeg_ctx *ctx;
+ struct vb2_v4l2_buffer *src_buf, *dst_buf;
+ enum vb2_buffer_state buf_state = VB2_BUF_STATE_ERROR;
+ u32 result_size;
+
+ ctx = v4l2_m2m_get_curr_priv(jpeg->m2m_dev);
+ if (!ctx) {
+ v4l2_err(&jpeg->v4l2_dev, "Context is NULL\n");
+ return IRQ_HANDLED;
+ }
+
+ src_buf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+ dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+
+ result_size = mtk_jpeg_enc_get_file_size(jpeg->reg_base);
+ vb2_set_plane_payload(&dst_buf->vb2_buf, 0, result_size);
+
+ buf_state = VB2_BUF_STATE_DONE;
+
+ v4l2_m2m_buf_done(src_buf, buf_state);
+ v4l2_m2m_buf_done(dst_buf, buf_state);
+ v4l2_m2m_job_finish(jpeg->m2m_dev, ctx->fh.m2m_ctx);
+ pm_runtime_put(ctx->jpeg->dev);
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t mtk_jpeg_enc_irq(int irq, void *priv)
+{
+ struct mtk_jpeg_dev *jpeg = priv;
+ u32 irq_status;
+ irqreturn_t ret = IRQ_NONE;
+
+ cancel_delayed_work(&jpeg->job_timeout_work);
+
+ irq_status = readl(jpeg->reg_base + JPEG_ENC_INT_STS) &
+ JPEG_ENC_INT_STATUS_MASK_ALLIRQ;
+ if (irq_status)
+ writel(0, jpeg->reg_base + JPEG_ENC_INT_STS);
+
+ if (!(irq_status & JPEG_ENC_INT_STATUS_DONE))
+ return ret;
+
+ ret = mtk_jpeg_enc_done(jpeg);
+ return ret;
+}
+
static irqreturn_t mtk_jpeg_dec_irq(int irq, void *priv)
{
struct mtk_jpeg_dev *jpeg = priv;
u32 dec_ret;
int i;
- dec_ret = mtk_jpeg_dec_get_int_status(jpeg->dec_reg_base);
+ cancel_delayed_work(&jpeg->job_timeout_work);
+
+ dec_ret = mtk_jpeg_dec_get_int_status(jpeg->reg_base);
dec_irq_ret = mtk_jpeg_dec_enum_result(dec_ret);
ctx = v4l2_m2m_get_curr_priv(jpeg->m2m_dev);
if (!ctx) {
jpeg_src_buf = mtk_jpeg_vb2_to_srcbuf(&src_buf->vb2_buf);
if (dec_irq_ret >= MTK_JPEG_DEC_RESULT_UNDERFLOW)
- mtk_jpeg_dec_reset(jpeg->dec_reg_base);
+ mtk_jpeg_dec_reset(jpeg->reg_base);
if (dec_irq_ret != MTK_JPEG_DEC_RESULT_EOF_DONE) {
dev_err(jpeg->dev, "decode failed\n");
v4l2_m2m_buf_done(src_buf, buf_state);
v4l2_m2m_buf_done(dst_buf, buf_state);
v4l2_m2m_job_finish(jpeg->m2m_dev, ctx->fh.m2m_ctx);
+ pm_runtime_put(ctx->jpeg->dev);
return IRQ_HANDLED;
}
static void mtk_jpeg_set_default_params(struct mtk_jpeg_ctx *ctx)
{
struct mtk_jpeg_q_data *q = &ctx->out_q;
- int i;
+ struct mtk_jpeg_dev *jpeg = ctx->jpeg;
- ctx->colorspace = V4L2_COLORSPACE_JPEG,
- ctx->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
- ctx->quantization = V4L2_QUANTIZATION_DEFAULT;
- ctx->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+ ctx->fh.ctrl_handler = &ctx->ctrl_hdl;
+ q->pix_mp.colorspace = V4L2_COLORSPACE_SRGB;
+ q->pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_601;
+ q->pix_mp.quantization = V4L2_QUANTIZATION_FULL_RANGE;
+ q->pix_mp.xfer_func = V4L2_XFER_FUNC_SRGB;
- q->fmt = mtk_jpeg_find_format(ctx, V4L2_PIX_FMT_JPEG,
- MTK_JPEG_FMT_TYPE_OUTPUT);
- q->w = MTK_JPEG_MIN_WIDTH;
- q->h = MTK_JPEG_MIN_HEIGHT;
- q->bytesperline[0] = 0;
- q->sizeimage[0] = MTK_JPEG_DEFAULT_SIZEIMAGE;
+ q->fmt = mtk_jpeg_find_format(jpeg->variant->formats,
+ jpeg->variant->num_formats,
+ jpeg->variant->out_q_default_fourcc,
+ MTK_JPEG_FMT_FLAG_OUTPUT);
+ q->pix_mp.width = MTK_JPEG_MIN_WIDTH;
+ q->pix_mp.height = MTK_JPEG_MIN_HEIGHT;
+ mtk_jpeg_try_fmt_mplane(&q->pix_mp, q->fmt);
q = &ctx->cap_q;
- q->fmt = mtk_jpeg_find_format(ctx, V4L2_PIX_FMT_YUV420M,
- MTK_JPEG_FMT_TYPE_CAPTURE);
- q->w = MTK_JPEG_MIN_WIDTH;
- q->h = MTK_JPEG_MIN_HEIGHT;
-
- for (i = 0; i < q->fmt->colplanes; i++) {
- u32 stride = q->w * q->fmt->h_sample[i] / 4;
- u32 h = q->h * q->fmt->v_sample[i] / 4;
-
- q->bytesperline[i] = stride;
- q->sizeimage[i] = stride * h;
- }
+ q->fmt = mtk_jpeg_find_format(jpeg->variant->formats,
+ jpeg->variant->num_formats,
+ jpeg->variant->cap_q_default_fourcc,
+ MTK_JPEG_FMT_FLAG_CAPTURE);
+ q->pix_mp.colorspace = V4L2_COLORSPACE_SRGB;
+ q->pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_601;
+ q->pix_mp.quantization = V4L2_QUANTIZATION_FULL_RANGE;
+ q->pix_mp.xfer_func = V4L2_XFER_FUNC_SRGB;
+ q->pix_mp.width = MTK_JPEG_MIN_WIDTH;
+ q->pix_mp.height = MTK_JPEG_MIN_HEIGHT;
+
+ mtk_jpeg_try_fmt_mplane(&q->pix_mp, q->fmt);
}
static int mtk_jpeg_open(struct file *file)
goto error;
}
+ if (jpeg->variant->cap_q_default_fourcc == V4L2_PIX_FMT_JPEG) {
+ ret = mtk_jpeg_enc_ctrls_setup(ctx);
+ if (ret) {
+ v4l2_err(&jpeg->v4l2_dev, "Failed to setup jpeg enc controls\n");
+ goto error;
+ }
+ } else {
+ v4l2_ctrl_handler_init(&ctx->ctrl_hdl, 0);
+ }
mtk_jpeg_set_default_params(ctx);
mutex_unlock(&jpeg->lock);
return 0;
mutex_lock(&jpeg->lock);
v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
+ v4l2_ctrl_handler_free(&ctx->ctrl_hdl);
v4l2_fh_del(&ctx->fh);
v4l2_fh_exit(&ctx->fh);
kfree(ctx);
.mmap = v4l2_m2m_fop_mmap,
};
+static struct clk_bulk_data mt8173_jpeg_dec_clocks[] = {
+ { .id = "jpgdec-smi" },
+ { .id = "jpgdec" },
+};
+
+static struct clk_bulk_data mtk_jpeg_clocks[] = {
+ { .id = "jpgenc" },
+};
+
static int mtk_jpeg_clk_init(struct mtk_jpeg_dev *jpeg)
{
struct device_node *node;
struct platform_device *pdev;
+ int ret;
node = of_parse_phandle(jpeg->dev->of_node, "mediatek,larb", 0);
if (!node)
jpeg->larb = &pdev->dev;
- jpeg->clk_jdec = devm_clk_get(jpeg->dev, "jpgdec");
- if (IS_ERR(jpeg->clk_jdec))
- return PTR_ERR(jpeg->clk_jdec);
+ ret = devm_clk_bulk_get(jpeg->dev, jpeg->variant->num_clks,
+ jpeg->variant->clks);
+ if (ret) {
+ dev_err(&pdev->dev, "failed to get jpeg clock:%d\n", ret);
+ return ret;
+ }
- jpeg->clk_jdec_smi = devm_clk_get(jpeg->dev, "jpgdec-smi");
- return PTR_ERR_OR_ZERO(jpeg->clk_jdec_smi);
+ return 0;
}
+static void mtk_jpeg_job_timeout_work(struct work_struct *work)
+{
+ struct mtk_jpeg_dev *jpeg = container_of(work, struct mtk_jpeg_dev,
+ job_timeout_work.work);
+ struct mtk_jpeg_ctx *ctx;
+ struct vb2_v4l2_buffer *src_buf, *dst_buf;
+
+ ctx = v4l2_m2m_get_curr_priv(jpeg->m2m_dev);
+ src_buf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
+ dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+
+ jpeg->variant->hw_reset(jpeg->reg_base);
+
+ pm_runtime_put(jpeg->dev);
+
+ v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_ERROR);
+ v4l2_m2m_buf_done(dst_buf, VB2_BUF_STATE_ERROR);
+ v4l2_m2m_job_finish(jpeg->m2m_dev, ctx->fh.m2m_ctx);
+}
static int mtk_jpeg_probe(struct platform_device *pdev)
{
struct mtk_jpeg_dev *jpeg;
struct resource *res;
- int dec_irq;
+ int jpeg_irq;
int ret;
jpeg = devm_kzalloc(&pdev->dev, sizeof(*jpeg), GFP_KERNEL);
mutex_init(&jpeg->lock);
spin_lock_init(&jpeg->hw_lock);
jpeg->dev = &pdev->dev;
+ jpeg->variant = of_device_get_match_data(jpeg->dev);
+ INIT_DELAYED_WORK(&jpeg->job_timeout_work, mtk_jpeg_job_timeout_work);
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- jpeg->dec_reg_base = devm_ioremap_resource(&pdev->dev, res);
- if (IS_ERR(jpeg->dec_reg_base)) {
- ret = PTR_ERR(jpeg->dec_reg_base);
+ jpeg->reg_base = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(jpeg->reg_base)) {
+ ret = PTR_ERR(jpeg->reg_base);
return ret;
}
- res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
- dec_irq = platform_get_irq(pdev, 0);
- if (!res || dec_irq < 0) {
- dev_err(&pdev->dev, "Failed to get dec_irq %d.\n", dec_irq);
- ret = -EINVAL;
- return ret;
+ jpeg_irq = platform_get_irq(pdev, 0);
+ if (jpeg_irq < 0) {
+ dev_err(&pdev->dev, "Failed to get jpeg_irq %d.\n", jpeg_irq);
+ return jpeg_irq;
}
- ret = devm_request_irq(&pdev->dev, dec_irq, mtk_jpeg_dec_irq, 0,
- pdev->name, jpeg);
+ ret = devm_request_irq(&pdev->dev, jpeg_irq,
+ jpeg->variant->irq_handler, 0, pdev->name, jpeg);
if (ret) {
- dev_err(&pdev->dev, "Failed to request dec_irq %d (%d)\n",
- dec_irq, ret);
- ret = -EINVAL;
+ dev_err(&pdev->dev, "Failed to request jpeg_irq %d (%d)\n",
+ jpeg_irq, ret);
goto err_req_irq;
}
goto err_dev_register;
}
- jpeg->m2m_dev = v4l2_m2m_init(&mtk_jpeg_m2m_ops);
+ jpeg->m2m_dev = v4l2_m2m_init(jpeg->variant->m2m_ops);
+
if (IS_ERR(jpeg->m2m_dev)) {
v4l2_err(&jpeg->v4l2_dev, "Failed to init mem2mem device\n");
ret = PTR_ERR(jpeg->m2m_dev);
goto err_m2m_init;
}
- jpeg->dec_vdev = video_device_alloc();
- if (!jpeg->dec_vdev) {
+ jpeg->vdev = video_device_alloc();
+ if (!jpeg->vdev) {
ret = -ENOMEM;
- goto err_dec_vdev_alloc;
+ goto err_vfd_jpeg_alloc;
}
- snprintf(jpeg->dec_vdev->name, sizeof(jpeg->dec_vdev->name),
- "%s-dec", MTK_JPEG_NAME);
- jpeg->dec_vdev->fops = &mtk_jpeg_fops;
- jpeg->dec_vdev->ioctl_ops = &mtk_jpeg_ioctl_ops;
- jpeg->dec_vdev->minor = -1;
- jpeg->dec_vdev->release = video_device_release;
- jpeg->dec_vdev->lock = &jpeg->lock;
- jpeg->dec_vdev->v4l2_dev = &jpeg->v4l2_dev;
- jpeg->dec_vdev->vfl_dir = VFL_DIR_M2M;
- jpeg->dec_vdev->device_caps = V4L2_CAP_STREAMING |
- V4L2_CAP_VIDEO_M2M_MPLANE;
-
- ret = video_register_device(jpeg->dec_vdev, VFL_TYPE_VIDEO, 3);
+ snprintf(jpeg->vdev->name, sizeof(jpeg->vdev->name),
+ "%s", jpeg->variant->dev_name);
+ jpeg->vdev->fops = &mtk_jpeg_fops;
+ jpeg->vdev->ioctl_ops = jpeg->variant->ioctl_ops;
+ jpeg->vdev->minor = -1;
+ jpeg->vdev->release = video_device_release;
+ jpeg->vdev->lock = &jpeg->lock;
+ jpeg->vdev->v4l2_dev = &jpeg->v4l2_dev;
+ jpeg->vdev->vfl_dir = VFL_DIR_M2M;
+ jpeg->vdev->device_caps = V4L2_CAP_STREAMING |
+ V4L2_CAP_VIDEO_M2M_MPLANE;
+
+ ret = video_register_device(jpeg->vdev, VFL_TYPE_VIDEO, -1);
if (ret) {
v4l2_err(&jpeg->v4l2_dev, "Failed to register video device\n");
- goto err_dec_vdev_register;
+ goto err_vfd_jpeg_register;
}
- video_set_drvdata(jpeg->dec_vdev, jpeg);
+ video_set_drvdata(jpeg->vdev, jpeg);
v4l2_info(&jpeg->v4l2_dev,
- "decoder device registered as /dev/video%d (%d,%d)\n",
- jpeg->dec_vdev->num, VIDEO_MAJOR, jpeg->dec_vdev->minor);
+ "%s device registered as /dev/video%d (%d,%d)\n",
+ jpeg->variant->dev_name, jpeg->vdev->num,
+ VIDEO_MAJOR, jpeg->vdev->minor);
platform_set_drvdata(pdev, jpeg);
return 0;
-err_dec_vdev_register:
- video_device_release(jpeg->dec_vdev);
+err_vfd_jpeg_register:
+ video_device_release(jpeg->vdev);
-err_dec_vdev_alloc:
+err_vfd_jpeg_alloc:
v4l2_m2m_release(jpeg->m2m_dev);
err_m2m_init:
struct mtk_jpeg_dev *jpeg = platform_get_drvdata(pdev);
pm_runtime_disable(&pdev->dev);
- video_unregister_device(jpeg->dec_vdev);
- video_device_release(jpeg->dec_vdev);
+ video_unregister_device(jpeg->vdev);
+ video_device_release(jpeg->vdev);
v4l2_m2m_release(jpeg->m2m_dev);
v4l2_device_unregister(&jpeg->v4l2_dev);
{
struct mtk_jpeg_dev *jpeg = dev_get_drvdata(dev);
- mtk_jpeg_dec_reset(jpeg->dec_reg_base);
mtk_jpeg_clk_off(jpeg);
return 0;
struct mtk_jpeg_dev *jpeg = dev_get_drvdata(dev);
mtk_jpeg_clk_on(jpeg);
- mtk_jpeg_dec_reset(jpeg->dec_reg_base);
return 0;
}
static __maybe_unused int mtk_jpeg_suspend(struct device *dev)
{
- int ret;
-
- if (pm_runtime_suspended(dev))
- return 0;
+ struct mtk_jpeg_dev *jpeg = dev_get_drvdata(dev);
- ret = mtk_jpeg_pm_suspend(dev);
- return ret;
+ v4l2_m2m_suspend(jpeg->m2m_dev);
+ return pm_runtime_force_suspend(dev);
}
static __maybe_unused int mtk_jpeg_resume(struct device *dev)
{
+ struct mtk_jpeg_dev *jpeg = dev_get_drvdata(dev);
int ret;
- if (pm_runtime_suspended(dev))
- return 0;
-
- ret = mtk_jpeg_pm_resume(dev);
+ ret = pm_runtime_force_resume(dev);
+ if (ret < 0)
+ return ret;
+ v4l2_m2m_resume(jpeg->m2m_dev);
return ret;
}
SET_RUNTIME_PM_OPS(mtk_jpeg_pm_suspend, mtk_jpeg_pm_resume, NULL)
};
+static const struct mtk_jpeg_variant mt8173_jpeg_drvdata = {
+ .clks = mt8173_jpeg_dec_clocks,
+ .num_clks = ARRAY_SIZE(mt8173_jpeg_dec_clocks),
+ .formats = mtk_jpeg_dec_formats,
+ .num_formats = MTK_JPEG_DEC_NUM_FORMATS,
+ .qops = &mtk_jpeg_dec_qops,
+ .irq_handler = mtk_jpeg_dec_irq,
+ .hw_reset = mtk_jpeg_dec_reset,
+ .m2m_ops = &mtk_jpeg_dec_m2m_ops,
+ .dev_name = "mtk-jpeg-dec",
+ .ioctl_ops = &mtk_jpeg_dec_ioctl_ops,
+ .out_q_default_fourcc = V4L2_PIX_FMT_JPEG,
+ .cap_q_default_fourcc = V4L2_PIX_FMT_YUV420M,
+};
+
+static const struct mtk_jpeg_variant mtk_jpeg_drvdata = {
+ .clks = mtk_jpeg_clocks,
+ .num_clks = ARRAY_SIZE(mtk_jpeg_clocks),
+ .formats = mtk_jpeg_enc_formats,
+ .num_formats = MTK_JPEG_ENC_NUM_FORMATS,
+ .qops = &mtk_jpeg_enc_qops,
+ .irq_handler = mtk_jpeg_enc_irq,
+ .hw_reset = mtk_jpeg_enc_reset,
+ .m2m_ops = &mtk_jpeg_enc_m2m_ops,
+ .dev_name = "mtk-jpeg-enc",
+ .ioctl_ops = &mtk_jpeg_enc_ioctl_ops,
+ .out_q_default_fourcc = V4L2_PIX_FMT_YUYV,
+ .cap_q_default_fourcc = V4L2_PIX_FMT_JPEG,
+};
+
static const struct of_device_id mtk_jpeg_match[] = {
{
.compatible = "mediatek,mt8173-jpgdec",
- .data = NULL,
+ .data = &mt8173_jpeg_drvdata,
},
{
.compatible = "mediatek,mt2701-jpgdec",
- .data = NULL,
+ .data = &mt8173_jpeg_drvdata,
+ },
+ {
+ .compatible = "mediatek,mtk-jpgenc",
+ .data = &mtk_jpeg_drvdata,
},
{},
};
* Copyright (c) 2016 MediaTek Inc.
* Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
* Rick Chang <rick.chang@mediatek.com>
+ * Xia Jiang <xia.jiang@mediatek.com>
*/
#ifndef _MTK_JPEG_CORE_H
#define MTK_JPEG_NAME "mtk-jpeg"
-#define MTK_JPEG_FMT_FLAG_DEC_OUTPUT BIT(0)
-#define MTK_JPEG_FMT_FLAG_DEC_CAPTURE BIT(1)
+#define MTK_JPEG_COMP_MAX 3
-#define MTK_JPEG_FMT_TYPE_OUTPUT 1
-#define MTK_JPEG_FMT_TYPE_CAPTURE 2
+#define MTK_JPEG_FMT_FLAG_OUTPUT BIT(0)
+#define MTK_JPEG_FMT_FLAG_CAPTURE BIT(1)
-#define MTK_JPEG_MIN_WIDTH 32
-#define MTK_JPEG_MIN_HEIGHT 32
-#define MTK_JPEG_MAX_WIDTH 8192
-#define MTK_JPEG_MAX_HEIGHT 8192
+#define MTK_JPEG_MIN_WIDTH 32U
+#define MTK_JPEG_MIN_HEIGHT 32U
+#define MTK_JPEG_MAX_WIDTH 65535U
+#define MTK_JPEG_MAX_HEIGHT 65535U
#define MTK_JPEG_DEFAULT_SIZEIMAGE (1 * 1024 * 1024)
+#define MTK_JPEG_HW_TIMEOUT_MSEC 1000
+
+#define MTK_JPEG_MAX_EXIF_SIZE (64 * 1024)
+
+/**
+ * enum mtk_jpeg_ctx_state - states of the context state machine
+ * @MTK_JPEG_INIT: current state is initialized
+ * @MTK_JPEG_RUNNING: current state is running
+ * @MTK_JPEG_SOURCE_CHANGE: current state is source resolution change
+ */
enum mtk_jpeg_ctx_state {
MTK_JPEG_INIT = 0,
MTK_JPEG_RUNNING,
MTK_JPEG_SOURCE_CHANGE,
};
+/**
+ * mtk_jpeg_variant - mtk jpeg driver variant
+ * @clks: clock names
+ * @num_clks: numbers of clock
+ * @format: jpeg driver's internal color format
+ * @num_format: number of format
+ * @qops: the callback of jpeg vb2_ops
+ * @irq_handler: jpeg irq handler callback
+ * @hw_reset: jpeg hardware reset callback
+ * @m2m_ops: the callback of jpeg v4l2_m2m_ops
+ * @dev_name: jpeg device name
+ * @ioctl_ops: the callback of jpeg v4l2_ioctl_ops
+ * @out_q_default_fourcc: output queue default fourcc
+ * @cap_q_default_fourcc: capture queue default fourcc
+ */
+struct mtk_jpeg_variant {
+ struct clk_bulk_data *clks;
+ int num_clks;
+ struct mtk_jpeg_fmt *formats;
+ int num_formats;
+ const struct vb2_ops *qops;
+ irqreturn_t (*irq_handler)(int irq, void *priv);
+ void (*hw_reset)(void __iomem *base);
+ const struct v4l2_m2m_ops *m2m_ops;
+ const char *dev_name;
+ const struct v4l2_ioctl_ops *ioctl_ops;
+ u32 out_q_default_fourcc;
+ u32 cap_q_default_fourcc;
+};
+
/**
* struct mt_jpeg - JPEG IP abstraction
* @lock: the mutex protecting this structure
* @v4l2_dev: v4l2 device for mem2mem mode
* @m2m_dev: v4l2 mem2mem device data
* @alloc_ctx: videobuf2 memory allocator's context
- * @dec_vdev: video device node for decoder mem2mem mode
- * @dec_reg_base: JPEG registers mapping
- * @clk_jdec: JPEG hw working clock
- * @clk_jdec_smi: JPEG SMI bus clock
+ * @vdev: video device node for jpeg mem2mem mode
+ * @reg_base: JPEG registers mapping
* @larb: SMI device
+ * @job_timeout_work: IRQ timeout structure
+ * @variant: driver variant to be used
*/
struct mtk_jpeg_dev {
struct mutex lock;
struct v4l2_device v4l2_dev;
struct v4l2_m2m_dev *m2m_dev;
void *alloc_ctx;
- struct video_device *dec_vdev;
- void __iomem *dec_reg_base;
- struct clk *clk_jdec;
- struct clk *clk_jdec_smi;
+ struct video_device *vdev;
+ void __iomem *reg_base;
struct device *larb;
+ struct delayed_work job_timeout_work;
+ const struct mtk_jpeg_variant *variant;
};
/**
* struct jpeg_fmt - driver's internal color format data
* @fourcc: the fourcc code, 0 if not applicable
+ * @hw_format: hardware format value
* @h_sample: horizontal sample count of plane in 4 * 4 pixel image
* @v_sample: vertical sample count of plane in 4 * 4 pixel image
* @colplanes: number of color planes (1 for packed formats)
*/
struct mtk_jpeg_fmt {
u32 fourcc;
+ u32 hw_format;
int h_sample[VIDEO_MAX_PLANES];
int v_sample[VIDEO_MAX_PLANES];
int colplanes;
/**
* mtk_jpeg_q_data - parameters of one queue
* @fmt: driver-specific format of this queue
- * @w: image width
- * @h: image height
- * @bytesperline: distance in bytes between the leftmost pixels in two adjacent
- * lines
- * @sizeimage: image buffer size in bytes
+ * @pix_mp: multiplanar format
+ * @enc_crop_rect: jpeg encoder crop information
*/
struct mtk_jpeg_q_data {
struct mtk_jpeg_fmt *fmt;
- u32 w;
- u32 h;
- u32 bytesperline[VIDEO_MAX_PLANES];
- u32 sizeimage[VIDEO_MAX_PLANES];
+ struct v4l2_pix_format_mplane pix_mp;
+ struct v4l2_rect enc_crop_rect;
};
/**
* @out_q: source (output) queue information
* @cap_q: destination (capture) queue queue information
* @fh: V4L2 file handle
- * @dec_param parameters for HW decoding
* @state: state of the context
- * @header_valid: set if header has been parsed and valid
- * @colorspace: enum v4l2_colorspace; supplemental to pixelformat
- * @ycbcr_enc: enum v4l2_ycbcr_encoding, Y'CbCr encoding
- * @quantization: enum v4l2_quantization, colorspace quantization
- * @xfer_func: enum v4l2_xfer_func, colorspace transfer function
+ * @enable_exif: enable exif mode of jpeg encoder
+ * @enc_quality: jpeg encoder quality
+ * @restart_interval: jpeg encoder restart interval
+ * @ctrl_hdl: controls handler
*/
struct mtk_jpeg_ctx {
struct mtk_jpeg_dev *jpeg;
struct mtk_jpeg_q_data cap_q;
struct v4l2_fh fh;
enum mtk_jpeg_ctx_state state;
-
- enum v4l2_colorspace colorspace;
- enum v4l2_ycbcr_encoding ycbcr_enc;
- enum v4l2_quantization quantization;
- enum v4l2_xfer_func xfer_func;
+ bool enable_exif;
+ u8 enc_quality;
+ u8 restart_interval;
+ struct v4l2_ctrl_handler ctrl_hdl;
};
#endif /* _MTK_JPEG_CORE_H */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2016 MediaTek Inc.
+ * Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
+ * Rick Chang <rick.chang@mediatek.com>
+ */
+
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <media/videobuf2-core.h>
+
+#include "mtk_jpeg_dec_hw.h"
+
+#define MTK_JPEG_DUNUM_MASK(val) (((val) - 1) & 0x3)
+
+enum mtk_jpeg_color {
+ MTK_JPEG_COLOR_420 = 0x00221111,
+ MTK_JPEG_COLOR_422 = 0x00211111,
+ MTK_JPEG_COLOR_444 = 0x00111111,
+ MTK_JPEG_COLOR_422V = 0x00121111,
+ MTK_JPEG_COLOR_422X2 = 0x00412121,
+ MTK_JPEG_COLOR_422VX2 = 0x00222121,
+ MTK_JPEG_COLOR_400 = 0x00110000
+};
+
+static inline int mtk_jpeg_verify_align(u32 val, int align, u32 reg)
+{
+ if (val & (align - 1)) {
+ pr_err("mtk-jpeg: write reg %x without %d align\n", reg, align);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int mtk_jpeg_decide_format(struct mtk_jpeg_dec_param *param)
+{
+ param->src_color = (param->sampling_w[0] << 20) |
+ (param->sampling_h[0] << 16) |
+ (param->sampling_w[1] << 12) |
+ (param->sampling_h[1] << 8) |
+ (param->sampling_w[2] << 4) |
+ (param->sampling_h[2]);
+
+ param->uv_brz_w = 0;
+ switch (param->src_color) {
+ case MTK_JPEG_COLOR_444:
+ param->uv_brz_w = 1;
+ param->dst_fourcc = V4L2_PIX_FMT_YUV422M;
+ break;
+ case MTK_JPEG_COLOR_422X2:
+ case MTK_JPEG_COLOR_422:
+ param->dst_fourcc = V4L2_PIX_FMT_YUV422M;
+ break;
+ case MTK_JPEG_COLOR_422V:
+ case MTK_JPEG_COLOR_422VX2:
+ param->uv_brz_w = 1;
+ param->dst_fourcc = V4L2_PIX_FMT_YUV420M;
+ break;
+ case MTK_JPEG_COLOR_420:
+ param->dst_fourcc = V4L2_PIX_FMT_YUV420M;
+ break;
+ case MTK_JPEG_COLOR_400:
+ param->dst_fourcc = V4L2_PIX_FMT_GREY;
+ break;
+ default:
+ param->dst_fourcc = 0;
+ return -1;
+ }
+
+ return 0;
+}
+
+static void mtk_jpeg_calc_mcu(struct mtk_jpeg_dec_param *param)
+{
+ u32 factor_w, factor_h;
+ u32 i, comp, blk;
+
+ factor_w = 2 + param->sampling_w[0];
+ factor_h = 2 + param->sampling_h[0];
+ param->mcu_w = (param->pic_w + (1 << factor_w) - 1) >> factor_w;
+ param->mcu_h = (param->pic_h + (1 << factor_h) - 1) >> factor_h;
+ param->total_mcu = param->mcu_w * param->mcu_h;
+ param->unit_num = ((param->pic_w + 7) >> 3) * ((param->pic_h + 7) >> 3);
+ param->blk_num = 0;
+ for (i = 0; i < MTK_JPEG_COMP_MAX; i++) {
+ param->blk_comp[i] = 0;
+ if (i >= param->comp_num)
+ continue;
+ param->blk_comp[i] = param->sampling_w[i] *
+ param->sampling_h[i];
+ param->blk_num += param->blk_comp[i];
+ }
+
+ param->membership = 0;
+ for (i = 0, blk = 0, comp = 0; i < MTK_JPEG_BLOCK_MAX; i++) {
+ if (i < param->blk_num && comp < param->comp_num) {
+ u32 tmp;
+
+ tmp = (0x04 + (comp & 0x3));
+ param->membership |= tmp << (i * 3);
+ if (++blk == param->blk_comp[comp]) {
+ comp++;
+ blk = 0;
+ }
+ } else {
+ param->membership |= 7 << (i * 3);
+ }
+ }
+}
+
+static void mtk_jpeg_calc_dma_group(struct mtk_jpeg_dec_param *param)
+{
+ u32 factor_mcu = 3;
+
+ if (param->src_color == MTK_JPEG_COLOR_444 &&
+ param->dst_fourcc == V4L2_PIX_FMT_YUV422M)
+ factor_mcu = 4;
+ else if (param->src_color == MTK_JPEG_COLOR_422V &&
+ param->dst_fourcc == V4L2_PIX_FMT_YUV420M)
+ factor_mcu = 4;
+ else if (param->src_color == MTK_JPEG_COLOR_422X2 &&
+ param->dst_fourcc == V4L2_PIX_FMT_YUV422M)
+ factor_mcu = 2;
+ else if (param->src_color == MTK_JPEG_COLOR_400 ||
+ (param->src_color & 0x0FFFF) == 0)
+ factor_mcu = 4;
+
+ param->dma_mcu = 1 << factor_mcu;
+ param->dma_group = param->mcu_w / param->dma_mcu;
+ param->dma_last_mcu = param->mcu_w % param->dma_mcu;
+ if (param->dma_last_mcu)
+ param->dma_group++;
+ else
+ param->dma_last_mcu = param->dma_mcu;
+}
+
+static int mtk_jpeg_calc_dst_size(struct mtk_jpeg_dec_param *param)
+{
+ u32 i, padding_w;
+ u32 ds_row_h[3];
+ u32 brz_w[3];
+
+ brz_w[0] = 0;
+ brz_w[1] = param->uv_brz_w;
+ brz_w[2] = brz_w[1];
+
+ for (i = 0; i < param->comp_num; i++) {
+ if (brz_w[i] > 3)
+ return -1;
+
+ padding_w = param->mcu_w * MTK_JPEG_DCTSIZE *
+ param->sampling_w[i];
+ /* output format is 420/422 */
+ param->comp_w[i] = padding_w >> brz_w[i];
+ param->comp_w[i] = round_up(param->comp_w[i],
+ MTK_JPEG_DCTSIZE);
+ param->img_stride[i] = i ? round_up(param->comp_w[i], 16)
+ : round_up(param->comp_w[i], 32);
+ ds_row_h[i] = (MTK_JPEG_DCTSIZE * param->sampling_h[i]);
+ }
+ param->dec_w = param->img_stride[0];
+ param->dec_h = ds_row_h[0] * param->mcu_h;
+
+ for (i = 0; i < MTK_JPEG_COMP_MAX; i++) {
+ /* They must be equal in frame mode. */
+ param->mem_stride[i] = param->img_stride[i];
+ param->comp_size[i] = param->mem_stride[i] * ds_row_h[i] *
+ param->mcu_h;
+ }
+
+ param->y_size = param->comp_size[0];
+ param->uv_size = param->comp_size[1];
+ param->dec_size = param->y_size + (param->uv_size << 1);
+
+ return 0;
+}
+
+int mtk_jpeg_dec_fill_param(struct mtk_jpeg_dec_param *param)
+{
+ if (mtk_jpeg_decide_format(param))
+ return -1;
+
+ mtk_jpeg_calc_mcu(param);
+ mtk_jpeg_calc_dma_group(param);
+ if (mtk_jpeg_calc_dst_size(param))
+ return -2;
+
+ return 0;
+}
+
+u32 mtk_jpeg_dec_get_int_status(void __iomem *base)
+{
+ u32 ret;
+
+ ret = readl(base + JPGDEC_REG_INTERRUPT_STATUS) & BIT_INQST_MASK_ALLIRQ;
+ if (ret)
+ writel(ret, base + JPGDEC_REG_INTERRUPT_STATUS);
+
+ return ret;
+}
+
+u32 mtk_jpeg_dec_enum_result(u32 irq_result)
+{
+ if (irq_result & BIT_INQST_MASK_EOF)
+ return MTK_JPEG_DEC_RESULT_EOF_DONE;
+ if (irq_result & BIT_INQST_MASK_PAUSE)
+ return MTK_JPEG_DEC_RESULT_PAUSE;
+ if (irq_result & BIT_INQST_MASK_UNDERFLOW)
+ return MTK_JPEG_DEC_RESULT_UNDERFLOW;
+ if (irq_result & BIT_INQST_MASK_OVERFLOW)
+ return MTK_JPEG_DEC_RESULT_OVERFLOW;
+ if (irq_result & BIT_INQST_MASK_ERROR_BS)
+ return MTK_JPEG_DEC_RESULT_ERROR_BS;
+
+ return MTK_JPEG_DEC_RESULT_ERROR_UNKNOWN;
+}
+
+void mtk_jpeg_dec_start(void __iomem *base)
+{
+ writel(0, base + JPGDEC_REG_TRIG);
+}
+
+static void mtk_jpeg_dec_soft_reset(void __iomem *base)
+{
+ writel(0x0000FFFF, base + JPGDEC_REG_INTERRUPT_STATUS);
+ writel(0x00, base + JPGDEC_REG_RESET);
+ writel(0x01, base + JPGDEC_REG_RESET);
+}
+
+static void mtk_jpeg_dec_hard_reset(void __iomem *base)
+{
+ writel(0x00, base + JPGDEC_REG_RESET);
+ writel(0x10, base + JPGDEC_REG_RESET);
+}
+
+void mtk_jpeg_dec_reset(void __iomem *base)
+{
+ mtk_jpeg_dec_soft_reset(base);
+ mtk_jpeg_dec_hard_reset(base);
+}
+
+static void mtk_jpeg_dec_set_brz_factor(void __iomem *base, u8 yscale_w,
+ u8 yscale_h, u8 uvscale_w, u8 uvscale_h)
+{
+ u32 val;
+
+ val = (uvscale_h << 12) | (uvscale_w << 8) |
+ (yscale_h << 4) | yscale_w;
+ writel(val, base + JPGDEC_REG_BRZ_FACTOR);
+}
+
+static void mtk_jpeg_dec_set_dst_bank0(void __iomem *base, u32 addr_y,
+ u32 addr_u, u32 addr_v)
+{
+ mtk_jpeg_verify_align(addr_y, 16, JPGDEC_REG_DEST_ADDR0_Y);
+ writel(addr_y, base + JPGDEC_REG_DEST_ADDR0_Y);
+ mtk_jpeg_verify_align(addr_u, 16, JPGDEC_REG_DEST_ADDR0_U);
+ writel(addr_u, base + JPGDEC_REG_DEST_ADDR0_U);
+ mtk_jpeg_verify_align(addr_v, 16, JPGDEC_REG_DEST_ADDR0_V);
+ writel(addr_v, base + JPGDEC_REG_DEST_ADDR0_V);
+}
+
+static void mtk_jpeg_dec_set_dst_bank1(void __iomem *base, u32 addr_y,
+ u32 addr_u, u32 addr_v)
+{
+ writel(addr_y, base + JPGDEC_REG_DEST_ADDR1_Y);
+ writel(addr_u, base + JPGDEC_REG_DEST_ADDR1_U);
+ writel(addr_v, base + JPGDEC_REG_DEST_ADDR1_V);
+}
+
+static void mtk_jpeg_dec_set_mem_stride(void __iomem *base, u32 stride_y,
+ u32 stride_uv)
+{
+ writel((stride_y & 0xFFFF), base + JPGDEC_REG_STRIDE_Y);
+ writel((stride_uv & 0xFFFF), base + JPGDEC_REG_STRIDE_UV);
+}
+
+static void mtk_jpeg_dec_set_img_stride(void __iomem *base, u32 stride_y,
+ u32 stride_uv)
+{
+ writel((stride_y & 0xFFFF), base + JPGDEC_REG_IMG_STRIDE_Y);
+ writel((stride_uv & 0xFFFF), base + JPGDEC_REG_IMG_STRIDE_UV);
+}
+
+static void mtk_jpeg_dec_set_pause_mcu_idx(void __iomem *base, u32 idx)
+{
+ writel(idx & 0x0003FFFFFF, base + JPGDEC_REG_PAUSE_MCU_NUM);
+}
+
+static void mtk_jpeg_dec_set_dec_mode(void __iomem *base, u32 mode)
+{
+ writel(mode & 0x03, base + JPGDEC_REG_OPERATION_MODE);
+}
+
+static void mtk_jpeg_dec_set_bs_write_ptr(void __iomem *base, u32 ptr)
+{
+ mtk_jpeg_verify_align(ptr, 16, JPGDEC_REG_FILE_BRP);
+ writel(ptr, base + JPGDEC_REG_FILE_BRP);
+}
+
+static void mtk_jpeg_dec_set_bs_info(void __iomem *base, u32 addr, u32 size)
+{
+ mtk_jpeg_verify_align(addr, 16, JPGDEC_REG_FILE_ADDR);
+ mtk_jpeg_verify_align(size, 128, JPGDEC_REG_FILE_TOTAL_SIZE);
+ writel(addr, base + JPGDEC_REG_FILE_ADDR);
+ writel(size, base + JPGDEC_REG_FILE_TOTAL_SIZE);
+}
+
+static void mtk_jpeg_dec_set_comp_id(void __iomem *base, u32 id_y, u32 id_u,
+ u32 id_v)
+{
+ u32 val;
+
+ val = ((id_y & 0x00FF) << 24) | ((id_u & 0x00FF) << 16) |
+ ((id_v & 0x00FF) << 8);
+ writel(val, base + JPGDEC_REG_COMP_ID);
+}
+
+static void mtk_jpeg_dec_set_total_mcu(void __iomem *base, u32 num)
+{
+ writel(num - 1, base + JPGDEC_REG_TOTAL_MCU_NUM);
+}
+
+static void mtk_jpeg_dec_set_comp0_du(void __iomem *base, u32 num)
+{
+ writel(num - 1, base + JPGDEC_REG_COMP0_DATA_UNIT_NUM);
+}
+
+static void mtk_jpeg_dec_set_du_membership(void __iomem *base, u32 member,
+ u32 gmc, u32 isgray)
+{
+ if (isgray)
+ member = 0x3FFFFFFC;
+ member |= (isgray << 31) | (gmc << 30);
+ writel(member, base + JPGDEC_REG_DU_CTRL);
+}
+
+static void mtk_jpeg_dec_set_q_table(void __iomem *base, u32 id0, u32 id1,
+ u32 id2)
+{
+ u32 val;
+
+ val = ((id0 & 0x0f) << 8) | ((id1 & 0x0f) << 4) | ((id2 & 0x0f) << 0);
+ writel(val, base + JPGDEC_REG_QT_ID);
+}
+
+static void mtk_jpeg_dec_set_dma_group(void __iomem *base, u32 mcu_group,
+ u32 group_num, u32 last_mcu)
+{
+ u32 val;
+
+ val = (((mcu_group - 1) & 0x00FF) << 16) |
+ (((group_num - 1) & 0x007F) << 8) |
+ ((last_mcu - 1) & 0x00FF);
+ writel(val, base + JPGDEC_REG_WDMA_CTRL);
+}
+
+static void mtk_jpeg_dec_set_sampling_factor(void __iomem *base, u32 comp_num,
+ u32 y_w, u32 y_h, u32 u_w,
+ u32 u_h, u32 v_w, u32 v_h)
+{
+ u32 val;
+ u32 y_wh = (MTK_JPEG_DUNUM_MASK(y_w) << 2) | MTK_JPEG_DUNUM_MASK(y_h);
+ u32 u_wh = (MTK_JPEG_DUNUM_MASK(u_w) << 2) | MTK_JPEG_DUNUM_MASK(u_h);
+ u32 v_wh = (MTK_JPEG_DUNUM_MASK(v_w) << 2) | MTK_JPEG_DUNUM_MASK(v_h);
+
+ if (comp_num == 1)
+ val = 0;
+ else
+ val = (y_wh << 8) | (u_wh << 4) | v_wh;
+ writel(val, base + JPGDEC_REG_DU_NUM);
+}
+
+void mtk_jpeg_dec_set_config(void __iomem *base,
+ struct mtk_jpeg_dec_param *config,
+ struct mtk_jpeg_bs *bs,
+ struct mtk_jpeg_fb *fb)
+{
+ mtk_jpeg_dec_set_brz_factor(base, 0, 0, config->uv_brz_w, 0);
+ mtk_jpeg_dec_set_dec_mode(base, 0);
+ mtk_jpeg_dec_set_comp0_du(base, config->unit_num);
+ mtk_jpeg_dec_set_total_mcu(base, config->total_mcu);
+ mtk_jpeg_dec_set_bs_info(base, bs->str_addr, bs->size);
+ mtk_jpeg_dec_set_bs_write_ptr(base, bs->end_addr);
+ mtk_jpeg_dec_set_du_membership(base, config->membership, 1,
+ (config->comp_num == 1) ? 1 : 0);
+ mtk_jpeg_dec_set_comp_id(base, config->comp_id[0], config->comp_id[1],
+ config->comp_id[2]);
+ mtk_jpeg_dec_set_q_table(base, config->qtbl_num[0],
+ config->qtbl_num[1], config->qtbl_num[2]);
+ mtk_jpeg_dec_set_sampling_factor(base, config->comp_num,
+ config->sampling_w[0],
+ config->sampling_h[0],
+ config->sampling_w[1],
+ config->sampling_h[1],
+ config->sampling_w[2],
+ config->sampling_h[2]);
+ mtk_jpeg_dec_set_mem_stride(base, config->mem_stride[0],
+ config->mem_stride[1]);
+ mtk_jpeg_dec_set_img_stride(base, config->img_stride[0],
+ config->img_stride[1]);
+ mtk_jpeg_dec_set_dst_bank0(base, fb->plane_addr[0],
+ fb->plane_addr[1], fb->plane_addr[2]);
+ mtk_jpeg_dec_set_dst_bank1(base, 0, 0, 0);
+ mtk_jpeg_dec_set_dma_group(base, config->dma_mcu, config->dma_group,
+ config->dma_last_mcu);
+ mtk_jpeg_dec_set_pause_mcu_idx(base, config->total_mcu);
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2016 MediaTek Inc.
+ * Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
+ * Rick Chang <rick.chang@mediatek.com>
+ * Xia Jiang <xia.jiang@mediatek.com>
+ */
+
+#ifndef _MTK_JPEG_DEC_HW_H
+#define _MTK_JPEG_DEC_HW_H
+
+#include <media/videobuf2-core.h>
+
+#include "mtk_jpeg_core.h"
+#include "mtk_jpeg_dec_reg.h"
+
+enum {
+ MTK_JPEG_DEC_RESULT_EOF_DONE = 0,
+ MTK_JPEG_DEC_RESULT_PAUSE = 1,
+ MTK_JPEG_DEC_RESULT_UNDERFLOW = 2,
+ MTK_JPEG_DEC_RESULT_OVERFLOW = 3,
+ MTK_JPEG_DEC_RESULT_ERROR_BS = 4,
+ MTK_JPEG_DEC_RESULT_ERROR_UNKNOWN = 6
+};
+
+struct mtk_jpeg_dec_param {
+ u32 pic_w;
+ u32 pic_h;
+ u32 dec_w;
+ u32 dec_h;
+ u32 src_color;
+ u32 dst_fourcc;
+ u32 mcu_w;
+ u32 mcu_h;
+ u32 total_mcu;
+ u32 unit_num;
+ u32 comp_num;
+ u32 comp_id[MTK_JPEG_COMP_MAX];
+ u32 sampling_w[MTK_JPEG_COMP_MAX];
+ u32 sampling_h[MTK_JPEG_COMP_MAX];
+ u32 qtbl_num[MTK_JPEG_COMP_MAX];
+ u32 blk_num;
+ u32 blk_comp[MTK_JPEG_COMP_MAX];
+ u32 membership;
+ u32 dma_mcu;
+ u32 dma_group;
+ u32 dma_last_mcu;
+ u32 img_stride[MTK_JPEG_COMP_MAX];
+ u32 mem_stride[MTK_JPEG_COMP_MAX];
+ u32 comp_w[MTK_JPEG_COMP_MAX];
+ u32 comp_size[MTK_JPEG_COMP_MAX];
+ u32 y_size;
+ u32 uv_size;
+ u32 dec_size;
+ u8 uv_brz_w;
+};
+
+struct mtk_jpeg_bs {
+ dma_addr_t str_addr;
+ dma_addr_t end_addr;
+ size_t size;
+};
+
+struct mtk_jpeg_fb {
+ dma_addr_t plane_addr[MTK_JPEG_COMP_MAX];
+ size_t size;
+};
+
+int mtk_jpeg_dec_fill_param(struct mtk_jpeg_dec_param *param);
+u32 mtk_jpeg_dec_get_int_status(void __iomem *dec_reg_base);
+u32 mtk_jpeg_dec_enum_result(u32 irq_result);
+void mtk_jpeg_dec_set_config(void __iomem *base,
+ struct mtk_jpeg_dec_param *config,
+ struct mtk_jpeg_bs *bs,
+ struct mtk_jpeg_fb *fb);
+void mtk_jpeg_dec_reset(void __iomem *dec_reg_base);
+void mtk_jpeg_dec_start(void __iomem *dec_reg_base);
+
+#endif /* _MTK_JPEG_HW_H */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2016 MediaTek Inc.
+ * Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
+ * Rick Chang <rick.chang@mediatek.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/videodev2.h>
+
+#include "mtk_jpeg_dec_parse.h"
+
+#define TEM 0x01
+#define SOF0 0xc0
+#define RST 0xd0
+#define SOI 0xd8
+#define EOI 0xd9
+
+struct mtk_jpeg_stream {
+ u8 *addr;
+ u32 size;
+ u32 curr;
+};
+
+static int read_byte(struct mtk_jpeg_stream *stream)
+{
+ if (stream->curr >= stream->size)
+ return -1;
+ return stream->addr[stream->curr++];
+}
+
+static int read_word_be(struct mtk_jpeg_stream *stream, u32 *word)
+{
+ u32 temp;
+ int byte;
+
+ byte = read_byte(stream);
+ if (byte == -1)
+ return -1;
+ temp = byte << 8;
+ byte = read_byte(stream);
+ if (byte == -1)
+ return -1;
+ *word = (u32)byte | temp;
+
+ return 0;
+}
+
+static void read_skip(struct mtk_jpeg_stream *stream, long len)
+{
+ if (len <= 0)
+ return;
+ while (len--)
+ read_byte(stream);
+}
+
+static bool mtk_jpeg_do_parse(struct mtk_jpeg_dec_param *param, u8 *src_addr_va,
+ u32 src_size)
+{
+ bool notfound = true;
+ struct mtk_jpeg_stream stream;
+
+ stream.addr = src_addr_va;
+ stream.size = src_size;
+ stream.curr = 0;
+
+ while (notfound) {
+ int i, length, byte;
+ u32 word;
+
+ byte = read_byte(&stream);
+ if (byte == -1)
+ return false;
+ if (byte != 0xff)
+ continue;
+ do
+ byte = read_byte(&stream);
+ while (byte == 0xff);
+ if (byte == -1)
+ return false;
+ if (byte == 0)
+ continue;
+
+ length = 0;
+ switch (byte) {
+ case SOF0:
+ /* length */
+ if (read_word_be(&stream, &word))
+ break;
+
+ /* precision */
+ if (read_byte(&stream) == -1)
+ break;
+
+ if (read_word_be(&stream, &word))
+ break;
+ param->pic_h = word;
+
+ if (read_word_be(&stream, &word))
+ break;
+ param->pic_w = word;
+
+ param->comp_num = read_byte(&stream);
+ if (param->comp_num != 1 && param->comp_num != 3)
+ break;
+
+ for (i = 0; i < param->comp_num; i++) {
+ param->comp_id[i] = read_byte(&stream);
+ if (param->comp_id[i] == -1)
+ break;
+
+ /* sampling */
+ byte = read_byte(&stream);
+ if (byte == -1)
+ break;
+ param->sampling_w[i] = (byte >> 4) & 0x0F;
+ param->sampling_h[i] = byte & 0x0F;
+
+ param->qtbl_num[i] = read_byte(&stream);
+ if (param->qtbl_num[i] == -1)
+ break;
+ }
+
+ notfound = !(i == param->comp_num);
+ break;
+ case RST ... RST + 7:
+ case SOI:
+ case EOI:
+ case TEM:
+ break;
+ default:
+ if (read_word_be(&stream, &word))
+ break;
+ length = (long)word - 2;
+ read_skip(&stream, length);
+ break;
+ }
+ }
+
+ return !notfound;
+}
+
+bool mtk_jpeg_parse(struct mtk_jpeg_dec_param *param, u8 *src_addr_va,
+ u32 src_size)
+{
+ if (!mtk_jpeg_do_parse(param, src_addr_va, src_size))
+ return false;
+ if (mtk_jpeg_dec_fill_param(param))
+ return false;
+
+ return true;
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2016 MediaTek Inc.
+ * Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
+ * Rick Chang <rick.chang@mediatek.com>
+ */
+
+#ifndef _MTK_JPEG_PARSE_H
+#define _MTK_JPEG_PARSE_H
+
+#include "mtk_jpeg_dec_hw.h"
+
+bool mtk_jpeg_parse(struct mtk_jpeg_dec_param *param, u8 *src_addr_va,
+ u32 src_size);
+
+#endif /* _MTK_JPEG_PARSE_H */
+
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2016 MediaTek Inc.
+ * Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
+ * Rick Chang <rick.chang@mediatek.com>
+ */
+
+#ifndef _MTK_JPEG_REG_H
+#define _MTK_JPEG_REG_H
+
+#define MTK_JPEG_BLOCK_MAX 10
+#define MTK_JPEG_DCTSIZE 8
+
+#define BIT_INQST_MASK_ERROR_BS 0x20
+#define BIT_INQST_MASK_PAUSE 0x10
+#define BIT_INQST_MASK_OVERFLOW 0x04
+#define BIT_INQST_MASK_UNDERFLOW 0x02
+#define BIT_INQST_MASK_EOF 0x01
+#define BIT_INQST_MASK_ALLIRQ 0x37
+
+#define JPGDEC_REG_RESET 0x0090
+#define JPGDEC_REG_BRZ_FACTOR 0x00f8
+#define JPGDEC_REG_DU_NUM 0x00fc
+#define JPGDEC_REG_DEST_ADDR0_Y 0x0140
+#define JPGDEC_REG_DEST_ADDR0_U 0x0144
+#define JPGDEC_REG_DEST_ADDR0_V 0x0148
+#define JPGDEC_REG_DEST_ADDR1_Y 0x014c
+#define JPGDEC_REG_DEST_ADDR1_U 0x0150
+#define JPGDEC_REG_DEST_ADDR1_V 0x0154
+#define JPGDEC_REG_STRIDE_Y 0x0158
+#define JPGDEC_REG_STRIDE_UV 0x015c
+#define JPGDEC_REG_IMG_STRIDE_Y 0x0160
+#define JPGDEC_REG_IMG_STRIDE_UV 0x0164
+#define JPGDEC_REG_WDMA_CTRL 0x016c
+#define JPGDEC_REG_PAUSE_MCU_NUM 0x0170
+#define JPGDEC_REG_OPERATION_MODE 0x017c
+#define JPGDEC_REG_FILE_ADDR 0x0200
+#define JPGDEC_REG_COMP_ID 0x020c
+#define JPGDEC_REG_TOTAL_MCU_NUM 0x0210
+#define JPGDEC_REG_COMP0_DATA_UNIT_NUM 0x0224
+#define JPGDEC_REG_DU_CTRL 0x023c
+#define JPGDEC_REG_TRIG 0x0240
+#define JPGDEC_REG_FILE_BRP 0x0248
+#define JPGDEC_REG_FILE_TOTAL_SIZE 0x024c
+#define JPGDEC_REG_QT_ID 0x0270
+#define JPGDEC_REG_INTERRUPT_STATUS 0x0274
+#define JPGDEC_REG_STATUS 0x0278
+
+#endif /* _MTK_JPEG_REG_H */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2019 MediaTek Inc.
+ * Author: Xia Jiang <xia.jiang@mediatek.com>
+ *
+ */
+
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include "mtk_jpeg_enc_hw.h"
+
+static const struct mtk_jpeg_enc_qlt mtk_jpeg_enc_quality[] = {
+ {.quality_param = 34, .hardware_value = JPEG_ENC_QUALITY_Q34},
+ {.quality_param = 39, .hardware_value = JPEG_ENC_QUALITY_Q39},
+ {.quality_param = 48, .hardware_value = JPEG_ENC_QUALITY_Q48},
+ {.quality_param = 60, .hardware_value = JPEG_ENC_QUALITY_Q60},
+ {.quality_param = 64, .hardware_value = JPEG_ENC_QUALITY_Q64},
+ {.quality_param = 68, .hardware_value = JPEG_ENC_QUALITY_Q68},
+ {.quality_param = 74, .hardware_value = JPEG_ENC_QUALITY_Q74},
+ {.quality_param = 80, .hardware_value = JPEG_ENC_QUALITY_Q80},
+ {.quality_param = 82, .hardware_value = JPEG_ENC_QUALITY_Q82},
+ {.quality_param = 84, .hardware_value = JPEG_ENC_QUALITY_Q84},
+ {.quality_param = 87, .hardware_value = JPEG_ENC_QUALITY_Q87},
+ {.quality_param = 90, .hardware_value = JPEG_ENC_QUALITY_Q90},
+ {.quality_param = 92, .hardware_value = JPEG_ENC_QUALITY_Q92},
+ {.quality_param = 95, .hardware_value = JPEG_ENC_QUALITY_Q95},
+ {.quality_param = 97, .hardware_value = JPEG_ENC_QUALITY_Q97},
+};
+
+void mtk_jpeg_enc_reset(void __iomem *base)
+{
+ writel(0, base + JPEG_ENC_RSTB);
+ writel(JPEG_ENC_RESET_BIT, base + JPEG_ENC_RSTB);
+ writel(0, base + JPEG_ENC_CODEC_SEL);
+}
+
+u32 mtk_jpeg_enc_get_file_size(void __iomem *base)
+{
+ return readl(base + JPEG_ENC_DMA_ADDR0) -
+ readl(base + JPEG_ENC_DST_ADDR0);
+}
+
+void mtk_jpeg_enc_start(void __iomem *base)
+{
+ u32 value;
+
+ value = readl(base + JPEG_ENC_CTRL);
+ value |= JPEG_ENC_CTRL_INT_EN_BIT | JPEG_ENC_CTRL_ENABLE_BIT;
+ writel(value, base + JPEG_ENC_CTRL);
+}
+
+void mtk_jpeg_set_enc_src(struct mtk_jpeg_ctx *ctx, void __iomem *base,
+ struct vb2_buffer *src_buf)
+{
+ int i;
+ dma_addr_t dma_addr;
+
+ for (i = 0; i < src_buf->num_planes; i++) {
+ dma_addr = vb2_dma_contig_plane_dma_addr(src_buf, i) +
+ src_buf->planes[i].data_offset;
+ if (!i)
+ writel(dma_addr, base + JPEG_ENC_SRC_LUMA_ADDR);
+ else
+ writel(dma_addr, base + JPEG_ENC_SRC_CHROMA_ADDR);
+ }
+}
+
+void mtk_jpeg_set_enc_dst(struct mtk_jpeg_ctx *ctx, void __iomem *base,
+ struct vb2_buffer *dst_buf)
+{
+ dma_addr_t dma_addr;
+ size_t size;
+ u32 dma_addr_offset;
+ u32 dma_addr_offsetmask;
+
+ dma_addr = vb2_dma_contig_plane_dma_addr(dst_buf, 0);
+ dma_addr_offset = ctx->enable_exif ? MTK_JPEG_MAX_EXIF_SIZE : 0;
+ dma_addr_offsetmask = dma_addr & JPEG_ENC_DST_ADDR_OFFSET_MASK;
+ size = vb2_plane_size(dst_buf, 0);
+
+ writel(dma_addr_offset & ~0xf, base + JPEG_ENC_OFFSET_ADDR);
+ writel(dma_addr_offsetmask & 0xf, base + JPEG_ENC_BYTE_OFFSET_MASK);
+ writel(dma_addr & ~0xf, base + JPEG_ENC_DST_ADDR0);
+ writel((dma_addr + size) & ~0xf, base + JPEG_ENC_STALL_ADDR0);
+}
+
+void mtk_jpeg_set_enc_params(struct mtk_jpeg_ctx *ctx, void __iomem *base)
+{
+ u32 value;
+ u32 width = ctx->out_q.enc_crop_rect.width;
+ u32 height = ctx->out_q.enc_crop_rect.height;
+ u32 enc_format = ctx->out_q.fmt->fourcc;
+ u32 bytesperline = ctx->out_q.pix_mp.plane_fmt[0].bytesperline;
+ u32 blk_num;
+ u32 img_stride;
+ u32 mem_stride;
+ u32 i, enc_quality;
+
+ value = width << 16 | height;
+ writel(value, base + JPEG_ENC_IMG_SIZE);
+
+ if (enc_format == V4L2_PIX_FMT_NV12M ||
+ enc_format == V4L2_PIX_FMT_NV21M)
+ /*
+ * Total 8 x 8 block number of luma and chroma.
+ * The number of blocks is counted from 0.
+ */
+ blk_num = DIV_ROUND_UP(width, 16) *
+ DIV_ROUND_UP(height, 16) * 6 - 1;
+ else
+ blk_num = DIV_ROUND_UP(width, 16) *
+ DIV_ROUND_UP(height, 8) * 4 - 1;
+ writel(blk_num, base + JPEG_ENC_BLK_NUM);
+
+ if (enc_format == V4L2_PIX_FMT_NV12M ||
+ enc_format == V4L2_PIX_FMT_NV21M) {
+ /* 4:2:0 */
+ img_stride = round_up(width, 16);
+ mem_stride = bytesperline;
+ } else {
+ /* 4:2:2 */
+ img_stride = round_up(width * 2, 32);
+ mem_stride = img_stride;
+ }
+ writel(img_stride, base + JPEG_ENC_IMG_STRIDE);
+ writel(mem_stride, base + JPEG_ENC_STRIDE);
+
+ enc_quality = mtk_jpeg_enc_quality[0].hardware_value;
+ for (i = 0; i < ARRAY_SIZE(mtk_jpeg_enc_quality); i++) {
+ if (ctx->enc_quality <= mtk_jpeg_enc_quality[i].quality_param) {
+ enc_quality = mtk_jpeg_enc_quality[i].hardware_value;
+ break;
+ }
+ }
+ writel(enc_quality, base + JPEG_ENC_QUALITY);
+
+ value = readl(base + JPEG_ENC_CTRL);
+ value &= ~JPEG_ENC_CTRL_YUV_FORMAT_MASK;
+ value |= (ctx->out_q.fmt->hw_format & 3) << 3;
+ if (ctx->enable_exif)
+ value |= JPEG_ENC_CTRL_FILE_FORMAT_BIT;
+ else
+ value &= ~JPEG_ENC_CTRL_FILE_FORMAT_BIT;
+ if (ctx->restart_interval)
+ value |= JPEG_ENC_CTRL_RESTART_EN_BIT;
+ else
+ value &= ~JPEG_ENC_CTRL_RESTART_EN_BIT;
+ writel(value, base + JPEG_ENC_CTRL);
+
+ writel(ctx->restart_interval, base + JPEG_ENC_RST_MCU_NUM);
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2019 MediaTek Inc.
+ * Author: Xia Jiang <xia.jiang@mediatek.com>
+ *
+ */
+
+#ifndef _MTK_JPEG_ENC_HW_H
+#define _MTK_JPEG_ENC_HW_H
+
+#include <media/videobuf2-core.h>
+
+#include "mtk_jpeg_core.h"
+
+#define JPEG_ENC_INT_STATUS_DONE BIT(0)
+#define JPEG_ENC_INT_STATUS_MASK_ALLIRQ 0x13
+
+#define JPEG_ENC_DST_ADDR_OFFSET_MASK GENMASK(3, 0)
+
+#define JPEG_ENC_CTRL_YUV_FORMAT_MASK 0x18
+#define JPEG_ENC_CTRL_RESTART_EN_BIT BIT(10)
+#define JPEG_ENC_CTRL_FILE_FORMAT_BIT BIT(5)
+#define JPEG_ENC_CTRL_INT_EN_BIT BIT(2)
+#define JPEG_ENC_CTRL_ENABLE_BIT BIT(0)
+#define JPEG_ENC_RESET_BIT BIT(0)
+
+#define JPEG_ENC_YUV_FORMAT_YUYV 0
+#define JPEG_ENC_YUV_FORMAT_YVYU 1
+#define JPEG_ENC_YUV_FORMAT_NV12 2
+#define JEPG_ENC_YUV_FORMAT_NV21 3
+
+#define JPEG_ENC_QUALITY_Q60 0x0
+#define JPEG_ENC_QUALITY_Q80 0x1
+#define JPEG_ENC_QUALITY_Q90 0x2
+#define JPEG_ENC_QUALITY_Q95 0x3
+#define JPEG_ENC_QUALITY_Q39 0x4
+#define JPEG_ENC_QUALITY_Q68 0x5
+#define JPEG_ENC_QUALITY_Q84 0x6
+#define JPEG_ENC_QUALITY_Q92 0x7
+#define JPEG_ENC_QUALITY_Q48 0x8
+#define JPEG_ENC_QUALITY_Q74 0xa
+#define JPEG_ENC_QUALITY_Q87 0xb
+#define JPEG_ENC_QUALITY_Q34 0xc
+#define JPEG_ENC_QUALITY_Q64 0xe
+#define JPEG_ENC_QUALITY_Q82 0xf
+#define JPEG_ENC_QUALITY_Q97 0x10
+
+#define JPEG_ENC_RSTB 0x100
+#define JPEG_ENC_CTRL 0x104
+#define JPEG_ENC_QUALITY 0x108
+#define JPEG_ENC_BLK_NUM 0x10C
+#define JPEG_ENC_BLK_CNT 0x110
+#define JPEG_ENC_INT_STS 0x11c
+#define JPEG_ENC_DST_ADDR0 0x120
+#define JPEG_ENC_DMA_ADDR0 0x124
+#define JPEG_ENC_STALL_ADDR0 0x128
+#define JPEG_ENC_OFFSET_ADDR 0x138
+#define JPEG_ENC_RST_MCU_NUM 0x150
+#define JPEG_ENC_IMG_SIZE 0x154
+#define JPEG_ENC_DEBUG_INFO0 0x160
+#define JPEG_ENC_DEBUG_INFO1 0x164
+#define JPEG_ENC_TOTAL_CYCLE 0x168
+#define JPEG_ENC_BYTE_OFFSET_MASK 0x16c
+#define JPEG_ENC_SRC_LUMA_ADDR 0x170
+#define JPEG_ENC_SRC_CHROMA_ADDR 0x174
+#define JPEG_ENC_STRIDE 0x178
+#define JPEG_ENC_IMG_STRIDE 0x17c
+#define JPEG_ENC_DCM_CTRL 0x300
+#define JPEG_ENC_CODEC_SEL 0x314
+#define JPEG_ENC_ULTRA_THRES 0x318
+
+/**
+ * struct mtk_jpeg_enc_qlt - JPEG encoder quality data
+ * @quality_param: quality value
+ * @hardware_value: hardware value of quality
+ */
+struct mtk_jpeg_enc_qlt {
+ u8 quality_param;
+ u8 hardware_value;
+};
+
+void mtk_jpeg_enc_reset(void __iomem *base);
+u32 mtk_jpeg_enc_get_file_size(void __iomem *base);
+void mtk_jpeg_enc_start(void __iomem *enc_reg_base);
+void mtk_jpeg_set_enc_src(struct mtk_jpeg_ctx *ctx, void __iomem *base,
+ struct vb2_buffer *src_buf);
+void mtk_jpeg_set_enc_dst(struct mtk_jpeg_ctx *ctx, void __iomem *base,
+ struct vb2_buffer *dst_buf);
+void mtk_jpeg_set_enc_params(struct mtk_jpeg_ctx *ctx, void __iomem *base);
+
+#endif /* _MTK_JPEG_ENC_HW_H */
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2016 MediaTek Inc.
- * Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
- * Rick Chang <rick.chang@mediatek.com>
- */
-
-#include <linux/io.h>
-#include <linux/kernel.h>
-#include <media/videobuf2-core.h>
-
-#include "mtk_jpeg_hw.h"
-
-#define MTK_JPEG_DUNUM_MASK(val) (((val) - 1) & 0x3)
-
-enum mtk_jpeg_color {
- MTK_JPEG_COLOR_420 = 0x00221111,
- MTK_JPEG_COLOR_422 = 0x00211111,
- MTK_JPEG_COLOR_444 = 0x00111111,
- MTK_JPEG_COLOR_422V = 0x00121111,
- MTK_JPEG_COLOR_422X2 = 0x00412121,
- MTK_JPEG_COLOR_422VX2 = 0x00222121,
- MTK_JPEG_COLOR_400 = 0x00110000
-};
-
-static inline int mtk_jpeg_verify_align(u32 val, int align, u32 reg)
-{
- if (val & (align - 1)) {
- pr_err("mtk-jpeg: write reg %x without %d align\n", reg, align);
- return -1;
- }
-
- return 0;
-}
-
-static int mtk_jpeg_decide_format(struct mtk_jpeg_dec_param *param)
-{
- param->src_color = (param->sampling_w[0] << 20) |
- (param->sampling_h[0] << 16) |
- (param->sampling_w[1] << 12) |
- (param->sampling_h[1] << 8) |
- (param->sampling_w[2] << 4) |
- (param->sampling_h[2]);
-
- param->uv_brz_w = 0;
- switch (param->src_color) {
- case MTK_JPEG_COLOR_444:
- param->uv_brz_w = 1;
- param->dst_fourcc = V4L2_PIX_FMT_YUV422M;
- break;
- case MTK_JPEG_COLOR_422X2:
- case MTK_JPEG_COLOR_422:
- param->dst_fourcc = V4L2_PIX_FMT_YUV422M;
- break;
- case MTK_JPEG_COLOR_422V:
- case MTK_JPEG_COLOR_422VX2:
- param->uv_brz_w = 1;
- param->dst_fourcc = V4L2_PIX_FMT_YUV420M;
- break;
- case MTK_JPEG_COLOR_420:
- param->dst_fourcc = V4L2_PIX_FMT_YUV420M;
- break;
- case MTK_JPEG_COLOR_400:
- param->dst_fourcc = V4L2_PIX_FMT_GREY;
- break;
- default:
- param->dst_fourcc = 0;
- return -1;
- }
-
- return 0;
-}
-
-static void mtk_jpeg_calc_mcu(struct mtk_jpeg_dec_param *param)
-{
- u32 factor_w, factor_h;
- u32 i, comp, blk;
-
- factor_w = 2 + param->sampling_w[0];
- factor_h = 2 + param->sampling_h[0];
- param->mcu_w = (param->pic_w + (1 << factor_w) - 1) >> factor_w;
- param->mcu_h = (param->pic_h + (1 << factor_h) - 1) >> factor_h;
- param->total_mcu = param->mcu_w * param->mcu_h;
- param->unit_num = ((param->pic_w + 7) >> 3) * ((param->pic_h + 7) >> 3);
- param->blk_num = 0;
- for (i = 0; i < MTK_JPEG_COMP_MAX; i++) {
- param->blk_comp[i] = 0;
- if (i >= param->comp_num)
- continue;
- param->blk_comp[i] = param->sampling_w[i] *
- param->sampling_h[i];
- param->blk_num += param->blk_comp[i];
- }
-
- param->membership = 0;
- for (i = 0, blk = 0, comp = 0; i < MTK_JPEG_BLOCK_MAX; i++) {
- if (i < param->blk_num && comp < param->comp_num) {
- u32 tmp;
-
- tmp = (0x04 + (comp & 0x3));
- param->membership |= tmp << (i * 3);
- if (++blk == param->blk_comp[comp]) {
- comp++;
- blk = 0;
- }
- } else {
- param->membership |= 7 << (i * 3);
- }
- }
-}
-
-static void mtk_jpeg_calc_dma_group(struct mtk_jpeg_dec_param *param)
-{
- u32 factor_mcu = 3;
-
- if (param->src_color == MTK_JPEG_COLOR_444 &&
- param->dst_fourcc == V4L2_PIX_FMT_YUV422M)
- factor_mcu = 4;
- else if (param->src_color == MTK_JPEG_COLOR_422V &&
- param->dst_fourcc == V4L2_PIX_FMT_YUV420M)
- factor_mcu = 4;
- else if (param->src_color == MTK_JPEG_COLOR_422X2 &&
- param->dst_fourcc == V4L2_PIX_FMT_YUV422M)
- factor_mcu = 2;
- else if (param->src_color == MTK_JPEG_COLOR_400 ||
- (param->src_color & 0x0FFFF) == 0)
- factor_mcu = 4;
-
- param->dma_mcu = 1 << factor_mcu;
- param->dma_group = param->mcu_w / param->dma_mcu;
- param->dma_last_mcu = param->mcu_w % param->dma_mcu;
- if (param->dma_last_mcu)
- param->dma_group++;
- else
- param->dma_last_mcu = param->dma_mcu;
-}
-
-static int mtk_jpeg_calc_dst_size(struct mtk_jpeg_dec_param *param)
-{
- u32 i, padding_w;
- u32 ds_row_h[3];
- u32 brz_w[3];
-
- brz_w[0] = 0;
- brz_w[1] = param->uv_brz_w;
- brz_w[2] = brz_w[1];
-
- for (i = 0; i < param->comp_num; i++) {
- if (brz_w[i] > 3)
- return -1;
-
- padding_w = param->mcu_w * MTK_JPEG_DCTSIZE *
- param->sampling_w[i];
- /* output format is 420/422 */
- param->comp_w[i] = padding_w >> brz_w[i];
- param->comp_w[i] = mtk_jpeg_align(param->comp_w[i],
- MTK_JPEG_DCTSIZE);
- param->img_stride[i] = i ? mtk_jpeg_align(param->comp_w[i], 16)
- : mtk_jpeg_align(param->comp_w[i], 32);
- ds_row_h[i] = (MTK_JPEG_DCTSIZE * param->sampling_h[i]);
- }
- param->dec_w = param->img_stride[0];
- param->dec_h = ds_row_h[0] * param->mcu_h;
-
- for (i = 0; i < MTK_JPEG_COMP_MAX; i++) {
- /* They must be equal in frame mode. */
- param->mem_stride[i] = param->img_stride[i];
- param->comp_size[i] = param->mem_stride[i] * ds_row_h[i] *
- param->mcu_h;
- }
-
- param->y_size = param->comp_size[0];
- param->uv_size = param->comp_size[1];
- param->dec_size = param->y_size + (param->uv_size << 1);
-
- return 0;
-}
-
-int mtk_jpeg_dec_fill_param(struct mtk_jpeg_dec_param *param)
-{
- if (mtk_jpeg_decide_format(param))
- return -1;
-
- mtk_jpeg_calc_mcu(param);
- mtk_jpeg_calc_dma_group(param);
- if (mtk_jpeg_calc_dst_size(param))
- return -2;
-
- return 0;
-}
-
-u32 mtk_jpeg_dec_get_int_status(void __iomem *base)
-{
- u32 ret;
-
- ret = readl(base + JPGDEC_REG_INTERRUPT_STATUS) & BIT_INQST_MASK_ALLIRQ;
- if (ret)
- writel(ret, base + JPGDEC_REG_INTERRUPT_STATUS);
-
- return ret;
-}
-
-u32 mtk_jpeg_dec_enum_result(u32 irq_result)
-{
- if (irq_result & BIT_INQST_MASK_EOF)
- return MTK_JPEG_DEC_RESULT_EOF_DONE;
- if (irq_result & BIT_INQST_MASK_PAUSE)
- return MTK_JPEG_DEC_RESULT_PAUSE;
- if (irq_result & BIT_INQST_MASK_UNDERFLOW)
- return MTK_JPEG_DEC_RESULT_UNDERFLOW;
- if (irq_result & BIT_INQST_MASK_OVERFLOW)
- return MTK_JPEG_DEC_RESULT_OVERFLOW;
- if (irq_result & BIT_INQST_MASK_ERROR_BS)
- return MTK_JPEG_DEC_RESULT_ERROR_BS;
-
- return MTK_JPEG_DEC_RESULT_ERROR_UNKNOWN;
-}
-
-void mtk_jpeg_dec_start(void __iomem *base)
-{
- writel(0, base + JPGDEC_REG_TRIG);
-}
-
-static void mtk_jpeg_dec_soft_reset(void __iomem *base)
-{
- writel(0x0000FFFF, base + JPGDEC_REG_INTERRUPT_STATUS);
- writel(0x00, base + JPGDEC_REG_RESET);
- writel(0x01, base + JPGDEC_REG_RESET);
-}
-
-static void mtk_jpeg_dec_hard_reset(void __iomem *base)
-{
- writel(0x00, base + JPGDEC_REG_RESET);
- writel(0x10, base + JPGDEC_REG_RESET);
-}
-
-void mtk_jpeg_dec_reset(void __iomem *base)
-{
- mtk_jpeg_dec_soft_reset(base);
- mtk_jpeg_dec_hard_reset(base);
-}
-
-static void mtk_jpeg_dec_set_brz_factor(void __iomem *base, u8 yscale_w,
- u8 yscale_h, u8 uvscale_w, u8 uvscale_h)
-{
- u32 val;
-
- val = (uvscale_h << 12) | (uvscale_w << 8) |
- (yscale_h << 4) | yscale_w;
- writel(val, base + JPGDEC_REG_BRZ_FACTOR);
-}
-
-static void mtk_jpeg_dec_set_dst_bank0(void __iomem *base, u32 addr_y,
- u32 addr_u, u32 addr_v)
-{
- mtk_jpeg_verify_align(addr_y, 16, JPGDEC_REG_DEST_ADDR0_Y);
- writel(addr_y, base + JPGDEC_REG_DEST_ADDR0_Y);
- mtk_jpeg_verify_align(addr_u, 16, JPGDEC_REG_DEST_ADDR0_U);
- writel(addr_u, base + JPGDEC_REG_DEST_ADDR0_U);
- mtk_jpeg_verify_align(addr_v, 16, JPGDEC_REG_DEST_ADDR0_V);
- writel(addr_v, base + JPGDEC_REG_DEST_ADDR0_V);
-}
-
-static void mtk_jpeg_dec_set_dst_bank1(void __iomem *base, u32 addr_y,
- u32 addr_u, u32 addr_v)
-{
- writel(addr_y, base + JPGDEC_REG_DEST_ADDR1_Y);
- writel(addr_u, base + JPGDEC_REG_DEST_ADDR1_U);
- writel(addr_v, base + JPGDEC_REG_DEST_ADDR1_V);
-}
-
-static void mtk_jpeg_dec_set_mem_stride(void __iomem *base, u32 stride_y,
- u32 stride_uv)
-{
- writel((stride_y & 0xFFFF), base + JPGDEC_REG_STRIDE_Y);
- writel((stride_uv & 0xFFFF), base + JPGDEC_REG_STRIDE_UV);
-}
-
-static void mtk_jpeg_dec_set_img_stride(void __iomem *base, u32 stride_y,
- u32 stride_uv)
-{
- writel((stride_y & 0xFFFF), base + JPGDEC_REG_IMG_STRIDE_Y);
- writel((stride_uv & 0xFFFF), base + JPGDEC_REG_IMG_STRIDE_UV);
-}
-
-static void mtk_jpeg_dec_set_pause_mcu_idx(void __iomem *base, u32 idx)
-{
- writel(idx & 0x0003FFFFFF, base + JPGDEC_REG_PAUSE_MCU_NUM);
-}
-
-static void mtk_jpeg_dec_set_dec_mode(void __iomem *base, u32 mode)
-{
- writel(mode & 0x03, base + JPGDEC_REG_OPERATION_MODE);
-}
-
-static void mtk_jpeg_dec_set_bs_write_ptr(void __iomem *base, u32 ptr)
-{
- mtk_jpeg_verify_align(ptr, 16, JPGDEC_REG_FILE_BRP);
- writel(ptr, base + JPGDEC_REG_FILE_BRP);
-}
-
-static void mtk_jpeg_dec_set_bs_info(void __iomem *base, u32 addr, u32 size)
-{
- mtk_jpeg_verify_align(addr, 16, JPGDEC_REG_FILE_ADDR);
- mtk_jpeg_verify_align(size, 128, JPGDEC_REG_FILE_TOTAL_SIZE);
- writel(addr, base + JPGDEC_REG_FILE_ADDR);
- writel(size, base + JPGDEC_REG_FILE_TOTAL_SIZE);
-}
-
-static void mtk_jpeg_dec_set_comp_id(void __iomem *base, u32 id_y, u32 id_u,
- u32 id_v)
-{
- u32 val;
-
- val = ((id_y & 0x00FF) << 24) | ((id_u & 0x00FF) << 16) |
- ((id_v & 0x00FF) << 8);
- writel(val, base + JPGDEC_REG_COMP_ID);
-}
-
-static void mtk_jpeg_dec_set_total_mcu(void __iomem *base, u32 num)
-{
- writel(num - 1, base + JPGDEC_REG_TOTAL_MCU_NUM);
-}
-
-static void mtk_jpeg_dec_set_comp0_du(void __iomem *base, u32 num)
-{
- writel(num - 1, base + JPGDEC_REG_COMP0_DATA_UNIT_NUM);
-}
-
-static void mtk_jpeg_dec_set_du_membership(void __iomem *base, u32 member,
- u32 gmc, u32 isgray)
-{
- if (isgray)
- member = 0x3FFFFFFC;
- member |= (isgray << 31) | (gmc << 30);
- writel(member, base + JPGDEC_REG_DU_CTRL);
-}
-
-static void mtk_jpeg_dec_set_q_table(void __iomem *base, u32 id0, u32 id1,
- u32 id2)
-{
- u32 val;
-
- val = ((id0 & 0x0f) << 8) | ((id1 & 0x0f) << 4) | ((id2 & 0x0f) << 0);
- writel(val, base + JPGDEC_REG_QT_ID);
-}
-
-static void mtk_jpeg_dec_set_dma_group(void __iomem *base, u32 mcu_group,
- u32 group_num, u32 last_mcu)
-{
- u32 val;
-
- val = (((mcu_group - 1) & 0x00FF) << 16) |
- (((group_num - 1) & 0x007F) << 8) |
- ((last_mcu - 1) & 0x00FF);
- writel(val, base + JPGDEC_REG_WDMA_CTRL);
-}
-
-static void mtk_jpeg_dec_set_sampling_factor(void __iomem *base, u32 comp_num,
- u32 y_w, u32 y_h, u32 u_w,
- u32 u_h, u32 v_w, u32 v_h)
-{
- u32 val;
- u32 y_wh = (MTK_JPEG_DUNUM_MASK(y_w) << 2) | MTK_JPEG_DUNUM_MASK(y_h);
- u32 u_wh = (MTK_JPEG_DUNUM_MASK(u_w) << 2) | MTK_JPEG_DUNUM_MASK(u_h);
- u32 v_wh = (MTK_JPEG_DUNUM_MASK(v_w) << 2) | MTK_JPEG_DUNUM_MASK(v_h);
-
- if (comp_num == 1)
- val = 0;
- else
- val = (y_wh << 8) | (u_wh << 4) | v_wh;
- writel(val, base + JPGDEC_REG_DU_NUM);
-}
-
-void mtk_jpeg_dec_set_config(void __iomem *base,
- struct mtk_jpeg_dec_param *config,
- struct mtk_jpeg_bs *bs,
- struct mtk_jpeg_fb *fb)
-{
- mtk_jpeg_dec_set_brz_factor(base, 0, 0, config->uv_brz_w, 0);
- mtk_jpeg_dec_set_dec_mode(base, 0);
- mtk_jpeg_dec_set_comp0_du(base, config->unit_num);
- mtk_jpeg_dec_set_total_mcu(base, config->total_mcu);
- mtk_jpeg_dec_set_bs_info(base, bs->str_addr, bs->size);
- mtk_jpeg_dec_set_bs_write_ptr(base, bs->end_addr);
- mtk_jpeg_dec_set_du_membership(base, config->membership, 1,
- (config->comp_num == 1) ? 1 : 0);
- mtk_jpeg_dec_set_comp_id(base, config->comp_id[0], config->comp_id[1],
- config->comp_id[2]);
- mtk_jpeg_dec_set_q_table(base, config->qtbl_num[0],
- config->qtbl_num[1], config->qtbl_num[2]);
- mtk_jpeg_dec_set_sampling_factor(base, config->comp_num,
- config->sampling_w[0],
- config->sampling_h[0],
- config->sampling_w[1],
- config->sampling_h[1],
- config->sampling_w[2],
- config->sampling_h[2]);
- mtk_jpeg_dec_set_mem_stride(base, config->mem_stride[0],
- config->mem_stride[1]);
- mtk_jpeg_dec_set_img_stride(base, config->img_stride[0],
- config->img_stride[1]);
- mtk_jpeg_dec_set_dst_bank0(base, fb->plane_addr[0],
- fb->plane_addr[1], fb->plane_addr[2]);
- mtk_jpeg_dec_set_dst_bank1(base, 0, 0, 0);
- mtk_jpeg_dec_set_dma_group(base, config->dma_mcu, config->dma_group,
- config->dma_last_mcu);
- mtk_jpeg_dec_set_pause_mcu_idx(base, config->total_mcu);
-}
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2016 MediaTek Inc.
- * Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
- * Rick Chang <rick.chang@mediatek.com>
- */
-
-#ifndef _MTK_JPEG_HW_H
-#define _MTK_JPEG_HW_H
-
-#include <media/videobuf2-core.h>
-
-#include "mtk_jpeg_core.h"
-#include "mtk_jpeg_reg.h"
-
-enum {
- MTK_JPEG_DEC_RESULT_EOF_DONE = 0,
- MTK_JPEG_DEC_RESULT_PAUSE = 1,
- MTK_JPEG_DEC_RESULT_UNDERFLOW = 2,
- MTK_JPEG_DEC_RESULT_OVERFLOW = 3,
- MTK_JPEG_DEC_RESULT_ERROR_BS = 4,
- MTK_JPEG_DEC_RESULT_ERROR_UNKNOWN = 6
-};
-
-struct mtk_jpeg_dec_param {
- u32 pic_w;
- u32 pic_h;
- u32 dec_w;
- u32 dec_h;
- u32 src_color;
- u32 dst_fourcc;
- u32 mcu_w;
- u32 mcu_h;
- u32 total_mcu;
- u32 unit_num;
- u32 comp_num;
- u32 comp_id[MTK_JPEG_COMP_MAX];
- u32 sampling_w[MTK_JPEG_COMP_MAX];
- u32 sampling_h[MTK_JPEG_COMP_MAX];
- u32 qtbl_num[MTK_JPEG_COMP_MAX];
- u32 blk_num;
- u32 blk_comp[MTK_JPEG_COMP_MAX];
- u32 membership;
- u32 dma_mcu;
- u32 dma_group;
- u32 dma_last_mcu;
- u32 img_stride[MTK_JPEG_COMP_MAX];
- u32 mem_stride[MTK_JPEG_COMP_MAX];
- u32 comp_w[MTK_JPEG_COMP_MAX];
- u32 comp_size[MTK_JPEG_COMP_MAX];
- u32 y_size;
- u32 uv_size;
- u32 dec_size;
- u8 uv_brz_w;
-};
-
-static inline u32 mtk_jpeg_align(u32 val, u32 align)
-{
- return (val + align - 1) & ~(align - 1);
-}
-
-struct mtk_jpeg_bs {
- dma_addr_t str_addr;
- dma_addr_t end_addr;
- size_t size;
-};
-
-struct mtk_jpeg_fb {
- dma_addr_t plane_addr[MTK_JPEG_COMP_MAX];
- size_t size;
-};
-
-int mtk_jpeg_dec_fill_param(struct mtk_jpeg_dec_param *param);
-u32 mtk_jpeg_dec_get_int_status(void __iomem *dec_reg_base);
-u32 mtk_jpeg_dec_enum_result(u32 irq_result);
-void mtk_jpeg_dec_set_config(void __iomem *base,
- struct mtk_jpeg_dec_param *config,
- struct mtk_jpeg_bs *bs,
- struct mtk_jpeg_fb *fb);
-void mtk_jpeg_dec_reset(void __iomem *dec_reg_base);
-void mtk_jpeg_dec_start(void __iomem *dec_reg_base);
-
-#endif /* _MTK_JPEG_HW_H */
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2016 MediaTek Inc.
- * Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
- * Rick Chang <rick.chang@mediatek.com>
- */
-
-#include <linux/kernel.h>
-#include <linux/videodev2.h>
-
-#include "mtk_jpeg_parse.h"
-
-#define TEM 0x01
-#define SOF0 0xc0
-#define RST 0xd0
-#define SOI 0xd8
-#define EOI 0xd9
-
-struct mtk_jpeg_stream {
- u8 *addr;
- u32 size;
- u32 curr;
-};
-
-static int read_byte(struct mtk_jpeg_stream *stream)
-{
- if (stream->curr >= stream->size)
- return -1;
- return stream->addr[stream->curr++];
-}
-
-static int read_word_be(struct mtk_jpeg_stream *stream, u32 *word)
-{
- u32 temp;
- int byte;
-
- byte = read_byte(stream);
- if (byte == -1)
- return -1;
- temp = byte << 8;
- byte = read_byte(stream);
- if (byte == -1)
- return -1;
- *word = (u32)byte | temp;
-
- return 0;
-}
-
-static void read_skip(struct mtk_jpeg_stream *stream, long len)
-{
- if (len <= 0)
- return;
- while (len--)
- read_byte(stream);
-}
-
-static bool mtk_jpeg_do_parse(struct mtk_jpeg_dec_param *param, u8 *src_addr_va,
- u32 src_size)
-{
- bool notfound = true;
- struct mtk_jpeg_stream stream;
-
- stream.addr = src_addr_va;
- stream.size = src_size;
- stream.curr = 0;
-
- while (notfound) {
- int i, length, byte;
- u32 word;
-
- byte = read_byte(&stream);
- if (byte == -1)
- return false;
- if (byte != 0xff)
- continue;
- do
- byte = read_byte(&stream);
- while (byte == 0xff);
- if (byte == -1)
- return false;
- if (byte == 0)
- continue;
-
- length = 0;
- switch (byte) {
- case SOF0:
- /* length */
- if (read_word_be(&stream, &word))
- break;
-
- /* precision */
- if (read_byte(&stream) == -1)
- break;
-
- if (read_word_be(&stream, &word))
- break;
- param->pic_h = word;
-
- if (read_word_be(&stream, &word))
- break;
- param->pic_w = word;
-
- param->comp_num = read_byte(&stream);
- if (param->comp_num != 1 && param->comp_num != 3)
- break;
-
- for (i = 0; i < param->comp_num; i++) {
- param->comp_id[i] = read_byte(&stream);
- if (param->comp_id[i] == -1)
- break;
-
- /* sampling */
- byte = read_byte(&stream);
- if (byte == -1)
- break;
- param->sampling_w[i] = (byte >> 4) & 0x0F;
- param->sampling_h[i] = byte & 0x0F;
-
- param->qtbl_num[i] = read_byte(&stream);
- if (param->qtbl_num[i] == -1)
- break;
- }
-
- notfound = !(i == param->comp_num);
- break;
- case RST ... RST + 7:
- case SOI:
- case EOI:
- case TEM:
- break;
- default:
- if (read_word_be(&stream, &word))
- break;
- length = (long)word - 2;
- read_skip(&stream, length);
- break;
- }
- }
-
- return !notfound;
-}
-
-bool mtk_jpeg_parse(struct mtk_jpeg_dec_param *param, u8 *src_addr_va,
- u32 src_size)
-{
- if (!mtk_jpeg_do_parse(param, src_addr_va, src_size))
- return false;
- if (mtk_jpeg_dec_fill_param(param))
- return false;
-
- return true;
-}
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2016 MediaTek Inc.
- * Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
- * Rick Chang <rick.chang@mediatek.com>
- */
-
-#ifndef _MTK_JPEG_PARSE_H
-#define _MTK_JPEG_PARSE_H
-
-#include "mtk_jpeg_hw.h"
-
-bool mtk_jpeg_parse(struct mtk_jpeg_dec_param *param, u8 *src_addr_va,
- u32 src_size);
-
-#endif /* _MTK_JPEG_PARSE_H */
-
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (c) 2016 MediaTek Inc.
- * Author: Ming Hsiu Tsai <minghsiu.tsai@mediatek.com>
- * Rick Chang <rick.chang@mediatek.com>
- */
-
-#ifndef _MTK_JPEG_REG_H
-#define _MTK_JPEG_REG_H
-
-#define MTK_JPEG_COMP_MAX 3
-#define MTK_JPEG_BLOCK_MAX 10
-#define MTK_JPEG_DCTSIZE 8
-
-#define BIT_INQST_MASK_ERROR_BS 0x20
-#define BIT_INQST_MASK_PAUSE 0x10
-#define BIT_INQST_MASK_OVERFLOW 0x04
-#define BIT_INQST_MASK_UNDERFLOW 0x02
-#define BIT_INQST_MASK_EOF 0x01
-#define BIT_INQST_MASK_ALLIRQ 0x37
-
-#define JPGDEC_REG_RESET 0x0090
-#define JPGDEC_REG_BRZ_FACTOR 0x00F8
-#define JPGDEC_REG_DU_NUM 0x00FC
-#define JPGDEC_REG_DEST_ADDR0_Y 0x0140
-#define JPGDEC_REG_DEST_ADDR0_U 0x0144
-#define JPGDEC_REG_DEST_ADDR0_V 0x0148
-#define JPGDEC_REG_DEST_ADDR1_Y 0x014C
-#define JPGDEC_REG_DEST_ADDR1_U 0x0150
-#define JPGDEC_REG_DEST_ADDR1_V 0x0154
-#define JPGDEC_REG_STRIDE_Y 0x0158
-#define JPGDEC_REG_STRIDE_UV 0x015C
-#define JPGDEC_REG_IMG_STRIDE_Y 0x0160
-#define JPGDEC_REG_IMG_STRIDE_UV 0x0164
-#define JPGDEC_REG_WDMA_CTRL 0x016C
-#define JPGDEC_REG_PAUSE_MCU_NUM 0x0170
-#define JPGDEC_REG_OPERATION_MODE 0x017C
-#define JPGDEC_REG_FILE_ADDR 0x0200
-#define JPGDEC_REG_COMP_ID 0x020C
-#define JPGDEC_REG_TOTAL_MCU_NUM 0x0210
-#define JPGDEC_REG_COMP0_DATA_UNIT_NUM 0x0224
-#define JPGDEC_REG_DU_CTRL 0x023C
-#define JPGDEC_REG_TRIG 0x0240
-#define JPGDEC_REG_FILE_BRP 0x0248
-#define JPGDEC_REG_FILE_TOTAL_SIZE 0x024C
-#define JPGDEC_REG_QT_ID 0x0270
-#define JPGDEC_REG_INTERRUPT_STATUS 0x0274
-#define JPGDEC_REG_STATUS 0x0278
-
-#endif /* _MTK_JPEG_REG_H */
void mtk_mdp_register_component(struct mtk_mdp_dev *mdp,
struct mtk_mdp_comp *comp)
{
- list_add(&mdp->comp_list, &comp->node);
+ list_add(&comp->node, &mdp->comp_list);
}
void mtk_mdp_unregister_component(struct mtk_mdp_dev *mdp,
mtk_vcodec_dec.o \
mtk_vcodec_dec_pm.o \
-
mtk-vcodec-enc-y := venc/venc_vp8_if.o \
venc/venc_h264_if.o \
mtk_vcodec_enc.o \
mtk-vcodec-common-y := mtk_vcodec_intr.o \
- mtk_vcodec_util.o\
-
-ccflags-y += -I$(srctree)/drivers/media/platform/mtk-vpu
+ mtk_vcodec_util.o \
+ mtk_vcodec_fw.o
vb->vb2_buf.index,
dstbuf->queued_in_vb2);
v4l2_m2m_buf_queue(ctx->m2m_ctx, vb);
- } else if ((dstbuf->queued_in_vb2 == false) &&
- (dstbuf->queued_in_v4l2 == true)) {
+ } else if (!dstbuf->queued_in_vb2 && dstbuf->queued_in_v4l2) {
/*
* If buffer in v4l2 driver but not in vb2 queue yet,
* and we get this buffer from free_list, it means
mutex_unlock(&ctx->lock);
}
v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_ERROR);
- } else if (res_chg == false) {
+ } else if (!res_chg) {
/*
* we only return src buffer with VB2_BUF_STATE_DONE
* when decode success without resolution change
buf = container_of(vb2_v4l2, struct mtk_video_dec_buf,
m2m_buf.vb);
mutex_lock(&ctx->lock);
- if (buf->used == false) {
+ if (!buf->used) {
v4l2_m2m_buf_queue(ctx->m2m_ctx, vb2_v4l2);
buf->queued_in_vb2 = true;
buf->queued_in_v4l2 = true;
dst_vq->dev = &ctx->dev->plat_dev->dev;
ret = vb2_queue_init(dst_vq);
- if (ret) {
- vb2_queue_release(src_vq);
+ if (ret)
mtk_v4l2_err("Failed to initialize videobuf2 queue(capture)");
- }
return ret;
}
#include "mtk_vcodec_dec_pm.h"
#include "mtk_vcodec_intr.h"
#include "mtk_vcodec_util.h"
-#include "mtk_vpu.h"
+#include "mtk_vcodec_fw.h"
#define VDEC_HW_ACTIVE 0x10
#define VDEC_IRQ_CFG 0x11
return IRQ_HANDLED;
}
-static void mtk_vcodec_dec_reset_handler(void *priv)
-{
- struct mtk_vcodec_dev *dev = priv;
- struct mtk_vcodec_ctx *ctx;
-
- mtk_v4l2_err("Watchdog timeout!!");
-
- mutex_lock(&dev->dev_mutex);
- list_for_each_entry(ctx, &dev->ctx_list, list) {
- ctx->state = MTK_STATE_ABORT;
- mtk_v4l2_debug(0, "[%d] Change to state MTK_STATE_ERROR",
- ctx->id);
- }
- mutex_unlock(&dev->dev_mutex);
-}
-
static int fops_vcodec_open(struct file *file)
{
struct mtk_vcodec_dev *dev = video_drvdata(file);
if (v4l2_fh_is_singular(&ctx->fh)) {
mtk_vcodec_dec_pw_on(&dev->pm);
/*
- * vpu_load_firmware checks if it was loaded already and
- * does nothing in that case
+ * Does nothing if firmware was already loaded.
*/
- ret = vpu_load_firmware(dev->vpu_plat_dev);
+ ret = mtk_vcodec_fw_load_firmware(dev->fw_handler);
if (ret < 0) {
/*
* Return 0 if downloading firmware successfully,
* otherwise it is failed
*/
- mtk_v4l2_err("vpu_load_firmware failed!");
+ mtk_v4l2_err("failed to load firmware!");
goto err_load_fw;
}
dev->dec_capability =
- vpu_get_vdec_hw_capa(dev->vpu_plat_dev);
+ mtk_vcodec_fw_get_vdec_capa(dev->fw_handler);
mtk_v4l2_debug(0, "decoder capability %x", dev->dec_capability);
}
struct mtk_vcodec_dev *dev;
struct video_device *vfd_dec;
struct resource *res;
+ phandle rproc_phandle;
+ enum mtk_vcodec_fw_type fw_type;
int i, ret;
dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
INIT_LIST_HEAD(&dev->ctx_list);
dev->plat_dev = pdev;
- dev->vpu_plat_dev = vpu_get_plat_device(dev->plat_dev);
- if (dev->vpu_plat_dev == NULL) {
- mtk_v4l2_err("[VPU] vpu device in not ready");
- return -EPROBE_DEFER;
+ if (!of_property_read_u32(pdev->dev.of_node, "mediatek,vpu",
+ &rproc_phandle)) {
+ fw_type = VPU;
+ } else if (!of_property_read_u32(pdev->dev.of_node, "mediatek,scp",
+ &rproc_phandle)) {
+ fw_type = SCP;
+ } else {
+ mtk_v4l2_err("Could not get vdec IPI device");
+ return -ENODEV;
+ }
+ if (!pdev->dev.dma_parms) {
+ pdev->dev.dma_parms = devm_kzalloc(&pdev->dev,
+ sizeof(*pdev->dev.dma_parms),
+ GFP_KERNEL);
+ if (!pdev->dev.dma_parms)
+ return -ENOMEM;
}
+ dma_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32));
- vpu_wdt_reg_handler(dev->vpu_plat_dev, mtk_vcodec_dec_reset_handler,
- dev, VPU_RST_DEC);
+ dev->fw_handler = mtk_vcodec_fw_select(dev, fw_type, VPU_RST_DEC);
+ if (IS_ERR(dev->fw_handler))
+ return PTR_ERR(dev->fw_handler);
ret = mtk_vcodec_init_dec_pm(dev);
if (ret < 0) {
dev_err(&pdev->dev, "Failed to get mt vcodec clock source");
- return ret;
+ goto err_dec_pm;
}
for (i = 0; i < NUM_MAX_VDEC_REG_BASE; i++) {
}
dev->dec_irq = platform_get_irq(pdev, 0);
+ irq_set_status_flags(dev->dec_irq, IRQ_NOAUTOEN);
ret = devm_request_irq(&pdev->dev, dev->dec_irq,
mtk_vcodec_dec_irq_handler, 0, pdev->name, dev);
if (ret) {
goto err_res;
}
- disable_irq(dev->dec_irq);
mutex_init(&dev->dec_mutex);
mutex_init(&dev->dev_mutex);
spin_lock_init(&dev->irqlock);
v4l2_device_unregister(&dev->v4l2_dev);
err_res:
mtk_vcodec_release_dec_pm(dev);
+err_dec_pm:
+ mtk_vcodec_fw_release(dev->fw_handler);
return ret;
}
v4l2_device_unregister(&dev->v4l2_dev);
mtk_vcodec_release_dec_pm(dev);
+ mtk_vcodec_fw_release(dev->fw_handler);
return 0;
}
#include "mtk_vcodec_dec_pm.h"
#include "mtk_vcodec_util.h"
-#include "mtk_vpu.h"
int mtk_vcodec_init_dec_pm(struct mtk_vcodec_dev *mtkdev)
{
};
+enum mtk_chip {
+ MTK_MT8173,
+ MTK_MT8183,
+};
+
+/**
+ * struct mtk_vcodec_enc_pdata - compatible data for each IC
+ *
+ * @chip: chip this encoder is compatible with
+ *
+ * @uses_ext: whether the encoder uses the extended firmware messaging format
+ * @has_lt_irq: whether the encoder uses the LT irq
+ * @min_birate: minimum supported encoding bitrate
+ * @max_bitrate: maximum supported encoding bitrate
+ * @capture_formats: array of supported capture formats
+ * @num_capture_formats: number of entries in capture_formats
+ * @output_formats: array of supported output formats
+ * @num_output_formats: number of entries in output_formats
+ */
+struct mtk_vcodec_enc_pdata {
+ enum mtk_chip chip;
+
+ bool uses_ext;
+ bool has_lt_irq;
+ unsigned long min_bitrate;
+ unsigned long max_bitrate;
+ const struct mtk_video_fmt *capture_formats;
+ size_t num_capture_formats;
+ const struct mtk_video_fmt *output_formats;
+ size_t num_output_formats;
+};
+
+#define MTK_ENC_CTX_IS_EXT(ctx) ((ctx)->dev->venc_pdata->uses_ext)
+
/**
* struct mtk_vcodec_dev - driver data
* @v4l2_dev: V4L2 device to register video devices for.
* @m2m_dev_dec: m2m device for decoder
* @m2m_dev_enc: m2m device for encoder.
* @plat_dev: platform device
- * @vpu_plat_dev: mtk vpu platform device
* @ctx_list: list of struct mtk_vcodec_ctx
* @irqlock: protect data access by irq handler and work thread
* @curr_ctx: The context that is waiting for codec hardware
*
* @reg_base: Mapped address of MTK Vcodec registers.
*
+ * @fw_handler: used to communicate with the firmware.
* @id_counter: used to identify current opened instance
*
* @encode_workqueue: encode work queue
struct v4l2_m2m_dev *m2m_dev_dec;
struct v4l2_m2m_dev *m2m_dev_enc;
struct platform_device *plat_dev;
- struct platform_device *vpu_plat_dev;
struct list_head ctx_list;
spinlock_t irqlock;
struct mtk_vcodec_ctx *curr_ctx;
void __iomem *reg_base[NUM_MAX_VCODEC_REG_BASE];
+ const struct mtk_vcodec_enc_pdata *venc_pdata;
+
+ struct mtk_vcodec_fw *fw_handler;
unsigned long id_counter;
#define DFT_CFG_WIDTH MTK_VENC_MIN_W
#define DFT_CFG_HEIGHT MTK_VENC_MIN_H
#define MTK_MAX_CTRLS_HINT 20
-#define OUT_FMT_IDX 0
-#define CAP_FMT_IDX 4
+#define MTK_DEFAULT_FRAMERATE_NUM 1001
+#define MTK_DEFAULT_FRAMERATE_DENOM 30000
static void mtk_venc_worker(struct work_struct *work);
-static const struct mtk_video_fmt mtk_video_formats[] = {
- {
- .fourcc = V4L2_PIX_FMT_NV12M,
- .type = MTK_FMT_FRAME,
- .num_planes = 2,
- },
- {
- .fourcc = V4L2_PIX_FMT_NV21M,
- .type = MTK_FMT_FRAME,
- .num_planes = 2,
- },
- {
- .fourcc = V4L2_PIX_FMT_YUV420M,
- .type = MTK_FMT_FRAME,
- .num_planes = 3,
- },
- {
- .fourcc = V4L2_PIX_FMT_YVU420M,
- .type = MTK_FMT_FRAME,
- .num_planes = 3,
- },
- {
- .fourcc = V4L2_PIX_FMT_H264,
- .type = MTK_FMT_ENC,
- .num_planes = 1,
- },
- {
- .fourcc = V4L2_PIX_FMT_VP8,
- .type = MTK_FMT_ENC,
- .num_planes = 1,
- },
-};
-
-#define NUM_FORMATS ARRAY_SIZE(mtk_video_formats)
-
-static const struct mtk_codec_framesizes mtk_venc_framesizes[] = {
- {
- .fourcc = V4L2_PIX_FMT_H264,
- .stepwise = { MTK_VENC_MIN_W, MTK_VENC_MAX_W, 16,
- MTK_VENC_MIN_H, MTK_VENC_MAX_H, 16 },
- },
- {
- .fourcc = V4L2_PIX_FMT_VP8,
- .stepwise = { MTK_VENC_MIN_W, MTK_VENC_MAX_W, 16,
- MTK_VENC_MIN_H, MTK_VENC_MAX_H, 16 },
- },
+static const struct v4l2_frmsize_stepwise mtk_venc_framesizes = {
+ MTK_VENC_MIN_W, MTK_VENC_MAX_W, 16,
+ MTK_VENC_MIN_H, MTK_VENC_MAX_H, 16,
};
#define NUM_SUPPORTED_FRAMESIZE ARRAY_SIZE(mtk_venc_framesizes)
.s_ctrl = vidioc_venc_s_ctrl,
};
-static int vidioc_enum_fmt(struct v4l2_fmtdesc *f, bool output_queue)
+static int vidioc_enum_fmt(struct v4l2_fmtdesc *f,
+ const struct mtk_video_fmt *formats,
+ size_t num_formats)
+{
+ if (f->index >= num_formats)
+ return -EINVAL;
+
+ f->pixelformat = formats[f->index].fourcc;
+ memset(f->reserved, 0, sizeof(f->reserved));
+
+ return 0;
+}
+
+static const struct mtk_video_fmt *
+mtk_venc_find_format(u32 fourcc, const struct mtk_vcodec_enc_pdata *pdata)
{
const struct mtk_video_fmt *fmt;
- int i, j = 0;
-
- for (i = 0; i < NUM_FORMATS; ++i) {
- if (output_queue && mtk_video_formats[i].type != MTK_FMT_FRAME)
- continue;
- if (!output_queue && mtk_video_formats[i].type != MTK_FMT_ENC)
- continue;
-
- if (j == f->index) {
- fmt = &mtk_video_formats[i];
- f->pixelformat = fmt->fourcc;
- memset(f->reserved, 0, sizeof(f->reserved));
- return 0;
- }
- ++j;
+ unsigned int k;
+
+ for (k = 0; k < pdata->num_capture_formats; k++) {
+ fmt = &pdata->capture_formats[k];
+ if (fmt->fourcc == fourcc)
+ return fmt;
+ }
+
+ for (k = 0; k < pdata->num_output_formats; k++) {
+ fmt = &pdata->output_formats[k];
+ if (fmt->fourcc == fourcc)
+ return fmt;
}
- return -EINVAL;
+ return NULL;
}
static int vidioc_enum_framesizes(struct file *file, void *fh,
struct v4l2_frmsizeenum *fsize)
{
- int i = 0;
+ const struct mtk_video_fmt *fmt;
if (fsize->index != 0)
return -EINVAL;
- for (i = 0; i < NUM_SUPPORTED_FRAMESIZE; ++i) {
- if (fsize->pixel_format != mtk_venc_framesizes[i].fourcc)
- continue;
+ fmt = mtk_venc_find_format(fsize->pixel_format,
+ fh_to_ctx(fh)->dev->venc_pdata);
+ if (!fmt)
+ return -EINVAL;
- fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
- fsize->stepwise = mtk_venc_framesizes[i].stepwise;
- return 0;
- }
+ fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
+ fsize->stepwise = mtk_venc_framesizes;
- return -EINVAL;
+ return 0;
}
static int vidioc_enum_fmt_vid_cap(struct file *file, void *priv,
struct v4l2_fmtdesc *f)
{
- return vidioc_enum_fmt(f, false);
+ const struct mtk_vcodec_enc_pdata *pdata =
+ fh_to_ctx(priv)->dev->venc_pdata;
+
+ return vidioc_enum_fmt(f, pdata->capture_formats,
+ pdata->num_capture_formats);
}
static int vidioc_enum_fmt_vid_out(struct file *file, void *priv,
struct v4l2_fmtdesc *f)
{
- return vidioc_enum_fmt(f, true);
+ const struct mtk_vcodec_enc_pdata *pdata =
+ fh_to_ctx(priv)->dev->venc_pdata;
+
+ return vidioc_enum_fmt(f, pdata->output_formats,
+ pdata->num_output_formats);
}
static int vidioc_venc_querycap(struct file *file, void *priv,
struct v4l2_streamparm *a)
{
struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv);
+ struct v4l2_fract *timeperframe = &a->parm.output.timeperframe;
if (a->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)
return -EINVAL;
- ctx->enc_params.framerate_num =
- a->parm.output.timeperframe.denominator;
- ctx->enc_params.framerate_denom =
- a->parm.output.timeperframe.numerator;
+ if (timeperframe->numerator == 0 || timeperframe->denominator == 0) {
+ timeperframe->numerator = MTK_DEFAULT_FRAMERATE_NUM;
+ timeperframe->denominator = MTK_DEFAULT_FRAMERATE_DENOM;
+ }
+
+ ctx->enc_params.framerate_num = timeperframe->denominator;
+ ctx->enc_params.framerate_denom = timeperframe->numerator;
ctx->param_change |= MTK_ENCODE_PARAM_FRAMERATE;
a->parm.output.capability = V4L2_CAP_TIMEPERFRAME;
return &ctx->q_data[MTK_Q_DATA_DST];
}
-static const struct mtk_video_fmt *mtk_venc_find_format(struct v4l2_format *f)
-{
- const struct mtk_video_fmt *fmt;
- unsigned int k;
-
- for (k = 0; k < NUM_FORMATS; k++) {
- fmt = &mtk_video_formats[k];
- if (fmt->fourcc == f->fmt.pix.pixelformat)
- return fmt;
- }
-
- return NULL;
-}
-
/* V4L2 specification suggests the driver corrects the format struct if any of
* the dimensions is unsupported
*/
pix_fmt_mp->num_planes = fmt->num_planes;
pix_fmt_mp->plane_fmt[0].sizeimage =
- pix_fmt_mp->width * pix_fmt_mp->height;
+ pix_fmt_mp->width * pix_fmt_mp->height +
+ ((ALIGN(pix_fmt_mp->width, 16) * 2) * 16);
pix_fmt_mp->plane_fmt[0].bytesperline = pix_fmt_mp->width;
if (pix_fmt_mp->num_planes == 2) {
pix_fmt_mp->plane_fmt[1].sizeimage =
- (pix_fmt_mp->width * pix_fmt_mp->height) / 2;
+ (pix_fmt_mp->width * pix_fmt_mp->height) / 2 +
+ (ALIGN(pix_fmt_mp->width, 16) * 16);
pix_fmt_mp->plane_fmt[2].sizeimage = 0;
pix_fmt_mp->plane_fmt[1].bytesperline =
pix_fmt_mp->width;
} else if (pix_fmt_mp->num_planes == 3) {
pix_fmt_mp->plane_fmt[1].sizeimage =
pix_fmt_mp->plane_fmt[2].sizeimage =
- (pix_fmt_mp->width * pix_fmt_mp->height) / 4;
+ (pix_fmt_mp->width * pix_fmt_mp->height) / 4 +
+ ((ALIGN(pix_fmt_mp->width, 16) / 2) * 16);
pix_fmt_mp->plane_fmt[1].bytesperline =
pix_fmt_mp->plane_fmt[2].bytesperline =
pix_fmt_mp->width / 2;
struct v4l2_format *f)
{
struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv);
+ const struct mtk_vcodec_enc_pdata *pdata = ctx->dev->venc_pdata;
struct vb2_queue *vq;
struct mtk_q_data *q_data;
int i, ret;
return -EINVAL;
}
- fmt = mtk_venc_find_format(f);
+ fmt = mtk_venc_find_format(f->fmt.pix.pixelformat, pdata);
if (!fmt) {
- f->fmt.pix.pixelformat = mtk_video_formats[CAP_FMT_IDX].fourcc;
- fmt = mtk_venc_find_format(f);
+ fmt = &ctx->dev->venc_pdata->capture_formats[0];
+ f->fmt.pix.pixelformat = fmt->fourcc;
}
q_data->fmt = fmt;
struct v4l2_format *f)
{
struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv);
+ const struct mtk_vcodec_enc_pdata *pdata = ctx->dev->venc_pdata;
struct vb2_queue *vq;
struct mtk_q_data *q_data;
int ret, i;
return -EINVAL;
}
- fmt = mtk_venc_find_format(f);
+ fmt = mtk_venc_find_format(f->fmt.pix.pixelformat, pdata);
if (!fmt) {
- f->fmt.pix.pixelformat = mtk_video_formats[OUT_FMT_IDX].fourcc;
- fmt = mtk_venc_find_format(f);
+ fmt = &ctx->dev->venc_pdata->output_formats[0];
+ f->fmt.pix.pixelformat = fmt->fourcc;
}
pix_fmt_mp->height = clamp(pix_fmt_mp->height,
{
const struct mtk_video_fmt *fmt;
struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv);
+ const struct mtk_vcodec_enc_pdata *pdata = ctx->dev->venc_pdata;
- fmt = mtk_venc_find_format(f);
+ fmt = mtk_venc_find_format(f->fmt.pix.pixelformat, pdata);
if (!fmt) {
- f->fmt.pix.pixelformat = mtk_video_formats[CAP_FMT_IDX].fourcc;
- fmt = mtk_venc_find_format(f);
+ fmt = &ctx->dev->venc_pdata->capture_formats[0];
+ f->fmt.pix.pixelformat = fmt->fourcc;
}
f->fmt.pix_mp.colorspace = ctx->colorspace;
f->fmt.pix_mp.ycbcr_enc = ctx->ycbcr_enc;
struct v4l2_format *f)
{
const struct mtk_video_fmt *fmt;
+ struct mtk_vcodec_ctx *ctx = fh_to_ctx(priv);
+ const struct mtk_vcodec_enc_pdata *pdata = ctx->dev->venc_pdata;
- fmt = mtk_venc_find_format(f);
+ fmt = mtk_venc_find_format(f->fmt.pix.pixelformat, pdata);
if (!fmt) {
- f->fmt.pix.pixelformat = mtk_video_formats[OUT_FMT_IDX].fourcc;
- fmt = mtk_venc_find_format(f);
+ fmt = &ctx->dev->venc_pdata->output_formats[0];
+ f->fmt.pix.pixelformat = fmt->fourcc;
}
if (!f->fmt.pix_mp.colorspace) {
f->fmt.pix_mp.colorspace = V4L2_COLORSPACE_REC709;
ctx->state = MTK_STATE_FREE;
}
+static int vb2ops_venc_buf_out_validate(struct vb2_buffer *vb)
+{
+ struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+
+ vbuf->field = V4L2_FIELD_NONE;
+ return 0;
+}
+
static const struct vb2_ops mtk_venc_vb2_ops = {
.queue_setup = vb2ops_venc_queue_setup,
+ .buf_out_validate = vb2ops_venc_buf_out_validate,
.buf_prepare = vb2ops_venc_buf_prepare,
.buf_queue = vb2ops_venc_buf_queue,
.wait_prepare = vb2_ops_wait_prepare,
q_data->coded_height = DFT_CFG_HEIGHT;
q_data->field = V4L2_FIELD_NONE;
- q_data->fmt = &mtk_video_formats[OUT_FMT_IDX];
+ q_data->fmt = &ctx->dev->venc_pdata->output_formats[0];
v4l_bound_align_image(&q_data->coded_width,
MTK_VENC_MIN_W,
memset(q_data, 0, sizeof(struct mtk_q_data));
q_data->coded_width = DFT_CFG_WIDTH;
q_data->coded_height = DFT_CFG_HEIGHT;
- q_data->fmt = &mtk_video_formats[CAP_FMT_IDX];
+ q_data->fmt = &ctx->dev->venc_pdata->capture_formats[0];
q_data->field = V4L2_FIELD_NONE;
ctx->q_data[MTK_Q_DATA_DST].sizeimage[0] =
DFT_CFG_WIDTH * DFT_CFG_HEIGHT;
ctx->q_data[MTK_Q_DATA_DST].bytesperline[0] = 0;
+ ctx->enc_params.framerate_num = MTK_DEFAULT_FRAMERATE_NUM;
+ ctx->enc_params.framerate_denom = MTK_DEFAULT_FRAMERATE_DENOM;
}
int mtk_vcodec_enc_ctrls_setup(struct mtk_vcodec_ctx *ctx)
v4l2_ctrl_handler_init(handler, MTK_MAX_CTRLS_HINT);
+ v4l2_ctrl_new_std(handler, ops, V4L2_CID_MIN_BUFFERS_FOR_OUTPUT,
+ 1, 1, 1, 1);
v4l2_ctrl_new_std(handler, ops, V4L2_CID_MPEG_VIDEO_BITRATE,
- 1, 4000000, 1, 4000000);
+ ctx->dev->venc_pdata->min_bitrate,
+ ctx->dev->venc_pdata->max_bitrate, 1, 4000000);
v4l2_ctrl_new_std(handler, ops, V4L2_CID_MPEG_VIDEO_B_FRAMES,
0, 2, 1, 0);
v4l2_ctrl_new_std(handler, ops, V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE,
#include "mtk_vcodec_enc_pm.h"
#include "mtk_vcodec_intr.h"
#include "mtk_vcodec_util.h"
-#include "mtk_vpu.h"
+#include "mtk_vcodec_fw.h"
module_param(mtk_v4l2_dbg_level, int, S_IRUGO | S_IWUSR);
module_param(mtk_vcodec_dbg, bool, S_IRUGO | S_IWUSR);
+static const struct mtk_video_fmt mtk_video_formats_output_mt8173[] = {
+ {
+ .fourcc = V4L2_PIX_FMT_NV12M,
+ .type = MTK_FMT_FRAME,
+ .num_planes = 2,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_NV21M,
+ .type = MTK_FMT_FRAME,
+ .num_planes = 2,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_YUV420M,
+ .type = MTK_FMT_FRAME,
+ .num_planes = 3,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_YVU420M,
+ .type = MTK_FMT_FRAME,
+ .num_planes = 3,
+ },
+};
+
+static const struct mtk_video_fmt mtk_video_formats_capture_mt8173[] = {
+ {
+ .fourcc = V4L2_PIX_FMT_H264,
+ .type = MTK_FMT_ENC,
+ .num_planes = 1,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_VP8,
+ .type = MTK_FMT_ENC,
+ .num_planes = 1,
+ },
+};
+
+static const struct mtk_video_fmt mtk_video_formats_capture_mt8183[] = {
+ {
+ .fourcc = V4L2_PIX_FMT_H264,
+ .type = MTK_FMT_ENC,
+ .num_planes = 1,
+ },
+};
+
/* Wake up context wait_queue */
static void wake_up_ctx(struct mtk_vcodec_ctx *ctx, unsigned int reason)
{
return IRQ_HANDLED;
}
-static void mtk_vcodec_enc_reset_handler(void *priv)
-{
- struct mtk_vcodec_dev *dev = priv;
- struct mtk_vcodec_ctx *ctx;
-
- mtk_v4l2_debug(0, "Watchdog timeout!!");
-
- mutex_lock(&dev->dev_mutex);
- list_for_each_entry(ctx, &dev->ctx_list, list) {
- ctx->state = MTK_STATE_ABORT;
- mtk_v4l2_debug(0, "[%d] Change to state MTK_STATE_ABORT",
- ctx->id);
- }
- mutex_unlock(&dev->dev_mutex);
-}
-
static int fops_vcodec_open(struct file *file)
{
struct mtk_vcodec_dev *dev = video_drvdata(file);
if (v4l2_fh_is_singular(&ctx->fh)) {
/*
- * vpu_load_firmware checks if it was loaded already and
+ * load fireware to checks if it was loaded already and
* does nothing in that case
*/
- ret = vpu_load_firmware(dev->vpu_plat_dev);
+ ret = mtk_vcodec_fw_load_firmware(dev->fw_handler);
if (ret < 0) {
/*
* Return 0 if downloading firmware successfully,
}
dev->enc_capability =
- vpu_get_venc_hw_capa(dev->vpu_plat_dev);
+ mtk_vcodec_fw_get_venc_capa(dev->fw_handler);
mtk_v4l2_debug(0, "encoder capability %x", dev->enc_capability);
}
struct mtk_vcodec_dev *dev;
struct video_device *vfd_enc;
struct resource *res;
- int i, j, ret;
+ phandle rproc_phandle;
+ enum mtk_vcodec_fw_type fw_type;
+ int ret;
dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
if (!dev)
INIT_LIST_HEAD(&dev->ctx_list);
dev->plat_dev = pdev;
- dev->vpu_plat_dev = vpu_get_plat_device(dev->plat_dev);
- if (dev->vpu_plat_dev == NULL) {
- mtk_v4l2_err("[VPU] vpu device in not ready");
- return -EPROBE_DEFER;
+ if (!of_property_read_u32(pdev->dev.of_node, "mediatek,vpu",
+ &rproc_phandle)) {
+ fw_type = VPU;
+ } else if (!of_property_read_u32(pdev->dev.of_node, "mediatek,scp",
+ &rproc_phandle)) {
+ fw_type = SCP;
+ } else {
+ mtk_v4l2_err("Could not get venc IPI device");
+ return -ENODEV;
}
+ if (!pdev->dev.dma_parms) {
+ pdev->dev.dma_parms = devm_kzalloc(&pdev->dev,
+ sizeof(*pdev->dev.dma_parms),
+ GFP_KERNEL);
+ if (!pdev->dev.dma_parms)
+ return -ENOMEM;
+ }
+ dma_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32));
- vpu_wdt_reg_handler(dev->vpu_plat_dev, mtk_vcodec_enc_reset_handler,
- dev, VPU_RST_ENC);
+ dev->fw_handler = mtk_vcodec_fw_select(dev, fw_type, VPU_RST_ENC);
+ if (IS_ERR(dev->fw_handler))
+ return PTR_ERR(dev->fw_handler);
+ dev->venc_pdata = of_device_get_match_data(&pdev->dev);
ret = mtk_vcodec_init_enc_pm(dev);
if (ret < 0) {
dev_err(&pdev->dev, "Failed to get mt vcodec clock source!");
- return ret;
+ goto err_enc_pm;
}
- for (i = VENC_SYS, j = 0; i < NUM_MAX_VCODEC_REG_BASE; i++, j++) {
- res = platform_get_resource(pdev, IORESOURCE_MEM, j);
- dev->reg_base[i] = devm_ioremap_resource(&pdev->dev, res);
- if (IS_ERR((__force void *)dev->reg_base[i])) {
- ret = PTR_ERR((__force void *)dev->reg_base[i]);
- goto err_res;
- }
- mtk_v4l2_debug(2, "reg[%d] base=0x%p", i, dev->reg_base[i]);
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ dev->reg_base[VENC_SYS] = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR((__force void *)dev->reg_base[VENC_SYS])) {
+ ret = PTR_ERR((__force void *)dev->reg_base[VENC_SYS]);
+ goto err_res;
}
+ mtk_v4l2_debug(2, "reg[%d] base=0x%p", i, dev->reg_base[VENC_SYS]);
res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
if (res == NULL) {
}
dev->enc_irq = platform_get_irq(pdev, 0);
+ irq_set_status_flags(dev->enc_irq, IRQ_NOAUTOEN);
ret = devm_request_irq(&pdev->dev, dev->enc_irq,
mtk_vcodec_enc_irq_handler,
0, pdev->name, dev);
goto err_res;
}
- dev->enc_lt_irq = platform_get_irq(pdev, 1);
- ret = devm_request_irq(&pdev->dev,
- dev->enc_lt_irq, mtk_vcodec_enc_lt_irq_handler,
- 0, pdev->name, dev);
- if (ret) {
- dev_err(&pdev->dev,
- "Failed to install dev->enc_lt_irq %d (%d)",
- dev->enc_lt_irq, ret);
- ret = -EINVAL;
- goto err_res;
+ if (dev->venc_pdata->has_lt_irq) {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ dev->reg_base[VENC_LT_SYS] = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR((__force void *)dev->reg_base[VENC_LT_SYS])) {
+ ret = PTR_ERR((__force void *)dev->reg_base[VENC_LT_SYS]);
+ goto err_res;
+ }
+ mtk_v4l2_debug(2, "reg[%d] base=0x%p", i, dev->reg_base[VENC_LT_SYS]);
+
+ dev->enc_lt_irq = platform_get_irq(pdev, 1);
+ irq_set_status_flags(dev->enc_lt_irq, IRQ_NOAUTOEN);
+ ret = devm_request_irq(&pdev->dev,
+ dev->enc_lt_irq,
+ mtk_vcodec_enc_lt_irq_handler,
+ 0, pdev->name, dev);
+ if (ret) {
+ dev_err(&pdev->dev,
+ "Failed to install dev->enc_lt_irq %d (%d)",
+ dev->enc_lt_irq, ret);
+ ret = -EINVAL;
+ goto err_res;
+ }
}
- disable_irq(dev->enc_irq);
- disable_irq(dev->enc_lt_irq); /* VENC_LT */
mutex_init(&dev->enc_mutex);
mutex_init(&dev->dev_mutex);
spin_lock_init(&dev->irqlock);
v4l2_device_unregister(&dev->v4l2_dev);
err_res:
mtk_vcodec_release_enc_pm(dev);
+err_enc_pm:
+ mtk_vcodec_fw_release(dev->fw_handler);
return ret;
}
+static const struct mtk_vcodec_enc_pdata mt8173_pdata = {
+ .chip = MTK_MT8173,
+ .has_lt_irq = true,
+ .capture_formats = mtk_video_formats_capture_mt8173,
+ .num_capture_formats = ARRAY_SIZE(mtk_video_formats_capture_mt8173),
+ .output_formats = mtk_video_formats_output_mt8173,
+ .num_output_formats = ARRAY_SIZE(mtk_video_formats_output_mt8173),
+ .min_bitrate = 1,
+ .max_bitrate = 4000000,
+};
+
+static const struct mtk_vcodec_enc_pdata mt8183_pdata = {
+ .chip = MTK_MT8183,
+ .has_lt_irq = false,
+ .uses_ext = true,
+ .capture_formats = mtk_video_formats_capture_mt8183,
+ .num_capture_formats = ARRAY_SIZE(mtk_video_formats_capture_mt8183),
+ /* MT8183 supports the same output formats as MT8173 */
+ .output_formats = mtk_video_formats_output_mt8173,
+ .num_output_formats = ARRAY_SIZE(mtk_video_formats_output_mt8173),
+ .min_bitrate = 64,
+ .max_bitrate = 40000000,
+};
+
static const struct of_device_id mtk_vcodec_enc_match[] = {
- {.compatible = "mediatek,mt8173-vcodec-enc",},
+ {.compatible = "mediatek,mt8173-vcodec-enc", .data = &mt8173_pdata},
+ {.compatible = "mediatek,mt8183-vcodec-enc", .data = &mt8183_pdata},
{},
};
MODULE_DEVICE_TABLE(of, mtk_vcodec_enc_match);
v4l2_device_unregister(&dev->v4l2_dev);
mtk_vcodec_release_enc_pm(dev);
+ mtk_vcodec_fw_release(dev->fw_handler);
return 0;
}
#include "mtk_vcodec_enc_pm.h"
#include "mtk_vcodec_util.h"
-#include "mtk_vpu.h"
-
int mtk_vcodec_init_enc_pm(struct mtk_vcodec_dev *mtkdev)
{
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+
+#include "mtk_vcodec_fw.h"
+#include "mtk_vcodec_util.h"
+#include "mtk_vcodec_drv.h"
+
+struct mtk_vcodec_fw_ops {
+ int (*load_firmware)(struct mtk_vcodec_fw *fw);
+ unsigned int (*get_vdec_capa)(struct mtk_vcodec_fw *fw);
+ unsigned int (*get_venc_capa)(struct mtk_vcodec_fw *fw);
+ void * (*map_dm_addr)(struct mtk_vcodec_fw *fw, u32 dtcm_dmem_addr);
+ int (*ipi_register)(struct mtk_vcodec_fw *fw, int id,
+ mtk_vcodec_ipi_handler handler, const char *name, void *priv);
+ int (*ipi_send)(struct mtk_vcodec_fw *fw, int id, void *buf,
+ unsigned int len, unsigned int wait);
+};
+
+struct mtk_vcodec_fw {
+ enum mtk_vcodec_fw_type type;
+ const struct mtk_vcodec_fw_ops *ops;
+ struct platform_device *pdev;
+ struct mtk_scp *scp;
+};
+
+static int mtk_vcodec_vpu_load_firmware(struct mtk_vcodec_fw *fw)
+{
+ return vpu_load_firmware(fw->pdev);
+}
+
+static unsigned int mtk_vcodec_vpu_get_vdec_capa(struct mtk_vcodec_fw *fw)
+{
+ return vpu_get_vdec_hw_capa(fw->pdev);
+}
+
+static unsigned int mtk_vcodec_vpu_get_venc_capa(struct mtk_vcodec_fw *fw)
+{
+ return vpu_get_venc_hw_capa(fw->pdev);
+}
+
+static void *mtk_vcodec_vpu_map_dm_addr(struct mtk_vcodec_fw *fw,
+ u32 dtcm_dmem_addr)
+{
+ return vpu_mapping_dm_addr(fw->pdev, dtcm_dmem_addr);
+}
+
+static int mtk_vcodec_vpu_set_ipi_register(struct mtk_vcodec_fw *fw, int id,
+ mtk_vcodec_ipi_handler handler,
+ const char *name, void *priv)
+{
+ /*
+ * The handler we receive takes a void * as its first argument. We
+ * cannot change this because it needs to be passed down to the rproc
+ * subsystem when SCP is used. VPU takes a const argument, which is
+ * more constrained, so the conversion below is safe.
+ */
+ ipi_handler_t handler_const = (ipi_handler_t)handler;
+
+ return vpu_ipi_register(fw->pdev, id, handler_const, name, priv);
+}
+
+static int mtk_vcodec_vpu_ipi_send(struct mtk_vcodec_fw *fw, int id, void *buf,
+ unsigned int len, unsigned int wait)
+{
+ return vpu_ipi_send(fw->pdev, id, buf, len);
+}
+
+static const struct mtk_vcodec_fw_ops mtk_vcodec_vpu_msg = {
+ .load_firmware = mtk_vcodec_vpu_load_firmware,
+ .get_vdec_capa = mtk_vcodec_vpu_get_vdec_capa,
+ .get_venc_capa = mtk_vcodec_vpu_get_venc_capa,
+ .map_dm_addr = mtk_vcodec_vpu_map_dm_addr,
+ .ipi_register = mtk_vcodec_vpu_set_ipi_register,
+ .ipi_send = mtk_vcodec_vpu_ipi_send,
+};
+
+static int mtk_vcodec_scp_load_firmware(struct mtk_vcodec_fw *fw)
+{
+ return rproc_boot(scp_get_rproc(fw->scp));
+}
+
+static unsigned int mtk_vcodec_scp_get_vdec_capa(struct mtk_vcodec_fw *fw)
+{
+ return scp_get_vdec_hw_capa(fw->scp);
+}
+
+static unsigned int mtk_vcodec_scp_get_venc_capa(struct mtk_vcodec_fw *fw)
+{
+ return scp_get_venc_hw_capa(fw->scp);
+}
+
+static void *mtk_vcodec_vpu_scp_dm_addr(struct mtk_vcodec_fw *fw,
+ u32 dtcm_dmem_addr)
+{
+ return scp_mapping_dm_addr(fw->scp, dtcm_dmem_addr);
+}
+
+static int mtk_vcodec_scp_set_ipi_register(struct mtk_vcodec_fw *fw, int id,
+ mtk_vcodec_ipi_handler handler,
+ const char *name, void *priv)
+{
+ return scp_ipi_register(fw->scp, id, handler, priv);
+}
+
+static int mtk_vcodec_scp_ipi_send(struct mtk_vcodec_fw *fw, int id, void *buf,
+ unsigned int len, unsigned int wait)
+{
+ return scp_ipi_send(fw->scp, id, buf, len, wait);
+}
+
+static const struct mtk_vcodec_fw_ops mtk_vcodec_rproc_msg = {
+ .load_firmware = mtk_vcodec_scp_load_firmware,
+ .get_vdec_capa = mtk_vcodec_scp_get_vdec_capa,
+ .get_venc_capa = mtk_vcodec_scp_get_venc_capa,
+ .map_dm_addr = mtk_vcodec_vpu_scp_dm_addr,
+ .ipi_register = mtk_vcodec_scp_set_ipi_register,
+ .ipi_send = mtk_vcodec_scp_ipi_send,
+};
+
+static void mtk_vcodec_reset_handler(void *priv)
+{
+ struct mtk_vcodec_dev *dev = priv;
+ struct mtk_vcodec_ctx *ctx;
+
+ mtk_v4l2_err("Watchdog timeout!!");
+
+ mutex_lock(&dev->dev_mutex);
+ list_for_each_entry(ctx, &dev->ctx_list, list) {
+ ctx->state = MTK_STATE_ABORT;
+ mtk_v4l2_debug(0, "[%d] Change to state MTK_STATE_ABORT",
+ ctx->id);
+ }
+ mutex_unlock(&dev->dev_mutex);
+}
+
+struct mtk_vcodec_fw *mtk_vcodec_fw_select(struct mtk_vcodec_dev *dev,
+ enum mtk_vcodec_fw_type type,
+ enum rst_id rst_id)
+{
+ const struct mtk_vcodec_fw_ops *ops;
+ struct mtk_vcodec_fw *fw;
+ struct platform_device *fw_pdev = NULL;
+ struct mtk_scp *scp = NULL;
+
+ switch (type) {
+ case VPU:
+ ops = &mtk_vcodec_vpu_msg;
+ fw_pdev = vpu_get_plat_device(dev->plat_dev);
+ if (!fw_pdev) {
+ mtk_v4l2_err("firmware device is not ready");
+ return ERR_PTR(-EINVAL);
+ }
+ vpu_wdt_reg_handler(fw_pdev, mtk_vcodec_reset_handler,
+ dev, rst_id);
+ break;
+ case SCP:
+ ops = &mtk_vcodec_rproc_msg;
+ scp = scp_get(dev->plat_dev);
+ if (!scp) {
+ mtk_v4l2_err("could not get vdec scp handle");
+ return ERR_PTR(-EPROBE_DEFER);
+ }
+ break;
+ default:
+ mtk_v4l2_err("invalid vcodec fw type");
+ return ERR_PTR(-EINVAL);
+ }
+
+ fw = devm_kzalloc(&dev->plat_dev->dev, sizeof(*fw), GFP_KERNEL);
+ if (!fw)
+ return ERR_PTR(-EINVAL);
+
+ fw->type = type;
+ fw->ops = ops;
+ fw->pdev = fw_pdev;
+ fw->scp = scp;
+
+ return fw;
+}
+EXPORT_SYMBOL_GPL(mtk_vcodec_fw_select);
+
+void mtk_vcodec_fw_release(struct mtk_vcodec_fw *fw)
+{
+ switch (fw->type) {
+ case VPU:
+ put_device(&fw->pdev->dev);
+ break;
+ case SCP:
+ scp_put(fw->scp);
+ break;
+ }
+}
+EXPORT_SYMBOL_GPL(mtk_vcodec_fw_release);
+
+int mtk_vcodec_fw_load_firmware(struct mtk_vcodec_fw *fw)
+{
+ return fw->ops->load_firmware(fw);
+}
+EXPORT_SYMBOL_GPL(mtk_vcodec_fw_load_firmware);
+
+unsigned int mtk_vcodec_fw_get_vdec_capa(struct mtk_vcodec_fw *fw)
+{
+ return fw->ops->get_vdec_capa(fw);
+}
+EXPORT_SYMBOL_GPL(mtk_vcodec_fw_get_vdec_capa);
+
+unsigned int mtk_vcodec_fw_get_venc_capa(struct mtk_vcodec_fw *fw)
+{
+ return fw->ops->get_venc_capa(fw);
+}
+EXPORT_SYMBOL_GPL(mtk_vcodec_fw_get_venc_capa);
+
+void *mtk_vcodec_fw_map_dm_addr(struct mtk_vcodec_fw *fw, u32 mem_addr)
+{
+ return fw->ops->map_dm_addr(fw, mem_addr);
+}
+EXPORT_SYMBOL_GPL(mtk_vcodec_fw_map_dm_addr);
+
+int mtk_vcodec_fw_ipi_register(struct mtk_vcodec_fw *fw, int id,
+ mtk_vcodec_ipi_handler handler,
+ const char *name, void *priv)
+{
+ return fw->ops->ipi_register(fw, id, handler, name, priv);
+}
+EXPORT_SYMBOL_GPL(mtk_vcodec_fw_ipi_register);
+
+int mtk_vcodec_fw_ipi_send(struct mtk_vcodec_fw *fw, int id, void *buf,
+ unsigned int len, unsigned int wait)
+{
+ return fw->ops->ipi_send(fw, id, buf, len, wait);
+}
+EXPORT_SYMBOL_GPL(mtk_vcodec_fw_ipi_send);
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _MTK_VCODEC_FW_H_
+#define _MTK_VCODEC_FW_H_
+
+#include <linux/remoteproc.h>
+#include <linux/remoteproc/mtk_scp.h>
+
+#include "../mtk-vpu/mtk_vpu.h"
+
+struct mtk_vcodec_dev;
+
+enum mtk_vcodec_fw_type {
+ VPU,
+ SCP,
+};
+
+struct mtk_vcodec_fw;
+
+typedef void (*mtk_vcodec_ipi_handler) (void *data,
+ unsigned int len, void *priv);
+
+struct mtk_vcodec_fw *mtk_vcodec_fw_select(struct mtk_vcodec_dev *dev,
+ enum mtk_vcodec_fw_type type,
+ enum rst_id rst_id);
+void mtk_vcodec_fw_release(struct mtk_vcodec_fw *fw);
+
+int mtk_vcodec_fw_load_firmware(struct mtk_vcodec_fw *fw);
+unsigned int mtk_vcodec_fw_get_vdec_capa(struct mtk_vcodec_fw *fw);
+unsigned int mtk_vcodec_fw_get_venc_capa(struct mtk_vcodec_fw *fw);
+void *mtk_vcodec_fw_map_dm_addr(struct mtk_vcodec_fw *fw, u32 mem_addr);
+int mtk_vcodec_fw_ipi_register(struct mtk_vcodec_fw *fw, int id,
+ mtk_vcodec_ipi_handler handler,
+ const char *name, void *priv);
+int mtk_vcodec_fw_ipi_send(struct mtk_vcodec_fw *fw, int id,
+ void *buf, unsigned int len, unsigned int wait);
+
+#endif /* _MTK_VCODEC_FW_H_ */
#include "mtk_vcodec_drv.h"
#include "mtk_vcodec_util.h"
-#include "mtk_vpu.h"
/* For encoder, this will enable logs in venc/*/
bool mtk_vcodec_dbg;
inst->ctx = ctx;
inst->vpu.id = IPI_VDEC_H264;
- inst->vpu.dev = ctx->dev->vpu_plat_dev;
inst->vpu.ctx = ctx;
err = vpu_dec_init(&inst->vpu);
inst->ctx = ctx;
inst->vpu.id = IPI_VDEC_VP8;
- inst->vpu.dev = ctx->dev->vpu_plat_dev;
inst->vpu.ctx = ctx;
err = vpu_dec_init(&inst->vpu);
inst->ctx = ctx;
inst->vpu.id = IPI_VDEC_VP9;
- inst->vpu.dev = ctx->dev->vpu_plat_dev;
inst->vpu.ctx = ctx;
if (vpu_dec_init(&inst->vpu)) {
goto DECODE_ERROR;
}
- if (vp9_decode_end_proc(inst) != true) {
+ if (!vp9_decode_end_proc(inst)) {
mtk_vcodec_err(inst, "vp9_decode_end_proc");
ret = -EINVAL;
goto DECODE_ERROR;
#ifndef _VDEC_DRV_BASE_
#define _VDEC_DRV_BASE_
-#include "mtk_vcodec_drv.h"
-
#include "vdec_drv_if.h"
struct vdec_common_if {
#include "mtk_vcodec_dec.h"
#include "vdec_drv_base.h"
#include "mtk_vcodec_dec_pm.h"
-#include "mtk_vpu.h"
int vdec_if_init(struct mtk_vcodec_ctx *ctx, unsigned int fourcc)
{
#include "mtk_vcodec_util.h"
#include "vdec_ipi_msg.h"
#include "vdec_vpu_if.h"
+#include "mtk_vcodec_fw.h"
static void handle_init_ack_msg(const struct vdec_vpu_ipi_init_ack *msg)
{
/* mapping VPU address to kernel virtual address */
/* the content in vsi is initialized to 0 in VPU */
- vpu->vsi = vpu_mapping_dm_addr(vpu->dev, msg->vpu_inst_addr);
+ vpu->vsi = mtk_vcodec_fw_map_dm_addr(vpu->ctx->dev->fw_handler,
+ msg->vpu_inst_addr);
vpu->inst_addr = msg->vpu_inst_addr;
mtk_vcodec_debug(vpu, "- vpu_inst_addr = 0x%x", vpu->inst_addr);
* This function runs in interrupt context and it means there's an IPI MSG
* from VPU.
*/
-static void vpu_dec_ipi_handler(const void *data, unsigned int len, void *priv)
+static void vpu_dec_ipi_handler(void *data, unsigned int len, void *priv)
{
const struct vdec_vpu_ipi_ack *msg = data;
struct vdec_vpu_inst *vpu = (struct vdec_vpu_inst *)
vpu->failure = 0;
vpu->signaled = 0;
- err = vpu_ipi_send(vpu->dev, vpu->id, msg, len);
+ err = mtk_vcodec_fw_ipi_send(vpu->ctx->dev->fw_handler, vpu->id, msg,
+ len, 2000);
if (err) {
mtk_vcodec_err(vpu, "send fail vpu_id=%d msg_id=%X status=%d",
vpu->id, *(uint32_t *)msg, err);
init_waitqueue_head(&vpu->wq);
vpu->handler = vpu_dec_ipi_handler;
- err = vpu_ipi_register(vpu->dev, vpu->id, vpu->handler, "vdec", NULL);
+ err = mtk_vcodec_fw_ipi_register(vpu->ctx->dev->fw_handler, vpu->id,
+ vpu->handler, "vdec", NULL);
if (err != 0) {
mtk_vcodec_err(vpu, "vpu_ipi_register fail status=%d", err);
return err;
#ifndef _VDEC_VPU_IF_H_
#define _VDEC_VPU_IF_H_
-#include "mtk_vpu.h"
+#include "mtk_vcodec_fw.h"
+
+struct mtk_vcodec_ctx;
/**
* struct vdec_vpu_inst - VPU instance for video codec
- * @ipi_id : ipi id for each decoder
+ * @id : ipi msg id for each decoder
* @vsi : driver structure allocated by VPU side and shared to AP side
* for control and info share
* @failure : VPU execution result status, 0: success, others: fail
* @handler : ipi handler for each decoder
*/
struct vdec_vpu_inst {
- enum ipi_id id;
+ int id;
void *vsi;
int32_t failure;
uint32_t inst_addr;
unsigned int signaled;
struct mtk_vcodec_ctx *ctx;
- struct platform_device *dev;
wait_queue_head_t wq;
- ipi_handler_t handler;
+ mtk_vcodec_ipi_handler handler;
};
/**
#include "../venc_drv_base.h"
#include "../venc_ipi_msg.h"
#include "../venc_vpu_if.h"
-#include "mtk_vpu.h"
static const char h264_filler_marker[] = {0x0, 0x0, 0x0, 0x1, 0xc};
#define H264_FILLER_MARKER_SIZE ARRAY_SIZE(h264_filler_marker)
#define VENC_PIC_BITSTREAM_BYTE_CNT 0x0098
+/*
+ * enum venc_h264_frame_type - h264 encoder output bitstream frame type
+ */
+enum venc_h264_frame_type {
+ VENC_H264_IDR_FRM,
+ VENC_H264_I_FRM,
+ VENC_H264_P_FRM,
+ VENC_H264_B_FRM,
+};
+
/*
* enum venc_h264_vpu_work_buf - h264 encoder buffer index
*/
struct mtk_vcodec_mem pps_buf;
bool work_buf_allocated;
unsigned int frm_cnt;
+ unsigned int skip_frm_cnt;
unsigned int prepend_hdr;
struct venc_vpu_inst vpu_inst;
struct venc_h264_vsi *vsi;
*/
inst->work_bufs[i].size = wb[i].size;
if (i == VENC_H264_VPU_WORK_BUF_SKIP_FRAME) {
- inst->work_bufs[i].va = vpu_mapping_dm_addr(
- inst->vpu_inst.dev, wb[i].vpua);
+ struct mtk_vcodec_fw *handler;
+
+ handler = inst->vpu_inst.ctx->dev->fw_handler;
+ inst->work_bufs[i].va =
+ mtk_vcodec_fw_map_dm_addr(handler, wb[i].vpua);
inst->work_bufs[i].dma_addr = 0;
} else {
ret = mtk_vcodec_mem_alloc(inst->ctx,
* setting in VPU side.
*/
if (i == VENC_H264_VPU_WORK_BUF_RC_CODE) {
+ struct mtk_vcodec_fw *handler;
void *tmp_va;
- tmp_va = vpu_mapping_dm_addr(inst->vpu_inst.dev,
- wb[i].vpua);
+ handler = inst->vpu_inst.ctx->dev->fw_handler;
+ tmp_va = mtk_vcodec_fw_map_dm_addr(handler,
+ wb[i].vpua);
memcpy(inst->work_bufs[i].va, tmp_va,
wb[i].size);
}
return irq_status;
}
+static int h264_frame_type(struct venc_h264_inst *inst)
+{
+ if ((inst->vsi->config.gop_size != 0 &&
+ (inst->frm_cnt % inst->vsi->config.gop_size) == 0) ||
+ (inst->frm_cnt == 0 && inst->vsi->config.gop_size == 0)) {
+ /* IDR frame */
+ return VENC_H264_IDR_FRM;
+ } else if ((inst->vsi->config.intra_period != 0 &&
+ (inst->frm_cnt % inst->vsi->config.intra_period) == 0) ||
+ (inst->frm_cnt == 0 && inst->vsi->config.intra_period == 0)) {
+ /* I frame */
+ return VENC_H264_I_FRM;
+ } else {
+ return VENC_H264_P_FRM; /* Note: B frames are not supported */
+ }
+}
static int h264_encode_sps(struct venc_h264_inst *inst,
struct mtk_vcodec_mem *bs_buf,
unsigned int *bs_size)
mtk_vcodec_debug_enter(inst);
ret = vpu_enc_encode(&inst->vpu_inst, H264_BS_MODE_SPS, NULL,
- bs_buf, bs_size);
+ bs_buf, bs_size, NULL);
if (ret)
return ret;
mtk_vcodec_debug_enter(inst);
ret = vpu_enc_encode(&inst->vpu_inst, H264_BS_MODE_PPS, NULL,
- bs_buf, bs_size);
+ bs_buf, bs_size, NULL);
if (ret)
return ret;
{
int ret = 0;
unsigned int irq_status;
+ struct venc_frame_info frame_info;
mtk_vcodec_debug_enter(inst);
-
+ mtk_vcodec_debug(inst, "frm_cnt = %d\n ", inst->frm_cnt);
+ frame_info.frm_count = inst->frm_cnt;
+ frame_info.skip_frm_count = inst->skip_frm_cnt;
+ frame_info.frm_type = h264_frame_type(inst);
+ mtk_vcodec_debug(inst, "frm_count = %d,skip_frm_count =%d,frm_type=%d.\n",
+ frame_info.frm_count, frame_info.skip_frm_count,
+ frame_info.frm_type);
ret = vpu_enc_encode(&inst->vpu_inst, H264_BS_MODE_FRAME, frm_buf,
- bs_buf, bs_size);
+ bs_buf, bs_size, &frame_info);
if (ret)
return ret;
inst->work_bufs[VENC_H264_VPU_WORK_BUF_SKIP_FRAME].va,
*bs_size);
++inst->frm_cnt;
+ ++inst->skip_frm_cnt;
return ret;
}
static int h264_enc_init(struct mtk_vcodec_ctx *ctx)
{
+ const bool is_ext = MTK_ENC_CTX_IS_EXT(ctx);
int ret = 0;
struct venc_h264_inst *inst;
inst->ctx = ctx;
inst->vpu_inst.ctx = ctx;
- inst->vpu_inst.dev = ctx->dev->vpu_plat_dev;
- inst->vpu_inst.id = IPI_VENC_H264;
+ inst->vpu_inst.id = is_ext ? SCP_IPI_VENC_H264 : IPI_VENC_H264;
inst->hw_base = mtk_vcodec_get_reg_addr(inst->ctx, VENC_SYS);
mtk_vcodec_debug_enter(inst);
inst->prepend_hdr = 1;
mtk_vcodec_debug(inst, "set prepend header mode");
break;
-
+ case VENC_SET_PARAM_FORCE_INTRA:
+ case VENC_SET_PARAM_GOP_SIZE:
+ case VENC_SET_PARAM_INTRA_PERIOD:
+ inst->frm_cnt = 0;
+ inst->skip_frm_cnt = 0;
+ fallthrough;
default:
ret = vpu_enc_set_param(&inst->vpu_inst, type, enc_prm);
break;
#include "../venc_drv_base.h"
#include "../venc_ipi_msg.h"
#include "../venc_vpu_if.h"
-#include "mtk_vpu.h"
#define VENC_BITSTREAM_FRAME_SIZE 0x0098
#define VENC_BITSTREAM_HEADER_LEN 0x00e8
if (i == VENC_VP8_VPU_WORK_BUF_RC_CODE ||
i == VENC_VP8_VPU_WORK_BUF_RC_CODE2 ||
i == VENC_VP8_VPU_WORK_BUF_RC_CODE3) {
+ struct mtk_vcodec_fw *handler;
void *tmp_va;
- tmp_va = vpu_mapping_dm_addr(inst->vpu_inst.dev,
- wb[i].vpua);
+ handler = inst->vpu_inst.ctx->dev->fw_handler;
+ tmp_va = mtk_vcodec_fw_map_dm_addr(handler,
+ wb[i].vpua);
memcpy(inst->work_bufs[i].va, tmp_va, wb[i].size);
}
wb[i].iova = inst->work_bufs[i].dma_addr;
mtk_vcodec_debug(inst, "->frm_cnt=%d", inst->frm_cnt);
- ret = vpu_enc_encode(&inst->vpu_inst, 0, frm_buf, bs_buf, bs_size);
+ ret = vpu_enc_encode(&inst->vpu_inst, 0, frm_buf, bs_buf, bs_size,
+ NULL);
if (ret)
return ret;
inst->ctx = ctx;
inst->vpu_inst.ctx = ctx;
- inst->vpu_inst.dev = ctx->dev->vpu_plat_dev;
inst->vpu_inst.id = IPI_VENC_VP8;
inst->hw_base = mtk_vcodec_get_reg_addr(inst->ctx, VENC_LT_SYS);
#include "mtk_vcodec_enc.h"
#include "mtk_vcodec_enc_pm.h"
-#include "mtk_vpu.h"
int venc_if_init(struct mtk_vcodec_ctx *ctx, unsigned int fourcc)
{
unsigned int gop_size;
};
+/**
+ * struct venc_frame_info - per-frame information to pass to the firmware.
+ *
+ * @frm_count: sequential number for this frame
+ * @skip_frm_count: number of frames skipped so far while decoding
+ * @frm_type: type of the frame, from enum venc_h264_frame_type
+ */
+struct venc_frame_info {
+ unsigned int frm_count; /* per frame update */
+ unsigned int skip_frm_count; /* per frame update */
+ unsigned int frm_type; /* per frame update */
+};
+
/*
* struct venc_frm_buf - frame buffer information used in venc_if_encode()
* @fb_addr: plane frame buffer addresses
uint32_t data[8];
};
+struct venc_ap_ipi_msg_set_param_ext {
+ struct venc_ap_ipi_msg_set_param base;
+ uint32_t data_ext[24];
+};
+
/**
* struct venc_ap_ipi_msg_enc - AP to VPU enc cmd structure
* @msg_id: message id (AP_IPIMSG_XXX_ENC_ENCODE)
uint32_t bs_size;
};
+/**
+ * struct venc_ap_ipi_msg_enc_ext - AP to SCP extended enc cmd structure
+ *
+ * @base: base msg structure
+ * @data_item: number of items in the data array
+ * @data[8]: data array to store the set parameters
+ */
+struct venc_ap_ipi_msg_enc_ext {
+ struct venc_ap_ipi_msg_enc base;
+ uint32_t data_item;
+ uint32_t data[32];
+};
+
/**
* struct venc_ap_ipi_msg_deinit - AP to VPU deinit cmd structure
* @msg_id: message id (AP_IPIMSG_XXX_ENC_DEINIT)
* @venc_inst: AP encoder instance (struct venc_vp8_inst/venc_h264_inst *)
* @vpu_inst_addr: VPU encoder instance addr
* (struct venc_vp8_vsi/venc_h264_vsi *)
- * @reserved: reserved for future use. vpu is running in 32bit. Without
- * this reserved field, if kernel run in 64bit. this struct size
- * will be different between kernel and vpu
+ * @venc_abi_version: ABI version of the firmware. Kernel can use it to
+ * ensure that it is compatible with the firmware.
+ * For MT8173 the value of this field is undefined and
+ * should not be used.
*/
struct venc_vpu_ipi_msg_init {
uint32_t msg_id;
uint32_t status;
uint64_t venc_inst;
uint32_t vpu_inst_addr;
- uint32_t reserved;
+ uint32_t venc_abi_version;
};
/**
* Author: PoChun Lin <pochun.lin@mediatek.com>
*/
-#include "mtk_vpu.h"
+#include "mtk_vcodec_drv.h"
+#include "mtk_vcodec_fw.h"
#include "venc_ipi_msg.h"
#include "venc_vpu_if.h"
const struct venc_vpu_ipi_msg_init *msg = data;
vpu->inst_addr = msg->vpu_inst_addr;
- vpu->vsi = vpu_mapping_dm_addr(vpu->dev, msg->vpu_inst_addr);
+ vpu->vsi = mtk_vcodec_fw_map_dm_addr(vpu->ctx->dev->fw_handler,
+ msg->vpu_inst_addr);
+
+ /* Firmware version field value is unspecified on MT8173. */
+ if (vpu->ctx->dev->venc_pdata->chip == MTK_MT8173)
+ return;
+
+ /* Check firmware version. */
+ mtk_vcodec_debug(vpu, "firmware version: 0x%x\n",
+ msg->venc_abi_version);
+ switch (msg->venc_abi_version) {
+ case 1:
+ break;
+ default:
+ mtk_vcodec_err(vpu, "unhandled firmware version 0x%x\n",
+ msg->venc_abi_version);
+ vpu->failure = 1;
+ break;
+ }
}
static void handle_enc_encode_msg(struct venc_vpu_inst *vpu, const void *data)
vpu->is_key_frm = msg->is_key_frm;
}
-static void vpu_enc_ipi_handler(const void *data, unsigned int len, void *priv)
+static void vpu_enc_ipi_handler(void *data, unsigned int len, void *priv)
{
const struct venc_vpu_ipi_msg_common *msg = data;
struct venc_vpu_inst *vpu =
mtk_vcodec_debug(vpu, "msg_id %x inst %p status %d",
msg->msg_id, vpu, msg->status);
+ vpu->signaled = 1;
+ vpu->failure = (msg->status != VENC_IPI_MSG_STATUS_OK);
+ if (vpu->failure)
+ goto failure;
+
switch (msg->msg_id) {
case VPU_IPIMSG_ENC_INIT_DONE:
handle_enc_init_msg(vpu, data);
break;
}
- vpu->signaled = 1;
- vpu->failure = (msg->status != VENC_IPI_MSG_STATUS_OK);
-
+failure:
mtk_vcodec_debug_leave(vpu);
}
mtk_vcodec_debug_enter(vpu);
- if (!vpu->dev) {
+ if (!vpu->ctx->dev->fw_handler) {
mtk_vcodec_err(vpu, "inst dev is NULL");
return -EINVAL;
}
- status = vpu_ipi_send(vpu->dev, vpu->id, msg, len);
+ status = mtk_vcodec_fw_ipi_send(vpu->ctx->dev->fw_handler, vpu->id, msg,
+ len, 2000);
if (status) {
mtk_vcodec_err(vpu, "vpu_ipi_send msg_id %x len %d fail %d",
*(uint32_t *)msg, len, status);
vpu->signaled = 0;
vpu->failure = 0;
- status = vpu_ipi_register(vpu->dev, vpu->id, vpu_enc_ipi_handler,
- NULL, NULL);
+ status = mtk_vcodec_fw_ipi_register(vpu->ctx->dev->fw_handler, vpu->id,
+ vpu_enc_ipi_handler, "venc", NULL);
+
if (status) {
mtk_vcodec_err(vpu, "vpu_ipi_register fail %d", status);
return -EINVAL;
return 0;
}
+static unsigned int venc_enc_param_crop_right(struct venc_vpu_inst *vpu,
+ struct venc_enc_param *enc_prm)
+{
+ unsigned int img_crop_right = enc_prm->buf_width - enc_prm->width;
+
+ return img_crop_right % 16;
+}
+
+static unsigned int venc_enc_param_crop_bottom(struct venc_enc_param *enc_prm)
+{
+ return round_up(enc_prm->height, 16) - enc_prm->height;
+}
+
+static unsigned int venc_enc_param_num_mb(struct venc_enc_param *enc_prm)
+{
+ return DIV_ROUND_UP(enc_prm->width, 16) *
+ DIV_ROUND_UP(enc_prm->height, 16);
+}
+
int vpu_enc_set_param(struct venc_vpu_inst *vpu,
enum venc_set_param_type id,
struct venc_enc_param *enc_param)
{
- struct venc_ap_ipi_msg_set_param out;
+ const bool is_ext = MTK_ENC_CTX_IS_EXT(vpu->ctx);
+ size_t msg_size = is_ext ?
+ sizeof(struct venc_ap_ipi_msg_set_param_ext) :
+ sizeof(struct venc_ap_ipi_msg_set_param);
+ struct venc_ap_ipi_msg_set_param_ext out;
mtk_vcodec_debug(vpu, "id %d ->", id);
memset(&out, 0, sizeof(out));
- out.msg_id = AP_IPIMSG_ENC_SET_PARAM;
- out.vpu_inst_addr = vpu->inst_addr;
- out.param_id = id;
+ out.base.msg_id = AP_IPIMSG_ENC_SET_PARAM;
+ out.base.vpu_inst_addr = vpu->inst_addr;
+ out.base.param_id = id;
switch (id) {
case VENC_SET_PARAM_ENC:
- out.data_item = 0;
+ if (is_ext) {
+ out.base.data_item = 3;
+ out.base.data[0] =
+ venc_enc_param_crop_right(vpu, enc_param);
+ out.base.data[1] =
+ venc_enc_param_crop_bottom(enc_param);
+ out.base.data[2] = venc_enc_param_num_mb(enc_param);
+ } else {
+ out.base.data_item = 0;
+ }
break;
case VENC_SET_PARAM_FORCE_INTRA:
- out.data_item = 0;
+ out.base.data_item = 0;
break;
case VENC_SET_PARAM_ADJUST_BITRATE:
- out.data_item = 1;
- out.data[0] = enc_param->bitrate;
+ out.base.data_item = 1;
+ out.base.data[0] = enc_param->bitrate;
break;
case VENC_SET_PARAM_ADJUST_FRAMERATE:
- out.data_item = 1;
- out.data[0] = enc_param->frm_rate;
+ out.base.data_item = 1;
+ out.base.data[0] = enc_param->frm_rate;
break;
case VENC_SET_PARAM_GOP_SIZE:
- out.data_item = 1;
- out.data[0] = enc_param->gop_size;
+ out.base.data_item = 1;
+ out.base.data[0] = enc_param->gop_size;
break;
case VENC_SET_PARAM_INTRA_PERIOD:
- out.data_item = 1;
- out.data[0] = enc_param->intra_period;
+ out.base.data_item = 1;
+ out.base.data[0] = enc_param->intra_period;
break;
case VENC_SET_PARAM_SKIP_FRAME:
- out.data_item = 0;
+ out.base.data_item = 0;
break;
default:
mtk_vcodec_err(vpu, "id %d not supported", id);
return -EINVAL;
}
- if (vpu_enc_send_msg(vpu, &out, sizeof(out))) {
+ if (vpu_enc_send_msg(vpu, &out, msg_size)) {
mtk_vcodec_err(vpu,
"AP_IPIMSG_ENC_SET_PARAM %d fail", id);
return -EINVAL;
int vpu_enc_encode(struct venc_vpu_inst *vpu, unsigned int bs_mode,
struct venc_frm_buf *frm_buf,
struct mtk_vcodec_mem *bs_buf,
- unsigned int *bs_size)
+ unsigned int *bs_size,
+ struct venc_frame_info *frame_info)
{
- struct venc_ap_ipi_msg_enc out;
+ const bool is_ext = MTK_ENC_CTX_IS_EXT(vpu->ctx);
+ size_t msg_size = is_ext ?
+ sizeof(struct venc_ap_ipi_msg_enc_ext) :
+ sizeof(struct venc_ap_ipi_msg_enc);
+ struct venc_ap_ipi_msg_enc_ext out;
mtk_vcodec_debug(vpu, "bs_mode %d ->", bs_mode);
memset(&out, 0, sizeof(out));
- out.msg_id = AP_IPIMSG_ENC_ENCODE;
- out.vpu_inst_addr = vpu->inst_addr;
- out.bs_mode = bs_mode;
+ out.base.msg_id = AP_IPIMSG_ENC_ENCODE;
+ out.base.vpu_inst_addr = vpu->inst_addr;
+ out.base.bs_mode = bs_mode;
if (frm_buf) {
if ((frm_buf->fb_addr[0].dma_addr % 16 == 0) &&
(frm_buf->fb_addr[1].dma_addr % 16 == 0) &&
(frm_buf->fb_addr[2].dma_addr % 16 == 0)) {
- out.input_addr[0] = frm_buf->fb_addr[0].dma_addr;
- out.input_addr[1] = frm_buf->fb_addr[1].dma_addr;
- out.input_addr[2] = frm_buf->fb_addr[2].dma_addr;
+ out.base.input_addr[0] = frm_buf->fb_addr[0].dma_addr;
+ out.base.input_addr[1] = frm_buf->fb_addr[1].dma_addr;
+ out.base.input_addr[2] = frm_buf->fb_addr[2].dma_addr;
} else {
mtk_vcodec_err(vpu, "dma_addr not align to 16");
return -EINVAL;
}
}
if (bs_buf) {
- out.bs_addr = bs_buf->dma_addr;
- out.bs_size = bs_buf->size;
+ out.base.bs_addr = bs_buf->dma_addr;
+ out.base.bs_size = bs_buf->size;
}
- if (vpu_enc_send_msg(vpu, &out, sizeof(out))) {
+ if (is_ext && frame_info) {
+ out.data_item = 3;
+ out.data[0] = frame_info->frm_count;
+ out.data[1] = frame_info->skip_frm_count;
+ out.data[2] = frame_info->frm_type;
+ }
+ if (vpu_enc_send_msg(vpu, &out, msg_size)) {
mtk_vcodec_err(vpu, "AP_IPIMSG_ENC_ENCODE %d fail",
bs_mode);
return -EINVAL;
#ifndef _VENC_VPU_IF_H_
#define _VENC_VPU_IF_H_
-#include "mtk_vpu.h"
+#include "mtk_vcodec_fw.h"
#include "venc_drv_if.h"
/*
int is_key_frm;
unsigned int inst_addr;
void *vsi;
- enum ipi_id id;
+ int id;
struct mtk_vcodec_ctx *ctx;
- struct platform_device *dev;
};
int vpu_enc_init(struct venc_vpu_inst *vpu);
int vpu_enc_encode(struct venc_vpu_inst *vpu, unsigned int bs_mode,
struct venc_frm_buf *frm_buf,
struct mtk_vcodec_mem *bs_buf,
- unsigned int *bs_size);
+ unsigned int *bs_size,
+ struct venc_frame_info *frame_info);
int vpu_enc_deinit(struct venc_vpu_inst *vpu);
#endif
#ifdef CONFIG_DEBUG_FS
vpu_debugfs = debugfs_create_file("mtk_vpu", S_IRUGO, NULL, (void *)dev,
&vpu_debug_fops);
- if (!vpu_debugfs) {
- ret = -ENOMEM;
- goto cleanup_ipi;
- }
#endif
/* Set PTCM to 96K and DTCM to 32K */
of_reserved_mem_device_release(dev);
#ifdef CONFIG_DEBUG_FS
debugfs_remove(vpu_debugfs);
-cleanup_ipi:
#endif
memset(vpu->ipi_desc, 0, sizeof(struct vpu_ipi_desc) * IPI_MAX);
vpu_mutex_destroy:
platform_set_drvdata(pdev, pcdev);
irq = platform_get_irq(pdev, 0);
- if (irq < 0)
- return irq;
+ if (irq < 0) {
+ ret = irq;
+ goto rel_vdev;
+ }
+
ret = devm_request_irq(&pdev->dev, irq, emmaprp_irq, 0,
dev_name(&pdev->dev), pcdev);
if (ret)
* readback the same register, in this case the revision register.
*
* See this link for reference:
- * http://www.mail-archive.com/linux-omap@vger.kernel.org/msg08149.html
+ * https://www.mail-archive.com/linux-omap@vger.kernel.org/msg08149.html
*/
void omap3isp_flush(struct isp_device *isp)
{
mem = platform_get_resource(pdev, IORESOURCE_MEM, i);
isp->mmio_base[map_idx] =
devm_ioremap_resource(isp->dev, mem);
- if (IS_ERR(isp->mmio_base[map_idx]))
- return PTR_ERR(isp->mmio_base[map_idx]);
+ if (IS_ERR(isp->mmio_base[map_idx])) {
+ ret = PTR_ERR(isp->mmio_base[map_idx]);
+ goto error;
+ }
}
ret = isp_get_clocks(isp);
* requested.
*/
format->fmt.pix.field = V4L2_FIELD_INTERLACED_TB;
- /* Fall-through */
+ fallthrough;
case V4L2_FIELD_INTERLACED_TB:
case V4L2_FIELD_INTERLACED_BT:
/* Interlaced orders are only supported at the CCDC output. */
return pxa_mbus_find_fmtdesc(code, mbus_fmt, ARRAY_SIZE(mbus_fmt));
}
-static unsigned int pxa_mbus_config_compatible(const struct v4l2_mbus_config *cfg,
- unsigned int flags)
-{
- unsigned long common_flags;
- bool hsync = true, vsync = true, pclk, data, mode;
- bool mipi_lanes, mipi_clock;
-
- common_flags = cfg->flags & flags;
-
- switch (cfg->type) {
- case V4L2_MBUS_PARALLEL:
- hsync = common_flags & (V4L2_MBUS_HSYNC_ACTIVE_HIGH |
- V4L2_MBUS_HSYNC_ACTIVE_LOW);
- vsync = common_flags & (V4L2_MBUS_VSYNC_ACTIVE_HIGH |
- V4L2_MBUS_VSYNC_ACTIVE_LOW);
- /* fall through */
- case V4L2_MBUS_BT656:
- pclk = common_flags & (V4L2_MBUS_PCLK_SAMPLE_RISING |
- V4L2_MBUS_PCLK_SAMPLE_FALLING);
- data = common_flags & (V4L2_MBUS_DATA_ACTIVE_HIGH |
- V4L2_MBUS_DATA_ACTIVE_LOW);
- mode = common_flags & (V4L2_MBUS_MASTER | V4L2_MBUS_SLAVE);
- return (!hsync || !vsync || !pclk || !data || !mode) ?
- 0 : common_flags;
- case V4L2_MBUS_CSI2_DPHY:
- mipi_lanes = common_flags & V4L2_MBUS_CSI2_LANES;
- mipi_clock = common_flags & (V4L2_MBUS_CSI2_NONCONTINUOUS_CLOCK |
- V4L2_MBUS_CSI2_CONTINUOUS_CLOCK);
- return (!mipi_lanes || !mipi_clock) ? 0 : common_flags;
- default:
- WARN_ON(1);
- return -EINVAL;
- }
- return 0;
-}
-
/**
* struct pxa_camera_format_xlate - match between host and sensor formats
* @code: code of a sensor provided format
clk_disable_unprepare(pcdev->clk);
}
-static void pxa_camera_eof(unsigned long arg)
+static void pxa_camera_eof(struct tasklet_struct *t)
{
- struct pxa_camera_dev *pcdev = (struct pxa_camera_dev *)arg;
+ struct pxa_camera_dev *pcdev = from_tasklet(pcdev, t, task_eof);
unsigned long cifr;
struct pxa_buffer *buf;
return IRQ_HANDLED;
}
-static int test_platform_param(struct pxa_camera_dev *pcdev,
- unsigned char buswidth, unsigned long *flags)
-{
- /*
- * Platform specified synchronization and pixel clock polarities are
- * only a recommendation and are only used during probing. The PXA270
- * quick capture interface supports both.
- */
- *flags = (pcdev->platform_flags & PXA_CAMERA_MASTER ?
- V4L2_MBUS_MASTER : V4L2_MBUS_SLAVE) |
- V4L2_MBUS_HSYNC_ACTIVE_HIGH |
- V4L2_MBUS_HSYNC_ACTIVE_LOW |
- V4L2_MBUS_VSYNC_ACTIVE_HIGH |
- V4L2_MBUS_VSYNC_ACTIVE_LOW |
- V4L2_MBUS_DATA_ACTIVE_HIGH |
- V4L2_MBUS_PCLK_SAMPLE_RISING |
- V4L2_MBUS_PCLK_SAMPLE_FALLING;
-
- /* If requested data width is supported by the platform, use it */
- if ((1 << (buswidth - 1)) & pcdev->width_flags)
- return 0;
-
- return -EINVAL;
-}
-
static void pxa_camera_setup_cicr(struct pxa_camera_dev *pcdev,
unsigned long flags, __u32 pixfmt)
{
*/
static int pxa_camera_set_bus_param(struct pxa_camera_dev *pcdev)
{
+ unsigned int bus_width = pcdev->current_fmt->host_fmt->bits_per_sample;
struct v4l2_mbus_config cfg = {.type = V4L2_MBUS_PARALLEL,};
u32 pixfmt = pcdev->current_fmt->host_fmt->fourcc;
- unsigned long bus_flags, common_flags;
+ int mbus_config;
int ret;
- ret = test_platform_param(pcdev,
- pcdev->current_fmt->host_fmt->bits_per_sample,
- &bus_flags);
- if (ret < 0)
- return ret;
-
- ret = sensor_call(pcdev, video, g_mbus_config, &cfg);
- if (!ret) {
- common_flags = pxa_mbus_config_compatible(&cfg,
- bus_flags);
- if (!common_flags) {
- dev_warn(pcdev_to_dev(pcdev),
- "Flags incompatible: camera 0x%x, host 0x%lx\n",
- cfg.flags, bus_flags);
- return -EINVAL;
- }
- } else if (ret != -ENOIOCTLCMD) {
- return ret;
- } else {
- common_flags = bus_flags;
+ if (!((1 << (bus_width - 1)) & pcdev->width_flags)) {
+ dev_err(pcdev_to_dev(pcdev), "Unsupported bus width %u",
+ bus_width);
+ return -EINVAL;
}
pcdev->channels = 1;
/* Make choices, based on platform preferences */
- if ((common_flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH) &&
- (common_flags & V4L2_MBUS_HSYNC_ACTIVE_LOW)) {
- if (pcdev->platform_flags & PXA_CAMERA_HSP)
- common_flags &= ~V4L2_MBUS_HSYNC_ACTIVE_HIGH;
- else
- common_flags &= ~V4L2_MBUS_HSYNC_ACTIVE_LOW;
- }
+ mbus_config = 0;
+ if (pcdev->platform_flags & PXA_CAMERA_MASTER)
+ mbus_config |= V4L2_MBUS_MASTER;
+ else
+ mbus_config |= V4L2_MBUS_SLAVE;
- if ((common_flags & V4L2_MBUS_VSYNC_ACTIVE_HIGH) &&
- (common_flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)) {
- if (pcdev->platform_flags & PXA_CAMERA_VSP)
- common_flags &= ~V4L2_MBUS_VSYNC_ACTIVE_HIGH;
- else
- common_flags &= ~V4L2_MBUS_VSYNC_ACTIVE_LOW;
- }
+ if (pcdev->platform_flags & PXA_CAMERA_HSP)
+ mbus_config |= V4L2_MBUS_HSYNC_ACTIVE_HIGH;
+ else
+ mbus_config |= V4L2_MBUS_HSYNC_ACTIVE_LOW;
- if ((common_flags & V4L2_MBUS_PCLK_SAMPLE_RISING) &&
- (common_flags & V4L2_MBUS_PCLK_SAMPLE_FALLING)) {
- if (pcdev->platform_flags & PXA_CAMERA_PCP)
- common_flags &= ~V4L2_MBUS_PCLK_SAMPLE_RISING;
- else
- common_flags &= ~V4L2_MBUS_PCLK_SAMPLE_FALLING;
- }
+ if (pcdev->platform_flags & PXA_CAMERA_VSP)
+ mbus_config |= V4L2_MBUS_VSYNC_ACTIVE_HIGH;
+ else
+ mbus_config |= V4L2_MBUS_VSYNC_ACTIVE_LOW;
- cfg.flags = common_flags;
- ret = sensor_call(pcdev, video, s_mbus_config, &cfg);
+ if (pcdev->platform_flags & PXA_CAMERA_PCP)
+ mbus_config |= V4L2_MBUS_PCLK_SAMPLE_RISING;
+ else
+ mbus_config |= V4L2_MBUS_PCLK_SAMPLE_FALLING;
+ mbus_config |= V4L2_MBUS_DATA_ACTIVE_HIGH;
+
+ cfg.flags = mbus_config;
+ ret = sensor_call(pcdev, pad, set_mbus_config, 0, &cfg);
if (ret < 0 && ret != -ENOIOCTLCMD) {
- dev_dbg(pcdev_to_dev(pcdev),
- "camera s_mbus_config(0x%lx) returned %d\n",
- common_flags, ret);
+ dev_err(pcdev_to_dev(pcdev),
+ "Failed to call set_mbus_config: %d\n", ret);
return ret;
}
- pxa_camera_setup_cicr(pcdev, common_flags, pixfmt);
-
- return 0;
-}
-
-static int pxa_camera_try_bus_param(struct pxa_camera_dev *pcdev,
- unsigned char buswidth)
-{
- struct v4l2_mbus_config cfg = {.type = V4L2_MBUS_PARALLEL,};
- unsigned long bus_flags, common_flags;
- int ret = test_platform_param(pcdev, buswidth, &bus_flags);
-
- if (ret < 0)
- return ret;
+ /*
+ * If the requested media bus configuration has not been fully applied
+ * make sure it is supported by the platform.
+ *
+ * PXA does not support V4L2_MBUS_DATA_ACTIVE_LOW and the bus mastering
+ * roles should match.
+ */
+ if (cfg.flags != mbus_config) {
+ unsigned int pxa_mbus_role = mbus_config & (V4L2_MBUS_MASTER |
+ V4L2_MBUS_SLAVE);
+ if (pxa_mbus_role != (cfg.flags & (V4L2_MBUS_MASTER |
+ V4L2_MBUS_SLAVE))) {
+ dev_err(pcdev_to_dev(pcdev),
+ "Unsupported mbus configuration: bus mastering\n");
+ return -EINVAL;
+ }
- ret = sensor_call(pcdev, video, g_mbus_config, &cfg);
- if (!ret) {
- common_flags = pxa_mbus_config_compatible(&cfg,
- bus_flags);
- if (!common_flags) {
- dev_warn(pcdev_to_dev(pcdev),
- "Flags incompatible: camera 0x%x, host 0x%lx\n",
- cfg.flags, bus_flags);
+ if (cfg.flags & V4L2_MBUS_DATA_ACTIVE_LOW) {
+ dev_err(pcdev_to_dev(pcdev),
+ "Unsupported mbus configuration: DATA_ACTIVE_LOW\n");
return -EINVAL;
}
- } else if (ret == -ENOIOCTLCMD) {
- ret = 0;
}
- return ret;
+ pxa_camera_setup_cicr(pcdev, cfg.flags, pixfmt);
+
+ return 0;
}
static const struct pxa_mbus_pixelfmt pxa_camera_formats[] = {
return 0;
}
- /* This also checks support for the requested bits-per-sample */
- ret = pxa_camera_try_bus_param(pcdev, fmt->bits_per_sample);
- if (ret < 0)
- return 0;
-
switch (code.code) {
case MEDIA_BUS_FMT_UYVY8_2X8:
formats++;
goto exit_free_dma;
}
- tasklet_init(&pcdev->task_eof, pxa_camera_eof, (unsigned long)pcdev);
+ tasklet_setup(&pcdev->task_eof, pxa_camera_eof);
pxa_camera_activate(pcdev);
int ret;
ret = pm_runtime_get_sync(dev);
- if (ret < 0)
+ if (ret < 0) {
+ pm_runtime_put_sync(dev);
return ret;
+ }
ret = csiphy_set_clock_rates(csiphy);
if (ret < 0) {
.flush_buffers = vfe_flush_buffers,
};
-void msm_vfe_stop_streaming(struct vfe_device *vfe)
-{
- int i;
-
- for (i = 0; i < ARRAY_SIZE(vfe->line); i++)
- msm_video_stop_streaming(&vfe->line[i].video_out);
-}
-
/*
* msm_vfe_register_entities - Register subdev node for VFE module
* @vfe: VFE device
void msm_vfe_get_vfe_id(struct media_entity *entity, u8 *id);
void msm_vfe_get_vfe_line_id(struct media_entity *entity, enum vfe_line_id *id);
-void msm_vfe_stop_streaming(struct vfe_device *vfe);
-
extern const struct vfe_hw_ops vfe_ops_4_1;
extern const struct vfe_hw_ops vfe_ops_4_7;
#include "camss-video.h"
#include "camss.h"
+#define CAMSS_FRAME_MIN_WIDTH 1
+#define CAMSS_FRAME_MAX_WIDTH 8191
+#define CAMSS_FRAME_MIN_HEIGHT 1
+#define CAMSS_FRAME_MAX_HEIGHT_RDI 8191
+#define CAMSS_FRAME_MAX_HEIGHT_PIX 4096
+
struct fract {
u8 numerator;
u8 denominator;
return 0;
}
-static int video_enum_fmt(struct file *file, void *fh, struct v4l2_fmtdesc *f)
+/*
+ * Returns the index in the video->formats[] array of the element which
+ * has the "ndx"th unique value of pixelformat field.
+ * If not found (no more unique pixelformat's) returns -EINVAL.
+ */
+static int video_get_unique_pixelformat_by_index(struct camss_video *video,
+ int ndx)
{
- struct camss_video *video = video_drvdata(file);
int i, j, k;
- if (f->type != video->type)
- return -EINVAL;
-
- if (f->index >= video->nformats)
- return -EINVAL;
-
/* find index "i" of "k"th unique pixelformat in formats array */
k = -1;
for (i = 0; i < video->nformats; i++) {
if (j == i)
k++;
- if (k == f->index)
- break;
+ if (k == ndx)
+ return i;
+ }
+
+ return -EINVAL;
+}
+
+/*
+ * Returns the index in the video->formats[] array of the element which
+ * has code equal to mcode.
+ * If not found returns -EINVAL.
+ */
+static int video_get_pixelformat_by_mbus_code(struct camss_video *video,
+ u32 mcode)
+{
+ int i;
+
+ for (i = 0; i < video->nformats; i++) {
+ if (video->formats[i].code == mcode)
+ return i;
+ }
+
+ return -EINVAL;
+}
+
+static int video_enum_fmt(struct file *file, void *fh, struct v4l2_fmtdesc *f)
+{
+ struct camss_video *video = video_drvdata(file);
+ int i;
+
+ if (f->type != video->type)
+ return -EINVAL;
+
+ if (f->index >= video->nformats)
+ return -EINVAL;
+
+ if (f->mbus_code) {
+ /* Each entry in formats[] table has unique mbus_code */
+ if (f->index > 0)
+ return -EINVAL;
+
+ i = video_get_pixelformat_by_mbus_code(video, f->mbus_code);
+ } else {
+ i = video_get_unique_pixelformat_by_index(video, f->index);
}
- if (k < f->index)
+ if (i < 0)
return -EINVAL;
f->pixelformat = video->formats[i].pixelformat;
return 0;
}
+static int video_enum_framesizes(struct file *file, void *fh,
+ struct v4l2_frmsizeenum *fsize)
+{
+ struct camss_video *video = video_drvdata(file);
+ int i;
+
+ if (fsize->index)
+ return -EINVAL;
+
+ /* Only accept pixel format present in the formats[] table */
+ for (i = 0; i < video->nformats; i++) {
+ if (video->formats[i].pixelformat == fsize->pixel_format)
+ break;
+ }
+
+ if (i == video->nformats)
+ return -EINVAL;
+
+ fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
+ fsize->stepwise.min_width = CAMSS_FRAME_MIN_WIDTH;
+ fsize->stepwise.max_width = CAMSS_FRAME_MAX_WIDTH;
+ fsize->stepwise.min_height = CAMSS_FRAME_MIN_HEIGHT;
+ fsize->stepwise.max_height = (video->line_based) ?
+ CAMSS_FRAME_MAX_HEIGHT_PIX : CAMSS_FRAME_MAX_HEIGHT_RDI;
+ fsize->stepwise.step_width = 1;
+ fsize->stepwise.step_height = 1;
+
+ return 0;
+}
+
static int video_g_fmt(struct file *file, void *fh, struct v4l2_format *f)
{
struct camss_video *video = video_drvdata(file);
1, 65528);
sizeimage[i] = clamp_t(u32, p->sizeimage,
bytesperline[i],
- bytesperline[i] * 4096);
+ bytesperline[i] * CAMSS_FRAME_MAX_HEIGHT_PIX);
}
for (j = 0; j < video->nformats; j++)
memset(pix_mp, 0, sizeof(*pix_mp));
pix_mp->pixelformat = fi->pixelformat;
- pix_mp->width = clamp_t(u32, width, 1, 8191);
- pix_mp->height = clamp_t(u32, height, 1, 8191);
+ pix_mp->width = clamp_t(u32, width, 1, CAMSS_FRAME_MAX_WIDTH);
+ pix_mp->height = clamp_t(u32, height, 1, CAMSS_FRAME_MAX_HEIGHT_RDI);
pix_mp->num_planes = fi->planes;
for (i = 0; i < pix_mp->num_planes; i++) {
bpl = pix_mp->width / fi->hsub[i].numerator *
1, 65528);
p->sizeimage = clamp_t(u32, p->sizeimage,
p->bytesperline,
- p->bytesperline * 4096);
+ p->bytesperline * CAMSS_FRAME_MAX_HEIGHT_PIX);
lines = p->sizeimage / p->bytesperline;
if (p->bytesperline < bytesperline[i])
static const struct v4l2_ioctl_ops msm_vid_ioctl_ops = {
.vidioc_querycap = video_querycap,
.vidioc_enum_fmt_vid_cap = video_enum_fmt,
+ .vidioc_enum_framesizes = video_enum_framesizes,
.vidioc_g_fmt_vid_cap_mplane = video_g_fmt,
.vidioc_s_fmt_vid_cap_mplane = video_s_fmt,
.vidioc_try_fmt_vid_cap_mplane = video_try_fmt,
if (ret < 0) {
dev_err(v4l2_dev->dev, "Failed to init video entity: %d\n",
ret);
- goto error_media_init;
+ goto error_vb2_init;
}
mutex_init(&video->lock);
}
vdev->fops = &msm_vid_fops;
- vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_STREAMING |
- V4L2_CAP_READWRITE;
+ vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_STREAMING
+ | V4L2_CAP_READWRITE | V4L2_CAP_IO_MC;
vdev->ioctl_ops = &msm_vid_ioctl_ops;
vdev->release = msm_video_release;
vdev->v4l2_dev = v4l2_dev;
error_video_register:
media_entity_cleanup(&vdev->entity);
mutex_destroy(&video->lock);
-error_media_init:
- vb2_queue_release(&video->vb2_q);
error_vb2_init:
mutex_destroy(&video->q_lock);
return ret;
}
-void msm_video_stop_streaming(struct camss_video *video)
-{
- if (vb2_is_streaming(&video->vb2_q))
- vb2_queue_release(&video->vb2_q);
-}
-
void msm_video_unregister(struct camss_video *video)
{
atomic_inc(&video->camss->ref_count);
- video_unregister_device(&video->vdev);
+ vb2_video_unregister_device(&video->vdev);
atomic_dec(&video->camss->ref_count);
}
unsigned int nformats;
};
-void msm_video_stop_streaming(struct camss_video *video);
-
int msm_video_register(struct camss_video *video, struct v4l2_device *v4l2_dev,
const char *name, int is_pix);
*/
static int camss_remove(struct platform_device *pdev)
{
- unsigned int i;
-
struct camss *camss = platform_get_drvdata(pdev);
- for (i = 0; i < camss->vfe_num; i++)
- msm_vfe_stop_streaming(&camss->vfe[i]);
-
v4l2_async_notifier_unregister(&camss->notifier);
v4l2_async_notifier_cleanup(&camss->notifier);
camss_unregister_entities(camss);
venus-core-objs += core.o helpers.o firmware.o \
hfi_venus.o hfi_msgs.o hfi_cmds.o hfi.o \
- hfi_parser.o pm_helpers.o
+ hfi_parser.o pm_helpers.o dbgfs.o
venus-dec-objs += vdec.o vdec_ctrls.o
venus-enc-objs += venc.o venc_ctrls.o
#include <linux/init.h>
#include <linux/interconnect.h>
#include <linux/ioctl.h>
+#include <linux/delay.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/of_device.h>
mutex_unlock(&core->lock);
disable_irq_nosync(core->irq);
-
- /*
- * Delay recovery to ensure venus has completed any pending cache
- * operations. Without this sleep, we see device reset when firmware is
- * unloaded after a system error.
- */
- schedule_delayed_work(&core->work, msecs_to_jiffies(100));
+ schedule_delayed_work(&core->work, msecs_to_jiffies(10));
}
static const struct hfi_core_ops venus_core_ops = {
container_of(work, struct venus_core, work.work);
int ret = 0;
- dev_warn(core->dev, "system error has occurred, starting recovery!\n");
-
pm_runtime_get_sync(core->dev);
hfi_core_deinit(core, true);
- hfi_destroy(core);
+
+ dev_warn(core->dev, "system error has occurred, starting recovery!\n");
+
mutex_lock(&core->lock);
+
+ while (pm_runtime_active(core->dev_dec) || pm_runtime_active(core->dev_enc))
+ msleep(10);
+
venus_shutdown(core);
pm_runtime_put_sync(core->dev);
- ret |= hfi_create(core, &venus_core_ops);
+ while (core->pmdomains[0] && pm_runtime_active(core->pmdomains[0]))
+ usleep_range(1000, 1500);
+
+ hfi_reinit(core);
pm_runtime_get_sync(core->dev);
ret |= venus_boot(core);
-
ret |= hfi_core_resume(core, true);
enable_irq(core->irq);
ret = dma_set_mask_and_coherent(dev, core->res->dma_mask);
if (ret)
- return ret;
+ goto err_core_put;
- if (!dev->dma_parms) {
- dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms),
- GFP_KERNEL);
- if (!dev->dma_parms)
- return -ENOMEM;
- }
- dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
+ dma_set_max_seg_size(dev, UINT_MAX);
INIT_LIST_HEAD(&core->instances);
mutex_init(&core->lock);
IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
"venus", core);
if (ret)
- return ret;
+ goto err_core_put;
ret = hfi_create(core, &venus_core_ops);
if (ret)
- return ret;
+ goto err_core_put;
pm_runtime_enable(dev);
goto err_core_deinit;
ret = pm_runtime_put_sync(dev);
- if (ret)
+ if (ret) {
+ pm_runtime_get_noresume(dev);
goto err_dev_unregister;
+ }
+
+ venus_dbgfs_init(core);
return 0;
err_venus_shutdown:
venus_shutdown(core);
err_runtime_disable:
+ pm_runtime_put_noidle(dev);
pm_runtime_set_suspended(dev);
pm_runtime_disable(dev);
hfi_destroy(core);
+err_core_put:
+ if (core->pm_ops->core_put)
+ core->pm_ops->core_put(dev);
return ret;
}
v4l2_device_unregister(&core->v4l2_dev);
mutex_destroy(&core->pm_lock);
mutex_destroy(&core->lock);
+ venus_dbgfs_deinit(core);
return ret;
}
.vcodec_clks_num = 2,
.vcodec_pmdomains = { "venus", "vcodec0", "vcodec1" },
.vcodec_pmdomains_num = 3,
+ .opp_pmdomain = (const char *[]) { "cx", NULL },
.vcodec_num = 2,
.max_load = 3110400, /* 4096x2160@90 */
.hfi_version = HFI_VERSION_4XX,
.vmem_size = 0,
.vmem_addr = 0,
.dma_mask = 0xe0000000 - 1,
+ .cp_start = 0,
+ .cp_size = 0x70800000,
+ .cp_nonpixel_start = 0x1000000,
+ .cp_nonpixel_size = 0x24800000,
.fwname = "qcom/venus-5.2/venus.mdt",
};
.vcodec_clks_num = 2,
.vcodec_pmdomains = { "venus", "vcodec0" },
.vcodec_pmdomains_num = 2,
+ .opp_pmdomain = (const char *[]) { "cx", NULL },
.vcodec_num = 1,
.hfi_version = HFI_VERSION_4XX,
.vmem_id = VIDC_RESOURCE_NONE,
#include <media/v4l2-ctrls.h>
#include <media/v4l2-device.h>
+#include "dbgfs.h"
#include "hfi.h"
+#define VDBGL "VenusLow : "
+#define VDBGM "VenusMed : "
+#define VDBGH "VenusHigh: "
+#define VDBGFW "VenusFW : "
+
#define VIDC_CLKS_NUM_MAX 4
#define VIDC_VCODEC_CLKS_NUM_MAX 2
#define VIDC_PMDOMAINS_NUM_MAX 3
+extern int venus_fw_debug;
+
struct freq_tbl {
unsigned int load;
unsigned long freq;
unsigned int vcodec_clks_num;
const char * const vcodec_pmdomains[VIDC_PMDOMAINS_NUM_MAX];
unsigned int vcodec_pmdomains_num;
+ const char **opp_pmdomain;
unsigned int vcodec_num;
enum hfi_version hfi_version;
u32 max_load;
unsigned int vmem_id;
u32 vmem_size;
u32 vmem_addr;
+ u32 cp_start;
+ u32 cp_size;
+ u32 cp_nonpixel_start;
+ u32 cp_nonpixel_size;
const char *fwname;
};
* @priv: a private filed for HFI operations
* @ops: the core HFI operations
* @work: a delayed work for handling system fatal error
+ * @root: debugfs root directory
*/
struct venus_core {
void __iomem *base;
struct clk *vcodec1_clks[VIDC_VCODEC_CLKS_NUM_MAX];
struct icc_path *video_path;
struct icc_path *cpucfg_path;
+ struct opp_table *opp_table;
+ bool has_opp_table;
struct device_link *pd_dl_venus;
struct device *pmdomains[VIDC_PMDOMAINS_NUM_MAX];
+ struct device_link *opp_dl_venus;
+ struct device *opp_pmdomain;
struct video_device *vdev_dec;
struct video_device *vdev_enc;
struct v4l2_device v4l2_dev;
unsigned int codecs_count;
unsigned int core0_usage_count;
unsigned int core1_usage_count;
+ struct dentry *root;
};
struct vdec_controls {
u32 bitrate;
u32 bitrate_peak;
u32 rc_enable;
+ u32 const_quality;
+ u32 frame_skip_mode;
u32 h264_i_period;
u32 h264_entropy_mode;
u32 header_mode;
- struct {
- u32 mpeg4;
- u32 h264;
- u32 vpx;
- u32 hevc;
- } profile;
- struct {
- u32 mpeg4;
- u32 h264;
- u32 hevc;
- } level;
+ u32 profile;
+ u32 level;
};
struct venus_buffer {
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Linaro Ltd.
+ */
+
+#include <linux/debugfs.h>
+
+#include "core.h"
+
+void venus_dbgfs_init(struct venus_core *core)
+{
+ core->root = debugfs_create_dir("venus", NULL);
+ debugfs_create_x32("fw_level", 0644, core->root, &venus_fw_debug);
+}
+
+void venus_dbgfs_deinit(struct venus_core *core)
+{
+ debugfs_remove_recursive(core->root);
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2020 Linaro Ltd. */
+
+#ifndef __VENUS_DBGFS_H__
+#define __VENUS_DBGFS_H__
+
+struct venus_core;
+
+void venus_dbgfs_init(struct venus_core *core);
+void venus_dbgfs_deinit(struct venus_core *core);
+
+#endif
int venus_boot(struct venus_core *core)
{
struct device *dev = core->dev;
+ const struct venus_resources *res = core->res;
phys_addr_t mem_phys;
size_t mem_size;
int ret;
else
ret = venus_boot_no_tz(core, mem_phys, mem_size);
- return ret;
+ if (ret)
+ return ret;
+
+ if (core->use_tz && res->cp_size) {
+ ret = qcom_scm_mem_protect_video_var(res->cp_start,
+ res->cp_size,
+ res->cp_nonpixel_start,
+ res->cp_nonpixel_size);
+ if (ret) {
+ qcom_scm_pas_shutdown(VENUS_PAS_ID);
+ dev_err(dev, "set virtual address ranges fail (%d)\n",
+ ret);
+ return ret;
+ }
+ }
+
+ return 0;
}
int venus_shutdown(struct venus_core *core)
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/slab.h>
+#include <linux/kernel.h>
#include <media/videobuf2-dma-sg.h>
#include <media/v4l2-mem2mem.h>
#include <asm/div64.h>
}
if (slot == -1) {
- dev_dbg(inst->core->dev, "%s: no free slot\n", __func__);
+ dev_dbg(inst->core->dev, VDBGL "no free slot\n");
return;
}
}
EXPORT_SYMBOL_GPL(venus_helper_get_bufreq);
+struct id_mapping {
+ u32 hfi_id;
+ u32 v4l2_id;
+};
+
+static const struct id_mapping mpeg4_profiles[] = {
+ { HFI_MPEG4_PROFILE_SIMPLE, V4L2_MPEG_VIDEO_MPEG4_PROFILE_SIMPLE },
+ { HFI_MPEG4_PROFILE_ADVANCEDSIMPLE, V4L2_MPEG_VIDEO_MPEG4_PROFILE_ADVANCED_SIMPLE },
+};
+
+static const struct id_mapping mpeg4_levels[] = {
+ { HFI_MPEG4_LEVEL_0, V4L2_MPEG_VIDEO_MPEG4_LEVEL_0 },
+ { HFI_MPEG4_LEVEL_0b, V4L2_MPEG_VIDEO_MPEG4_LEVEL_0B },
+ { HFI_MPEG4_LEVEL_1, V4L2_MPEG_VIDEO_MPEG4_LEVEL_1 },
+ { HFI_MPEG4_LEVEL_2, V4L2_MPEG_VIDEO_MPEG4_LEVEL_2 },
+ { HFI_MPEG4_LEVEL_3, V4L2_MPEG_VIDEO_MPEG4_LEVEL_3 },
+ { HFI_MPEG4_LEVEL_4, V4L2_MPEG_VIDEO_MPEG4_LEVEL_4 },
+ { HFI_MPEG4_LEVEL_5, V4L2_MPEG_VIDEO_MPEG4_LEVEL_5 },
+};
+
+static const struct id_mapping mpeg2_profiles[] = {
+ { HFI_MPEG2_PROFILE_SIMPLE, V4L2_MPEG_VIDEO_MPEG2_PROFILE_SIMPLE },
+ { HFI_MPEG2_PROFILE_MAIN, V4L2_MPEG_VIDEO_MPEG2_PROFILE_MAIN },
+ { HFI_MPEG2_PROFILE_SNR, V4L2_MPEG_VIDEO_MPEG2_PROFILE_SNR_SCALABLE },
+ { HFI_MPEG2_PROFILE_SPATIAL, V4L2_MPEG_VIDEO_MPEG2_PROFILE_SPATIALLY_SCALABLE },
+ { HFI_MPEG2_PROFILE_HIGH, V4L2_MPEG_VIDEO_MPEG2_PROFILE_HIGH },
+};
+
+static const struct id_mapping mpeg2_levels[] = {
+ { HFI_MPEG2_LEVEL_LL, V4L2_MPEG_VIDEO_MPEG2_LEVEL_LOW },
+ { HFI_MPEG2_LEVEL_ML, V4L2_MPEG_VIDEO_MPEG2_LEVEL_MAIN },
+ { HFI_MPEG2_LEVEL_H14, V4L2_MPEG_VIDEO_MPEG2_LEVEL_HIGH_1440 },
+ { HFI_MPEG2_LEVEL_HL, V4L2_MPEG_VIDEO_MPEG2_LEVEL_HIGH },
+};
+
+static const struct id_mapping h264_profiles[] = {
+ { HFI_H264_PROFILE_BASELINE, V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE },
+ { HFI_H264_PROFILE_MAIN, V4L2_MPEG_VIDEO_H264_PROFILE_MAIN },
+ { HFI_H264_PROFILE_HIGH, V4L2_MPEG_VIDEO_H264_PROFILE_HIGH },
+ { HFI_H264_PROFILE_STEREO_HIGH, V4L2_MPEG_VIDEO_H264_PROFILE_STEREO_HIGH },
+ { HFI_H264_PROFILE_MULTIVIEW_HIGH, V4L2_MPEG_VIDEO_H264_PROFILE_MULTIVIEW_HIGH },
+ { HFI_H264_PROFILE_CONSTRAINED_BASE, V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE },
+ { HFI_H264_PROFILE_CONSTRAINED_HIGH, V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH },
+};
+
+static const struct id_mapping h264_levels[] = {
+ { HFI_H264_LEVEL_1, V4L2_MPEG_VIDEO_H264_LEVEL_1_0 },
+ { HFI_H264_LEVEL_1b, V4L2_MPEG_VIDEO_H264_LEVEL_1B },
+ { HFI_H264_LEVEL_11, V4L2_MPEG_VIDEO_H264_LEVEL_1_1 },
+ { HFI_H264_LEVEL_12, V4L2_MPEG_VIDEO_H264_LEVEL_1_2 },
+ { HFI_H264_LEVEL_13, V4L2_MPEG_VIDEO_H264_LEVEL_1_3 },
+ { HFI_H264_LEVEL_2, V4L2_MPEG_VIDEO_H264_LEVEL_2_0 },
+ { HFI_H264_LEVEL_21, V4L2_MPEG_VIDEO_H264_LEVEL_2_1 },
+ { HFI_H264_LEVEL_22, V4L2_MPEG_VIDEO_H264_LEVEL_2_2 },
+ { HFI_H264_LEVEL_3, V4L2_MPEG_VIDEO_H264_LEVEL_3_0 },
+ { HFI_H264_LEVEL_31, V4L2_MPEG_VIDEO_H264_LEVEL_3_1 },
+ { HFI_H264_LEVEL_32, V4L2_MPEG_VIDEO_H264_LEVEL_3_2 },
+ { HFI_H264_LEVEL_4, V4L2_MPEG_VIDEO_H264_LEVEL_4_0 },
+ { HFI_H264_LEVEL_41, V4L2_MPEG_VIDEO_H264_LEVEL_4_1 },
+ { HFI_H264_LEVEL_42, V4L2_MPEG_VIDEO_H264_LEVEL_4_2 },
+ { HFI_H264_LEVEL_5, V4L2_MPEG_VIDEO_H264_LEVEL_5_0 },
+ { HFI_H264_LEVEL_51, V4L2_MPEG_VIDEO_H264_LEVEL_5_1 },
+ { HFI_H264_LEVEL_52, V4L2_MPEG_VIDEO_H264_LEVEL_5_1 },
+};
+
+static const struct id_mapping hevc_profiles[] = {
+ { HFI_HEVC_PROFILE_MAIN, V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN },
+ { HFI_HEVC_PROFILE_MAIN_STILL_PIC, V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE },
+ { HFI_HEVC_PROFILE_MAIN10, V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10 },
+};
+
+static const struct id_mapping hevc_levels[] = {
+ { HFI_HEVC_LEVEL_1, V4L2_MPEG_VIDEO_HEVC_LEVEL_1 },
+ { HFI_HEVC_LEVEL_2, V4L2_MPEG_VIDEO_HEVC_LEVEL_2 },
+ { HFI_HEVC_LEVEL_21, V4L2_MPEG_VIDEO_HEVC_LEVEL_2_1 },
+ { HFI_HEVC_LEVEL_3, V4L2_MPEG_VIDEO_HEVC_LEVEL_3 },
+ { HFI_HEVC_LEVEL_31, V4L2_MPEG_VIDEO_HEVC_LEVEL_3_1 },
+ { HFI_HEVC_LEVEL_4, V4L2_MPEG_VIDEO_HEVC_LEVEL_4 },
+ { HFI_HEVC_LEVEL_41, V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1 },
+ { HFI_HEVC_LEVEL_5, V4L2_MPEG_VIDEO_HEVC_LEVEL_5 },
+ { HFI_HEVC_LEVEL_51, V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1 },
+ { HFI_HEVC_LEVEL_52, V4L2_MPEG_VIDEO_HEVC_LEVEL_5_2 },
+ { HFI_HEVC_LEVEL_6, V4L2_MPEG_VIDEO_HEVC_LEVEL_6 },
+ { HFI_HEVC_LEVEL_61, V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1 },
+ { HFI_HEVC_LEVEL_62, V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2 },
+};
+
+static const struct id_mapping vp8_profiles[] = {
+ { HFI_VPX_PROFILE_VERSION_0, V4L2_MPEG_VIDEO_VP8_PROFILE_0 },
+ { HFI_VPX_PROFILE_VERSION_1, V4L2_MPEG_VIDEO_VP8_PROFILE_1 },
+ { HFI_VPX_PROFILE_VERSION_2, V4L2_MPEG_VIDEO_VP8_PROFILE_2 },
+ { HFI_VPX_PROFILE_VERSION_3, V4L2_MPEG_VIDEO_VP8_PROFILE_3 },
+};
+
+static const struct id_mapping vp9_profiles[] = {
+ { HFI_VP9_PROFILE_P0, V4L2_MPEG_VIDEO_VP9_PROFILE_0 },
+ { HFI_VP9_PROFILE_P2_10B, V4L2_MPEG_VIDEO_VP9_PROFILE_2 },
+};
+
+static const struct id_mapping vp9_levels[] = {
+ { HFI_VP9_LEVEL_1, V4L2_MPEG_VIDEO_VP9_LEVEL_1_0 },
+ { HFI_VP9_LEVEL_11, V4L2_MPEG_VIDEO_VP9_LEVEL_1_1 },
+ { HFI_VP9_LEVEL_2, V4L2_MPEG_VIDEO_VP9_LEVEL_2_0},
+ { HFI_VP9_LEVEL_21, V4L2_MPEG_VIDEO_VP9_LEVEL_2_1 },
+ { HFI_VP9_LEVEL_3, V4L2_MPEG_VIDEO_VP9_LEVEL_3_0},
+ { HFI_VP9_LEVEL_31, V4L2_MPEG_VIDEO_VP9_LEVEL_3_1 },
+ { HFI_VP9_LEVEL_4, V4L2_MPEG_VIDEO_VP9_LEVEL_4_0 },
+ { HFI_VP9_LEVEL_41, V4L2_MPEG_VIDEO_VP9_LEVEL_4_1 },
+ { HFI_VP9_LEVEL_5, V4L2_MPEG_VIDEO_VP9_LEVEL_5_0 },
+ { HFI_VP9_LEVEL_51, V4L2_MPEG_VIDEO_VP9_LEVEL_5_1 },
+ { HFI_VP9_LEVEL_6, V4L2_MPEG_VIDEO_VP9_LEVEL_6_0 },
+ { HFI_VP9_LEVEL_61, V4L2_MPEG_VIDEO_VP9_LEVEL_6_1 },
+};
+
+static u32 find_v4l2_id(u32 hfi_id, const struct id_mapping *array, unsigned int array_sz)
+{
+ unsigned int i;
+
+ if (!array || !array_sz)
+ return 0;
+
+ for (i = 0; i < array_sz; i++)
+ if (hfi_id == array[i].hfi_id)
+ return array[i].v4l2_id;
+
+ return 0;
+}
+
+static u32 find_hfi_id(u32 v4l2_id, const struct id_mapping *array, unsigned int array_sz)
+{
+ unsigned int i;
+
+ if (!array || !array_sz)
+ return 0;
+
+ for (i = 0; i < array_sz; i++)
+ if (v4l2_id == array[i].v4l2_id)
+ return array[i].hfi_id;
+
+ return 0;
+}
+
+static void
+v4l2_id_profile_level(u32 hfi_codec, struct hfi_profile_level *pl, u32 *profile, u32 *level)
+{
+ u32 hfi_pf = pl->profile;
+ u32 hfi_lvl = pl->level;
+
+ switch (hfi_codec) {
+ case HFI_VIDEO_CODEC_H264:
+ *profile = find_v4l2_id(hfi_pf, h264_profiles, ARRAY_SIZE(h264_profiles));
+ *level = find_v4l2_id(hfi_lvl, h264_levels, ARRAY_SIZE(h264_levels));
+ break;
+ case HFI_VIDEO_CODEC_MPEG2:
+ *profile = find_v4l2_id(hfi_pf, mpeg2_profiles, ARRAY_SIZE(mpeg2_profiles));
+ *level = find_v4l2_id(hfi_lvl, mpeg2_levels, ARRAY_SIZE(mpeg2_levels));
+ break;
+ case HFI_VIDEO_CODEC_MPEG4:
+ *profile = find_v4l2_id(hfi_pf, mpeg4_profiles, ARRAY_SIZE(mpeg4_profiles));
+ *level = find_v4l2_id(hfi_lvl, mpeg4_levels, ARRAY_SIZE(mpeg4_levels));
+ break;
+ case HFI_VIDEO_CODEC_VP8:
+ *profile = find_v4l2_id(hfi_pf, vp8_profiles, ARRAY_SIZE(vp8_profiles));
+ *level = 0;
+ break;
+ case HFI_VIDEO_CODEC_VP9:
+ *profile = find_v4l2_id(hfi_pf, vp9_profiles, ARRAY_SIZE(vp9_profiles));
+ *level = find_v4l2_id(hfi_lvl, vp9_levels, ARRAY_SIZE(vp9_levels));
+ break;
+ case HFI_VIDEO_CODEC_HEVC:
+ *profile = find_v4l2_id(hfi_pf, hevc_profiles, ARRAY_SIZE(hevc_profiles));
+ *level = find_v4l2_id(hfi_lvl, hevc_levels, ARRAY_SIZE(hevc_levels));
+ break;
+ default:
+ break;
+ }
+}
+
+static void
+hfi_id_profile_level(u32 hfi_codec, u32 v4l2_pf, u32 v4l2_lvl, struct hfi_profile_level *pl)
+{
+ switch (hfi_codec) {
+ case HFI_VIDEO_CODEC_H264:
+ pl->profile = find_hfi_id(v4l2_pf, h264_profiles, ARRAY_SIZE(h264_profiles));
+ pl->level = find_hfi_id(v4l2_lvl, h264_levels, ARRAY_SIZE(h264_levels));
+ break;
+ case HFI_VIDEO_CODEC_MPEG2:
+ pl->profile = find_hfi_id(v4l2_pf, mpeg2_profiles, ARRAY_SIZE(mpeg2_profiles));
+ pl->level = find_hfi_id(v4l2_lvl, mpeg2_levels, ARRAY_SIZE(mpeg2_levels));
+ break;
+ case HFI_VIDEO_CODEC_MPEG4:
+ pl->profile = find_hfi_id(v4l2_pf, mpeg4_profiles, ARRAY_SIZE(mpeg4_profiles));
+ pl->level = find_hfi_id(v4l2_lvl, mpeg4_levels, ARRAY_SIZE(mpeg4_levels));
+ break;
+ case HFI_VIDEO_CODEC_VP8:
+ pl->profile = find_hfi_id(v4l2_pf, vp8_profiles, ARRAY_SIZE(vp8_profiles));
+ pl->level = 0;
+ break;
+ case HFI_VIDEO_CODEC_VP9:
+ pl->profile = find_hfi_id(v4l2_pf, vp9_profiles, ARRAY_SIZE(vp9_profiles));
+ pl->level = find_hfi_id(v4l2_lvl, vp9_levels, ARRAY_SIZE(vp9_levels));
+ break;
+ case HFI_VIDEO_CODEC_HEVC:
+ pl->profile = find_hfi_id(v4l2_pf, hevc_profiles, ARRAY_SIZE(hevc_profiles));
+ pl->level = find_hfi_id(v4l2_lvl, hevc_levels, ARRAY_SIZE(hevc_levels));
+ break;
+ default:
+ break;
+ }
+}
+
+int venus_helper_get_profile_level(struct venus_inst *inst, u32 *profile, u32 *level)
+{
+ const u32 ptype = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT;
+ union hfi_get_property hprop;
+ int ret;
+
+ ret = hfi_session_get_property(inst, ptype, &hprop);
+ if (ret)
+ return ret;
+
+ v4l2_id_profile_level(inst->hfi_codec, &hprop.profile_level, profile, level);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(venus_helper_get_profile_level);
+
+int venus_helper_set_profile_level(struct venus_inst *inst, u32 profile, u32 level)
+{
+ const u32 ptype = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT;
+ struct hfi_profile_level pl;
+
+ hfi_id_profile_level(inst->hfi_codec, profile, level, &pl);
+
+ return hfi_session_set_property(inst, ptype, &pl);
+}
+EXPORT_SYMBOL_GPL(venus_helper_set_profile_level);
+
static u32 get_framesize_raw_nv12(u32 width, u32 height)
{
u32 y_stride, uv_stride, y_plane;
int venus_helper_process_initial_out_bufs(struct venus_inst *inst);
void venus_helper_get_ts_metadata(struct venus_inst *inst, u64 timestamp_us,
struct vb2_v4l2_buffer *vbuf);
+int venus_helper_get_profile_level(struct venus_inst *inst, u32 *profile, u32 *level);
+int venus_helper_set_profile_level(struct venus_inst *inst, u32 profile, u32 level);
#endif
{
venus_hfi_destroy(core);
}
+
+void hfi_reinit(struct venus_core *core)
+{
+ venus_hfi_queues_reinit(core);
+}
int hfi_create(struct venus_core *core, const struct hfi_core_ops *ops);
void hfi_destroy(struct venus_core *core);
+void hfi_reinit(struct venus_core *core);
int hfi_core_init(struct venus_core *core);
int hfi_core_deinit(struct venus_core *core, bool blocking);
case HFI_RATE_CONTROL_CBR_VFR:
case HFI_RATE_CONTROL_VBR_CFR:
case HFI_RATE_CONTROL_VBR_VFR:
+ case HFI_RATE_CONTROL_CQ:
break;
default:
ret = -EINVAL;
return 0;
}
+static int
+pkt_session_set_property_6xx(struct hfi_session_set_property_pkt *pkt,
+ void *cookie, u32 ptype, void *pdata)
+{
+ void *prop_data;
+
+ if (!pkt || !cookie || !pdata)
+ return -EINVAL;
+
+ prop_data = &pkt->data[1];
+
+ pkt->shdr.hdr.size = sizeof(*pkt);
+ pkt->shdr.hdr.pkt_type = HFI_CMD_SESSION_SET_PROPERTY;
+ pkt->shdr.session_id = hash32_ptr(cookie);
+ pkt->num_properties = 1;
+ pkt->data[0] = ptype;
+
+ switch (ptype) {
+ case HFI_PROPERTY_CONFIG_HEIC_FRAME_QUALITY: {
+ struct hfi_heic_frame_quality *in = pdata, *cq = prop_data;
+
+ cq->frame_quality = in->frame_quality;
+ pkt->shdr.hdr.size += sizeof(u32) + sizeof(*cq);
+ break;
+ } default:
+ return pkt_session_set_property_4xx(pkt, cookie, ptype, pdata);
+ }
+
+ return 0;
+}
+
int pkt_session_get_property(struct hfi_session_get_property_pkt *pkt,
void *cookie, u32 ptype)
{
if (hfi_ver == HFI_VERSION_3XX)
return pkt_session_set_property_3xx(pkt, cookie, ptype, pdata);
- return pkt_session_set_property_4xx(pkt, cookie, ptype, pdata);
+ if (hfi_ver == HFI_VERSION_4XX)
+ return pkt_session_set_property_4xx(pkt, cookie, ptype, pdata);
+
+ return pkt_session_set_property_6xx(pkt, cookie, ptype, pdata);
}
void pkt_set_version(enum hfi_version version)
#define HFI_RATE_CONTROL_VBR_CFR 0x1000003
#define HFI_RATE_CONTROL_CBR_VFR 0x1000004
#define HFI_RATE_CONTROL_CBR_CFR 0x1000005
+#define HFI_RATE_CONTROL_CQ 0x1000008
#define HFI_VIDEO_CODEC_H264 0x00000002
#define HFI_VIDEO_CODEC_H263 0x00000004
#define HFI_HEVC_TIER_MAIN 0x1
#define HFI_HEVC_TIER_HIGH0 0x2
+/* VP9 Profile 0, 8-bit */
+#define HFI_VP9_PROFILE_P0 0x00000001
+/* VP9 Profile 2, 10-bit */
+#define HFI_VP9_PROFILE_P2_10B 0x00000004
+
+#define HFI_VP9_LEVEL_1 0x00000001
+#define HFI_VP9_LEVEL_11 0x00000002
+#define HFI_VP9_LEVEL_2 0x00000004
+#define HFI_VP9_LEVEL_21 0x00000008
+#define HFI_VP9_LEVEL_3 0x00000010
+#define HFI_VP9_LEVEL_31 0x00000020
+#define HFI_VP9_LEVEL_4 0x00000040
+#define HFI_VP9_LEVEL_41 0x00000080
+#define HFI_VP9_LEVEL_5 0x00000100
+#define HFI_VP9_LEVEL_51 0x00000200
+#define HFI_VP9_LEVEL_6 0x00000400
+#define HFI_VP9_LEVEL_61 0x00000800
+
#define HFI_BUFFER_INPUT 0x1
#define HFI_BUFFER_OUTPUT 0x2
#define HFI_BUFFER_OUTPUT2 0x3
#define HFI_PROPERTY_CONFIG_VENC_HIER_P_ENH_LAYER 0x200600b
#define HFI_PROPERTY_CONFIG_VENC_LTRPERIOD 0x200600c
#define HFI_PROPERTY_CONFIG_VENC_PERF_MODE 0x200600e
+#define HFI_PROPERTY_CONFIG_HEIC_FRAME_QUALITY 0x2006014
/*
* HFI_PROPERTY_PARAM_VPE_COMMON_START
enum hfi_version {
HFI_VERSION_1XX,
HFI_VERSION_3XX,
- HFI_VERSION_4XX
+ HFI_VERSION_4XX,
+ HFI_VERSION_6XX,
};
struct hfi_buffer_info {
u32 quality_vs_speed;
};
+struct hfi_heic_frame_quality {
+ u32 frame_quality;
+ u32 reserved[3];
+};
+
struct hfi_quantization {
u32 qp_i;
u32 qp_p;
struct hfi_msg_event_notify_pkt *pkt)
{
if (pkt)
- dev_dbg(core->dev,
+ dev_dbg(core->dev, VDBGH
"sys error (session id:%x, data1:%x, data2:%x)\n",
pkt->shdr.session_id, pkt->event_data1,
pkt->event_data2);
{
struct device *dev = core->dev;
- dev_dbg(dev, "session error: event id:%x, session id:%x\n",
+ dev_dbg(dev, VDBGH "session error: event id:%x, session id:%x\n",
pkt->event_data1, pkt->shdr.session_id);
if (!inst)
/* bad packet */
return;
- dev_dbg(dev, "F/W version: %s\n", (u8 *)&pkt->data[1]);
+ dev_dbg(dev, VDBGL "F/W version: %s\n", (u8 *)&pkt->data[1]);
}
static void hfi_sys_property_info(struct venus_core *core,
struct device *dev = core->dev;
if (!pkt->num_properties) {
- dev_dbg(dev, "%s: no properties\n", __func__);
+ dev_dbg(dev, VDBGL "no properties\n");
return;
}
sys_get_prop_image_version(dev, pkt);
break;
default:
- dev_dbg(dev, "%s: unknown property data\n", __func__);
+ dev_dbg(dev, VDBGL "unknown property data\n");
break;
}
}
static void hfi_sys_idle_done(struct venus_core *core, struct venus_inst *inst,
void *packet)
{
- dev_dbg(core->dev, "sys idle\n");
+ dev_dbg(core->dev, VDBGL "sys idle\n");
}
static void hfi_sys_pc_prepare_done(struct venus_core *core,
{
struct hfi_msg_sys_pc_prep_done_pkt *pkt = packet;
- dev_dbg(core->dev, "pc prepare done (error %x)\n", pkt->error_type);
+ dev_dbg(core->dev, VDBGL "pc prepare done (error %x)\n",
+ pkt->error_type);
}
static unsigned int
case HFI_PROPERTY_CONFIG_VDEC_ENTROPY:
break;
default:
- dev_dbg(dev, "%s: unknown property id:%x\n", __func__,
- pkt->data[0]);
+ dev_dbg(dev, VDBGM "unknown property id:%x\n", pkt->data[0]);
return;
}
parser_init(inst, &codecs, &domain);
+ core->codecs_count = 0;
+ memset(core->caps, 0, sizeof(core->caps));
+
while (words_count) {
data = word + 1;
};
static bool venus_pkt_debug;
-static int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
+int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
static bool venus_sys_idle_indicator;
static bool venus_fw_low_power_mode = true;
static int venus_hw_rsp_timeout = 1000;
minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;
- dev_dbg(dev, "venus hw version %x.%x.%x\n", major, minor, step);
+ dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step);
return major;
}
if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
struct hfi_msg_sys_debug_pkt *pkt = packet;
- dev_dbg(dev, "%s", pkt->msg_data);
+ dev_dbg(dev, VDBGFW "%s", pkt->msg_data);
}
}
}
venus_set_state(hdev, VENUS_STATE_DEINIT);
- /*
- * Once SYS_ERROR received from HW, it is safe to halt the AXI.
- * With SYS_ERROR, Venus FW may have crashed and HW might be
- * active and causing unnecessary transactions. Hence it is
- * safe to stop all AXI transactions from venus subsystem.
- */
- venus_halt_axi(hdev);
venus_sfr_print(hdev);
}
res = hdev->core->res;
pkt = hdev->pkt_buf;
- if (hdev->irq_status & WRAPPER_INTR_STATUS_A2HWD_MASK) {
- venus_sfr_print(hdev);
- hfi_process_watchdog_timeout(core);
- }
while (!venus_iface_msgq_read(hdev, pkt)) {
msg_ret = hfi_process_msg_packet(core, pkt);
struct hfi_session_init_pkt pkt;
int ret;
+ ret = venus_sys_set_debug(hdev, venus_fw_debug);
+ if (ret)
+ goto err;
+
ret = pkt_session_init(&pkt, inst, session_type, codec);
if (ret)
goto err;
core->ops = NULL;
return ret;
}
+
+void venus_hfi_queues_reinit(struct venus_core *core)
+{
+ struct venus_hfi_device *hdev = to_hfi_priv(core);
+ struct hfi_queue_table_header *tbl_hdr;
+ struct iface_queue *queue;
+ struct hfi_sfr *sfr;
+ unsigned int i;
+
+ mutex_lock(&hdev->lock);
+
+ for (i = 0; i < IFACEQ_NUM; i++) {
+ queue = &hdev->queues[i];
+ queue->qhdr =
+ IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
+
+ venus_set_qhdr_defaults(queue->qhdr);
+
+ queue->qhdr->start_addr = queue->qmem.da;
+
+ if (i == IFACEQ_CMD_IDX)
+ queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
+ else if (i == IFACEQ_MSG_IDX)
+ queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
+ else if (i == IFACEQ_DBG_IDX)
+ queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
+ }
+
+ tbl_hdr = hdev->ifaceq_table.kva;
+ tbl_hdr->version = 0;
+ tbl_hdr->size = IFACEQ_TABLE_SIZE;
+ tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
+ tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
+ tbl_hdr->num_q = IFACEQ_NUM;
+ tbl_hdr->num_active_q = IFACEQ_NUM;
+
+ /*
+ * Set receive request to zero on debug queue as there is no
+ * need of interrupt from video hardware for debug messages
+ */
+ queue = &hdev->queues[IFACEQ_DBG_IDX];
+ queue->qhdr->rx_req = 0;
+
+ sfr = hdev->sfr.kva;
+ sfr->buf_size = ALIGNED_SFR_SIZE;
+
+ /* ensure table and queue header structs are settled in memory */
+ wmb();
+
+ mutex_unlock(&hdev->lock);
+}
void venus_hfi_destroy(struct venus_core *core);
int venus_hfi_create(struct venus_core *core);
+void venus_hfi_queues_reinit(struct venus_core *core);
#endif
#include <linux/iopoll.h>
#include <linux/kernel.h>
#include <linux/pm_domain.h>
+#include <linux/pm_opp.h>
#include <linux/pm_runtime.h>
#include <linux/types.h>
#include <media/v4l2-mem2mem.h>
static int core_clks_set_rate(struct venus_core *core, unsigned long freq)
{
- struct clk *clk = core->clks[0];
int ret;
- ret = clk_set_rate(clk, freq);
+ ret = dev_pm_opp_set_rate(core->dev, freq);
if (ret)
return ret;
}
mutex_unlock(&core->lock);
- dev_dbg(core->dev, "total: avg_bw: %u, peak_bw: %u\n",
+ dev_dbg(core->dev, VDBGL "total: avg_bw: %u, peak_bw: %u\n",
total_avg, total_peak);
return icc_set_bw(core->video_path, total_avg, total_peak);
static int vcodec_domains_get(struct device *dev)
{
+ int ret;
+ struct opp_table *opp_table;
+ struct device **opp_virt_dev;
struct venus_core *core = dev_get_drvdata(dev);
const struct venus_resources *res = core->res;
struct device *pd;
unsigned int i;
if (!res->vcodec_pmdomains_num)
- return -ENODEV;
+ goto skip_pmdomains;
for (i = 0; i < res->vcodec_pmdomains_num; i++) {
pd = dev_pm_domain_attach_by_name(dev,
if (!core->pd_dl_venus)
return -ENODEV;
+skip_pmdomains:
+ if (!core->has_opp_table)
+ return 0;
+
+ /* Attach the power domain for setting performance state */
+ opp_table = dev_pm_opp_attach_genpd(dev, res->opp_pmdomain, &opp_virt_dev);
+ if (IS_ERR(opp_table)) {
+ ret = PTR_ERR(opp_table);
+ goto opp_attach_err;
+ }
+
+ core->opp_pmdomain = *opp_virt_dev;
+ core->opp_dl_venus = device_link_add(dev, core->opp_pmdomain,
+ DL_FLAG_RPM_ACTIVE |
+ DL_FLAG_PM_RUNTIME |
+ DL_FLAG_STATELESS);
+ if (!core->opp_dl_venus) {
+ ret = -ENODEV;
+ goto opp_dl_add_err;
+ }
+
return 0;
+
+opp_dl_add_err:
+ dev_pm_domain_detach(core->opp_pmdomain, true);
+opp_attach_err:
+ if (core->pd_dl_venus) {
+ device_link_del(core->pd_dl_venus);
+ for (i = 0; i < res->vcodec_pmdomains_num; i++) {
+ if (IS_ERR_OR_NULL(core->pmdomains[i]))
+ continue;
+ dev_pm_domain_detach(core->pmdomains[i], true);
+ }
+ }
+ return ret;
}
static void vcodec_domains_put(struct device *dev)
unsigned int i;
if (!res->vcodec_pmdomains_num)
- return;
+ goto skip_pmdomains;
if (core->pd_dl_venus)
device_link_del(core->pd_dl_venus);
continue;
dev_pm_domain_detach(core->pmdomains[i], true);
}
+
+skip_pmdomains:
+ if (!core->has_opp_table)
+ return;
+
+ if (core->opp_dl_venus)
+ device_link_del(core->opp_dl_venus);
+
+ dev_pm_domain_detach(core->opp_pmdomain, true);
}
static int core_get_v4(struct device *dev)
if (legacy_binding)
return 0;
+ core->opp_table = dev_pm_opp_set_clkname(dev, "core");
+ if (IS_ERR(core->opp_table))
+ return PTR_ERR(core->opp_table);
+
+ if (core->res->opp_pmdomain) {
+ ret = dev_pm_opp_of_add_table(dev);
+ if (!ret) {
+ core->has_opp_table = true;
+ } else if (ret != -ENODEV) {
+ dev_err(dev, "invalid OPP table in device tree\n");
+ dev_pm_opp_put_clkname(core->opp_table);
+ return ret;
+ }
+ }
+
ret = vcodec_domains_get(dev);
- if (ret)
+ if (ret) {
+ if (core->has_opp_table)
+ dev_pm_opp_of_remove_table(dev);
+ dev_pm_opp_put_clkname(core->opp_table);
return ret;
+ }
return 0;
}
static void core_put_v4(struct device *dev)
{
+ struct venus_core *core = dev_get_drvdata(dev);
+
if (legacy_binding)
return;
vcodec_domains_put(dev);
+
+ if (core->has_opp_table)
+ dev_pm_opp_of_remove_table(dev);
+ if (core->opp_table)
+ dev_pm_opp_put_clkname(core->opp_table);
+
}
static int core_power_v4(struct device *dev, int on)
struct venus_core *core = dev_get_drvdata(dev);
int ret = 0;
- if (on == POWER_ON)
+ if (on == POWER_ON) {
ret = core_clks_enable(core);
- else
+ } else {
+ /* Drop the performance state vote */
+ if (core->opp_pmdomain)
+ dev_pm_opp_set_rate(dev, 0);
+
core_clks_disable(core);
+ }
return ret;
}
if (!(inst->codec_state == VENUS_DEC_STATE_CAPTURE_SETUP) ||
!inst->reconfig)
- dev_dbg(inst->core->dev, "%s: wrong state\n", __func__);
+ dev_dbg(inst->core->dev, VDBGH "wrong state\n");
done:
return 0;
switch (inst->codec_state) {
case VENUS_DEC_STATE_DECODING:
ret = hfi_session_flush(inst, HFI_FLUSH_ALL, true);
- /* fallthrough */
+ fallthrough;
case VENUS_DEC_STATE_DRAIN:
vdec_cancel_dst_buffers(inst);
inst->codec_state = VENUS_DEC_STATE_STOPPED;
break;
}
- INIT_LIST_HEAD(&inst->registeredbufs);
-
return ret;
}
static void vdec_buf_cleanup(struct vb2_buffer *vb)
{
struct venus_inst *inst = vb2_get_drv_priv(vb->vb2_queue);
+ struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+ struct venus_buffer *buf = to_venus_buffer(vbuf);
+
+ mutex_lock(&inst->lock);
+ if (vb->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
+ if (!list_empty(&inst->registeredbufs))
+ list_del_init(&buf->reg_list);
+ mutex_unlock(&inst->lock);
inst->buf_count--;
if (!inst->buf_count)
if (inst->bit_depth != ev_data->bit_depth)
inst->bit_depth = ev_data->bit_depth;
- dev_dbg(dev, "event %s sufficient resources (%ux%u)\n",
+ dev_dbg(dev, VDBGM "event %s sufficient resources (%ux%u)\n",
sufficient ? "" : "not", ev_data->width, ev_data->height);
if (sufficient) {
ret = hfi_session_flush(inst, HFI_FLUSH_OUTPUT, false);
if (ret)
- dev_dbg(dev, "flush output error %d\n", ret);
+ dev_dbg(dev, VDBGH "flush output error %d\n", ret);
}
inst->reconfig = true;
dst_vq->allow_zero_bytesused = 1;
dst_vq->min_buffers_needed = 0;
dst_vq->dev = inst->core->dev;
- ret = vb2_queue_init(dst_vq);
- if (ret) {
- vb2_queue_release(src_vq);
- return ret;
- }
-
- return 0;
+ return vb2_queue_init(dst_vq);
}
static int vdec_open(struct file *file)
case V4L2_CID_MPEG_VIDEO_H264_PROFILE:
case V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE:
case V4L2_CID_MPEG_VIDEO_VP8_PROFILE:
+ case V4L2_CID_MPEG_VIDEO_VP9_PROFILE:
ctr->profile = ctrl->val;
break;
case V4L2_CID_MPEG_VIDEO_H264_LEVEL:
case V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL:
+ case V4L2_CID_MPEG_VIDEO_VP9_LEVEL:
ctr->level = ctrl->val;
break;
default:
struct venus_inst *inst = ctrl_to_inst(ctrl);
struct vdec_controls *ctr = &inst->controls.dec;
struct hfi_buffer_requirements bufreq;
- union hfi_get_property hprop;
enum hfi_version ver = inst->core->res->hfi_version;
- u32 ptype = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT;
+ u32 profile, level;
int ret;
switch (ctrl->id) {
case V4L2_CID_MPEG_VIDEO_H264_PROFILE:
case V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE:
case V4L2_CID_MPEG_VIDEO_VP8_PROFILE:
- ret = hfi_session_get_property(inst, ptype, &hprop);
+ case V4L2_CID_MPEG_VIDEO_VP9_PROFILE:
+ ret = venus_helper_get_profile_level(inst, &profile, &level);
if (!ret)
- ctr->profile = hprop.profile_level.profile;
+ ctr->profile = profile;
ctrl->val = ctr->profile;
break;
case V4L2_CID_MPEG_VIDEO_H264_LEVEL:
case V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL:
- ret = hfi_session_get_property(inst, ptype, &hprop);
+ case V4L2_CID_MPEG_VIDEO_VP9_LEVEL:
+ ret = venus_helper_get_profile_level(inst, &profile, &level);
if (!ret)
- ctr->level = hprop.profile_level.level;
+ ctr->level = level;
ctrl->val = ctr->level;
break;
case V4L2_CID_MPEG_VIDEO_DECODER_MPEG4_DEBLOCK_FILTER:
struct v4l2_ctrl *ctrl;
int ret;
- ret = v4l2_ctrl_handler_init(&inst->ctrl_handler, 7);
+ ret = v4l2_ctrl_handler_init(&inst->ctrl_handler, 9);
if (ret)
return ret;
if (ctrl)
ctrl->flags |= V4L2_CTRL_FLAG_VOLATILE;
+ ctrl = v4l2_ctrl_new_std_menu(&inst->ctrl_handler, &vdec_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_VP9_PROFILE,
+ V4L2_MPEG_VIDEO_VP9_PROFILE_3,
+ 0, V4L2_MPEG_VIDEO_VP9_PROFILE_0);
+ if (ctrl)
+ ctrl->flags |= V4L2_CTRL_FLAG_VOLATILE;
+
+ ctrl = v4l2_ctrl_new_std_menu(&inst->ctrl_handler, &vdec_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_VP9_LEVEL,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_6_2,
+ 0, V4L2_MPEG_VIDEO_VP9_LEVEL_1_0);
+ if (ctrl)
+ ctrl->flags |= V4L2_CTRL_FLAG_VOLATILE;
+
v4l2_ctrl_new_std(&inst->ctrl_handler, &vdec_ctrl_ops,
V4L2_CID_MPEG_VIDEO_DECODER_MPEG4_DEBLOCK_FILTER, 0, 1, 1, 0);
static int venc_v4l2_to_hfi(int id, int value)
{
switch (id) {
- case V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL:
- switch (value) {
- case V4L2_MPEG_VIDEO_MPEG4_LEVEL_0:
- default:
- return HFI_MPEG4_LEVEL_0;
- case V4L2_MPEG_VIDEO_MPEG4_LEVEL_0B:
- return HFI_MPEG4_LEVEL_0b;
- case V4L2_MPEG_VIDEO_MPEG4_LEVEL_1:
- return HFI_MPEG4_LEVEL_1;
- case V4L2_MPEG_VIDEO_MPEG4_LEVEL_2:
- return HFI_MPEG4_LEVEL_2;
- case V4L2_MPEG_VIDEO_MPEG4_LEVEL_3:
- return HFI_MPEG4_LEVEL_3;
- case V4L2_MPEG_VIDEO_MPEG4_LEVEL_4:
- return HFI_MPEG4_LEVEL_4;
- case V4L2_MPEG_VIDEO_MPEG4_LEVEL_5:
- return HFI_MPEG4_LEVEL_5;
- }
- case V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE:
- switch (value) {
- case V4L2_MPEG_VIDEO_MPEG4_PROFILE_SIMPLE:
- default:
- return HFI_MPEG4_PROFILE_SIMPLE;
- case V4L2_MPEG_VIDEO_MPEG4_PROFILE_ADVANCED_SIMPLE:
- return HFI_MPEG4_PROFILE_ADVANCEDSIMPLE;
- }
- case V4L2_CID_MPEG_VIDEO_H264_PROFILE:
- switch (value) {
- case V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE:
- return HFI_H264_PROFILE_BASELINE;
- case V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE:
- return HFI_H264_PROFILE_CONSTRAINED_BASE;
- case V4L2_MPEG_VIDEO_H264_PROFILE_MAIN:
- return HFI_H264_PROFILE_MAIN;
- case V4L2_MPEG_VIDEO_H264_PROFILE_HIGH:
- default:
- return HFI_H264_PROFILE_HIGH;
- }
- case V4L2_CID_MPEG_VIDEO_H264_LEVEL:
- switch (value) {
- case V4L2_MPEG_VIDEO_H264_LEVEL_1_0:
- return HFI_H264_LEVEL_1;
- case V4L2_MPEG_VIDEO_H264_LEVEL_1B:
- return HFI_H264_LEVEL_1b;
- case V4L2_MPEG_VIDEO_H264_LEVEL_1_1:
- return HFI_H264_LEVEL_11;
- case V4L2_MPEG_VIDEO_H264_LEVEL_1_2:
- return HFI_H264_LEVEL_12;
- case V4L2_MPEG_VIDEO_H264_LEVEL_1_3:
- return HFI_H264_LEVEL_13;
- case V4L2_MPEG_VIDEO_H264_LEVEL_2_0:
- return HFI_H264_LEVEL_2;
- case V4L2_MPEG_VIDEO_H264_LEVEL_2_1:
- return HFI_H264_LEVEL_21;
- case V4L2_MPEG_VIDEO_H264_LEVEL_2_2:
- return HFI_H264_LEVEL_22;
- case V4L2_MPEG_VIDEO_H264_LEVEL_3_0:
- return HFI_H264_LEVEL_3;
- case V4L2_MPEG_VIDEO_H264_LEVEL_3_1:
- return HFI_H264_LEVEL_31;
- case V4L2_MPEG_VIDEO_H264_LEVEL_3_2:
- return HFI_H264_LEVEL_32;
- case V4L2_MPEG_VIDEO_H264_LEVEL_4_0:
- return HFI_H264_LEVEL_4;
- case V4L2_MPEG_VIDEO_H264_LEVEL_4_1:
- return HFI_H264_LEVEL_41;
- case V4L2_MPEG_VIDEO_H264_LEVEL_4_2:
- return HFI_H264_LEVEL_42;
- case V4L2_MPEG_VIDEO_H264_LEVEL_5_0:
- default:
- return HFI_H264_LEVEL_5;
- case V4L2_MPEG_VIDEO_H264_LEVEL_5_1:
- return HFI_H264_LEVEL_51;
- }
case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE:
switch (value) {
case V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC:
case V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC:
return HFI_H264_ENTROPY_CABAC;
}
- case V4L2_CID_MPEG_VIDEO_VP8_PROFILE:
- switch (value) {
- case 0:
- default:
- return HFI_VPX_PROFILE_VERSION_0;
- case 1:
- return HFI_VPX_PROFILE_VERSION_1;
- case 2:
- return HFI_VPX_PROFILE_VERSION_2;
- case 3:
- return HFI_VPX_PROFILE_VERSION_3;
- }
case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE:
switch (value) {
case V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_ENABLED:
case V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_DISABLED_AT_SLICE_BOUNDARY:
return HFI_H264_DB_MODE_SKIP_SLICE_BOUNDARY;
}
- case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE:
- switch (value) {
- case V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN:
- default:
- return HFI_HEVC_PROFILE_MAIN;
- case V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE:
- return HFI_HEVC_PROFILE_MAIN_STILL_PIC;
- case V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10:
- return HFI_HEVC_PROFILE_MAIN10;
- }
- case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL:
- switch (value) {
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_1:
- default:
- return HFI_HEVC_LEVEL_1;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_2:
- return HFI_HEVC_LEVEL_2;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_2_1:
- return HFI_HEVC_LEVEL_21;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_3:
- return HFI_HEVC_LEVEL_3;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_3_1:
- return HFI_HEVC_LEVEL_31;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_4:
- return HFI_HEVC_LEVEL_4;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1:
- return HFI_HEVC_LEVEL_41;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_5:
- return HFI_HEVC_LEVEL_5;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1:
- return HFI_HEVC_LEVEL_51;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_5_2:
- return HFI_HEVC_LEVEL_52;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_6:
- return HFI_HEVC_LEVEL_6;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1:
- return HFI_HEVC_LEVEL_61;
- case V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2:
- return HFI_HEVC_LEVEL_62;
- }
}
return 0;
{
struct venus_inst *inst = to_inst(file);
const struct venus_format *fmt;
+ unsigned int framerate_factor = 1;
fival->type = V4L2_FRMIVAL_TYPE_STEPWISE;
fival->height < frame_height_min(inst))
return -EINVAL;
+ if (IS_V1(inst->core)) {
+ /* framerate is reported in 1/65535 fps unit */
+ framerate_factor = (1 << 16);
+ }
+
fival->stepwise.min.numerator = 1;
- fival->stepwise.min.denominator = frate_max(inst);
+ fival->stepwise.min.denominator = frate_max(inst) / framerate_factor;
fival->stepwise.max.numerator = 1;
- fival->stepwise.max.denominator = frate_min(inst);
+ fival->stepwise.max.denominator = frate_min(inst) / framerate_factor;
fival->stepwise.step.numerator = 1;
- fival->stepwise.step.denominator = frate_max(inst);
+ fival->stepwise.step.denominator = frate_max(inst) / framerate_factor;
return 0;
}
{
struct venc_controls *ctr = &inst->controls.enc;
struct hfi_intra_period intra_period;
- struct hfi_profile_level pl;
struct hfi_framerate frate;
struct hfi_bitrate brate;
struct hfi_idr_period idrp;
struct hfi_quantization quant;
struct hfi_quantization_range quant_range;
- u32 ptype, rate_control, bitrate, profile = 0, level = 0;
+ u32 ptype, rate_control, bitrate;
int ret;
ret = venus_helper_set_work_mode(inst, VIDC_WORK_MODE_2);
if (!ctr->rc_enable)
rate_control = HFI_RATE_CONTROL_OFF;
else if (ctr->bitrate_mode == V4L2_MPEG_VIDEO_BITRATE_MODE_VBR)
- rate_control = HFI_RATE_CONTROL_VBR_CFR;
- else
- rate_control = HFI_RATE_CONTROL_CBR_CFR;
+ rate_control = ctr->frame_skip_mode ? HFI_RATE_CONTROL_VBR_VFR :
+ HFI_RATE_CONTROL_VBR_CFR;
+ else if (ctr->bitrate_mode == V4L2_MPEG_VIDEO_BITRATE_MODE_CBR)
+ rate_control = ctr->frame_skip_mode ? HFI_RATE_CONTROL_CBR_VFR :
+ HFI_RATE_CONTROL_CBR_CFR;
+ else if (ctr->bitrate_mode == V4L2_MPEG_VIDEO_BITRATE_MODE_CQ)
+ rate_control = HFI_RATE_CONTROL_CQ;
ptype = HFI_PROPERTY_PARAM_VENC_RATE_CONTROL;
ret = hfi_session_set_property(inst, ptype, &rate_control);
if (ret)
return ret;
+ if (rate_control == HFI_RATE_CONTROL_CQ && ctr->const_quality) {
+ struct hfi_heic_frame_quality quality = {};
+
+ ptype = HFI_PROPERTY_CONFIG_HEIC_FRAME_QUALITY;
+ quality.frame_quality = ctr->const_quality;
+ ret = hfi_session_set_property(inst, ptype, &quality);
+ if (ret)
+ return ret;
+ }
+
if (!ctr->bitrate)
bitrate = 64000;
else
if (ret)
return ret;
- if (inst->fmt_cap->pixfmt == V4L2_PIX_FMT_H264) {
- profile = venc_v4l2_to_hfi(V4L2_CID_MPEG_VIDEO_H264_PROFILE,
- ctr->profile.h264);
- level = venc_v4l2_to_hfi(V4L2_CID_MPEG_VIDEO_H264_LEVEL,
- ctr->level.h264);
- } else if (inst->fmt_cap->pixfmt == V4L2_PIX_FMT_VP8) {
- profile = venc_v4l2_to_hfi(V4L2_CID_MPEG_VIDEO_VP8_PROFILE,
- ctr->profile.vpx);
- level = 0;
- } else if (inst->fmt_cap->pixfmt == V4L2_PIX_FMT_MPEG4) {
- profile = venc_v4l2_to_hfi(V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE,
- ctr->profile.mpeg4);
- level = venc_v4l2_to_hfi(V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL,
- ctr->level.mpeg4);
- } else if (inst->fmt_cap->pixfmt == V4L2_PIX_FMT_H263) {
- profile = 0;
- level = 0;
- } else if (inst->fmt_cap->pixfmt == V4L2_PIX_FMT_HEVC) {
- profile = venc_v4l2_to_hfi(V4L2_CID_MPEG_VIDEO_HEVC_PROFILE,
- ctr->profile.hevc);
- level = venc_v4l2_to_hfi(V4L2_CID_MPEG_VIDEO_HEVC_LEVEL,
- ctr->level.hevc);
- }
-
- ptype = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT;
- pl.profile = profile;
- pl.level = level;
-
- ret = hfi_session_set_property(inst, ptype, &pl);
+ ret = venus_helper_set_profile_level(inst, ctr->profile, ctr->level);
if (ret)
return ret;
dst_vq->allow_zero_bytesused = 1;
dst_vq->min_buffers_needed = 1;
dst_vq->dev = inst->core->dev;
- ret = vb2_queue_init(dst_vq);
- if (ret) {
- vb2_queue_release(src_vq);
- return ret;
- }
-
- return 0;
+ return vb2_queue_init(dst_vq);
}
static void venc_inst_init(struct venus_inst *inst)
ctr->h264_entropy_mode = ctrl->val;
break;
case V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE:
- ctr->profile.mpeg4 = ctrl->val;
- break;
case V4L2_CID_MPEG_VIDEO_H264_PROFILE:
- ctr->profile.h264 = ctrl->val;
- break;
case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE:
- ctr->profile.hevc = ctrl->val;
- break;
case V4L2_CID_MPEG_VIDEO_VP8_PROFILE:
- ctr->profile.vpx = ctrl->val;
+ ctr->profile = ctrl->val;
break;
case V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL:
- ctr->level.mpeg4 = ctrl->val;
- break;
case V4L2_CID_MPEG_VIDEO_H264_LEVEL:
- ctr->level.h264 = ctrl->val;
- break;
case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL:
- ctr->level.hevc = ctrl->val;
+ ctr->level = ctrl->val;
break;
case V4L2_CID_MPEG_VIDEO_H264_I_FRAME_QP:
ctr->h264_i_qp = ctrl->val;
case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE:
ctr->rc_enable = ctrl->val;
break;
+ case V4L2_CID_MPEG_VIDEO_CONSTANT_QUALITY:
+ ctr->const_quality = ctrl->val;
+ break;
+ case V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE:
+ ctr->frame_skip_mode = ctrl->val;
+ break;
default:
return -EINVAL;
}
{
int ret;
- ret = v4l2_ctrl_handler_init(&inst->ctrl_handler, 31);
+ ret = v4l2_ctrl_handler_init(&inst->ctrl_handler, 33);
if (ret)
return ret;
V4L2_CID_MPEG_VIDEO_BITRATE_MODE,
V4L2_MPEG_VIDEO_BITRATE_MODE_CBR,
~((1 << V4L2_MPEG_VIDEO_BITRATE_MODE_VBR) |
- (1 << V4L2_MPEG_VIDEO_BITRATE_MODE_CBR)),
+ (1 << V4L2_MPEG_VIDEO_BITRATE_MODE_CBR) |
+ (1 << V4L2_MPEG_VIDEO_BITRATE_MODE_CQ)),
V4L2_MPEG_VIDEO_BITRATE_MODE_VBR);
v4l2_ctrl_new_std_menu(&inst->ctrl_handler, &venc_ctrl_ops,
v4l2_ctrl_new_std(&inst->ctrl_handler, &venc_ctrl_ops,
V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE, 0, 1, 1, 1);
+ v4l2_ctrl_new_std(&inst->ctrl_handler, &venc_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_CONSTANT_QUALITY, 0, 100, 1, 0);
+
+ v4l2_ctrl_new_std_menu(&inst->ctrl_handler, &venc_ctrl_ops,
+ V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE,
+ V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT,
+ ~((1 << V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED) |
+ (1 << V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT)),
+ V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED);
+
ret = inst->ctrl_handler.error;
if (ret)
goto err;
struct rcar_fcp_device {
struct list_head list;
struct device *dev;
- struct device_dma_parameters dma_parms;
};
static LIST_HEAD(fcp_devices);
return 0;
ret = pm_runtime_get_sync(fcp->dev);
- if (ret < 0)
+ if (ret < 0) {
+ pm_runtime_put_noidle(fcp->dev);
return ret;
+ }
return 0;
}
fcp->dev = &pdev->dev;
- fcp->dev->dma_parms = &fcp->dma_parms;
- dma_set_max_seg_size(fcp->dev, DMA_BIT_MASK(32));
+ dma_set_max_seg_size(fcp->dev, UINT_MAX);
pm_runtime_enable(&pdev->dev);
select V4L2_FWNODE
help
Support for Renesas R-Car MIPI CSI-2 receiver.
- Supports R-Car Gen3 SoCs.
+ Supports R-Car Gen3 and RZ/G2 SoCs.
To compile this driver as a module, choose M here: the
module will be called rcar-csi2.
select V4L2_FWNODE
help
Support for Renesas R-Car Video Input (VIN) driver.
- Supports R-Car Gen2 and Gen3 SoCs.
+ Supports R-Car Gen{2,3} and RZ/G{1,2} SoCs.
To compile this driver as a module, choose M here: the
module will be called rcar-vin.
static void rvin_group_cleanup(struct rvin_group *group)
{
- media_device_unregister(&group->mdev);
media_device_cleanup(&group->mdev);
mutex_destroy(&group->lock);
}
struct media_device *mdev = &group->mdev;
const struct of_device_id *match;
struct device_node *np;
- int ret;
mutex_init(&group->lock);
media_device_init(mdev);
- ret = media_device_register(&group->mdev);
- if (ret)
- rvin_group_cleanup(group);
-
- return ret;
+ return 0;
}
static void rvin_group_release(struct kref *kref)
switch (vin->parallel->mbus_type) {
case V4L2_MBUS_PARALLEL:
- vin_dbg(vin, "Found PARALLEL media bus\n");
- vin->parallel->mbus_flags = vep->bus.parallel.flags;
- break;
case V4L2_MBUS_BT656:
- vin_dbg(vin, "Found BT656 media bus\n");
- vin->parallel->mbus_flags = 0;
+ vin_dbg(vin, "Found %s media bus\n",
+ vin->parallel->mbus_type == V4L2_MBUS_PARALLEL ?
+ "PARALLEL" : "BT656");
+ vin->parallel->bus = vep->bus.parallel;
break;
default:
vin_err(vin, "Unknown media bus type\n");
unsigned int i;
int ret;
+ ret = media_device_register(&vin->group->mdev);
+ if (ret)
+ return ret;
+
ret = v4l2_device_register_subdev_nodes(&vin->v4l2_dev);
if (ret) {
vin_err(vin, "Failed to register subdev nodes\n");
}
mutex_unlock(&vin->group->lock);
+
+ media_device_unregister(&vin->group->mdev);
}
static int rvin_group_notify_bound(struct v4l2_async_notifier *notifier,
.max_height = 2048,
};
+static const struct rvin_group_route rcar_info_r8a774e1_routes[] = {
+ { .csi = RVIN_CSI40, .channel = 0, .vin = 0, .mask = BIT(0) | BIT(3) },
+ { .csi = RVIN_CSI20, .channel = 0, .vin = 0, .mask = BIT(1) | BIT(4) },
+ { .csi = RVIN_CSI40, .channel = 1, .vin = 0, .mask = BIT(2) },
+ { .csi = RVIN_CSI20, .channel = 0, .vin = 1, .mask = BIT(0) },
+ { .csi = RVIN_CSI40, .channel = 1, .vin = 1, .mask = BIT(1) | BIT(3) },
+ { .csi = RVIN_CSI40, .channel = 0, .vin = 1, .mask = BIT(2) },
+ { .csi = RVIN_CSI20, .channel = 1, .vin = 1, .mask = BIT(4) },
+ { .csi = RVIN_CSI20, .channel = 1, .vin = 2, .mask = BIT(0) },
+ { .csi = RVIN_CSI40, .channel = 0, .vin = 2, .mask = BIT(1) },
+ { .csi = RVIN_CSI20, .channel = 0, .vin = 2, .mask = BIT(2) },
+ { .csi = RVIN_CSI40, .channel = 2, .vin = 2, .mask = BIT(3) },
+ { .csi = RVIN_CSI20, .channel = 2, .vin = 2, .mask = BIT(4) },
+ { .csi = RVIN_CSI40, .channel = 1, .vin = 3, .mask = BIT(0) },
+ { .csi = RVIN_CSI20, .channel = 1, .vin = 3, .mask = BIT(1) | BIT(2) },
+ { .csi = RVIN_CSI40, .channel = 3, .vin = 3, .mask = BIT(3) },
+ { .csi = RVIN_CSI20, .channel = 3, .vin = 3, .mask = BIT(4) },
+ { .csi = RVIN_CSI20, .channel = 0, .vin = 4, .mask = BIT(1) | BIT(4) },
+ { .csi = RVIN_CSI20, .channel = 0, .vin = 5, .mask = BIT(0) },
+ { .csi = RVIN_CSI20, .channel = 1, .vin = 5, .mask = BIT(4) },
+ { .csi = RVIN_CSI20, .channel = 1, .vin = 6, .mask = BIT(0) },
+ { .csi = RVIN_CSI20, .channel = 0, .vin = 6, .mask = BIT(2) },
+ { .csi = RVIN_CSI20, .channel = 2, .vin = 6, .mask = BIT(4) },
+ { .csi = RVIN_CSI20, .channel = 1, .vin = 7, .mask = BIT(1) | BIT(2) },
+ { .csi = RVIN_CSI20, .channel = 3, .vin = 7, .mask = BIT(4) },
+ { /* Sentinel */ }
+};
+
+static const struct rvin_info rcar_info_r8a774e1 = {
+ .model = RCAR_GEN3,
+ .use_mc = true,
+ .max_width = 4096,
+ .max_height = 4096,
+ .routes = rcar_info_r8a774e1_routes,
+};
+
static const struct rvin_group_route rcar_info_r8a7795_routes[] = {
{ .csi = RVIN_CSI40, .channel = 0, .vin = 0, .mask = BIT(0) | BIT(3) },
{ .csi = RVIN_CSI20, .channel = 0, .vin = 0, .mask = BIT(1) | BIT(4) },
.compatible = "renesas,vin-r8a774c0",
.data = &rcar_info_r8a77990,
},
+ {
+ .compatible = "renesas,vin-r8a774e1",
+ .data = &rcar_info_r8a774e1,
+ },
{
.compatible = "renesas,vin-r8a7778",
.data = &rcar_info_m1,
v4l2_async_notifier_cleanup(&vin->notifier);
if (vin->info->use_mc) {
- mutex_lock(&vin->group->lock);
- if (&vin->v4l2_dev == vin->group->notifier.v4l2_dev) {
- v4l2_async_notifier_unregister(&vin->group->notifier);
- v4l2_async_notifier_cleanup(&vin->group->notifier);
- }
- mutex_unlock(&vin->group->lock);
+ v4l2_async_notifier_unregister(&vin->group->notifier);
+ v4l2_async_notifier_cleanup(&vin->group->notifier);
rvin_group_put(vin);
}
{ .code = MEDIA_BUS_FMT_YUYV8_1X16, .datatype = 0x1e, .bpp = 16 },
{ .code = MEDIA_BUS_FMT_UYVY8_2X8, .datatype = 0x1e, .bpp = 16 },
{ .code = MEDIA_BUS_FMT_YUYV10_2X10, .datatype = 0x1e, .bpp = 20 },
+ { .code = MEDIA_BUS_FMT_SBGGR8_1X8, .datatype = 0x2a, .bpp = 8 },
+ { .code = MEDIA_BUS_FMT_SGBRG8_1X8, .datatype = 0x2a, .bpp = 8 },
+ { .code = MEDIA_BUS_FMT_SGRBG8_1X8, .datatype = 0x2a, .bpp = 8 },
{ .code = MEDIA_BUS_FMT_SRGGB8_1X8, .datatype = 0x2a, .bpp = 8 },
};
struct media_pad pads[NR_OF_RCAR_CSI2_PAD];
struct v4l2_async_notifier notifier;
- struct v4l2_async_subdev asd;
struct v4l2_subdev *remote;
+ unsigned int remote_pad;
struct v4l2_mbus_framefmt mf;
reset_control_deassert(priv->rstc);
}
-static int rcsi2_wait_phy_start(struct rcar_csi2 *priv)
+static int rcsi2_wait_phy_start(struct rcar_csi2 *priv,
+ unsigned int lanes)
{
unsigned int timeout;
/* Wait for the clock and data lanes to enter LP-11 state. */
for (timeout = 0; timeout <= 20; timeout++) {
- const u32 lane_mask = (1 << priv->lanes) - 1;
+ const u32 lane_mask = (1 << lanes) - 1;
if ((rcsi2_read(priv, PHCLM_REG) & PHCLM_STOPSTATECKL) &&
(rcsi2_read(priv, PHDLM_REG) & lane_mask) == lane_mask)
return 0;
}
-static int rcsi2_calc_mbps(struct rcar_csi2 *priv, unsigned int bpp)
+static int rcsi2_calc_mbps(struct rcar_csi2 *priv, unsigned int bpp,
+ unsigned int lanes)
{
struct v4l2_subdev *source;
struct v4l2_ctrl *ctrl;
* bps = link_freq * 2
*/
mbps = v4l2_ctrl_g_ctrl_int64(ctrl) * bpp;
- do_div(mbps, priv->lanes * 1000000);
+ do_div(mbps, lanes * 1000000);
return mbps;
}
+static int rcsi2_get_active_lanes(struct rcar_csi2 *priv,
+ unsigned int *lanes)
+{
+ struct v4l2_mbus_config mbus_config = { 0 };
+ unsigned int num_lanes = UINT_MAX;
+ int ret;
+
+ *lanes = priv->lanes;
+
+ ret = v4l2_subdev_call(priv->remote, pad, get_mbus_config,
+ priv->remote_pad, &mbus_config);
+ if (ret == -ENOIOCTLCMD) {
+ dev_dbg(priv->dev, "No remote mbus configuration available\n");
+ return 0;
+ }
+
+ if (ret) {
+ dev_err(priv->dev, "Failed to get remote mbus configuration\n");
+ return ret;
+ }
+
+ if (mbus_config.type != V4L2_MBUS_CSI2_DPHY) {
+ dev_err(priv->dev, "Unsupported media bus type %u\n",
+ mbus_config.type);
+ return -EINVAL;
+ }
+
+ if (mbus_config.flags & V4L2_MBUS_CSI2_1_LANE)
+ num_lanes = 1;
+ else if (mbus_config.flags & V4L2_MBUS_CSI2_2_LANE)
+ num_lanes = 2;
+ else if (mbus_config.flags & V4L2_MBUS_CSI2_3_LANE)
+ num_lanes = 3;
+ else if (mbus_config.flags & V4L2_MBUS_CSI2_4_LANE)
+ num_lanes = 4;
+
+ if (num_lanes > priv->lanes) {
+ dev_err(priv->dev,
+ "Unsupported mbus config: too many data lanes %u\n",
+ num_lanes);
+ return -EINVAL;
+ }
+
+ *lanes = num_lanes;
+
+ return 0;
+}
+
static int rcsi2_start_receiver(struct rcar_csi2 *priv)
{
const struct rcar_csi2_format *format;
u32 phycnt, vcdt = 0, vcdt2 = 0, fld = 0;
+ unsigned int lanes;
unsigned int i;
int mbps, ret;
fld |= FLD_FLD_NUM(1);
}
+ /*
+ * Get the number of active data lanes inspecting the remote mbus
+ * configuration.
+ */
+ ret = rcsi2_get_active_lanes(priv, &lanes);
+ if (ret)
+ return ret;
+
phycnt = PHYCNT_ENABLECLK;
- phycnt |= (1 << priv->lanes) - 1;
+ phycnt |= (1 << lanes) - 1;
- mbps = rcsi2_calc_mbps(priv, format->bpp);
+ mbps = rcsi2_calc_mbps(priv, format->bpp, lanes);
if (mbps < 0)
return mbps;
rcsi2_write(priv, PHYCNT_REG, phycnt | PHYCNT_SHUTDOWNZ);
rcsi2_write(priv, PHYCNT_REG, phycnt | PHYCNT_SHUTDOWNZ | PHYCNT_RSTZ);
- ret = rcsi2_wait_phy_start(priv);
+ ret = rcsi2_wait_phy_start(priv, lanes);
if (ret)
return ret;
}
priv->remote = subdev;
+ priv->remote_pad = pad;
dev_dbg(priv->dev, "Bound %s pad: %d\n", subdev->name, pad);
static int rcsi2_parse_dt(struct rcar_csi2 *priv)
{
+ struct v4l2_async_subdev *asd;
+ struct fwnode_handle *fwnode;
struct device_node *ep;
struct v4l2_fwnode_endpoint v4l2_ep = { .bus_type = 0 };
int ret;
return ret;
}
- priv->asd.match.fwnode =
- fwnode_graph_get_remote_endpoint(of_fwnode_handle(ep));
- priv->asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
-
+ fwnode = fwnode_graph_get_remote_endpoint(of_fwnode_handle(ep));
of_node_put(ep);
- v4l2_async_notifier_init(&priv->notifier);
-
- ret = v4l2_async_notifier_add_subdev(&priv->notifier, &priv->asd);
- if (ret) {
- fwnode_handle_put(priv->asd.match.fwnode);
- return ret;
- }
+ dev_dbg(priv->dev, "Found '%pOF'\n", to_of_node(fwnode));
+ v4l2_async_notifier_init(&priv->notifier);
priv->notifier.ops = &rcar_csi2_notify_ops;
- dev_dbg(priv->dev, "Found '%pOF'\n",
- to_of_node(priv->asd.match.fwnode));
+ asd = v4l2_async_notifier_add_fwnode_subdev(&priv->notifier, fwnode,
+ sizeof(*asd));
+ fwnode_handle_put(fwnode);
+ if (IS_ERR(asd))
+ return PTR_ERR(asd);
ret = v4l2_async_subdev_notifier_register(&priv->subdev,
&priv->notifier);
.compatible = "renesas,r8a774c0-csi2",
.data = &rcar_csi2_info_r8a77990,
},
+ {
+ .compatible = "renesas,r8a774e1-csi2",
+ .data = &rcar_csi2_info_r8a7795,
+ },
{
.compatible = "renesas,r8a7795-csi2",
.data = &rcar_csi2_info_r8a7795,
#define VNDMR2_VPS (1 << 30)
#define VNDMR2_HPS (1 << 29)
#define VNDMR2_CES (1 << 28)
+#define VNDMR2_YDS (1 << 22)
#define VNDMR2_FTEV (1 << 17)
#define VNDMR2_VLV(n) ((n & 0xf) << 12)
/* For RAW8 format bpp is 1, but the hardware process RAW8
* format in 2 pixel unit hence configure VNIS_REG as stride / 2.
*/
- if (vin->format.pixelformat == V4L2_PIX_FMT_SRGGB8)
+ switch (vin->format.pixelformat) {
+ case V4L2_PIX_FMT_SBGGR8:
+ case V4L2_PIX_FMT_SGBRG8:
+ case V4L2_PIX_FMT_SGRBG8:
+ case V4L2_PIX_FMT_SRGGB8:
stride /= 2;
+ break;
+ default:
+ break;
+ }
rvin_write(vin, stride, VNIS_REG);
}
input_is_yuv = true;
break;
+ case MEDIA_BUS_FMT_SBGGR8_1X8:
+ case MEDIA_BUS_FMT_SGBRG8_1X8:
+ case MEDIA_BUS_FMT_SGRBG8_1X8:
case MEDIA_BUS_FMT_SRGGB8_1X8:
vnmc |= VNMC_INF_RAW8;
break;
if (!vin->is_csi) {
/* Hsync Signal Polarity Select */
- if (!(vin->parallel->mbus_flags & V4L2_MBUS_HSYNC_ACTIVE_LOW))
+ if (!(vin->parallel->bus.flags & V4L2_MBUS_HSYNC_ACTIVE_LOW))
dmr2 |= VNDMR2_HPS;
/* Vsync Signal Polarity Select */
- if (!(vin->parallel->mbus_flags & V4L2_MBUS_VSYNC_ACTIVE_LOW))
+ if (!(vin->parallel->bus.flags & V4L2_MBUS_VSYNC_ACTIVE_LOW))
dmr2 |= VNDMR2_VPS;
/* Data Enable Polarity Select */
- if (vin->parallel->mbus_flags & V4L2_MBUS_DATA_ENABLE_LOW)
+ if (vin->parallel->bus.flags & V4L2_MBUS_DATA_ENABLE_LOW)
dmr2 |= VNDMR2_CES;
+
+ switch (vin->mbus_code) {
+ case MEDIA_BUS_FMT_UYVY8_2X8:
+ if (vin->parallel->bus.bus_width == 8 &&
+ vin->parallel->bus.data_shift == 8)
+ dmr2 |= VNDMR2_YDS;
+ break;
+ default:
+ break;
+ }
}
/*
case V4L2_PIX_FMT_ABGR32:
dmr = VNDMR_A8BIT(vin->alpha) | VNDMR_EXRGB | VNDMR_DTMD_ARGB;
break;
+ case V4L2_PIX_FMT_SBGGR8:
+ case V4L2_PIX_FMT_SGBRG8:
+ case V4L2_PIX_FMT_SGRBG8:
case V4L2_PIX_FMT_SRGGB8:
dmr = 0;
break;
case MEDIA_BUS_FMT_UYVY10_2X10:
case MEDIA_BUS_FMT_RGB888_1X24:
break;
+ case MEDIA_BUS_FMT_SBGGR8_1X8:
+ if (vin->format.pixelformat != V4L2_PIX_FMT_SBGGR8)
+ return -EPIPE;
+ break;
+ case MEDIA_BUS_FMT_SGBRG8_1X8:
+ if (vin->format.pixelformat != V4L2_PIX_FMT_SGBRG8)
+ return -EPIPE;
+ break;
+ case MEDIA_BUS_FMT_SGRBG8_1X8:
+ if (vin->format.pixelformat != V4L2_PIX_FMT_SGRBG8)
+ return -EPIPE;
+ break;
case MEDIA_BUS_FMT_SRGGB8_1X8:
if (vin->format.pixelformat != V4L2_PIX_FMT_SRGGB8)
return -EPIPE;
int ret;
ret = pm_runtime_get_sync(vin->dev);
- if (ret < 0)
+ if (ret < 0) {
+ pm_runtime_put_noidle(vin->dev);
return ret;
+ }
/* Make register writes take effect immediately. */
vnmc = rvin_read(vin, VNMC_REG);
.fourcc = V4L2_PIX_FMT_ABGR32,
.bpp = 4,
},
+ {
+ .fourcc = V4L2_PIX_FMT_SBGGR8,
+ .bpp = 1,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_SGBRG8,
+ .bpp = 1,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_SGRBG8,
+ .bpp = 1,
+ },
{
.fourcc = V4L2_PIX_FMT_SRGGB8,
.bpp = 1,
case MEDIA_BUS_FMT_UYVY10_2X10:
case MEDIA_BUS_FMT_RGB888_1X24:
break;
+ case MEDIA_BUS_FMT_SBGGR8_1X8:
+ if (f->index)
+ return -EINVAL;
+ f->pixelformat = V4L2_PIX_FMT_SBGGR8;
+ return 0;
+ case MEDIA_BUS_FMT_SGBRG8_1X8:
+ if (f->index)
+ return -EINVAL;
+ f->pixelformat = V4L2_PIX_FMT_SGBRG8;
+ return 0;
+ case MEDIA_BUS_FMT_SGRBG8_1X8:
+ if (f->index)
+ return -EINVAL;
+ f->pixelformat = V4L2_PIX_FMT_SGRBG8;
+ return 0;
case MEDIA_BUS_FMT_SRGGB8_1X8:
if (f->index)
return -EINVAL;
int ret;
ret = pm_runtime_get_sync(vin->dev);
- if (ret < 0)
+ if (ret < 0) {
+ pm_runtime_put_noidle(vin->dev);
return ret;
+ }
ret = mutex_lock_interruptible(&vin->lock);
if (ret)
#include <media/v4l2-ctrls.h>
#include <media/v4l2-dev.h>
#include <media/v4l2-device.h>
+#include <media/v4l2-fwnode.h>
#include <media/videobuf2-v4l2.h>
/* Number of HW buffers */
* @asd: sub-device descriptor for async framework
* @subdev: subdevice matched using async framework
* @mbus_type: media bus type
- * @mbus_flags: media bus configuration flags
+ * @bus: media bus parallel configuration
* @source_pad: source pad of remote subdevice
* @sink_pad: sink pad of remote subdevice
*
struct v4l2_subdev *subdev;
enum v4l2_mbus_type mbus_type;
- unsigned int mbus_flags;
+ struct v4l2_fwnode_bus_parallel bus;
unsigned int source_pad;
unsigned int sink_pad;
/* OF graph endpoint's V4L2 async data */
struct rcar_drif_graph_ep {
struct v4l2_subdev *subdev; /* Async matched subdev */
- struct v4l2_async_subdev asd; /* Async sub-device descriptor */
};
/* DMA buffer */
struct rcar_drif_sdr *sdr =
container_of(notifier, struct rcar_drif_sdr, notifier);
- if (sdr->ep.asd.match.fwnode !=
- of_fwnode_handle(subdev->dev->of_node)) {
- rdrif_err(sdr, "subdev %s cannot bind\n", subdev->name);
- return -EINVAL;
- }
-
v4l2_set_subdev_hostdata(subdev, sdr);
sdr->ep.subdev = subdev;
rdrif_dbg(sdr, "bound asd %s\n", subdev->name);
{
struct v4l2_async_notifier *notifier = &sdr->notifier;
struct fwnode_handle *fwnode, *ep;
- int ret;
+ struct v4l2_async_subdev *asd;
v4l2_async_notifier_init(notifier);
if (!ep)
return 0;
+ /* Get the endpoint properties */
+ rcar_drif_get_ep_properties(sdr, ep);
+
fwnode = fwnode_graph_get_remote_port_parent(ep);
+ fwnode_handle_put(ep);
if (!fwnode) {
dev_warn(sdr->dev, "bad remote port parent\n");
- fwnode_handle_put(ep);
return -EINVAL;
}
- sdr->ep.asd.match.fwnode = fwnode;
- sdr->ep.asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
- ret = v4l2_async_notifier_add_subdev(notifier, &sdr->ep.asd);
- if (ret) {
- fwnode_handle_put(fwnode);
- return ret;
- }
-
- /* Get the endpoint properties */
- rcar_drif_get_ep_properties(sdr, ep);
-
+ asd = v4l2_async_notifier_add_fwnode_subdev(notifier, fwnode,
+ sizeof(*asd));
fwnode_handle_put(fwnode);
- fwnode_handle_put(ep);
+ if (IS_ERR(asd))
+ return PTR_ERR(asd);
return 0;
}
/* Non-swapped planar image capture mode. */
case V4L2_PIX_FMT_NV16:
cdocr |= CEU_CDOCR_NO_DOWSAMPLE;
- /* fall-through */
+ fallthrough;
case V4L2_PIX_FMT_NV12:
if (mbus_fmt->swapped)
camcr = mbus_fmt->fmt_order_swap;
/* Swapped planar image capture mode. */
case V4L2_PIX_FMT_NV61:
cdocr |= CEU_CDOCR_NO_DOWSAMPLE;
- /* fall-through */
+ fallthrough;
case V4L2_PIX_FMT_NV21:
if (mbus_fmt->swapped)
camcr = mbus_fmt->fmt_order;
ret = pm_runtime_get_sync(rga->dev);
if (ret < 0) {
+ pm_runtime_put_noidle(rga->dev);
rga_buf_return_buffers(q, VB2_BUF_STATE_QUEUED);
return ret;
}
ret = camif_media_dev_init(camif);
if (ret < 0)
- goto err_alloc;
+ goto err_pm;
ret = camif_register_sensor(camif);
if (ret < 0)
media_device_unregister(&camif->media_dev);
media_device_cleanup(&camif->media_dev);
camif_unregister_media_entities(camif);
-err_alloc:
+err_pm:
pm_runtime_put(dev);
pm_runtime_disable(dev);
-err_pm:
camif_clk_put(camif);
err_clk:
s3c_camif_unregister_subdev(camif);
.menu_skip_mask = 0,
.default_value = V4L2_MPEG_MFC51_VIDEO_FRAME_SKIP_MODE_DISABLED,
},
+ {
+ .id = V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE,
+ .type = V4L2_CTRL_TYPE_MENU,
+ .maximum = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT,
+ .default_value = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED,
+ },
{
.id = V4L2_CID_MPEG_MFC51_VIDEO_RC_FIXED_TARGET_BIT,
.type = V4L2_CTRL_TYPE_BOOLEAN,
p->seq_hdr_mode = ctrl->val;
break;
case V4L2_CID_MPEG_MFC51_VIDEO_FRAME_SKIP_MODE:
+ case V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE:
p->frame_skip_mode = ctrl->val;
break;
case V4L2_CID_MPEG_MFC51_VIDEO_RC_FIXED_TARGET_BIT:
int i, ret = 0;
ret = pm_runtime_get_sync(pm->device);
- if (ret < 0)
+ if (ret < 0) {
+ pm_runtime_put_noidle(pm->device);
return ret;
+ }
/* clock control */
for (i = 0; i < pm->num_clocks; i++) {
DEFINE_SHOW_ATTRIBUTE(last_request);
DEFINE_SHOW_ATTRIBUTE(perf);
-int bdisp_debugfs_create(struct bdisp_dev *bdisp)
+void bdisp_debugfs_create(struct bdisp_dev *bdisp)
{
char dirname[16];
snprintf(dirname, sizeof(dirname), "%s%d", BDISP_NAME, bdisp->id);
bdisp->dbg.debugfs_entry = debugfs_create_dir(dirname, NULL);
- if (!bdisp->dbg.debugfs_entry)
- goto err;
- if (!bdisp_dbg_create_entry(regs))
- goto err;
-
- if (!bdisp_dbg_create_entry(last_nodes))
- goto err;
-
- if (!bdisp_dbg_create_entry(last_nodes_raw))
- goto err;
-
- if (!bdisp_dbg_create_entry(last_request))
- goto err;
-
- if (!bdisp_dbg_create_entry(perf))
- goto err;
-
- return 0;
-
-err:
- bdisp_debugfs_remove(bdisp);
- return -ENOMEM;
+ bdisp_dbg_create_entry(regs);
+ bdisp_dbg_create_entry(last_nodes);
+ bdisp_dbg_create_entry(last_nodes_raw);
+ bdisp_dbg_create_entry(last_request);
+ bdisp_dbg_create_entry(perf);
}
void bdisp_debugfs_remove(struct bdisp_dev *bdisp)
}
/* Debug */
- ret = bdisp_debugfs_create(bdisp);
- if (ret) {
- dev_err(dev, "failed to create debugfs\n");
- goto err_v4l2;
- }
+ bdisp_debugfs_create(bdisp);
/* Power management */
pm_runtime_enable(dev);
ret = pm_runtime_get_sync(dev);
if (ret < 0) {
dev_err(dev, "failed to set PM\n");
- goto err_dbg;
+ goto err_pm;
}
/* Filters */
bdisp_hw_free_filters(bdisp->dev);
err_pm:
pm_runtime_put(dev);
-err_dbg:
bdisp_debugfs_remove(bdisp);
-err_v4l2:
v4l2_device_unregister(&bdisp->v4l2_dev);
err_clk:
if (!IS_ERR(bdisp->clock))
int bdisp_hw_update(struct bdisp_ctx *ctx);
void bdisp_debugfs_remove(struct bdisp_dev *bdisp);
-int bdisp_debugfs_create(struct bdisp_dev *bdisp);
+void bdisp_debugfs_create(struct bdisp_dev *bdisp);
void bdisp_dbg_perf_begin(struct bdisp_dev *bdisp);
void bdisp_dbg_perf_end(struct bdisp_dev *bdisp);
add_timer(&fei->timer);
}
-static void channel_swdemux_tsklet(unsigned long data)
+static void channel_swdemux_tsklet(struct tasklet_struct *t)
{
- struct channel_info *channel = (struct channel_info *)data;
+ struct channel_info *channel = from_tasklet(channel, t, tsklet);
struct c8sectpfei *fei;
unsigned long wp, rp;
int pos, num_packets, n, size;
dev_dbg(fei->dev, "Starting channel=%p\n", channel);
- tasklet_init(&channel->tsklet, channel_swdemux_tsklet,
- (unsigned long) channel);
+ tasklet_setup(&channel->tsklet, channel_swdemux_tsklet);
/* Reset the internal inputblock sram pointers */
writel(channel->fifo,
writel(tsin->back_buffer_busaddr, tsin->irec + DMA_PRDS_BUSRP_TP(0));
/* initialize tasklet */
- tasklet_init(&tsin->tsklet, channel_swdemux_tsklet,
- (unsigned long) tsin);
+ tasklet_setup(&tsin->tsklet, channel_swdemux_tsklet);
return 0;
/* enable the hardware */
if (!dec->pm) {
ret = delta_get_sync(ctx);
- if (ret)
+ if (ret) {
+ delta_put_autosuspend(ctx);
goto err;
+ }
}
/* decode this access unit */
void hva_debugfs_create(struct hva_dev *hva)
{
hva->dbg.debugfs_entry = debugfs_create_dir(HVA_NAME, NULL);
- if (!hva->dbg.debugfs_entry)
- goto err;
- if (!hva_dbg_create_entry(device))
- goto err;
-
- if (!hva_dbg_create_entry(encoders))
- goto err;
-
- if (!hva_dbg_create_entry(last))
- goto err;
-
- if (!hva_dbg_create_entry(regs))
- goto err;
-
- return;
-
-err:
- hva_debugfs_remove(hva);
+ hva_dbg_create_entry(device);
+ hva_dbg_create_entry(encoders);
+ hva_dbg_create_entry(last);
+ hva_dbg_create_entry(regs);
}
void hva_debugfs_remove(struct hva_dev *hva)
if (pm_runtime_get_sync(dev) < 0) {
dev_err(dev, "%s failed to get pm_runtime\n", HVA_PREFIX);
+ pm_runtime_put_noidle(dev);
mutex_unlock(&hva->protect_mutex);
return -EFAULT;
}
ret = pm_runtime_get_sync(dev);
if (ret < 0) {
dev_err(dev, "%s failed to set PM\n", HVA_PREFIX);
- goto err_clk;
+ goto err_pm;
}
/* check IP hardware version */
if (pm_runtime_get_sync(dev) < 0) {
seq_puts(s, "Cannot wake up IP\n");
+ pm_runtime_put_noidle(dev);
mutex_unlock(&hva->protect_mutex);
return;
}
if (ret < 0) {
dev_err(dcmi->dev, "%s: Failed to start streaming, cannot get sync (%d)\n",
__func__, ret);
- goto err_release_buffers;
+ goto err_pm_put;
}
ret = media_pipeline_start(&dcmi->vdev->entity, &dcmi->pipeline);
err_pm_put:
pm_runtime_put(dcmi->dev);
-
-err_release_buffers:
spin_lock_irq(&dcmi->irqlock);
/*
* Return all buffers to vb2 in QUEUED state.
v4l2_async_notifier_unregister(&csi->notifier);
v4l2_async_notifier_cleanup(&csi->notifier);
+ vb2_video_unregister_device(&csi->vdev);
media_device_unregister(&csi->mdev);
sun4i_csi_dma_unregister(csi);
media_device_cleanup(&csi->mdev);
ret = v4l2_device_register(csi->dev, &csi->v4l);
if (ret) {
dev_err(csi->dev, "Couldn't register the v4l2 device\n");
- goto err_free_queue;
+ goto err_free_mutex;
}
ret = devm_request_irq(csi->dev, irq, sun4i_csi_irq, 0,
err_unregister_device:
v4l2_device_unregister(&csi->v4l);
-err_free_queue:
- vb2_queue_release(q);
-
err_free_mutex:
mutex_destroy(&csi->lock);
return ret;
void sun4i_csi_dma_unregister(struct sun4i_csi *csi)
{
v4l2_device_unregister(&csi->v4l);
- vb2_queue_release(&csi->queue);
mutex_destroy(&csi->lock);
}
if (ret < 0) {
v4l2_err(&csi->v4l2_dev,
"video_register_device failed: %d\n", ret);
- goto release_vb2;
+ goto clean_entity;
}
return 0;
-release_vb2:
- vb2_queue_release(&video->vb2_vidq);
clean_entity:
media_entity_cleanup(&video->vdev.entity);
mutex_destroy(&video->lock);
void sun6i_video_cleanup(struct sun6i_video *video)
{
- video_unregister_device(&video->vdev);
+ vb2_video_unregister_device(&video->vdev);
media_entity_cleanup(&video->vdev.entity);
- vb2_queue_release(&video->vb2_vidq);
mutex_destroy(&video->lock);
}
dev->dev = &pdev->dev;
irq = platform_get_irq(pdev, 0);
- if (irq <= 0) {
- dev_err(dev->dev, "Failed to get IRQ\n");
-
+ if (irq <= 0)
return irq;
- }
ret = devm_request_irq(dev->dev, irq, rotate_irq,
0, dev_name(dev->dev), dev);
r = pm_runtime_get_sync(&pdev->dev);
WARN_ON(r < 0);
+ if (r)
+ pm_runtime_put_noidle(&pdev->dev);
return r < 0 ? r : 0;
}
int ret;
ret = pm_runtime_get_sync(vsp1->dev);
- return ret < 0 ? ret : 0;
+ if (ret < 0) {
+ pm_runtime_put_noidle(vsp1->dev);
+ return ret;
+ }
+
+ return 0;
}
/*
/* Configure device parameters based on the version register. */
pm_runtime_enable(&pdev->dev);
- ret = pm_runtime_get_sync(&pdev->dev);
+ ret = vsp1_device_get(vsp1);
if (ret < 0)
goto done;
vsp1->version = vsp1_read(vsp1, VI6_IP_VERSION);
- pm_runtime_put_sync(&pdev->dev);
+ vsp1_device_put(vsp1);
for (i = 0; i < ARRAY_SIZE(vsp1_device_infos); ++i) {
if ((vsp1->version & VI6_IP_VERSION_MODEL_MASK) ==
};
-static int si476x_radio_init_debugfs(struct si476x_radio *radio)
+static void si476x_radio_init_debugfs(struct si476x_radio *radio)
{
- struct dentry *dentry;
- int ret;
+ radio->debugfs = debugfs_create_dir(dev_name(radio->v4l2dev.dev), NULL);
- dentry = debugfs_create_dir(dev_name(radio->v4l2dev.dev), NULL);
- if (IS_ERR(dentry)) {
- ret = PTR_ERR(dentry);
- goto exit;
- }
- radio->debugfs = dentry;
-
- dentry = debugfs_create_file("acf", S_IRUGO,
- radio->debugfs, radio, &radio_acf_fops);
- if (IS_ERR(dentry)) {
- ret = PTR_ERR(dentry);
- goto cleanup;
- }
+ debugfs_create_file("acf", S_IRUGO, radio->debugfs, radio,
+ &radio_acf_fops);
- dentry = debugfs_create_file("rds_blckcnt", S_IRUGO,
- radio->debugfs, radio,
- &radio_rds_blckcnt_fops);
- if (IS_ERR(dentry)) {
- ret = PTR_ERR(dentry);
- goto cleanup;
- }
+ debugfs_create_file("rds_blckcnt", S_IRUGO, radio->debugfs, radio,
+ &radio_rds_blckcnt_fops);
- dentry = debugfs_create_file("agc", S_IRUGO,
- radio->debugfs, radio, &radio_agc_fops);
- if (IS_ERR(dentry)) {
- ret = PTR_ERR(dentry);
- goto cleanup;
- }
+ debugfs_create_file("agc", S_IRUGO, radio->debugfs, radio,
+ &radio_agc_fops);
- dentry = debugfs_create_file("rsq", S_IRUGO,
- radio->debugfs, radio, &radio_rsq_fops);
- if (IS_ERR(dentry)) {
- ret = PTR_ERR(dentry);
- goto cleanup;
- }
+ debugfs_create_file("rsq", S_IRUGO, radio->debugfs, radio,
+ &radio_rsq_fops);
- dentry = debugfs_create_file("rsq_primary", S_IRUGO,
- radio->debugfs, radio,
- &radio_rsq_primary_fops);
- if (IS_ERR(dentry)) {
- ret = PTR_ERR(dentry);
- goto cleanup;
- }
-
- return 0;
-cleanup:
- debugfs_remove_recursive(radio->debugfs);
-exit:
- return ret;
+ debugfs_create_file("rsq_primary", S_IRUGO, radio->debugfs, radio,
+ &radio_rsq_primary_fops);
}
goto exit;
}
- rval = si476x_radio_init_debugfs(radio);
- if (rval < 0) {
- dev_err(&pdev->dev, "Could not create debugfs interface\n");
- goto exit;
- }
+ si476x_radio_init_debugfs(radio);
return 0;
exit:
* V4L2_CID_TUNE_POWER_LEVEL. */
if (force)
break;
- /* fall through */
+ fallthrough;
case V4L2_CID_TUNE_POWER_LEVEL:
ret = si4713_tx_tune_power(sdev,
sdev->tune_pwr_level->val, sdev->tune_ant_cap->val);
* Author: Manjunatha Halli <manjunatha_halli@ti.com>
*/
-#include <linux/module.h>
-#include <linux/firmware.h>
#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/nospec.h>
+
#include "fmdrv.h"
#include "fmdrv_v4l2.h"
#include "fmdrv_common.h"
* FM common sub-module will schedule this tasklet whenever it receives
* FM packet from ST driver.
*/
-static void recv_tasklet(unsigned long arg)
+static void recv_tasklet(struct tasklet_struct *t)
{
struct fmdev *fmdev;
struct fm_irq *irq_info;
u8 num_fm_hci_cmds;
unsigned long flags;
- fmdev = (struct fmdev *)arg;
+ fmdev = from_tasklet(fmdev, t, tx_task);
irq_info = &fmdev->irq_info;
/* Process all packets in the RX queue */
while ((skb = skb_dequeue(&fmdev->rx_q))) {
}
/* FM send tasklet: is scheduled when FM packet has to be sent to chip */
-static void send_tasklet(unsigned long arg)
+static void send_tasklet(struct tasklet_struct *t)
{
struct fmdev *fmdev;
struct sk_buff *skb;
int len;
- fmdev = (struct fmdev *)arg;
+ fmdev = from_tasklet(fmdev, t, tx_task);
if (!atomic_read(&fmdev->tx_cnt))
return;
struct fm_rds *rds = &fmdev->rx.rds;
unsigned long group_idx, flags;
u8 *rds_data, meta_data, tmpbuf[FM_RDS_BLK_SIZE];
- u8 type, blk_idx;
+ u8 type, blk_idx, idx;
u16 cur_picode;
u32 rds_len;
}
/* Skip checkword (control) byte and copy only data byte */
- memcpy(&rds_fmt.data.groupdatabuff.
- buff[blk_idx * (FM_RDS_BLK_SIZE - 1)],
- rds_data, (FM_RDS_BLK_SIZE - 1));
+ idx = array_index_nospec(blk_idx * (FM_RDS_BLK_SIZE - 1),
+ FM_RX_RDS_INFO_FIELD_MAX - (FM_RDS_BLK_SIZE - 1));
+
+ memcpy(&rds_fmt.data.groupdatabuff.buff[idx], rds_data,
+ FM_RDS_BLK_SIZE - 1);
rds->last_blk_idx = blk_idx;
/* Initialize TX queue and TX tasklet */
skb_queue_head_init(&fmdev->tx_q);
- tasklet_init(&fmdev->tx_task, send_tasklet, (unsigned long)fmdev);
+ tasklet_setup(&fmdev->tx_task, send_tasklet);
/* Initialize RX Queue and RX tasklet */
skb_queue_head_init(&fmdev->rx_q);
- tasklet_init(&fmdev->rx_task, recv_tasklet, (unsigned long)fmdev);
+ tasklet_setup(&fmdev->rx_task, recv_tasklet);
fmdev->irq_info.stage = 0;
atomic_set(&fmdev->tx_cnt, 1);
err("%s: endpoint_in message size==0? \n", __func__);
return -ENODEV;
}
+ if (!usb_endpoint_is_int_out(endpoint_out)) {
+ err("%s: Unexpected endpoint_out\n", __func__);
+ return -ENODEV;
+ }
ati_remote = kzalloc(sizeof (struct ati_remote), GFP_KERNEL);
rc_dev = rc_allocate_device(RC_DRIVER_SCANCODE);
select_timeout:
if (dev->rx_fan_input_inuse) {
- dev->rdev->rx_resolution = US_TO_NS(ENE_FW_SAMPLE_PERIOD_FAN);
+ dev->rdev->rx_resolution = ENE_FW_SAMPLE_PERIOD_FAN;
/* Fan input doesn't support timeouts, it just ends the
input with a maximum sample */
dev->rdev->min_timeout = dev->rdev->max_timeout =
- US_TO_NS(ENE_FW_SMPL_BUF_FAN_MSK *
- ENE_FW_SAMPLE_PERIOD_FAN);
+ ENE_FW_SMPL_BUF_FAN_MSK *
+ ENE_FW_SAMPLE_PERIOD_FAN;
} else {
- dev->rdev->rx_resolution = US_TO_NS(sample_period);
+ dev->rdev->rx_resolution = sample_period;
/* Theoreticly timeout is unlimited, but we cap it
* because it was seen that on one device, it
* would stop sending spaces after around 250 msec.
* Besides, this is close to 2^32 anyway and timeout is u32.
*/
- dev->rdev->min_timeout = US_TO_NS(127 * sample_period);
- dev->rdev->max_timeout = US_TO_NS(200000);
+ dev->rdev->min_timeout = 127 * sample_period;
+ dev->rdev->max_timeout = 200000;
}
if (dev->hw_learning_and_tx_capable)
- dev->rdev->tx_resolution = US_TO_NS(sample_period);
+ dev->rdev->tx_resolution = sample_period;
if (dev->rdev->timeout > dev->rdev->max_timeout)
dev->rdev->timeout = dev->rdev->max_timeout;
dbg("RX: %d (%s)", hw_sample, pulse ? "pulse" : "space");
- ev.duration = US_TO_NS(hw_sample);
+ ev.duration = hw_sample;
ev.pulse = pulse;
ir_raw_event_store_with_filter(dev->rdev, &ev);
}
dev->learning_mode_enabled = learning_mode_force;
/* Set reasonable default timeout */
- dev->rdev->timeout = US_TO_NS(150000);
+ dev->rdev->timeout = MS_TO_US(150);
}
/* Upload all hardware settings at once. Used at load and resume time */
case PARSE_IRDATA:
fintek->rem--;
rawir.pulse = ((sample & BUF_PULSE_BIT) != 0);
- rawir.duration = US_TO_NS((sample & BUF_SAMPLE_MASK)
- * CIR_SAMPLE_PERIOD);
+ rawir.duration = (sample & BUF_SAMPLE_MASK)
+ * CIR_SAMPLE_PERIOD;
fit_dbg("Storing %s with duration %d",
rawir.pulse ? "pulse" : "space",
rdev->dev.parent = &pdev->dev;
rdev->driver_name = FINTEK_DRIVER_NAME;
rdev->map_name = RC_MAP_RC6_MCE;
- rdev->timeout = US_TO_NS(1000);
+ rdev->timeout = 1000;
/* rx resolution is hardwired to 50us atm, 1, 25, 100 also possible */
- rdev->rx_resolution = US_TO_NS(CIR_SAMPLE_PERIOD);
+ rdev->rx_resolution = CIR_SAMPLE_PERIOD;
fintek->rdev = rdev;
#include <linux/of.h>
#include <linux/of_gpio.h>
#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/pm_qos.h>
#include <linux/irq.h>
#include <media/rc-core.h>
struct rc_dev *rcdev;
struct gpio_desc *gpiod;
int irq;
+ struct device *pmdev;
+ struct pm_qos_request qos;
};
static irqreturn_t gpio_ir_recv_irq(int irq, void *dev_id)
{
int val;
struct gpio_rc_dev *gpio_dev = dev_id;
+ struct device *pmdev = gpio_dev->pmdev;
+
+ /*
+ * For some cpuidle systems, not all:
+ * Respond to interrupt taking more latency when cpu in idle.
+ * Invoke asynchronous pm runtime get from interrupt context,
+ * this may introduce a millisecond delay to call resume callback,
+ * where to disable cpuilde.
+ *
+ * Two issues lead to fail to decode first frame, one is latency to
+ * respond to interrupt, another is delay introduced by async api.
+ */
+ if (pmdev)
+ pm_runtime_get(pmdev);
val = gpiod_get_value(gpio_dev->gpiod);
if (val >= 0)
ir_raw_event_store_edge(gpio_dev->rcdev, val == 1);
+ if (pmdev) {
+ pm_runtime_mark_last_busy(pmdev);
+ pm_runtime_put_autosuspend(pmdev);
+ }
+
return IRQ_HANDLED;
}
struct device_node *np = dev->of_node;
struct gpio_rc_dev *gpio_dev;
struct rc_dev *rcdev;
+ u32 period = 0;
int rc;
if (!np)
return rc;
}
+ of_property_read_u32(np, "linux,autosuspend-period", &period);
+ if (period) {
+ gpio_dev->pmdev = dev;
+ pm_runtime_set_autosuspend_delay(dev, period);
+ pm_runtime_use_autosuspend(dev);
+ pm_runtime_set_suspended(dev);
+ pm_runtime_enable(dev);
+ }
+
platform_set_drvdata(pdev, gpio_dev);
return devm_request_irq(dev, gpio_dev->irq, gpio_ir_recv_irq,
return 0;
}
+static int gpio_ir_recv_runtime_suspend(struct device *dev)
+{
+ struct gpio_rc_dev *gpio_dev = dev_get_drvdata(dev);
+
+ cpu_latency_qos_remove_request(&gpio_dev->qos);
+
+ return 0;
+}
+
+static int gpio_ir_recv_runtime_resume(struct device *dev)
+{
+ struct gpio_rc_dev *gpio_dev = dev_get_drvdata(dev);
+
+ cpu_latency_qos_add_request(&gpio_dev->qos, 0);
+
+ return 0;
+}
+
static const struct dev_pm_ops gpio_ir_recv_pm_ops = {
.suspend = gpio_ir_recv_suspend,
.resume = gpio_ir_recv_resume,
+ .runtime_suspend = gpio_ir_recv_runtime_suspend,
+ .runtime_resume = gpio_ir_recv_runtime_resume,
};
#endif
overflow);
do {
- rawir.duration = ir->buf_in[i] * 85333;
+ rawir.duration = ir->buf_in[i] * 85;
rawir.pulse = i & 1;
ir_raw_event_store_with_filter(ir->rc, &rawir);
rc->priv = ir;
rc->driver_name = DRIVER_NAME;
rc->map_name = RC_MAP_HAUPPAUGE;
- rc->timeout = MS_TO_NS(100);
- rc->rx_resolution = 85333;
+ rc->timeout = MS_TO_US(100);
+ rc->rx_resolution = 85;
ir->rc = rc;
ret = rc_register_device(rc);
#define MAX_IN_PACKET 8u
#define MAX_OUT_PACKET (sizeof(struct send_packet) + BUF_SIZE)
#define TIMEOUT 1000
-#define RX_RESOLUTION 21333
+#define RX_RESOLUTION 21
struct packet {
uint16_t start;
break;
case CMD_TX_OVERFLOW:
ir->tx_overflow = true;
- /* fall through */
+ fallthrough;
case CMD_RECEIVER_OFF:
case CMD_RECEIVER_ON:
case CMD_SEND:
for (i = 0; i < 7; i++) {
if (ir->buf_in[i] == 0x80) {
rawir.pulse = false;
- rawir.duration = US_TO_NS(21845);
+ rawir.duration = 21845;
} else {
rawir.pulse = (ir->buf_in[i] & 0x80) == 0;
rawir.duration = ((ir->buf_in[i] & 0x7f) + 1) *
#include <media/rc-core.h>
/* Each bit is 250us */
-#define BIT_DURATION 250000
+#define BIT_DURATION 250
struct imon {
struct device *dev;
data_h = ((symb_val >> 16) & 0xffff) * 10;
symb_time = (data_l + data_h) / 10;
- ev.duration = US_TO_NS(data_l);
+ ev.duration = data_l;
ev.pulse = true;
ir_raw_event_store(priv->rdev, &ev);
if (symb_time < IR_CFG_SYMBOL_MAXWIDTH) {
- ev.duration = US_TO_NS(data_h);
+ ev.duration = data_h;
ev.pulse = false;
ir_raw_event_store(priv->rdev, &ev);
} else {
rdev->input_id.vendor = 0x0001;
rdev->input_id.product = 0x0001;
rdev->input_id.version = 0x0100;
- rdev->rx_resolution = US_TO_NS(10);
- rdev->timeout = US_TO_NS(IR_CFG_SYMBOL_MAXWIDTH * 10);
+ rdev->rx_resolution = 10;
+ rdev->timeout = IR_CFG_SYMBOL_MAXWIDTH * 10;
ret = rc_register_device(rdev);
if (ret < 0)
#include <linux/module.h>
#include "rc-core-priv.h"
-#define IMON_UNIT 415662 /* ns */
+#define IMON_UNIT 416 /* us */
#define IMON_BITS 30
#define IMON_CHKBITS (BIT(30) | BIT(25) | BIT(24) | BIT(22) | \
BIT(21) | BIT(20) | BIT(19) | BIT(18) | \
dev_dbg(&dev->dev,
"iMON decode started at state %d bitno %d (%uus %s)\n",
- data->state, data->count, TO_US(ev.duration),
- TO_STR(ev.pulse));
+ data->state, data->count, ev.duration, TO_STR(ev.pulse));
/*
* Since iMON protocol is a series of bits, if at any point
* we're at a new scancode.
*/
if (data->state == STATE_ERROR) {
- if (!ev.pulse && ev.duration > MS_TO_NS(10))
+ if (!ev.pulse && ev.duration > MS_TO_US(10))
data->state = STATE_INACTIVE;
return 0;
}
err_out:
dev_dbg(&dev->dev,
"iMON decode failed at state %d bitno %d (%uus %s)\n",
- data->state, data->count, TO_US(ev.duration),
- TO_STR(ev.pulse));
+ data->state, data->count, ev.duration, TO_STR(ev.pulse));
data->state = STATE_ERROR;
#include "rc-core-priv.h"
#define JVC_NBITS 16 /* dev(8) + func(8) */
-#define JVC_UNIT 525000 /* ns */
+#define JVC_UNIT 525 /* us */
#define JVC_HEADER_PULSE (16 * JVC_UNIT) /* lack of header -> repeat */
#define JVC_HEADER_SPACE (8 * JVC_UNIT)
#define JVC_BIT_PULSE (1 * JVC_UNIT)
goto out;
dev_dbg(&dev->dev, "JVC decode started at state %d (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
again:
switch (data->state) {
out:
dev_dbg(&dev->dev, "JVC decode failed at state %d (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
data->state = STATE_INACTIVE;
return -EINVAL;
}
* input device for the remote, rather than the keyboard/mouse one.
*/
-#define MCIR2_UNIT 333333 /* ns */
+#define MCIR2_UNIT 333 /* us */
#define MCIR2_HEADER_NBITS 5
#define MCIR2_MOUSE_NBITS 29
#define MCIR2_KEYBOARD_NBITS 32
again:
dev_dbg(&dev->dev, "started at state %i (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
if (!geq_margin(ev.duration, MCIR2_UNIT, MCIR2_UNIT / 2))
return 0;
}
lsc.scancode = scancode;
- ir_lirc_scancode_event(dev, &lsc);
+ lirc_scancode_event(dev, &lsc);
data->state = STATE_INACTIVE;
input_event(dev->input_dev, EV_MSC, MSC_SCAN, scancode);
input_sync(dev->input_dev);
out:
dev_dbg(&dev->dev, "failed at state %i (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
data->state = STATE_INACTIVE;
return -EINVAL;
}
#include "rc-core-priv.h"
#define NEC_NBITS 32
-#define NEC_UNIT 562500 /* ns */
+#define NEC_UNIT 563 /* us */
#define NEC_HEADER_PULSE (16 * NEC_UNIT)
#define NECX_HEADER_PULSE (8 * NEC_UNIT) /* Less common NEC variant */
#define NEC_HEADER_SPACE (8 * NEC_UNIT)
}
dev_dbg(&dev->dev, "NEC decode started at state %d (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
switch (data->state) {
}
dev_dbg(&dev->dev, "NEC decode failed at count %d state %d (%uus %s)\n",
- data->count, data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->count, data->state, ev.duration, TO_STR(ev.pulse));
data->state = STATE_INACTIVE;
return -EINVAL;
}
#define RC5_SZ_NBITS 15
#define RC5X_NBITS 20
#define CHECK_RC5X_NBITS 8
-#define RC5_UNIT 888888 /* ns */
+#define RC5_UNIT 889 /* us */
#define RC5_BIT_START (1 * RC5_UNIT)
#define RC5_BIT_END (1 * RC5_UNIT)
#define RC5X_SPACE (4 * RC5_UNIT)
again:
dev_dbg(&dev->dev, "RC5(x/sz) decode started at state %i (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
if (!geq_margin(ev.duration, RC5_UNIT, RC5_UNIT / 2))
return 0;
out:
dev_dbg(&dev->dev, "RC5(x/sz) decode failed at state %i count %d (%uus %s)\n",
- data->state, data->count, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, data->count, ev.duration, TO_STR(ev.pulse));
data->state = STATE_INACTIVE;
return -EINVAL;
}
* RC6-6A-32 (MCE version with toggle bit in body)
*/
-#define RC6_UNIT 444444 /* nanosecs */
+#define RC6_UNIT 444 /* microseconds */
#define RC6_HEADER_NBITS 4 /* not including toggle bit */
#define RC6_0_NBITS 16
#define RC6_6A_32_NBITS 32
again:
dev_dbg(&dev->dev, "RC6 decode started at state %i (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
if (!geq_margin(ev.duration, RC6_UNIT, RC6_UNIT / 2))
return 0;
out:
dev_dbg(&dev->dev, "RC6 decode failed at state %i (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
data->state = STATE_INACTIVE;
return -EINVAL;
}
#include "rc-core-priv.h"
#include <linux/module.h>
-#define RCMM_UNIT 166667 /* nanosecs */
-#define RCMM_PREFIX_PULSE 416666 /* 166666.666666666*2.5 */
-#define RCMM_PULSE_0 277777 /* 166666.666666666*(1+2/3) */
-#define RCMM_PULSE_1 444444 /* 166666.666666666*(2+2/3) */
-#define RCMM_PULSE_2 611111 /* 166666.666666666*(3+2/3) */
-#define RCMM_PULSE_3 777778 /* 166666.666666666*(4+2/3) */
+#define RCMM_UNIT 166 /* microseconds */
+#define RCMM_PREFIX_PULSE 417 /* 166.666666666666*2.5 */
+#define RCMM_PULSE_0 278 /* 166.666666666666*(1+2/3) */
+#define RCMM_PULSE_1 444 /* 166.666666666666*(2+2/3) */
+#define RCMM_PULSE_2 611 /* 166.666666666666*(3+2/3) */
+#define RCMM_PULSE_3 778 /* 166.666666666666*(4+2/3) */
enum rcmm_state {
STATE_INACTIVE,
int value;
if (!(dev->enabled_protocols & (RC_PROTO_BIT_RCMM32 |
- RC_PROTO_BIT_RCMM24 |
- RC_PROTO_BIT_RCMM12)))
+ RC_PROTO_BIT_RCMM24 |
+ RC_PROTO_BIT_RCMM12)))
return 0;
if (!is_timing_event(ev)) {
}
dev_dbg(&dev->dev, "RC-MM decode failed at count %d state %d (%uus %s)\n",
- data->count, data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->count, data->state, ev.duration, TO_STR(ev.pulse));
data->state = STATE_INACTIVE;
return -EINVAL;
}
#include "rc-core-priv.h"
#define SANYO_NBITS (13+13+8+8)
-#define SANYO_UNIT 562500 /* ns */
+#define SANYO_UNIT 563 /* us */
#define SANYO_HEADER_PULSE (16 * SANYO_UNIT)
#define SANYO_HEADER_SPACE (8 * SANYO_UNIT)
#define SANYO_BIT_PULSE (1 * SANYO_UNIT)
}
dev_dbg(&dev->dev, "SANYO decode started at state %d (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
switch (data->state) {
}
dev_dbg(&dev->dev, "SANYO decode failed at count %d state %d (%uus %s)\n",
- data->count, data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->count, data->state, ev.duration, TO_STR(ev.pulse));
data->state = STATE_INACTIVE;
return -EINVAL;
}
#include "rc-core-priv.h"
#define SHARP_NBITS 15
-#define SHARP_UNIT 40000 /* ns */
+#define SHARP_UNIT 40 /* us */
#define SHARP_BIT_PULSE (8 * SHARP_UNIT) /* 320us */
#define SHARP_BIT_0_PERIOD (25 * SHARP_UNIT) /* 1ms (680us space) */
#define SHARP_BIT_1_PERIOD (50 * SHARP_UNIT) /* 2ms (1680ms space) */
}
dev_dbg(&dev->dev, "Sharp decode started at state %d (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
switch (data->state) {
}
dev_dbg(&dev->dev, "Sharp decode failed at count %d state %d (%uus %s)\n",
- data->count, data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->count, data->state, ev.duration, TO_STR(ev.pulse));
data->state = STATE_INACTIVE;
return -EINVAL;
}
#include <linux/module.h>
#include "rc-core-priv.h"
-#define SONY_UNIT 600000 /* ns */
+#define SONY_UNIT 600 /* us */
#define SONY_HEADER_PULSE (4 * SONY_UNIT)
#define SONY_HEADER_SPACE (1 * SONY_UNIT)
#define SONY_BIT_0_PULSE (1 * SONY_UNIT)
goto out;
dev_dbg(&dev->dev, "Sony decode started at state %d (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
switch (data->state) {
out:
dev_dbg(&dev->dev, "Sony decode failed at state %d (%uus %s)\n",
- data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, ev.duration, TO_STR(ev.pulse));
data->state = STATE_INACTIVE;
return -EINVAL;
#include <linux/module.h>
#include "rc-core-priv.h"
-#define XMP_UNIT 136000 /* ns */
-#define XMP_LEADER 210000 /* ns */
-#define XMP_NIBBLE_PREFIX 760000 /* ns */
-#define XMP_HALFFRAME_SPACE 13800000 /* ns */
-#define XMP_TRAILER_SPACE 20000000 /* should be 80ms but not all dureation supliers can go that high */
+#define XMP_UNIT 136 /* us */
+#define XMP_LEADER 210 /* us */
+#define XMP_NIBBLE_PREFIX 760 /* us */
+#define XMP_HALFFRAME_SPACE 13800 /* us */
+/* should be 80ms but not all duration supliers can go that high */
+#define XMP_TRAILER_SPACE 20000
enum xmp_state {
STATE_INACTIVE,
}
dev_dbg(&dev->dev, "XMP decode started at state %d %d (%uus %s)\n",
- data->state, data->count, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->state, data->count, ev.duration, TO_STR(ev.pulse));
switch (data->state) {
}
dev_dbg(&dev->dev, "XMP decode failed at count %d state %d (%uus %s)\n",
- data->count, data->state, TO_US(ev.duration), TO_STR(ev.pulse));
+ data->count, data->state, ev.duration, TO_STR(ev.pulse));
data->state = STATE_INACTIVE;
return -EINVAL;
}
#define LEN_SAMPLEMODEPROTO 3
#define MIN_FW_VERSION 20
-#define UNIT_NS 21333
-#define MAX_TIMEOUT_NS (UNIT_NS * U16_MAX)
+#define UNIT_US 21
+#define MAX_TIMEOUT_US (UNIT_US * U16_MAX)
#define MAX_PACKET 64
if (v == 0xffff) {
rawir.pulse = false;
} else {
- rawir.duration = v * UNIT_NS;
+ rawir.duration = v * UNIT_US;
ir_raw_event_store_with_timeout(irtoy->rc,
&rawir);
}
return -ENOMEM;
for (i = 0; i < count; i++) {
- u16 v = DIV_ROUND_CLOSEST(US_TO_NS(txbuf[i]), UNIT_NS);
+ u16 v = DIV_ROUND_CLOSEST(txbuf[i], UNIT_US);
if (!v)
v = 1;
rc->tx_ir = irtoy_tx;
rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
rc->map_name = RC_MAP_RC6_MCE;
- rc->rx_resolution = UNIT_NS;
+ rc->rx_resolution = UNIT_US;
rc->timeout = IR_DEFAULT_TIMEOUT;
/*
*
* So, make timeout a largish minimum which works with most protocols.
*/
- rc->min_timeout = MS_TO_NS(40);
- rc->max_timeout = MAX_TIMEOUT_NS;
+ rc->min_timeout = MS_TO_US(40);
+ rc->max_timeout = MAX_TIMEOUT_US;
err = rc_register_device(rc);
if (err)
if (next_one > 0) {
ev.pulse = true;
ev.duration =
- ITE_BITS_TO_NS(next_one, sample_period);
+ ITE_BITS_TO_US(next_one, sample_period);
ir_raw_event_store_with_filter(dev->rdev, &ev);
}
while (next_one < size) {
next_zero = find_next_zero_bit_le(ldata, size, next_one + 1);
ev.pulse = false;
- ev.duration = ITE_BITS_TO_NS(next_zero - next_one, sample_period);
+ ev.duration = ITE_BITS_TO_US(next_zero - next_one, sample_period);
ir_raw_event_store_with_filter(dev->rdev, &ev);
if (next_zero < size) {
next_zero + 1);
ev.pulse = true;
ev.duration =
- ITE_BITS_TO_NS(next_one - next_zero,
+ ITE_BITS_TO_US(next_one - next_zero,
sample_period);
ir_raw_event_store_with_filter
(dev->rdev, &ev);
rdev->timeout = IR_DEFAULT_TIMEOUT;
rdev->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
rdev->rx_resolution = ITE_BAUDRATE_DIVISOR *
- itdev->params.sample_period;
+ itdev->params.sample_period / 1000;
rdev->tx_resolution = ITE_BAUDRATE_DIVISOR *
- itdev->params.sample_period;
+ itdev->params.sample_period / 1000;
/* set up transmitter related values if needed */
if (itdev->params.hw_tx_capable) {
#define ITE_DEFAULT_CARRIER_FREQ 38000
/* convert bits to us */
-#define ITE_BITS_TO_NS(bits, sample_period) \
-((u32) ((bits) * ITE_BAUDRATE_DIVISOR * sample_period))
+#define ITE_BITS_TO_US(bits, sample_period) \
+((u32)((bits) * ITE_BAUDRATE_DIVISOR * (sample_period) / 1000))
/*
* n in RDCR produces a tolerance of +/- n * 6.25% around the center
static struct class *lirc_class;
/**
- * ir_lirc_raw_event() - Send raw IR data to lirc to be relayed to userspace
+ * lirc_raw_event() - Send raw IR data to lirc to be relayed to userspace
*
* @dev: the struct rc_dev descriptor of the device
* @ev: the struct ir_raw_event descriptor of the pulse/space
*/
-void ir_lirc_raw_event(struct rc_dev *dev, struct ir_raw_event ev)
+void lirc_raw_event(struct rc_dev *dev, struct ir_raw_event ev)
{
unsigned long flags;
struct lirc_fh *fh;
dev->gap = true;
dev->gap_duration = ev.duration;
- sample = LIRC_TIMEOUT(ev.duration / 1000);
+ sample = LIRC_TIMEOUT(ev.duration);
dev_dbg(&dev->dev, "timeout report (duration: %d)\n", sample);
/* Normal sample */
} else {
if (dev->gap) {
- dev->gap_duration += ktime_to_ns(ktime_sub(ktime_get(),
+ dev->gap_duration += ktime_to_us(ktime_sub(ktime_get(),
dev->gap_start));
- /* Convert to ms and cap by LIRC_VALUE_MASK */
- do_div(dev->gap_duration, 1000);
+ /* Cap by LIRC_VALUE_MASK */
dev->gap_duration = min_t(u64, dev->gap_duration,
LIRC_VALUE_MASK);
dev->gap = false;
}
- sample = ev.pulse ? LIRC_PULSE(ev.duration / 1000) :
- LIRC_SPACE(ev.duration / 1000);
+ sample = ev.pulse ? LIRC_PULSE(ev.duration) :
+ LIRC_SPACE(ev.duration);
dev_dbg(&dev->dev, "delivering %uus %s to lirc_dev\n",
- TO_US(ev.duration), TO_STR(ev.pulse));
+ ev.duration, TO_STR(ev.pulse));
}
/*
}
/**
- * ir_lirc_scancode_event() - Send scancode data to lirc to be relayed to
+ * lirc_scancode_event() - Send scancode data to lirc to be relayed to
* userspace. This can be called in atomic context.
* @dev: the struct rc_dev descriptor of the device
* @lsc: the struct lirc_scancode describing the decoded scancode
*/
-void ir_lirc_scancode_event(struct rc_dev *dev, struct lirc_scancode *lsc)
+void lirc_scancode_event(struct rc_dev *dev, struct lirc_scancode *lsc)
{
unsigned long flags;
struct lirc_fh *fh;
}
spin_unlock_irqrestore(&dev->lirc_fh_lock, flags);
}
-EXPORT_SYMBOL_GPL(ir_lirc_scancode_event);
+EXPORT_SYMBOL_GPL(lirc_scancode_event);
-static int ir_lirc_open(struct inode *inode, struct file *file)
+static int lirc_open(struct inode *inode, struct file *file)
{
struct rc_dev *dev = container_of(inode->i_cdev, struct rc_dev,
lirc_cdev);
return retval;
}
-static int ir_lirc_close(struct inode *inode, struct file *file)
+static int lirc_close(struct inode *inode, struct file *file)
{
struct lirc_fh *fh = file->private_data;
struct rc_dev *dev = fh->rc;
return 0;
}
-static ssize_t ir_lirc_transmit_ir(struct file *file, const char __user *buf,
- size_t n, loff_t *ppos)
+static ssize_t lirc_transmit(struct file *file, const char __user *buf,
+ size_t n, loff_t *ppos)
{
struct lirc_fh *fh = file->private_data;
struct rc_dev *dev = fh->rc;
}
for (i = 0; i < count; i++)
- /* Convert from NS to US */
- txbuf[i] = DIV_ROUND_UP(raw[i].duration, 1000);
+ txbuf[i] = raw[i].duration;
if (dev->s_tx_carrier) {
int carrier = ir_raw_encode_carrier(scan.rc_proto);
}
for (i = 0; i < count; i++) {
- if (txbuf[i] > IR_MAX_DURATION / 1000 - duration || !txbuf[i]) {
+ if (txbuf[i] > IR_MAX_DURATION - duration || !txbuf[i]) {
ret = -EINVAL;
goto out_kfree;
}
return ret;
}
-static long ir_lirc_ioctl(struct file *file, unsigned int cmd,
- unsigned long arg)
+static long lirc_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
struct lirc_fh *fh = file->private_data;
struct rc_dev *dev = fh->rc;
if (!dev->rx_resolution)
ret = -ENOTTY;
else
- val = dev->rx_resolution / 1000;
+ val = dev->rx_resolution;
break;
case LIRC_SET_WIDEBAND_RECEIVER:
if (!dev->max_timeout)
ret = -ENOTTY;
else
- val = DIV_ROUND_UP(dev->min_timeout, 1000);
+ val = dev->min_timeout;
break;
case LIRC_GET_MAX_TIMEOUT:
if (!dev->max_timeout)
ret = -ENOTTY;
else
- val = dev->max_timeout / 1000;
+ val = dev->max_timeout;
break;
case LIRC_SET_REC_TIMEOUT:
if (!dev->max_timeout) {
ret = -ENOTTY;
- } else if (val > U32_MAX / 1000) {
- /* Check for multiply overflow */
- ret = -EINVAL;
} else {
- u32 tmp = val * 1000;
-
- if (tmp < dev->min_timeout || tmp > dev->max_timeout)
+ if (val < dev->min_timeout || val > dev->max_timeout)
ret = -EINVAL;
else if (dev->s_timeout)
- ret = dev->s_timeout(dev, tmp);
+ ret = dev->s_timeout(dev, val);
else
- dev->timeout = tmp;
+ dev->timeout = val;
}
break;
if (!dev->timeout)
ret = -ENOTTY;
else
- val = DIV_ROUND_UP(dev->timeout, 1000);
+ val = dev->timeout;
break;
case LIRC_SET_REC_TIMEOUT_REPORTS:
return ret;
}
-static __poll_t ir_lirc_poll(struct file *file, struct poll_table_struct *wait)
+static __poll_t lirc_poll(struct file *file, struct poll_table_struct *wait)
{
struct lirc_fh *fh = file->private_data;
struct rc_dev *rcdev = fh->rc;
return events;
}
-static ssize_t ir_lirc_read_mode2(struct file *file, char __user *buffer,
- size_t length)
+static ssize_t lirc_read_mode2(struct file *file, char __user *buffer,
+ size_t length)
{
struct lirc_fh *fh = file->private_data;
struct rc_dev *rcdev = fh->rc;
return copied;
}
-static ssize_t ir_lirc_read_scancode(struct file *file, char __user *buffer,
- size_t length)
+static ssize_t lirc_read_scancode(struct file *file, char __user *buffer,
+ size_t length)
{
struct lirc_fh *fh = file->private_data;
struct rc_dev *rcdev = fh->rc;
return copied;
}
-static ssize_t ir_lirc_read(struct file *file, char __user *buffer,
- size_t length, loff_t *ppos)
+static ssize_t lirc_read(struct file *file, char __user *buffer, size_t length,
+ loff_t *ppos)
{
struct lirc_fh *fh = file->private_data;
struct rc_dev *rcdev = fh->rc;
return -ENODEV;
if (fh->rec_mode == LIRC_MODE_MODE2)
- return ir_lirc_read_mode2(file, buffer, length);
+ return lirc_read_mode2(file, buffer, length);
else /* LIRC_MODE_SCANCODE */
- return ir_lirc_read_scancode(file, buffer, length);
+ return lirc_read_scancode(file, buffer, length);
}
static const struct file_operations lirc_fops = {
.owner = THIS_MODULE,
- .write = ir_lirc_transmit_ir,
- .unlocked_ioctl = ir_lirc_ioctl,
+ .write = lirc_transmit,
+ .unlocked_ioctl = lirc_ioctl,
.compat_ioctl = compat_ptr_ioctl,
- .read = ir_lirc_read,
- .poll = ir_lirc_poll,
- .open = ir_lirc_open,
- .release = ir_lirc_close,
+ .read = lirc_read,
+ .poll = lirc_poll,
+ .open = lirc_open,
+ .release = lirc_close,
.llseek = no_llseek,
};
put_device(&rcdev->dev);
}
-int ir_lirc_register(struct rc_dev *dev)
+int lirc_register(struct rc_dev *dev)
{
const char *rx_type, *tx_type;
int err, minor;
return err;
}
-void ir_lirc_unregister(struct rc_dev *dev)
+void lirc_unregister(struct rc_dev *dev)
{
unsigned long flags;
struct lirc_fh *fh;
return PTR_ERR(lirc_class);
}
- retval = alloc_chrdev_region(&lirc_base_dev, 0, RC_DEV_MAX,
- "BaseRemoteCtl");
+ retval = alloc_chrdev_region(&lirc_base_dev, 0, RC_DEV_MAX, "lirc");
if (retval) {
class_destroy(lirc_class);
pr_err("alloc_chrdev_region failed\n");
struct mceusb_dev *ir = dev->priv;
unsigned int units;
- units = DIV_ROUND_CLOSEST(timeout, US_TO_NS(MCE_TIME_UNIT));
+ units = DIV_ROUND_CLOSEST(timeout, MCE_TIME_UNIT);
cmdbuf[2] = units >> 8;
cmdbuf[3] = units;
switch (subcmd) {
/* 2-byte return value commands */
case MCE_RSP_EQIRTIMEOUT:
- ir->rc->timeout = US_TO_NS((*hi << 8 | *lo) * MCE_TIME_UNIT);
+ ir->rc->timeout = (*hi << 8 | *lo) * MCE_TIME_UNIT;
break;
case MCE_RSP_EQIRNUMPORTS:
ir->num_txports = *hi;
ir->pulse_tunit += rawir.duration;
ir->pulse_count++;
}
- rawir.duration *= US_TO_NS(MCE_TIME_UNIT);
+ rawir.duration *= MCE_TIME_UNIT;
- dev_dbg(ir->dev, "Storing %s %u ns (%02x)",
+ dev_dbg(ir->dev, "Storing %s %u us (%02x)",
rawir.pulse ? "pulse" : "space",
rawir.duration, ir->buf_in[i]);
rc->dev.parent = dev;
rc->priv = ir;
rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
- rc->min_timeout = US_TO_NS(MCE_TIME_UNIT);
- rc->timeout = MS_TO_NS(100);
+ rc->min_timeout = MCE_TIME_UNIT;
+ rc->timeout = MS_TO_US(100);
if (!mceusb_model[ir->model].broken_irtimeout) {
rc->s_timeout = mceusb_set_timeout;
rc->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
duration = readl_relaxed(ir->reg + IR_DEC_REG1);
duration = FIELD_GET(REG1_TIME_IV_MASK, duration);
- rawir.duration = US_TO_NS(duration * MESON_TRATE);
+ rawir.duration = duration * MESON_TRATE;
status = readl_relaxed(ir->reg + IR_DEC_STATUS);
rawir.pulse = !!(status & STATUS_IR_DEC_IN);
map_name = of_get_property(node, "linux,rc-map-name", NULL);
ir->rc->map_name = map_name ? map_name : RC_MAP_EMPTY;
ir->rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
- ir->rc->rx_resolution = US_TO_NS(MESON_TRATE);
+ ir->rc->rx_resolution = MESON_TRATE;
ir->rc->min_timeout = 1;
ir->rc->timeout = IR_DEFAULT_TIMEOUT;
ir->rc->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
#define MTK_IR_END(v, p) ((v) == MTK_MAX_SAMPLES && (p) == 0)
/* Number of registers to record the pulse width */
#define MTK_CHKDATA_SZ 17
-/* Sample period in ns */
-#define MTK_IR_SAMPLE 46000
+/* Sample period in us */
+#define MTK_IR_SAMPLE 46
enum mtk_fields {
/* Register to setting software sampling period */
/* Inspect the ir samples */
for (i = 0, count = 0; i < ret && count < WAKEUP_MAX_SIZE; ++i) {
- /* NS to US */
- val = DIV_ROUND_UP(raw[i].duration, 1000L) / SAMPLE_PERIOD;
+ val = raw[i].duration / SAMPLE_PERIOD;
/* Split too large values into several smaller ones */
while (val > 0 && count < WAKEUP_MAX_SIZE) {
sample = nvt->buf[i];
rawir.pulse = ((sample & BUF_PULSE_BIT) != 0);
- rawir.duration = US_TO_NS((sample & BUF_LEN_MASK)
- * SAMPLE_PERIOD);
+ rawir.duration = (sample & BUF_LEN_MASK) * SAMPLE_PERIOD;
nvt_dbg("Storing %s with duration %d",
rawir.pulse ? "pulse" : "space", rawir.duration);
rdev->input_id.version = nvt->chip_minor;
rdev->driver_name = NVT_DRIVER_NAME;
rdev->map_name = RC_MAP_RC6_MCE;
- rdev->timeout = MS_TO_NS(100);
+ rdev->timeout = MS_TO_US(100);
/* rx resolution is hardwired to 50us atm, 1, 25, 100 also possible */
- rdev->rx_resolution = US_TO_NS(CIR_SAMPLE_PERIOD);
+ rdev->rx_resolution = CIR_SAMPLE_PERIOD;
#if 0
rdev->min_timeout = XYZ;
rdev->max_timeout = XYZ;
#define CIR_IOREG_LENGTH 0x0f
/* RX limit length, 8 high bits for SLCH, 8 low bits for SLCL */
-#define CIR_RX_LIMIT_COUNT (IR_DEFAULT_TIMEOUT / US_TO_NS(SAMPLE_PERIOD))
+#define CIR_RX_LIMIT_COUNT (IR_DEFAULT_TIMEOUT / SAMPLE_PERIOD)
/* CIR Regs */
#define CIR_IRCON 0x00
return !ev.carrier_report && !ev.reset;
}
-#define TO_US(duration) DIV_ROUND_CLOSEST((duration), 1000)
#define TO_STR(is_pulse) ((is_pulse) ? "pulse" : "space")
/* functions for IR encoders */
#ifdef CONFIG_LIRC
int lirc_dev_init(void);
void lirc_dev_exit(void);
-void ir_lirc_raw_event(struct rc_dev *dev, struct ir_raw_event ev);
-void ir_lirc_scancode_event(struct rc_dev *dev, struct lirc_scancode *lsc);
-int ir_lirc_register(struct rc_dev *dev);
-void ir_lirc_unregister(struct rc_dev *dev);
+void lirc_raw_event(struct rc_dev *dev, struct ir_raw_event ev);
+void lirc_scancode_event(struct rc_dev *dev, struct lirc_scancode *lsc);
+int lirc_register(struct rc_dev *dev);
+void lirc_unregister(struct rc_dev *dev);
struct rc_dev *rc_dev_get_from_fd(int fd);
#else
static inline int lirc_dev_init(void) { return 0; }
static inline void lirc_dev_exit(void) {}
-static inline void ir_lirc_raw_event(struct rc_dev *dev,
- struct ir_raw_event ev) { }
-static inline void ir_lirc_scancode_event(struct rc_dev *dev,
- struct lirc_scancode *lsc) { }
-static inline int ir_lirc_register(struct rc_dev *dev) { return 0; }
-static inline void ir_lirc_unregister(struct rc_dev *dev) { }
+static inline void lirc_raw_event(struct rc_dev *dev,
+ struct ir_raw_event ev) { }
+static inline void lirc_scancode_event(struct rc_dev *dev,
+ struct lirc_scancode *lsc) { }
+static inline int lirc_register(struct rc_dev *dev) { return 0; }
+static inline void lirc_unregister(struct rc_dev *dev) { }
#endif
/*
if (dev->enabled_protocols &
handler->protocols || !handler->protocols)
handler->decode(dev, ev);
- ir_lirc_raw_event(dev, ev);
+ lirc_raw_event(dev, ev);
raw->prev_ev = ev;
}
mutex_unlock(&ir_raw_handler_lock);
return -EINVAL;
dev_dbg(&dev->dev, "sample: (%05dus %s)\n",
- TO_US(ev->duration), TO_STR(ev->pulse));
+ ev->duration, TO_STR(ev->pulse));
if (!kfifo_put(&dev->raw->kfifo, *ev)) {
dev_err(&dev->dev, "IR event FIFO is full!\n");
return -EINVAL;
now = ktime_get();
- ev.duration = ktime_to_ns(ktime_sub(now, dev->raw->last_event));
+ ev.duration = ktime_to_us(ktime_sub(now, dev->raw->last_event));
ev.pulse = !pulse;
return ir_raw_event_store_with_timeout(dev, &ev);
if (timeout == 0)
timeout = IR_DEFAULT_TIMEOUT;
else
- timeout += MS_TO_NS(10);
+ timeout += MS_TO_US(10);
if (timeout < dev->min_timeout)
timeout = dev->min_timeout;
spin_lock_irqsave(&dev->raw->edge_spinlock, flags);
interval = ktime_sub(ktime_get(), dev->raw->last_event);
- if (ktime_to_ns(interval) >= dev->timeout) {
+ if (ktime_to_us(interval) >= dev->timeout) {
struct ir_raw_event ev = {
.timeout = true,
- .duration = ktime_to_ns(interval)
+ .duration = ktime_to_us(interval)
};
ir_raw_event_store(dev, &ev);
} else {
mod_timer(&dev->raw->edge_handle,
- jiffies + nsecs_to_jiffies(dev->timeout -
- ktime_to_ns(interval)));
+ jiffies + usecs_to_jiffies(dev->timeout -
+ ktime_to_us(interval)));
}
spin_unlock_irqrestore(&dev->raw->edge_spinlock, flags);
for (i = 0; i < count; i++) {
rawir.pulse = i % 2 ? false : true;
- rawir.duration = txbuf[i] * 1000;
+ rawir.duration = txbuf[i];
if (rawir.duration)
ir_raw_event_store_with_filter(dev, &rawir);
}
rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
rc->allowed_wakeup_protocols = RC_PROTO_BIT_ALL_IR_ENCODER;
rc->encode_wakeup = true;
- rc->timeout = 100 * 1000 * 1000; /* 100 ms */
+ rc->timeout = MS_TO_US(100); /* 100 ms */
rc->min_timeout = 1;
rc->max_timeout = UINT_MAX;
- rc->rx_resolution = 1000;
- rc->tx_resolution = 1000;
+ rc->rx_resolution = 1;
+ rc->tx_resolution = 1;
rc->s_tx_mask = loop_set_tx_mask;
rc->s_tx_carrier = loop_set_tx_carrier;
rc->s_tx_duty_cycle = loop_set_tx_duty_cycle;
};
if (dev->allowed_protocols != RC_PROTO_BIT_CEC)
- ir_lirc_scancode_event(dev, &sc);
+ lirc_scancode_event(dev, &sc);
spin_lock_irqsave(&dev->keylock, flags);
};
if (dev->allowed_protocols != RC_PROTO_BIT_CEC)
- ir_lirc_scancode_event(dev, &sc);
+ lirc_scancode_event(dev, &sc);
if (new_event && dev->keypressed)
ir_do_keyup(dev, false);
* keycodes with rc_keydown, so lirc must be registered first.
*/
if (dev->allowed_protocols != RC_PROTO_BIT_CEC) {
- rc = ir_lirc_register(dev);
+ rc = lirc_register(dev);
if (rc < 0)
goto out_dev;
}
rc_free_rx_device(dev);
out_lirc:
if (dev->allowed_protocols != RC_PROTO_BIT_CEC)
- ir_lirc_unregister(dev);
+ lirc_unregister(dev);
out_dev:
device_del(&dev->dev);
out_rx_free:
* that userspace polling will get notified.
*/
if (dev->allowed_protocols != RC_PROTO_BIT_CEC)
- ir_lirc_unregister(dev);
+ lirc_unregister(dev);
device_del(&dev->dev);
{
struct ir_raw_event rawir = {};
struct device *dev;
- unsigned int i, sig_size, single_len, offset, val;
+ unsigned int i, sig_size, offset, val;
u32 mod_freq;
dev = rr3->dev;
for (i = 0; i < sig_size; i++) {
offset = rr3->irdata.sigdata[i];
val = get_unaligned_be16(&rr3->irdata.lens[offset]);
- single_len = redrat3_len_to_us(val);
/* we should always get pulse/space/pulse/space samples */
if (i % 2)
else
rawir.pulse = true;
- rawir.duration = US_TO_NS(single_len);
+ rawir.duration = redrat3_len_to_us(val);
/* cap the value to IR_MAX_DURATION */
rawir.duration = (rawir.duration > IR_MAX_DURATION) ?
IR_MAX_DURATION : rawir.duration;
return timeout;
}
-static int redrat3_set_timeout(struct rc_dev *rc_dev, unsigned int timeoutns)
+static int redrat3_set_timeout(struct rc_dev *rc_dev, unsigned int timeoutus)
{
struct redrat3_dev *rr3 = rc_dev->priv;
struct usb_device *udev = rr3->udev;
if (!timeout)
return -ENOMEM;
- *timeout = cpu_to_be32(redrat3_us_to_len(timeoutns / 1000));
+ *timeout = cpu_to_be32(redrat3_us_to_len(timeoutus));
ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0), RR3_SET_IR_PARAM,
USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
RR3_IR_IO_SIG_TIMEOUT, 0, timeout, sizeof(*timeout),
rc->dev.parent = dev;
rc->priv = rr3;
rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
- rc->min_timeout = MS_TO_NS(RR3_RX_MIN_TIMEOUT);
- rc->max_timeout = MS_TO_NS(RR3_RX_MAX_TIMEOUT);
- rc->timeout = US_TO_NS(redrat3_get_timeout(rr3));
+ rc->min_timeout = MS_TO_US(RR3_RX_MIN_TIMEOUT);
+ rc->max_timeout = MS_TO_US(RR3_RX_MAX_TIMEOUT);
+ rc->timeout = redrat3_get_timeout(rr3);
rc->s_timeout = redrat3_set_timeout;
rc->tx_ir = redrat3_transmit_ir;
rc->s_tx_carrier = redrat3_set_tx_carrier;
rc->s_carrier_report = redrat3_wideband_receiver;
rc->driver_name = DRIVER_NAME;
- rc->rx_resolution = US_TO_NS(2);
+ rc->rx_resolution = 2;
rc->map_name = RC_MAP_HAUPPAUGE;
ret = rc_register_device(rc);
if (ptr > 0 && is_pulse) {
pulse += l;
- if (pulse > 250000) {
+ if (pulse > 250) {
ev.duration = space;
ev.pulse = false;
ir_raw_event_store_with_filter(serial_ir.rcdev, &ev);
}
if (!is_pulse) {
if (ptr == 0) {
- if (l > 20000000) {
+ if (l > 20000) {
space = l;
ptr++;
return;
}
} else {
- if (l > 20000000) {
+ if (l > 20000) {
space += pulse;
if (space > IR_MAX_DURATION)
space = IR_MAX_DURATION;
sense = sense ? 0 : 1;
}
} else {
- data = ktime_to_ns(delkt);
+ data = ktime_to_us(delkt);
}
frbwrite(data, !(dcd ^ sense));
serial_ir.lastkt = kt;
rcdev->min_timeout = 1;
rcdev->timeout = IR_DEFAULT_TIMEOUT;
rcdev->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
- rcdev->rx_resolution = 250000;
+ rcdev->rx_resolution = 250;
serial_ir.rcdev = rcdev;
/* Reserve io region. */
if ((iommap &&
- (devm_request_mem_region(&dev->dev, iommap, 8 << ioshift,
+ (devm_request_mem_region(&dev->dev, iommap, 8UL << ioshift,
KBUILD_MODNAME) == NULL)) ||
(!iommap && (devm_request_region(&dev->dev, io, 8,
KBUILD_MODNAME) == NULL))) {
} else {
val += TIME_CONST / 2;
}
- ev.duration = US_TO_NS(val);
+ ev.duration = val;
ir_raw_event_store_with_filter(rcdev, &ev);
}
mark /= dev->sample_div;
}
- ev.duration = US_TO_NS(mark);
+ ev.duration = mark;
ev.pulse = true;
ir_raw_event_store(dev->rdev, &ev);
if (!last_symbol) {
- ev.duration = US_TO_NS(symbol);
+ ev.duration = symbol;
ev.pulse = false;
ir_raw_event_store(dev->rdev, &ev);
} else {
rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
/* rx sampling rate is 10Mhz */
rdev->rx_resolution = 100;
- rdev->timeout = US_TO_NS(MAX_SYMB_TIME);
+ rdev->timeout = MAX_SYMB_TIME;
rdev->priv = rc_dev;
rdev->open = st_rc_open;
rdev->close = st_rc_close;
} else {
rawir.duration = delta;
rawir.duration -= sz->sum;
- rawir.duration = US_TO_NS(rawir.duration);
rawir.duration = (rawir.duration > IR_MAX_DURATION) ?
IR_MAX_DURATION : rawir.duration;
}
rawir.duration = ((int) value) * SZ_RESOLUTION;
rawir.duration += SZ_RESOLUTION / 2;
sz->sum += rawir.duration;
- rawir.duration = US_TO_NS(rawir.duration);
rawir.duration = (rawir.duration > IR_MAX_DURATION) ?
IR_MAX_DURATION : rawir.duration;
sz_push(sz, rawir);
rawir.duration = ((int) value) * SZ_RESOLUTION;
rawir.duration += SZ_RESOLUTION / 2;
sz->sum += rawir.duration;
- rawir.duration = US_TO_NS(rawir.duration);
sz_push(sz, rawir);
}
sz->decoder_state = PulseSpace;
/* FIXME: don't yet have a way to set this */
sz->timeout_enabled = true;
- sz->rdev->timeout = ((US_TO_NS(SZ_TIMEOUT * SZ_RESOLUTION) &
- IR_MAX_DURATION) | 0x03000000);
+ sz->rdev->timeout = SZ_TIMEOUT * SZ_RESOLUTION;
#if 0
/* not yet supported, depends on patches from maxim */
/* see also: LIRC_GET_REC_RESOLUTION and LIRC_SET_REC_TIMEOUT */
- sz->min_timeout = US_TO_NS(SZ_TIMEOUT * SZ_RESOLUTION);
- sz->max_timeout = US_TO_NS(SZ_TIMEOUT * SZ_RESOLUTION);
+ sz->min_timeout = SZ_TIMEOUT * SZ_RESOLUTION;
+ sz->max_timeout = SZ_TIMEOUT * SZ_RESOLUTION;
#endif
sz->signal_start = ktime_get_real();
ir->rc->dev.parent = dev;
ir->rc->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
/* Frequency after IR internal divider with sample period in ns */
- ir->rc->rx_resolution = (1000000000ul / (b_clk_freq / 64));
- ir->rc->timeout = MS_TO_NS(SUNXI_IR_TIMEOUT);
+ ir->rc->rx_resolution = (USEC_PER_SEC / (b_clk_freq / 64));
+ ir->rc->timeout = MS_TO_US(SUNXI_IR_TIMEOUT);
ir->rc->driver_name = SUNXI_IR_DEV;
ret = rc_register_device(ir->rc);
* messages per second (!), whether IR is idle or not.
*/
#define NUM_URBS 4
-#define NS_PER_BYTE 62500
-#define NS_PER_BIT (NS_PER_BYTE/8)
+#define US_PER_BYTE 62
+#define US_PER_BIT (US_PER_BYTE / 8)
struct ttusbir {
struct rc_dev *rc;
switch (v) {
case 0xfe:
rawir.pulse = false;
- rawir.duration = NS_PER_BYTE;
+ rawir.duration = US_PER_BYTE;
if (ir_raw_event_store_with_filter(tt->rc, &rawir))
event = true;
break;
case 0:
rawir.pulse = true;
- rawir.duration = NS_PER_BYTE;
+ rawir.duration = US_PER_BYTE;
if (ir_raw_event_store_with_filter(tt->rc, &rawir))
event = true;
break;
rawir.pulse = false;
}
- rawir.duration = NS_PER_BIT * (8 - b);
+ rawir.duration = US_PER_BIT * (8 - b);
if (ir_raw_event_store_with_filter(tt->rc, &rawir))
event = true;
rawir.pulse = !rawir.pulse;
- rawir.duration = NS_PER_BIT * b;
+ rawir.duration = US_PER_BIT * b;
if (ir_raw_event_store_with_filter(tt->rc, &rawir))
event = true;
break;
rc->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
/*
- * The precision is NS_PER_BIT, but since every 8th bit can be
- * overwritten with garbage the accuracy is at best 2 * NS_PER_BIT.
+ * The precision is US_PER_BIT, but since every 8th bit can be
+ * overwritten with garbage the accuracy is at best 2 * US_PER_BIT.
*/
- rc->rx_resolution = NS_PER_BIT;
+ rc->rx_resolution = 2 * US_PER_BIT;
ret = rc_register_device(rc);
if (ret) {
{
u8 irdata;
struct ir_raw_event rawir = {};
- unsigned duration;
/* Since RXHDLEV is set, at least 8 bytes are in the FIFO */
while (inb(data->sbase + WBCIR_REG_SP3_LSR) & WBCIR_RX_AVAIL) {
if (data->rxstate == WBCIR_RXSTATE_ERROR)
continue;
- duration = ((irdata & 0x7F) + 1) *
+ rawir.duration = ((irdata & 0x7F) + 1) *
(data->carrier_report_enabled ? 2 : 10);
rawir.pulse = irdata & 0x80 ? false : true;
- rawir.duration = US_TO_NS(duration);
if (rawir.pulse)
- data->pulse_duration += duration;
+ data->pulse_duration += rawir.duration;
ir_raw_event_store_with_filter(data->dev, &rawir);
}
/* Set a higher sampling resolution if carrier reports are enabled */
wbcir_select_bank(data, WBCIR_BANK_2);
- data->dev->rx_resolution = US_TO_NS(enable ? 2 : 10);
+ data->dev->rx_resolution = enable ? 2 : 10;
outb(enable ? 0x03 : 0x0f, data->sbase + WBCIR_REG_SP3_BGDL);
outb(0x00, data->sbase + WBCIR_REG_SP3_BGDH);
data->dev->min_timeout = 1;
data->dev->timeout = IR_DEFAULT_TIMEOUT;
data->dev->max_timeout = 10 * IR_DEFAULT_TIMEOUT;
- data->dev->rx_resolution = US_TO_NS(2);
+ data->dev->rx_resolution = 2;
data->dev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER;
data->dev->allowed_wakeup_protocols = RC_PROTO_BIT_NEC |
RC_PROTO_BIT_NECX | RC_PROTO_BIT_NEC32 | RC_PROTO_BIT_RC5 |
rdev->device_name = xbox_remote->rc_name;
rdev->input_phys = xbox_remote->rc_phys;
- rdev->timeout = MS_TO_NS(10);
+ rdev->timeout = MS_TO_US(10);
usb_to_input_id(xbox_remote->udev, &rdev->input_id);
rdev->dev.parent = &xbox_remote->interface->dev;
source "drivers/media/test-drivers/vicodec/Kconfig"
endif #V4L_TEST_DRIVERS
+
+menuconfig DVB_TEST_DRIVERS
+ bool "DVB test drivers"
+ depends on DVB_CORE && MEDIA_SUPPORT && I2C
+ help
+ Enables DVB test drivers.
+
+ This enables the DVB test drivers. They are meant as an aid for
+ DVB device driver writers and developers working on userspace
+ media applications.
+
+if DVB_TEST_DRIVERS
+
+source "drivers/media/test-drivers/vidtv/Kconfig"
+
+endif #DVB_TEST_DRIVERS
obj-$(CONFIG_VIDEO_VIVID) += vivid/
obj-$(CONFIG_VIDEO_VIM2M) += vim2m.o
obj-$(CONFIG_VIDEO_VICODEC) += vicodec/
+obj-$(CONFIG_DVB_VIDTV) += vidtv/
case V4L2_EVENT_SOURCE_CHANGE:
if (ctx->is_enc)
return -EINVAL;
- /* fall through */
+ fallthrough;
case V4L2_EVENT_EOS:
if (ctx->is_stateless)
return -EINVAL;
ctx->comp_size = 0;
ctx->header_size = 0;
ctx->comp_magic_cnt = 0;
- ctx->comp_has_frame = 0;
- ctx->comp_has_next_frame = 0;
+ ctx->comp_has_frame = false;
+ ctx->comp_has_next_frame = false;
}
}
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0-only
+config DVB_VIDTV
+ tristate "Virtual DVB Driver (vidtv)"
+ depends on DVB_CORE && MEDIA_SUPPORT && I2C
+ help
+ The virtual DVB test driver serves as a reference DVB driver and helps
+ validate the existing APIs in the media subsystem. It can also aid developers
+ working on userspace applications.
+
+
+ When in doubt, say N.
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+
+dvb-vidtv-tuner-objs := vidtv_tuner.o
+dvb-vidtv-demod-objs := vidtv_demod.o
+dvb-vidtv-bridge-objs := vidtv_bridge.o vidtv_common.o vidtv_ts.o vidtv_psi.o \
+ vidtv_pes.o vidtv_s302m.o vidtv_channel.o vidtv_mux.o
+
+obj-$(CONFIG_DVB_VIDTV) += dvb-vidtv-tuner.o dvb-vidtv-demod.o \
+ dvb-vidtv-bridge.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * The Virtual DTV test driver serves as a reference DVB driver and helps
+ * validate the existing APIs in the media subsystem. It can also aid
+ * developers working on userspace applications.
+ *
+ * When this module is loaded, it will attempt to modprobe 'dvb_vidtv_tuner' and 'dvb_vidtv_demod'.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#include <linux/moduleparam.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/dev_printk.h>
+#include <linux/time.h>
+#include <linux/types.h>
+#include <linux/workqueue.h>
+
+#include "vidtv_bridge.h"
+#include "vidtv_demod.h"
+#include "vidtv_tuner.h"
+#include "vidtv_ts.h"
+#include "vidtv_mux.h"
+#include "vidtv_common.h"
+
+//#define MUX_BUF_MAX_SZ
+//#define MUX_BUF_MIN_SZ
+#define TUNER_DEFAULT_ADDR 0x68
+#define DEMOD_DEFAULT_ADDR 0x60
+
+/* LNBf fake parameters: ranges used by an Universal (extended) European LNBf */
+#define LNB_CUT_FREQUENCY 11700000
+#define LNB_LOW_FREQ 9750000
+#define LNB_HIGH_FREQ 10600000
+
+
+static unsigned int drop_tslock_prob_on_low_snr;
+module_param(drop_tslock_prob_on_low_snr, uint, 0);
+MODULE_PARM_DESC(drop_tslock_prob_on_low_snr,
+ "Probability of losing the TS lock if the signal quality is bad");
+
+static unsigned int recover_tslock_prob_on_good_snr;
+module_param(recover_tslock_prob_on_good_snr, uint, 0);
+MODULE_PARM_DESC(recover_tslock_prob_on_good_snr,
+ "Probability recovering the TS lock when the signal improves");
+
+static unsigned int mock_power_up_delay_msec;
+module_param(mock_power_up_delay_msec, uint, 0);
+MODULE_PARM_DESC(mock_power_up_delay_msec, "Simulate a power up delay");
+
+static unsigned int mock_tune_delay_msec;
+module_param(mock_tune_delay_msec, uint, 0);
+MODULE_PARM_DESC(mock_tune_delay_msec, "Simulate a tune delay");
+
+static unsigned int vidtv_valid_dvb_t_freqs[NUM_VALID_TUNER_FREQS] = {
+ 474000000
+};
+
+module_param_array(vidtv_valid_dvb_t_freqs, uint, NULL, 0);
+MODULE_PARM_DESC(vidtv_valid_dvb_t_freqs,
+ "Valid DVB-T frequencies to simulate, in Hz");
+
+static unsigned int vidtv_valid_dvb_c_freqs[NUM_VALID_TUNER_FREQS] = {
+ 474000000
+};
+
+module_param_array(vidtv_valid_dvb_c_freqs, uint, NULL, 0);
+MODULE_PARM_DESC(vidtv_valid_dvb_c_freqs,
+ "Valid DVB-C frequencies to simulate, in Hz");
+
+static unsigned int vidtv_valid_dvb_s_freqs[NUM_VALID_TUNER_FREQS] = {
+ 11362000
+};
+module_param_array(vidtv_valid_dvb_s_freqs, uint, NULL, 0);
+MODULE_PARM_DESC(vidtv_valid_dvb_s_freqs,
+ "Valid DVB-S/S2 frequencies to simulate at Ku-Band, in kHz");
+
+static unsigned int max_frequency_shift_hz;
+module_param(max_frequency_shift_hz, uint, 0);
+MODULE_PARM_DESC(max_frequency_shift_hz,
+ "Maximum shift in HZ allowed when tuning in a channel");
+
+DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nums);
+
+/*
+ * Influences the signal acquisition time. See ISO/IEC 13818-1 : 2000. p. 113.
+ */
+static unsigned int si_period_msec = 40;
+module_param(si_period_msec, uint, 0);
+MODULE_PARM_DESC(si_period_msec, "How often to send SI packets. Default: 40ms");
+
+static unsigned int pcr_period_msec = 40;
+module_param(pcr_period_msec, uint, 0);
+MODULE_PARM_DESC(pcr_period_msec, "How often to send PCR packets. Default: 40ms");
+
+static unsigned int mux_rate_kbytes_sec = 4096;
+module_param(mux_rate_kbytes_sec, uint, 0);
+MODULE_PARM_DESC(mux_rate_kbytes_sec, "Mux rate: will pad stream if below");
+
+static unsigned int pcr_pid = 0x200;
+module_param(pcr_pid, uint, 0);
+MODULE_PARM_DESC(pcr_pid, "PCR PID for all channels: defaults to 0x200");
+
+static unsigned int mux_buf_sz_pkts;
+module_param(mux_buf_sz_pkts, uint, 0);
+MODULE_PARM_DESC(mux_buf_sz_pkts, "Size for the internal mux buffer in multiples of 188 bytes");
+
+#define MUX_BUF_MIN_SZ 90164
+#define MUX_BUF_MAX_SZ (MUX_BUF_MIN_SZ * 10)
+
+static u32 vidtv_bridge_mux_buf_sz_for_mux_rate(void)
+{
+ u32 max_elapsed_time_msecs = VIDTV_MAX_SLEEP_USECS / USEC_PER_MSEC;
+ u32 nbytes_expected;
+ u32 mux_buf_sz = mux_buf_sz_pkts * TS_PACKET_LEN;
+
+ nbytes_expected = mux_rate_kbytes_sec;
+ nbytes_expected *= max_elapsed_time_msecs;
+
+ mux_buf_sz = roundup(nbytes_expected, TS_PACKET_LEN);
+ mux_buf_sz += mux_buf_sz / 10;
+
+ if (mux_buf_sz < MUX_BUF_MIN_SZ)
+ mux_buf_sz = MUX_BUF_MIN_SZ;
+
+ if (mux_buf_sz > MUX_BUF_MAX_SZ)
+ mux_buf_sz = MUX_BUF_MAX_SZ;
+
+ return mux_buf_sz;
+}
+
+static bool vidtv_bridge_check_demod_lock(struct vidtv_dvb *dvb, u32 n)
+{
+ enum fe_status status;
+
+ dvb->fe[n]->ops.read_status(dvb->fe[n], &status);
+
+ return status == (FE_HAS_SIGNAL |
+ FE_HAS_CARRIER |
+ FE_HAS_VITERBI |
+ FE_HAS_SYNC |
+ FE_HAS_LOCK);
+}
+
+static void
+vidtv_bridge_on_new_pkts_avail(void *priv, u8 *buf, u32 npkts)
+{
+ /*
+ * called on a separate thread by the mux when new packets become
+ * available
+ */
+ struct vidtv_dvb *dvb = (struct vidtv_dvb *)priv;
+
+ /* drop packets if we lose the lock */
+ if (vidtv_bridge_check_demod_lock(dvb, 0))
+ dvb_dmx_swfilter_packets(&dvb->demux, buf, npkts);
+}
+
+static int vidtv_start_streaming(struct vidtv_dvb *dvb)
+{
+ struct vidtv_mux_init_args mux_args = {0};
+ struct device *dev = &dvb->pdev->dev;
+ u32 mux_buf_sz;
+
+ if (dvb->streaming) {
+ dev_warn_ratelimited(dev, "Already streaming. Skipping.\n");
+ return 0;
+ }
+
+ mux_buf_sz = (mux_buf_sz_pkts) ? mux_buf_sz_pkts : vidtv_bridge_mux_buf_sz_for_mux_rate();
+
+ mux_args.mux_rate_kbytes_sec = mux_rate_kbytes_sec;
+ mux_args.on_new_packets_available_cb = vidtv_bridge_on_new_pkts_avail;
+ mux_args.mux_buf_sz = mux_buf_sz;
+ mux_args.pcr_period_usecs = pcr_period_msec * 1000;
+ mux_args.si_period_usecs = si_period_msec * 1000;
+ mux_args.pcr_pid = pcr_pid;
+ mux_args.transport_stream_id = VIDTV_DEFAULT_TS_ID;
+ mux_args.priv = dvb;
+
+ dvb->streaming = true;
+ dvb->mux = vidtv_mux_init(dvb->fe[0], dev, mux_args);
+ vidtv_mux_start_thread(dvb->mux);
+
+ dev_dbg_ratelimited(dev, "Started streaming\n");
+ return 0;
+}
+
+static int vidtv_stop_streaming(struct vidtv_dvb *dvb)
+{
+ struct device *dev = &dvb->pdev->dev;
+
+ dvb->streaming = false;
+ vidtv_mux_stop_thread(dvb->mux);
+ vidtv_mux_destroy(dvb->mux);
+ dvb->mux = NULL;
+
+ dev_dbg_ratelimited(dev, "Stopped streaming\n");
+ return 0;
+}
+
+static int vidtv_start_feed(struct dvb_demux_feed *feed)
+{
+ struct dvb_demux *demux = feed->demux;
+ struct vidtv_dvb *dvb = demux->priv;
+ int rc;
+ int ret;
+
+ if (!demux->dmx.frontend)
+ return -EINVAL;
+
+ mutex_lock(&dvb->feed_lock);
+
+ dvb->nfeeds++;
+ rc = dvb->nfeeds;
+
+ if (dvb->nfeeds == 1) {
+ ret = vidtv_start_streaming(dvb);
+ if (ret < 0)
+ rc = ret;
+ }
+
+ mutex_unlock(&dvb->feed_lock);
+ return rc;
+}
+
+static int vidtv_stop_feed(struct dvb_demux_feed *feed)
+{
+ struct dvb_demux *demux = feed->demux;
+ struct vidtv_dvb *dvb = demux->priv;
+ int err = 0;
+
+ mutex_lock(&dvb->feed_lock);
+ dvb->nfeeds--;
+
+ if (!dvb->nfeeds)
+ err = vidtv_stop_streaming(dvb);
+
+ mutex_unlock(&dvb->feed_lock);
+ return err;
+}
+
+static struct dvb_frontend *vidtv_get_frontend_ptr(struct i2c_client *c)
+{
+ /* the demod will set this when its probe function runs */
+ struct vidtv_demod_state *state = i2c_get_clientdata(c);
+
+ return &state->frontend;
+}
+
+static int vidtv_master_xfer(struct i2c_adapter *i2c_adap,
+ struct i2c_msg msgs[],
+ int num)
+{
+ return 0;
+}
+
+static u32 vidtv_i2c_func(struct i2c_adapter *adapter)
+{
+ return I2C_FUNC_I2C;
+}
+
+static const struct i2c_algorithm vidtv_i2c_algorithm = {
+ .master_xfer = vidtv_master_xfer,
+ .functionality = vidtv_i2c_func,
+};
+
+static int vidtv_bridge_i2c_register_adap(struct vidtv_dvb *dvb)
+{
+ struct i2c_adapter *i2c_adapter = &dvb->i2c_adapter;
+
+ strscpy(i2c_adapter->name, "vidtv_i2c", sizeof(i2c_adapter->name));
+ i2c_adapter->owner = THIS_MODULE;
+ i2c_adapter->algo = &vidtv_i2c_algorithm;
+ i2c_adapter->algo_data = NULL;
+ i2c_adapter->timeout = 500;
+ i2c_adapter->retries = 3;
+ i2c_adapter->dev.parent = &dvb->pdev->dev;
+
+ i2c_set_adapdata(i2c_adapter, dvb);
+ return i2c_add_adapter(&dvb->i2c_adapter);
+}
+
+static int vidtv_bridge_register_adap(struct vidtv_dvb *dvb)
+{
+ int ret = 0;
+
+ ret = dvb_register_adapter(&dvb->adapter,
+ KBUILD_MODNAME,
+ THIS_MODULE,
+ &dvb->i2c_adapter.dev,
+ adapter_nums);
+
+ return ret;
+}
+
+static int vidtv_bridge_dmx_init(struct vidtv_dvb *dvb)
+{
+ dvb->demux.dmx.capabilities = DMX_TS_FILTERING |
+ DMX_SECTION_FILTERING;
+
+ dvb->demux.priv = dvb;
+ dvb->demux.filternum = 256;
+ dvb->demux.feednum = 256;
+ dvb->demux.start_feed = vidtv_start_feed;
+ dvb->demux.stop_feed = vidtv_stop_feed;
+
+ return dvb_dmx_init(&dvb->demux);
+}
+
+static int vidtv_bridge_dmxdev_init(struct vidtv_dvb *dvb)
+{
+ dvb->dmx_dev.filternum = 256;
+ dvb->dmx_dev.demux = &dvb->demux.dmx;
+ dvb->dmx_dev.capabilities = 0;
+
+ return dvb_dmxdev_init(&dvb->dmx_dev, &dvb->adapter);
+}
+
+static int vidtv_bridge_probe_demod(struct vidtv_dvb *dvb, u32 n)
+{
+ struct vidtv_demod_config cfg = {};
+
+ cfg.drop_tslock_prob_on_low_snr = drop_tslock_prob_on_low_snr;
+ cfg.recover_tslock_prob_on_good_snr = recover_tslock_prob_on_good_snr;
+
+ dvb->i2c_client_demod[n] = dvb_module_probe("dvb_vidtv_demod",
+ NULL,
+ &dvb->i2c_adapter,
+ DEMOD_DEFAULT_ADDR,
+ &cfg);
+
+ /* driver will not work anyways so bail out */
+ if (!dvb->i2c_client_demod[n])
+ return -ENODEV;
+
+ /* retrieve a ptr to the frontend state */
+ dvb->fe[n] = vidtv_get_frontend_ptr(dvb->i2c_client_demod[n]);
+
+ return 0;
+}
+
+static int vidtv_bridge_probe_tuner(struct vidtv_dvb *dvb, u32 n)
+{
+ struct vidtv_tuner_config cfg = {};
+ u32 freq;
+ int i;
+
+ cfg.fe = dvb->fe[n];
+ cfg.mock_power_up_delay_msec = mock_power_up_delay_msec;
+ cfg.mock_tune_delay_msec = mock_tune_delay_msec;
+
+ /* TODO: check if the frequencies are at a valid range */
+
+ memcpy(cfg.vidtv_valid_dvb_t_freqs,
+ vidtv_valid_dvb_t_freqs,
+ sizeof(vidtv_valid_dvb_t_freqs));
+
+ memcpy(cfg.vidtv_valid_dvb_c_freqs,
+ vidtv_valid_dvb_c_freqs,
+ sizeof(vidtv_valid_dvb_c_freqs));
+
+ /*
+ * Convert Satellite frequencies from Ku-band in kHZ into S-band
+ * frequencies in Hz.
+ */
+ for (i = 0; i < ARRAY_SIZE(vidtv_valid_dvb_s_freqs); i++) {
+ freq = vidtv_valid_dvb_s_freqs[i];
+ if (freq) {
+ if (freq < LNB_CUT_FREQUENCY)
+ freq = abs(freq - LNB_LOW_FREQ);
+ else
+ freq = abs(freq - LNB_HIGH_FREQ);
+ }
+ cfg.vidtv_valid_dvb_s_freqs[i] = freq;
+ }
+
+ cfg.max_frequency_shift_hz = max_frequency_shift_hz;
+
+ dvb->i2c_client_tuner[n] = dvb_module_probe("dvb_vidtv_tuner",
+ NULL,
+ &dvb->i2c_adapter,
+ TUNER_DEFAULT_ADDR,
+ &cfg);
+
+ return (dvb->i2c_client_tuner[n]) ? 0 : -ENODEV;
+}
+
+static int vidtv_bridge_dvb_init(struct vidtv_dvb *dvb)
+{
+ int ret;
+ int i;
+ int j;
+
+ ret = vidtv_bridge_i2c_register_adap(dvb);
+ if (ret < 0)
+ goto fail_i2c;
+
+ ret = vidtv_bridge_register_adap(dvb);
+ if (ret < 0)
+ goto fail_adapter;
+
+ for (i = 0; i < NUM_FE; ++i) {
+ ret = vidtv_bridge_probe_demod(dvb, i);
+ if (ret < 0)
+ goto fail_demod_probe;
+
+ ret = vidtv_bridge_probe_tuner(dvb, i);
+ if (ret < 0)
+ goto fail_tuner_probe;
+
+ ret = dvb_register_frontend(&dvb->adapter, dvb->fe[i]);
+ if (ret < 0)
+ goto fail_fe;
+ }
+
+ ret = vidtv_bridge_dmx_init(dvb);
+ if (ret < 0)
+ goto fail_dmx;
+
+ ret = vidtv_bridge_dmxdev_init(dvb);
+ if (ret < 0)
+ goto fail_dmx_dev;
+
+ for (j = 0; j < NUM_FE; ++j) {
+ ret = dvb->demux.dmx.connect_frontend(&dvb->demux.dmx,
+ &dvb->dmx_fe[j]);
+ if (ret < 0)
+ goto fail_dmx_conn;
+
+ /*
+ * The source of the demux is a frontend connected
+ * to the demux.
+ */
+ dvb->dmx_fe[j].source = DMX_FRONTEND_0;
+ }
+
+ return ret;
+
+fail_dmx_conn:
+ for (j = j - 1; j >= 0; --j)
+ dvb->demux.dmx.remove_frontend(&dvb->demux.dmx,
+ &dvb->dmx_fe[j]);
+fail_dmx_dev:
+ dvb_dmxdev_release(&dvb->dmx_dev);
+fail_dmx:
+ dvb_dmx_release(&dvb->demux);
+fail_fe:
+ for (j = i; j >= 0; --j)
+ dvb_unregister_frontend(dvb->fe[j]);
+fail_tuner_probe:
+ for (j = i; j >= 0; --j)
+ if (dvb->i2c_client_tuner[j])
+ dvb_module_release(dvb->i2c_client_tuner[j]);
+
+fail_demod_probe:
+ for (j = i; j >= 0; --j)
+ if (dvb->i2c_client_demod[j])
+ dvb_module_release(dvb->i2c_client_demod[j]);
+
+fail_adapter:
+ dvb_unregister_adapter(&dvb->adapter);
+
+fail_i2c:
+ i2c_del_adapter(&dvb->i2c_adapter);
+
+ return ret;
+}
+
+static int vidtv_bridge_probe(struct platform_device *pdev)
+{
+ struct vidtv_dvb *dvb;
+ int ret;
+
+ dvb = kzalloc(sizeof(*dvb), GFP_KERNEL);
+ if (!dvb)
+ return -ENOMEM;
+
+ dvb->pdev = pdev;
+
+ ret = vidtv_bridge_dvb_init(dvb);
+ if (ret < 0)
+ goto err_dvb;
+
+ mutex_init(&dvb->feed_lock);
+
+ platform_set_drvdata(pdev, dvb);
+
+ dev_info(&pdev->dev, "Successfully initialized vidtv!\n");
+ return ret;
+
+err_dvb:
+ kfree(dvb);
+ return ret;
+}
+
+static int vidtv_bridge_remove(struct platform_device *pdev)
+{
+ struct vidtv_dvb *dvb;
+ u32 i;
+
+ dvb = platform_get_drvdata(pdev);
+
+ mutex_destroy(&dvb->feed_lock);
+
+ for (i = 0; i < NUM_FE; ++i) {
+ dvb_unregister_frontend(dvb->fe[i]);
+ dvb_module_release(dvb->i2c_client_tuner[i]);
+ dvb_module_release(dvb->i2c_client_demod[i]);
+ }
+
+ dvb_dmxdev_release(&dvb->dmx_dev);
+ dvb_dmx_release(&dvb->demux);
+ dvb_unregister_adapter(&dvb->adapter);
+
+ return 0;
+}
+
+static void vidtv_bridge_dev_release(struct device *dev)
+{
+}
+
+static struct platform_device vidtv_bridge_dev = {
+ .name = "vidtv_bridge",
+ .dev.release = vidtv_bridge_dev_release,
+};
+
+static struct platform_driver vidtv_bridge_driver = {
+ .driver = {
+ .name = "vidtv_bridge",
+ .suppress_bind_attrs = true,
+ },
+ .probe = vidtv_bridge_probe,
+ .remove = vidtv_bridge_remove,
+};
+
+static void __exit vidtv_bridge_exit(void)
+{
+ platform_driver_unregister(&vidtv_bridge_driver);
+ platform_device_unregister(&vidtv_bridge_dev);
+}
+
+static int __init vidtv_bridge_init(void)
+{
+ int ret;
+
+ ret = platform_device_register(&vidtv_bridge_dev);
+ if (ret)
+ return ret;
+
+ ret = platform_driver_register(&vidtv_bridge_driver);
+ if (ret)
+ platform_device_unregister(&vidtv_bridge_dev);
+
+ return ret;
+}
+
+module_init(vidtv_bridge_init);
+module_exit(vidtv_bridge_exit);
+
+MODULE_DESCRIPTION("Virtual Digital TV Test Driver");
+MODULE_AUTHOR("Daniel W. S. Almeida");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("vidtv");
+MODULE_ALIAS("dvb_vidtv");
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * The Virtual DTV test driver serves as a reference DVB driver and helps
+ * validate the existing APIs in the media subsystem. It can also aid
+ * developers working on userspace applications.
+ *
+ * When this module is loaded, it will attempt to modprobe 'dvb_vidtv_tuner' and 'dvb_vidtv_demod'.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#ifndef VIDTV_BRIDGE_H
+#define VIDTV_BRIDGE_H
+
+/*
+ * For now, only one frontend is supported. See vidtv_start_streaming()
+ */
+#define NUM_FE 1
+
+#include <linux/i2c.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include "vidtv_mux.h"
+
+/**
+ * struct vidtv_dvb - Vidtv bridge state
+ * @pdev: The platform device. Obtained when the bridge is probed.
+ * @fe: The frontends. Obtained when probing the demodulator modules.
+ * @adapter: Represents a DTV adapter. See 'dvb_register_adapter'.
+ * @demux: The demux used by the dvb_dmx_swfilter_packets() call.
+ * @dmx_dev: Represents a demux device.
+ * @dmx_frontend: The frontends associated with the demux.
+ * @i2c_adapter: The i2c_adapter associated with the bridge driver.
+ * @i2c_client_demod: The i2c_clients associated with the demodulator modules.
+ * @i2c_client_tuner: The i2c_clients associated with the tuner modules.
+ * @nfeeds: The number of feeds active.
+ * @feed_lock: Protects access to the start/stop stream logic/data.
+ * @streaming: Whether we are streaming now.
+ * @mux: The abstraction responsible for delivering MPEG TS packets to the bridge.
+ */
+struct vidtv_dvb {
+ struct platform_device *pdev;
+ struct dvb_frontend *fe[NUM_FE];
+ struct dvb_adapter adapter;
+ struct dvb_demux demux;
+ struct dmxdev dmx_dev;
+ struct dmx_frontend dmx_fe[NUM_FE];
+ struct i2c_adapter i2c_adapter;
+ struct i2c_client *i2c_client_demod[NUM_FE];
+ struct i2c_client *i2c_client_tuner[NUM_FE];
+
+ u32 nfeeds;
+ struct mutex feed_lock; /* Protects access to the start/stop stream logic/data. */
+
+ bool streaming;
+
+ struct vidtv_mux *mux;
+};
+
+#endif // VIDTV_BRIDG_H
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Vidtv serves as a reference DVB driver and helps validate the existing APIs
+ * in the media subsystem. It can also aid developers working on userspace
+ * applications.
+ *
+ * This file contains the code for a 'channel' abstraction.
+ *
+ * When vidtv boots, it will create some hardcoded channels.
+ * Their services will be concatenated to populate the SDT.
+ * Their programs will be concatenated to populate the PAT
+ * For each program in the PAT, a PMT section will be created
+ * The PMT section for a channel will be assigned its streams.
+ * Every stream will have its corresponding encoder polled to produce TS packets
+ * These packets may be interleaved by the mux and then delivered to the bridge
+ *
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/dev_printk.h>
+#include <linux/ratelimit.h>
+
+#include "vidtv_channel.h"
+#include "vidtv_psi.h"
+#include "vidtv_encoder.h"
+#include "vidtv_mux.h"
+#include "vidtv_common.h"
+#include "vidtv_s302m.h"
+
+static void vidtv_channel_encoder_destroy(struct vidtv_encoder *e)
+{
+ struct vidtv_encoder *curr = e;
+ struct vidtv_encoder *tmp = NULL;
+
+ while (curr) {
+ /* forward the call to the derived type */
+ tmp = curr;
+ curr = curr->next;
+ tmp->destroy(tmp);
+ }
+}
+
+#define ENCODING_ISO8859_15 "\x0b"
+
+struct vidtv_channel
+*vidtv_channel_s302m_init(struct vidtv_channel *head, u16 transport_stream_id)
+{
+ /*
+ * init an audio only channel with a s302m encoder
+ */
+ const u16 s302m_service_id = 0x880;
+ const u16 s302m_program_num = 0x880;
+ const u16 s302m_program_pid = 0x101; /* packet id for PMT*/
+ const u16 s302m_es_pid = 0x111; /* packet id for the ES */
+ const __be32 s302m_fid = cpu_to_be32(VIDTV_S302M_FORMAT_IDENTIFIER);
+
+ char *name = ENCODING_ISO8859_15 "Beethoven";
+ char *provider = ENCODING_ISO8859_15 "LinuxTV.org";
+
+ struct vidtv_channel *s302m = kzalloc(sizeof(*s302m), GFP_KERNEL);
+ struct vidtv_s302m_encoder_init_args encoder_args = {};
+
+ s302m->name = kstrdup(name, GFP_KERNEL);
+
+ s302m->service = vidtv_psi_sdt_service_init(NULL, s302m_service_id);
+
+ s302m->service->descriptor = (struct vidtv_psi_desc *)
+ vidtv_psi_service_desc_init(NULL,
+ DIGITAL_TELEVISION_SERVICE,
+ name,
+ provider);
+
+ s302m->transport_stream_id = transport_stream_id;
+
+ s302m->program = vidtv_psi_pat_program_init(NULL,
+ s302m_service_id,
+ s302m_program_pid);
+
+ s302m->program_num = s302m_program_num;
+
+ s302m->streams = vidtv_psi_pmt_stream_init(NULL,
+ STREAM_PRIVATE_DATA,
+ s302m_es_pid);
+
+ s302m->streams->descriptor = (struct vidtv_psi_desc *)
+ vidtv_psi_registration_desc_init(NULL,
+ s302m_fid,
+ NULL,
+ 0);
+ encoder_args.es_pid = s302m_es_pid;
+
+ s302m->encoders = vidtv_s302m_encoder_init(encoder_args);
+
+ if (head) {
+ while (head->next)
+ head = head->next;
+
+ head->next = s302m;
+ }
+
+ return s302m;
+}
+
+static struct vidtv_psi_table_sdt_service
+*vidtv_channel_sdt_serv_cat_into_new(struct vidtv_mux *m)
+{
+ /* Concatenate the services */
+ const struct vidtv_channel *cur_chnl = m->channels;
+
+ struct vidtv_psi_table_sdt_service *curr = NULL;
+ struct vidtv_psi_table_sdt_service *head = NULL;
+ struct vidtv_psi_table_sdt_service *tail = NULL;
+
+ struct vidtv_psi_desc *desc = NULL;
+ u16 service_id;
+
+ if (!cur_chnl)
+ return NULL;
+
+ while (cur_chnl) {
+ curr = cur_chnl->service;
+
+ if (!curr)
+ dev_warn_ratelimited(m->dev,
+ "No services found for channel %s\n", cur_chnl->name);
+
+ while (curr) {
+ service_id = be16_to_cpu(curr->service_id);
+ tail = vidtv_psi_sdt_service_init(tail, service_id);
+
+ desc = vidtv_psi_desc_clone(curr->descriptor);
+ vidtv_psi_desc_assign(&tail->descriptor, desc);
+
+ if (!head)
+ head = tail;
+
+ curr = curr->next;
+ }
+
+ cur_chnl = cur_chnl->next;
+ }
+
+ return head;
+}
+
+static struct vidtv_psi_table_pat_program*
+vidtv_channel_pat_prog_cat_into_new(struct vidtv_mux *m)
+{
+ /* Concatenate the programs */
+ const struct vidtv_channel *cur_chnl = m->channels;
+ struct vidtv_psi_table_pat_program *curr = NULL;
+ struct vidtv_psi_table_pat_program *head = NULL;
+ struct vidtv_psi_table_pat_program *tail = NULL;
+ u16 serv_id;
+ u16 pid;
+
+ if (!cur_chnl)
+ return NULL;
+
+ while (cur_chnl) {
+ curr = cur_chnl->program;
+
+ if (!curr)
+ dev_warn_ratelimited(m->dev,
+ "No programs found for channel %s\n",
+ cur_chnl->name);
+
+ while (curr) {
+ serv_id = be16_to_cpu(curr->service_id);
+ pid = vidtv_psi_get_pat_program_pid(curr);
+ tail = vidtv_psi_pat_program_init(tail,
+ serv_id,
+ pid);
+
+ if (!head)
+ head = tail;
+
+ curr = curr->next;
+ }
+
+ cur_chnl = cur_chnl->next;
+ }
+
+ return head;
+}
+
+static void
+vidtv_channel_pmt_match_sections(struct vidtv_channel *channels,
+ struct vidtv_psi_table_pmt **sections,
+ u32 nsections)
+{
+ /*
+ * Match channels to their respective PMT sections, then assign the
+ * streams
+ */
+ struct vidtv_psi_table_pmt *curr_section = NULL;
+ struct vidtv_channel *cur_chnl = channels;
+
+ struct vidtv_psi_table_pmt_stream *s = NULL;
+ struct vidtv_psi_table_pmt_stream *head = NULL;
+ struct vidtv_psi_table_pmt_stream *tail = NULL;
+
+ struct vidtv_psi_desc *desc = NULL;
+ u32 j;
+ u16 curr_id;
+ u16 e_pid; /* elementary stream pid */
+
+ while (cur_chnl) {
+ for (j = 0; j < nsections; ++j) {
+ curr_section = sections[j];
+
+ if (!curr_section)
+ continue;
+
+ curr_id = be16_to_cpu(curr_section->header.id);
+
+ /* we got a match */
+ if (curr_id == cur_chnl->program_num) {
+ s = cur_chnl->streams;
+
+ /* clone the streams for the PMT */
+ while (s) {
+ e_pid = vidtv_psi_pmt_stream_get_elem_pid(s);
+ tail = vidtv_psi_pmt_stream_init(tail,
+ s->type,
+ e_pid);
+
+ if (!head)
+ head = tail;
+
+ desc = vidtv_psi_desc_clone(s->descriptor);
+ vidtv_psi_desc_assign(&tail->descriptor, desc);
+
+ s = s->next;
+ }
+
+ vidtv_psi_pmt_stream_assign(curr_section, head);
+ break;
+ }
+ }
+
+ cur_chnl = cur_chnl->next;
+ }
+}
+
+void vidtv_channel_si_init(struct vidtv_mux *m)
+{
+ struct vidtv_psi_table_pat_program *programs = NULL;
+ struct vidtv_psi_table_sdt_service *services = NULL;
+
+ m->si.pat = vidtv_psi_pat_table_init(m->transport_stream_id);
+
+ m->si.sdt = vidtv_psi_sdt_table_init(m->transport_stream_id);
+
+ programs = vidtv_channel_pat_prog_cat_into_new(m);
+ services = vidtv_channel_sdt_serv_cat_into_new(m);
+
+ /* assemble all programs and assign to PAT */
+ vidtv_psi_pat_program_assign(m->si.pat, programs);
+
+ /* assemble all services and assign to SDT */
+ vidtv_psi_sdt_service_assign(m->si.sdt, services);
+
+ m->si.pmt_secs = vidtv_psi_pmt_create_sec_for_each_pat_entry(m->si.pat, m->pcr_pid);
+
+ vidtv_channel_pmt_match_sections(m->channels,
+ m->si.pmt_secs,
+ m->si.pat->programs);
+}
+
+void vidtv_channel_si_destroy(struct vidtv_mux *m)
+{
+ u32 i;
+ u16 num_programs = m->si.pat->programs;
+
+ vidtv_psi_pat_table_destroy(m->si.pat);
+
+ for (i = 0; i < num_programs; ++i)
+ vidtv_psi_pmt_table_destroy(m->si.pmt_secs[i]);
+
+ kfree(m->si.pmt_secs);
+ vidtv_psi_sdt_table_destroy(m->si.sdt);
+}
+
+void vidtv_channels_init(struct vidtv_mux *m)
+{
+ /* this is the place to add new 'channels' for vidtv */
+ m->channels = vidtv_channel_s302m_init(NULL, m->transport_stream_id);
+}
+
+void vidtv_channels_destroy(struct vidtv_mux *m)
+{
+ struct vidtv_channel *curr = m->channels;
+ struct vidtv_channel *tmp = NULL;
+
+ while (curr) {
+ kfree(curr->name);
+ vidtv_psi_sdt_service_destroy(curr->service);
+ vidtv_psi_pat_program_destroy(curr->program);
+ vidtv_psi_pmt_stream_destroy(curr->streams);
+ vidtv_channel_encoder_destroy(curr->encoders);
+
+ tmp = curr;
+ curr = curr->next;
+ kfree(tmp);
+ }
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Vidtv serves as a reference DVB driver and helps validate the existing APIs
+ * in the media subsystem. It can also aid developers working on userspace
+ * applications.
+ *
+ * This file contains the code for a 'channel' abstraction.
+ *
+ * When vidtv boots, it will create some hardcoded channels.
+ * Their services will be concatenated to populate the SDT.
+ * Their programs will be concatenated to populate the PAT
+ * For each program in the PAT, a PMT section will be created
+ * The PMT section for a channel will be assigned its streams.
+ * Every stream will have its corresponding encoder polled to produce TS packets
+ * These packets may be interleaved by the mux and then delivered to the bridge
+ *
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#ifndef VIDTV_CHANNEL_H
+#define VIDTV_CHANNEL_H
+
+#include <linux/types.h>
+#include "vidtv_psi.h"
+#include "vidtv_encoder.h"
+#include "vidtv_mux.h"
+
+/**
+ * struct vidtv_channel - A 'channel' abstraction
+ *
+ * When vidtv boots, it will create some hardcoded channels.
+ * Their services will be concatenated to populate the SDT.
+ * Their programs will be concatenated to populate the PAT
+ * For each program in the PAT, a PMT section will be created
+ * The PMT section for a channel will be assigned its streams.
+ * Every stream will have its corresponding encoder polled to produce TS packets
+ * These packets may be interleaved by the mux and then delivered to the bridge
+ *
+ * @transport_stream_id: a number to identify the TS, chosen at will.
+ * @service: A _single_ service. Will be concatenated into the SDT.
+ * @program_num: The link between PAT, PMT and SDT.
+ * @program: A _single_ program with one or more streams associated with it.
+ * Will be concatenated into the PAT.
+ * @streams: A stream loop used to populate the PMT section for 'program'
+ * @encoders: A encoder loop. There must be one encoder for each stream.
+ * @next: Optionally chain this channel.
+ */
+struct vidtv_channel {
+ char *name;
+ u16 transport_stream_id;
+ struct vidtv_psi_table_sdt_service *service;
+ u16 program_num;
+ struct vidtv_psi_table_pat_program *program;
+ struct vidtv_psi_table_pmt_stream *streams;
+ struct vidtv_encoder *encoders;
+ struct vidtv_channel *next;
+};
+
+/**
+ * vidtv_channel_si_init - Init the PSI tables from the channels in the mux
+ * @m: The mux containing the channels.
+ */
+void vidtv_channel_si_init(struct vidtv_mux *m);
+void vidtv_channel_si_destroy(struct vidtv_mux *m);
+
+/**
+ * vidtv_channels_init - Init hardcoded, fake 'channels'.
+ * @m: The mux to store the channels into.
+ */
+void vidtv_channels_init(struct vidtv_mux *m);
+struct vidtv_channel
+*vidtv_channel_s302m_init(struct vidtv_channel *head, u16 transport_stream_id);
+void vidtv_channels_destroy(struct vidtv_mux *m);
+
+#endif //VIDTV_CHANNEL_H
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * The Virtual DVB test driver serves as a reference DVB driver and helps
+ * validate the existing APIs in the media subsystem. It can also aid
+ * developers working on userspace applications.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s, %d: " fmt, __func__, __LINE__
+
+#include <linux/printk.h>
+#include <linux/ratelimit.h>
+#include <linux/string.h>
+#include <linux/types.h>
+
+#include "vidtv_common.h"
+
+/**
+ * vidtv_memcpy() - wrapper routine to be used by MPEG-TS
+ * generator, in order to avoid going past the
+ * output buffer.
+ * @to: Starting element to where a MPEG-TS packet will
+ * be copied.
+ * @to_offset: Starting position of the @to buffer to be filled.
+ * @to_size: Size of the @to buffer.
+ * @from: Starting element of the buffer to be copied.
+ * @len: Number of elements to be copy from @from buffer
+ * into @to+ @to_offset buffer.
+ *
+ * Note:
+ * Real digital TV demod drivers should not have memcpy
+ * wrappers. We use it here because emulating MPEG-TS
+ * generation at kernelspace requires some extra care.
+ *
+ * Return:
+ * Returns the number of bytes written
+ */
+u32 vidtv_memcpy(void *to,
+ size_t to_offset,
+ size_t to_size,
+ const void *from,
+ size_t len)
+{
+ if (unlikely(to_offset + len > to_size)) {
+ pr_err_ratelimited("overflow detected, skipping. Try increasing the buffer size. Needed %zu, had %zu\n",
+ to_offset + len,
+ to_size);
+ return 0;
+ }
+
+ memcpy(to + to_offset, from, len);
+ return len;
+}
+
+/**
+ * vidtv_memset() - wrapper routine to be used by MPEG-TS
+ * generator, in order to avoid going past the
+ * output buffer.
+ * @to: Starting element to set
+ * @to_offset: Starting position of the @to buffer to be filled.
+ * @to_size: Size of the @to buffer.
+ * @c: The value to set the memory to.
+ * @len: Number of elements to be copy from @from buffer
+ * into @to+ @to_offset buffer.
+ *
+ * Note:
+ * Real digital TV demod drivers should not have memset
+ * wrappers. We use it here because emulating MPEG-TS
+ * generation at kernelspace requires some extra care.
+ *
+ * Return:
+ * Returns the number of bytes written
+ */
+u32 vidtv_memset(void *to,
+ size_t to_offset,
+ size_t to_size,
+ const int c,
+ size_t len)
+{
+ if (unlikely(to_offset + len > to_size)) {
+ pr_err_ratelimited("overflow detected, skipping. Try increasing the buffer size. Needed %zu, had %zu\n",
+ to_offset + len,
+ to_size);
+ return 0;
+ }
+
+ memset(to + to_offset, c, len);
+ return len;
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * The Virtual DVB test driver serves as a reference DVB driver and helps
+ * validate the existing APIs in the media subsystem. It can also aid
+ * developers working on userspace applications.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#ifndef VIDTV_COMMON_H
+#define VIDTV_COMMON_H
+
+#include <linux/types.h>
+
+#define CLOCK_UNIT_90KHZ 90000
+#define CLOCK_UNIT_27MHZ 27000000
+#define VIDTV_SLEEP_USECS 10000
+#define VIDTV_MAX_SLEEP_USECS (2 * VIDTV_SLEEP_USECS)
+#define VIDTV_DEFAULT_TS_ID 0x744
+
+u32 vidtv_memcpy(void *to,
+ size_t to_offset,
+ size_t to_size,
+ const void *from,
+ size_t len);
+
+u32 vidtv_memset(void *to,
+ size_t to_offset,
+ size_t to_size,
+ int c,
+ size_t len);
+
+#endif // VIDTV_COMMON_H
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * The Virtual DVB test driver serves as a reference DVB driver and helps
+ * validate the existing APIs in the media subsystem. It can also aid
+ * developers working on userspace applications.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ * Based on the example driver written by Emard <emard@softhome.net>
+ */
+
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/printk.h>
+#include <linux/random.h>
+#include <linux/ratelimit.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/workqueue.h>
+#include <media/dvb_frontend.h>
+
+#include "vidtv_demod.h"
+
+#define POLL_THRD_TIME 2000 /* ms */
+
+static const struct vidtv_demod_cnr_to_qual_s vidtv_demod_c_cnr_2_qual[] = {
+ /* from libdvbv5 source code, in milli db */
+ { QAM_256, FEC_NONE, 34000, 38000},
+ { QAM_64, FEC_NONE, 30000, 34000},
+};
+
+static const struct vidtv_demod_cnr_to_qual_s vidtv_demod_s_cnr_2_qual[] = {
+ /* from libdvbv5 source code, in milli db */
+ { QPSK, FEC_1_2, 7000, 10000},
+ { QPSK, FEC_2_3, 9000, 12000},
+ { QPSK, FEC_3_4, 10000, 13000},
+ { QPSK, FEC_5_6, 11000, 14000},
+ { QPSK, FEC_7_8, 12000, 15000},
+};
+
+static const struct vidtv_demod_cnr_to_qual_s vidtv_demod_s2_cnr_2_qual[] = {
+ /* from libdvbv5 source code, in milli db */
+ { QPSK, FEC_1_2, 9000, 12000},
+ { QPSK, FEC_2_3, 11000, 14000},
+ { QPSK, FEC_3_4, 12000, 15000},
+ { QPSK, FEC_5_6, 12000, 15000},
+ { QPSK, FEC_8_9, 13000, 16000},
+ { QPSK, FEC_9_10, 13500, 16500},
+ { PSK_8, FEC_2_3, 14500, 17500},
+ { PSK_8, FEC_3_4, 16000, 19000},
+ { PSK_8, FEC_5_6, 17500, 20500},
+ { PSK_8, FEC_8_9, 19000, 22000},
+};
+
+static const struct vidtv_demod_cnr_to_qual_s vidtv_demod_t_cnr_2_qual[] = {
+ /* from libdvbv5 source code, in milli db*/
+ { QPSK, FEC_1_2, 4100, 5900},
+ { QPSK, FEC_2_3, 6100, 9600},
+ { QPSK, FEC_3_4, 7200, 12400},
+ { QPSK, FEC_5_6, 8500, 15600},
+ { QPSK, FEC_7_8, 9200, 17500},
+ { QAM_16, FEC_1_2, 9800, 11800},
+ { QAM_16, FEC_2_3, 12100, 15300},
+ { QAM_16, FEC_3_4, 13400, 18100},
+ { QAM_16, FEC_5_6, 14800, 21300},
+ { QAM_16, FEC_7_8, 15700, 23600},
+ { QAM_64, FEC_1_2, 14000, 16000},
+ { QAM_64, FEC_2_3, 19900, 25400},
+ { QAM_64, FEC_3_4, 24900, 27900},
+ { QAM_64, FEC_5_6, 21300, 23300},
+ { QAM_64, FEC_7_8, 22000, 24000},
+};
+
+static const struct vidtv_demod_cnr_to_qual_s *vidtv_match_cnr_s(struct dvb_frontend *fe)
+{
+ const struct vidtv_demod_cnr_to_qual_s *cnr2qual = NULL;
+ struct device *dev = fe->dvb->device;
+ struct dtv_frontend_properties *c;
+ u32 array_size = 0;
+ u32 i;
+
+ c = &fe->dtv_property_cache;
+
+ switch (c->delivery_system) {
+ case SYS_DVBT:
+ case SYS_DVBT2:
+ cnr2qual = vidtv_demod_t_cnr_2_qual;
+ array_size = ARRAY_SIZE(vidtv_demod_t_cnr_2_qual);
+ break;
+
+ case SYS_DVBS:
+ cnr2qual = vidtv_demod_s_cnr_2_qual;
+ array_size = ARRAY_SIZE(vidtv_demod_s_cnr_2_qual);
+ break;
+
+ case SYS_DVBS2:
+ cnr2qual = vidtv_demod_s2_cnr_2_qual;
+ array_size = ARRAY_SIZE(vidtv_demod_s2_cnr_2_qual);
+ break;
+
+ case SYS_DVBC_ANNEX_A:
+ cnr2qual = vidtv_demod_c_cnr_2_qual;
+ array_size = ARRAY_SIZE(vidtv_demod_c_cnr_2_qual);
+ break;
+
+ default:
+ dev_warn_ratelimited(dev,
+ "%s: unsupported delivery system: %u\n",
+ __func__,
+ c->delivery_system);
+ break;
+ }
+
+ for (i = 0; i < array_size; i++)
+ if (cnr2qual[i].modulation == c->modulation &&
+ cnr2qual[i].fec == c->fec_inner)
+ return &cnr2qual[i];
+
+ return NULL; /* not found */
+}
+
+static void vidtv_clean_stats(struct dvb_frontend *fe)
+{
+ struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+
+ /* Fill the length of each status counter */
+
+ /* Signal is always available */
+ c->strength.len = 1;
+ c->strength.stat[0].scale = FE_SCALE_DECIBEL;
+ c->strength.stat[0].svalue = 0;
+
+ /* Usually available only after Viterbi lock */
+ c->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+ c->cnr.stat[0].svalue = 0;
+ c->cnr.len = 1;
+
+ /* Those depends on full lock */
+ c->pre_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+ c->pre_bit_error.stat[0].uvalue = 0;
+ c->pre_bit_error.len = 1;
+ c->pre_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+ c->pre_bit_count.stat[0].uvalue = 0;
+ c->pre_bit_count.len = 1;
+ c->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+ c->post_bit_error.stat[0].uvalue = 0;
+ c->post_bit_error.len = 1;
+ c->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+ c->post_bit_count.stat[0].uvalue = 0;
+ c->post_bit_count.len = 1;
+ c->block_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+ c->block_error.stat[0].uvalue = 0;
+ c->block_error.len = 1;
+ c->block_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+ c->block_count.stat[0].uvalue = 0;
+ c->block_count.len = 1;
+}
+
+static void vidtv_demod_update_stats(struct dvb_frontend *fe)
+{
+ struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+ struct vidtv_demod_state *state = fe->demodulator_priv;
+ u32 scale;
+
+ if (state->status & FE_HAS_LOCK) {
+ scale = FE_SCALE_COUNTER;
+ c->cnr.stat[0].scale = FE_SCALE_DECIBEL;
+ } else {
+ scale = FE_SCALE_NOT_AVAILABLE;
+ c->cnr.stat[0].scale = scale;
+ }
+
+ c->pre_bit_error.stat[0].scale = scale;
+ c->pre_bit_count.stat[0].scale = scale;
+ c->post_bit_error.stat[0].scale = scale;
+ c->post_bit_count.stat[0].scale = scale;
+ c->block_error.stat[0].scale = scale;
+ c->block_count.stat[0].scale = scale;
+
+ /*
+ * Add a 0.5% of randomness at the signal strength and CNR,
+ * and make them different, as we want to have something closer
+ * to a real case scenario.
+ *
+ * Also, usually, signal strength is a negative number in dBm.
+ */
+ c->strength.stat[0].svalue = state->tuner_cnr;
+ c->strength.stat[0].svalue -= prandom_u32_max(state->tuner_cnr / 50);
+ c->strength.stat[0].svalue -= 68000; /* Adjust to a better range */
+
+ c->cnr.stat[0].svalue = state->tuner_cnr;
+ c->cnr.stat[0].svalue -= prandom_u32_max(state->tuner_cnr / 50);
+
+}
+
+static int vidtv_demod_read_status(struct dvb_frontend *fe,
+ enum fe_status *status)
+{
+ struct vidtv_demod_state *state = fe->demodulator_priv;
+ const struct vidtv_demod_cnr_to_qual_s *cnr2qual = NULL;
+ struct vidtv_demod_config *config = &state->config;
+ u16 snr = 0;
+
+ /* Simulate random lost of signal due to a bad-tuned channel */
+ cnr2qual = vidtv_match_cnr_s(&state->frontend);
+
+ if (cnr2qual && state->tuner_cnr < cnr2qual->cnr_good &&
+ state->frontend.ops.tuner_ops.get_rf_strength) {
+ state->frontend.ops.tuner_ops.get_rf_strength(&state->frontend,
+ &snr);
+
+ if (snr < cnr2qual->cnr_ok) {
+ /* eventually lose the TS lock */
+ if (prandom_u32_max(100) < config->drop_tslock_prob_on_low_snr)
+ state->status = 0;
+ } else {
+ /* recover if the signal improves */
+ if (prandom_u32_max(100) <
+ config->recover_tslock_prob_on_good_snr)
+ state->status = FE_HAS_SIGNAL |
+ FE_HAS_CARRIER |
+ FE_HAS_VITERBI |
+ FE_HAS_SYNC |
+ FE_HAS_LOCK;
+ }
+ }
+
+ vidtv_demod_update_stats(&state->frontend);
+
+ *status = state->status;
+
+ return 0;
+}
+
+static int vidtv_demod_read_signal_strength(struct dvb_frontend *fe,
+ u16 *strength)
+{
+ struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+
+ *strength = c->strength.stat[0].uvalue;
+
+ return 0;
+}
+
+/*
+ * NOTE:
+ * This is implemented here just to be used as an example for real
+ * demod drivers.
+ *
+ * Should only be implemented if it actually reads something from the hardware.
+ * Also, it should check for the locks, in order to avoid report wrong data
+ * to userspace.
+ */
+static int vidtv_demod_get_frontend(struct dvb_frontend *fe,
+ struct dtv_frontend_properties *p)
+{
+ return 0;
+}
+
+static int vidtv_demod_set_frontend(struct dvb_frontend *fe)
+{
+ struct vidtv_demod_state *state = fe->demodulator_priv;
+ u32 tuner_status = 0;
+ int ret;
+
+ if (!fe->ops.tuner_ops.set_params)
+ return 0;
+
+ fe->ops.tuner_ops.set_params(fe);
+
+ /* store the CNR returned by the tuner */
+ ret = fe->ops.tuner_ops.get_rf_strength(fe, &state->tuner_cnr);
+ if (ret < 0)
+ return ret;
+
+ fe->ops.tuner_ops.get_status(fe, &tuner_status);
+ state->status = (state->tuner_cnr > 0) ? FE_HAS_SIGNAL |
+ FE_HAS_CARRIER |
+ FE_HAS_VITERBI |
+ FE_HAS_SYNC |
+ FE_HAS_LOCK :
+ 0;
+
+ vidtv_demod_update_stats(fe);
+
+ if (fe->ops.i2c_gate_ctrl)
+ fe->ops.i2c_gate_ctrl(fe, 0);
+
+ return 0;
+}
+
+/*
+ * NOTE:
+ * This is implemented here just to be used as an example for real
+ * demod drivers.
+ *
+ * Should only be implemented if the demod has support for DVB-S or DVB-S2
+ */
+static int vidtv_demod_set_tone(struct dvb_frontend *fe,
+ enum fe_sec_tone_mode tone)
+{
+ return 0;
+}
+
+/*
+ * NOTE:
+ * This is implemented here just to be used as an example for real
+ * demod drivers.
+ *
+ * Should only be implemented if the demod has support for DVB-S or DVB-S2
+ */
+static int vidtv_demod_set_voltage(struct dvb_frontend *fe,
+ enum fe_sec_voltage voltage)
+{
+ return 0;
+}
+
+/*
+ * NOTE:
+ * This is implemented here just to be used as an example for real
+ * demod drivers.
+ *
+ * Should only be implemented if the demod has support for DVB-S or DVB-S2
+ */
+static int vidtv_send_diseqc_msg(struct dvb_frontend *fe,
+ struct dvb_diseqc_master_cmd *cmd)
+{
+ return 0;
+}
+
+/*
+ * NOTE:
+ * This is implemented here just to be used as an example for real
+ * demod drivers.
+ *
+ * Should only be implemented if the demod has support for DVB-S or DVB-S2
+ */
+static int vidtv_diseqc_send_burst(struct dvb_frontend *fe,
+ enum fe_sec_mini_cmd burst)
+{
+ return 0;
+}
+
+static void vidtv_demod_release(struct dvb_frontend *fe)
+{
+ struct vidtv_demod_state *state = fe->demodulator_priv;
+
+ kfree(state);
+}
+
+static const struct dvb_frontend_ops vidtv_demod_ops = {
+ .delsys = {
+ SYS_DVBT,
+ SYS_DVBT2,
+ SYS_DVBC_ANNEX_A,
+ SYS_DVBS,
+ SYS_DVBS2,
+ },
+
+ .info = {
+ .name = "Dummy demod for DVB-T/T2/C/S/S2",
+ .frequency_min_hz = 51 * MHz,
+ .frequency_max_hz = 2150 * MHz,
+ .frequency_stepsize_hz = 62500,
+ .frequency_tolerance_hz = 29500 * kHz,
+ .symbol_rate_min = 1000000,
+ .symbol_rate_max = 45000000,
+
+ .caps = FE_CAN_FEC_1_2 |
+ FE_CAN_FEC_2_3 |
+ FE_CAN_FEC_3_4 |
+ FE_CAN_FEC_4_5 |
+ FE_CAN_FEC_5_6 |
+ FE_CAN_FEC_6_7 |
+ FE_CAN_FEC_7_8 |
+ FE_CAN_FEC_8_9 |
+ FE_CAN_QAM_16 |
+ FE_CAN_QAM_64 |
+ FE_CAN_QAM_32 |
+ FE_CAN_QAM_128 |
+ FE_CAN_QAM_256 |
+ FE_CAN_QAM_AUTO |
+ FE_CAN_QPSK |
+ FE_CAN_FEC_AUTO |
+ FE_CAN_INVERSION_AUTO |
+ FE_CAN_TRANSMISSION_MODE_AUTO |
+ FE_CAN_GUARD_INTERVAL_AUTO |
+ FE_CAN_HIERARCHY_AUTO,
+ },
+
+ .release = vidtv_demod_release,
+
+ .set_frontend = vidtv_demod_set_frontend,
+ .get_frontend = vidtv_demod_get_frontend,
+
+ .read_status = vidtv_demod_read_status,
+ .read_signal_strength = vidtv_demod_read_signal_strength,
+
+ /* For DVB-S/S2 */
+ .set_voltage = vidtv_demod_set_voltage,
+ .set_tone = vidtv_demod_set_tone,
+ .diseqc_send_master_cmd = vidtv_send_diseqc_msg,
+ .diseqc_send_burst = vidtv_diseqc_send_burst,
+
+};
+
+static const struct i2c_device_id vidtv_demod_i2c_id_table[] = {
+ {"dvb_vidtv_demod", 0},
+ {}
+};
+MODULE_DEVICE_TABLE(i2c, vidtv_demod_i2c_id_table);
+
+static int vidtv_demod_i2c_probe(struct i2c_client *client,
+ const struct i2c_device_id *id)
+{
+ struct vidtv_tuner_config *config = client->dev.platform_data;
+ struct vidtv_demod_state *state;
+
+ /* allocate memory for the internal state */
+ state = kzalloc(sizeof(*state), GFP_KERNEL);
+ if (!state)
+ return -ENOMEM;
+
+ /* create dvb_frontend */
+ memcpy(&state->frontend.ops,
+ &vidtv_demod_ops,
+ sizeof(struct dvb_frontend_ops));
+
+ memcpy(&state->config, config, sizeof(state->config));
+
+ state->frontend.demodulator_priv = state;
+ i2c_set_clientdata(client, state);
+
+ vidtv_clean_stats(&state->frontend);
+
+ return 0;
+}
+
+static int vidtv_demod_i2c_remove(struct i2c_client *client)
+{
+ struct vidtv_demod_state *state = i2c_get_clientdata(client);
+
+ kfree(state);
+
+ return 0;
+}
+
+static struct i2c_driver vidtv_demod_i2c_driver = {
+ .driver = {
+ .name = "dvb_vidtv_demod",
+ .suppress_bind_attrs = true,
+ },
+ .probe = vidtv_demod_i2c_probe,
+ .remove = vidtv_demod_i2c_remove,
+ .id_table = vidtv_demod_i2c_id_table,
+};
+
+module_i2c_driver(vidtv_demod_i2c_driver);
+
+MODULE_DESCRIPTION("Virtual DVB Demodulator Driver");
+MODULE_AUTHOR("Daniel W. S. Almeida");
+MODULE_LICENSE("GPL");
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * The Virtual DTV test driver serves as a reference DVB driver and helps
+ * validate the existing APIs in the media subsystem. It can also aid
+ * developers working on userspace applications.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ * Based on the example driver written by Emard <emard@softhome.net>
+ */
+
+#ifndef VIDTV_DEMOD_H
+#define VIDTV_DEMOD_H
+
+#include <linux/dvb/frontend.h>
+#include <media/dvb_frontend.h>
+
+/**
+ * struct vidtv_demod_cnr_to_qual_s - Map CNR values to a given combination of
+ * modulation and fec_inner
+ * @modulation: see enum fe_modulation
+ * @fec: see enum fe_fec_rate
+ *
+ * This struct matches values for 'good' and 'ok' CNRs given the combination
+ * of modulation and fec_inner in use. We might simulate some noise if the
+ * signal quality is not too good.
+ *
+ * The values were taken from libdvbv5.
+ */
+struct vidtv_demod_cnr_to_qual_s {
+ u32 modulation;
+ u32 fec;
+ u32 cnr_ok;
+ u32 cnr_good;
+};
+
+/**
+ * struct vidtv_demod_config - Configuration used to init the demod
+ * @drop_tslock_prob_on_low_snr: probability of losing the lock due to low snr
+ * @recover_tslock_prob_on_good_snr: probability of recovering when the signal
+ * improves
+ *
+ * The configuration used to init the demodulator module, usually filled
+ * by a bridge driver. For vidtv, this is filled by vidtv_bridge before the
+ * demodulator module is probed.
+ */
+struct vidtv_demod_config {
+ u8 drop_tslock_prob_on_low_snr;
+ u8 recover_tslock_prob_on_good_snr;
+};
+
+/**
+ * struct vidtv_demod_state - The demodulator state
+ * @frontend: The frontend structure allocated by the demod.
+ * @config: The config used to init the demod.
+ * @poll_snr: The task responsible for periodically checking the simulated
+ * signal quality, eventually dropping or reacquiring the TS lock.
+ * @status: the demod status.
+ * @cold_start: Whether the demod has not been init yet.
+ * @poll_snr_thread_running: Whether the task responsible for periodically
+ * checking the simulated signal quality is running.
+ * @poll_snr_thread_restart: Whether we should restart the poll_snr task.
+ */
+struct vidtv_demod_state {
+ struct dvb_frontend frontend;
+ struct vidtv_demod_config config;
+ enum fe_status status;
+ u16 tuner_cnr;
+};
+#endif // VIDTV_DEMOD_H
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Vidtv serves as a reference DVB driver and helps validate the existing APIs
+ * in the media subsystem. It can also aid developers working on userspace
+ * applications.
+ *
+ * This file contains a generic encoder type that can provide data for a stream
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#ifndef VIDTV_ENCODER_H
+#define VIDTV_ENCODER_H
+
+#include <linux/types.h>
+
+enum vidtv_encoder_id {
+ /* add IDs here when implementing new encoders */
+ S302M,
+};
+
+struct vidtv_access_unit {
+ u32 num_samples;
+ u64 pts;
+ u64 dts;
+ u32 nbytes;
+ u32 offset;
+ struct vidtv_access_unit *next;
+};
+
+/* Some musical notes, used by a tone generator */
+enum musical_notes {
+ NOTE_SILENT = 0,
+
+ NOTE_C_2 = 65,
+ NOTE_CS_2 = 69,
+ NOTE_D_2 = 73,
+ NOTE_DS_2 = 78,
+ NOTE_E_2 = 82,
+ NOTE_F_2 = 87,
+ NOTE_FS_2 = 93,
+ NOTE_G_2 = 98,
+ NOTE_GS_2 = 104,
+ NOTE_A_2 = 110,
+ NOTE_AS_2 = 117,
+ NOTE_B_2 = 123,
+ NOTE_C_3 = 131,
+ NOTE_CS_3 = 139,
+ NOTE_D_3 = 147,
+ NOTE_DS_3 = 156,
+ NOTE_E_3 = 165,
+ NOTE_F_3 = 175,
+ NOTE_FS_3 = 185,
+ NOTE_G_3 = 196,
+ NOTE_GS_3 = 208,
+ NOTE_A_3 = 220,
+ NOTE_AS_3 = 233,
+ NOTE_B_3 = 247,
+ NOTE_C_4 = 262,
+ NOTE_CS_4 = 277,
+ NOTE_D_4 = 294,
+ NOTE_DS_4 = 311,
+ NOTE_E_4 = 330,
+ NOTE_F_4 = 349,
+ NOTE_FS_4 = 370,
+ NOTE_G_4 = 392,
+ NOTE_GS_4 = 415,
+ NOTE_A_4 = 440,
+ NOTE_AS_4 = 466,
+ NOTE_B_4 = 494,
+ NOTE_C_5 = 523,
+ NOTE_CS_5 = 554,
+ NOTE_D_5 = 587,
+ NOTE_DS_5 = 622,
+ NOTE_E_5 = 659,
+ NOTE_F_5 = 698,
+ NOTE_FS_5 = 740,
+ NOTE_G_5 = 784,
+ NOTE_GS_5 = 831,
+ NOTE_A_5 = 880,
+ NOTE_AS_5 = 932,
+ NOTE_B_5 = 988,
+ NOTE_C_6 = 1047,
+ NOTE_CS_6 = 1109,
+ NOTE_D_6 = 1175,
+ NOTE_DS_6 = 1245,
+ NOTE_E_6 = 1319,
+ NOTE_F_6 = 1397,
+ NOTE_FS_6 = 1480,
+ NOTE_G_6 = 1568,
+ NOTE_GS_6 = 1661,
+ NOTE_A_6 = 1760,
+ NOTE_AS_6 = 1865,
+ NOTE_B_6 = 1976,
+ NOTE_C_7 = 2093
+};
+
+/**
+ * struct vidtv_encoder - A generic encoder type.
+ * @id: So we can cast to a concrete implementation when needed.
+ * @name: Usually the same as the stream name.
+ * @encoder_buf: The encoder internal buffer for the access units.
+ * @encoder_buf_sz: The encoder buffer size, in bytes
+ * @encoder_buf_offset: Our byte position in the encoder buffer.
+ * @sample_count: How many samples we have encoded in total.
+ * @src_buf: The source of raw data to be encoded, encoder might set a
+ * default if null.
+ * @src_buf_offset: Our position in the source buffer.
+ * @is_video_encoder: Whether this a video encoder (as opposed to audio)
+ * @ctx: Encoder-specific state.
+ * @stream_id: Examples: Audio streams (0xc0-0xdf), Video streams
+ * (0xe0-0xef).
+ * @es_id: The TS PID to use for the elementary stream in this encoder.
+ * @encode: Prepare enough AUs for the given amount of time.
+ * @clear: Clear the encoder output.
+ * @sync: Attempt to synchronize with this encoder.
+ * @sampling_rate_hz: The sampling rate (or fps, if video) used.
+ * @last_sample_cb: Called when the encoder runs out of data.This is
+ * so the source can read data in a
+ * piecemeal fashion instead of having to
+ * provide it all at once.
+ * @destroy: Destroy this encoder, freeing allocated resources.
+ * @next: Next in the chain
+ */
+struct vidtv_encoder {
+ enum vidtv_encoder_id id;
+ char *name;
+
+ u8 *encoder_buf;
+ u32 encoder_buf_sz;
+ u32 encoder_buf_offset;
+
+ u64 sample_count;
+ int last_duration;
+ int note_offset;
+ enum musical_notes last_tone;
+
+ struct vidtv_access_unit *access_units;
+
+ void *src_buf;
+ u32 src_buf_sz;
+ u32 src_buf_offset;
+
+ bool is_video_encoder;
+ void *ctx;
+
+ __be16 stream_id;
+
+ __be16 es_pid;
+
+ void *(*encode)(struct vidtv_encoder *e);
+
+ u32 (*clear)(struct vidtv_encoder *e);
+
+ struct vidtv_encoder *sync;
+
+ u32 sampling_rate_hz;
+
+ void (*last_sample_cb)(u32 sample_no);
+
+ void (*destroy)(struct vidtv_encoder *e);
+
+ struct vidtv_encoder *next;
+};
+
+#endif /* VIDTV_ENCODER_H */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Vidtv serves as a reference DVB driver and helps validate the existing APIs
+ * in the media subsystem. It can also aid developers working on userspace
+ * applications.
+ *
+ * This file contains the multiplexer logic for TS packets from different
+ * elementary streams
+ *
+ * Loosely based on libavcodec/mpegtsenc.c
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/jiffies.h>
+#include <linux/kernel.h>
+#include <linux/dev_printk.h>
+#include <linux/ratelimit.h>
+#include <linux/delay.h>
+#include <linux/vmalloc.h>
+#include <linux/math64.h>
+
+#include "vidtv_mux.h"
+#include "vidtv_ts.h"
+#include "vidtv_pes.h"
+#include "vidtv_encoder.h"
+#include "vidtv_channel.h"
+#include "vidtv_common.h"
+#include "vidtv_psi.h"
+
+static struct vidtv_mux_pid_ctx
+*vidtv_mux_get_pid_ctx(struct vidtv_mux *m, u16 pid)
+{
+ struct vidtv_mux_pid_ctx *ctx;
+
+ hash_for_each_possible(m->pid_ctx, ctx, h, pid)
+ if (ctx->pid == pid)
+ return ctx;
+ return NULL;
+}
+
+static struct vidtv_mux_pid_ctx
+*vidtv_mux_create_pid_ctx_once(struct vidtv_mux *m, u16 pid)
+{
+ struct vidtv_mux_pid_ctx *ctx;
+
+ ctx = vidtv_mux_get_pid_ctx(m, pid);
+
+ if (ctx)
+ goto end;
+
+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+ ctx->pid = pid;
+ ctx->cc = 0;
+ hash_add(m->pid_ctx, &ctx->h, pid);
+
+end:
+ return ctx;
+}
+
+static void vidtv_mux_pid_ctx_init(struct vidtv_mux *m)
+{
+ struct vidtv_psi_table_pat_program *p = m->si.pat->program;
+ u16 pid;
+
+ hash_init(m->pid_ctx);
+ /* push the pcr pid ctx */
+ vidtv_mux_create_pid_ctx_once(m, m->pcr_pid);
+ /* push the null packet pid ctx */
+ vidtv_mux_create_pid_ctx_once(m, TS_NULL_PACKET_PID);
+ /* push the PAT pid ctx */
+ vidtv_mux_create_pid_ctx_once(m, VIDTV_PAT_PID);
+ /* push the SDT pid ctx */
+ vidtv_mux_create_pid_ctx_once(m, VIDTV_SDT_PID);
+
+ /* add a ctx for all PMT sections */
+ while (p) {
+ pid = vidtv_psi_get_pat_program_pid(p);
+ vidtv_mux_create_pid_ctx_once(m, pid);
+ p = p->next;
+ }
+}
+
+static void vidtv_mux_pid_ctx_destroy(struct vidtv_mux *m)
+{
+ int bkt;
+ struct vidtv_mux_pid_ctx *ctx;
+ struct hlist_node *tmp;
+
+ hash_for_each_safe(m->pid_ctx, bkt, tmp, ctx, h) {
+ hash_del(&ctx->h);
+ kfree(ctx);
+ }
+}
+
+static void vidtv_mux_update_clk(struct vidtv_mux *m)
+{
+ /* call this at every thread iteration */
+ u64 elapsed_time;
+
+ m->timing.past_jiffies = m->timing.current_jiffies;
+ m->timing.current_jiffies = get_jiffies_64();
+
+ elapsed_time = jiffies_to_usecs(m->timing.current_jiffies -
+ m->timing.past_jiffies);
+
+ /* update the 27Mhz clock proportionally to the elapsed time */
+ m->timing.clk += (CLOCK_UNIT_27MHZ / USEC_PER_SEC) * elapsed_time;
+}
+
+static u32 vidtv_mux_push_si(struct vidtv_mux *m)
+{
+ u32 initial_offset = m->mux_buf_offset;
+
+ struct vidtv_mux_pid_ctx *pat_ctx;
+ struct vidtv_mux_pid_ctx *pmt_ctx;
+ struct vidtv_mux_pid_ctx *sdt_ctx;
+
+ struct vidtv_psi_pat_write_args pat_args = {};
+ struct vidtv_psi_pmt_write_args pmt_args = {};
+ struct vidtv_psi_sdt_write_args sdt_args = {};
+
+ u32 nbytes; /* the number of bytes written by this function */
+ u16 pmt_pid;
+ u32 i;
+
+ pat_ctx = vidtv_mux_get_pid_ctx(m, VIDTV_PAT_PID);
+ sdt_ctx = vidtv_mux_get_pid_ctx(m, VIDTV_SDT_PID);
+
+ pat_args.buf = m->mux_buf;
+ pat_args.offset = m->mux_buf_offset;
+ pat_args.pat = m->si.pat;
+ pat_args.buf_sz = m->mux_buf_sz;
+ pat_args.continuity_counter = &pat_ctx->cc;
+
+ m->mux_buf_offset += vidtv_psi_pat_write_into(pat_args);
+
+ for (i = 0; i < m->si.pat->programs; ++i) {
+ pmt_pid = vidtv_psi_pmt_get_pid(m->si.pmt_secs[i],
+ m->si.pat);
+
+ if (pmt_pid > TS_LAST_VALID_PID) {
+ dev_warn_ratelimited(m->dev,
+ "PID: %d not found\n", pmt_pid);
+ continue;
+ }
+
+ pmt_ctx = vidtv_mux_get_pid_ctx(m, pmt_pid);
+
+ pmt_args.buf = m->mux_buf;
+ pmt_args.offset = m->mux_buf_offset;
+ pmt_args.pmt = m->si.pmt_secs[i];
+ pmt_args.pid = pmt_pid;
+ pmt_args.buf_sz = m->mux_buf_sz;
+ pmt_args.continuity_counter = &pmt_ctx->cc;
+ pmt_args.pcr_pid = m->pcr_pid;
+
+ /* write each section into buffer */
+ m->mux_buf_offset += vidtv_psi_pmt_write_into(pmt_args);
+ }
+
+ sdt_args.buf = m->mux_buf;
+ sdt_args.offset = m->mux_buf_offset;
+ sdt_args.sdt = m->si.sdt;
+ sdt_args.buf_sz = m->mux_buf_sz;
+ sdt_args.continuity_counter = &sdt_ctx->cc;
+
+ m->mux_buf_offset += vidtv_psi_sdt_write_into(sdt_args);
+
+ nbytes = m->mux_buf_offset - initial_offset;
+
+ m->num_streamed_si++;
+
+ return nbytes;
+}
+
+static u32 vidtv_mux_push_pcr(struct vidtv_mux *m)
+{
+ struct pcr_write_args args = {};
+ struct vidtv_mux_pid_ctx *ctx;
+ u32 nbytes = 0;
+
+ ctx = vidtv_mux_get_pid_ctx(m, m->pcr_pid);
+ args.dest_buf = m->mux_buf;
+ args.pid = m->pcr_pid;
+ args.buf_sz = m->mux_buf_sz;
+ args.continuity_counter = &ctx->cc;
+
+ /* the 27Mhz clock will feed both parts of the PCR bitfield */
+ args.pcr = m->timing.clk;
+
+ nbytes += vidtv_ts_pcr_write_into(args);
+ m->mux_buf_offset += nbytes;
+
+ m->num_streamed_pcr++;
+
+ return nbytes;
+}
+
+static bool vidtv_mux_should_push_pcr(struct vidtv_mux *m)
+{
+ u64 next_pcr_at;
+
+ if (m->num_streamed_pcr == 0)
+ return true;
+
+ next_pcr_at = m->timing.start_jiffies +
+ usecs_to_jiffies(m->num_streamed_pcr *
+ m->timing.pcr_period_usecs);
+
+ return time_after64(m->timing.current_jiffies, next_pcr_at);
+}
+
+static bool vidtv_mux_should_push_si(struct vidtv_mux *m)
+{
+ u64 next_si_at;
+
+ if (m->num_streamed_si == 0)
+ return true;
+
+ next_si_at = m->timing.start_jiffies +
+ usecs_to_jiffies(m->num_streamed_si *
+ m->timing.si_period_usecs);
+
+ return time_after64(m->timing.current_jiffies, next_si_at);
+}
+
+static u32 vidtv_mux_packetize_access_units(struct vidtv_mux *m,
+ struct vidtv_encoder *e)
+{
+ u32 nbytes = 0;
+
+ struct pes_write_args args = {};
+ u32 initial_offset = m->mux_buf_offset;
+ struct vidtv_access_unit *au = e->access_units;
+
+ u8 *buf = NULL;
+ struct vidtv_mux_pid_ctx *pid_ctx = vidtv_mux_create_pid_ctx_once(m,
+ be16_to_cpu(e->es_pid));
+
+ args.dest_buf = m->mux_buf;
+ args.dest_buf_sz = m->mux_buf_sz;
+ args.pid = be16_to_cpu(e->es_pid);
+ args.encoder_id = e->id;
+ args.continuity_counter = &pid_ctx->cc;
+ args.stream_id = be16_to_cpu(e->stream_id);
+ args.send_pts = true;
+
+ while (au) {
+ buf = e->encoder_buf + au->offset;
+ args.from = buf;
+ args.access_unit_len = au->nbytes;
+ args.dest_offset = m->mux_buf_offset;
+ args.pts = au->pts;
+ args.pcr = m->timing.clk;
+
+ m->mux_buf_offset += vidtv_pes_write_into(args);
+
+ au = au->next;
+ }
+
+ /*
+ * clear the encoder state once the ES data has been written to the mux
+ * buffer
+ */
+ e->clear(e);
+
+ nbytes = m->mux_buf_offset - initial_offset;
+ return nbytes;
+}
+
+static u32 vidtv_mux_poll_encoders(struct vidtv_mux *m)
+{
+ u32 nbytes = 0;
+ u32 au_nbytes;
+ struct vidtv_channel *cur_chnl = m->channels;
+ struct vidtv_encoder *e = NULL;
+
+ while (cur_chnl) {
+ e = cur_chnl->encoders;
+
+ while (e) {
+ e->encode(e);
+ /* get the TS packets into the mux buffer */
+ au_nbytes = vidtv_mux_packetize_access_units(m, e);
+ nbytes += au_nbytes;
+ m->mux_buf_offset += au_nbytes;
+ /* grab next encoder */
+ e = e->next;
+ }
+
+ /* grab the next channel */
+ cur_chnl = cur_chnl->next;
+ }
+
+ return nbytes;
+}
+
+static u32 vidtv_mux_pad_with_nulls(struct vidtv_mux *m, u32 npkts)
+{
+ struct null_packet_write_args args = {};
+ u32 initial_offset = m->mux_buf_offset;
+ u32 nbytes; /* the number of bytes written by this function */
+ u32 i;
+ struct vidtv_mux_pid_ctx *ctx;
+
+ ctx = vidtv_mux_get_pid_ctx(m, TS_NULL_PACKET_PID);
+
+ args.dest_buf = m->mux_buf;
+ args.buf_sz = m->mux_buf_sz;
+ args.continuity_counter = &ctx->cc;
+ args.dest_offset = m->mux_buf_offset;
+
+ for (i = 0; i < npkts; ++i) {
+ m->mux_buf_offset += vidtv_ts_null_write_into(args);
+ args.dest_offset = m->mux_buf_offset;
+ }
+
+ nbytes = m->mux_buf_offset - initial_offset;
+
+ /* sanity check */
+ if (nbytes != npkts * TS_PACKET_LEN)
+ dev_err_ratelimited(m->dev, "%d != %d\n",
+ nbytes, npkts * TS_PACKET_LEN);
+
+ return nbytes;
+}
+
+static void vidtv_mux_clear(struct vidtv_mux *m)
+{
+ /* clear the packets currently in the mux */
+ memset(m->mux_buf, 0, m->mux_buf_sz * sizeof(*m->mux_buf));
+ /* point to the beginning of the buffer again */
+ m->mux_buf_offset = 0;
+}
+
+#define ERR_RATE 10000000
+static void vidtv_mux_tick(struct work_struct *work)
+{
+ struct vidtv_mux *m = container_of(work,
+ struct vidtv_mux,
+ mpeg_thread);
+ struct dtv_frontend_properties *c = &m->fe->dtv_property_cache;
+ u32 nbytes;
+ u32 npkts;
+ u32 tot_bits = 0;
+
+ while (m->streaming) {
+ nbytes = 0;
+
+ vidtv_mux_update_clk(m);
+
+ if (vidtv_mux_should_push_pcr(m))
+ nbytes += vidtv_mux_push_pcr(m);
+
+ if (vidtv_mux_should_push_si(m))
+ nbytes += vidtv_mux_push_si(m);
+
+ nbytes += vidtv_mux_poll_encoders(m);
+ nbytes += vidtv_mux_pad_with_nulls(m, 256);
+
+ npkts = nbytes / TS_PACKET_LEN;
+
+ /* if the buffer is not aligned there is a bug somewhere */
+ if (nbytes % TS_PACKET_LEN)
+ dev_err_ratelimited(m->dev, "Misaligned buffer\n");
+
+ if (m->on_new_packets_available_cb)
+ m->on_new_packets_available_cb(m->priv,
+ m->mux_buf,
+ npkts);
+
+ vidtv_mux_clear(m);
+
+ /*
+ * Update bytes and packet counts at DVBv5 stats
+ *
+ * For now, both pre and post bit counts are identical,
+ * but post BER count can be lower than pre BER, if the error
+ * correction logic discards packages.
+ */
+ c->pre_bit_count.stat[0].uvalue = nbytes * 8;
+ c->post_bit_count.stat[0].uvalue = nbytes * 8;
+ c->block_count.stat[0].uvalue += npkts;
+
+ /*
+ * Even without any visible errors for the user, the pre-BER
+ * stats usually have an error range up to 1E-6. So,
+ * add some random error increment count to it.
+ *
+ * Please notice that this is a poor guy's implementation,
+ * as it will produce one corrected bit error every time
+ * ceil(total bytes / ERR_RATE) is incremented, without
+ * any sort of (pseudo-)randomness.
+ */
+ tot_bits += nbytes * 8;
+ if (tot_bits > ERR_RATE) {
+ c->pre_bit_error.stat[0].uvalue++;
+ tot_bits -= ERR_RATE;
+ }
+
+ usleep_range(VIDTV_SLEEP_USECS, VIDTV_MAX_SLEEP_USECS);
+ }
+}
+
+void vidtv_mux_start_thread(struct vidtv_mux *m)
+{
+ if (m->streaming) {
+ dev_warn_ratelimited(m->dev, "Already streaming. Skipping.\n");
+ return;
+ }
+
+ m->streaming = true;
+ m->timing.start_jiffies = get_jiffies_64();
+ schedule_work(&m->mpeg_thread);
+}
+
+void vidtv_mux_stop_thread(struct vidtv_mux *m)
+{
+ if (m->streaming) {
+ m->streaming = false; /* thread will quit */
+ cancel_work_sync(&m->mpeg_thread);
+ }
+}
+
+struct vidtv_mux *vidtv_mux_init(struct dvb_frontend *fe,
+ struct device *dev,
+ struct vidtv_mux_init_args args)
+{
+ struct vidtv_mux *m = kzalloc(sizeof(*m), GFP_KERNEL);
+
+ m->dev = dev;
+ m->fe = fe;
+ m->timing.pcr_period_usecs = args.pcr_period_usecs;
+ m->timing.si_period_usecs = args.si_period_usecs;
+
+ m->mux_rate_kbytes_sec = args.mux_rate_kbytes_sec;
+
+ m->on_new_packets_available_cb = args.on_new_packets_available_cb;
+
+ m->mux_buf = vzalloc(args.mux_buf_sz);
+ m->mux_buf_sz = args.mux_buf_sz;
+
+ m->pcr_pid = args.pcr_pid;
+ m->transport_stream_id = args.transport_stream_id;
+ m->priv = args.priv;
+ m->timing.current_jiffies = get_jiffies_64();
+
+ if (args.channels)
+ m->channels = args.channels;
+ else
+ vidtv_channels_init(m);
+
+ /* will alloc data for pmt_sections after initializing pat */
+ vidtv_channel_si_init(m);
+
+ INIT_WORK(&m->mpeg_thread, vidtv_mux_tick);
+
+ vidtv_mux_pid_ctx_init(m);
+
+ return m;
+}
+
+void vidtv_mux_destroy(struct vidtv_mux *m)
+{
+ vidtv_mux_stop_thread(m);
+ vidtv_mux_pid_ctx_destroy(m);
+ vidtv_channel_si_destroy(m);
+ vidtv_channels_destroy(m);
+ vfree(m->mux_buf);
+ kfree(m);
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Vidtv serves as a reference DVB driver and helps validate the existing APIs
+ * in the media subsystem. It can also aid developers working on userspace
+ * applications.
+ *
+ * This file contains the muxer logic for TS packets from different
+ * elementary streams.
+ *
+ * Loosely based on libavcodec/mpegtsenc.c
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#ifndef VIDTV_MUX_H
+#define VIDTV_MUX_H
+
+#include <linux/types.h>
+#include <linux/hashtable.h>
+#include <linux/workqueue.h>
+#include <media/dvb_frontend.h>
+
+#include "vidtv_psi.h"
+
+/**
+ * struct vidtv_mux_timing - Timing related information
+ *
+ * This is used to decide when PCR or PSI packets should be sent. This will also
+ * provide storage for the clock, which is used to compute the value for the PCR.
+ *
+ * @start_jiffies: The value of 'jiffies' when we started the mux thread.
+ * @current_jiffies: The value of 'jiffies' for the current iteration.
+ * @past_jiffies: The value of 'jiffies' for the past iteration.
+ * @clk: A 27Mhz clock from which we will drive the PCR. Updated proportionally
+ * on every iteration.
+ * @pcr_period_usecs: How often we should send PCR packets.
+ * @si_period_usecs: How often we should send PSI packets.
+ */
+struct vidtv_mux_timing {
+ u64 start_jiffies;
+ u64 current_jiffies;
+ u64 past_jiffies;
+
+ u64 clk;
+
+ u64 pcr_period_usecs;
+ u64 si_period_usecs;
+};
+
+/**
+ * struct vidtv_mux_si - Store the PSI context.
+ *
+ * This is used to store the PAT, PMT sections and SDT in use by the muxer.
+ *
+ * The muxer acquire these by looking into the hardcoded channels in
+ * vidtv_channel and then periodically sends the TS packets for them>
+ *
+ * @pat: The PAT in use by the muxer.
+ * @pmt_secs: The PMT sections in use by the muxer. One for each program in the PAT.
+ * @sdt: The SDT in use by the muxer.
+ */
+struct vidtv_mux_si {
+ /* the SI tables */
+ struct vidtv_psi_table_pat *pat;
+ struct vidtv_psi_table_pmt **pmt_secs; /* the PMT sections */
+ struct vidtv_psi_table_sdt *sdt;
+};
+
+/**
+ * struct vidtv_mux_pid_ctx - Store the context for a given TS PID.
+ * @pid: The TS PID.
+ * @cc: The continuity counter for this PID. It is incremented on every TS
+ * pack and it will wrap around at 0xf0. If the decoder notices a sudden jump in
+ * this counter this will trigger a discontinuity state.
+ * @h: This is embedded in a hash table, mapping pid -> vidtv_mux_pid_ctx
+ */
+struct vidtv_mux_pid_ctx {
+ u16 pid;
+ u8 cc; /* continuity counter */
+ struct hlist_node h;
+};
+
+/**
+ * struct vidtv_mux - A muxer abstraction loosely based in libavcodec/mpegtsenc.c
+ * @mux_rate_kbytes_sec: The bit rate for the TS, in kbytes.
+ * @timing: Keeps track of timing related information.
+ * @pid_ctx: A hash table to keep track of per-PID metadata.
+ * @on_new_packets_available_cb: A callback to inform of new TS packets ready.
+ * @mux_buf: A pointer to a buffer for this muxer. TS packets are stored there
+ * and then passed on to the bridge driver.
+ * @mux_buf_sz: The size for 'mux_buf'.
+ * @mux_buf_offset: The current offset into 'mux_buf'.
+ * @channels: The channels associated with this muxer.
+ * @si: Keeps track of the PSI context.
+ * @num_streamed_pcr: Number of PCR packets streamed.
+ * @num_streamed_si: The number of PSI packets streamed.
+ * @mpeg_thread: Thread responsible for the muxer loop.
+ * @streaming: whether 'mpeg_thread' is running.
+ * @pcr_pid: The TS PID used for the PSI packets. All channels will share the
+ * same PCR.
+ * @transport_stream_id: The transport stream ID
+ * @priv: Private data.
+ */
+struct vidtv_mux {
+ struct dvb_frontend *fe;
+ struct device *dev;
+
+ struct vidtv_mux_timing timing;
+
+ u32 mux_rate_kbytes_sec;
+
+ DECLARE_HASHTABLE(pid_ctx, 3);
+
+ void (*on_new_packets_available_cb)(void *priv, u8 *buf, u32 npackets);
+
+ u8 *mux_buf;
+ u32 mux_buf_sz;
+ u32 mux_buf_offset;
+
+ struct vidtv_channel *channels;
+
+ struct vidtv_mux_si si;
+ u64 num_streamed_pcr;
+ u64 num_streamed_si;
+
+ struct work_struct mpeg_thread;
+ bool streaming;
+
+ u16 pcr_pid;
+ u16 transport_stream_id;
+ void *priv;
+};
+
+/**
+ * struct vidtv_mux_init_args - Arguments used to inix the muxer.
+ * @mux_rate_kbytes_sec: The bit rate for the TS, in kbytes.
+ * @on_new_packets_available_cb: A callback to inform of new TS packets ready.
+ * @mux_buf_sz: The size for 'mux_buf'.
+ * @pcr_period_usecs: How often we should send PCR packets.
+ * @si_period_usecs: How often we should send PSI packets.
+ * @pcr_pid: The TS PID used for the PSI packets. All channels will share the
+ * same PCR.
+ * @transport_stream_id: The transport stream ID
+ * @channels: an optional list of channels to use
+ * @priv: Private data.
+ */
+struct vidtv_mux_init_args {
+ u32 mux_rate_kbytes_sec;
+ void (*on_new_packets_available_cb)(void *priv, u8 *buf, u32 npackets);
+ u32 mux_buf_sz;
+ u64 pcr_period_usecs;
+ u64 si_period_usecs;
+ u16 pcr_pid;
+ u16 transport_stream_id;
+ struct vidtv_channel *channels;
+ void *priv;
+};
+
+struct vidtv_mux *vidtv_mux_init(struct dvb_frontend *fe,
+ struct device *dev,
+ struct vidtv_mux_init_args args);
+void vidtv_mux_destroy(struct vidtv_mux *m);
+
+void vidtv_mux_start_thread(struct vidtv_mux *m);
+void vidtv_mux_stop_thread(struct vidtv_mux *m);
+
+#endif //VIDTV_MUX_H
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Vidtv serves as a reference DVB driver and helps validate the existing APIs
+ * in the media subsystem. It can also aid developers working on userspace
+ * applications.
+ *
+ * This file contains the logic to translate the ES data for one access unit
+ * from an encoder into MPEG TS packets. It does so by first encapsulating it
+ * with a PES header and then splitting it into TS packets.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s, %d: " fmt, __func__, __LINE__
+
+#include <linux/types.h>
+#include <linux/printk.h>
+#include <linux/ratelimit.h>
+#include <asm/byteorder.h>
+
+#include "vidtv_pes.h"
+#include "vidtv_common.h"
+#include "vidtv_encoder.h"
+#include "vidtv_ts.h"
+
+#define PRIVATE_STREAM_1_ID 0xbd /* private_stream_1. See SMPTE 302M-2007 p.6 */
+#define PES_HEADER_MAX_STUFFING_BYTES 32
+#define PES_TS_HEADER_MAX_STUFFING_BYTES 182
+
+static u32 vidtv_pes_op_get_len(bool send_pts, bool send_dts)
+{
+ u32 len = 0;
+
+ /* the flags must always be sent */
+ len += sizeof(struct vidtv_pes_optional);
+
+ /* From all optionals, we might send these for now */
+ if (send_pts && send_dts)
+ len += sizeof(struct vidtv_pes_optional_pts_dts);
+ else if (send_pts)
+ len += sizeof(struct vidtv_pes_optional_pts);
+
+ return len;
+}
+
+#define SIZE_PCR (6 + sizeof(struct vidtv_mpeg_ts_adaption))
+
+static u32 vidtv_pes_h_get_len(bool send_pts, bool send_dts)
+{
+ u32 len = 0;
+
+ /* PES header length notwithstanding stuffing bytes */
+
+ len += sizeof(struct vidtv_mpeg_pes);
+ len += vidtv_pes_op_get_len(send_pts, send_dts);
+
+ return len;
+}
+
+static u32 vidtv_pes_write_header_stuffing(struct pes_header_write_args args)
+{
+ /*
+ * This is a fixed 8-bit value equal to '0xFF' that can be inserted
+ * by the encoder, for example to meet the requirements of the channel.
+ * It is discarded by the decoder. No more than 32 stuffing bytes shall
+ * be present in one PES packet header.
+ */
+ if (args.n_pes_h_s_bytes > PES_HEADER_MAX_STUFFING_BYTES) {
+ pr_warn_ratelimited("More than %d stuffing bytes in PES packet header\n",
+ PES_HEADER_MAX_STUFFING_BYTES);
+ args.n_pes_h_s_bytes = PES_HEADER_MAX_STUFFING_BYTES;
+ }
+
+ return vidtv_memset(args.dest_buf,
+ args.dest_offset,
+ args.dest_buf_sz,
+ TS_FILL_BYTE,
+ args.n_pes_h_s_bytes);
+}
+
+static u32 vidtv_pes_write_pts_dts(struct pes_header_write_args args)
+{
+ u32 nbytes = 0; /* the number of bytes written by this function */
+
+ struct vidtv_pes_optional_pts pts = {};
+ struct vidtv_pes_optional_pts_dts pts_dts = {};
+ void *op = NULL;
+ size_t op_sz = 0;
+ u64 mask1;
+ u64 mask2;
+ u64 mask3;
+
+ if (!args.send_pts && args.send_dts)
+ return 0;
+
+ mask1 = GENMASK_ULL(32, 30);
+ mask2 = GENMASK_ULL(29, 15);
+ mask3 = GENMASK_ULL(14, 0);
+
+ /* see ISO/IEC 13818-1 : 2000 p. 32 */
+ if (args.send_pts && args.send_dts) {
+ pts_dts.pts1 = (0x3 << 4) | ((args.pts & mask1) >> 29) | 0x1;
+ pts_dts.pts2 = cpu_to_be16(((args.pts & mask2) >> 14) | 0x1);
+ pts_dts.pts3 = cpu_to_be16(((args.pts & mask3) << 1) | 0x1);
+
+ pts_dts.dts1 = (0x1 << 4) | ((args.dts & mask1) >> 29) | 0x1;
+ pts_dts.dts2 = cpu_to_be16(((args.dts & mask2) >> 14) | 0x1);
+ pts_dts.dts3 = cpu_to_be16(((args.dts & mask3) << 1) | 0x1);
+
+ op = &pts_dts;
+ op_sz = sizeof(pts_dts);
+
+ } else if (args.send_pts) {
+ pts.pts1 = (0x1 << 5) | ((args.pts & mask1) >> 29) | 0x1;
+ pts.pts2 = cpu_to_be16(((args.pts & mask2) >> 14) | 0x1);
+ pts.pts3 = cpu_to_be16(((args.pts & mask3) << 1) | 0x1);
+
+ op = &pts;
+ op_sz = sizeof(pts);
+ }
+
+ /* copy PTS/DTS optional */
+ nbytes += vidtv_memcpy(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.dest_buf_sz,
+ op,
+ op_sz);
+
+ return nbytes;
+}
+
+static u32 vidtv_pes_write_h(struct pes_header_write_args args)
+{
+ u32 nbytes = 0; /* the number of bytes written by this function */
+
+ struct vidtv_mpeg_pes pes_header = {};
+ struct vidtv_pes_optional pes_optional = {};
+ struct pes_header_write_args pts_dts_args = args;
+ u32 stream_id = (args.encoder_id == S302M) ? PRIVATE_STREAM_1_ID : args.stream_id;
+ u16 pes_opt_bitfield = 0x01 << 15;
+
+ pes_header.bitfield = cpu_to_be32((PES_START_CODE_PREFIX << 8) | stream_id);
+
+ pes_header.length = cpu_to_be16(vidtv_pes_op_get_len(args.send_pts,
+ args.send_dts) +
+ args.access_unit_len);
+
+ if (args.send_pts && args.send_dts)
+ pes_opt_bitfield |= (0x3 << 6);
+ else if (args.send_pts)
+ pes_opt_bitfield |= (0x1 << 7);
+
+ pes_optional.bitfield = cpu_to_be16(pes_opt_bitfield);
+ pes_optional.length = vidtv_pes_op_get_len(args.send_pts, args.send_dts) +
+ args.n_pes_h_s_bytes -
+ sizeof(struct vidtv_pes_optional);
+
+ /* copy header */
+ nbytes += vidtv_memcpy(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.dest_buf_sz,
+ &pes_header,
+ sizeof(pes_header));
+
+ /* copy optional header bits */
+ nbytes += vidtv_memcpy(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.dest_buf_sz,
+ &pes_optional,
+ sizeof(pes_optional));
+
+ /* copy the timing information */
+ pts_dts_args.dest_offset = args.dest_offset + nbytes;
+ nbytes += vidtv_pes_write_pts_dts(pts_dts_args);
+
+ /* write any PES header stuffing */
+ nbytes += vidtv_pes_write_header_stuffing(args);
+
+ return nbytes;
+}
+
+static u32 vidtv_pes_write_pcr_bits(u8 *to, u32 to_offset, u64 pcr)
+{
+ /* Exact same from ffmpeg. PCR is a counter driven by a 27Mhz clock */
+ u64 div;
+ u64 rem;
+ u8 *buf = to + to_offset;
+ u64 pcr_low;
+ u64 pcr_high;
+
+ div = div64_u64_rem(pcr, 300, &rem);
+
+ pcr_low = rem; /* pcr_low = pcr % 300 */
+ pcr_high = div; /* pcr_high = pcr / 300 */
+
+ *buf++ = pcr_high >> 25;
+ *buf++ = pcr_high >> 17;
+ *buf++ = pcr_high >> 9;
+ *buf++ = pcr_high >> 1;
+ *buf++ = pcr_high << 7 | pcr_low >> 8 | 0x7e;
+ *buf++ = pcr_low;
+
+ return 6;
+}
+
+static u32 vidtv_pes_write_stuffing(struct pes_ts_header_write_args *args,
+ u32 dest_offset, bool need_pcr,
+ u64 *last_pcr)
+{
+ struct vidtv_mpeg_ts_adaption ts_adap = {};
+ int stuff_nbytes;
+ u32 nbytes = 0;
+
+ if (!args->n_stuffing_bytes)
+ return 0;
+
+ ts_adap.random_access = 1;
+
+ /* length _immediately_ following 'adaptation_field_length' */
+ if (need_pcr) {
+ ts_adap.PCR = 1;
+ ts_adap.length = SIZE_PCR;
+ } else {
+ ts_adap.length = sizeof(ts_adap);
+ }
+ stuff_nbytes = args->n_stuffing_bytes - ts_adap.length;
+
+ ts_adap.length -= sizeof(ts_adap.length);
+
+ if (unlikely(stuff_nbytes < 0))
+ stuff_nbytes = 0;
+
+ ts_adap.length += stuff_nbytes;
+
+ /* write the adap after the TS header */
+ nbytes += vidtv_memcpy(args->dest_buf,
+ dest_offset + nbytes,
+ args->dest_buf_sz,
+ &ts_adap,
+ sizeof(ts_adap));
+
+ /* write the optional PCR */
+ if (need_pcr) {
+ nbytes += vidtv_pes_write_pcr_bits(args->dest_buf,
+ dest_offset + nbytes,
+ args->pcr);
+
+ *last_pcr = args->pcr;
+ }
+
+ /* write the stuffing bytes, if are there anything left */
+ if (stuff_nbytes)
+ nbytes += vidtv_memset(args->dest_buf,
+ dest_offset + nbytes,
+ args->dest_buf_sz,
+ TS_FILL_BYTE,
+ stuff_nbytes);
+
+ /*
+ * The n_stuffing_bytes contain a pre-calculated value of
+ * the amount of data that this function would read, made from
+ * vidtv_pes_h_get_len(). If something went wrong, print a warning
+ */
+ if (nbytes != args->n_stuffing_bytes)
+ pr_warn_ratelimited("write size was %d, expected %d\n",
+ nbytes, args->n_stuffing_bytes);
+
+ return nbytes;
+}
+
+static u32 vidtv_pes_write_ts_h(struct pes_ts_header_write_args args,
+ bool need_pcr, u64 *last_pcr)
+{
+ /* number of bytes written by this function */
+ u32 nbytes = 0;
+ struct vidtv_mpeg_ts ts_header = {};
+ u16 payload_start = !args.wrote_pes_header;
+
+ ts_header.sync_byte = TS_SYNC_BYTE;
+ ts_header.bitfield = cpu_to_be16((payload_start << 14) | args.pid);
+ ts_header.scrambling = 0;
+ ts_header.adaptation_field = (args.n_stuffing_bytes) > 0;
+ ts_header.payload = (args.n_stuffing_bytes) < PES_TS_HEADER_MAX_STUFFING_BYTES;
+
+ ts_header.continuity_counter = *args.continuity_counter;
+
+ vidtv_ts_inc_cc(args.continuity_counter);
+
+ /* write the TS header */
+ nbytes += vidtv_memcpy(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.dest_buf_sz,
+ &ts_header,
+ sizeof(ts_header));
+
+ /* write stuffing, if any */
+ nbytes += vidtv_pes_write_stuffing(&args, args.dest_offset + nbytes,
+ need_pcr, last_pcr);
+
+ return nbytes;
+}
+
+u32 vidtv_pes_write_into(struct pes_write_args args)
+{
+ u32 unaligned_bytes = (args.dest_offset % TS_PACKET_LEN);
+ struct pes_ts_header_write_args ts_header_args = {};
+ struct pes_header_write_args pes_header_args = {};
+ u32 remaining_len = args.access_unit_len;
+ bool wrote_pes_header = false;
+ u64 last_pcr = args.pcr;
+ bool need_pcr = true;
+ u32 available_space;
+ u32 payload_size;
+ u32 stuff_bytes;
+ u32 nbytes = 0;
+
+ if (unaligned_bytes) {
+ pr_warn_ratelimited("buffer is misaligned, while starting PES\n");
+
+ /* forcibly align and hope for the best */
+ nbytes += vidtv_memset(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.dest_buf_sz,
+ TS_FILL_BYTE,
+ TS_PACKET_LEN - unaligned_bytes);
+ }
+
+ if (args.send_dts && !args.send_pts) {
+ pr_warn_ratelimited("forbidden value '01' for PTS_DTS flags\n");
+ args.send_pts = true;
+ args.pts = args.dts;
+ }
+
+ /* see SMPTE 302M clause 6.4 */
+ if (args.encoder_id == S302M) {
+ args.send_dts = false;
+ args.send_pts = true;
+ }
+
+ while (remaining_len) {
+ available_space = TS_PAYLOAD_LEN;
+ /*
+ * The amount of space initially available in the TS packet.
+ * if this is the beginning of the PES packet, take into account
+ * the space needed for the TS header _and_ for the PES header
+ */
+ if (!wrote_pes_header)
+ available_space -= vidtv_pes_h_get_len(args.send_pts,
+ args.send_dts);
+
+ /*
+ * if the encoder has inserted stuffing bytes in the PES
+ * header, account for them.
+ */
+ available_space -= args.n_pes_h_s_bytes;
+
+ /* Take the extra adaptation into account if need to send PCR */
+ if (need_pcr) {
+ available_space -= SIZE_PCR;
+ stuff_bytes = SIZE_PCR;
+ } else {
+ stuff_bytes = 0;
+ }
+
+ /*
+ * how much of the _actual_ payload should be written in this
+ * packet.
+ */
+ if (remaining_len >= available_space) {
+ payload_size = available_space;
+ } else {
+ /* Last frame should ensure 188-bytes PS alignment */
+ payload_size = remaining_len;
+ stuff_bytes += available_space - payload_size;
+
+ /*
+ * Ensure that the stuff bytes will be within the
+ * allowed range, decrementing the number of payload
+ * bytes to write if needed.
+ */
+ if (stuff_bytes > PES_TS_HEADER_MAX_STUFFING_BYTES) {
+ u32 tmp = stuff_bytes - PES_TS_HEADER_MAX_STUFFING_BYTES;
+
+ stuff_bytes = PES_TS_HEADER_MAX_STUFFING_BYTES;
+ payload_size -= tmp;
+ }
+ }
+
+ /* write ts header */
+ ts_header_args.dest_buf = args.dest_buf;
+ ts_header_args.dest_offset = args.dest_offset + nbytes;
+ ts_header_args.dest_buf_sz = args.dest_buf_sz;
+ ts_header_args.pid = args.pid;
+ ts_header_args.pcr = args.pcr;
+ ts_header_args.continuity_counter = args.continuity_counter;
+ ts_header_args.wrote_pes_header = wrote_pes_header;
+ ts_header_args.n_stuffing_bytes = stuff_bytes;
+
+ nbytes += vidtv_pes_write_ts_h(ts_header_args, need_pcr,
+ &last_pcr);
+
+ need_pcr = false;
+
+ if (!wrote_pes_header) {
+ /* write the PES header only once */
+ pes_header_args.dest_buf = args.dest_buf;
+
+ pes_header_args.dest_offset = args.dest_offset +
+ nbytes;
+
+ pes_header_args.dest_buf_sz = args.dest_buf_sz;
+ pes_header_args.encoder_id = args.encoder_id;
+ pes_header_args.send_pts = args.send_pts;
+ pes_header_args.pts = args.pts;
+ pes_header_args.send_dts = args.send_dts;
+ pes_header_args.dts = args.dts;
+ pes_header_args.stream_id = args.stream_id;
+ pes_header_args.n_pes_h_s_bytes = args.n_pes_h_s_bytes;
+ pes_header_args.access_unit_len = args.access_unit_len;
+
+ nbytes += vidtv_pes_write_h(pes_header_args);
+ wrote_pes_header = true;
+ }
+
+ /* write as much of the payload as we possibly can */
+ nbytes += vidtv_memcpy(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.dest_buf_sz,
+ args.from,
+ payload_size);
+
+ args.from += payload_size;
+
+ remaining_len -= payload_size;
+ }
+
+ return nbytes;
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Vidtv serves as a reference DVB driver and helps validate the existing APIs
+ * in the media subsystem. It can also aid developers working on userspace
+ * applications.
+ *
+ * This file contains the logic to translate the ES data for one access unit
+ * from an encoder into MPEG TS packets. It does so by first encapsulating it
+ * with a PES header and then splitting it into TS packets.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#ifndef VIDTV_PES_H
+#define VIDTV_PES_H
+
+#include <asm/byteorder.h>
+#include <linux/types.h>
+
+#include "vidtv_common.h"
+
+#define PES_MAX_LEN 65536 /* Set 'length' to 0 if greater. Only possible for video. */
+#define PES_START_CODE_PREFIX 0x001 /* 00 00 01 */
+
+/* Used when sending PTS, but not DTS */
+struct vidtv_pes_optional_pts {
+ u8 pts1;
+ __be16 pts2;
+ __be16 pts3;
+} __packed;
+
+/* Used when sending both PTS and DTS */
+struct vidtv_pes_optional_pts_dts {
+ u8 pts1;
+ __be16 pts2;
+ __be16 pts3;
+
+ u8 dts1;
+ __be16 dts2;
+ __be16 dts3;
+} __packed;
+
+/* PES optional flags */
+struct vidtv_pes_optional {
+ /*
+ * These flags show which components are actually
+ * present in the "optional fields" in the optional PES
+ * header and which are not
+ *
+ * u16 two:2; //0x2
+ * u16 PES_scrambling_control:2;
+ * u16 PES_priority:1;
+ * u16 data_alignment_indicator:1; // unused
+ * u16 copyright:1;
+ * u16 original_or_copy:1;
+ * u16 PTS_DTS:2;
+ * u16 ESCR:1;
+ * u16 ES_rate:1;
+ * u16 DSM_trick_mode:1;
+ * u16 additional_copy_info:1;
+ * u16 PES_CRC:1;
+ * u16 PES_extension:1;
+ */
+ __be16 bitfield;
+ u8 length;
+} __packed;
+
+/* The PES header */
+struct vidtv_mpeg_pes {
+ __be32 bitfield; /* packet_start_code_prefix:24, stream_id: 8 */
+ /* after this field until the end of the PES data payload */
+ __be16 length;
+ struct vidtv_pes_optional optional[];
+} __packed;
+
+/**
+ * struct pes_header_write_args - Arguments to write a PES header.
+ * @dest_buf: The buffer to write into.
+ * @dest_offset: where to start writing in the dest_buffer.
+ * @dest_buf_sz: The size of the dest_buffer
+ * @encoder_id: Encoder id (see vidtv_encoder.h)
+ * @send_pts: Should we send PTS?
+ * @pts: PTS value to send.
+ * @send_dts: Should we send DTS?
+ * @dts: DTS value to send.
+ * @stream_id: The stream id to use. Ex: Audio streams (0xc0-0xdf), Video
+ * streams (0xe0-0xef).
+ * @n_pes_h_s_bytes: Padding bytes. Might be used by an encoder if needed, gets
+ * discarded by the decoder.
+ * @access_unit_len: The size of _one_ access unit (with any headers it might need)
+ */
+struct pes_header_write_args {
+ void *dest_buf;
+ u32 dest_offset;
+ u32 dest_buf_sz;
+ u32 encoder_id;
+
+ bool send_pts;
+ u64 pts;
+
+ bool send_dts;
+ u64 dts;
+
+ u16 stream_id;
+ /* might be used by an encoder if needed, gets discarded by decoder */
+ u32 n_pes_h_s_bytes;
+ u32 access_unit_len;
+};
+
+/**
+ * struct pes_ts_header_write_args - Arguments to write a TS header.
+ * @dest_buf: The buffer to write into.
+ * @dest_offset: where to start writing in the dest_buffer.
+ * @dest_buf_sz: The size of the dest_buffer
+ * @pid: The PID to use for the TS packets.
+ * @continuity_counter: Incremented on every new TS packet.
+ * @n_pes_h_s_bytes: Padding bytes. Might be used by an encoder if needed, gets
+ * discarded by the decoder.
+ */
+struct pes_ts_header_write_args {
+ void *dest_buf;
+ u32 dest_offset;
+ u32 dest_buf_sz;
+ u16 pid;
+ u8 *continuity_counter;
+ bool wrote_pes_header;
+ u32 n_stuffing_bytes;
+ u64 pcr;
+};
+
+/**
+ * struct pes_write_args - Arguments for the packetizer.
+ * @dest_buf: The buffer to write into.
+ * @from: A pointer to the encoder buffer containing one access unit.
+ * @access_unit_len: The size of _one_ access unit (with any headers it might need)
+ * @dest_offset: where to start writing in the dest_buffer.
+ * @dest_buf_sz: The size of the dest_buffer
+ * @pid: The PID to use for the TS packets.
+ * @encoder_id: Encoder id (see vidtv_encoder.h)
+ * @continuity_counter: Incremented on every new TS packet.
+ * @stream_id: The stream id to use. Ex: Audio streams (0xc0-0xdf), Video
+ * streams (0xe0-0xef).
+ * @send_pts: Should we send PTS?
+ * @pts: PTS value to send.
+ * @send_dts: Should we send DTS?
+ * @dts: DTS value to send.
+ * @n_pes_h_s_bytes: Padding bytes. Might be used by an encoder if needed, gets
+ * discarded by the decoder.
+ */
+struct pes_write_args {
+ void *dest_buf;
+ void *from;
+ u32 access_unit_len;
+
+ u32 dest_offset;
+ u32 dest_buf_sz;
+ u16 pid;
+
+ u32 encoder_id;
+
+ u8 *continuity_counter;
+
+ u16 stream_id;
+
+ bool send_pts;
+ u64 pts;
+
+ bool send_dts;
+ u64 dts;
+
+ u32 n_pes_h_s_bytes;
+ u64 pcr;
+};
+
+/**
+ * vidtv_pes_write_into - Write a PES packet as MPEG-TS packets into a buffer.
+ * @args: The args to use when writing
+ *
+ * This function translate the ES data for one access unit
+ * from an encoder into MPEG TS packets. It does so by first encapsulating it
+ * with a PES header and then splitting it into TS packets.
+ *
+ * The data is then written into the buffer pointed to by 'args.buf'
+ *
+ * Return: The number of bytes written into the buffer. This is usually NOT
+ * equal to the size of the access unit, since we need space for PES headers, TS headers
+ * and padding bytes, if any.
+ */
+u32 vidtv_pes_write_into(struct pes_write_args args);
+
+#endif // VIDTV_PES_H
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file contains the logic to work with MPEG Program-Specific Information.
+ * These are defined both in ISO/IEC 13818-1 (systems) and ETSI EN 300 468.
+ * PSI is carried in the form of table structures, and although each table might
+ * technically be broken into one or more sections, we do not do this here,
+ * hence 'table' and 'section' are interchangeable for vidtv.
+ *
+ * This code currently supports three tables: PAT, PMT and SDT. These are the
+ * bare minimum to get userspace to recognize our MPEG transport stream. It can
+ * be extended to support more PSI tables in the future.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s, %d: " fmt, __func__, __LINE__
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/crc32.h>
+#include <linux/string.h>
+#include <linux/printk.h>
+#include <linux/ratelimit.h>
+#include <linux/string.h>
+#include <asm/byteorder.h>
+
+#include "vidtv_psi.h"
+#include "vidtv_common.h"
+#include "vidtv_ts.h"
+
+#define CRC_SIZE_IN_BYTES 4
+#define MAX_VERSION_NUM 32
+
+static const u32 CRC_LUT[256] = {
+ /* from libdvbv5 */
+ 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc, 0x17c56b6b,
+ 0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61,
+ 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, 0x4c11db70, 0x48d0c6c7,
+ 0x4593e01e, 0x4152fda9, 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75,
+ 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3,
+ 0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039,
+ 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, 0xbe2b5b58, 0xbaea46ef,
+ 0xb7a96036, 0xb3687d81, 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d,
+ 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb,
+ 0xceb42022, 0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1,
+ 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077, 0x30476dc0,
+ 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072,
+ 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, 0x018aeb13, 0x054bf6a4,
+ 0x0808d07d, 0x0cc9cdca, 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde,
+ 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08,
+ 0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba,
+ 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc,
+ 0xb6238b25, 0xb2e29692, 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6,
+ 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050,
+ 0xe9362689, 0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2,
+ 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683, 0xd1799b34,
+ 0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637,
+ 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, 0x4f040d56, 0x4bc510e1,
+ 0x46863638, 0x42472b8f, 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53,
+ 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5,
+ 0x3f9b762c, 0x3b5a6b9b, 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff,
+ 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, 0xf12f560e, 0xf5ee4bb9,
+ 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b,
+ 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd,
+ 0xcda1f604, 0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7,
+ 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6, 0x9ff77d71,
+ 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3,
+ 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2,
+ 0x470cdd2b, 0x43cdc09c, 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8,
+ 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e,
+ 0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec,
+ 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, 0x2497d08d, 0x2056cd3a,
+ 0x2d15ebe3, 0x29d4f654, 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0,
+ 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676,
+ 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4,
+ 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662,
+ 0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668,
+ 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4
+};
+
+static inline u32 dvb_crc32(u32 crc, u8 *data, u32 len)
+{
+ /* from libdvbv5 */
+ while (len--)
+ crc = (crc << 8) ^ CRC_LUT[((crc >> 24) ^ *data++) & 0xff];
+ return crc;
+}
+
+static void vidtv_psi_update_version_num(struct vidtv_psi_table_header *h)
+{
+ h->version++;
+}
+
+static inline u16 vidtv_psi_sdt_serv_get_desc_loop_len(struct vidtv_psi_table_sdt_service *s)
+{
+ u16 mask;
+ u16 ret;
+
+ mask = GENMASK(11, 0);
+
+ ret = be16_to_cpu(s->bitfield) & mask;
+ return ret;
+}
+
+static inline u16 vidtv_psi_pmt_stream_get_desc_loop_len(struct vidtv_psi_table_pmt_stream *s)
+{
+ u16 mask;
+ u16 ret;
+
+ mask = GENMASK(9, 0);
+
+ ret = be16_to_cpu(s->bitfield2) & mask;
+ return ret;
+}
+
+static inline u16 vidtv_psi_pmt_get_desc_loop_len(struct vidtv_psi_table_pmt *p)
+{
+ u16 mask;
+ u16 ret;
+
+ mask = GENMASK(9, 0);
+
+ ret = be16_to_cpu(p->bitfield2) & mask;
+ return ret;
+}
+
+static inline u16 vidtv_psi_get_sec_len(struct vidtv_psi_table_header *h)
+{
+ u16 mask;
+ u16 ret;
+
+ mask = GENMASK(11, 0);
+
+ ret = be16_to_cpu(h->bitfield) & mask;
+ return ret;
+}
+
+inline u16 vidtv_psi_get_pat_program_pid(struct vidtv_psi_table_pat_program *p)
+{
+ u16 mask;
+ u16 ret;
+
+ mask = GENMASK(12, 0);
+
+ ret = be16_to_cpu(p->bitfield) & mask;
+ return ret;
+}
+
+inline u16 vidtv_psi_pmt_stream_get_elem_pid(struct vidtv_psi_table_pmt_stream *s)
+{
+ u16 mask;
+ u16 ret;
+
+ mask = GENMASK(12, 0);
+
+ ret = be16_to_cpu(s->bitfield) & mask;
+ return ret;
+}
+
+static inline void vidtv_psi_set_desc_loop_len(__be16 *bitfield, u16 new_len, u8 desc_len_nbits)
+{
+ u16 mask;
+ __be16 new;
+
+ mask = GENMASK(15, desc_len_nbits);
+
+ new = cpu_to_be16((be16_to_cpu(*bitfield) & mask) | new_len);
+ *bitfield = new;
+}
+
+static void vidtv_psi_set_sec_len(struct vidtv_psi_table_header *h, u16 new_len)
+{
+ u16 old_len = vidtv_psi_get_sec_len(h);
+ __be16 new;
+ u16 mask;
+
+ mask = GENMASK(15, 13);
+
+ new = cpu_to_be16((be16_to_cpu(h->bitfield) & mask) | new_len);
+
+ if (old_len > MAX_SECTION_LEN)
+ pr_warn_ratelimited("section length: %d > %d, old len was %d\n",
+ new_len,
+ MAX_SECTION_LEN,
+ old_len);
+
+ h->bitfield = new;
+}
+
+static u32 vidtv_psi_ts_psi_write_into(struct psi_write_args args)
+{
+ /*
+ * Packetize PSI sections into TS packets:
+ * push a TS header (4bytes) every 184 bytes
+ * manage the continuity_counter
+ * add stuffing (i.e. padding bytes) after the CRC
+ */
+
+ u32 nbytes_past_boundary = (args.dest_offset % TS_PACKET_LEN);
+ bool aligned = (nbytes_past_boundary == 0);
+ struct vidtv_mpeg_ts ts_header = {};
+
+ /* number of bytes written by this function */
+ u32 nbytes = 0;
+ /* how much there is left to write */
+ u32 remaining_len = args.len;
+ /* how much we can be written in this packet */
+ u32 payload_write_len = 0;
+ /* where we are in the source */
+ u32 payload_offset = 0;
+
+ const u16 PAYLOAD_START = args.new_psi_section;
+
+ if (!args.crc && !args.is_crc)
+ pr_warn_ratelimited("Missing CRC for chunk\n");
+
+ if (args.crc)
+ *args.crc = dvb_crc32(*args.crc, args.from, args.len);
+
+ if (args.new_psi_section && !aligned) {
+ pr_warn_ratelimited("Cannot write a new PSI section in a misaligned buffer\n");
+
+ /* forcibly align and hope for the best */
+ nbytes += vidtv_memset(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.dest_buf_sz,
+ TS_FILL_BYTE,
+ TS_PACKET_LEN - nbytes_past_boundary);
+ }
+
+ while (remaining_len) {
+ nbytes_past_boundary = (args.dest_offset + nbytes) % TS_PACKET_LEN;
+ aligned = (nbytes_past_boundary == 0);
+
+ if (aligned) {
+ /* if at a packet boundary, write a new TS header */
+ ts_header.sync_byte = TS_SYNC_BYTE;
+ ts_header.bitfield = cpu_to_be16((PAYLOAD_START << 14) | args.pid);
+ ts_header.scrambling = 0;
+ ts_header.continuity_counter = *args.continuity_counter;
+ ts_header.payload = 1;
+ /* no adaptation field */
+ ts_header.adaptation_field = 0;
+
+ /* copy the header */
+ nbytes += vidtv_memcpy(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.dest_buf_sz,
+ &ts_header,
+ sizeof(ts_header));
+ /*
+ * This will trigger a discontinuity if the buffer is full,
+ * effectively dropping the packet.
+ */
+ vidtv_ts_inc_cc(args.continuity_counter);
+ }
+
+ /* write the pointer_field in the first byte of the payload */
+ if (args.new_psi_section)
+ nbytes += vidtv_memset(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.dest_buf_sz,
+ 0x0,
+ 1);
+
+ /* write as much of the payload as possible */
+ nbytes_past_boundary = (args.dest_offset + nbytes) % TS_PACKET_LEN;
+ payload_write_len = min(TS_PACKET_LEN - nbytes_past_boundary, remaining_len);
+
+ nbytes += vidtv_memcpy(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.dest_buf_sz,
+ args.from + payload_offset,
+ payload_write_len);
+
+ /* 'payload_write_len' written from a total of 'len' requested*/
+ remaining_len -= payload_write_len;
+ payload_offset += payload_write_len;
+ }
+
+ /*
+ * fill the rest of the packet if there is any remaining space unused
+ */
+
+ nbytes_past_boundary = (args.dest_offset + nbytes) % TS_PACKET_LEN;
+
+ if (args.is_crc)
+ nbytes += vidtv_memset(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.dest_buf_sz,
+ TS_FILL_BYTE,
+ TS_PACKET_LEN - nbytes_past_boundary);
+
+ return nbytes;
+}
+
+static u32 table_section_crc32_write_into(struct crc32_write_args args)
+{
+ /* the CRC is the last entry in the section */
+ u32 nbytes = 0;
+ struct psi_write_args psi_args = {};
+
+ psi_args.dest_buf = args.dest_buf;
+ psi_args.from = &args.crc;
+ psi_args.len = CRC_SIZE_IN_BYTES;
+ psi_args.dest_offset = args.dest_offset;
+ psi_args.pid = args.pid;
+ psi_args.new_psi_section = false;
+ psi_args.continuity_counter = args.continuity_counter;
+ psi_args.is_crc = true;
+ psi_args.dest_buf_sz = args.dest_buf_sz;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+
+ return nbytes;
+}
+
+struct vidtv_psi_desc_service *vidtv_psi_service_desc_init(struct vidtv_psi_desc *head,
+ enum service_type service_type,
+ char *service_name,
+ char *provider_name)
+{
+ struct vidtv_psi_desc_service *desc;
+ u32 service_name_len = service_name ? strlen(service_name) : 0;
+ u32 provider_name_len = provider_name ? strlen(provider_name) : 0;
+
+ desc = kzalloc(sizeof(*desc), GFP_KERNEL);
+
+ desc->type = SERVICE_DESCRIPTOR;
+
+ desc->length = sizeof_field(struct vidtv_psi_desc_service, service_type)
+ + sizeof_field(struct vidtv_psi_desc_service, provider_name_len)
+ + provider_name_len
+ + sizeof_field(struct vidtv_psi_desc_service, service_name_len)
+ + service_name_len;
+
+ desc->service_type = service_type;
+
+ desc->service_name_len = service_name_len;
+
+ if (service_name && service_name_len)
+ desc->service_name = kstrdup(service_name, GFP_KERNEL);
+
+ desc->provider_name_len = provider_name_len;
+
+ if (provider_name && provider_name_len)
+ desc->provider_name = kstrdup(provider_name, GFP_KERNEL);
+
+ if (head) {
+ while (head->next)
+ head = head->next;
+
+ head->next = (struct vidtv_psi_desc *)desc;
+ }
+ return desc;
+}
+
+struct vidtv_psi_desc_registration
+*vidtv_psi_registration_desc_init(struct vidtv_psi_desc *head,
+ __be32 format_id,
+ u8 *additional_ident_info,
+ u32 additional_info_len)
+{
+ struct vidtv_psi_desc_registration *desc;
+
+ desc = kzalloc(sizeof(*desc) + sizeof(format_id) + additional_info_len, GFP_KERNEL);
+
+ desc->type = REGISTRATION_DESCRIPTOR;
+
+ desc->length = sizeof_field(struct vidtv_psi_desc_registration, format_id)
+ + additional_info_len;
+
+ desc->format_id = format_id;
+
+ if (additional_ident_info && additional_info_len)
+ memcpy(desc->additional_identification_info,
+ additional_ident_info,
+ additional_info_len);
+
+ if (head) {
+ while (head->next)
+ head = head->next;
+
+ head->next = (struct vidtv_psi_desc *)desc;
+ }
+
+ return desc;
+}
+
+struct vidtv_psi_desc *vidtv_psi_desc_clone(struct vidtv_psi_desc *desc)
+{
+ struct vidtv_psi_desc *head = NULL;
+ struct vidtv_psi_desc *prev = NULL;
+ struct vidtv_psi_desc *curr = NULL;
+
+ struct vidtv_psi_desc_service *service;
+
+ while (desc) {
+ switch (desc->type) {
+ case SERVICE_DESCRIPTOR:
+ service = (struct vidtv_psi_desc_service *)desc;
+ curr = (struct vidtv_psi_desc *)
+ vidtv_psi_service_desc_init(head,
+ service->service_type,
+ service->service_name,
+ service->provider_name);
+ break;
+
+ case REGISTRATION_DESCRIPTOR:
+ default:
+ curr = kzalloc(sizeof(*desc) + desc->length, GFP_KERNEL);
+ memcpy(curr, desc, sizeof(*desc) + desc->length);
+ break;
+ }
+
+ if (curr)
+ curr->next = NULL;
+ if (!head)
+ head = curr;
+ if (prev)
+ prev->next = curr;
+
+ prev = curr;
+ desc = desc->next;
+ }
+
+ return head;
+}
+
+void vidtv_psi_desc_destroy(struct vidtv_psi_desc *desc)
+{
+ struct vidtv_psi_desc *curr = desc;
+ struct vidtv_psi_desc *tmp = NULL;
+
+ while (curr) {
+ tmp = curr;
+ curr = curr->next;
+
+ switch (tmp->type) {
+ case SERVICE_DESCRIPTOR:
+ kfree(((struct vidtv_psi_desc_service *)tmp)->provider_name);
+ kfree(((struct vidtv_psi_desc_service *)tmp)->service_name);
+
+ break;
+ case REGISTRATION_DESCRIPTOR:
+ /* nothing to do */
+ break;
+
+ default:
+ pr_warn_ratelimited("Possible leak: not handling descriptor type %d\n",
+ tmp->type);
+ break;
+ }
+
+ kfree(tmp);
+ }
+}
+
+static u16
+vidtv_psi_desc_comp_loop_len(struct vidtv_psi_desc *desc)
+{
+ u32 length = 0;
+
+ if (!desc)
+ return 0;
+
+ while (desc) {
+ length += sizeof_field(struct vidtv_psi_desc, type);
+ length += sizeof_field(struct vidtv_psi_desc, length);
+ length += desc->length; /* from 'length' field until the end of the descriptor */
+ desc = desc->next;
+ }
+
+ return length;
+}
+
+void vidtv_psi_desc_assign(struct vidtv_psi_desc **to,
+ struct vidtv_psi_desc *desc)
+{
+ if (desc == *to)
+ return;
+
+ if (*to)
+ vidtv_psi_desc_destroy(*to);
+
+ *to = desc;
+}
+
+void vidtv_pmt_desc_assign(struct vidtv_psi_table_pmt *pmt,
+ struct vidtv_psi_desc **to,
+ struct vidtv_psi_desc *desc)
+{
+ vidtv_psi_desc_assign(to, desc);
+ vidtv_psi_pmt_table_update_sec_len(pmt);
+
+ if (vidtv_psi_get_sec_len(&pmt->header) > MAX_SECTION_LEN)
+ vidtv_psi_desc_assign(to, NULL);
+
+ vidtv_psi_update_version_num(&pmt->header);
+}
+
+void vidtv_sdt_desc_assign(struct vidtv_psi_table_sdt *sdt,
+ struct vidtv_psi_desc **to,
+ struct vidtv_psi_desc *desc)
+{
+ vidtv_psi_desc_assign(to, desc);
+ vidtv_psi_sdt_table_update_sec_len(sdt);
+
+ if (vidtv_psi_get_sec_len(&sdt->header) > MAX_SECTION_LEN)
+ vidtv_psi_desc_assign(to, NULL);
+
+ vidtv_psi_update_version_num(&sdt->header);
+}
+
+static u32 vidtv_psi_desc_write_into(struct desc_write_args args)
+{
+ /* the number of bytes written by this function */
+ u32 nbytes = 0;
+ struct psi_write_args psi_args = {};
+
+ psi_args.dest_buf = args.dest_buf;
+ psi_args.from = &args.desc->type;
+
+ psi_args.len = sizeof_field(struct vidtv_psi_desc, type) +
+ sizeof_field(struct vidtv_psi_desc, length);
+
+ psi_args.dest_offset = args.dest_offset + nbytes;
+ psi_args.pid = args.pid;
+ psi_args.new_psi_section = false;
+ psi_args.continuity_counter = args.continuity_counter;
+ psi_args.is_crc = false;
+ psi_args.dest_buf_sz = args.dest_buf_sz;
+ psi_args.crc = args.crc;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+
+ switch (args.desc->type) {
+ case SERVICE_DESCRIPTOR:
+ psi_args.dest_offset = args.dest_offset + nbytes;
+ psi_args.len = sizeof_field(struct vidtv_psi_desc_service, service_type) +
+ sizeof_field(struct vidtv_psi_desc_service, provider_name_len);
+ psi_args.from = &((struct vidtv_psi_desc_service *)args.desc)->service_type;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+
+ psi_args.dest_offset = args.dest_offset + nbytes;
+ psi_args.len = ((struct vidtv_psi_desc_service *)args.desc)->provider_name_len;
+ psi_args.from = ((struct vidtv_psi_desc_service *)args.desc)->provider_name;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+
+ psi_args.dest_offset = args.dest_offset + nbytes;
+ psi_args.len = sizeof_field(struct vidtv_psi_desc_service, service_name_len);
+ psi_args.from = &((struct vidtv_psi_desc_service *)args.desc)->service_name_len;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+
+ psi_args.dest_offset = args.dest_offset + nbytes;
+ psi_args.len = ((struct vidtv_psi_desc_service *)args.desc)->service_name_len;
+ psi_args.from = ((struct vidtv_psi_desc_service *)args.desc)->service_name;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+ break;
+
+ case REGISTRATION_DESCRIPTOR:
+ default:
+ psi_args.dest_offset = args.dest_offset + nbytes;
+ psi_args.len = args.desc->length;
+ psi_args.from = &args.desc->data;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+ break;
+ }
+
+ return nbytes;
+}
+
+static u32
+vidtv_psi_table_header_write_into(struct header_write_args args)
+{
+ /* the number of bytes written by this function */
+ u32 nbytes = 0;
+ struct psi_write_args psi_args = {};
+
+ psi_args.dest_buf = args.dest_buf;
+ psi_args.from = args.h;
+ psi_args.len = sizeof(struct vidtv_psi_table_header);
+ psi_args.dest_offset = args.dest_offset;
+ psi_args.pid = args.pid;
+ psi_args.new_psi_section = true;
+ psi_args.continuity_counter = args.continuity_counter;
+ psi_args.is_crc = false;
+ psi_args.dest_buf_sz = args.dest_buf_sz;
+ psi_args.crc = args.crc;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+
+ return nbytes;
+}
+
+void
+vidtv_psi_pat_table_update_sec_len(struct vidtv_psi_table_pat *pat)
+{
+ /* see ISO/IEC 13818-1 : 2000 p.43 */
+ u16 length = 0;
+ u32 i;
+
+ /* from immediately after 'section_length' until 'last_section_number'*/
+ length += PAT_LEN_UNTIL_LAST_SECTION_NUMBER;
+
+ /* do not count the pointer */
+ for (i = 0; i < pat->programs; ++i)
+ length += sizeof(struct vidtv_psi_table_pat_program) -
+ sizeof(struct vidtv_psi_table_pat_program *);
+
+ length += CRC_SIZE_IN_BYTES;
+
+ vidtv_psi_set_sec_len(&pat->header, length);
+}
+
+void vidtv_psi_pmt_table_update_sec_len(struct vidtv_psi_table_pmt *pmt)
+{
+ /* see ISO/IEC 13818-1 : 2000 p.46 */
+ u16 length = 0;
+ struct vidtv_psi_table_pmt_stream *s = pmt->stream;
+ u16 desc_loop_len;
+
+ /* from immediately after 'section_length' until 'program_info_length'*/
+ length += PMT_LEN_UNTIL_PROGRAM_INFO_LENGTH;
+
+ desc_loop_len = vidtv_psi_desc_comp_loop_len(pmt->descriptor);
+ vidtv_psi_set_desc_loop_len(&pmt->bitfield2, desc_loop_len, 10);
+
+ length += desc_loop_len;
+
+ while (s) {
+ /* skip both pointers at the end */
+ length += sizeof(struct vidtv_psi_table_pmt_stream) -
+ sizeof(struct vidtv_psi_desc *) -
+ sizeof(struct vidtv_psi_table_pmt_stream *);
+
+ desc_loop_len = vidtv_psi_desc_comp_loop_len(s->descriptor);
+ vidtv_psi_set_desc_loop_len(&s->bitfield2, desc_loop_len, 10);
+
+ length += desc_loop_len;
+
+ s = s->next;
+ }
+
+ length += CRC_SIZE_IN_BYTES;
+
+ vidtv_psi_set_sec_len(&pmt->header, length);
+}
+
+void vidtv_psi_sdt_table_update_sec_len(struct vidtv_psi_table_sdt *sdt)
+{
+ /* see ETSI EN 300 468 V 1.10.1 p.24 */
+ u16 length = 0;
+ struct vidtv_psi_table_sdt_service *s = sdt->service;
+ u16 desc_loop_len;
+
+ /*
+ * from immediately after 'section_length' until
+ * 'reserved_for_future_use'
+ */
+ length += SDT_LEN_UNTIL_RESERVED_FOR_FUTURE_USE;
+
+ while (s) {
+ /* skip both pointers at the end */
+ length += sizeof(struct vidtv_psi_table_sdt_service) -
+ sizeof(struct vidtv_psi_desc *) -
+ sizeof(struct vidtv_psi_table_sdt_service *);
+
+ desc_loop_len = vidtv_psi_desc_comp_loop_len(s->descriptor);
+ vidtv_psi_set_desc_loop_len(&s->bitfield, desc_loop_len, 12);
+
+ length += desc_loop_len;
+
+ s = s->next;
+ }
+
+ length += CRC_SIZE_IN_BYTES;
+
+ vidtv_psi_set_sec_len(&sdt->header, length);
+}
+
+struct vidtv_psi_table_pat_program*
+vidtv_psi_pat_program_init(struct vidtv_psi_table_pat_program *head,
+ u16 service_id,
+ u16 program_map_pid)
+{
+ struct vidtv_psi_table_pat_program *program;
+ const u16 RESERVED = 0x07;
+
+ program = kzalloc(sizeof(*program), GFP_KERNEL);
+
+ program->service_id = cpu_to_be16(service_id);
+
+ /* pid for the PMT section in the TS */
+ program->bitfield = cpu_to_be16((RESERVED << 13) | program_map_pid);
+ program->next = NULL;
+
+ if (head) {
+ while (head->next)
+ head = head->next;
+
+ head->next = program;
+ }
+
+ return program;
+}
+
+void
+vidtv_psi_pat_program_destroy(struct vidtv_psi_table_pat_program *p)
+{
+ struct vidtv_psi_table_pat_program *curr = p;
+ struct vidtv_psi_table_pat_program *tmp = NULL;
+
+ while (curr) {
+ tmp = curr;
+ curr = curr->next;
+ kfree(tmp);
+ }
+}
+
+void
+vidtv_psi_pat_program_assign(struct vidtv_psi_table_pat *pat,
+ struct vidtv_psi_table_pat_program *p)
+{
+ /* This function transfers ownership of p to the table */
+
+ u16 program_count = 0;
+ struct vidtv_psi_table_pat_program *program = p;
+
+ if (p == pat->program)
+ return;
+
+ while (program) {
+ ++program_count;
+ program = program->next;
+ }
+
+ pat->programs = program_count;
+ pat->program = p;
+
+ /* Recompute section length */
+ vidtv_psi_pat_table_update_sec_len(pat);
+
+ if (vidtv_psi_get_sec_len(&pat->header) > MAX_SECTION_LEN)
+ vidtv_psi_pat_program_assign(pat, NULL);
+
+ vidtv_psi_update_version_num(&pat->header);
+}
+
+struct vidtv_psi_table_pat *vidtv_psi_pat_table_init(u16 transport_stream_id)
+{
+ struct vidtv_psi_table_pat *pat = kzalloc(sizeof(*pat), GFP_KERNEL);
+ const u16 SYNTAX = 0x1;
+ const u16 ZERO = 0x0;
+ const u16 ONES = 0x03;
+
+ pat->header.table_id = 0x0;
+
+ pat->header.bitfield = cpu_to_be16((SYNTAX << 15) | (ZERO << 14) | (ONES << 12));
+ pat->header.id = cpu_to_be16(transport_stream_id);
+ pat->header.current_next = 0x1;
+
+ pat->header.version = 0x1f;
+
+ pat->header.one2 = 0x03;
+ pat->header.section_id = 0x0;
+ pat->header.last_section = 0x0;
+
+ pat->programs = 0;
+
+ vidtv_psi_pat_table_update_sec_len(pat);
+
+ return pat;
+}
+
+u32 vidtv_psi_pat_write_into(struct vidtv_psi_pat_write_args args)
+{
+ /* the number of bytes written by this function */
+ u32 nbytes = 0;
+ const u16 pat_pid = VIDTV_PAT_PID;
+ u32 crc = 0xffffffff;
+
+ struct vidtv_psi_table_pat_program *p = args.pat->program;
+
+ struct header_write_args h_args = {};
+ struct psi_write_args psi_args = {};
+ struct crc32_write_args c_args = {};
+
+ vidtv_psi_pat_table_update_sec_len(args.pat);
+
+ h_args.dest_buf = args.buf;
+ h_args.dest_offset = args.offset;
+ h_args.h = &args.pat->header;
+ h_args.pid = pat_pid;
+ h_args.continuity_counter = args.continuity_counter;
+ h_args.dest_buf_sz = args.buf_sz;
+ h_args.crc = &crc;
+
+ nbytes += vidtv_psi_table_header_write_into(h_args);
+
+ /* note that the field 'u16 programs' is not really part of the PAT */
+
+ psi_args.dest_buf = args.buf;
+ psi_args.pid = pat_pid;
+ psi_args.new_psi_section = false;
+ psi_args.continuity_counter = args.continuity_counter;
+ psi_args.is_crc = false;
+ psi_args.dest_buf_sz = args.buf_sz;
+ psi_args.crc = &crc;
+
+ while (p) {
+ /* copy the PAT programs */
+ psi_args.from = p;
+ /* skip the pointer */
+ psi_args.len = sizeof(*p) -
+ sizeof(struct vidtv_psi_table_pat_program *);
+ psi_args.dest_offset = args.offset + nbytes;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+
+ p = p->next;
+ }
+
+ c_args.dest_buf = args.buf;
+ c_args.dest_offset = args.offset + nbytes;
+ c_args.crc = cpu_to_be32(crc);
+ c_args.pid = pat_pid;
+ c_args.continuity_counter = args.continuity_counter;
+ c_args.dest_buf_sz = args.buf_sz;
+
+ /* Write the CRC32 at the end */
+ nbytes += table_section_crc32_write_into(c_args);
+
+ return nbytes;
+}
+
+void
+vidtv_psi_pat_table_destroy(struct vidtv_psi_table_pat *p)
+{
+ vidtv_psi_pat_program_destroy(p->program);
+ kfree(p);
+}
+
+struct vidtv_psi_table_pmt_stream*
+vidtv_psi_pmt_stream_init(struct vidtv_psi_table_pmt_stream *head,
+ enum vidtv_psi_stream_types stream_type,
+ u16 es_pid)
+{
+ struct vidtv_psi_table_pmt_stream *stream;
+ const u16 RESERVED1 = 0x07;
+ const u16 RESERVED2 = 0x0f;
+ const u16 ZERO = 0x0;
+ u16 desc_loop_len;
+
+ stream = kzalloc(sizeof(*stream), GFP_KERNEL);
+
+ stream->type = stream_type;
+
+ stream->bitfield = cpu_to_be16((RESERVED1 << 13) | es_pid);
+
+ desc_loop_len = vidtv_psi_desc_comp_loop_len(stream->descriptor);
+
+ stream->bitfield2 = cpu_to_be16((RESERVED2 << 12) |
+ (ZERO << 10) |
+ desc_loop_len);
+ stream->next = NULL;
+
+ if (head) {
+ while (head->next)
+ head = head->next;
+
+ head->next = stream;
+ }
+
+ return stream;
+}
+
+void vidtv_psi_pmt_stream_destroy(struct vidtv_psi_table_pmt_stream *s)
+{
+ struct vidtv_psi_table_pmt_stream *curr_stream = s;
+ struct vidtv_psi_table_pmt_stream *tmp_stream = NULL;
+
+ while (curr_stream) {
+ tmp_stream = curr_stream;
+ curr_stream = curr_stream->next;
+ vidtv_psi_desc_destroy(tmp_stream->descriptor);
+ kfree(tmp_stream);
+ }
+}
+
+void vidtv_psi_pmt_stream_assign(struct vidtv_psi_table_pmt *pmt,
+ struct vidtv_psi_table_pmt_stream *s)
+{
+ /* This function transfers ownership of s to the table */
+ if (s == pmt->stream)
+ return;
+
+ pmt->stream = s;
+ vidtv_psi_pmt_table_update_sec_len(pmt);
+
+ if (vidtv_psi_get_sec_len(&pmt->header) > MAX_SECTION_LEN)
+ vidtv_psi_pmt_stream_assign(pmt, NULL);
+
+ vidtv_psi_update_version_num(&pmt->header);
+}
+
+u16 vidtv_psi_pmt_get_pid(struct vidtv_psi_table_pmt *section,
+ struct vidtv_psi_table_pat *pat)
+{
+ struct vidtv_psi_table_pat_program *program = pat->program;
+
+ /*
+ * service_id is the same as program_number in the
+ * corresponding program_map_section
+ * see ETSI EN 300 468 v1.15.1 p. 24
+ */
+ while (program) {
+ if (program->service_id == section->header.id)
+ return vidtv_psi_get_pat_program_pid(program);
+
+ program = program->next;
+ }
+
+ return TS_LAST_VALID_PID + 1; /* not found */
+}
+
+struct vidtv_psi_table_pmt *vidtv_psi_pmt_table_init(u16 program_number,
+ u16 pcr_pid)
+{
+ struct vidtv_psi_table_pmt *pmt = kzalloc(sizeof(*pmt), GFP_KERNEL);
+ const u16 SYNTAX = 0x1;
+ const u16 ZERO = 0x0;
+ const u16 ONES = 0x03;
+ const u16 RESERVED1 = 0x07;
+ const u16 RESERVED2 = 0x0f;
+ u16 desc_loop_len;
+
+ if (!pcr_pid)
+ pcr_pid = 0x1fff;
+
+ pmt->header.table_id = 0x2;
+
+ pmt->header.bitfield = cpu_to_be16((SYNTAX << 15) | (ZERO << 14) | (ONES << 12));
+
+ pmt->header.id = cpu_to_be16(program_number);
+ pmt->header.current_next = 0x1;
+
+ pmt->header.version = 0x1f;
+
+ pmt->header.one2 = ONES;
+ pmt->header.section_id = 0;
+ pmt->header.last_section = 0;
+
+ pmt->bitfield = cpu_to_be16((RESERVED1 << 13) | pcr_pid);
+
+ desc_loop_len = vidtv_psi_desc_comp_loop_len(pmt->descriptor);
+
+ pmt->bitfield2 = cpu_to_be16((RESERVED2 << 12) |
+ (ZERO << 10) |
+ desc_loop_len);
+
+ vidtv_psi_pmt_table_update_sec_len(pmt);
+
+ return pmt;
+}
+
+u32 vidtv_psi_pmt_write_into(struct vidtv_psi_pmt_write_args args)
+{
+ /* the number of bytes written by this function */
+ u32 nbytes = 0;
+ u32 crc = 0xffffffff;
+
+ struct vidtv_psi_desc *table_descriptor = args.pmt->descriptor;
+ struct vidtv_psi_table_pmt_stream *stream = args.pmt->stream;
+ struct vidtv_psi_desc *stream_descriptor = (stream) ?
+ args.pmt->stream->descriptor :
+ NULL;
+
+ struct header_write_args h_args = {};
+ struct psi_write_args psi_args = {};
+ struct desc_write_args d_args = {};
+ struct crc32_write_args c_args = {};
+
+ vidtv_psi_pmt_table_update_sec_len(args.pmt);
+
+ h_args.dest_buf = args.buf;
+ h_args.dest_offset = args.offset;
+ h_args.h = &args.pmt->header;
+ h_args.pid = args.pid;
+ h_args.continuity_counter = args.continuity_counter;
+ h_args.dest_buf_sz = args.buf_sz;
+ h_args.crc = &crc;
+
+ nbytes += vidtv_psi_table_header_write_into(h_args);
+
+ /* write the two bitfields */
+ psi_args.dest_buf = args.buf;
+ psi_args.from = &args.pmt->bitfield;
+ psi_args.len = sizeof_field(struct vidtv_psi_table_pmt, bitfield) +
+ sizeof_field(struct vidtv_psi_table_pmt, bitfield2);
+
+ psi_args.dest_offset = args.offset + nbytes;
+ psi_args.pid = args.pid;
+ psi_args.new_psi_section = false;
+ psi_args.continuity_counter = args.continuity_counter;
+ psi_args.is_crc = false;
+ psi_args.dest_buf_sz = args.buf_sz;
+ psi_args.crc = &crc;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+
+ while (table_descriptor) {
+ /* write the descriptors, if any */
+ d_args.dest_buf = args.buf;
+ d_args.dest_offset = args.offset + nbytes;
+ d_args.desc = table_descriptor;
+ d_args.pid = args.pid;
+ d_args.continuity_counter = args.continuity_counter;
+ d_args.dest_buf_sz = args.buf_sz;
+ d_args.crc = &crc;
+
+ nbytes += vidtv_psi_desc_write_into(d_args);
+
+ table_descriptor = table_descriptor->next;
+ }
+
+ while (stream) {
+ /* write the streams, if any */
+ psi_args.from = stream;
+ psi_args.len = sizeof_field(struct vidtv_psi_table_pmt_stream, type) +
+ sizeof_field(struct vidtv_psi_table_pmt_stream, bitfield) +
+ sizeof_field(struct vidtv_psi_table_pmt_stream, bitfield2);
+ psi_args.dest_offset = args.offset + nbytes;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+
+ while (stream_descriptor) {
+ /* write the stream descriptors, if any */
+ d_args.dest_buf = args.buf;
+ d_args.dest_offset = args.offset + nbytes;
+ d_args.desc = stream_descriptor;
+ d_args.pid = args.pid;
+ d_args.continuity_counter = args.continuity_counter;
+ d_args.dest_buf_sz = args.buf_sz;
+ d_args.crc = &crc;
+
+ nbytes += vidtv_psi_desc_write_into(d_args);
+
+ stream_descriptor = stream_descriptor->next;
+ }
+
+ stream = stream->next;
+ }
+
+ c_args.dest_buf = args.buf;
+ c_args.dest_offset = args.offset + nbytes;
+ c_args.crc = cpu_to_be32(crc);
+ c_args.pid = args.pid;
+ c_args.continuity_counter = args.continuity_counter;
+ c_args.dest_buf_sz = args.buf_sz;
+
+ /* Write the CRC32 at the end */
+ nbytes += table_section_crc32_write_into(c_args);
+
+ return nbytes;
+}
+
+void vidtv_psi_pmt_table_destroy(struct vidtv_psi_table_pmt *pmt)
+{
+ vidtv_psi_desc_destroy(pmt->descriptor);
+ vidtv_psi_pmt_stream_destroy(pmt->stream);
+ kfree(pmt);
+}
+
+struct vidtv_psi_table_sdt *vidtv_psi_sdt_table_init(u16 transport_stream_id)
+{
+ struct vidtv_psi_table_sdt *sdt = kzalloc(sizeof(*sdt), GFP_KERNEL);
+ const u16 SYNTAX = 0x1;
+ const u16 ONE = 0x1;
+ const u16 ONES = 0x03;
+ const u16 RESERVED = 0xff;
+
+ sdt->header.table_id = 0x42;
+
+ sdt->header.bitfield = cpu_to_be16((SYNTAX << 15) | (ONE << 14) | (ONES << 12));
+
+ /*
+ * This is a 16-bit field which serves as a label for identification
+ * of the TS, about which the SDT informs, from any other multiplex
+ * within the delivery system.
+ */
+ sdt->header.id = cpu_to_be16(transport_stream_id);
+ sdt->header.current_next = ONE;
+
+ sdt->header.version = 0x1f;
+
+ sdt->header.one2 = ONES;
+ sdt->header.section_id = 0;
+ sdt->header.last_section = 0;
+
+ /*
+ * FIXME: The network_id range from 0xFF01 to 0xFFFF is used to
+ * indicate temporary private use. For now, let's use the first
+ * value.
+ * This can be changed to something more useful, when support for
+ * NIT gets added
+ */
+ sdt->network_id = cpu_to_be16(0xff01);
+ sdt->reserved = RESERVED;
+
+ vidtv_psi_sdt_table_update_sec_len(sdt);
+
+ return sdt;
+}
+
+u32 vidtv_psi_sdt_write_into(struct vidtv_psi_sdt_write_args args)
+{
+ u32 nbytes = 0;
+ u16 sdt_pid = VIDTV_SDT_PID; /* see ETSI EN 300 468 v1.15.1 p. 11 */
+
+ u32 crc = 0xffffffff;
+
+ struct vidtv_psi_table_sdt_service *service = args.sdt->service;
+ struct vidtv_psi_desc *service_desc = (args.sdt->service) ?
+ args.sdt->service->descriptor :
+ NULL;
+
+ struct header_write_args h_args = {};
+ struct psi_write_args psi_args = {};
+ struct desc_write_args d_args = {};
+ struct crc32_write_args c_args = {};
+
+ vidtv_psi_sdt_table_update_sec_len(args.sdt);
+
+ h_args.dest_buf = args.buf;
+ h_args.dest_offset = args.offset;
+ h_args.h = &args.sdt->header;
+ h_args.pid = sdt_pid;
+ h_args.continuity_counter = args.continuity_counter;
+ h_args.dest_buf_sz = args.buf_sz;
+ h_args.crc = &crc;
+
+ nbytes += vidtv_psi_table_header_write_into(h_args);
+
+ psi_args.dest_buf = args.buf;
+ psi_args.from = &args.sdt->network_id;
+
+ psi_args.len = sizeof_field(struct vidtv_psi_table_sdt, network_id) +
+ sizeof_field(struct vidtv_psi_table_sdt, reserved);
+
+ psi_args.dest_offset = args.offset + nbytes;
+ psi_args.pid = sdt_pid;
+ psi_args.new_psi_section = false;
+ psi_args.continuity_counter = args.continuity_counter;
+ psi_args.is_crc = false;
+ psi_args.dest_buf_sz = args.buf_sz;
+ psi_args.crc = &crc;
+
+ /* copy u16 network_id + u8 reserved)*/
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+
+ while (service) {
+ /* copy the services, if any */
+ psi_args.from = service;
+ /* skip both pointers at the end */
+ psi_args.len = sizeof(struct vidtv_psi_table_sdt_service) -
+ sizeof(struct vidtv_psi_desc *) -
+ sizeof(struct vidtv_psi_table_sdt_service *);
+ psi_args.dest_offset = args.offset + nbytes;
+
+ nbytes += vidtv_psi_ts_psi_write_into(psi_args);
+
+ while (service_desc) {
+ /* copy the service descriptors, if any */
+ d_args.dest_buf = args.buf;
+ d_args.dest_offset = args.offset + nbytes;
+ d_args.desc = service_desc;
+ d_args.pid = sdt_pid;
+ d_args.continuity_counter = args.continuity_counter;
+ d_args.dest_buf_sz = args.buf_sz;
+ d_args.crc = &crc;
+
+ nbytes += vidtv_psi_desc_write_into(d_args);
+
+ service_desc = service_desc->next;
+ }
+
+ service = service->next;
+ }
+
+ c_args.dest_buf = args.buf;
+ c_args.dest_offset = args.offset + nbytes;
+ c_args.crc = cpu_to_be32(crc);
+ c_args.pid = sdt_pid;
+ c_args.continuity_counter = args.continuity_counter;
+ c_args.dest_buf_sz = args.buf_sz;
+
+ /* Write the CRC at the end */
+ nbytes += table_section_crc32_write_into(c_args);
+
+ return nbytes;
+}
+
+void vidtv_psi_sdt_table_destroy(struct vidtv_psi_table_sdt *sdt)
+{
+ vidtv_psi_sdt_service_destroy(sdt->service);
+ kfree(sdt);
+}
+
+struct vidtv_psi_table_sdt_service
+*vidtv_psi_sdt_service_init(struct vidtv_psi_table_sdt_service *head,
+ u16 service_id)
+{
+ struct vidtv_psi_table_sdt_service *service;
+
+ service = kzalloc(sizeof(*service), GFP_KERNEL);
+
+ /*
+ * ETSI 300 468: this is a 16bit field which serves as a label to
+ * identify this service from any other service within the TS.
+ * The service id is the same as the program number in the
+ * corresponding program_map_section
+ */
+ service->service_id = cpu_to_be16(service_id);
+ service->EIT_schedule = 0x0;
+ service->EIT_present_following = 0x0;
+ service->reserved = 0x3f;
+
+ service->bitfield = cpu_to_be16(RUNNING << 13);
+
+ if (head) {
+ while (head->next)
+ head = head->next;
+
+ head->next = service;
+ }
+
+ return service;
+}
+
+void
+vidtv_psi_sdt_service_destroy(struct vidtv_psi_table_sdt_service *service)
+{
+ struct vidtv_psi_table_sdt_service *curr = service;
+ struct vidtv_psi_table_sdt_service *tmp = NULL;
+
+ while (curr) {
+ tmp = curr;
+ curr = curr->next;
+ vidtv_psi_desc_destroy(tmp->descriptor);
+ kfree(tmp);
+ }
+}
+
+void
+vidtv_psi_sdt_service_assign(struct vidtv_psi_table_sdt *sdt,
+ struct vidtv_psi_table_sdt_service *service)
+{
+ if (service == sdt->service)
+ return;
+
+ sdt->service = service;
+
+ /* recompute section length */
+ vidtv_psi_sdt_table_update_sec_len(sdt);
+
+ if (vidtv_psi_get_sec_len(&sdt->header) > MAX_SECTION_LEN)
+ vidtv_psi_sdt_service_assign(sdt, NULL);
+
+ vidtv_psi_update_version_num(&sdt->header);
+}
+
+struct vidtv_psi_table_pmt**
+vidtv_psi_pmt_create_sec_for_each_pat_entry(struct vidtv_psi_table_pat *pat, u16 pcr_pid)
+
+{
+ /*
+ * PMTs contain information about programs. For each program,
+ * there is one PMT section. This function will create a section
+ * for each program found in the PAT
+ */
+ struct vidtv_psi_table_pat_program *program = pat->program;
+ struct vidtv_psi_table_pmt **pmt_secs;
+ u32 i = 0;
+
+ /* a section for each program_id */
+ pmt_secs = kcalloc(pat->programs,
+ sizeof(struct vidtv_psi_table_pmt *),
+ GFP_KERNEL);
+
+ while (program) {
+ pmt_secs[i] = vidtv_psi_pmt_table_init(be16_to_cpu(program->service_id), pcr_pid);
+ ++i;
+ program = program->next;
+ }
+
+ return pmt_secs;
+}
+
+struct vidtv_psi_table_pmt
+*vidtv_psi_find_pmt_sec(struct vidtv_psi_table_pmt **pmt_sections,
+ u16 nsections,
+ u16 program_num)
+{
+ /* find the PMT section associated with 'program_num' */
+ struct vidtv_psi_table_pmt *sec = NULL;
+ u32 i;
+
+ for (i = 0; i < nsections; ++i) {
+ sec = pmt_sections[i];
+ if (be16_to_cpu(sec->header.id) == program_num)
+ return sec;
+ }
+
+ return NULL; /* not found */
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This file contains the logic to work with MPEG Program-Specific Information.
+ * These are defined both in ISO/IEC 13818-1 (systems) and ETSI EN 300 468.
+ * PSI is carried in the form of table structures, and although each table might
+ * technically be broken into one or more sections, we do not do this here,
+ * hence 'table' and 'section' are interchangeable for vidtv.
+ *
+ * This code currently supports three tables: PAT, PMT and SDT. These are the
+ * bare minimum to get userspace to recognize our MPEG transport stream. It can
+ * be extended to support more PSI tables in the future.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#ifndef VIDTV_PSI_H
+#define VIDTV_PSI_H
+
+#include <linux/types.h>
+#include <asm/byteorder.h>
+
+/*
+ * all section lengths start immediately after the 'section_length' field
+ * see ISO/IEC 13818-1 : 2000 and ETSI EN 300 468 V 1.10.1 for
+ * reference
+ */
+#define PAT_LEN_UNTIL_LAST_SECTION_NUMBER 5
+#define PMT_LEN_UNTIL_PROGRAM_INFO_LENGTH 9
+#define SDT_LEN_UNTIL_RESERVED_FOR_FUTURE_USE 8
+#define MAX_SECTION_LEN 1021
+#define VIDTV_PAT_PID 0 /* mandated by the specs */
+#define VIDTV_SDT_PID 0x0011 /* mandated by the specs */
+
+enum vidtv_psi_descriptors {
+ REGISTRATION_DESCRIPTOR = 0x05, /* See ISO/IEC 13818-1 section 2.6.8 */
+ SERVICE_DESCRIPTOR = 0x48, /* See ETSI EN 300 468 section 6.2.33 */
+};
+
+enum vidtv_psi_stream_types {
+ STREAM_PRIVATE_DATA = 0x06, /* see ISO/IEC 13818-1 2000 p. 48 */
+};
+
+/**
+ * struct vidtv_psi_desc - A generic PSI descriptor type.
+ * The descriptor length is an 8-bit field specifying the total number of bytes of the data portion
+ * of the descriptor following the byte defining the value of this field.
+ */
+struct vidtv_psi_desc {
+ struct vidtv_psi_desc *next;
+ u8 type;
+ u8 length;
+ u8 data[];
+} __packed;
+
+/**
+ * struct vidtv_psi_desc_service - Service descriptor.
+ * See ETSI EN 300 468 section 6.2.33.
+ */
+struct vidtv_psi_desc_service {
+ struct vidtv_psi_desc *next;
+ u8 type;
+ u8 length;
+
+ u8 service_type;
+ u8 provider_name_len;
+ char *provider_name;
+ u8 service_name_len;
+ char *service_name;
+} __packed;
+
+/**
+ * struct vidtv_psi_desc_registration - A registration descriptor.
+ * See ISO/IEC 13818-1 section 2.6.8
+ */
+struct vidtv_psi_desc_registration {
+ struct vidtv_psi_desc *next;
+ u8 type;
+ u8 length;
+
+ /*
+ * The format_identifier is a 32-bit value obtained from a Registration
+ * Authority as designated by ISO/IEC JTC 1/SC 29.
+ */
+ __be32 format_id;
+ /*
+ * The meaning of additional_identification_info bytes, if any, are
+ * defined by the assignee of that format_identifier, and once defined
+ * they shall not change.
+ */
+ u8 additional_identification_info[];
+} __packed;
+
+/**
+ * struct vidtv_psi_table_header - A header that is present for all PSI tables.
+ */
+struct vidtv_psi_table_header {
+ u8 table_id;
+
+ __be16 bitfield; /* syntax: 1, zero: 1, one: 2, section_length: 13 */
+
+ __be16 id; /* TS ID */
+ u8 current_next:1;
+ u8 version:5;
+ u8 one2:2;
+ u8 section_id; /* section_number */
+ u8 last_section; /* last_section_number */
+} __packed;
+
+/**
+ * struct vidtv_psi_table_pat_program - A single program in the PAT
+ * See ISO/IEC 13818-1 : 2000 p.43
+ */
+struct vidtv_psi_table_pat_program {
+ __be16 service_id;
+ __be16 bitfield; /* reserved: 3, program_map_pid/network_pid: 13 */
+ struct vidtv_psi_table_pat_program *next;
+} __packed;
+
+/**
+ * struct vidtv_psi_table_pat - The Program Allocation Table (PAT)
+ * See ISO/IEC 13818-1 : 2000 p.43
+ */
+struct vidtv_psi_table_pat {
+ struct vidtv_psi_table_header header;
+ u16 programs; /* Included by libdvbv5, not part of the table and not actually serialized */
+ struct vidtv_psi_table_pat_program *program;
+} __packed;
+
+/**
+ * struct vidtv_psi_table_sdt_service - Represents a service in the SDT.
+ * see ETSI EN 300 468 v1.15.1 section 5.2.3.
+ */
+struct vidtv_psi_table_sdt_service {
+ __be16 service_id;
+ u8 EIT_present_following:1;
+ u8 EIT_schedule:1;
+ u8 reserved:6;
+ __be16 bitfield; /* running_status: 3, free_ca:1, desc_loop_len:12 */
+ struct vidtv_psi_desc *descriptor;
+ struct vidtv_psi_table_sdt_service *next;
+} __packed;
+
+/**
+ * struct vidtv_psi_table_sdt - Represents the Service Description Table
+ * see ETSI EN 300 468 v1.15.1 section 5.2.3.
+ */
+
+struct vidtv_psi_table_sdt {
+ struct vidtv_psi_table_header header;
+ __be16 network_id; /* original_network_id */
+ u8 reserved;
+ struct vidtv_psi_table_sdt_service *service;
+} __packed;
+
+/**
+ * enum service_running_status - Status of a SDT service.
+ * see ETSI EN 300 468 v1.15.1 section 5.2.3 table 6.
+ */
+enum service_running_status {
+ RUNNING = 0x4,
+};
+
+/**
+ * enum service_type - The type of a SDT service.
+ * see ETSI EN 300 468 v1.15.1 section 6.2.33, table 81.
+ */
+enum service_type {
+ /* see ETSI EN 300 468 v1.15.1 p. 77 */
+ DIGITAL_TELEVISION_SERVICE = 0x1,
+};
+
+/**
+ * struct vidtv_psi_table_pmt_stream - A single stream in the PMT.
+ * See ISO/IEC 13818-1 : 2000 p.46.
+ */
+struct vidtv_psi_table_pmt_stream {
+ u8 type;
+ __be16 bitfield; /* reserved: 3, elementary_pid: 13 */
+ __be16 bitfield2; /*reserved: 4, zero: 2, desc_length: 10 */
+ struct vidtv_psi_desc *descriptor;
+ struct vidtv_psi_table_pmt_stream *next;
+} __packed;
+
+/**
+ * struct vidtv_psi_table_pmt - The Program Map Table (PMT).
+ * See ISO/IEC 13818-1 : 2000 p.46.
+ */
+struct vidtv_psi_table_pmt {
+ struct vidtv_psi_table_header header;
+ __be16 bitfield; /* reserved:3, pcr_pid: 13 */
+ __be16 bitfield2; /* reserved: 4, zero: 2, desc_len: 10 */
+ struct vidtv_psi_desc *descriptor;
+ struct vidtv_psi_table_pmt_stream *stream;
+} __packed;
+
+/**
+ * struct psi_write_args - Arguments for the PSI packetizer.
+ * @dest_buf: The buffer to write into.
+ * @from: PSI data to be copied.
+ * @len: How much to write.
+ * @dest_offset: where to start writing in the dest_buffer.
+ * @pid: TS packet ID.
+ * @new_psi_section: Set when starting a table section.
+ * @continuity_counter: Incremented on every new packet.
+ * @is_crc: Set when writing the CRC at the end.
+ * @dest_buf_sz: The size of the dest_buffer
+ * @crc: a pointer to store the crc for this chunk
+ */
+struct psi_write_args {
+ void *dest_buf;
+ void *from;
+ size_t len;
+ u32 dest_offset;
+ u16 pid;
+ bool new_psi_section;
+ u8 *continuity_counter;
+ bool is_crc;
+ u32 dest_buf_sz;
+ u32 *crc;
+};
+
+/**
+ * struct desc_write_args - Arguments in order to write a descriptor.
+ * @dest_buf: The buffer to write into.
+ * @dest_offset: where to start writing in the dest_buffer.
+ * @desc: A pointer to the descriptor
+ * @pid: TS packet ID.
+ * @continuity_counter: Incremented on every new packet.
+ * @dest_buf_sz: The size of the dest_buffer
+ * @crc: a pointer to store the crc for this chunk
+ */
+struct desc_write_args {
+ void *dest_buf;
+ u32 dest_offset;
+ struct vidtv_psi_desc *desc;
+ u16 pid;
+ u8 *continuity_counter;
+ u32 dest_buf_sz;
+ u32 *crc;
+};
+
+/**
+ * struct crc32_write_args - Arguments in order to write the CRC at the end of
+ * the PSI tables.
+ * @dest_buf: The buffer to write into.
+ * @dest_offset: where to start writing in the dest_buffer.
+ * @crc: the CRC value to write
+ * @pid: TS packet ID.
+ * @continuity_counter: Incremented on every new packet.
+ * @dest_buf_sz: The size of the dest_buffer
+ */
+struct crc32_write_args {
+ void *dest_buf;
+ u32 dest_offset;
+ __be32 crc;
+ u16 pid;
+ u8 *continuity_counter;
+ u32 dest_buf_sz;
+};
+
+/**
+ * struct header_write_args - Arguments in order to write the common table
+ * header
+ * @dest_buf: The buffer to write into.
+ * @dest_offset: where to start writing in the dest_buffer.
+ * @h: a pointer to the header.
+ * @pid: TS packet ID.
+ * @continuity_counter: Incremented on every new packet.
+ * @dest_buf_sz: The size of the dest_buffer
+ * @crc: a pointer to store the crc for this chunk
+ */
+struct header_write_args {
+ void *dest_buf;
+ u32 dest_offset;
+ struct vidtv_psi_table_header *h;
+ u16 pid;
+ u8 *continuity_counter;
+ u32 dest_buf_sz;
+ u32 *crc;
+};
+
+struct vidtv_psi_desc_service *vidtv_psi_service_desc_init(struct vidtv_psi_desc *head,
+ enum service_type service_type,
+ char *service_name,
+ char *provider_name);
+
+struct vidtv_psi_desc_registration
+*vidtv_psi_registration_desc_init(struct vidtv_psi_desc *head,
+ __be32 format_id,
+ u8 *additional_ident_info,
+ u32 additional_info_len);
+
+struct vidtv_psi_table_pat_program
+*vidtv_psi_pat_program_init(struct vidtv_psi_table_pat_program *head,
+ u16 service_id,
+ u16 program_map_pid);
+
+struct vidtv_psi_table_pmt_stream*
+vidtv_psi_pmt_stream_init(struct vidtv_psi_table_pmt_stream *head,
+ enum vidtv_psi_stream_types stream_type,
+ u16 es_pid);
+
+struct vidtv_psi_table_pat *vidtv_psi_pat_table_init(u16 transport_stream_id);
+
+struct vidtv_psi_table_pmt *vidtv_psi_pmt_table_init(u16 program_number,
+ u16 pcr_pid);
+
+struct vidtv_psi_table_sdt *vidtv_psi_sdt_table_init(u16 transport_stream_id);
+
+struct vidtv_psi_table_sdt_service*
+vidtv_psi_sdt_service_init(struct vidtv_psi_table_sdt_service *head,
+ u16 service_id);
+
+void
+vidtv_psi_desc_destroy(struct vidtv_psi_desc *desc);
+
+void
+vidtv_psi_pat_program_destroy(struct vidtv_psi_table_pat_program *p);
+
+void
+vidtv_psi_pat_table_destroy(struct vidtv_psi_table_pat *p);
+
+void
+vidtv_psi_pmt_stream_destroy(struct vidtv_psi_table_pmt_stream *s);
+
+void
+vidtv_psi_pmt_table_destroy(struct vidtv_psi_table_pmt *pmt);
+
+void
+vidtv_psi_sdt_table_destroy(struct vidtv_psi_table_sdt *sdt);
+
+void
+vidtv_psi_sdt_service_destroy(struct vidtv_psi_table_sdt_service *service);
+
+/**
+ * vidtv_psi_sdt_service_assign - Assigns the service loop to the SDT.
+ * @sdt: The SDT to assign to.
+ * @service: The service loop (one or more services)
+ *
+ * This will free the previous service loop in the table.
+ * This will assign ownership of the service loop to the table, i.e. the table
+ * will free this service loop when a call to its destroy function is made.
+ */
+void
+vidtv_psi_sdt_service_assign(struct vidtv_psi_table_sdt *sdt,
+ struct vidtv_psi_table_sdt_service *service);
+
+/**
+ * vidtv_psi_desc_assign - Assigns a descriptor loop at some point
+ * @to: Where to assign this descriptor loop to
+ * @desc: The descriptor loop that will be assigned.
+ *
+ * This will free the loop in 'to', if any.
+ */
+void vidtv_psi_desc_assign(struct vidtv_psi_desc **to,
+ struct vidtv_psi_desc *desc);
+
+/**
+ * vidtv_psi_pmt_desc_assign - Assigns a descriptor loop at some point in a PMT section.
+ * @pmt: The PMT section that will contain the descriptor loop
+ * @to: Where in the PMT to assign this descriptor loop to
+ * @desc: The descriptor loop that will be assigned.
+ *
+ * This will free the loop in 'to', if any.
+ * This will assign ownership of the loop to the table, i.e. the table
+ * will free this loop when a call to its destroy function is made.
+ */
+void vidtv_pmt_desc_assign(struct vidtv_psi_table_pmt *pmt,
+ struct vidtv_psi_desc **to,
+ struct vidtv_psi_desc *desc);
+
+/**
+ * vidtv_psi_sdt_desc_assign - Assigns a descriptor loop at some point in a SDT.
+ * @sdt: The SDT that will contain the descriptor loop
+ * @to: Where in the PMT to assign this descriptor loop to
+ * @desc: The descriptor loop that will be assigned.
+ *
+ * This will free the loop in 'to', if any.
+ * This will assign ownership of the loop to the table, i.e. the table
+ * will free this loop when a call to its destroy function is made.
+ */
+void vidtv_sdt_desc_assign(struct vidtv_psi_table_sdt *sdt,
+ struct vidtv_psi_desc **to,
+ struct vidtv_psi_desc *desc);
+
+/**
+ * vidtv_psi_pat_program_assign - Assigns the program loop to the PAT.
+ * @pat: The PAT to assign to.
+ * @p: The program loop (one or more programs)
+ *
+ * This will free the previous program loop in the table.
+ * This will assign ownership of the program loop to the table, i.e. the table
+ * will free this program loop when a call to its destroy function is made.
+ */
+void vidtv_psi_pat_program_assign(struct vidtv_psi_table_pat *pat,
+ struct vidtv_psi_table_pat_program *p);
+
+/**
+ * vidtv_psi_pmt_stream_assign - Assigns the stream loop to the PAT.
+ * @pmt: The PMT to assign to.
+ * @s: The stream loop (one or more streams)
+ *
+ * This will free the previous stream loop in the table.
+ * This will assign ownership of the stream loop to the table, i.e. the table
+ * will free this stream loop when a call to its destroy function is made.
+ */
+void vidtv_psi_pmt_stream_assign(struct vidtv_psi_table_pmt *pmt,
+ struct vidtv_psi_table_pmt_stream *s);
+
+struct vidtv_psi_desc *vidtv_psi_desc_clone(struct vidtv_psi_desc *desc);
+
+/**
+ * vidtv_psi_create_sec_for_each_pat_entry - Create a PMT section for each
+ * program found in the PAT
+ * @pat: The PAT to look for programs.
+ * @s: The stream loop (one or more streams)
+ * @pcr_pid: packet ID for the PCR to be used for the program described in this
+ * PMT section
+ */
+struct vidtv_psi_table_pmt**
+vidtv_psi_pmt_create_sec_for_each_pat_entry(struct vidtv_psi_table_pat *pat, u16 pcr_pid);
+
+/**
+ * vidtv_psi_pmt_get_pid - Get the TS PID for a PMT section.
+ * @section: The PMT section whose PID we want to retrieve.
+ * @pat: The PAT table to look into.
+ *
+ * Returns: the TS PID for 'section'
+ */
+u16 vidtv_psi_pmt_get_pid(struct vidtv_psi_table_pmt *section,
+ struct vidtv_psi_table_pat *pat);
+
+/**
+ * vidtv_psi_pat_table_update_sec_len - Recompute and update the PAT section length.
+ * @pat: The PAT whose length is to be updated.
+ *
+ * This will traverse the table and accumulate the length of its components,
+ * which is then used to replace the 'section_length' field.
+ *
+ * If section_length > MAX_SECTION_LEN, the operation fails.
+ */
+void vidtv_psi_pat_table_update_sec_len(struct vidtv_psi_table_pat *pat);
+
+/**
+ * vidtv_psi_pmt_table_update_sec_len - Recompute and update the PMT section length.
+ * @pmt: The PMT whose length is to be updated.
+ *
+ * This will traverse the table and accumulate the length of its components,
+ * which is then used to replace the 'section_length' field.
+ *
+ * If section_length > MAX_SECTION_LEN, the operation fails.
+ */
+void vidtv_psi_pmt_table_update_sec_len(struct vidtv_psi_table_pmt *pmt);
+
+/**
+ * vidtv_psi_sdt_table_update_sec_len - Recompute and update the SDT section length.
+ * @sdt: The SDT whose length is to be updated.
+ *
+ * This will traverse the table and accumulate the length of its components,
+ * which is then used to replace the 'section_length' field.
+ *
+ * If section_length > MAX_SECTION_LEN, the operation fails.
+ */
+void vidtv_psi_sdt_table_update_sec_len(struct vidtv_psi_table_sdt *sdt);
+
+/**
+ * struct vidtv_psi_pat_write_args - Arguments for writing a PAT table
+ * @buf: The destination buffer.
+ * @offset: The offset into the destination buffer.
+ * @pat: A pointer to the PAT.
+ * @buf_sz: The size of the destination buffer.
+ * @continuity_counter: A pointer to the CC. Incremented on every new packet.
+ *
+ */
+struct vidtv_psi_pat_write_args {
+ char *buf;
+ u32 offset;
+ struct vidtv_psi_table_pat *pat;
+ u32 buf_sz;
+ u8 *continuity_counter;
+};
+
+/**
+ * vidtv_psi_pat_write_into - Write PAT as MPEG-TS packets into a buffer.
+ * @args: An instance of struct vidtv_psi_pat_write_args
+ *
+ * This function writes the MPEG TS packets for a PAT table into a buffer.
+ * Calling code will usually generate the PAT via a call to its init function
+ * and thus is responsible for freeing it.
+ *
+ * Return: The number of bytes written into the buffer. This is NOT
+ * equal to the size of the PAT, since more space is needed for TS headers during TS
+ * encapsulation.
+ */
+u32 vidtv_psi_pat_write_into(struct vidtv_psi_pat_write_args args);
+
+/**
+ * struct vidtv_psi_sdt_write_args - Arguments for writing a SDT table
+ * @buf: The destination buffer.
+ * @offset: The offset into the destination buffer.
+ * @sdt: A pointer to the SDT.
+ * @buf_sz: The size of the destination buffer.
+ * @continuity_counter: A pointer to the CC. Incremented on every new packet.
+ *
+ */
+
+struct vidtv_psi_sdt_write_args {
+ char *buf;
+ u32 offset;
+ struct vidtv_psi_table_sdt *sdt;
+ u32 buf_sz;
+ u8 *continuity_counter;
+};
+
+/**
+ * vidtv_psi_sdt_write_into - Write SDT as MPEG-TS packets into a buffer.
+ * @args: an instance of struct vidtv_psi_sdt_write_args
+ *
+ * This function writes the MPEG TS packets for a SDT table into a buffer.
+ * Calling code will usually generate the SDT via a call to its init function
+ * and thus is responsible for freeing it.
+ *
+ * Return: The number of bytes written into the buffer. This is NOT
+ * equal to the size of the SDT, since more space is needed for TS headers during TS
+ * encapsulation.
+ */
+u32 vidtv_psi_sdt_write_into(struct vidtv_psi_sdt_write_args args);
+
+/**
+ * struct vidtv_psi_pmt_write_args - Arguments for writing a PMT section
+ * @buf: The destination buffer.
+ * @offset: The offset into the destination buffer.
+ * @pmt: A pointer to the PMT.
+ * @buf_sz: The size of the destination buffer.
+ * @continuity_counter: A pointer to the CC. Incremented on every new packet.
+ *
+ */
+struct vidtv_psi_pmt_write_args {
+ char *buf;
+ u32 offset;
+ struct vidtv_psi_table_pmt *pmt;
+ u16 pid;
+ u32 buf_sz;
+ u8 *continuity_counter;
+ u16 pcr_pid;
+};
+
+/**
+ * vidtv_psi_pmt_write_into - Write PMT as MPEG-TS packets into a buffer.
+ * @args: an instance of struct vidtv_psi_pmt_write_args
+ *
+ * This function writes the MPEG TS packets for a PMT section into a buffer.
+ * Calling code will usually generate the PMT section via a call to its init function
+ * and thus is responsible for freeing it.
+ *
+ * Return: The number of bytes written into the buffer. This is NOT
+ * equal to the size of the PMT section, since more space is needed for TS headers
+ * during TS encapsulation.
+ */
+u32 vidtv_psi_pmt_write_into(struct vidtv_psi_pmt_write_args args);
+
+/**
+ * vidtv_psi_find_pmt_sec - Finds the PMT section for 'program_num'
+ * @pmt_sections: The sections to look into.
+ * @nsections: The number of sections.
+ * @program_num: The 'program_num' from PAT pointing to a PMT section.
+ *
+ * Return: A pointer to the PMT, if found, or NULL.
+ */
+struct vidtv_psi_table_pmt *vidtv_psi_find_pmt_sec(struct vidtv_psi_table_pmt **pmt_sections,
+ u16 nsections,
+ u16 program_num);
+
+u16 vidtv_psi_get_pat_program_pid(struct vidtv_psi_table_pat_program *p);
+u16 vidtv_psi_pmt_stream_get_elem_pid(struct vidtv_psi_table_pmt_stream *s);
+
+#endif // VIDTV_PSI_H
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Vidtv serves as a reference DVB driver and helps validate the existing APIs
+ * in the media subsystem. It can also aid developers working on userspace
+ * applications.
+ *
+ * This file contains the code for an AES3 (also known as AES/EBU) encoder.
+ * It is based on EBU Tech 3250 and SMPTE 302M technical documents.
+ *
+ * This encoder currently supports 16bit AES3 subframes using 16bit signed
+ * integers.
+ *
+ * Note: AU stands for Access Unit, and AAU stands for Audio Access Unit
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s, %d: " fmt, __func__, __LINE__
+
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/crc32.h>
+#include <linux/vmalloc.h>
+#include <linux/string.h>
+#include <linux/kernel.h>
+#include <linux/jiffies.h>
+#include <linux/printk.h>
+#include <linux/ratelimit.h>
+#include <linux/fixp-arith.h>
+
+#include <linux/math64.h>
+#include <asm/byteorder.h>
+
+#include "vidtv_s302m.h"
+#include "vidtv_encoder.h"
+#include "vidtv_common.h"
+
+#define S302M_SAMPLING_RATE_HZ 48000
+#define PES_PRIVATE_STREAM_1 0xbd /* PES: private_stream_1 */
+#define S302M_BLOCK_SZ 192
+#define S302M_SIN_LUT_NUM_ELEM 1024
+
+/* these are retrieved empirically from ffmpeg/libavcodec */
+#define FF_S302M_DEFAULT_NUM_FRAMES 1115
+#define FF_S302M_DEFAULT_PTS_INCREMENT 2090
+#define FF_S302M_DEFAULT_PTS_OFFSET 100000
+
+/* Used by the tone generator: number of samples for PI */
+#define PI 180
+
+static const u8 reverse[256] = {
+ /* from ffmpeg */
+ 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, 0x10, 0x90, 0x50, 0xD0,
+ 0x30, 0xB0, 0x70, 0xF0, 0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8,
+ 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8, 0x04, 0x84, 0x44, 0xC4,
+ 0x24, 0xA4, 0x64, 0xE4, 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4,
+ 0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, 0x1C, 0x9C, 0x5C, 0xDC,
+ 0x3C, 0xBC, 0x7C, 0xFC, 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2,
+ 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2, 0x0A, 0x8A, 0x4A, 0xCA,
+ 0x2A, 0xAA, 0x6A, 0xEA, 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA,
+ 0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, 0x16, 0x96, 0x56, 0xD6,
+ 0x36, 0xB6, 0x76, 0xF6, 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE,
+ 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE, 0x01, 0x81, 0x41, 0xC1,
+ 0x21, 0xA1, 0x61, 0xE1, 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1,
+ 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, 0x19, 0x99, 0x59, 0xD9,
+ 0x39, 0xB9, 0x79, 0xF9, 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5,
+ 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5, 0x0D, 0x8D, 0x4D, 0xCD,
+ 0x2D, 0xAD, 0x6D, 0xED, 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD,
+ 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, 0x13, 0x93, 0x53, 0xD3,
+ 0x33, 0xB3, 0x73, 0xF3, 0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB,
+ 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB, 0x07, 0x87, 0x47, 0xC7,
+ 0x27, 0xA7, 0x67, 0xE7, 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7,
+ 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, 0x1F, 0x9F, 0x5F, 0xDF,
+ 0x3F, 0xBF, 0x7F, 0xFF,
+};
+
+struct tone_duration {
+ enum musical_notes note;
+ int duration;
+};
+
+#define COMPASS 120 /* beats per minute (Allegro) */
+static const struct tone_duration beethoven_5th_symphony[] = {
+ { NOTE_E_6, 128}, { NOTE_DS_6, 128}, { NOTE_E_6, 128},
+ { NOTE_DS_6, 128}, { NOTE_E_6, 128}, { NOTE_B_5, 128},
+ { NOTE_D_6, 128}, { NOTE_C_6, 128}, { NOTE_A_3, 128},
+ { NOTE_E_4, 128}, { NOTE_A_4, 128}, { NOTE_C_5, 128},
+ { NOTE_E_5, 128}, { NOTE_A_5, 128}, { NOTE_E_3, 128},
+ { NOTE_E_4, 128}, { NOTE_GS_4, 128}, { NOTE_E_5, 128},
+ { NOTE_GS_5, 128}, { NOTE_B_5, 128}, { NOTE_A_3, 128},
+ { NOTE_E_4, 128}, { NOTE_A_4, 128}, { NOTE_E_5, 128},
+ { NOTE_E_6, 128}, { NOTE_DS_6, 128}, { NOTE_E_6, 128},
+ { NOTE_DS_6, 128}, { NOTE_E_6, 128}, { NOTE_B_5, 128},
+ { NOTE_D_6, 128}, { NOTE_C_6, 128}, { NOTE_A_3, 128},
+ { NOTE_E_4, 128}, { NOTE_A_4, 128}, { NOTE_C_5, 128},
+ { NOTE_E_5, 128}, { NOTE_A_5, 128}, { NOTE_E_3, 128},
+ { NOTE_E_4, 128}, { NOTE_GS_4, 128}, { NOTE_E_5, 128},
+ { NOTE_C_6, 128}, { NOTE_B_5, 128}, { NOTE_A_3, 128},
+ { NOTE_E_4, 128}, { NOTE_A_4, 128}, { NOTE_SILENT, 128},
+
+ { NOTE_E_6, 128}, { NOTE_DS_6, 128}, { NOTE_E_6, 128},
+ { NOTE_DS_6, 128}, { NOTE_E_6, 128}, { NOTE_B_5, 128},
+ { NOTE_D_6, 128}, { NOTE_C_6, 128}, { NOTE_A_3, 128},
+ { NOTE_E_4, 128}, { NOTE_A_4, 128}, { NOTE_C_5, 128},
+ { NOTE_E_5, 128}, { NOTE_A_5, 128}, { NOTE_E_3, 128},
+ { NOTE_E_4, 128}, { NOTE_GS_4, 128}, { NOTE_E_5, 128},
+ { NOTE_GS_5, 128}, { NOTE_B_5, 128}, { NOTE_A_3, 128},
+ { NOTE_E_4, 128}, { NOTE_A_4, 128}, { NOTE_E_5, 128},
+ { NOTE_E_6, 128}, { NOTE_DS_6, 128}, { NOTE_E_6, 128},
+ { NOTE_DS_6, 128}, { NOTE_E_6, 128}, { NOTE_B_5, 128},
+ { NOTE_D_6, 128}, { NOTE_C_6, 128}, { NOTE_A_3, 128},
+ { NOTE_E_4, 128}, { NOTE_A_4, 128}, { NOTE_C_5, 128},
+ { NOTE_E_5, 128}, { NOTE_A_5, 128}, { NOTE_E_3, 128},
+ { NOTE_E_4, 128}, { NOTE_GS_4, 128}, { NOTE_E_5, 128},
+ { NOTE_C_6, 128}, { NOTE_B_5, 128}, { NOTE_A_3, 128},
+ { NOTE_E_4, 128}, { NOTE_A_4, 128}, { NOTE_B_4, 128},
+ { NOTE_C_5, 128}, { NOTE_D_5, 128}, { NOTE_C_4, 128},
+ { NOTE_G_4, 128}, { NOTE_C_5, 128}, { NOTE_G_4, 128},
+ { NOTE_F_5, 128}, { NOTE_E_5, 128}, { NOTE_G_3, 128},
+ { NOTE_G_4, 128}, { NOTE_B_3, 128}, { NOTE_F_4, 128},
+ { NOTE_E_5, 128}, { NOTE_D_5, 128}, { NOTE_A_3, 128},
+ { NOTE_E_4, 128}, { NOTE_A_4, 128}, { NOTE_E_4, 128},
+ { NOTE_D_5, 128}, { NOTE_C_5, 128}, { NOTE_E_3, 128},
+ { NOTE_E_4, 128}, { NOTE_E_5, 255}, { NOTE_E_6, 128},
+ { NOTE_E_5, 128}, { NOTE_E_6, 128}, { NOTE_E_5, 255},
+ { NOTE_DS_5, 128}, { NOTE_E_5, 128}, { NOTE_DS_6, 128},
+ { NOTE_E_6, 128}, { NOTE_DS_5, 128}, { NOTE_E_5, 128},
+ { NOTE_DS_6, 128}, { NOTE_E_6, 128}, { NOTE_DS_6, 128},
+ { NOTE_E_6, 128}, { NOTE_DS_6, 128}, { NOTE_E_6, 128},
+ { NOTE_B_5, 128}, { NOTE_D_6, 128}, { NOTE_C_6, 128},
+ { NOTE_A_3, 128}, { NOTE_E_4, 128}, { NOTE_A_4, 128},
+ { NOTE_C_5, 128}, { NOTE_E_5, 128}, { NOTE_A_5, 128},
+ { NOTE_E_3, 128}, { NOTE_E_4, 128}, { NOTE_GS_4, 128},
+ { NOTE_E_5, 128}, { NOTE_GS_5, 128}, { NOTE_B_5, 128},
+ { NOTE_A_3, 128}, { NOTE_E_4, 128}, { NOTE_A_4, 128},
+ { NOTE_E_5, 128}, { NOTE_E_6, 128}, { NOTE_DS_6, 128},
+ { NOTE_E_6, 128}, { NOTE_DS_6, 128}, { NOTE_E_6, 128},
+ { NOTE_B_5, 128}, { NOTE_D_6, 128}, { NOTE_C_6, 128},
+ { NOTE_A_3, 128}, { NOTE_E_4, 128}, { NOTE_A_4, 128},
+ { NOTE_C_5, 128}, { NOTE_E_5, 128}, { NOTE_A_5, 128},
+ { NOTE_E_3, 128}, { NOTE_E_4, 128}, { NOTE_GS_4, 128},
+ { NOTE_E_5, 128}, { NOTE_C_6, 128}, { NOTE_B_5, 128},
+ { NOTE_C_5, 255}, { NOTE_C_5, 255}, { NOTE_SILENT, 512},
+};
+
+static struct vidtv_access_unit *vidtv_s302m_access_unit_init(struct vidtv_access_unit *head)
+{
+ struct vidtv_access_unit *au = kzalloc(sizeof(*au), GFP_KERNEL);
+
+ if (head) {
+ while (head->next)
+ head = head->next;
+
+ head->next = au;
+ }
+
+ return au;
+}
+
+static void vidtv_s302m_access_unit_destroy(struct vidtv_encoder *e)
+{
+ struct vidtv_access_unit *head = e->access_units;
+ struct vidtv_access_unit *tmp = NULL;
+
+ while (head) {
+ tmp = head;
+ head = head->next;
+ kfree(tmp);
+ }
+
+ e->access_units = NULL;
+}
+
+static void vidtv_s302m_alloc_au(struct vidtv_encoder *e)
+{
+ struct vidtv_access_unit *sync_au = NULL;
+ struct vidtv_access_unit *temp = NULL;
+
+ if (e->sync && e->sync->is_video_encoder) {
+ sync_au = e->sync->access_units;
+
+ while (sync_au) {
+ temp = vidtv_s302m_access_unit_init(e->access_units);
+ if (!e->access_units)
+ e->access_units = temp;
+
+ sync_au = sync_au->next;
+ }
+
+ return;
+ }
+
+ e->access_units = vidtv_s302m_access_unit_init(NULL);
+}
+
+static void
+vidtv_s302m_compute_sample_count_from_video(struct vidtv_encoder *e)
+{
+ struct vidtv_access_unit *au = e->access_units;
+ struct vidtv_access_unit *sync_au = e->sync->access_units;
+ u32 vau_duration_usecs;
+ u32 sample_duration_usecs;
+ u32 s;
+
+ vau_duration_usecs = USEC_PER_SEC / e->sync->sampling_rate_hz;
+ sample_duration_usecs = USEC_PER_SEC / e->sampling_rate_hz;
+
+ while (au && sync_au) {
+ s = DIV_ROUND_UP(vau_duration_usecs, sample_duration_usecs);
+ au->num_samples = s;
+ au = au->next;
+ sync_au = sync_au->next;
+ }
+}
+
+static void vidtv_s302m_compute_pts_from_video(struct vidtv_encoder *e)
+{
+ struct vidtv_access_unit *au = e->access_units;
+ struct vidtv_access_unit *sync_au = e->sync->access_units;
+
+ /* use the same pts from the video access unit*/
+ while (au && sync_au) {
+ au->pts = sync_au->pts;
+ au = au->next;
+ sync_au = sync_au->next;
+ }
+}
+
+static u16 vidtv_s302m_get_sample(struct vidtv_encoder *e)
+{
+ u16 sample;
+ int pos;
+
+ if (!e->src_buf) {
+ /*
+ * Simple tone generator: play the tones at the
+ * beethoven_5th_symphony array.
+ */
+ if (e->last_duration <= 0) {
+ if (e->src_buf_offset >= ARRAY_SIZE(beethoven_5th_symphony))
+ e->src_buf_offset = 0;
+
+ e->last_tone = beethoven_5th_symphony[e->src_buf_offset].note;
+ e->last_duration = beethoven_5th_symphony[e->src_buf_offset].duration * S302M_SAMPLING_RATE_HZ / COMPASS / 5;
+ e->src_buf_offset++;
+ e->note_offset = 0;
+ } else {
+ e->last_duration--;
+ }
+
+ /* Handle silent */
+ if (!e->last_tone) {
+ e->src_buf_offset = 0;
+ return 0x8000;
+ }
+
+ pos = (2 * PI * e->note_offset * e->last_tone / S302M_SAMPLING_RATE_HZ);
+
+ if (pos == 360)
+ e->note_offset = 0;
+ else
+ e->note_offset++;
+
+ return (fixp_sin32(pos % (2 * PI)) >> 16) + 0x8000;
+ }
+
+ /* bug somewhere */
+ if (e->src_buf_offset > e->src_buf_sz) {
+ pr_err_ratelimited("overflow detected: %d > %d, wrapping.\n",
+ e->src_buf_offset,
+ e->src_buf_sz);
+
+ e->src_buf_offset = 0;
+ }
+
+ if (e->src_buf_offset >= e->src_buf_sz) {
+ /* let the source know we are out of data */
+ if (e->last_sample_cb)
+ e->last_sample_cb(e->sample_count);
+
+ e->src_buf_offset = 0;
+ }
+
+ sample = *(u16 *)(e->src_buf + e->src_buf_offset);
+
+ return sample;
+}
+
+static u32 vidtv_s302m_write_frame(struct vidtv_encoder *e,
+ u16 sample)
+{
+ u32 nbytes = 0;
+ struct vidtv_s302m_frame_16 f = {};
+ struct vidtv_s302m_ctx *ctx = e->ctx;
+
+ /* from ffmpeg: see s302enc.c */
+
+ u8 vucf = ctx->frame_index == 0 ? 0x10 : 0;
+
+ f.data[0] = sample & 0xFF;
+ f.data[1] = (sample & 0xFF00) >> 8;
+ f.data[2] = ((sample & 0x0F) << 4) | vucf;
+ f.data[3] = (sample & 0x0FF0) >> 4;
+ f.data[4] = (sample & 0xF000) >> 12;
+
+ f.data[0] = reverse[f.data[0]];
+ f.data[1] = reverse[f.data[1]];
+ f.data[2] = reverse[f.data[2]];
+ f.data[3] = reverse[f.data[3]];
+ f.data[4] = reverse[f.data[4]];
+
+ nbytes += vidtv_memcpy(e->encoder_buf,
+ e->encoder_buf_offset,
+ VIDTV_S302M_BUF_SZ,
+ &f,
+ sizeof(f));
+
+ e->encoder_buf_offset += nbytes;
+
+ ctx->frame_index++;
+ if (ctx->frame_index >= S302M_BLOCK_SZ)
+ ctx->frame_index = 0;
+
+ return nbytes;
+}
+
+static u32 vidtv_s302m_write_h(struct vidtv_encoder *e, u32 p_sz)
+{
+ struct vidtv_smpte_s302m_es h = {};
+ u32 nbytes = 0;
+
+ /* 2 channels, ident: 0, 16 bits per sample */
+ h.bitfield = cpu_to_be32((p_sz << 16));
+
+ nbytes += vidtv_memcpy(e->encoder_buf,
+ e->encoder_buf_offset,
+ e->encoder_buf_sz,
+ &h,
+ sizeof(h));
+
+ e->encoder_buf_offset += nbytes;
+ return nbytes;
+}
+
+static void vidtv_s302m_write_frames(struct vidtv_encoder *e)
+{
+ struct vidtv_access_unit *au = e->access_units;
+ struct vidtv_s302m_ctx *ctx = e->ctx;
+ u32 nbytes_per_unit = 0;
+ u32 nbytes = 0;
+ u32 au_sz = 0;
+ u16 sample;
+ u32 j;
+
+ while (au) {
+ au_sz = au->num_samples *
+ sizeof(struct vidtv_s302m_frame_16);
+
+ nbytes_per_unit = vidtv_s302m_write_h(e, au_sz);
+
+ for (j = 0; j < au->num_samples; ++j) {
+ sample = vidtv_s302m_get_sample(e);
+ nbytes_per_unit += vidtv_s302m_write_frame(e, sample);
+
+ if (e->src_buf)
+ e->src_buf_offset += sizeof(u16);
+
+ e->sample_count++;
+ }
+
+ au->nbytes = nbytes_per_unit;
+
+ if (au_sz + sizeof(struct vidtv_smpte_s302m_es) != nbytes_per_unit) {
+ pr_warn_ratelimited("write size was %u, expected %zu\n",
+ nbytes_per_unit,
+ au_sz + sizeof(struct vidtv_smpte_s302m_es));
+ }
+
+ nbytes += nbytes_per_unit;
+ au->offset = nbytes - nbytes_per_unit;
+
+ nbytes_per_unit = 0;
+ ctx->au_count++;
+
+ au = au->next;
+ }
+}
+
+static void *vidtv_s302m_encode(struct vidtv_encoder *e)
+{
+ /*
+ * According to SMPTE 302M, an audio access unit is specified as those
+ * AES3 words that are associated with a corresponding video frame.
+ * Therefore, there is one audio access unit for every video access unit
+ * in the corresponding video encoder ('sync'), using the same values
+ * for PTS as used by the video encoder.
+ *
+ * Assuming that it is also possible to send audio without any
+ * associated video, as in a radio-like service, a single audio access unit
+ * is created with values for 'num_samples' and 'pts' taken empirically from
+ * ffmpeg
+ */
+
+ struct vidtv_s302m_ctx *ctx = e->ctx;
+
+ vidtv_s302m_access_unit_destroy(e);
+ vidtv_s302m_alloc_au(e);
+
+ if (e->sync && e->sync->is_video_encoder) {
+ vidtv_s302m_compute_sample_count_from_video(e);
+ vidtv_s302m_compute_pts_from_video(e);
+ } else {
+ e->access_units->num_samples = FF_S302M_DEFAULT_NUM_FRAMES;
+ e->access_units->pts = (ctx->au_count * FF_S302M_DEFAULT_PTS_INCREMENT) +
+ FF_S302M_DEFAULT_PTS_OFFSET;
+ }
+
+ vidtv_s302m_write_frames(e);
+
+ return e->encoder_buf;
+}
+
+static u32 vidtv_s302m_clear(struct vidtv_encoder *e)
+{
+ struct vidtv_access_unit *au = e->access_units;
+ u32 count = 0;
+
+ while (au) {
+ count++;
+ au = au->next;
+ }
+
+ vidtv_s302m_access_unit_destroy(e);
+ memset(e->encoder_buf, 0, VIDTV_S302M_BUF_SZ);
+ e->encoder_buf_offset = 0;
+
+ return count;
+}
+
+struct vidtv_encoder
+*vidtv_s302m_encoder_init(struct vidtv_s302m_encoder_init_args args)
+{
+ struct vidtv_encoder *e = kzalloc(sizeof(*e), GFP_KERNEL);
+ u32 priv_sz = sizeof(struct vidtv_s302m_ctx);
+
+ e->id = S302M;
+
+ if (args.name)
+ e->name = kstrdup(args.name, GFP_KERNEL);
+
+ e->encoder_buf = vzalloc(VIDTV_S302M_BUF_SZ);
+ e->encoder_buf_sz = VIDTV_S302M_BUF_SZ;
+ e->encoder_buf_offset = 0;
+
+ e->sample_count = 0;
+ e->last_duration = 0;
+
+ e->src_buf = (args.src_buf) ? args.src_buf : NULL;
+ e->src_buf_sz = (args.src_buf) ? args.src_buf_sz : 0;
+ e->src_buf_offset = 0;
+
+ e->is_video_encoder = false;
+ e->ctx = kzalloc(priv_sz, GFP_KERNEL);
+
+ e->encode = vidtv_s302m_encode;
+ e->clear = vidtv_s302m_clear;
+
+ e->es_pid = cpu_to_be16(args.es_pid);
+ e->stream_id = cpu_to_be16(PES_PRIVATE_STREAM_1);
+
+ e->sync = args.sync;
+ e->sampling_rate_hz = S302M_SAMPLING_RATE_HZ;
+
+ e->last_sample_cb = args.last_sample_cb;
+
+ e->destroy = vidtv_s302m_encoder_destroy;
+
+ if (args.head) {
+ while (args.head->next)
+ args.head = args.head->next;
+
+ args.head->next = e;
+ }
+
+ e->next = NULL;
+
+ return e;
+}
+
+void vidtv_s302m_encoder_destroy(struct vidtv_encoder *e)
+{
+ if (e->id != S302M) {
+ pr_err_ratelimited("Encoder type mismatch, skipping.\n");
+ return;
+ }
+
+ vidtv_s302m_access_unit_destroy(e);
+ kfree(e->name);
+ vfree(e->encoder_buf);
+ kfree(e->ctx);
+ kfree(e);
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Vidtv serves as a reference DVB driver and helps validate the existing APIs
+ * in the media subsystem. It can also aid developers working on userspace
+ * applications.
+ *
+ * This file contains the code for an AES3 (also known as AES/EBU) encoder.
+ * It is based on EBU Tech 3250 and SMPTE 302M technical documents.
+ *
+ * This encoder currently supports 16bit AES3 subframes using 16bit signed
+ * integers.
+ *
+ * Note: AU stands for Access Unit, and AAU stands for Audio Access Unit
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#ifndef VIDTV_S302M_H
+#define VIDTV_S302M_H
+
+#include <linux/types.h>
+#include <asm/byteorder.h>
+
+#include "vidtv_encoder.h"
+
+/* see SMPTE 302M 2007 clause 7.3 */
+#define VIDTV_S302M_BUF_SZ 65024
+
+/* see ETSI TS 102 154 v.1.2.1 clause 7.3.5 */
+#define VIDTV_S302M_FORMAT_IDENTIFIER 0x42535344
+
+/**
+ * struct vidtv_s302m_ctx - s302m encoder context.
+ * @enc: A pointer to the containing encoder structure.
+ * @frame_index: The current frame in a block
+ * @au_count: The total number of access units encoded up to now
+ */
+struct vidtv_s302m_ctx {
+ struct vidtv_encoder *enc;
+ u32 frame_index;
+ u32 au_count;
+};
+
+/**
+ * struct vidtv_smpte_s302m_es - s302m MPEG Elementary Stream header.
+ *
+ * See SMPTE 302M 2007 table 1.
+ */
+struct vidtv_smpte_s302m_es {
+ /*
+ *
+ * audio_packet_size:16;
+ * num_channels:2;
+ * channel_identification:8;
+ * bits_per_sample:2; // 0x0 for 16bits
+ * zero:4;
+ */
+ __be32 bitfield;
+} __packed;
+
+struct vidtv_s302m_frame_16 {
+ u8 data[5];
+} __packed;
+
+/**
+ * struct vidtv_s302m_encoder_init_args - Args for the s302m encoder.
+ *
+ * @name: A name to identify this particular instance
+ * @src_buf: The source buffer, encoder will default to a sine wave if this is NULL.
+ * @src_buf_sz: The size of the source buffer.
+ * @es_pid: The MPEG Elementary Stream PID to use.
+ * @sync: Attempt to synchronize audio with this video encoder, if not NULL.
+ * @last_sample_cb: A callback called when the encoder runs out of data.
+ * @head: Add to this chain
+ */
+struct vidtv_s302m_encoder_init_args {
+ char *name;
+ void *src_buf;
+ u32 src_buf_sz;
+ u16 es_pid;
+ struct vidtv_encoder *sync;
+ void (*last_sample_cb)(u32 sample_no);
+
+ struct vidtv_encoder *head;
+};
+
+struct vidtv_encoder
+*vidtv_s302m_encoder_init(struct vidtv_s302m_encoder_init_args args);
+
+void vidtv_s302m_encoder_destroy(struct vidtv_encoder *encoder);
+
+#endif /* VIDTV_S302M_H */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * The Virtual DVB test driver serves as a reference DVB driver and helps
+ * validate the existing APIs in the media subsystem. It can also aid
+ * developers working on userspace applications.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s, %d: " fmt, __func__, __LINE__
+
+#include <linux/printk.h>
+#include <linux/ratelimit.h>
+#include <linux/types.h>
+#include <linux/math64.h>
+#include <asm/byteorder.h>
+
+#include "vidtv_ts.h"
+#include "vidtv_common.h"
+
+static u32 vidtv_ts_write_pcr_bits(u8 *to, u32 to_offset, u64 pcr)
+{
+ /* Exact same from ffmpeg. PCR is a counter driven by a 27Mhz clock */
+ u64 div;
+ u64 rem;
+ u8 *buf = to + to_offset;
+ u64 pcr_low;
+ u64 pcr_high;
+
+ div = div64_u64_rem(pcr, 300, &rem);
+
+ pcr_low = rem; /* pcr_low = pcr % 300 */
+ pcr_high = div; /* pcr_high = pcr / 300 */
+
+ *buf++ = pcr_high >> 25;
+ *buf++ = pcr_high >> 17;
+ *buf++ = pcr_high >> 9;
+ *buf++ = pcr_high >> 1;
+ *buf++ = pcr_high << 7 | pcr_low >> 8 | 0x7e;
+ *buf++ = pcr_low;
+
+ return 6;
+}
+
+void vidtv_ts_inc_cc(u8 *continuity_counter)
+{
+ ++*continuity_counter;
+ if (*continuity_counter > TS_CC_MAX_VAL)
+ *continuity_counter = 0;
+}
+
+u32 vidtv_ts_null_write_into(struct null_packet_write_args args)
+{
+ u32 nbytes = 0;
+ struct vidtv_mpeg_ts ts_header = {};
+
+ ts_header.sync_byte = TS_SYNC_BYTE;
+ ts_header.bitfield = cpu_to_be16(TS_NULL_PACKET_PID);
+ ts_header.payload = 1;
+ ts_header.continuity_counter = *args.continuity_counter;
+
+ /* copy TS header */
+ nbytes += vidtv_memcpy(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.buf_sz,
+ &ts_header,
+ sizeof(ts_header));
+
+ vidtv_ts_inc_cc(args.continuity_counter);
+
+ /* fill the rest with empty data */
+ nbytes += vidtv_memset(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.buf_sz,
+ TS_FILL_BYTE,
+ TS_PACKET_LEN - nbytes);
+
+ /* we should have written exactly _one_ 188byte packet */
+ if (nbytes != TS_PACKET_LEN)
+ pr_warn_ratelimited("Expected exactly %d bytes, got %d\n",
+ TS_PACKET_LEN,
+ nbytes);
+
+ return nbytes;
+}
+
+u32 vidtv_ts_pcr_write_into(struct pcr_write_args args)
+{
+ u32 nbytes = 0;
+ struct vidtv_mpeg_ts ts_header = {};
+ struct vidtv_mpeg_ts_adaption ts_adap = {};
+
+ ts_header.sync_byte = TS_SYNC_BYTE;
+ ts_header.bitfield = cpu_to_be16(args.pid);
+ ts_header.scrambling = 0;
+ /* cc is not incremented, but it is needed. see 13818-1 clause 2.4.3.3 */
+ ts_header.continuity_counter = *args.continuity_counter;
+ ts_header.payload = 0;
+ ts_header.adaptation_field = 1;
+
+ /* 13818-1 clause 2.4.3.5 */
+ ts_adap.length = 183;
+ ts_adap.PCR = 1;
+
+ /* copy TS header */
+ nbytes += vidtv_memcpy(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.buf_sz,
+ &ts_header,
+ sizeof(ts_header));
+
+ /* write the adap after the TS header */
+ nbytes += vidtv_memcpy(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.buf_sz,
+ &ts_adap,
+ sizeof(ts_adap));
+
+ /* write the PCR optional */
+ nbytes += vidtv_ts_write_pcr_bits(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.pcr);
+
+ nbytes += vidtv_memset(args.dest_buf,
+ args.dest_offset + nbytes,
+ args.buf_sz,
+ TS_FILL_BYTE,
+ TS_PACKET_LEN - nbytes);
+
+ /* we should have written exactly _one_ 188byte packet */
+ if (nbytes != TS_PACKET_LEN)
+ pr_warn_ratelimited("Expected exactly %d bytes, got %d\n",
+ TS_PACKET_LEN,
+ nbytes);
+
+ return nbytes;
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * The Virtual DVB test driver serves as a reference DVB driver and helps
+ * validate the existing APIs in the media subsystem. It can also aid
+ * developers working on userspace applications.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#ifndef VIDTV_TS_H
+#define VIDTV_TS_H
+
+#include <linux/types.h>
+#include <asm/byteorder.h>
+
+#define TS_SYNC_BYTE 0x47
+#define TS_PACKET_LEN 188
+#define TS_PAYLOAD_LEN 184
+#define TS_NULL_PACKET_PID 0x1fff
+#define TS_CC_MAX_VAL 0x0f /* 4 bits */
+#define TS_LAST_VALID_PID 8191
+#define TS_FILL_BYTE 0xff /* the byte used in packet stuffing */
+
+struct vidtv_mpeg_ts_adaption {
+ u8 length;
+ struct {
+ u8 extension:1;
+ u8 private_data:1;
+ u8 splicing_point:1;
+ u8 OPCR:1;
+ u8 PCR:1;
+ u8 priority:1;
+ u8 random_access:1;
+ u8 discontinued:1;
+ } __packed;
+ u8 data[];
+} __packed;
+
+struct vidtv_mpeg_ts {
+ u8 sync_byte;
+ __be16 bitfield; /* tei: 1, payload_start:1 priority: 1, pid:13 */
+ struct {
+ u8 continuity_counter:4;
+ u8 payload:1;
+ u8 adaptation_field:1;
+ u8 scrambling:2;
+ } __packed;
+ struct vidtv_mpeg_ts_adaption adaption[];
+} __packed;
+
+/**
+ * struct pcr_write_args - Arguments for the pcr_write_into function.
+ * @dest_buf: The buffer to write into.
+ * @dest_offset: The byte offset into the buffer.
+ * @pid: The TS PID for the PCR packets.
+ * @buf_sz: The size of the buffer in bytes.
+ * @countinuity_counter: The TS continuity_counter.
+ * @pcr: A sample from the system clock.
+ */
+struct pcr_write_args {
+ void *dest_buf;
+ u32 dest_offset;
+ u16 pid;
+ u32 buf_sz;
+ u8 *continuity_counter;
+ u64 pcr;
+};
+
+/**
+ * struct null_packet_write_args - Arguments for the null_write_into function
+ * @dest_buf: The buffer to write into.
+ * @dest_offset: The byte offset into the buffer.
+ * @buf_sz: The size of the buffer in bytes.
+ * @countinuity_counter: The TS continuity_counter.
+ */
+struct null_packet_write_args {
+ void *dest_buf;
+ u32 dest_offset;
+ u32 buf_sz;
+ u8 *continuity_counter;
+};
+
+/* Increment the continuity counter */
+void vidtv_ts_inc_cc(u8 *continuity_counter);
+
+/**
+ * vidtv_ts_null_write_into - Write a TS null packet into a buffer.
+ * @args: the arguments to use when writing.
+ *
+ * This function will write a null packet into a buffer. This is usually used to
+ * pad TS streams.
+ *
+ * Return: The number of bytes written into the buffer.
+ */
+u32 vidtv_ts_null_write_into(struct null_packet_write_args args);
+
+/**
+ * vidtv_ts_pcr_write_into - Write a PCR packet into a buffer.
+ * @args: the arguments to use when writing.
+ *
+ * This function will write a PCR packet into a buffer. This is used to
+ * synchronize the clocks between encoders and decoders.
+ *
+ * Return: The number of bytes written into the buffer.
+ */
+u32 vidtv_ts_pcr_write_into(struct pcr_write_args args);
+
+#endif //VIDTV_TS_H
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * The Virtual DVB test driver serves as a reference DVB driver and helps
+ * validate the existing APIs in the media subsystem. It can also aid
+ * developers working on userspace applications.
+ *
+ * The vidtv tuner should support common TV standards such as
+ * DVB-T/T2/S/S2, ISDB-T and ATSC when completed.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <media/dvb_frontend.h>
+#include <linux/printk.h>
+#include <linux/ratelimit.h>
+
+#include "vidtv_tuner.h"
+
+struct vidtv_tuner_cnr_to_qual_s {
+ /* attempt to use the same values as libdvbv5 */
+ u32 modulation;
+ u32 fec;
+ u32 cnr_ok;
+ u32 cnr_good;
+};
+
+static const struct vidtv_tuner_cnr_to_qual_s vidtv_tuner_c_cnr_2_qual[] = {
+ /* from libdvbv5 source code, in milli db */
+ { QAM_256, FEC_NONE, 34000, 38000},
+ { QAM_64, FEC_NONE, 30000, 34000},
+};
+
+static const struct vidtv_tuner_cnr_to_qual_s vidtv_tuner_s_cnr_2_qual[] = {
+ /* from libdvbv5 source code, in milli db */
+ { QPSK, FEC_1_2, 7000, 10000},
+ { QPSK, FEC_2_3, 9000, 12000},
+ { QPSK, FEC_3_4, 10000, 13000},
+ { QPSK, FEC_5_6, 11000, 14000},
+ { QPSK, FEC_7_8, 12000, 15000},
+};
+
+static const struct vidtv_tuner_cnr_to_qual_s vidtv_tuner_s2_cnr_2_qual[] = {
+ /* from libdvbv5 source code, in milli db */
+ { QPSK, FEC_1_2, 9000, 12000},
+ { QPSK, FEC_2_3, 11000, 14000},
+ { QPSK, FEC_3_4, 12000, 15000},
+ { QPSK, FEC_5_6, 12000, 15000},
+ { QPSK, FEC_8_9, 13000, 16000},
+ { QPSK, FEC_9_10, 13500, 16500},
+ { PSK_8, FEC_2_3, 14500, 17500},
+ { PSK_8, FEC_3_4, 16000, 19000},
+ { PSK_8, FEC_5_6, 17500, 20500},
+ { PSK_8, FEC_8_9, 19000, 22000},
+};
+
+static const struct vidtv_tuner_cnr_to_qual_s vidtv_tuner_t_cnr_2_qual[] = {
+ /* from libdvbv5 source code, in milli db*/
+ { QPSK, FEC_1_2, 4100, 5900},
+ { QPSK, FEC_2_3, 6100, 9600},
+ { QPSK, FEC_3_4, 7200, 12400},
+ { QPSK, FEC_5_6, 8500, 15600},
+ { QPSK, FEC_7_8, 9200, 17500},
+ { QAM_16, FEC_1_2, 9800, 11800},
+ { QAM_16, FEC_2_3, 12100, 15300},
+ { QAM_16, FEC_3_4, 13400, 18100},
+ { QAM_16, FEC_5_6, 14800, 21300},
+ { QAM_16, FEC_7_8, 15700, 23600},
+ { QAM_64, FEC_1_2, 14000, 16000},
+ { QAM_64, FEC_2_3, 19900, 25400},
+ { QAM_64, FEC_3_4, 24900, 27900},
+ { QAM_64, FEC_5_6, 21300, 23300},
+ { QAM_64, FEC_7_8, 22000, 24000},
+};
+
+/**
+ * struct vidtv_tuner_hardware_state - Simulate the tuner hardware status
+ * @asleep: whether the tuner is asleep, i.e whether _sleep() or _suspend() was
+ * called.
+ * @lock_status: Whether the tuner has managed to lock on the requested
+ * frequency.
+ * @if_frequency: The tuner's intermediate frequency. Hardcoded for the purposes
+ * of simulation.
+ * @tuned_frequency: The actual tuned frequency.
+ * @bandwidth: The actual bandwidth.
+ *
+ * This structure is meant to simulate the status of the tuner hardware, as if
+ * we had a physical tuner hardware.
+ */
+struct vidtv_tuner_hardware_state {
+ bool asleep;
+ u32 lock_status;
+ u32 if_frequency;
+ u32 tuned_frequency;
+ u32 bandwidth;
+};
+
+/**
+ * struct vidtv_tuner_dev - The tuner struct
+ * @fe: A pointer to the dvb_frontend structure allocated by vidtv_demod
+ * @hw_state: A struct to simulate the tuner's hardware state as if we had a
+ * physical tuner hardware.
+ * @config: The configuration used to start the tuner module, usually filled
+ * by a bridge driver. For vidtv, this is filled by vidtv_bridge before the
+ * tuner module is probed.
+ */
+struct vidtv_tuner_dev {
+ struct dvb_frontend *fe;
+ struct vidtv_tuner_hardware_state hw_state;
+ struct vidtv_tuner_config config;
+};
+
+static struct vidtv_tuner_dev*
+vidtv_tuner_get_dev(struct dvb_frontend *fe)
+{
+ return i2c_get_clientdata(fe->tuner_priv);
+}
+
+static int vidtv_tuner_check_frequency_shift(struct dvb_frontend *fe)
+{
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+ struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+ struct vidtv_tuner_config config = tuner_dev->config;
+ u32 *valid_freqs = NULL;
+ u32 array_sz = 0;
+ u32 i;
+ u32 shift;
+
+ switch (c->delivery_system) {
+ case SYS_DVBT:
+ case SYS_DVBT2:
+ valid_freqs = config.vidtv_valid_dvb_t_freqs;
+ array_sz = ARRAY_SIZE(config.vidtv_valid_dvb_t_freqs);
+ break;
+ case SYS_DVBS:
+ case SYS_DVBS2:
+ valid_freqs = config.vidtv_valid_dvb_s_freqs;
+ array_sz = ARRAY_SIZE(config.vidtv_valid_dvb_s_freqs);
+ break;
+ case SYS_DVBC_ANNEX_A:
+ valid_freqs = config.vidtv_valid_dvb_c_freqs;
+ array_sz = ARRAY_SIZE(config.vidtv_valid_dvb_c_freqs);
+ break;
+
+ default:
+ dev_warn(fe->dvb->device,
+ "%s: unsupported delivery system: %u\n",
+ __func__,
+ c->delivery_system);
+
+ return -EINVAL;
+ }
+
+ for (i = 0; i < array_sz; i++) {
+ if (!valid_freqs[i])
+ break;
+ shift = abs(c->frequency - valid_freqs[i]);
+
+ if (!shift)
+ return 0;
+
+ /*
+ * This will provide a value from 0 to 100 that would
+ * indicate how far is the tuned frequency from the
+ * right one.
+ */
+ if (shift < config.max_frequency_shift_hz)
+ return shift * 100 / config.max_frequency_shift_hz;
+ }
+
+ return -EINVAL;
+}
+
+static int
+vidtv_tuner_get_signal_strength(struct dvb_frontend *fe, u16 *strength)
+{
+ struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+ const struct vidtv_tuner_cnr_to_qual_s *cnr2qual = NULL;
+ struct device *dev = fe->dvb->device;
+ u32 array_size = 0;
+ s32 shift;
+ u32 i;
+
+ shift = vidtv_tuner_check_frequency_shift(fe);
+ if (shift < 0) {
+ tuner_dev->hw_state.lock_status = 0;
+ *strength = 0;
+ return 0;
+ }
+
+ switch (c->delivery_system) {
+ case SYS_DVBT:
+ case SYS_DVBT2:
+ cnr2qual = vidtv_tuner_t_cnr_2_qual;
+ array_size = ARRAY_SIZE(vidtv_tuner_t_cnr_2_qual);
+ break;
+
+ case SYS_DVBS:
+ cnr2qual = vidtv_tuner_s_cnr_2_qual;
+ array_size = ARRAY_SIZE(vidtv_tuner_s_cnr_2_qual);
+ break;
+
+ case SYS_DVBS2:
+ cnr2qual = vidtv_tuner_s2_cnr_2_qual;
+ array_size = ARRAY_SIZE(vidtv_tuner_s2_cnr_2_qual);
+ break;
+
+ case SYS_DVBC_ANNEX_A:
+ cnr2qual = vidtv_tuner_c_cnr_2_qual;
+ array_size = ARRAY_SIZE(vidtv_tuner_c_cnr_2_qual);
+ break;
+
+ default:
+ dev_warn_ratelimited(dev,
+ "%s: unsupported delivery system: %u\n",
+ __func__,
+ c->delivery_system);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < array_size; i++) {
+ if (cnr2qual[i].modulation != c->modulation ||
+ cnr2qual[i].fec != c->fec_inner)
+ continue;
+
+ if (!shift) {
+ *strength = cnr2qual[i].cnr_good;
+ return 0;
+ }
+ /*
+ * Channel tuned at wrong frequency. Simulate that the
+ * Carrier S/N ratio is not too good.
+ */
+
+ *strength = cnr2qual[i].cnr_ok -
+ (cnr2qual[i].cnr_good - cnr2qual[i].cnr_ok);
+ return 0;
+ }
+
+ /*
+ * do a linear interpolation between 34dB and 10dB if we can't
+ * match against the table
+ */
+ *strength = 34000 - 24000 * shift / 100;
+ return 0;
+}
+
+static int vidtv_tuner_init(struct dvb_frontend *fe)
+{
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+ struct vidtv_tuner_config config = tuner_dev->config;
+
+ msleep_interruptible(config.mock_power_up_delay_msec);
+
+ tuner_dev->hw_state.asleep = false;
+ tuner_dev->hw_state.if_frequency = 5000;
+
+ return 0;
+}
+
+static int vidtv_tuner_sleep(struct dvb_frontend *fe)
+{
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+
+ tuner_dev->hw_state.asleep = true;
+ return 0;
+}
+
+static int vidtv_tuner_suspend(struct dvb_frontend *fe)
+{
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+
+ tuner_dev->hw_state.asleep = true;
+ return 0;
+}
+
+static int vidtv_tuner_resume(struct dvb_frontend *fe)
+{
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+
+ tuner_dev->hw_state.asleep = false;
+ return 0;
+}
+
+static int vidtv_tuner_set_params(struct dvb_frontend *fe)
+{
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+ struct vidtv_tuner_config config = tuner_dev->config;
+ struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+ s32 shift;
+
+ u32 min_freq = fe->ops.tuner_ops.info.frequency_min_hz;
+ u32 max_freq = fe->ops.tuner_ops.info.frequency_max_hz;
+ u32 min_bw = fe->ops.tuner_ops.info.bandwidth_min;
+ u32 max_bw = fe->ops.tuner_ops.info.bandwidth_max;
+
+ if (c->frequency < min_freq || c->frequency > max_freq ||
+ c->bandwidth_hz < min_bw || c->bandwidth_hz > max_bw) {
+ tuner_dev->hw_state.lock_status = 0;
+ return -EINVAL;
+ }
+
+ tuner_dev->hw_state.tuned_frequency = c->frequency;
+ tuner_dev->hw_state.bandwidth = c->bandwidth_hz;
+ tuner_dev->hw_state.lock_status = TUNER_STATUS_LOCKED;
+
+ msleep_interruptible(config.mock_tune_delay_msec);
+
+ shift = vidtv_tuner_check_frequency_shift(fe);
+ if (shift < 0) {
+ tuner_dev->hw_state.lock_status = 0;
+ return shift;
+ }
+
+ return 0;
+}
+
+static int vidtv_tuner_set_config(struct dvb_frontend *fe,
+ void *priv_cfg)
+{
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+
+ memcpy(&tuner_dev->config, priv_cfg, sizeof(tuner_dev->config));
+
+ return 0;
+}
+
+static int vidtv_tuner_get_frequency(struct dvb_frontend *fe,
+ u32 *frequency)
+{
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+
+ *frequency = tuner_dev->hw_state.tuned_frequency;
+
+ return 0;
+}
+
+static int vidtv_tuner_get_bandwidth(struct dvb_frontend *fe,
+ u32 *bandwidth)
+{
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+
+ *bandwidth = tuner_dev->hw_state.bandwidth;
+
+ return 0;
+}
+
+static int vidtv_tuner_get_if_frequency(struct dvb_frontend *fe,
+ u32 *frequency)
+{
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+
+ *frequency = tuner_dev->hw_state.if_frequency;
+
+ return 0;
+}
+
+static int vidtv_tuner_get_status(struct dvb_frontend *fe, u32 *status)
+{
+ struct vidtv_tuner_dev *tuner_dev = vidtv_tuner_get_dev(fe);
+
+ *status = tuner_dev->hw_state.lock_status;
+
+ return 0;
+}
+
+static const struct dvb_tuner_ops vidtv_tuner_ops = {
+ .init = vidtv_tuner_init,
+ .sleep = vidtv_tuner_sleep,
+ .suspend = vidtv_tuner_suspend,
+ .resume = vidtv_tuner_resume,
+ .set_params = vidtv_tuner_set_params,
+ .set_config = vidtv_tuner_set_config,
+ .get_bandwidth = vidtv_tuner_get_bandwidth,
+ .get_frequency = vidtv_tuner_get_frequency,
+ .get_if_frequency = vidtv_tuner_get_if_frequency,
+ .get_status = vidtv_tuner_get_status,
+ .get_rf_strength = vidtv_tuner_get_signal_strength
+};
+
+static const struct i2c_device_id vidtv_tuner_i2c_id_table[] = {
+ {"dvb_vidtv_tuner", 0},
+ {}
+};
+MODULE_DEVICE_TABLE(i2c, vidtv_tuner_i2c_id_table);
+
+static int vidtv_tuner_i2c_probe(struct i2c_client *client,
+ const struct i2c_device_id *id)
+{
+ struct vidtv_tuner_config *config = client->dev.platform_data;
+ struct dvb_frontend *fe = config->fe;
+ struct vidtv_tuner_dev *tuner_dev = NULL;
+
+ tuner_dev = kzalloc(sizeof(*tuner_dev), GFP_KERNEL);
+ if (!tuner_dev)
+ return -ENOMEM;
+
+ tuner_dev->fe = config->fe;
+ i2c_set_clientdata(client, tuner_dev);
+
+ memcpy(&fe->ops.tuner_ops,
+ &vidtv_tuner_ops,
+ sizeof(struct dvb_tuner_ops));
+
+ memcpy(&tuner_dev->config, config, sizeof(tuner_dev->config));
+ fe->tuner_priv = client;
+
+ return 0;
+}
+
+static int vidtv_tuner_i2c_remove(struct i2c_client *client)
+{
+ struct vidtv_tuner_dev *tuner_dev = i2c_get_clientdata(client);
+
+ kfree(tuner_dev);
+
+ return 0;
+}
+
+static struct i2c_driver vidtv_tuner_i2c_driver = {
+ .driver = {
+ .name = "dvb_vidtv_tuner",
+ .suppress_bind_attrs = true,
+ },
+ .probe = vidtv_tuner_i2c_probe,
+ .remove = vidtv_tuner_i2c_remove,
+ .id_table = vidtv_tuner_i2c_id_table,
+};
+module_i2c_driver(vidtv_tuner_i2c_driver);
+
+MODULE_DESCRIPTION("Virtual DVB Tuner");
+MODULE_AUTHOR("Daniel W. S. Almeida");
+MODULE_LICENSE("GPL");
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * The Virtual DTV test driver serves as a reference DVB driver and helps
+ * validate the existing APIs in the media subsystem. It can also aid
+ * developers working on userspace applications.
+ *
+ * Copyright (C) 2020 Daniel W. S. Almeida
+ */
+
+#ifndef VIDTV_TUNER_H
+#define VIDTV_TUNER_H
+
+#include <linux/types.h>
+#include <media/dvb_frontend.h>
+
+#define NUM_VALID_TUNER_FREQS 8
+
+/**
+ * struct vidtv_tuner_config - Configuration used to init the tuner.
+ * @fe: A pointer to the dvb_frontend structure allocated by vidtv_demod.
+ * @mock_power_up_delay_msec: Simulate a power-up delay.
+ * @mock_tune_delay_msec: Simulate a tune delay.
+ * @vidtv_valid_dvb_t_freqs: The valid DVB-T frequencies to simulate.
+ * @vidtv_valid_dvb_c_freqs: The valid DVB-C frequencies to simulate.
+ * @vidtv_valid_dvb_s_freqs: The valid DVB-S frequencies to simulate.
+ * @max_frequency_shift_hz: The maximum frequency shift in HZ allowed when
+ * tuning in a channel
+ *
+ * The configuration used to init the tuner module, usually filled
+ * by a bridge driver. For vidtv, this is filled by vidtv_bridge before the
+ * tuner module is probed.
+ */
+struct vidtv_tuner_config {
+ struct dvb_frontend *fe;
+ u32 mock_power_up_delay_msec;
+ u32 mock_tune_delay_msec;
+ u32 vidtv_valid_dvb_t_freqs[NUM_VALID_TUNER_FREQS];
+ u32 vidtv_valid_dvb_c_freqs[NUM_VALID_TUNER_FREQS];
+ u32 vidtv_valid_dvb_s_freqs[NUM_VALID_TUNER_FREQS];
+ u8 max_frequency_shift_hz;
+};
+
+#endif //VIDTV_TUNER_H
struct vimc_cap_device *vcap =
container_of(ved, struct vimc_cap_device, ved);
- vb2_queue_release(&vcap->queue);
- video_unregister_device(&vcap->vdev);
+ vb2_video_unregister_device(&vcap->vdev);
}
static void *vimc_cap_process_frame(struct vimc_ent_device *ved,
if (ret) {
dev_err(vimc->mdev.dev, "%s: video register failed (err=%d)\n",
vcap->vdev.name, ret);
- goto err_release_queue;
+ goto err_clean_m_ent;
}
return &vcap->ved;
-err_release_queue:
- vb2_queue_release(q);
err_clean_m_ent:
media_entity_cleanup(&vcap->vdev.entity);
err_free_vcap:
return vb2_queue_init(q);
}
-static int vivid_create_instance(struct platform_device *pdev, int inst)
+static int vivid_detect_feature_set(struct vivid_dev *dev, int inst,
+ unsigned node_type,
+ bool *has_tuner,
+ bool *has_modulator,
+ int *ccs_cap,
+ int *ccs_out,
+ unsigned in_type_counter[4],
+ unsigned out_type_counter[4])
{
- static const struct v4l2_dv_timings def_dv_timings =
- V4L2_DV_BT_CEA_1280X720P60;
- unsigned in_type_counter[4] = { 0, 0, 0, 0 };
- unsigned out_type_counter[4] = { 0, 0, 0, 0 };
- int ccs_cap = ccs_cap_mode[inst];
- int ccs_out = ccs_out_mode[inst];
- bool has_tuner;
- bool has_modulator;
- struct vivid_dev *dev;
- struct video_device *vfd;
- unsigned node_type = node_types[inst];
- v4l2_std_id tvnorms_cap = 0, tvnorms_out = 0;
- int ret;
int i;
-#ifdef CONFIG_VIDEO_VIVID_CEC
- unsigned int cec_tx_bus_cnt = 0;
-#endif
-
- /* allocate main vivid state structure */
- dev = kzalloc(sizeof(*dev), GFP_KERNEL);
- if (!dev)
- return -ENOMEM;
-
- dev->inst = inst;
-
-#ifdef CONFIG_MEDIA_CONTROLLER
- dev->v4l2_dev.mdev = &dev->mdev;
-
- /* Initialize media device */
- strscpy(dev->mdev.model, VIVID_MODULE_NAME, sizeof(dev->mdev.model));
- snprintf(dev->mdev.bus_info, sizeof(dev->mdev.bus_info),
- "platform:%s-%03d", VIVID_MODULE_NAME, inst);
- dev->mdev.dev = &pdev->dev;
- media_device_init(&dev->mdev);
- dev->mdev.ops = &vivid_media_ops;
-#endif
-
- /* register v4l2_device */
- snprintf(dev->v4l2_dev.name, sizeof(dev->v4l2_dev.name),
- "%s-%03d", VIVID_MODULE_NAME, inst);
- ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev);
- if (ret) {
- kfree(dev);
- return ret;
- }
- dev->v4l2_dev.release = vivid_dev_release;
-
- /* start detecting feature set */
/* do we use single- or multi-planar? */
dev->multiplanar = multiplanar[inst] > 1;
!dev->has_vid_cap && !dev->has_meta_cap) {
v4l2_warn(&dev->v4l2_dev,
"Webcam or HDMI input without video or metadata nodes\n");
- kfree(dev);
return -EINVAL;
}
if ((in_type_counter[TV] || in_type_counter[SVID]) &&
!dev->has_vid_cap && !dev->has_vbi_cap && !dev->has_meta_cap) {
v4l2_warn(&dev->v4l2_dev,
"TV or S-Video input without video, VBI or metadata nodes\n");
- kfree(dev);
return -EINVAL;
}
!dev->has_vid_out && !dev->has_vbi_out && !dev->has_meta_out) {
v4l2_warn(&dev->v4l2_dev,
"S-Video output without video, VBI or metadata nodes\n");
- kfree(dev);
return -EINVAL;
}
if (out_type_counter[HDMI] && !dev->has_vid_out && !dev->has_meta_out) {
v4l2_warn(&dev->v4l2_dev,
"HDMI output without video or metadata nodes\n");
- kfree(dev);
return -EINVAL;
}
dev->has_tv_tuner = in_type_counter[TV];
/* do we have a tuner? */
- has_tuner = ((dev->has_vid_cap || dev->has_vbi_cap) && in_type_counter[TV]) ||
- dev->has_radio_rx || dev->has_sdr_cap;
+ *has_tuner = ((dev->has_vid_cap || dev->has_vbi_cap) && in_type_counter[TV]) ||
+ dev->has_radio_rx || dev->has_sdr_cap;
/* do we have a modulator? */
- has_modulator = dev->has_radio_tx;
+ *has_modulator = dev->has_radio_tx;
if (dev->has_vid_cap)
/* do we have a framebuffer for overlay testing? */
dev->has_fb = node_type & 0x10000;
/* can we do crop/compose/scaling while capturing? */
- if (no_error_inj && ccs_cap == -1)
- ccs_cap = 7;
+ if (no_error_inj && *ccs_cap == -1)
+ *ccs_cap = 7;
/* if ccs_cap == -1, then the user can select it using controls */
- if (ccs_cap != -1) {
- dev->has_crop_cap = ccs_cap & 1;
- dev->has_compose_cap = ccs_cap & 2;
- dev->has_scaler_cap = ccs_cap & 4;
+ if (*ccs_cap != -1) {
+ dev->has_crop_cap = *ccs_cap & 1;
+ dev->has_compose_cap = *ccs_cap & 2;
+ dev->has_scaler_cap = *ccs_cap & 4;
v4l2_info(&dev->v4l2_dev, "Capture Crop: %c Compose: %c Scaler: %c\n",
dev->has_crop_cap ? 'Y' : 'N',
dev->has_compose_cap ? 'Y' : 'N',
}
/* can we do crop/compose/scaling with video output? */
- if (no_error_inj && ccs_out == -1)
- ccs_out = 7;
+ if (no_error_inj && *ccs_out == -1)
+ *ccs_out = 7;
/* if ccs_out == -1, then the user can select it using controls */
- if (ccs_out != -1) {
- dev->has_crop_out = ccs_out & 1;
- dev->has_compose_out = ccs_out & 2;
- dev->has_scaler_out = ccs_out & 4;
+ if (*ccs_out != -1) {
+ dev->has_crop_out = *ccs_out & 1;
+ dev->has_compose_out = *ccs_out & 2;
+ dev->has_scaler_out = *ccs_out & 4;
v4l2_info(&dev->v4l2_dev, "Output Crop: %c Compose: %c Scaler: %c\n",
dev->has_crop_out ? 'Y' : 'N',
dev->has_compose_out ? 'Y' : 'N',
/* do we create a touch capture device */
dev->has_touch_cap = node_type & 0x80000;
- /* end detecting feature set */
+ return 0;
+}
+static void vivid_set_capabilities(struct vivid_dev *dev)
+{
if (dev->has_vid_cap) {
/* set up the capabilities of the video capture device */
dev->vid_cap_caps = dev->multiplanar ?
dev->touch_cap_caps |= dev->multiplanar ?
V4L2_CAP_VIDEO_CAPTURE_MPLANE : V4L2_CAP_VIDEO_CAPTURE;
}
+}
- ret = -ENOMEM;
- /* initialize the test pattern generator */
- tpg_init(&dev->tpg, 640, 360);
- if (tpg_alloc(&dev->tpg, array_size(MAX_WIDTH, MAX_ZOOM)))
- goto free_dev;
- dev->scaled_line = vzalloc(array_size(MAX_WIDTH, MAX_ZOOM));
- if (!dev->scaled_line)
- goto free_dev;
- dev->blended_line = vzalloc(array_size(MAX_WIDTH, MAX_ZOOM));
- if (!dev->blended_line)
- goto free_dev;
-
- /* load the edid */
- dev->edid = vmalloc(array_size(256, 128));
- if (!dev->edid)
- goto free_dev;
-
- while (v4l2_dv_timings_presets[dev->query_dv_timings_size].bt.width)
- dev->query_dv_timings_size++;
-
- /*
- * Create a char pointer array that points to the names of all the
- * preset timings
- */
- dev->query_dv_timings_qmenu = kmalloc_array(dev->query_dv_timings_size,
- sizeof(char *), GFP_KERNEL);
- /*
- * Create a string array containing the names of all the preset
- * timings. Each name is max 31 chars long (+ terminating 0).
- */
- dev->query_dv_timings_qmenu_strings =
- kmalloc_array(dev->query_dv_timings_size, 32, GFP_KERNEL);
-
- if (!dev->query_dv_timings_qmenu ||
- !dev->query_dv_timings_qmenu_strings)
- goto free_dev;
-
- for (i = 0; i < dev->query_dv_timings_size; i++) {
- const struct v4l2_bt_timings *bt = &v4l2_dv_timings_presets[i].bt;
- char *p = dev->query_dv_timings_qmenu_strings + i * 32;
- u32 htot, vtot;
-
- dev->query_dv_timings_qmenu[i] = p;
-
- htot = V4L2_DV_BT_FRAME_WIDTH(bt);
- vtot = V4L2_DV_BT_FRAME_HEIGHT(bt);
- snprintf(p, 32, "%ux%u%s%u",
- bt->width, bt->height, bt->interlaced ? "i" : "p",
- (u32)bt->pixelclock / (htot * vtot));
- }
-
+static void vivid_disable_unused_ioctls(struct vivid_dev *dev,
+ bool has_tuner,
+ bool has_modulator,
+ unsigned in_type_counter[4],
+ unsigned out_type_counter[4])
+{
/* disable invalid ioctls based on the feature set */
if (!dev->has_audio_inputs) {
v4l2_disable_ioctl(&dev->vid_cap_dev, VIDIOC_S_AUDIO);
v4l2_disable_ioctl(&dev->touch_cap_dev, VIDIOC_S_PARM);
v4l2_disable_ioctl(&dev->touch_cap_dev, VIDIOC_ENUM_FRAMESIZES);
v4l2_disable_ioctl(&dev->touch_cap_dev, VIDIOC_ENUM_FRAMEINTERVALS);
+}
- /* configure internal data */
- dev->fmt_cap = &vivid_formats[0];
- dev->fmt_out = &vivid_formats[0];
- if (!dev->multiplanar)
- vivid_formats[0].data_offset[0] = 0;
- dev->webcam_size_idx = 1;
- dev->webcam_ival_idx = 3;
- tpg_s_fourcc(&dev->tpg, dev->fmt_cap->fourcc);
- dev->std_out = V4L2_STD_PAL;
- if (dev->input_type[0] == TV || dev->input_type[0] == SVID)
- tvnorms_cap = V4L2_STD_ALL;
- if (dev->output_type[0] == SVID)
- tvnorms_out = V4L2_STD_ALL;
- for (i = 0; i < MAX_INPUTS; i++) {
- dev->dv_timings_cap[i] = def_dv_timings;
- dev->std_cap[i] = V4L2_STD_PAL;
- }
- dev->dv_timings_out = def_dv_timings;
- dev->tv_freq = 2804 /* 175.25 * 16 */;
- dev->tv_audmode = V4L2_TUNER_MODE_STEREO;
- dev->tv_field_cap = V4L2_FIELD_INTERLACED;
- dev->tv_field_out = V4L2_FIELD_INTERLACED;
- dev->radio_rx_freq = 95000 * 16;
- dev->radio_rx_audmode = V4L2_TUNER_MODE_STEREO;
- if (dev->has_radio_tx) {
- dev->radio_tx_freq = 95500 * 16;
- dev->radio_rds_loop = false;
- }
- dev->radio_tx_subchans = V4L2_TUNER_SUB_STEREO | V4L2_TUNER_SUB_RDS;
- dev->sdr_adc_freq = 300000;
- dev->sdr_fm_freq = 50000000;
- dev->sdr_pixelformat = V4L2_SDR_FMT_CU8;
- dev->sdr_buffersize = SDR_CAP_SAMPLES_PER_BUF * 2;
-
- dev->edid_max_blocks = dev->edid_blocks = 2;
- memcpy(dev->edid, vivid_hdmi_edid, sizeof(vivid_hdmi_edid));
- dev->radio_rds_init_time = ktime_get();
-
- /* create all controls */
- ret = vivid_create_controls(dev, ccs_cap == -1, ccs_out == -1, no_error_inj,
- in_type_counter[TV] || in_type_counter[SVID] ||
- out_type_counter[SVID],
- in_type_counter[HDMI] || out_type_counter[HDMI]);
- if (ret)
- goto unreg_dev;
+static int vivid_init_dv_timings(struct vivid_dev *dev)
+{
+ int i;
- /* enable/disable interface specific controls */
- if (dev->num_outputs && dev->output_type[0] != HDMI)
- v4l2_ctrl_activate(dev->ctrl_display_present, false);
- if (dev->num_inputs && dev->input_type[0] != HDMI) {
- v4l2_ctrl_activate(dev->ctrl_dv_timings_signal_mode, false);
- v4l2_ctrl_activate(dev->ctrl_dv_timings, false);
- } else if (dev->num_inputs && dev->input_type[0] == HDMI) {
- v4l2_ctrl_activate(dev->ctrl_std_signal_mode, false);
- v4l2_ctrl_activate(dev->ctrl_standard, false);
- }
+ while (v4l2_dv_timings_presets[dev->query_dv_timings_size].bt.width)
+ dev->query_dv_timings_size++;
/*
- * update the capture and output formats to do a proper initial
- * configuration.
+ * Create a char pointer array that points to the names of all the
+ * preset timings
*/
- vivid_update_format_cap(dev, false);
- vivid_update_format_out(dev);
-
- /* initialize overlay */
- dev->fb_cap.fmt.width = dev->src_rect.width;
- dev->fb_cap.fmt.height = dev->src_rect.height;
- dev->fb_cap.fmt.pixelformat = dev->fmt_cap->fourcc;
- dev->fb_cap.fmt.bytesperline = dev->src_rect.width * tpg_g_twopixelsize(&dev->tpg, 0) / 2;
- dev->fb_cap.fmt.sizeimage = dev->src_rect.height * dev->fb_cap.fmt.bytesperline;
+ dev->query_dv_timings_qmenu = kmalloc_array(dev->query_dv_timings_size,
+ sizeof(char *), GFP_KERNEL);
+ /*
+ * Create a string array containing the names of all the preset
+ * timings. Each name is max 31 chars long (+ terminating 0).
+ */
+ dev->query_dv_timings_qmenu_strings =
+ kmalloc_array(dev->query_dv_timings_size, 32, GFP_KERNEL);
- /* update touch configuration */
- dev->timeperframe_tch_cap.numerator = 1;
- dev->timeperframe_tch_cap.denominator = 10;
- vivid_set_touch(dev, 0);
+ if (!dev->query_dv_timings_qmenu ||
+ !dev->query_dv_timings_qmenu_strings)
+ return -ENOMEM;
- /* initialize locks */
- spin_lock_init(&dev->slock);
- mutex_init(&dev->mutex);
+ for (i = 0; i < dev->query_dv_timings_size; i++) {
+ const struct v4l2_bt_timings *bt = &v4l2_dv_timings_presets[i].bt;
+ char *p = dev->query_dv_timings_qmenu_strings + i * 32;
+ u32 htot, vtot;
- /* init dma queues */
- INIT_LIST_HEAD(&dev->vid_cap_active);
- INIT_LIST_HEAD(&dev->vid_out_active);
- INIT_LIST_HEAD(&dev->vbi_cap_active);
- INIT_LIST_HEAD(&dev->vbi_out_active);
- INIT_LIST_HEAD(&dev->sdr_cap_active);
- INIT_LIST_HEAD(&dev->meta_cap_active);
- INIT_LIST_HEAD(&dev->meta_out_active);
- INIT_LIST_HEAD(&dev->touch_cap_active);
+ dev->query_dv_timings_qmenu[i] = p;
- INIT_LIST_HEAD(&dev->cec_work_list);
- spin_lock_init(&dev->cec_slock);
- /*
- * Same as create_singlethread_workqueue, but now I can use the
- * string formatting of alloc_ordered_workqueue.
- */
- dev->cec_workqueue =
- alloc_ordered_workqueue("vivid-%03d-cec", WQ_MEM_RECLAIM, inst);
- if (!dev->cec_workqueue) {
- ret = -ENOMEM;
- goto unreg_dev;
+ htot = V4L2_DV_BT_FRAME_WIDTH(bt);
+ vtot = V4L2_DV_BT_FRAME_HEIGHT(bt);
+ snprintf(p, 32, "%ux%u%s%u",
+ bt->width, bt->height, bt->interlaced ? "i" : "p",
+ (u32)bt->pixelclock / (htot * vtot));
}
- if (allocators[inst] == 1)
- dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+ return 0;
+}
+
+static int vivid_create_queues(struct vivid_dev *dev)
+{
+ int ret;
/* start creating the vb2 queues */
if (dev->has_vid_cap) {
V4L2_BUF_TYPE_VIDEO_CAPTURE, 2,
&vivid_vid_cap_qops);
if (ret)
- goto unreg_dev;
+ return ret;
}
if (dev->has_vid_out) {
V4L2_BUF_TYPE_VIDEO_OUTPUT, 2,
&vivid_vid_out_qops);
if (ret)
- goto unreg_dev;
+ return ret;
}
if (dev->has_vbi_cap) {
V4L2_BUF_TYPE_VBI_CAPTURE, 2,
&vivid_vbi_cap_qops);
if (ret)
- goto unreg_dev;
+ return ret;
}
if (dev->has_vbi_out) {
V4L2_BUF_TYPE_VBI_OUTPUT, 2,
&vivid_vbi_out_qops);
if (ret)
- goto unreg_dev;
+ return ret;
}
if (dev->has_sdr_cap) {
V4L2_BUF_TYPE_SDR_CAPTURE, 8,
&vivid_sdr_cap_qops);
if (ret)
- goto unreg_dev;
+ return ret;
}
if (dev->has_meta_cap) {
V4L2_BUF_TYPE_META_CAPTURE, 2,
&vivid_meta_cap_qops);
if (ret)
- goto unreg_dev;
+ return ret;
}
if (dev->has_meta_out) {
V4L2_BUF_TYPE_META_OUTPUT, 1,
&vivid_meta_out_qops);
if (ret)
- goto unreg_dev;
+ return ret;
}
if (dev->has_touch_cap) {
V4L2_BUF_TYPE_VIDEO_CAPTURE, 1,
&vivid_touch_cap_qops);
if (ret)
- goto unreg_dev;
+ return ret;
}
if (dev->has_fb) {
/* Create framebuffer for testing capture/output overlay */
ret = vivid_fb_init(dev);
if (ret)
- goto unreg_dev;
+ return ret;
v4l2_info(&dev->v4l2_dev, "Framebuffer device registered as fb%d\n",
dev->fb_info.node);
}
+ return 0;
+}
-#ifdef CONFIG_VIDEO_VIVID_CEC
- if (dev->has_vid_cap && in_type_counter[HDMI]) {
- struct cec_adapter *adap;
+static int vivid_create_devnodes(struct platform_device *pdev,
+ struct vivid_dev *dev, int inst,
+ unsigned int cec_tx_bus_cnt,
+ v4l2_std_id tvnorms_cap,
+ v4l2_std_id tvnorms_out,
+ unsigned in_type_counter[4],
+ unsigned out_type_counter[4])
+{
+ struct video_device *vfd;
+ int ret;
- adap = vivid_cec_alloc_adap(dev, 0, false);
- ret = PTR_ERR_OR_ZERO(adap);
- if (ret < 0)
- goto unreg_dev;
- dev->cec_rx_adap = adap;
- }
-
- if (dev->has_vid_out) {
- for (i = 0; i < dev->num_outputs; i++) {
- struct cec_adapter *adap;
-
- if (dev->output_type[i] != HDMI)
- continue;
-
- dev->cec_output2bus_map[i] = cec_tx_bus_cnt;
- adap = vivid_cec_alloc_adap(dev, cec_tx_bus_cnt, true);
- ret = PTR_ERR_OR_ZERO(adap);
- if (ret < 0) {
- for (i = 0; i < dev->num_outputs; i++)
- cec_delete_adapter(dev->cec_tx_adap[i]);
- goto unreg_dev;
- }
-
- dev->cec_tx_adap[cec_tx_bus_cnt] = adap;
- cec_tx_bus_cnt++;
- }
- }
-#endif
-
- v4l2_ctrl_handler_setup(&dev->ctrl_hdl_vid_cap);
- v4l2_ctrl_handler_setup(&dev->ctrl_hdl_vid_out);
- v4l2_ctrl_handler_setup(&dev->ctrl_hdl_vbi_cap);
- v4l2_ctrl_handler_setup(&dev->ctrl_hdl_vbi_out);
- v4l2_ctrl_handler_setup(&dev->ctrl_hdl_radio_rx);
- v4l2_ctrl_handler_setup(&dev->ctrl_hdl_radio_tx);
- v4l2_ctrl_handler_setup(&dev->ctrl_hdl_sdr_cap);
- v4l2_ctrl_handler_setup(&dev->ctrl_hdl_meta_cap);
- v4l2_ctrl_handler_setup(&dev->ctrl_hdl_meta_out);
- v4l2_ctrl_handler_setup(&dev->ctrl_hdl_touch_cap);
-
- /* finally start creating the device nodes */
if (dev->has_vid_cap) {
vfd = &dev->vid_cap_dev;
snprintf(vfd->name, sizeof(vfd->name),
dev->vid_cap_pad.flags = MEDIA_PAD_FL_SINK;
ret = media_entity_pads_init(&vfd->entity, 1, &dev->vid_cap_pad);
if (ret)
- goto unreg_dev;
+ return ret;
#endif
#ifdef CONFIG_VIDEO_VIVID_CEC
if (ret < 0) {
cec_delete_adapter(dev->cec_rx_adap);
dev->cec_rx_adap = NULL;
- goto unreg_dev;
+ return ret;
}
cec_s_phys_addr(dev->cec_rx_adap, 0, false);
v4l2_info(&dev->v4l2_dev, "CEC adapter %s registered for HDMI input 0\n",
ret = video_register_device(vfd, VFL_TYPE_VIDEO, vid_cap_nr[inst]);
if (ret < 0)
- goto unreg_dev;
+ return ret;
v4l2_info(&dev->v4l2_dev, "V4L2 capture device registered as %s\n",
video_device_node_name(vfd));
}
if (dev->has_vid_out) {
+#ifdef CONFIG_VIDEO_VIVID_CEC
+ int i;
+#endif
vfd = &dev->vid_out_dev;
snprintf(vfd->name, sizeof(vfd->name),
"vivid-%03d-vid-out", inst);
dev->vid_out_pad.flags = MEDIA_PAD_FL_SOURCE;
ret = media_entity_pads_init(&vfd->entity, 1, &dev->vid_out_pad);
if (ret)
- goto unreg_dev;
+ return ret;
#endif
#ifdef CONFIG_VIDEO_VIVID_CEC
cec_delete_adapter(dev->cec_tx_adap[i]);
dev->cec_tx_adap[i] = NULL;
}
- goto unreg_dev;
+ return ret;
}
v4l2_info(&dev->v4l2_dev, "CEC adapter %s registered for HDMI output %d\n",
dev_name(&dev->cec_tx_adap[i]->devnode.dev), i);
ret = video_register_device(vfd, VFL_TYPE_VIDEO, vid_out_nr[inst]);
if (ret < 0)
- goto unreg_dev;
+ return ret;
v4l2_info(&dev->v4l2_dev, "V4L2 output device registered as %s\n",
video_device_node_name(vfd));
}
dev->vbi_cap_pad.flags = MEDIA_PAD_FL_SINK;
ret = media_entity_pads_init(&vfd->entity, 1, &dev->vbi_cap_pad);
if (ret)
- goto unreg_dev;
+ return ret;
#endif
ret = video_register_device(vfd, VFL_TYPE_VBI, vbi_cap_nr[inst]);
if (ret < 0)
- goto unreg_dev;
+ return ret;
v4l2_info(&dev->v4l2_dev, "V4L2 capture device registered as %s, supports %s VBI\n",
video_device_node_name(vfd),
(dev->has_raw_vbi_cap && dev->has_sliced_vbi_cap) ?
dev->vbi_out_pad.flags = MEDIA_PAD_FL_SOURCE;
ret = media_entity_pads_init(&vfd->entity, 1, &dev->vbi_out_pad);
if (ret)
- goto unreg_dev;
+ return ret;
#endif
ret = video_register_device(vfd, VFL_TYPE_VBI, vbi_out_nr[inst]);
if (ret < 0)
- goto unreg_dev;
+ return ret;
v4l2_info(&dev->v4l2_dev, "V4L2 output device registered as %s, supports %s VBI\n",
video_device_node_name(vfd),
(dev->has_raw_vbi_out && dev->has_sliced_vbi_out) ?
dev->sdr_cap_pad.flags = MEDIA_PAD_FL_SINK;
ret = media_entity_pads_init(&vfd->entity, 1, &dev->sdr_cap_pad);
if (ret)
- goto unreg_dev;
+ return ret;
#endif
ret = video_register_device(vfd, VFL_TYPE_SDR, sdr_cap_nr[inst]);
if (ret < 0)
- goto unreg_dev;
+ return ret;
v4l2_info(&dev->v4l2_dev, "V4L2 capture device registered as %s\n",
video_device_node_name(vfd));
}
ret = video_register_device(vfd, VFL_TYPE_RADIO, radio_rx_nr[inst]);
if (ret < 0)
- goto unreg_dev;
+ return ret;
v4l2_info(&dev->v4l2_dev, "V4L2 receiver device registered as %s\n",
video_device_node_name(vfd));
}
ret = video_register_device(vfd, VFL_TYPE_RADIO, radio_tx_nr[inst]);
if (ret < 0)
- goto unreg_dev;
+ return ret;
v4l2_info(&dev->v4l2_dev, "V4L2 transmitter device registered as %s\n",
video_device_node_name(vfd));
}
ret = media_entity_pads_init(&vfd->entity, 1,
&dev->meta_cap_pad);
if (ret)
- goto unreg_dev;
+ return ret;
#endif
ret = video_register_device(vfd, VFL_TYPE_VIDEO,
meta_cap_nr[inst]);
if (ret < 0)
- goto unreg_dev;
+ return ret;
v4l2_info(&dev->v4l2_dev,
"V4L2 metadata capture device registered as %s\n",
video_device_node_name(vfd));
ret = media_entity_pads_init(&vfd->entity, 1,
&dev->meta_out_pad);
if (ret)
- goto unreg_dev;
+ return ret;
#endif
ret = video_register_device(vfd, VFL_TYPE_VIDEO,
meta_out_nr[inst]);
if (ret < 0)
- goto unreg_dev;
+ return ret;
v4l2_info(&dev->v4l2_dev,
"V4L2 metadata output device registered as %s\n",
video_device_node_name(vfd));
ret = media_entity_pads_init(&vfd->entity, 1,
&dev->touch_cap_pad);
if (ret)
- goto unreg_dev;
+ return ret;
#endif
ret = video_register_device(vfd, VFL_TYPE_TOUCH,
touch_cap_nr[inst]);
if (ret < 0)
- goto unreg_dev;
+ return ret;
v4l2_info(&dev->v4l2_dev,
"V4L2 touch capture device registered as %s\n",
video_device_node_name(vfd));
if (ret) {
dev_err(dev->mdev.dev,
"media device register failed (err=%d)\n", ret);
+ return ret;
+ }
+#endif
+ return 0;
+}
+
+static int vivid_create_instance(struct platform_device *pdev, int inst)
+{
+ static const struct v4l2_dv_timings def_dv_timings =
+ V4L2_DV_BT_CEA_1280X720P60;
+ unsigned in_type_counter[4] = { 0, 0, 0, 0 };
+ unsigned out_type_counter[4] = { 0, 0, 0, 0 };
+ int ccs_cap = ccs_cap_mode[inst];
+ int ccs_out = ccs_out_mode[inst];
+ bool has_tuner;
+ bool has_modulator;
+ struct vivid_dev *dev;
+ unsigned node_type = node_types[inst];
+ v4l2_std_id tvnorms_cap = 0, tvnorms_out = 0;
+ unsigned int cec_tx_bus_cnt = 0;
+ int ret;
+ int i;
+
+ /* allocate main vivid state structure */
+ dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+ if (!dev)
+ return -ENOMEM;
+
+ dev->inst = inst;
+
+#ifdef CONFIG_MEDIA_CONTROLLER
+ dev->v4l2_dev.mdev = &dev->mdev;
+
+ /* Initialize media device */
+ strscpy(dev->mdev.model, VIVID_MODULE_NAME, sizeof(dev->mdev.model));
+ snprintf(dev->mdev.bus_info, sizeof(dev->mdev.bus_info),
+ "platform:%s-%03d", VIVID_MODULE_NAME, inst);
+ dev->mdev.dev = &pdev->dev;
+ media_device_init(&dev->mdev);
+ dev->mdev.ops = &vivid_media_ops;
+#endif
+
+ /* register v4l2_device */
+ snprintf(dev->v4l2_dev.name, sizeof(dev->v4l2_dev.name),
+ "%s-%03d", VIVID_MODULE_NAME, inst);
+ ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev);
+ if (ret) {
+ kfree(dev);
+ return ret;
+ }
+ dev->v4l2_dev.release = vivid_dev_release;
+
+ ret = vivid_detect_feature_set(dev, inst, node_type,
+ &has_tuner, &has_modulator,
+ &ccs_cap, &ccs_out,
+ in_type_counter, out_type_counter);
+ if (ret)
+ goto free_dev;
+
+ vivid_set_capabilities(dev);
+
+ ret = -ENOMEM;
+ /* initialize the test pattern generator */
+ tpg_init(&dev->tpg, 640, 360);
+ if (tpg_alloc(&dev->tpg, array_size(MAX_WIDTH, MAX_ZOOM)))
+ goto free_dev;
+ dev->scaled_line = vzalloc(array_size(MAX_WIDTH, MAX_ZOOM));
+ if (!dev->scaled_line)
+ goto free_dev;
+ dev->blended_line = vzalloc(array_size(MAX_WIDTH, MAX_ZOOM));
+ if (!dev->blended_line)
+ goto free_dev;
+
+ /* load the edid */
+ dev->edid = vmalloc(array_size(256, 128));
+ if (!dev->edid)
+ goto free_dev;
+
+ ret = vivid_init_dv_timings(dev);
+ if (ret < 0)
+ goto free_dev;
+
+ vivid_disable_unused_ioctls(dev, has_tuner, has_modulator,
+ in_type_counter, out_type_counter);
+
+ /* configure internal data */
+ dev->fmt_cap = &vivid_formats[0];
+ dev->fmt_out = &vivid_formats[0];
+ if (!dev->multiplanar)
+ vivid_formats[0].data_offset[0] = 0;
+ dev->webcam_size_idx = 1;
+ dev->webcam_ival_idx = 3;
+ tpg_s_fourcc(&dev->tpg, dev->fmt_cap->fourcc);
+ dev->std_out = V4L2_STD_PAL;
+ if (dev->input_type[0] == TV || dev->input_type[0] == SVID)
+ tvnorms_cap = V4L2_STD_ALL;
+ if (dev->output_type[0] == SVID)
+ tvnorms_out = V4L2_STD_ALL;
+ for (i = 0; i < MAX_INPUTS; i++) {
+ dev->dv_timings_cap[i] = def_dv_timings;
+ dev->std_cap[i] = V4L2_STD_PAL;
+ }
+ dev->dv_timings_out = def_dv_timings;
+ dev->tv_freq = 2804 /* 175.25 * 16 */;
+ dev->tv_audmode = V4L2_TUNER_MODE_STEREO;
+ dev->tv_field_cap = V4L2_FIELD_INTERLACED;
+ dev->tv_field_out = V4L2_FIELD_INTERLACED;
+ dev->radio_rx_freq = 95000 * 16;
+ dev->radio_rx_audmode = V4L2_TUNER_MODE_STEREO;
+ if (dev->has_radio_tx) {
+ dev->radio_tx_freq = 95500 * 16;
+ dev->radio_rds_loop = false;
+ }
+ dev->radio_tx_subchans = V4L2_TUNER_SUB_STEREO | V4L2_TUNER_SUB_RDS;
+ dev->sdr_adc_freq = 300000;
+ dev->sdr_fm_freq = 50000000;
+ dev->sdr_pixelformat = V4L2_SDR_FMT_CU8;
+ dev->sdr_buffersize = SDR_CAP_SAMPLES_PER_BUF * 2;
+
+ dev->edid_max_blocks = dev->edid_blocks = 2;
+ memcpy(dev->edid, vivid_hdmi_edid, sizeof(vivid_hdmi_edid));
+ dev->radio_rds_init_time = ktime_get();
+
+ /* create all controls */
+ ret = vivid_create_controls(dev, ccs_cap == -1, ccs_out == -1, no_error_inj,
+ in_type_counter[TV] || in_type_counter[SVID] ||
+ out_type_counter[SVID],
+ in_type_counter[HDMI] || out_type_counter[HDMI]);
+ if (ret)
+ goto unreg_dev;
+
+ /* enable/disable interface specific controls */
+ if (dev->num_outputs && dev->output_type[0] != HDMI)
+ v4l2_ctrl_activate(dev->ctrl_display_present, false);
+ if (dev->num_inputs && dev->input_type[0] != HDMI) {
+ v4l2_ctrl_activate(dev->ctrl_dv_timings_signal_mode, false);
+ v4l2_ctrl_activate(dev->ctrl_dv_timings, false);
+ } else if (dev->num_inputs && dev->input_type[0] == HDMI) {
+ v4l2_ctrl_activate(dev->ctrl_std_signal_mode, false);
+ v4l2_ctrl_activate(dev->ctrl_standard, false);
+ }
+
+ /*
+ * update the capture and output formats to do a proper initial
+ * configuration.
+ */
+ vivid_update_format_cap(dev, false);
+ vivid_update_format_out(dev);
+
+ /* initialize overlay */
+ dev->fb_cap.fmt.width = dev->src_rect.width;
+ dev->fb_cap.fmt.height = dev->src_rect.height;
+ dev->fb_cap.fmt.pixelformat = dev->fmt_cap->fourcc;
+ dev->fb_cap.fmt.bytesperline = dev->src_rect.width * tpg_g_twopixelsize(&dev->tpg, 0) / 2;
+ dev->fb_cap.fmt.sizeimage = dev->src_rect.height * dev->fb_cap.fmt.bytesperline;
+
+ /* update touch configuration */
+ dev->timeperframe_tch_cap.numerator = 1;
+ dev->timeperframe_tch_cap.denominator = 10;
+ vivid_set_touch(dev, 0);
+
+ /* initialize locks */
+ spin_lock_init(&dev->slock);
+ mutex_init(&dev->mutex);
+
+ /* init dma queues */
+ INIT_LIST_HEAD(&dev->vid_cap_active);
+ INIT_LIST_HEAD(&dev->vid_out_active);
+ INIT_LIST_HEAD(&dev->vbi_cap_active);
+ INIT_LIST_HEAD(&dev->vbi_out_active);
+ INIT_LIST_HEAD(&dev->sdr_cap_active);
+ INIT_LIST_HEAD(&dev->meta_cap_active);
+ INIT_LIST_HEAD(&dev->meta_out_active);
+ INIT_LIST_HEAD(&dev->touch_cap_active);
+
+ INIT_LIST_HEAD(&dev->cec_work_list);
+ spin_lock_init(&dev->cec_slock);
+ /*
+ * Same as create_singlethread_workqueue, but now I can use the
+ * string formatting of alloc_ordered_workqueue.
+ */
+ dev->cec_workqueue = alloc_ordered_workqueue("vivid-%03d-cec",
+ WQ_MEM_RECLAIM, inst);
+ if (!dev->cec_workqueue) {
+ ret = -ENOMEM;
goto unreg_dev;
}
+
+ if (allocators[inst] == 1)
+ dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+
+ ret = vivid_create_queues(dev);
+ if (ret)
+ goto unreg_dev;
+
+#ifdef CONFIG_VIDEO_VIVID_CEC
+ if (dev->has_vid_cap && in_type_counter[HDMI]) {
+ struct cec_adapter *adap;
+
+ adap = vivid_cec_alloc_adap(dev, 0, false);
+ ret = PTR_ERR_OR_ZERO(adap);
+ if (ret < 0)
+ goto unreg_dev;
+ dev->cec_rx_adap = adap;
+ }
+
+ if (dev->has_vid_out) {
+ for (i = 0; i < dev->num_outputs; i++) {
+ struct cec_adapter *adap;
+
+ if (dev->output_type[i] != HDMI)
+ continue;
+
+ dev->cec_output2bus_map[i] = cec_tx_bus_cnt;
+ adap = vivid_cec_alloc_adap(dev, cec_tx_bus_cnt, true);
+ ret = PTR_ERR_OR_ZERO(adap);
+ if (ret < 0) {
+ for (i = 0; i < dev->num_outputs; i++)
+ cec_delete_adapter(dev->cec_tx_adap[i]);
+ goto unreg_dev;
+ }
+
+ dev->cec_tx_adap[cec_tx_bus_cnt] = adap;
+ cec_tx_bus_cnt++;
+ }
+ }
#endif
+ v4l2_ctrl_handler_setup(&dev->ctrl_hdl_vid_cap);
+ v4l2_ctrl_handler_setup(&dev->ctrl_hdl_vid_out);
+ v4l2_ctrl_handler_setup(&dev->ctrl_hdl_vbi_cap);
+ v4l2_ctrl_handler_setup(&dev->ctrl_hdl_vbi_out);
+ v4l2_ctrl_handler_setup(&dev->ctrl_hdl_radio_rx);
+ v4l2_ctrl_handler_setup(&dev->ctrl_hdl_radio_tx);
+ v4l2_ctrl_handler_setup(&dev->ctrl_hdl_sdr_cap);
+ v4l2_ctrl_handler_setup(&dev->ctrl_hdl_meta_cap);
+ v4l2_ctrl_handler_setup(&dev->ctrl_hdl_meta_out);
+ v4l2_ctrl_handler_setup(&dev->ctrl_hdl_touch_cap);
+
+ /* finally start creating the device nodes */
+ ret = vivid_create_devnodes(pdev, dev, inst, cec_tx_bus_cnt,
+ tvnorms_cap, tvnorms_out,
+ in_type_counter, out_type_counter);
+ if (ret)
+ goto unreg_dev;
+
/* Now that everything is fine, let's add it to device list */
vivid_devs[inst] = dev;
return 0;
unreg_dev:
- video_unregister_device(&dev->touch_cap_dev);
- video_unregister_device(&dev->meta_out_dev);
- video_unregister_device(&dev->meta_cap_dev);
+ vb2_video_unregister_device(&dev->touch_cap_dev);
+ vb2_video_unregister_device(&dev->meta_out_dev);
+ vb2_video_unregister_device(&dev->meta_cap_dev);
video_unregister_device(&dev->radio_tx_dev);
video_unregister_device(&dev->radio_rx_dev);
- video_unregister_device(&dev->sdr_cap_dev);
- video_unregister_device(&dev->vbi_out_dev);
- video_unregister_device(&dev->vbi_cap_dev);
- video_unregister_device(&dev->vid_out_dev);
- video_unregister_device(&dev->vid_cap_dev);
+ vb2_video_unregister_device(&dev->sdr_cap_dev);
+ vb2_video_unregister_device(&dev->vbi_out_dev);
+ vb2_video_unregister_device(&dev->vbi_cap_dev);
+ vb2_video_unregister_device(&dev->vid_out_dev);
+ vb2_video_unregister_device(&dev->vid_cap_dev);
cec_unregister_adapter(dev->cec_rx_adap);
for (i = 0; i < MAX_OUTPUTS; i++)
cec_unregister_adapter(dev->cec_tx_adap[i]);
if (dev->has_vid_cap) {
v4l2_info(&dev->v4l2_dev, "unregistering %s\n",
video_device_node_name(&dev->vid_cap_dev));
- video_unregister_device(&dev->vid_cap_dev);
+ vb2_video_unregister_device(&dev->vid_cap_dev);
}
if (dev->has_vid_out) {
v4l2_info(&dev->v4l2_dev, "unregistering %s\n",
video_device_node_name(&dev->vid_out_dev));
- video_unregister_device(&dev->vid_out_dev);
+ vb2_video_unregister_device(&dev->vid_out_dev);
}
if (dev->has_vbi_cap) {
v4l2_info(&dev->v4l2_dev, "unregistering %s\n",
video_device_node_name(&dev->vbi_cap_dev));
- video_unregister_device(&dev->vbi_cap_dev);
+ vb2_video_unregister_device(&dev->vbi_cap_dev);
}
if (dev->has_vbi_out) {
v4l2_info(&dev->v4l2_dev, "unregistering %s\n",
video_device_node_name(&dev->vbi_out_dev));
- video_unregister_device(&dev->vbi_out_dev);
+ vb2_video_unregister_device(&dev->vbi_out_dev);
}
if (dev->has_sdr_cap) {
v4l2_info(&dev->v4l2_dev, "unregistering %s\n",
video_device_node_name(&dev->sdr_cap_dev));
- video_unregister_device(&dev->sdr_cap_dev);
+ vb2_video_unregister_device(&dev->sdr_cap_dev);
}
if (dev->has_radio_rx) {
v4l2_info(&dev->v4l2_dev, "unregistering %s\n",
if (dev->has_meta_cap) {
v4l2_info(&dev->v4l2_dev, "unregistering %s\n",
video_device_node_name(&dev->meta_cap_dev));
- video_unregister_device(&dev->meta_cap_dev);
+ vb2_video_unregister_device(&dev->meta_cap_dev);
}
if (dev->has_meta_out) {
v4l2_info(&dev->v4l2_dev, "unregistering %s\n",
video_device_node_name(&dev->meta_out_dev));
- video_unregister_device(&dev->meta_out_dev);
+ vb2_video_unregister_device(&dev->meta_out_dev);
}
if (dev->has_touch_cap) {
v4l2_info(&dev->v4l2_dev, "unregistering %s\n",
video_device_node_name(&dev->touch_cap_dev));
- video_unregister_device(&dev->touch_cap_dev);
+ vb2_video_unregister_device(&dev->touch_cap_dev);
}
cec_unregister_adapter(dev->cec_rx_adap);
for (j = 0; j < MAX_OUTPUTS; j++)
{
struct vivid_meta_out_buf *meta = vb2_plane_vaddr(&buf->vb.vb2_buf, 0);
- tpg_s_brightness(&dev->tpg, meta->brightness);
- tpg_s_contrast(&dev->tpg, meta->contrast);
- tpg_s_saturation(&dev->tpg, meta->saturation);
- tpg_s_hue(&dev->tpg, meta->hue);
+ v4l2_ctrl_s_ctrl(dev->brightness, meta->brightness);
+ v4l2_ctrl_s_ctrl(dev->contrast, meta->contrast);
+ v4l2_ctrl_s_ctrl(dev->saturation, meta->saturation);
+ v4l2_ctrl_s_ctrl(dev->hue, meta->hue);
+
dprintk(dev, 2, " %s brightness %u contrast %u saturation %u hue %d\n",
__func__, meta->brightness, meta->contrast,
meta->saturation, meta->hue);
switch (frame) {
case 0:
vivid_vbi_gen_set_time_of_day(vbi->time_of_day_packet);
- /* fall through */
+ fallthrough;
case 1 ... 7:
data1->data[0] = vbi->time_of_day_packet[frame * 2];
data1->data[1] = vbi->time_of_day_packet[frame * 2 + 1];
unsigned factor = 1;
unsigned w, h;
unsigned p;
+ bool user_set_csc = !!(mp->flags & V4L2_PIX_FMT_FLAG_SET_CSC);
fmt = vivid_get_format(dev, mp->pixelformat);
if (!fmt) {
(fmt->bit_depth[p] / fmt->vdownsampling[p])) /
(fmt->bit_depth[0] / fmt->vdownsampling[0]);
- mp->colorspace = vivid_colorspace_cap(dev);
- if (fmt->color_enc == TGP_COLOR_ENC_HSV)
- mp->hsv_enc = vivid_hsv_enc_cap(dev);
- else
+ if (!user_set_csc || !v4l2_is_colorspace_valid(mp->colorspace))
+ mp->colorspace = vivid_colorspace_cap(dev);
+
+ if (!user_set_csc || !v4l2_is_xfer_func_valid(mp->xfer_func))
+ mp->xfer_func = vivid_xfer_func_cap(dev);
+
+ if (fmt->color_enc == TGP_COLOR_ENC_HSV) {
+ if (!user_set_csc || !v4l2_is_hsv_enc_valid(mp->hsv_enc))
+ mp->hsv_enc = vivid_hsv_enc_cap(dev);
+ } else if (fmt->color_enc == TGP_COLOR_ENC_YCBCR) {
+ if (!user_set_csc || !v4l2_is_ycbcr_enc_valid(mp->ycbcr_enc))
+ mp->ycbcr_enc = vivid_ycbcr_enc_cap(dev);
+ } else {
mp->ycbcr_enc = vivid_ycbcr_enc_cap(dev);
- mp->xfer_func = vivid_xfer_func_cap(dev);
- mp->quantization = vivid_quantization_cap(dev);
+ }
+
+ if (fmt->color_enc == TGP_COLOR_ENC_YCBCR ||
+ fmt->color_enc == TGP_COLOR_ENC_RGB) {
+ if (!user_set_csc || !v4l2_is_quant_valid(mp->quantization))
+ mp->quantization = vivid_quantization_cap(dev);
+ } else {
+ mp->quantization = vivid_quantization_cap(dev);
+ }
+
memset(mp->reserved, 0, sizeof(mp->reserved));
return 0;
}
if (vivid_is_sdtv_cap(dev))
dev->tv_field_cap = mp->field;
tpg_update_mv_step(&dev->tpg);
+ dev->tpg.colorspace = mp->colorspace;
+ dev->tpg.xfer_func = mp->xfer_func;
+ if (dev->fmt_cap->color_enc == TGP_COLOR_ENC_YCBCR)
+ dev->tpg.ycbcr_enc = mp->ycbcr_enc;
+ else
+ dev->tpg.hsv_enc = mp->hsv_enc;
+ dev->tpg.quantization = mp->quantization;
+
return 0;
}
fmt = &vivid_formats[f->index];
f->pixelformat = fmt->fourcc;
+
+ if (f->type != V4L2_BUF_TYPE_VIDEO_CAPTURE &&
+ f->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)
+ return 0;
+ /*
+ * For capture devices, we support the CSC API.
+ * We allow userspace to:
+ * 1. set the colorspace
+ * 2. set the xfer_func
+ * 3. set the ycbcr_enc on YUV formats
+ * 4. set the hsv_enc on HSV formats
+ * 5. set the quantization on YUV and RGB formats
+ */
+ f->flags |= V4L2_FMT_FLAG_CSC_COLORSPACE;
+ f->flags |= V4L2_FMT_FLAG_CSC_XFER_FUNC;
+
+ if (fmt->color_enc == TGP_COLOR_ENC_YCBCR) {
+ f->flags |= V4L2_FMT_FLAG_CSC_YCBCR_ENC;
+ f->flags |= V4L2_FMT_FLAG_CSC_QUANTIZATION;
+ } else if (fmt->color_enc == TGP_COLOR_ENC_HSV) {
+ f->flags |= V4L2_FMT_FLAG_CSC_HSV_ENC;
+ } else if (fmt->color_enc == TGP_COLOR_ENC_RGB) {
+ f->flags |= V4L2_FMT_FLAG_CSC_QUANTIZATION;
+ }
+
return 0;
}
dev_warn(&priv->i2c->dev, "Unsupported bandwidth %u kHz. Using 6000 kHz.\n",
bandwidth);
bandwidth = 6000;
- /* fallthrough */
+ fallthrough;
case 6000:
regs[FC11_REG_VCOSEL] |= FC11_VCOSEL_BW6M;
break;
{ QT1010_WR, reg, reg_init_val },
{ QT1010_WR, 0x1e, 0x00 },
{ QT1010_WR, 0x1e, oper },
- { QT1010_RD, reg, 0xff }
};
for (i = 0; i < ARRAY_SIZE(i2c_data); i++) {
- if (i2c_data[i].oper == QT1010_WR) {
- err = qt1010_writereg(priv, i2c_data[i].reg,
- i2c_data[i].val);
- } else {
- err = qt1010_readreg(priv, i2c_data[i].reg, &val2);
- }
- if (err) return err;
+ err = qt1010_writereg(priv, i2c_data[i].reg,
+ i2c_data[i].val);
+ if (err)
+ return err;
}
+ err = qt1010_readreg(priv, reg, &val2);
+ if (err)
+ return err;
do {
val1 = val2;
err = qt1010_readreg(priv, reg, &val2);
- if (err) return err;
+ if (err)
+ return err;
+
dev_dbg(&priv->i2c->dev, "%s: compare reg:%02x %02x %02x\n",
__func__, reg, val1, val2);
} while (val1 != val2);
static int qt1010_init_meas2(struct qt1010_priv *priv,
u8 reg_init_val, u8 *retval)
{
- u8 i, val;
+ u8 i, val = 0xff;
int err;
qt1010_i2c_oper_t i2c_data[] = {
{ QT1010_WR, 0x07, reg_init_val },
{ QT1010_WR, 0x1e, 0x00 },
{ QT1010_WR, 0x22, 0xff }
};
+
for (i = 0; i < ARRAY_SIZE(i2c_data); i++) {
if (i2c_data[i].oper == QT1010_WR) {
err = qt1010_writereg(priv, i2c_data[i].reg,
} else {
err = qt1010_readreg(priv, i2c_data[i].reg, &val);
}
- if (err) return err;
+ if (err)
+ return err;
}
*retval = val;
return 0;
break;
case SYS_DVBC_ANNEX_B:
bw = 6000000;
- /* fall through */
+ fallthrough;
case SYS_DVBC_ANNEX_A:
case SYS_DVBC_ANNEX_C:
if (bw <= 6000000) {
case TUNER_TENA_9533_DI:
case TUNER_YMEC_TVF_5533MF:
tuner_dbg("This tuner doesn't have FM. Most cards have a TEA5767 for FM\n");
- return 0;
+ return -EINVAL;
case TUNER_PHILIPS_FM1216ME_MK3:
case TUNER_PHILIPS_FM1236_MK3:
case TUNER_PHILIPS_FMD1216ME_MK3:
TUNER_RATIO_SELECT_50; /* 50 kHz step */
/* Bandswitch byte */
- simple_radio_bandswitch(fe, &buffer[0]);
+ if (simple_radio_bandswitch(fe, &buffer[0]))
+ return 0;
/* Convert from 1/16 kHz V4L steps to 1/20 MHz (=50 kHz) PLL steps
freq * (1 Mhz / 16000 V4L steps) * (20 PLL steps / 1 MHz) =
/* Remote Controller time units */
-#define AU8522_UNIT 200000 /* ns */
-#define NEC_START_SPACE (4500000 / AU8522_UNIT)
-#define NEC_START_PULSE (562500 * 16)
+#define AU8522_UNIT 200 /* us */
+#define NEC_START_SPACE (4500 / AU8522_UNIT)
+#define NEC_START_PULSE (563 * 16)
#define RC5_START_SPACE (4 * AU8522_UNIT)
-#define RC5_START_PULSE 888888
+#define RC5_START_PULSE 889
static int au0828_get_key_au8522(struct au0828_rc *ir)
{
return 0;
mutex_lock(&au0828_sysfs_lock);
- video_unregister_device(&dev->vdev);
- video_unregister_device(&dev->vbi_dev);
+ vb2_video_unregister_device(&dev->vdev);
+ vb2_video_unregister_device(&dev->vbi_dev);
mutex_unlock(&au0828_sysfs_lock);
v4l2_device_disconnect(&dev->v4l2_dev);
if (retval != 0) {
dprintk(1, "unable to register video device (error = %d).\n",
retval);
- ret = -ENODEV;
- goto err_reg_vdev;
+ return -ENODEV;
}
/* Register the vbi device */
return 0;
err_reg_vbi_dev:
- video_unregister_device(&dev->vdev);
-err_reg_vdev:
- vb2_queue_release(&dev->vb_vidq);
- vb2_queue_release(&dev->vb_vbiq);
+ vb2_video_unregister_device(&dev->vdev);
return ret;
}
usb_free_urb(fc_usb->iso_urb[i]);
}
- if (fc_usb->iso_buffer != NULL)
- usb_free_coherent(fc_usb->udev,
- fc_usb->buffer_size, fc_usb->iso_buffer,
- fc_usb->dma_addr);
+ usb_free_coherent(fc_usb->udev, fc_usb->buffer_size,
+ fc_usb->iso_buffer, fc_usb->dma_addr);
+
}
static int flexcop_usb_transfer_init(struct flexcop_usb *fc_usb)
if (fc_usb->uintf->cur_altsetting->desc.bNumEndpoints < 1)
return -ENODEV;
+ if (!usb_endpoint_is_isoc_in(&fc_usb->uintf->cur_altsetting->endpoint[1].desc))
+ return -ENODEV;
switch (fc_usb->udev->speed) {
case USB_SPEED_LOW:
#define B2C2_USB_CTRL_PIPE_IN usb_rcvctrlpipe(fc_usb->udev, 0)
#define B2C2_USB_CTRL_PIPE_OUT usb_sndctrlpipe(fc_usb->udev, 0)
-#define B2C2_USB_DATA_PIPE usb_rcvisocpipe(fc_usb->udev, 0x81)
+#define B2C2_USB_DATA_PIPE usb_rcvisocpipe(fc_usb->udev, 1)
struct flexcop_usb {
struct usb_device *udev;
struct cx231xx_ops *ops = NULL;
mutex_lock(&cx231xx_devlist_mutex);
- if (!list_empty(&cx231xx_extension_devlist)) {
- list_for_each_entry(ops, &cx231xx_extension_devlist, next) {
- if (ops->init)
- ops->init(dev);
- }
+ list_for_each_entry(ops, &cx231xx_extension_devlist, next) {
+ if (ops->init)
+ ops->init(dev);
}
mutex_unlock(&cx231xx_devlist_mutex);
}
struct cx231xx_ops *ops = NULL;
mutex_lock(&cx231xx_devlist_mutex);
- if (!list_empty(&cx231xx_extension_devlist)) {
- list_for_each_entry(ops, &cx231xx_extension_devlist, next) {
- if (ops->fini)
- ops->fini(dev);
- }
+ list_for_each_entry(ops, &cx231xx_extension_devlist, next) {
+ if (ops->fini)
+ ops->fini(dev);
}
mutex_unlock(&cx231xx_devlist_mutex);
}
case READ_I2C:
write = 0;
state->buf[2] |= 0x01; /* set I2C direction */
- /* fall through */
+ fallthrough;
case WRITE_I2C:
state->buf[0] = READ_WRITE_I2C;
break;
switch (request) {
case CMD_WRITE:
memcpy(ctx->buf, data, size);
- /* Fall through */
+ fallthrough;
case CMD_WRITE_SHORT:
pipe = usb_sndctrlpipe(d->udev, 0);
requesttype = USB_TYPE_VENDOR | USB_DIR_OUT;
cold = 0;
break;
}
- /* fall through */
+ fallthrough;
case TUNER_LG:
fw_lme = fw_lg;
ret = request_firmware(&fw, fw_lme, &udev->dev);
cold = 0;
break;
}
- /* fall through */
+ fallthrough;
case TUNER_LG:
fw_lme = fw_c_lg;
ret = request_firmware(&fw, fw_lme, &udev->dev);
st->dvb_usb_lme2510_firmware = TUNER_LG;
break;
}
- /* fall through */
+ fallthrough;
case TUNER_S0194:
fw_lme = fw_c_s0194;
ret = request_firmware(&fw, fw_lme, &udev->dev);
}
break;
}
- /* fall through */
+ fallthrough;
case 0x22f0:
st->i2c_gate = 5;
adap->fe[0] = dvb_attach(m88rs2000_attach,
default:
mxl_printk(KERN_ERR,
"gpio_port_expander undefined, assuming PCA9534");
- /* fall-thru */
+ fallthrough;
case mxl111sf_PCA9534:
return pca9534_set_gpio(state, gpio, val);
case mxl111sf_gpio_hw:
default:
mxl_printk(KERN_ERR,
"gpio_port_expander undefined, assuming PCA9534");
- /* fall-thru */
+ fallthrough;
case mxl111sf_PCA9534:
return pca9534_init_port_expander(state);
case mxl111sf_gpio_hw:
/* pass data to Kernel IR decoder */
for (i = 0; i < len; i++) {
ev.pulse = buf[i] >> 7;
- ev.duration = 50800 * (buf[i] & 0x7f);
+ ev.duration = 51 * (buf[i] & 0x7f);
ir_raw_event_store_with_filter(d->rc_dev, &ev);
}
rc->query = rtl2832u_rc_query;
rc->interval = 200;
/* we program idle len to 0xc0, set timeout to one less */
- rc->timeout = 0xbf * 50800;
+ rc->timeout = 0xbf * 51;
return 0;
}
tristate "Pinnacle PCTV HDTV Pro USB device/TT Connect S2-3600"
depends on DVB_USB
select TTPCI_EEPROM
+ select DVB_ISL6423 if MEDIA_SUBDRV_AUTOSELECT
select DVB_LNBP22 if MEDIA_SUBDRV_AUTOSELECT
select DVB_STB0899 if MEDIA_SUBDRV_AUTOSELECT
select DVB_STB6100 if MEDIA_SUBDRV_AUTOSELECT
cxusb_vprintk(dvbdev, OPS, "video device release\n");
- vb2_queue_release(vdev->queue);
-
video_device_release(vdev);
}
cxdev->videodev = video_device_alloc();
if (!cxdev->videodev) {
dev_err(&dvbdev->udev->dev, "video device alloc failed\n");
- ret = -ENOMEM;
- goto ret_qrelease;
+ return -ENOMEM;
}
cxdev->videodev->device_caps = videocaps;
ret_vrelease:
video_device_release(cxdev->videodev);
-
-ret_qrelease:
- vb2_queue_release(&cxdev->videoqueue);
-
return ret;
}
return 0;
ret_vunreg:
- video_unregister_device(cxdev->videodev);
+ vb2_video_unregister_device(cxdev->videodev);
ret_unregister:
v4l2_device_put(&cxdev->v4l2dev);
cxusb_vprintk(dvbdev, OPS, "unregistering analog\n");
video_unregister_device(cxdev->radiodev);
- video_unregister_device(cxdev->videodev);
+ vb2_video_unregister_device(cxdev->videodev);
v4l2_device_put(&cxdev->v4l2dev);
wait_for_completion(&cxdev->v4l2_release);
struct dib0700_adapter_state *st = adap->priv;
struct i2c_adapter *tun_i2c = st->dib8000_ops.get_i2c_master(adap->fe_adap[0].fe, DIBX000_I2C_INTERFACE_TUNER, 1);
- if (adap->id == 0) {
- if (dvb_attach(dib0090_register, adap->fe_adap[0].fe, tun_i2c, &dib809x_dib0090_config) == NULL)
- return -ENODEV;
- } else {
- /* FIXME: check if it is fe_adap[1] */
- if (dvb_attach(dib0090_register, adap->fe_adap[0].fe, tun_i2c, &dib809x_dib0090_config) == NULL)
- return -ENODEV;
- }
+ /* FIXME: if adap->id != 0, check if it is fe_adap[1] */
+ if (!dvb_attach(dib0090_register, adap->fe_adap[0].fe, tun_i2c, &dib809x_dib0090_config))
+ return -ENODEV;
st->set_param_save = adap->fe_adap[0].fe->ops.tuner_ops.set_params;
adap->fe_adap[0].fe->ops.tuner_ops.set_params = dib8096_set_param_override;
#include "stb6100.h"
#include "stb6100_cfg.h"
/* FE Power */
+#include "isl6423.h"
#include "lnbp22.h"
#include <media/dvb_ca_en50221.h>
{ 0, 0 }
};
+static struct isl6423_config pctv452e_isl6423_config = {
+ .current_max = SEC_CURRENT_515m,
+ .curlim = SEC_CURRENT_LIM_ON,
+ .mod_extern = 1,
+ .addr = 0x08,
+};
+
/*
* stores all private variables for communication with the PCTV452e DVB-S2
*/
&a->dev->i2c_adap);
if (!a->fe_adap[0].fe)
return -ENODEV;
- if ((dvb_attach(lnbp22_attach, a->fe_adap[0].fe,
- &a->dev->i2c_adap)) == NULL)
- err("Cannot attach lnbp22\n");
id = a->dev->desc->warm_ids[0];
- if (USB_VID_TECHNOTREND == id->idVendor
- && USB_PID_TECHNOTREND_CONNECT_S2_3650_CI == id->idProduct)
+ if (id->idVendor == USB_VID_TECHNOTREND &&
+ id->idProduct == USB_PID_TECHNOTREND_CONNECT_S2_3650_CI) {
+ if (dvb_attach(lnbp22_attach,
+ a->fe_adap[0].fe,
+ &a->dev->i2c_adap) == NULL) {
+ err("Cannot attach lnbp22\n");
+ }
/* Error ignored. */
tt3650_ci_init(a);
+ } else if (dvb_attach(isl6423_attach,
+ a->fe_adap[0].fe,
+ &a->dev->i2c_adap,
+ &pctv452e_isl6423_config) == NULL) {
+ err("Cannot attach isl6423\n");
+ }
return 0;
}
for (i = 1; i < ARRAY_SIZE(state->buf); i++) {
if (buf[i] == 0xff) {
ev.pulse = 0;
- ev.duration = 888888*2;
+ ev.duration = 889 * 2;
ir_raw_event_store(d->rc_dev, &ev);
break;
}
ev.pulse = !ev.pulse;
ev.duration = (buf[i] * FIRMWARE_CLOCK_DIVISOR *
- FIRMWARE_CLOCK_TICK) / 1000;
+ FIRMWARE_CLOCK_TICK) / (1000 * 1000);
ir_raw_event_store(d->rc_dev, &ev);
}
return -ENODEV;
switch (cmd) {
- case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: /* fall through */
- case SNDRV_PCM_TRIGGER_RESUME: /* fall through */
+ case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+ case SNDRV_PCM_TRIGGER_RESUME:
case SNDRV_PCM_TRIGGER_START:
atomic_set(&dev->adev.stream_started, 1);
break;
- case SNDRV_PCM_TRIGGER_PAUSE_PUSH: /* fall through */
- case SNDRV_PCM_TRIGGER_SUSPEND: /* fall through */
+ case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+ case SNDRV_PCM_TRIGGER_SUSPEND:
case SNDRV_PCM_TRIGGER_STOP:
atomic_set(&dev->adev.stream_started, 0);
break;
.amux = EM28XX_AMUX_LINE_IN,
} },
},
+ /*
+ * 1f4d:1abe MyGica iGrabber
+ * (same as several other EM2860 devices)
+ * Empia EM2860, Philips SAA7113, Empia EMP202, No Tuner
+ */
+ [EM2860_BOARD_MYGICA_IGRABBER] = {
+ .name = "MyGica iGrabber",
+ .vchannels = 2,
+ .tuner_type = TUNER_ABSENT,
+ .decoder = EM28XX_SAA711X,
+ .input = { {
+ .type = EM28XX_VMUX_COMPOSITE,
+ .vmux = SAA7115_COMPOSITE0,
+ .amux = EM28XX_AMUX_LINE_IN,
+ }, {
+ .type = EM28XX_VMUX_SVIDEO,
+ .vmux = SAA7115_SVIDEO3,
+ .amux = EM28XX_AMUX_LINE_IN,
+ } },
+ },
};
EXPORT_SYMBOL_GPL(em28xx_boards);
.driver_info = EM2860_BOARD_EASYCAP },
{ USB_DEVICE(0x1b80, 0xe425),
.driver_info = EM2874_BOARD_MAXMEDIA_UB425_TC },
+ { USB_DEVICE(0x1f4d, 0x1abe),
+ .driver_info = EM2860_BOARD_MYGICA_IGRABBER },
{ USB_DEVICE(0x2304, 0x0242),
.driver_info = EM2884_BOARD_PCTV_510E },
{ USB_DEVICE(0x2013, 0x0251),
#define EM2882_BOARD_ZOLID_HYBRID_TV_STICK 102
#define EM2861_BOARD_MAGIX_VIDEOWANDLER2 103
#define EM28178_BOARD_PCTV_461E_V2 104
+#define EM2860_BOARD_MYGICA_IGRABBER 105
/* Limits minimum and default number of buffers */
#define EM28XX_MIN_BUF 4
case 0xD8:
if (go->format == V4L2_PIX_FMT_MJPEG)
vb = frame_boundary(go, vb);
- /* fall through */
+ fallthrough;
default:
store_byte(vb, 0xFF);
store_byte(vb, buf[i]);
switch (gspca_dev->pixfmt.width) {
case 160:
data[9] |= 0x04; /* reg 8, 2:1 scale down from 320 */
- /* fall through */
+ fallthrough;
case 320:
default:
data[3] = 0x28; /* reg 2, H size/8 */
break;
case 176:
data[9] |= 0x04; /* reg 8, 2:1 scale down from 352 */
- /* fall through */
+ fallthrough;
case 352:
data[3] = 0x2c; /* reg 2, H size/8 */
data[4] = 0x48; /* reg 3, V size/4 */
switch (gspca_dev->pixfmt.width) {
case 160:
data[9] |= 0x0c; /* reg 8, 4:1 scale down */
- /* fall through */
+ fallthrough;
case 320:
data[9] |= 0x04; /* reg 8, 2:1 scale down */
- /* fall through */
+ fallthrough;
case 640:
default:
data[3] = 0x50; /* reg 2, H size/8 */
case 176:
data[9] |= 0x04; /* reg 8, 2:1 scale down */
- /* fall through */
+ fallthrough;
case 352:
data[3] = 0x2c; /* reg 2, H size */
data[4] = 0x48; /* reg 3, V size */
V4L2_CID_AUTOGAIN, 0, 1, 1, 1);
gspca_dev->gain = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
V4L2_CID_GAIN, 0, 253, 1, 128);
- /* fall through */
+ fallthrough;
case Cvideopro:
case DvcV6:
case Kritter:
break;
case BRIDGE_OVFX2:
req = 0x0a;
- /* fall through */
+ fallthrough;
case BRIDGE_W9968CF:
gspca_dbg(gspca_dev, D_USBO, "SET %02x %04x %04x\n",
req, value, index);
case SEN_OV76BE:
if (sd->gspca_dev.pixfmt.width == 320)
interlaced = 1;
- /* Fall through */
+ fallthrough;
case SEN_OV6630:
case SEN_OV7610:
case SEN_OV7670:
break;
}
/* For 640x480 case */
- /* fall through */
+ fallthrough;
default:
/* case 20: */
/* case 15: */
break;
case SENSOR_HV7131R:
sd->i2c_intf = 0x81; /* i2c 400 Kb/s */
- /* fall through */
+ fallthrough;
default:
cam->cam_mode = vga_mode;
cam->nmodes = ARRAY_SIZE(vga_mode);
case BRIDGE_SPCA504:
case BRIDGE_SPCA504C:
pollreg = 0;
- /* fall through */
+ fallthrough;
default:
/* case BRIDGE_SPCA533: */
/* case BRIDGE_SPCA504B: */
reg_w_riv(gspca_dev, 0x00, 0x2000, 0x00);
reg_w_riv(gspca_dev, 0x00, 0x2301, 0x13);
reg_w_riv(gspca_dev, 0x00, 0x2306, 0x00);
- /* fall through */
+ fallthrough;
case BRIDGE_SPCA533:
spca504B_PollingDataReady(gspca_dev);
spca50x_GetFirmware(gspca_dev);
case CIT_MODEL0:
case CIT_MODEL1:
cit_write_reg(gspca_dev, 0x0001, 0x0114);
- /* Fall through */
+ fallthrough;
case CIT_MODEL2:
case CIT_MODEL4:
cit_write_reg(gspca_dev, 0x00c0, 0x010c); /* Go! */
break;
case CIT_MODEL2:
v4l2_ctrl_grab(sd->lighting, false);
- /* Fall through! */
+ fallthrough;
case CIT_MODEL4:
cit_model2_Packet1(gspca_dev, 0x0030, 0x0004);
case SENSOR_HV7131R:
case SENSOR_TAS5130C:
reg_r(gspca_dev, 0x0008);
- /* fall through */
+ fallthrough;
case SENSOR_PO2030:
reg_w(gspca_dev, 0x03, 0x0008);
break;
case SENSOR_TAS5130C:
reg_w(gspca_dev, 0x09, 0x01ad); /* (from win traces) */
reg_w(gspca_dev, 0x15, 0x01ae);
- /* fall through */
+ fallthrough;
case SENSOR_PAS202B:
case SENSOR_PO2030:
/* reg_w(gspca_dev, 0x40, ZC3XX_R117_GGAIN); in win traces */
const char *bufPtr,unsigned int bufSize,
int *mskp,int *valp)
{
- int ret;
v4l2_std_id id;
- ret = pvr2_std_str_to_id(&id,bufPtr,bufSize);
- if (ret < 0) return ret;
+ if (!pvr2_std_str_to_id(&id, bufPtr, bufSize))
+ return -EINVAL;
if (mskp) *mskp = id;
if (valp) *valp = id;
return 0;
if (!DEVICE_USE_CODEC3(pdev->type))
break;
/* For CODEC3 where autogain also controls expo */
- /* fall through */
+ fallthrough;
case V4L2_CID_EXPOSURE_AUTO:
if (pdev->exposure_valid && time_before(jiffies,
pdev->last_exposure_update + HZ / 4)) {
break;
case SMS_UNKNOWN_TYPE:
pr_err("Unspecified sms device type!\n");
- /* fall-thru */
+ fallthrough;
default:
dev->buffer_size = USB2_BUFFER_SIZE;
dev->response_alignment = align;
int err = 0;
switch (cmd) {
- case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: /* fall through */
- case SNDRV_PCM_TRIGGER_RESUME: /* fall through */
+ case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+ case SNDRV_PCM_TRIGGER_RESUME:
case SNDRV_PCM_TRIGGER_START:
atomic_set(&core->stream_started, 1);
break;
- case SNDRV_PCM_TRIGGER_PAUSE_PUSH: /* fall through */
- case SNDRV_PCM_TRIGGER_SUSPEND: /* fall through */
+ case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+ case SNDRV_PCM_TRIGGER_SUSPEND:
case SNDRV_PCM_TRIGGER_STOP:
atomic_set(&core->stream_started, 0);
break;
/* FIXME: tm6000_extension_devlist_lock should be a spinlock */
- if (!list_empty(&tm6000_extension_devlist)) {
- list_for_each_entry(ops, &tm6000_extension_devlist, next) {
- if (ops->fillbuf && ops->type == type)
- ops->fillbuf(dev, buf, size);
- }
+ list_for_each_entry(ops, &tm6000_extension_devlist, next) {
+ if (ops->fillbuf && ops->type == type)
+ ops->fillbuf(dev, buf, size);
}
return 0;
struct tm6000_ops *ops = NULL;
mutex_lock(&tm6000_devlist_mutex);
- if (!list_empty(&tm6000_extension_devlist)) {
- list_for_each_entry(ops, &tm6000_extension_devlist, next) {
- if (ops->init)
- ops->init(dev);
- }
+ list_for_each_entry(ops, &tm6000_extension_devlist, next) {
+ if (ops->init)
+ ops->init(dev);
}
mutex_unlock(&tm6000_devlist_mutex);
}
struct tm6000_ops *ops = NULL;
mutex_lock(&tm6000_devlist_mutex);
- if (!list_empty(&tm6000_extension_devlist)) {
- list_for_each_entry(ops, &tm6000_extension_devlist, next) {
- if (ops->fini)
- ops->fini(dev);
- }
+ list_for_each_entry(ops, &tm6000_extension_devlist, next) {
+ if (ops->fini)
+ ops->fini(dev);
}
mutex_unlock(&tm6000_devlist_mutex);
}
* Copyright (c) 2002 Holger Waechtler <holger@convergence.de>
* Copyright (c) 2003 Felix Domke <tmbinc@elitedvb.net>
*/
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/wait.h>
DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr);
-#define dprintk(x...) do { if (debug) printk(KERN_DEBUG x); } while (0)
+#define dprintk(fmt, arg...) do { \
+ if (debug) \
+ printk(KERN_DEBUG pr_fmt("%s: " fmt), \
+ __func__, ##arg); \
+} while (0)
+
#define ISO_BUF_COUNT 4
#define FRAMES_PER_ISO_BUF 4
#define TTUSB_REV_2_2 0x22
#define TTUSB_BUDGET_NAME "ttusb_stc_fw"
+#define MAX_SEND 0x28
+#define MAX_RCV 0x20
+
/*
* since we're casting (struct ttusb*) <-> (struct dvb_demux*) around
* the dvb_demux field must be the first in struct!!
int cc; /* MuxCounter - will increment on EVERY MUX PACKET */
/* (including stuffing. yes. really.) */
- u8 last_result[32];
+ u8 send_buf[MAX_SEND];
+ u8 last_result[MAX_RCV];
int revision;
struct dvb_frontend* fe;
};
-/* ugly workaround ... don't know why it's necessary to read */
-/* all result codes. */
-
-static int ttusb_cmd(struct ttusb *ttusb,
- const u8 * data, int len, int needresult)
+static int ttusb_cmd(struct ttusb *ttusb, u8 *data, int len, int len_result)
{
int actual_len;
int err;
- int i;
-
- if (debug >= 3) {
- printk(KERN_DEBUG ">");
- for (i = 0; i < len; ++i)
- printk(KERN_CONT " %02x", data[i]);
- printk(KERN_CONT "\n");
- }
if (mutex_lock_interruptible(&ttusb->semusb) < 0)
return -EAGAIN;
+ if (debug >= 3)
+ dprintk("> %*ph\n", len, data);
+
+ memcpy(data, ttusb->send_buf, len);
+
err = usb_bulk_msg(ttusb->dev, ttusb->bulk_out_pipe,
- (u8 *) data, len, &actual_len, 1000);
+ ttusb->send_buf, len, &actual_len, 1000);
if (err != 0) {
- dprintk("%s: usb_bulk_msg(send) failed, err == %i!\n",
- __func__, err);
- mutex_unlock(&ttusb->semusb);
- return err;
+ dprintk("usb_bulk_msg(send) failed, err == %i!\n", err);
+ goto err;
}
if (actual_len != len) {
- dprintk("%s: only wrote %d of %d bytes\n", __func__,
+ err = -EIO;
+ dprintk("only wrote %d of %d bytes\n",
actual_len, len);
- mutex_unlock(&ttusb->semusb);
- return -1;
+ goto err;
}
err = usb_bulk_msg(ttusb->dev, ttusb->bulk_in_pipe,
- ttusb->last_result, 32, &actual_len, 1000);
+ ttusb->last_result, MAX_RCV, &actual_len, 1000);
if (err != 0) {
- printk("%s: failed, receive error %d\n", __func__,
- err);
- mutex_unlock(&ttusb->semusb);
- return err;
+ pr_err("cmd xter failed, receive error %d\n", err);
+ goto err;
}
if (debug >= 3) {
actual_len = ttusb->last_result[3] + 4;
- printk(KERN_DEBUG "<");
- for (i = 0; i < actual_len; ++i)
- printk(KERN_CONT " %02x", ttusb->last_result[i]);
- printk(KERN_CONT "\n");
+ dprintk("< %*ph\n", actual_len, ttusb->last_result);
}
- if (!needresult)
- mutex_unlock(&ttusb->semusb);
- return 0;
-}
+ if (len_result)
+ memcpy(ttusb->send_buf, ttusb->last_result, len_result);
-static int ttusb_result(struct ttusb *ttusb, u8 * data, int len)
-{
- memcpy(data, ttusb->last_result, len);
+err:
mutex_unlock(&ttusb->semusb);
- return 0;
+ return err;
}
static int ttusb_i2c_msg(struct ttusb *ttusb,
u8 addr, u8 * snd_buf, u8 snd_len, u8 * rcv_buf,
u8 rcv_len)
{
- u8 b[0x28];
+ u8 b[MAX_SEND];
u8 id = ++ttusb->c;
int i, err;
- if (snd_len > 0x28 - 7 || rcv_len > 0x20 - 7)
+ if (snd_len > MAX_SEND - 7 || rcv_len > MAX_RCV - 7)
return -EINVAL;
b[0] = 0xaa;
for (i = 0; i < snd_len; i++)
b[7 + i] = snd_buf[i];
- err = ttusb_cmd(ttusb, b, snd_len + 7, 1);
+ err = ttusb_cmd(ttusb, b, snd_len + 7, MAX_RCV);
if (err)
return -EREMOTEIO;
- err = ttusb_result(ttusb, b, 0x20);
-
/* check if the i2c transaction was successful */
if ((snd_len != b[5]) || (rcv_len != b[6])) return -EREMOTEIO;
if (rcv_len > 0) {
if (err || b[0] != 0x55 || b[1] != id) {
- dprintk
- ("%s: usb_bulk_msg(recv) failed, err == %i, id == %02x, b == ",
- __func__, err, id);
+ dprintk("usb_bulk_msg(recv) failed, err == %i, id == %02x, b == ",
+ err, id);
return -EREMOTEIO;
}
snd_buf, snd_len, rcv_buf, rcv_len);
if (err < rcv_len) {
- dprintk("%s: i == %i\n", __func__, i);
+ dprintk("i == %i\n", i);
break;
}
err = request_firmware(&fw, "ttusb-budget/dspbootcode.bin",
&ttusb->dev->dev);
if (err) {
- printk(KERN_ERR "ttusb-budget: failed to request firmware\n");
+ pr_err("failed to request firmware\n");
return err;
}
done:
release_firmware(fw);
if (err) {
- dprintk("%s: usb_bulk_msg() failed, return value %i!\n",
- __func__, err);
+ dprintk("usb_bulk_msg() failed, return value %i!\n", err);
}
return err;
/* i2c write read: 5 bytes, addr 0x10, 0x02 bytes write, 1 bytes read. */
u8 b3[] =
{ 0xaa, ++ttusb->c, 0x31, 5, 0x10, 0x02, 0x01, 0x00, 0x1e };
- u8 b4[] =
- { 0x55, ttusb->c, 0x31, 4, 0x10, 0x02, 0x01, 0x00, 0x1e };
u8 get_version[] = { 0xaa, ++ttusb->c, 0x17, 5, 0, 0, 0, 0, 0 };
u8 get_dsp_version[0x20] =
if ((err = ttusb_cmd(ttusb, b2, sizeof(b2), 0)))
return err;
- if ((err = ttusb_cmd(ttusb, b3, sizeof(b3), 1)))
- return err;
-
- err = ttusb_result(ttusb, b4, sizeof(b4));
-
- if ((err = ttusb_cmd(ttusb, get_version, sizeof(get_version), 1)))
+ if ((err = ttusb_cmd(ttusb, b3, sizeof(b3), 0)))
return err;
- if ((err = ttusb_result(ttusb, get_version, sizeof(get_version))))
+ if ((err = ttusb_cmd(ttusb, get_version,
+ sizeof(get_version), sizeof(get_version))))
return err;
- dprintk("%s: stc-version: %c%c%c%c%c\n", __func__,
- get_version[4], get_version[5], get_version[6],
- get_version[7], get_version[8]);
+ dprintk("stc-version: %c%c%c%c%c\n", get_version[4], get_version[5],
+ get_version[6], get_version[7], get_version[8]);
if (memcmp(get_version + 4, "V 0.0", 5) &&
memcmp(get_version + 4, "V 1.1", 5) &&
memcmp(get_version + 4, "V 2.1", 5) &&
memcmp(get_version + 4, "V 2.2", 5)) {
- printk
- ("%s: unknown STC version %c%c%c%c%c, please report!\n",
- __func__, get_version[4], get_version[5],
- get_version[6], get_version[7], get_version[8]);
+ pr_err("unknown STC version %c%c%c%c%c, please report!\n",
+ get_version[4], get_version[5],
+ get_version[6], get_version[7], get_version[8]);
}
ttusb->revision = ((get_version[6] - '0') << 4) |
(get_version[8] - '0');
err =
- ttusb_cmd(ttusb, get_dsp_version, sizeof(get_dsp_version), 1);
+ ttusb_cmd(ttusb, get_dsp_version,
+ sizeof(get_dsp_version), sizeof(get_dsp_version));
if (err)
return err;
- err =
- ttusb_result(ttusb, get_dsp_version, sizeof(get_dsp_version));
- if (err)
- return err;
- printk("%s: dsp-version: %c%c%c\n", __func__,
+ pr_info("dsp-version: %c%c%c\n",
get_dsp_version[4], get_dsp_version[5], get_dsp_version[6]);
return 0;
}
/* Diseqc */
if ((err = ttusb_cmd(ttusb, b, 4 + b[3], 0))) {
- dprintk("%s: usb_bulk_msg() failed, return value %i!\n",
- __func__, err);
+ dprintk("usb_bulk_msg() failed, return value %i!\n", err);
}
return err;
/* SetLNB */
if ((err = ttusb_cmd(ttusb, b, sizeof(b), 0))) {
- dprintk("%s: usb_bulk_msg() failed, return value %i!\n",
- __func__, err);
+ dprintk("usb_bulk_msg() failed, return value %i!\n", err);
}
return err;
err = ttusb_cmd(ttusb, b, sizeof(b), 0);
if (err) {
- dprintk("%s: usb_bulk_msg() failed, return value %i!\n",
- __func__, err);
+ dprintk("usb_bulk_msg() failed, return value %i!\n", err);
}
}
#endif
int i;
if (len < 4 || len & 0x1) {
- pr_warn("%s: muxpack has invalid len %d\n", __func__, len);
+ pr_warn("muxpack has invalid len %d\n", len);
numinvalid++;
return;
}
for (i = 0; i < len; i += 2)
csum ^= le16_to_cpup((__le16 *) (muxpack + i));
if (csum) {
- printk("%s: muxpack with incorrect checksum, ignoring\n",
- __func__);
+ pr_warn("muxpack with incorrect checksum, ignoring\n");
numinvalid++;
return;
}
cc = (muxpack[len - 4] << 8) | muxpack[len - 3];
cc &= 0x7FFF;
if ((cc != ttusb->cc) && (ttusb->cc != -1))
- printk("%s: cc discontinuity (%d frames missing)\n",
- __func__, (cc - ttusb->cc) & 0x7FFF);
+ pr_warn("cc discontinuity (%d frames missing)\n",
+ (cc - ttusb->cc) & 0x7FFF);
ttusb->cc = (cc + 1) & 0x7FFF;
if (muxpack[0] & 0x80) {
#ifdef TTUSB_HWSECTIONS
!!(ttusb->muxpack[1] & 1))
data++;
#warning TODO: pusi
- printk("cc: %04x\n", (data[0] << 8) | data[1]);
+ dprintk("cc: %04x\n", (data[0] << 8) | data[1]);
#endif
numsec++;
} else if (muxpack[0] == 0x47) {
dvb_dmx_swfilter_packets(&ttusb->dvb_demux, muxpack, 1);
} else if (muxpack[0] != 0) {
numinvalid++;
- printk("illegal muxpack type %02x\n", muxpack[0]);
+ pr_err("illegal muxpack type %02x\n", muxpack[0]);
} else
numstuff++;
}
int maxwork = 1024;
while (len) {
if (!(maxwork--)) {
- printk("%s: too much work\n", __func__);
+ pr_err("too much work\n");
break;
}
else {
ttusb->mux_state = 0;
if (ttusb->insync) {
- dprintk("%s: %02x\n",
- __func__, data[-1]);
- printk(KERN_INFO "%s: lost sync.\n",
- __func__);
+ pr_info("lost sync.\n");
ttusb->insync = 0;
}
}
ttusb->muxpack[1] + 2 +
4;
else {
- dprintk
- ("%s: invalid state: first byte is %x\n",
- __func__,
- ttusb->muxpack[0]);
+ dprintk("invalid state: first byte is %x\n",
+ ttusb->muxpack[0]);
ttusb->mux_state = 0;
}
}
if (!ttusb->iso_streaming)
return;
-#if 0
- printk("%s: status %d, errcount == %d, length == %i\n",
- __func__,
- urb->status, urb->error_count, urb->actual_length);
-#endif
-
if (!urb->status) {
for (i = 0; i < urb->number_of_packets; ++i) {
numpkt++;
int i, j, err, buffer_offset = 0;
if (ttusb->iso_streaming) {
- printk("%s: iso xfer already running!\n", __func__);
+ pr_err("iso xfer already running!\n");
return 0;
}
for (i = 0; i < ISO_BUF_COUNT; i++) {
if ((err = usb_submit_urb(ttusb->iso_urb[i], GFP_ATOMIC))) {
ttusb_stop_iso_xfer(ttusb);
- printk
- ("%s: failed urb submission (%i: err = %i)!\n",
- __func__, i, err);
+ pr_err("failed urb submission (%i: err = %i)!\n",
+ i, err);
return err;
}
}
if (fe->ops.i2c_gate_ctrl)
fe->ops.i2c_gate_ctrl(fe, 1);
if (i2c_transfer(&ttusb->i2c_adap, &tuner_msg, 1) != 1) {
- printk("dvb-ttusb-budget: dvbc_philips_tdm1316l_pll_set Error 1\n");
+ pr_err("dvbc_philips_tdm1316l_pll_set Error 1\n");
return -EIO;
}
if (fe->ops.i2c_gate_ctrl)
fe->ops.i2c_gate_ctrl(fe, 1);
if (i2c_transfer(&ttusb->i2c_adap, &tuner_msg, 1) != 1) {
- printk("dvb-ttusb-budget: dvbc_philips_tdm1316l_pll_set Error 2\n");
+ pr_err("dvbc_philips_tdm1316l_pll_set Error 2\n");
return -EIO;
}
}
if (ttusb->fe == NULL) {
- printk("dvb-ttusb-budget: A frontend driver was not found for device [%04x:%04x]\n",
+ pr_err("no frontend driver found for device [%04x:%04x]\n",
le16_to_cpu(ttusb->dev->descriptor.idVendor),
le16_to_cpu(ttusb->dev->descriptor.idProduct));
} else {
if (dvb_register_frontend(&ttusb->adapter, ttusb->fe)) {
- printk("dvb-ttusb-budget: Frontend registration failed!\n");
+ pr_err("Frontend registration failed!\n");
dvb_frontend_detach(ttusb->fe);
ttusb->fe = NULL;
}
struct ttusb *ttusb;
int result;
- dprintk("%s: TTUSB DVB connected\n", __func__);
+ dprintk("TTUSB DVB connected\n");
udev = interface_to_usbdev(intf);
result = ttusb_alloc_iso_urbs(ttusb);
if (result < 0) {
- dprintk("%s: ttusb_alloc_iso_urbs - failed\n", __func__);
+ dprintk("ttusb_alloc_iso_urbs - failed\n");
mutex_unlock(&ttusb->semi2c);
kfree(ttusb);
return result;
}
if (ttusb_init_controller(ttusb))
- printk("ttusb_init_controller: error\n");
+ pr_err("ttusb_init_controller: error\n");
mutex_unlock(&ttusb->semi2c);
result = dvb_dmx_init(&ttusb->dvb_demux);
if (result < 0) {
- printk("ttusb_dvb: dvb_dmx_init failed (errno = %d)\n", result);
+ pr_err("dvb_dmx_init failed (errno = %d)\n", result);
result = -ENODEV;
goto err_i2c_del_adapter;
}
result = dvb_dmxdev_init(&ttusb->dmxdev, &ttusb->adapter);
if (result < 0) {
- printk("ttusb_dvb: dvb_dmxdev_init failed (errno = %d)\n",
+ pr_err("dvb_dmxdev_init failed (errno = %d)\n",
result);
result = -ENODEV;
goto err_release_dmx;
}
if (dvb_net_init(&ttusb->adapter, &ttusb->dvbnet, &ttusb->dvb_demux.dmx)) {
- printk("ttusb_dvb: dvb_net_init failed!\n");
+ pr_err("dvb_net_init failed!\n");
result = -ENODEV;
goto err_release_dmxdev;
}
kfree(ttusb);
- dprintk("%s: TTUSB DVB disconnected\n", __func__);
+ dprintk("TTUSB DVB disconnected\n");
}
static const struct usb_device_id ttusb_table[] = {
}
}
-static void ttusb_dec_process_urb_frame_list(unsigned long data)
+static void ttusb_dec_process_urb_frame_list(struct tasklet_struct *t)
{
- struct ttusb_dec *dec = (struct ttusb_dec *)data;
+ struct ttusb_dec *dec = from_tasklet(dec, t, urb_tasklet);
struct list_head *item;
struct urb_frame *frame;
unsigned long flags;
{
spin_lock_init(&dec->urb_frame_list_lock);
INIT_LIST_HEAD(&dec->urb_frame_list);
- tasklet_init(&dec->urb_tasklet, ttusb_dec_process_urb_frame_list,
- (unsigned long)dec);
+ tasklet_setup(&dec->urb_tasklet, ttusb_dec_process_urb_frame_list);
}
static int ttusb_init_rc( struct ttusb_dec *dec)
usbtv_audio_fail:
/* we must not free at this point */
- usb_get_dev(usbtv->udev);
+ v4l2_device_get(&usbtv->v4l2_dev);
+ /* this will undo the v4l2_device_get() */
usbtv_video_free(usbtv);
usbtv_video_fail:
v4l2_device_unregister(&usbtv->v4l2_dev);
v4l2_ctrl_handler_free(&usbtv->ctrl);
- vb2_queue_release(&usbtv->vb2q);
kfree(usbtv);
}
v4l2_fail:
ctrl_fail:
v4l2_ctrl_handler_free(&usbtv->ctrl);
- vb2_queue_release(&usbtv->vb2q);
return ret;
}
mutex_lock(&usbtv->v4l2_lock);
usbtv_stop(usbtv);
- video_unregister_device(&usbtv->vdev);
+ vb2_video_unregister_device(&usbtv->vdev);
v4l2_device_disconnect(&usbtv->v4l2_dev);
mutex_unlock(&usbtv->v4l2_lock);
offset &= 7;
mask = ((1LL << bits) - 1) << offset;
- for (; bits > 0; data++) {
+ while (1) {
u8 byte = *data & mask;
value |= offset > 0 ? (byte >> offset) : (byte << (-offset));
bits -= 8 - (offset > 0 ? offset : 0);
+ if (bits <= 0)
+ break;
+
offset -= 8;
mask = (1 << bits) - 1;
+ data++;
}
/* Sign-extend the value if needed. */
{
struct uvc_entity *entity;
struct uvc_control *ctrl;
- unsigned int i, found = 0;
+ unsigned int i;
+ bool found;
u32 reqflags;
u16 size;
u8 *data = NULL;
int ret;
/* Find the extension unit. */
+ found = false;
list_for_each_entry(entity, &chain->entities, chain) {
if (UVC_ENTITY_TYPE(entity) == UVC_VC_EXTENSION_UNIT &&
- entity->id == xqry->unit)
+ entity->id == xqry->unit) {
+ found = true;
break;
+ }
}
- if (entity->id != xqry->unit) {
+ if (!found) {
uvc_trace(UVC_TRACE_CONTROL, "Extension unit %u not found.\n",
xqry->unit);
return -ENOENT;
}
/* Find the control and perform delayed initialization if needed. */
+ found = false;
for (i = 0; i < entity->ncontrols; ++i) {
ctrl = &entity->controls[i];
if (ctrl->index == xqry->selector - 1) {
- found = 1;
+ found = true;
break;
}
}
static int uvc_ctrl_add_info(struct uvc_device *dev, struct uvc_control *ctrl,
const struct uvc_control_info *info)
{
- int ret = 0;
-
ctrl->info = *info;
INIT_LIST_HEAD(&ctrl->info.mappings);
/* Allocate an array to save control values (cur, def, max, etc.) */
ctrl->uvc_data = kzalloc(ctrl->info.size * UVC_CTRL_DATA_LAST + 1,
GFP_KERNEL);
- if (ctrl->uvc_data == NULL) {
- ret = -ENOMEM;
- goto done;
- }
-
- /*
- * Retrieve control flags from the device. Ignore errors and work with
- * default flag values from the uvc_ctrl array when the device doesn't
- * properly implement GET_INFO on standard controls.
- */
- uvc_ctrl_get_flags(dev, ctrl, &ctrl->info);
+ if (!ctrl->uvc_data)
+ return -ENOMEM;
ctrl->initialized = 1;
"entity %u\n", ctrl->info.entity, ctrl->info.selector,
dev->udev->devpath, ctrl->entity->id);
-done:
- if (ret < 0)
- kfree(ctrl->uvc_data);
- return ret;
+ return 0;
}
/*
if (uvc_entity_match_guid(ctrl->entity, info->entity) &&
ctrl->index == info->index) {
uvc_ctrl_add_info(dev, ctrl, info);
+ /*
+ * Retrieve control flags from the device. Ignore errors
+ * and work with default flag values from the uvc_ctrl
+ * array when the device doesn't properly implement
+ * GET_INFO on standard controls.
+ */
+ uvc_ctrl_get_flags(dev, ctrl, &ctrl->info);
break;
}
}
void uvc_debugfs_init_stream(struct uvc_streaming *stream)
{
struct usb_device *udev = stream->dev->udev;
- struct dentry *dent;
char dir_name[33];
if (uvc_debugfs_root_dir == NULL)
snprintf(dir_name, sizeof(dir_name), "%u-%u-%u", udev->bus->busnum,
udev->devnum, stream->intfnum);
- dent = debugfs_create_dir(dir_name, uvc_debugfs_root_dir);
- if (IS_ERR_OR_NULL(dent)) {
- uvc_printk(KERN_INFO, "Unable to create debugfs %s "
- "directory.\n", dir_name);
- return;
- }
-
- stream->debugfs_dir = dent;
+ stream->debugfs_dir = debugfs_create_dir(dir_name,
+ uvc_debugfs_root_dir);
- dent = debugfs_create_file("stats", 0444, stream->debugfs_dir,
- stream, &uvc_debugfs_stats_fops);
- if (IS_ERR_OR_NULL(dent)) {
- uvc_printk(KERN_INFO, "Unable to create debugfs stats file.\n");
- uvc_debugfs_cleanup_stream(stream);
- return;
- }
+ debugfs_create_file("stats", 0444, stream->debugfs_dir, stream,
+ &uvc_debugfs_stats_fops);
}
void uvc_debugfs_cleanup_stream(struct uvc_streaming *stream)
.guid = UVC_GUID_FORMAT_CNF4,
.fcc = V4L2_PIX_FMT_CNF4,
},
+ {
+ .name = "HEVC",
+ .guid = UVC_GUID_FORMAT_HEVC,
+ .fcc = V4L2_PIX_FMT_HEVC,
+ },
};
/* ------------------------------------------------------------------------
return NULL;
}
-static u32 uvc_colorspace(const u8 primaries)
+static enum v4l2_colorspace uvc_colorspace(const u8 primaries)
{
- static const u8 colorprimaries[] = {
- 0,
+ static const enum v4l2_colorspace colorprimaries[] = {
+ V4L2_COLORSPACE_DEFAULT, /* Unspecified */
V4L2_COLORSPACE_SRGB,
V4L2_COLORSPACE_470_SYSTEM_M,
V4L2_COLORSPACE_470_SYSTEM_BG,
if (primaries < ARRAY_SIZE(colorprimaries))
return colorprimaries[primaries];
- return 0;
+ return V4L2_COLORSPACE_DEFAULT; /* Reserved */
+}
+
+static enum v4l2_xfer_func uvc_xfer_func(const u8 transfer_characteristics)
+{
+ /*
+ * V4L2 does not currently have definitions for all possible values of
+ * UVC transfer characteristics. If v4l2_xfer_func is extended with new
+ * values, the mapping below should be updated.
+ *
+ * Substitutions are taken from the mapping given for
+ * V4L2_XFER_FUNC_DEFAULT documented in videodev2.h.
+ */
+ static const enum v4l2_xfer_func xfer_funcs[] = {
+ V4L2_XFER_FUNC_DEFAULT, /* Unspecified */
+ V4L2_XFER_FUNC_709,
+ V4L2_XFER_FUNC_709, /* Substitution for BT.470-2 M */
+ V4L2_XFER_FUNC_709, /* Substitution for BT.470-2 B, G */
+ V4L2_XFER_FUNC_709, /* Substitution for SMPTE 170M */
+ V4L2_XFER_FUNC_SMPTE240M,
+ V4L2_XFER_FUNC_NONE,
+ V4L2_XFER_FUNC_SRGB,
+ };
+
+ if (transfer_characteristics < ARRAY_SIZE(xfer_funcs))
+ return xfer_funcs[transfer_characteristics];
+
+ return V4L2_XFER_FUNC_DEFAULT; /* Reserved */
+}
+
+static enum v4l2_ycbcr_encoding uvc_ycbcr_enc(const u8 matrix_coefficients)
+{
+ /*
+ * V4L2 does not currently have definitions for all possible values of
+ * UVC matrix coefficients. If v4l2_ycbcr_encoding is extended with new
+ * values, the mapping below should be updated.
+ *
+ * Substitutions are taken from the mapping given for
+ * V4L2_YCBCR_ENC_DEFAULT documented in videodev2.h.
+ *
+ * FCC is assumed to be close enough to 601.
+ */
+ static const enum v4l2_ycbcr_encoding ycbcr_encs[] = {
+ V4L2_YCBCR_ENC_DEFAULT, /* Unspecified */
+ V4L2_YCBCR_ENC_709,
+ V4L2_YCBCR_ENC_601, /* Substitution for FCC */
+ V4L2_YCBCR_ENC_601, /* Substitution for BT.470-2 B, G */
+ V4L2_YCBCR_ENC_601,
+ V4L2_YCBCR_ENC_SMPTE240M,
+ };
+
+ if (matrix_coefficients < ARRAY_SIZE(ycbcr_encs))
+ return ycbcr_encs[matrix_coefficients];
+
+ return V4L2_YCBCR_ENC_DEFAULT; /* Reserved */
}
/* Simplify a fraction using a simple continued fraction decomposition. The
return;
/* Convert the fraction to a simple continued fraction. See
- * http://mathforum.org/dr.math/faq/faq.fractions.html
+ * https://mathforum.org/dr.math/faq/faq.fractions.html
* Stop if the current term is bigger than or equal to the given
* threshold.
*/
}
format->colorspace = uvc_colorspace(buffer[3]);
+ format->xfer_func = uvc_xfer_func(buffer[4]);
+ format->ycbcr_enc = uvc_ycbcr_enc(buffer[5]);
buflen -= buffer[0];
buffer += buffer[0];
int ret;
if (UVC_ENTITY_TYPE(entity) != UVC_TT_STREAMING) {
+ u32 function;
+
v4l2_subdev_init(&entity->subdev, &uvc_subdev_ops);
strscpy(entity->subdev.name, entity->name,
sizeof(entity->subdev.name));
+ switch (UVC_ENTITY_TYPE(entity)) {
+ case UVC_VC_SELECTOR_UNIT:
+ function = MEDIA_ENT_F_VID_MUX;
+ break;
+ case UVC_VC_PROCESSING_UNIT:
+ case UVC_VC_EXTENSION_UNIT:
+ /* For lack of a better option. */
+ function = MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER;
+ break;
+ case UVC_COMPOSITE_CONNECTOR:
+ case UVC_COMPONENT_CONNECTOR:
+ function = MEDIA_ENT_F_CONN_COMPOSITE;
+ break;
+ case UVC_SVIDEO_CONNECTOR:
+ function = MEDIA_ENT_F_CONN_SVIDEO;
+ break;
+ case UVC_ITT_CAMERA:
+ function = MEDIA_ENT_F_CAM_SENSOR;
+ break;
+ case UVC_TT_VENDOR_SPECIFIC:
+ case UVC_ITT_VENDOR_SPECIFIC:
+ case UVC_ITT_MEDIA_TRANSPORT_INPUT:
+ case UVC_OTT_VENDOR_SPECIFIC:
+ case UVC_OTT_DISPLAY:
+ case UVC_OTT_MEDIA_TRANSPORT_OUTPUT:
+ case UVC_EXTERNAL_VENDOR_SPECIFIC:
+ default:
+ function = MEDIA_ENT_F_V4L2_SUBDEV_UNKNOWN;
+ break;
+ }
+
+ entity->subdev.entity.function = function;
+
ret = media_entity_pads_init(&entity->subdev.entity,
entity->num_pads, entity->pads);
if (ret < 0)
goto done;
+ /* After the probe, update fmt with the values returned from
+ * negotiation with the device.
+ */
+ for (i = 0; i < stream->nformats; ++i) {
+ if (probe->bFormatIndex == stream->format[i].index) {
+ format = &stream->format[i];
+ break;
+ }
+ }
+
+ if (i == stream->nformats) {
+ uvc_trace(UVC_TRACE_FORMAT, "Unknown bFormatIndex %u\n",
+ probe->bFormatIndex);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < format->nframes; ++i) {
+ if (probe->bFrameIndex == format->frame[i].bFrameIndex) {
+ frame = &format->frame[i];
+ break;
+ }
+ }
+
+ if (i == format->nframes) {
+ uvc_trace(UVC_TRACE_FORMAT, "Unknown bFrameIndex %u\n",
+ probe->bFrameIndex);
+ return -EINVAL;
+ }
+
fmt->fmt.pix.width = frame->wWidth;
fmt->fmt.pix.height = frame->wHeight;
fmt->fmt.pix.field = V4L2_FIELD_NONE;
fmt->fmt.pix.bytesperline = uvc_v4l2_get_bytesperline(format, frame);
fmt->fmt.pix.sizeimage = probe->dwMaxVideoFrameSize;
+ fmt->fmt.pix.pixelformat = format->fcc;
fmt->fmt.pix.colorspace = format->colorspace;
+ fmt->fmt.pix.xfer_func = format->xfer_func;
+ fmt->fmt.pix.ycbcr_enc = format->ycbcr_enc;
if (uvc_format != NULL)
*uvc_format = format;
fmt->fmt.pix.bytesperline = uvc_v4l2_get_bytesperline(format, frame);
fmt->fmt.pix.sizeimage = stream->ctrl.dwMaxVideoFrameSize;
fmt->fmt.pix.colorspace = format->colorspace;
+ fmt->fmt.pix.xfer_func = format->xfer_func;
+ fmt->fmt.pix.ycbcr_enc = format->ycbcr_enc;
done:
mutex_unlock(&stream->mutex);
* to avoid losing precision in the division. Similarly, the host timestamp is
* computed with
*
- * TS = ((TS2 - TS1) * PTS + TS1 * SOF2 - TS2 * SOF1) / (SOF2 - SOF1) (2)
+ * TS = ((TS2 - TS1) * SOF + TS1 * SOF2 - TS2 * SOF1) / (SOF2 - SOF1) (2)
*
* SOF values are coded on 11 bits by USB. We extend their precision with 16
* decimal bits, leading to a 11.16 coding.
default:
uvc_printk(KERN_WARNING, "Non-zero status (%d) in video "
"completion handler.\n", urb->status);
- /* fall through */
+ fallthrough;
case -ENOENT: /* usb_poison_urb() called. */
if (stream->frozen)
return;
- /* fall through */
+ fallthrough;
case -ECONNRESET: /* usb_unlink_urb() called. */
case -ESHUTDOWN: /* The endpoint is being disabled. */
uvc_queue_cancel(queue, urb->status == -ESHUTDOWN);
{0x32, 0x00, 0x00, 0x00, 0x02, 0x00, 0x10, 0x00, \
0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}
+#define UVC_GUID_FORMAT_HEVC \
+ { 'H', 'E', 'V', 'C', 0x00, 0x00, 0x10, 0x00, \
+ 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}
+
/* ------------------------------------------------------------------------
* Driver specific constants.
u8 type;
u8 index;
u8 bpp;
- u8 colorspace;
+ enum v4l2_colorspace colorspace;
+ enum v4l2_xfer_func xfer_func;
+ enum v4l2_ycbcr_encoding ycbcr_enc;
u32 fcc;
u32 flags;
* @memory: buffer memory type
* @format: frame format, for which buffers are requested
* @capabilities: capabilities of this buffer type.
- * @flags: additional buffer management attributes (ignored unless the
- * queue has V4L2_BUF_CAP_SUPPORTS_MMAP_CACHE_HINTS capability and
- * configured for MMAP streaming I/O).
* @reserved: future extensions
*/
struct v4l2_create_buffers32 {
__u32 memory; /* enum v4l2_memory */
struct v4l2_format32 format;
__u32 capabilities;
- __u32 flags;
- __u32 reserved[6];
+ __u32 reserved[7];
};
static int __bufsize_v4l2_format(struct v4l2_format32 __user *p32, u32 *size)
{
if (!access_ok(p32, sizeof(*p32)) ||
copy_in_user(p64, p32,
- offsetof(struct v4l2_create_buffers32, format)) ||
- assign_in_user(&p64->flags, &p32->flags))
+ offsetof(struct v4l2_create_buffers32, format)))
return -EFAULT;
return __get_v4l2_format32(&p64->format, &p32->format,
aux_buf, aux_space);
copy_in_user(p32, p64,
offsetof(struct v4l2_create_buffers32, format)) ||
assign_in_user(&p32->capabilities, &p64->capabilities) ||
- assign_in_user(&p32->flags, &p64->flags) ||
copy_in_user(p32->reserved, p64->reserved, sizeof(p64->reserved)))
return -EFAULT;
return __put_v4l2_format32(&p64->format, &p32->format);
static const char * const mpeg_video_bitrate_mode[] = {
"Variable Bitrate",
"Constant Bitrate",
+ "Constant Quality",
NULL
};
static const char * const mpeg_stream_type[] = {
"3",
NULL,
};
+ static const char * const vp9_level[] = {
+ "1",
+ "1.1",
+ "2",
+ "2.1",
+ "3",
+ "3.1",
+ "4",
+ "4.1",
+ "5",
+ "5.1",
+ "5.2",
+ "6",
+ "6.1",
+ "6.2",
+ NULL,
+ };
static const char * const flash_led_mode[] = {
"Off",
"External",
NULL,
};
+ static const char * const mpeg_video_frame_skip[] = {
+ "Disabled",
+ "Level Limit",
+ "VBV/CPB Limit",
+ NULL,
+ };
switch (id) {
case V4L2_CID_MPEG_AUDIO_SAMPLING_FREQ:
return flash_strobe_source;
case V4L2_CID_MPEG_VIDEO_HEADER_MODE:
return header_mode;
+ case V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE:
+ return mpeg_video_frame_skip;
case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE:
return multi_slice;
case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE:
return vp8_profile;
case V4L2_CID_MPEG_VIDEO_VP9_PROFILE:
return vp9_profile;
+ case V4L2_CID_MPEG_VIDEO_VP9_LEVEL:
+ return vp9_level;
case V4L2_CID_JPEG_CHROMA_SUBSAMPLING:
return jpeg_chroma_subsampling;
case V4L2_CID_DV_TX_MODE:
case V4L2_CID_MPEG_VIDEO_GOP_CLOSURE: return "Video GOP Closure";
case V4L2_CID_MPEG_VIDEO_PULLDOWN: return "Video Pulldown";
case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: return "Video Bitrate Mode";
+ case V4L2_CID_MPEG_VIDEO_CONSTANT_QUALITY: return "Constant Quality";
case V4L2_CID_MPEG_VIDEO_BITRATE: return "Video Bitrate";
case V4L2_CID_MPEG_VIDEO_BITRATE_PEAK: return "Video Peak Bitrate";
case V4L2_CID_MPEG_VIDEO_TEMPORAL_DECIMATION: return "Video Temporal Decimation";
case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE: return "H264 MB Level Rate Control";
case V4L2_CID_MPEG_VIDEO_HEADER_MODE: return "Sequence Header Mode";
case V4L2_CID_MPEG_VIDEO_MAX_REF_PIC: return "Max Number of Reference Pics";
+ case V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE: return "Frame Skip Mode";
case V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP: return "H263 I-Frame QP Value";
case V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP: return "H263 P-Frame QP Value";
case V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP: return "H263 B-Frame QP Value";
case V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS: return "H264 Decode Parameters";
case V4L2_CID_MPEG_VIDEO_H264_DECODE_MODE: return "H264 Decode Mode";
case V4L2_CID_MPEG_VIDEO_H264_START_CODE: return "H264 Start Code";
+ case V4L2_CID_MPEG_VIDEO_H264_PRED_WEIGHTS: return "H264 Prediction Weight Table";
case V4L2_CID_MPEG_VIDEO_MPEG2_LEVEL: return "MPEG2 Level";
case V4L2_CID_MPEG_VIDEO_MPEG2_PROFILE: return "MPEG2 Profile";
case V4L2_CID_MPEG_VIDEO_MPEG4_I_FRAME_QP: return "MPEG4 I-Frame QP Value";
case V4L2_CID_MPEG_VIDEO_VPX_P_FRAME_QP: return "VPX P-Frame QP Value";
case V4L2_CID_MPEG_VIDEO_VP8_PROFILE: return "VP8 Profile";
case V4L2_CID_MPEG_VIDEO_VP9_PROFILE: return "VP9 Profile";
+ case V4L2_CID_MPEG_VIDEO_VP9_LEVEL: return "VP9 Level";
case V4L2_CID_MPEG_VIDEO_VP8_FRAME_HEADER: return "VP8 Frame Header";
/* HEVC controls */
case V4L2_CID_FLASH_LED_MODE:
case V4L2_CID_FLASH_STROBE_SOURCE:
case V4L2_CID_MPEG_VIDEO_HEADER_MODE:
+ case V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE:
case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE:
case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE:
case V4L2_CID_MPEG_VIDEO_H264_LEVEL:
case V4L2_CID_MPEG_VIDEO_VPX_GOLDEN_FRAME_SEL:
case V4L2_CID_MPEG_VIDEO_VP8_PROFILE:
case V4L2_CID_MPEG_VIDEO_VP9_PROFILE:
+ case V4L2_CID_MPEG_VIDEO_VP9_LEVEL:
case V4L2_CID_DETECT_MD_MODE:
case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE:
case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL:
case V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS:
*type = V4L2_CTRL_TYPE_H264_DECODE_PARAMS;
break;
+ case V4L2_CID_MPEG_VIDEO_H264_PRED_WEIGHTS:
+ *type = V4L2_CTRL_TYPE_H264_PRED_WEIGHTS;
+ break;
case V4L2_CID_MPEG_VIDEO_VP8_FRAME_HEADER:
*type = V4L2_CTRL_TYPE_VP8_FRAME_HEADER;
break;
#define zero_padding(s) \
memset(&(s).padding, 0, sizeof((s).padding))
+#define zero_reserved(s) \
+ memset(&(s).reserved, 0, sizeof((s).reserved))
/*
* Compound controls validation requires setting unused fields/flags to zero
{
struct v4l2_ctrl_mpeg2_slice_params *p_mpeg2_slice_params;
struct v4l2_ctrl_vp8_frame_header *p_vp8_frame_header;
+ struct v4l2_ctrl_h264_slice_params *p_h264_slice_params;
+ struct v4l2_ctrl_h264_decode_params *p_h264_dec_params;
struct v4l2_ctrl_hevc_sps *p_hevc_sps;
struct v4l2_ctrl_hevc_pps *p_hevc_pps;
struct v4l2_ctrl_hevc_slice_params *p_hevc_slice_params;
case V4L2_CTRL_TYPE_H264_SPS:
case V4L2_CTRL_TYPE_H264_PPS:
case V4L2_CTRL_TYPE_H264_SCALING_MATRIX:
+ case V4L2_CTRL_TYPE_H264_PRED_WEIGHTS:
+ break;
+
case V4L2_CTRL_TYPE_H264_SLICE_PARAMS:
+ p_h264_slice_params = p;
+
+ zero_reserved(*p_h264_slice_params);
+ break;
+
case V4L2_CTRL_TYPE_H264_DECODE_PARAMS:
+ p_h264_dec_params = p;
+
+ for (i = 0; i < V4L2_H264_NUM_DPB_ENTRIES; i++) {
+ struct v4l2_h264_dpb_entry *dpb_entry =
+ &p_h264_dec_params->dpb[i];
+
+ zero_reserved(*dpb_entry);
+ }
+ zero_reserved(*p_h264_dec_params);
break;
case V4L2_CTRL_TYPE_VP8_FRAME_HEADER:
case V4L2_CTRL_TYPE_H264_DECODE_PARAMS:
elem_size = sizeof(struct v4l2_ctrl_h264_decode_params);
break;
+ case V4L2_CTRL_TYPE_H264_PRED_WEIGHTS:
+ elem_size = sizeof(struct v4l2_ctrl_h264_pred_weights);
+ break;
case V4L2_CTRL_TYPE_VP8_FRAME_HEADER:
elem_size = sizeof(struct v4l2_ctrl_vp8_frame_header);
break;
}
for (i = 0; i < vep->nr_of_link_frequencies; i++)
- pr_info("link-frequencies %u value %llu\n", i,
- vep->link_frequencies[i]);
+ pr_debug("link-frequencies %u value %llu\n", i,
+ vep->link_frequencies[i]);
}
pr_debug("===== end parsing endpoint %pfw\n", fwnode);
*
* @b: the builder context to initialize
* @dec_params: decode parameters control
- * @slice_params: first slice parameters control
* @sps: SPS control
* @dpb: DPB to use when creating the reference list
*/
void
v4l2_h264_init_reflist_builder(struct v4l2_h264_reflist_builder *b,
const struct v4l2_ctrl_h264_decode_params *dec_params,
- const struct v4l2_ctrl_h264_slice_params *slice_params,
const struct v4l2_ctrl_h264_sps *sps,
const struct v4l2_h264_dpb_entry dpb[V4L2_H264_NUM_DPB_ENTRIES])
{
unsigned int i;
max_frame_num = 1 << (sps->log2_max_frame_num_minus4 + 4);
- cur_frame_num = slice_params->frame_num;
+ cur_frame_num = dec_params->frame_num;
memset(b, 0, sizeof(*b));
- if (!(slice_params->flags & V4L2_H264_SLICE_FLAG_FIELD_PIC))
+ if (!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC))
b->cur_pic_order_count = min(dec_params->bottom_field_order_cnt,
dec_params->top_field_order_cnt);
- else if (slice_params->flags & V4L2_H264_SLICE_FLAG_BOTTOM_FIELD)
+ else if (dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD)
b->cur_pic_order_count = dec_params->bottom_field_order_cnt;
else
b->cur_pic_order_count = dec_params->top_field_order_cnt;
else
b->refs[i].frame_num = dpb[i].frame_num;
- if (!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD))
+ if (dpb[i].fields == V4L2_H264_FRAME_REF)
pic_order_count = min(dpb[i].top_field_order_cnt,
dpb[i].bottom_field_order_cnt);
- else if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_BOTTOM_FIELD)
+ else if (dpb[i].fields & V4L2_H264_BOTTOM_FIELD_REF)
pic_order_count = dpb[i].bottom_field_order_cnt;
else
pic_order_count = dpb[i].top_field_order_cnt;
if (ret)
return ret;
+
+ CLEAR_AFTER_FIELD(p, capabilities);
+
return ops->vidioc_reqbufs(file, fh, p);
}
if (ret)
return ret;
- CLEAR_AFTER_FIELD(create, flags);
+ CLEAR_AFTER_FIELD(create, capabilities);
v4l_sanitize_format(&create->format);
#define TRANS_ABORT (1 << 2)
+/* The job queue is not running new jobs */
+#define QUEUE_PAUSED (1 << 0)
+
+
/* Offset base for buffers on the destination queue - used to distinguish
* between source and destination buffers when mmapping - they receive the same
* offsets but for different queues */
* @job_queue: instances queued to run
* @job_spinlock: protects job_queue
* @job_work: worker to run queued jobs.
+ * @job_queue_flags: flags of the queue status, %QUEUE_PAUSED.
* @m2m_ops: driver callbacks
*/
struct v4l2_m2m_dev {
struct list_head job_queue;
spinlock_t job_spinlock;
struct work_struct job_work;
+ unsigned long job_queue_flags;
const struct v4l2_m2m_ops *m2m_ops;
};
return;
}
+ if (m2m_dev->job_queue_flags & QUEUE_PAUSED) {
+ spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
+ dprintk("Running new jobs is paused\n");
+ return;
+ }
+
m2m_dev->curr_ctx = list_first_entry(&m2m_dev->job_queue,
struct v4l2_m2m_ctx, queue);
m2m_dev->curr_ctx->job_flags |= TRANS_RUNNING;
}
EXPORT_SYMBOL(v4l2_m2m_buf_done_and_job_finish);
+void v4l2_m2m_suspend(struct v4l2_m2m_dev *m2m_dev)
+{
+ unsigned long flags;
+ struct v4l2_m2m_ctx *curr_ctx;
+
+ spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
+ m2m_dev->job_queue_flags |= QUEUE_PAUSED;
+ curr_ctx = m2m_dev->curr_ctx;
+ spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
+
+ if (curr_ctx)
+ wait_event(curr_ctx->finished,
+ !(curr_ctx->job_flags & TRANS_RUNNING));
+}
+EXPORT_SYMBOL(v4l2_m2m_suspend);
+
+void v4l2_m2m_resume(struct v4l2_m2m_dev *m2m_dev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
+ m2m_dev->job_queue_flags &= ~QUEUE_PAUSED;
+ spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
+
+ v4l2_m2m_try_run(m2m_dev);
+}
+EXPORT_SYMBOL(v4l2_m2m_resume);
+
int v4l2_m2m_reqbufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
struct v4l2_requestbuffers *reqbufs)
{
struct poll_table_struct *wait)
{
struct vb2_queue *src_q, *dst_q;
- struct vb2_buffer *src_vb = NULL, *dst_vb = NULL;
__poll_t rc = 0;
unsigned long flags;
list_empty(&dst_q->queued_list)))
return EPOLLERR;
- spin_lock_irqsave(&dst_q->done_lock, flags);
- if (list_empty(&dst_q->done_list)) {
- /*
- * If the last buffer was dequeued from the capture queue,
- * return immediately. DQBUF will return -EPIPE.
- */
- if (dst_q->last_buffer_dequeued) {
- spin_unlock_irqrestore(&dst_q->done_lock, flags);
- return EPOLLIN | EPOLLRDNORM;
- }
- }
- spin_unlock_irqrestore(&dst_q->done_lock, flags);
-
spin_lock_irqsave(&src_q->done_lock, flags);
if (!list_empty(&src_q->done_list))
- src_vb = list_first_entry(&src_q->done_list, struct vb2_buffer,
- done_entry);
- if (src_vb && (src_vb->state == VB2_BUF_STATE_DONE
- || src_vb->state == VB2_BUF_STATE_ERROR))
rc |= EPOLLOUT | EPOLLWRNORM;
spin_unlock_irqrestore(&src_q->done_lock, flags);
spin_lock_irqsave(&dst_q->done_lock, flags);
- if (!list_empty(&dst_q->done_list))
- dst_vb = list_first_entry(&dst_q->done_list, struct vb2_buffer,
- done_entry);
- if (dst_vb && (dst_vb->state == VB2_BUF_STATE_DONE
- || dst_vb->state == VB2_BUF_STATE_ERROR))
+ /*
+ * If the last buffer was dequeued from the capture queue, signal
+ * userspace. DQBUF(CAPTURE) will return -EPIPE.
+ */
+ if (!list_empty(&dst_q->done_list) || dst_q->last_buffer_dequeued)
rc |= EPOLLIN | EPOLLRDNORM;
spin_unlock_irqrestore(&dst_q->done_lock, flags);
sd->ops->pad->enum_dv_timings(sd, dvt);
}
+static int call_get_mbus_config(struct v4l2_subdev *sd, unsigned int pad,
+ struct v4l2_mbus_config *config)
+{
+ return check_pad(sd, pad) ? :
+ sd->ops->pad->get_mbus_config(sd, pad, config);
+}
+
+static int call_set_mbus_config(struct v4l2_subdev *sd, unsigned int pad,
+ struct v4l2_mbus_config *config)
+{
+ return check_pad(sd, pad) ? :
+ sd->ops->pad->get_mbus_config(sd, pad, config);
+}
+
static const struct v4l2_subdev_pad_ops v4l2_subdev_call_pad_wrappers = {
.get_fmt = call_get_fmt,
.set_fmt = call_set_fmt,
.set_edid = call_set_edid,
.dv_timings_cap = call_dv_timings_cap,
.enum_dv_timings = call_enum_dv_timings,
+ .get_mbus_config = call_get_mbus_config,
+ .set_mbus_config = call_set_mbus_config,
};
static const struct v4l2_subdev_video_ops v4l2_subdev_call_video_wrappers = {
if (rw == READ)
flags |= FOLL_WRITE;
- dprintk(1, "init user [0x%lx+0x%lx => %d pages]\n",
+ dprintk(1, "init user [0x%lx+0x%lx => %lu pages]\n",
data, size, dma->nr_pages);
err = pin_user_pages(data & PAGE_MASK, dma->nr_pages,
if (err != dma->nr_pages) {
dma->nr_pages = (err >= 0) ? err : 0;
- dprintk(1, "pin_user_pages: err=%d [%d]\n", err,
+ dprintk(1, "pin_user_pages: err=%d [%lu]\n", err,
dma->nr_pages);
return err < 0 ? err : -EINVAL;
}
}
static int videobuf_dma_init_kernel(struct videobuf_dmabuf *dma, int direction,
- int nr_pages)
+ unsigned long nr_pages)
{
int i;
- dprintk(1, "init kernel [%d pages]\n", nr_pages);
+ dprintk(1, "init kernel [%lu pages]\n", nr_pages);
dma->direction = direction;
dma->vaddr_pages = kcalloc(nr_pages, sizeof(*dma->vaddr_pages),
dma->vaddr = vmap(dma->vaddr_pages, nr_pages, VM_MAP | VM_IOREMAP,
PAGE_KERNEL);
if (NULL == dma->vaddr) {
- dprintk(1, "vmalloc_32(%d pages) failed\n", nr_pages);
+ dprintk(1, "vmalloc_32(%lu pages) failed\n", nr_pages);
goto out_free_pages;
}
- dprintk(1, "vmalloc is at addr %p, size=%d\n",
+ dprintk(1, "vmalloc is at addr %p, size=%lu\n",
dma->vaddr, nr_pages << PAGE_SHIFT);
memset(dma->vaddr, 0, nr_pages << PAGE_SHIFT);
}
static int videobuf_dma_init_overlay(struct videobuf_dmabuf *dma, int direction,
- dma_addr_t addr, int nr_pages)
+ dma_addr_t addr, unsigned long nr_pages)
{
- dprintk(1, "init overlay [%d pages @ bus 0x%lx]\n",
+ dprintk(1, "init overlay [%lu pages @ bus 0x%lx]\n",
nr_pages, (unsigned long)addr);
dma->direction = direction;
struct videobuf_buffer *vb,
struct v4l2_framebuffer *fbuf)
{
- int err, pages;
- dma_addr_t bus;
struct videobuf_dma_sg_memory *mem = vb->priv;
+ unsigned long pages;
+ dma_addr_t bus;
+ int err;
+
BUG_ON(!mem);
MAGIC_CHECK(mem->magic, MAGIC_SG_MEM);
} else if (host->card->stop)
host->card->stop(host->card);
+ if (host->removing)
+ goto out_power_off;
+
card = memstick_alloc_card(host);
if (!card) {
*/
void memstick_remove_host(struct memstick_host *host)
{
+ host->removing = 1;
flush_workqueue(workqueue);
mutex_lock(&host->lock);
if (host->card)
}
if (be16_to_cpu(page->header.block_id) != MS_BLOCK_BOOT_ID) {
- dbg("the pba at %d doesn' contain boot block ID", pba);
+ dbg("the pba at %d doesn't contain boot block ID", pba);
continue;
}
* This file supports the user system call for file open, close, mmap, etc.
* This also incudes the driver initialization code.
*
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
* Copyright (c) 2008-2014 Silicon Graphics, Inc. All Rights Reserved.
*/
#if defined CONFIG_IA64
gru_start_paddr = 0xd000000000UL; /* ZZZZZZZZZZZZZZZZZZZ fixme */
#else
- gru_start_paddr = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR) &
+ gru_start_paddr = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG) &
0x7fffffffffffUL;
#endif
gru_start_vaddr = __va(gru_start_paddr);
* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
* Copyright (C) 2004-2008 Silicon Graphics, Inc. All rights reserved.
*/
#if defined CONFIG_X86_UV || defined CONFIG_IA64_SGI_UV
#include <asm/uv/uv.h>
-#define is_uv() is_uv_system()
-#endif
-
-#ifndef is_uv
-#define is_uv() 0
#endif
#ifdef USE_DBUG_ON
#define XPC_MSG_SIZE(_payload_size) \
ALIGN(XPC_MSG_HDR_MAX_SIZE + (_payload_size), \
- is_uv() ? 64 : 128)
+ is_uv_system() ? 64 : 128)
/*
* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
* Copyright (c) 2004-2008 Silicon Graphics, Inc. All Rights Reserved.
*/
for (ch_number = 0; ch_number < XPC_MAX_NCHANNELS; ch_number++)
mutex_init(&xpc_registrations[ch_number].mutex);
- if (is_uv())
+ if (is_uv_system())
ret = xp_init_uv();
else
ret = 0;
static void __exit
xp_exit(void)
{
- if (is_uv())
+ if (is_uv_system())
xp_exit_uv();
}
* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
* Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
*/
enum xp_retval
xp_init_uv(void)
{
- BUG_ON(!is_uv());
+ WARN_ON(!is_uv_system());
+ if (!is_uv_system())
+ return xpUnsupported;
xp_max_npartitions = XP_MAX_NPARTITIONS_UV;
#ifdef CONFIG_X86
void
xp_exit_uv(void)
{
- BUG_ON(!is_uv());
+ WARN_ON(!is_uv_system());
}
* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
* Copyright (c) 2004-2009 Silicon Graphics, Inc. All Rights Reserved.
*/
xpc_teardown_partitions();
- if (is_uv())
+ if (is_uv_system())
xpc_exit_uv();
}
dev_set_name(xpc_part, "part");
dev_set_name(xpc_chan, "chan");
- if (is_uv()) {
+ if (is_uv_system()) {
ret = xpc_init_uv();
} else {
xpc_teardown_partitions();
out_1:
- if (is_uv())
+ if (is_uv_system())
xpc_exit_uv();
return ret;
}
* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
* Copyright (c) 2004-2008 Silicon Graphics, Inc. All Rights Reserved.
*/
*/
region_size = xp_region_size;
- if (is_uv())
+ if (is_uv_system())
max_regions = 256;
else {
max_regions = 64;
* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
+ * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
* Copyright (C) 1999-2009 Silicon Graphics, Inc. All rights reserved.
*/
{
int result;
- if (!is_uv())
+ if (!is_uv_system())
return -ENODEV;
dev_info(xpnet, "registering network device %s\n", XPNET_DEVICE_NAME);
static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)
{
- int pasid;
+ u32 pasid;
struct iommu_sva *handle;
if (!(uacce->flags & UACCE_DEV_SVA))
* whole block device, not on a partition. This prevents overspray
* between sibling partitions.
*/
- if ((!capable(CAP_SYS_RAWIO)) || (bdev != bdev->bd_contains))
+ if (!capable(CAP_SYS_RAWIO) || bdev_is_partition(bdev))
return -EPERM;
return 0;
}
{
struct mmc_card *card = mmc_dev_to_card(dev);
const char *type;
+ unsigned int i;
int retval = 0;
switch (card->type) {
card->cis.vendor, card->cis.device);
if (retval)
return retval;
+
+ retval = add_uevent_var(env, "SDIO_REVISION=%u.%u",
+ card->major_rev, card->minor_rev);
+ if (retval)
+ return retval;
+
+ for (i = 0; i < card->num_info; i++) {
+ retval = add_uevent_var(env, "SDIO_INFO%u=%s", i+1, card->info[i]);
+ if (retval)
+ return retval;
+ }
}
/*
host->ops->hw_reset(host);
}
+/**
+ * mmc_hw_reset - reset the card in hardware
+ * @host: MMC host to which the card is attached
+ *
+ * Hard reset the card. This function is only for upper layers, like the
+ * block layer or card drivers. You cannot use it in host drivers (struct
+ * mmc_card might be gone then).
+ *
+ * Return: 0 on success, -errno on failure
+ */
int mmc_hw_reset(struct mmc_host *host)
{
int ret;
}
EXPORT_SYMBOL(mmc_of_parse_voltage);
+/**
+ * mmc_first_nonreserved_index() - get the first index that is not reserved
+ */
+static int mmc_first_nonreserved_index(void)
+{
+ int max;
+
+ max = of_alias_get_highest_id("mmc");
+ if (max < 0)
+ return 0;
+
+ return max + 1;
+}
+
/**
* mmc_alloc_host - initialise the per-host structure.
* @extra: sizeof private data structure
{
int err;
struct mmc_host *host;
+ int alias_id, min_idx, max_idx;
host = kzalloc(sizeof(struct mmc_host) + extra, GFP_KERNEL);
if (!host)
/* scanning will be enabled when we're ready */
host->rescan_disable = 1;
- err = ida_simple_get(&mmc_host_ida, 0, 0, GFP_KERNEL);
+ alias_id = of_alias_get_id(dev->of_node, "mmc");
+ if (alias_id >= 0) {
+ min_idx = alias_id;
+ max_idx = alias_id + 1;
+ } else {
+ min_idx = mmc_first_nonreserved_index();
+ max_idx = 0;
+ }
+
+ err = ida_simple_get(&mmc_host_ida, min_idx, max_idx, GFP_KERNEL);
if (err < 0) {
kfree(host);
return NULL;
return err;
}
- /* Set host controller to HS timing */
- mmc_set_timing(card->host, MMC_TIMING_MMC_HS);
-
/* Prepare host to downgrade to HS timing */
if (host->ops->hs400_downgrade)
host->ops->hs400_downgrade(host);
+ /* Set host controller to HS timing */
+ mmc_set_timing(host, MMC_TIMING_MMC_HS);
+
/* Reduce frequency to HS frequency */
max_dtr = card->ext_csd.hs_max_dtr;
mmc_set_clock(host, max_dtr);
if (err)
goto out_err;
+ if (host->ops->hs400_downgrade)
+ host->ops->hs400_downgrade(host);
+
mmc_set_timing(host, MMC_TIMING_MMC_DDR52);
err = mmc_switch_status(card, true);
mmc_set_timing(host, MMC_TIMING_MMC_HS);
- if (host->ops->hs400_downgrade)
- host->ops->hs400_downgrade(host);
-
err = mmc_switch_status(card, true);
if (err)
goto out_err;
goto free_card;
if (mmc_card_hs200(card)) {
+ host->doing_init_tune = 1;
+
err = mmc_hs200_tuning(card);
- if (err)
- goto free_card;
+ if (!err)
+ err = mmc_select_hs400(card);
+
+ host->doing_init_tune = 0;
- err = mmc_select_hs400(card);
if (err)
goto free_card;
+
} else if (!mmc_card_hs400es(card)) {
/* Select the desired bus width optionally */
err = mmc_select_bus_width(card);
},
{
- .name = "Correct xfer_size at write (start failure)",
+ .name = "Proper xfer_size at write (start failure)",
.run = mmc_test_xfersize_write,
},
{
- .name = "Correct xfer_size at read (start failure)",
+ .name = "Proper xfer_size at read (start failure)",
.run = mmc_test_xfersize_read,
},
{
- .name = "Correct xfer_size at write (midway failure)",
+ .name = "Proper xfer_size at write (midway failure)",
.run = mmc_test_multi_xfersize_write,
},
{
- .name = "Correct xfer_size at read (midway failure)",
+ .name = "Proper xfer_size at read (midway failure)",
.run = mmc_test_multi_xfersize_read,
},
q->limits.discard_granularity = card->pref_erase << 9;
/* granularity must not be greater than max. discard */
if (card->pref_erase > max_discard)
- q->limits.discard_granularity = 0;
+ q->limits.discard_granularity = SECTOR_SIZE;
if (mmc_can_secure_erase_trim(card))
blk_queue_flag_set(QUEUE_FLAG_SECERASE, q);
}
MMC_DEV_ATTR(vendor, "0x%04x\n", card->cis.vendor);
MMC_DEV_ATTR(device, "0x%04x\n", card->cis.device);
+MMC_DEV_ATTR(revision, "%u.%u\n", card->major_rev, card->minor_rev);
+
+#define sdio_info_attr(num) \
+static ssize_t info##num##_show(struct device *dev, struct device_attribute *attr, char *buf) \
+{ \
+ struct mmc_card *card = mmc_dev_to_card(dev); \
+ \
+ if (num > card->num_info) \
+ return -ENODATA; \
+ if (!card->info[num-1][0]) \
+ return 0; \
+ return sprintf(buf, "%s\n", card->info[num-1]); \
+} \
+static DEVICE_ATTR_RO(info##num)
+
+sdio_info_attr(1);
+sdio_info_attr(2);
+sdio_info_attr(3);
+sdio_info_attr(4);
static struct attribute *sd_std_attrs[] = {
&dev_attr_vendor.attr,
&dev_attr_device.attr,
+ &dev_attr_revision.attr,
+ &dev_attr_info1.attr,
+ &dev_attr_info2.attr,
+ &dev_attr_info3.attr,
+ &dev_attr_info4.attr,
&dev_attr_cid.attr,
&dev_attr_csd.attr,
&dev_attr_scr.attr,
static umode_t sd_std_is_visible(struct kobject *kobj, struct attribute *attr,
int index)
{
- struct device *dev = container_of(kobj, struct device, kobj);
+ struct device *dev = kobj_to_dev(kobj);
struct mmc_card *card = mmc_dev_to_card(dev);
- /* CIS vendor and device ids are available only for Combo cards */
- if ((attr == &dev_attr_vendor.attr || attr == &dev_attr_device.attr) &&
- card->type != MMC_TYPE_SD_COMBO)
+ /* CIS vendor and device ids, revision and info string are available only for Combo cards */
+ if ((attr == &dev_attr_vendor.attr ||
+ attr == &dev_attr_device.attr ||
+ attr == &dev_attr_revision.attr ||
+ attr == &dev_attr_info1.attr ||
+ attr == &dev_attr_info2.attr ||
+ attr == &dev_attr_info3.attr ||
+ attr == &dev_attr_info4.attr
+ ) && card->type != MMC_TYPE_SD_COMBO)
return 0;
return attr->mode;
MMC_DEV_ATTR(vendor, "0x%04x\n", card->cis.vendor);
MMC_DEV_ATTR(device, "0x%04x\n", card->cis.device);
+MMC_DEV_ATTR(revision, "%u.%u\n", card->major_rev, card->minor_rev);
MMC_DEV_ATTR(ocr, "0x%08x\n", card->ocr);
MMC_DEV_ATTR(rca, "0x%04x\n", card->rca);
+#define sdio_info_attr(num) \
+static ssize_t info##num##_show(struct device *dev, struct device_attribute *attr, char *buf) \
+{ \
+ struct mmc_card *card = mmc_dev_to_card(dev); \
+ \
+ if (num > card->num_info) \
+ return -ENODATA; \
+ if (!card->info[num-1][0]) \
+ return 0; \
+ return sprintf(buf, "%s\n", card->info[num-1]); \
+} \
+static DEVICE_ATTR_RO(info##num)
+
+sdio_info_attr(1);
+sdio_info_attr(2);
+sdio_info_attr(3);
+sdio_info_attr(4);
+
static struct attribute *sdio_std_attrs[] = {
&dev_attr_vendor.attr,
&dev_attr_device.attr,
+ &dev_attr_revision.attr,
+ &dev_attr_info1.attr,
+ &dev_attr_info2.attr,
+ &dev_attr_info3.attr,
+ &dev_attr_info4.attr,
&dev_attr_ocr.attr,
&dev_attr_rca.attr,
NULL,
#define to_sdio_driver(d) container_of(d, struct sdio_driver, drv)
/* show configuration fields */
-#define sdio_config_attr(field, format_string) \
+#define sdio_config_attr(field, format_string, args...) \
static ssize_t \
field##_show(struct device *dev, struct device_attribute *attr, char *buf) \
{ \
struct sdio_func *func; \
\
func = dev_to_sdio_func (dev); \
- return sprintf (buf, format_string, func->field); \
+ return sprintf(buf, format_string, args); \
} \
static DEVICE_ATTR_RO(field)
-sdio_config_attr(class, "0x%02x\n");
-sdio_config_attr(vendor, "0x%04x\n");
-sdio_config_attr(device, "0x%04x\n");
-
-static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, char *buf)
-{
- struct sdio_func *func = dev_to_sdio_func (dev);
-
- return sprintf(buf, "sdio:c%02Xv%04Xd%04X\n",
- func->class, func->vendor, func->device);
-}
-static DEVICE_ATTR_RO(modalias);
+sdio_config_attr(class, "0x%02x\n", func->class);
+sdio_config_attr(vendor, "0x%04x\n", func->vendor);
+sdio_config_attr(device, "0x%04x\n", func->device);
+sdio_config_attr(revision, "%u.%u\n", func->major_rev, func->minor_rev);
+sdio_config_attr(modalias, "sdio:c%02Xv%04Xd%04X\n", func->class, func->vendor, func->device);
+
+#define sdio_info_attr(num) \
+static ssize_t info##num##_show(struct device *dev, struct device_attribute *attr, char *buf) \
+{ \
+ struct sdio_func *func = dev_to_sdio_func(dev); \
+ \
+ if (num > func->num_info) \
+ return -ENODATA; \
+ if (!func->info[num-1][0]) \
+ return 0; \
+ return sprintf(buf, "%s\n", func->info[num-1]); \
+} \
+static DEVICE_ATTR_RO(info##num)
+
+sdio_info_attr(1);
+sdio_info_attr(2);
+sdio_info_attr(3);
+sdio_info_attr(4);
static struct attribute *sdio_dev_attrs[] = {
&dev_attr_class.attr,
&dev_attr_vendor.attr,
&dev_attr_device.attr,
+ &dev_attr_revision.attr,
+ &dev_attr_info1.attr,
+ &dev_attr_info2.attr,
+ &dev_attr_info3.attr,
+ &dev_attr_info4.attr,
&dev_attr_modalias.attr,
NULL,
};
sdio_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
{
struct sdio_func *func = dev_to_sdio_func(dev);
+ unsigned int i;
if (add_uevent_var(env,
"SDIO_CLASS=%02X", func->class))
"SDIO_ID=%04X:%04X", func->vendor, func->device))
return -ENOMEM;
+ if (add_uevent_var(env,
+ "SDIO_REVISION=%u.%u", func->major_rev, func->minor_rev))
+ return -ENOMEM;
+
+ for (i = 0; i < func->num_info; i++) {
+ if (add_uevent_var(env, "SDIO_INFO%u=%s", i+1, func->info[i]))
+ return -ENOMEM;
+ }
+
if (add_uevent_var(env,
"MODALIAS=sdio:c%02Xv%04Xd%04X",
func->class, func->vendor, func->device))
static int cistpl_vers_1(struct mmc_card *card, struct sdio_func *func,
const unsigned char *buf, unsigned size)
{
+ u8 major_rev, minor_rev;
unsigned i, nr_strings;
char **buffer, *string;
+ if (size < 2)
+ return 0;
+
+ major_rev = buf[0];
+ minor_rev = buf[1];
+
/* Find all null-terminated (including zero length) strings in
the TPLLV1_INFO field. Trailing garbage is ignored. */
buf += 2;
}
if (func) {
+ func->major_rev = major_rev;
+ func->minor_rev = minor_rev;
func->num_info = nr_strings;
func->info = (const char**)buffer;
} else {
+ card->major_rev = major_rev;
+ card->minor_rev = minor_rev;
card->num_info = nr_strings;
card->info = (const char**)buffer;
}
config MMC_SDHCI_OF_ESDHC
tristate "SDHCI OF support for the Freescale eSDHC controller"
depends on MMC_SDHCI_PLTFM
- depends on PPC || ARCH_MXC || ARCH_LAYERSCAPE
+ depends on PPC || ARCH_MXC || ARCH_LAYERSCAPE || COMPILE_TEST
select MMC_SDHCI_IO_ACCESSORS
select FSL_GUTS
help
If you have a controller with this interface, say Y or M here.
If unsure, say N.
+config MMC_SDHCI_OF_SPARX5
+ tristate "SDHCI OF support for the MCHP Sparx5 SoC"
+ depends on MMC_SDHCI_PLTFM
+ depends on ARCH_SPARX5 || COMPILE_TEST
+ help
+ This selects the Secure Digital Host Controller Interface (SDHCI)
+ found in the MCHP Sparx5 SoC.
+
+ If you have a Sparx5 SoC with this interface, say Y or M here.
+
+ If unsure, say N.
+
config MMC_SDHCI_CADENCE
tristate "SDHCI support for the Cadence SD/SDIO/eMMC controller"
depends on MMC_SDHCI_PLTFM
config MMC_SDHCI_CNS3XXX
tristate "SDHCI support on the Cavium Networks CNS3xxx SoC"
- depends on ARCH_CNS3XXX
+ depends on ARCH_CNS3XXX || COMPILE_TEST
depends on MMC_SDHCI_PLTFM
help
This selects the SDHCI support for CNS3xxx System-on-Chip devices.
config MMC_SDHCI_ESDHC_IMX
tristate "SDHCI support for the Freescale eSDHC/uSDHC i.MX controller"
- depends on ARCH_MXC
+ depends on ARCH_MXC || COMPILE_TEST
depends on MMC_SDHCI_PLTFM
select MMC_SDHCI_IO_ACCESSORS
select MMC_CQHCI
config MMC_SDHCI_DOVE
tristate "SDHCI support on Marvell's Dove SoC"
- depends on ARCH_DOVE || MACH_DOVE
+ depends on ARCH_DOVE || MACH_DOVE || COMPILE_TEST
depends on MMC_SDHCI_PLTFM
select MMC_SDHCI_IO_ACCESSORS
help
config MMC_SDHCI_TEGRA
tristate "SDHCI platform support for the Tegra SD/MMC Controller"
- depends on ARCH_TEGRA
+ depends on ARCH_TEGRA || COMPILE_TEST
depends on MMC_SDHCI_PLTFM
select MMC_SDHCI_IO_ACCESSORS
select MMC_CQHCI
config MMC_SDHCI_S3C
tristate "SDHCI support on Samsung S3C SoC"
- depends on MMC_SDHCI && PLAT_SAMSUNG
+ depends on MMC_SDHCI
+ depends on PLAT_SAMSUNG || COMPILE_TEST
help
This selects the Secure Digital Host Controller Interface (SDHCI)
often referrered to as the HSMMC block in some of the Samsung S3C
config MMC_SDHCI_SIRF
tristate "SDHCI support on CSR SiRFprimaII and SiRFmarco SoCs"
- depends on ARCH_SIRF
+ depends on ARCH_SIRF || COMPILE_TEST
depends on MMC_SDHCI_PLTFM
select MMC_SDHCI_IO_ACCESSORS
help
config MMC_SDHCI_SPEAR
tristate "SDHCI support on ST SPEAr platform"
- depends on MMC_SDHCI && PLAT_SPEAR
+ depends on MMC_SDHCI
+ depends on PLAT_SPEAR || COMPILE_TEST
depends on OF
help
This selects the Secure Digital Host Controller Interface (SDHCI)
config MMC_SDHCI_BCM_KONA
tristate "SDHCI support on Broadcom KONA platform"
- depends on ARCH_BCM_MOBILE
+ depends on ARCH_BCM_MOBILE || COMPILE_TEST
depends on MMC_SDHCI_PLTFM
help
This selects the Broadcom Kona Secure Digital Host Controller
config MMC_MESON_GX
tristate "Amlogic S905/GX*/AXG SD/MMC Host Controller support"
- depends on ARCH_MESON && MMC
+ depends on ARCH_MESON|| COMPILE_TEST
+ depends on COMMON_CLK
help
This selects support for the Amlogic SD/MMC Host Controller
found on the S905/GX*/AXG family of SoCs. This controller is
config MMC_MOXART
tristate "MOXART SD/MMC Host Controller support"
- depends on ARCH_MOXART && MMC
+ depends on ARCH_MOXART || COMPILE_TEST
help
This selects support for the MOXART SD/MMC Host Controller.
MOXA provides one multi-functional card reader which can
config MMC_SDHCI_ST
tristate "SDHCI support on STMicroelectronics SoC"
- depends on ARCH_STI || FSP2
+ depends on ARCH_STI || FSP2 || COMPILE_TEST
depends on MMC_SDHCI_PLTFM
select MMC_SDHCI_IO_ACCESSORS
help
config MMC_SDHCI_MSM
tristate "Qualcomm SDHCI Controller Support"
- depends on ARCH_QCOM || (ARM && COMPILE_TEST)
+ depends on ARCH_QCOM || COMPILE_TEST
depends on MMC_SDHCI_PLTFM
select MMC_SDHCI_IO_ACCESSORS
select MMC_CQHCI
config MMC_MVSDIO
tristate "Marvell MMC/SD/SDIO host driver"
- depends on PLAT_ORION
+ depends on PLAT_ORION || (COMPILE_TEST && ARM)
depends on OF
help
This selects the Marvell SDIO host driver.
config MMC_DAVINCI
tristate "TI DAVINCI Multimedia Card Interface support"
- depends on ARCH_DAVINCI
+ depends on ARCH_DAVINCI || COMPILE_TEST
help
This selects the TI DAVINCI Multimedia card Interface.
If you have an DAVINCI board with a Multimedia Card slot,
config MMC_SDHCI_SPRD
tristate "Spreadtrum SDIO host Controller"
- depends on ARCH_SPRD
+ depends on ARCH_SPRD || COMPILE_TEST
depends on MMC_SDHCI_PLTFM
select MMC_SDHCI_IO_ACCESSORS
select MMC_HSQ
config MMC_TMIO
tristate "Toshiba Mobile IO Controller (TMIO) MMC/SD function support"
- depends on MFD_TMIO || MFD_ASIC3
+ depends on MFD_TMIO || MFD_ASIC3 || COMPILE_TEST
select MMC_TMIO_CORE
help
This provides support for the SD/MMC cell found in TC6393XB,
config MMC_DW
tristate "Synopsys DesignWare Memory Card Interface"
- depends on ARC || ARM || ARM64 || MIPS || COMPILE_TEST
+ depends on ARC || ARM || ARM64 || MIPS || RISCV || CSKY || COMPILE_TEST
help
This selects support for the Synopsys DesignWare Mobile Storage IP
block, this provides host support for SD and MMC interfaces, in both
config MMC_SUNXI
tristate "Allwinner sunxi SD/MMC Host Controller support"
- depends on ARCH_SUNXI
+ depends on ARCH_SUNXI || COMPILE_TEST
help
This selects support for the SD/MMC Host Controller on
Allwinner sunxi SoCs.
obj-$(CONFIG_MMC_SDHCI_OF_ESDHC) += sdhci-of-esdhc.o
obj-$(CONFIG_MMC_SDHCI_OF_HLWD) += sdhci-of-hlwd.o
obj-$(CONFIG_MMC_SDHCI_OF_DWCMSHC) += sdhci-of-dwcmshc.o
+obj-$(CONFIG_MMC_SDHCI_OF_SPARX5) += sdhci-of-sparx5.o
obj-$(CONFIG_MMC_SDHCI_BCM_KONA) += sdhci-bcm-kona.o
obj-$(CONFIG_MMC_SDHCI_IPROC) += sdhci-iproc.o
obj-$(CONFIG_MMC_SDHCI_MSM) += sdhci-msm.o
.id_table = alcor_pci_sdmmc_ids,
.driver = {
.name = DRV_NAME_ALCOR_PCI_SDMMC,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &alcor_mmc_pm_ops
},
};
.remove = goldfish_mmc_remove,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
},
};
.remove = atmci_remove,
.driver = {
.name = "atmel_mci",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(atmci_dt_ids),
.pm = &atmci_dev_pm_ops,
},
.resume = au1xmmc_resume,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
},
};
clk = devm_clk_get(dev, NULL);
if (IS_ERR(clk)) {
- ret = PTR_ERR(clk);
- if (ret != -EPROBE_DEFER)
- dev_err(dev, "could not get clk: %d\n", ret);
+ ret = dev_err_probe(dev, PTR_ERR(clk), "could not get clk\n");
goto err;
}
.remove = bcm2835_remove,
.driver = {
.name = "sdhost-bcm2835",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = bcm2835_match,
},
};
.remove = octeon_mmc_remove,
.driver = {
.name = KBUILD_MODNAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = octeon_mmc_match,
},
};
else
pr_debug("%s: cqhci: CQE off\n", mmc_hostname(mmc));
+ if (cq_host->ops->post_disable)
+ cq_host->ops->post_disable(mmc);
+
mmc->cqe_on = false;
}
__cqhci_enable(cq_host);
if (!mmc->cqe_on) {
+ if (cq_host->ops->pre_enable)
+ cq_host->ops->pre_enable(mmc);
+
cqhci_writel(cq_host, 0, CQHCI_CTL);
mmc->cqe_on = true;
pr_debug("%s: cqhci: CQE on\n", mmc_hostname(mmc));
void (*disable)(struct mmc_host *mmc, bool recovery);
void (*update_dcmd_desc)(struct mmc_host *mmc, struct mmc_request *mrq,
u64 *data);
+ void (*pre_enable)(struct mmc_host *mmc);
+ void (*post_disable)(struct mmc_host *mmc);
};
static inline void cqhci_writel(struct cqhci_host *host, u32 val, int reg)
if (qstatus & MMCST0_RSPDNE) {
/* End of command phase */
- end_command = (int) host->cmd;
+ end_command = host->cmd ? 1 : 0;
}
if (end_command)
pdev->id_entry = match->data;
ret = mmc_of_parse(mmc);
if (ret) {
- if (ret != -EPROBE_DEFER)
- dev_err(&pdev->dev,
- "could not parse of data: %d\n", ret);
+ dev_err_probe(&pdev->dev, ret,
+ "could not parse of data\n");
goto parse_fail;
}
} else {
static struct platform_driver davinci_mmcsd_driver = {
.driver = {
.name = "davinci_mmc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = davinci_mmcsd_pm_ops,
.of_match_table = davinci_mmc_dt_ids,
},
.remove = dw_mci_pltfm_remove,
.driver = {
.name = "dwmmc_bluefield",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = dw_mci_bluefield_match,
.pm = &dw_mci_pltfm_pmops,
},
.remove = dw_mci_exynos_remove,
.driver = {
.name = "dwmmc_exynos",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = dw_mci_exynos_match,
.pm = &dw_mci_exynos_pmops,
},
.remove = dw_mci_hi3798cv200_remove,
.driver = {
.name = "dwmmc_hi3798cv200",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = dw_mci_hi3798cv200_match,
},
};
.remove = dw_mci_pltfm_remove,
.driver = {
.name = "dwmmc_k3",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = dw_mci_k3_match,
.pm = &dw_mci_k3_dev_pm_ops,
},
.remove = dw_mci_pltfm_remove,
.driver = {
.name = "dw_mmc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = dw_mci_pltfm_match,
.pm = &dw_mci_pltfm_pmops,
},
.remove = dw_mci_rockchip_remove,
.driver = {
.name = "dwmmc_rockchip",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = dw_mci_rockchip_match,
.pm = &dw_mci_rockchip_dev_pm_ops,
},
struct device_node *node;
struct dw_mci_zx_priv_data *priv;
struct regmap *sysc_base;
- int ret;
/* syscon is needed only by emmc */
node = of_parse_phandle(np, "zte,aon-syscon", 0);
sysc_base = syscon_node_to_regmap(node);
of_node_put(node);
- if (IS_ERR(sysc_base)) {
- ret = PTR_ERR(sysc_base);
- if (ret != -EPROBE_DEFER)
- dev_err(host->dev, "Can't get syscon: %d\n",
- ret);
- return ret;
- }
+ if (IS_ERR(sysc_base))
+ return dev_err_probe(host->dev, PTR_ERR(sysc_base),
+ "Can't get syscon\n");
} else {
return 0;
}
.remove = dw_mci_pltfm_remove,
.driver = {
.name = "dwmmc_zx",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = dw_mci_zx_match,
.pm = &dw_mci_zx_dev_pm_ops,
},
if (!host->pdata) {
host->pdata = dw_mci_parse_dt(host);
- if (PTR_ERR(host->pdata) == -EPROBE_DEFER) {
- return -EPROBE_DEFER;
- } else if (IS_ERR(host->pdata)) {
- dev_err(host->dev, "platform data not available\n");
- return -EINVAL;
- }
+ if (IS_ERR(host->pdata))
+ return dev_err_probe(host->dev, PTR_ERR(host->pdata),
+ "platform data not available\n");
}
host->biu_clk = devm_clk_get(host->dev, "biu");
ret = mmc_of_parse(mmc);
if (ret) {
- if (ret != -EPROBE_DEFER)
- dev_err(&pdev->dev,
- "could not parse device properties: %d\n", ret);
+ dev_err_probe(&pdev->dev, ret, "could not parse device properties\n");
goto err_free_host;
}
.remove = jz4740_mmc_remove,
.driver = {
.name = "jz4740-mmc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(jz4740_mmc_of_match),
.pm = pm_ptr(&jz4740_mmc_pm_ops),
},
snprintf(name, sizeof(name), "clkin%d", i);
clk = devm_clk_get(host->dev, name);
- if (IS_ERR(clk)) {
- if (clk != ERR_PTR(-EPROBE_DEFER))
- dev_err(host->dev, "Missing clock %s\n", name);
- return PTR_ERR(clk);
- }
+ if (IS_ERR(clk))
+ return dev_err_probe(host->dev, PTR_ERR(clk),
+ "Missing clock %s\n", name);
mux_parent_names[i] = __clk_get_name(clk);
}
val |= ADJUST_ADJ_EN;
writel(val, host->regs + host->data->adjust);
- if (mmc->doing_retune)
+ if (mmc_doing_retune(mmc))
dly = FIELD_GET(ADJUST_ADJ_DELAY_MASK, val) + 1;
else
dly = 0;
}
ret = device_reset_optional(&pdev->dev);
- if (ret) {
- if (ret != -EPROBE_DEFER)
- dev_err(&pdev->dev, "device reset failed: %d\n", ret);
-
- return ret;
- }
+ if (ret)
+ return dev_err_probe(&pdev->dev, ret, "device reset failed\n");
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
host->regs = devm_ioremap_resource(&pdev->dev, res);
.remove = meson_mmc_remove,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(meson_mmc_of_match),
},
};
.remove = meson_mx_sdhc_remove,
.driver = {
.name = "meson-mx-sdhc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(meson_mx_sdhc_of_match),
},
};
.remove = meson_mx_mmc_remove,
.driver = {
.name = "meson-mx-sdio",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(meson_mx_mmc_of_match),
},
};
else
clock_rate = spi->max_speed_hz;
- timeout = data->timeout_ns +
+ timeout = data->timeout_ns / 1000 +
data->timeout_clks * 1000000 / clock_rate;
- timeout = usecs_to_jiffies((unsigned int)(timeout / 1000)) + 1;
+ timeout = usecs_to_jiffies((unsigned int)timeout) + 1;
/* Handle scatterlist segments one at a time, with synch for
* each 512-byte block
DMA_BIDIRECTIONAL);
}
#else
-static inline mmc_spi_dma_alloc(struct mmc_spi_host *host) { return 0; }
+static inline int mmc_spi_dma_alloc(struct mmc_spi_host *host) { return 0; }
static inline void mmc_spi_dma_free(struct mmc_spi_host *host) {}
#endif
dev_set_drvdata(&pdev->dev, NULL);
- if (mmc) {
- if (!IS_ERR(host->dma_chan_tx))
- dma_release_channel(host->dma_chan_tx);
- if (!IS_ERR(host->dma_chan_rx))
- dma_release_channel(host->dma_chan_rx);
- mmc_remove_host(mmc);
- mmc_free_host(mmc);
+ if (!IS_ERR(host->dma_chan_tx))
+ dma_release_channel(host->dma_chan_tx);
+ if (!IS_ERR(host->dma_chan_rx))
+ dma_release_channel(host->dma_chan_rx);
+ mmc_remove_host(mmc);
+ mmc_free_host(mmc);
+
+ writel(0, host->base + REG_INTERRUPT_MASK);
+ writel(0, host->base + REG_POWER_CONTROL);
+ writel(readl(host->base + REG_CLOCK_CONTROL) | CLK_OFF,
+ host->base + REG_CLOCK_CONTROL);
- writel(0, host->base + REG_INTERRUPT_MASK);
- writel(0, host->base + REG_POWER_CONTROL);
- writel(readl(host->base + REG_CLOCK_CONTROL) | CLK_OFF,
- host->base + REG_CLOCK_CONTROL);
- }
return 0;
}
.remove = moxart_remove,
.driver = {
.name = "mmc-moxart",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = moxart_mmc_match,
},
};
struct msdc_host {
struct device *dev;
const struct mtk_mmc_compatible *dev_comp;
- struct mmc_host *mmc; /* mmc structure */
int cmd_rsp;
spinlock_t lock;
static u64 msdc_timeout_cal(struct msdc_host *host, u64 ns, u64 clks)
{
+ struct mmc_host *mmc = mmc_from_priv(host);
u64 timeout, clk_ns;
u32 mode = 0;
- if (host->mmc->actual_clock == 0) {
+ if (mmc->actual_clock == 0) {
timeout = 0;
} else {
clk_ns = 1000000000ULL;
- do_div(clk_ns, host->mmc->actual_clock);
+ do_div(clk_ns, mmc->actual_clock);
timeout = ns + clk_ns - 1;
do_div(timeout, clk_ns);
timeout += clks;
static void msdc_set_mclk(struct msdc_host *host, unsigned char timing, u32 hz)
{
+ struct mmc_host *mmc = mmc_from_priv(host);
u32 mode;
u32 flags;
u32 div;
if (!hz) {
dev_dbg(host->dev, "set mclk to 0\n");
host->mclk = 0;
- host->mmc->actual_clock = 0;
+ mmc->actual_clock = 0;
sdr_clr_bits(host->base + MSDC_CFG, MSDC_CFG_CKPDN);
return;
}
while (!(readl(host->base + MSDC_CFG) & MSDC_CFG_CKSTB))
cpu_relax();
sdr_set_bits(host->base + MSDC_CFG, MSDC_CFG_CKPDN);
- host->mmc->actual_clock = sclk;
+ mmc->actual_clock = sclk;
host->mclk = hz;
host->timing = timing;
/* need because clk changed. */
* mmc_select_hs400() will drop to 50Mhz and High speed mode,
* tune result of hs200/200Mhz is not suitable for 50Mhz
*/
- if (host->mmc->actual_clock <= 52000000) {
+ if (mmc->actual_clock <= 52000000) {
writel(host->def_tune_para.iocon, host->base + MSDC_IOCON);
if (host->top_base) {
writel(host->def_tune_para.emmc_top_control,
sdr_set_field(host->base + tune_reg,
MSDC_PAD_TUNE_CMDRRDLY,
host->hs400_cmd_int_delay);
- dev_dbg(host->dev, "sclk: %d, timing: %d\n", host->mmc->actual_clock,
+ dev_dbg(host->dev, "sclk: %d, timing: %d\n", mmc->actual_clock,
timing);
}
static inline u32 msdc_cmd_prepare_raw_cmd(struct msdc_host *host,
struct mmc_request *mrq, struct mmc_command *cmd)
{
+ struct mmc_host *mmc = mmc_from_priv(host);
/* rawcmd :
* vol_swt << 30 | auto_cmd << 28 | blklen << 16 | go_irq << 15 |
* stop << 14 | rw << 13 | dtype << 11 | rsptyp << 7 | brk << 6 | opcode
struct mmc_data *data = cmd->data;
if (mmc_op_multi(opcode)) {
- if (mmc_card_mmc(host->mmc->card) && mrq->sbc &&
+ if (mmc_card_mmc(mmc->card) && mrq->sbc &&
!(mrq->sbc->arg & 0xFFFF0000))
rawcmd |= 0x2 << 28; /* AutoCMD23 */
}
*/
static void msdc_recheck_sdio_irq(struct msdc_host *host)
{
+ struct mmc_host *mmc = mmc_from_priv(host);
u32 reg_int, reg_inten, reg_ps;
- if (host->mmc->caps & MMC_CAP_SDIO_IRQ) {
+ if (mmc->caps & MMC_CAP_SDIO_IRQ) {
reg_inten = readl(host->base + MSDC_INTEN);
if (reg_inten & MSDC_INTEN_SDIOIRQ) {
reg_int = readl(host->base + MSDC_INT);
if (!(reg_int & MSDC_INT_SDIOIRQ ||
reg_ps & MSDC_PS_DATA1)) {
__msdc_enable_sdio_irq(host, 0);
- sdio_signal_irq(host->mmc);
+ sdio_signal_irq(mmc);
}
}
}
msdc_unprepare_data(host, mrq);
if (host->error)
msdc_reset_hw(host);
- mmc_request_done(host->mmc, mrq);
+ mmc_request_done(mmc_from_priv(host), mrq);
if (host->dev_comp->recheck_sdio_irq)
msdc_recheck_sdio_irq(host);
}
static irqreturn_t msdc_cmdq_irq(struct msdc_host *host, u32 intsts)
{
+ struct mmc_host *mmc = mmc_from_priv(host);
int cmd_err = 0, dat_err = 0;
if (intsts & MSDC_INT_RSPCRCERR) {
cmd_err, dat_err, intsts);
}
- return cqhci_irq(host->mmc, 0, cmd_err, dat_err);
+ return cqhci_irq(mmc, 0, cmd_err, dat_err);
}
static irqreturn_t msdc_irq(int irq, void *dev_id)
{
struct msdc_host *host = (struct msdc_host *) dev_id;
+ struct mmc_host *mmc = mmc_from_priv(host);
while (true) {
unsigned long flags;
spin_unlock_irqrestore(&host->lock, flags);
if ((events & event_mask) & MSDC_INT_SDIOIRQ)
- sdio_signal_irq(host->mmc);
+ sdio_signal_irq(mmc);
if ((events & event_mask) & MSDC_INT_CDSC) {
if (host->internal_cd)
- mmc_detect_change(host->mmc, msecs_to_jiffies(20));
+ mmc_detect_change(mmc, msecs_to_jiffies(20));
events &= ~MSDC_INT_CDSC;
}
if (!(events & (event_mask & ~MSDC_INT_SDIOIRQ)))
break;
- if ((host->mmc->caps2 & MMC_CAP2_CQE) &&
+ if ((mmc->caps2 & MMC_CAP2_CQE) &&
(events & MSDC_INT_CMDQ)) {
msdc_cmdq_irq(host, events);
/* clear interrupts */
}
}
+static void msdc_cqe_pre_enable(struct mmc_host *mmc)
+{
+ struct cqhci_host *cq_host = mmc->cqe_private;
+ u32 reg;
+
+ reg = cqhci_readl(cq_host, CQHCI_CFG);
+ reg |= CQHCI_ENABLE;
+ cqhci_writel(cq_host, reg, CQHCI_CFG);
+}
+
+static void msdc_cqe_post_disable(struct mmc_host *mmc)
+{
+ struct cqhci_host *cq_host = mmc->cqe_private;
+ u32 reg;
+
+ reg = cqhci_readl(cq_host, CQHCI_CFG);
+ reg &= ~CQHCI_ENABLE;
+ cqhci_writel(cq_host, reg, CQHCI_CFG);
+}
+
static const struct mmc_host_ops mt_msdc_ops = {
.post_req = msdc_post_req,
.pre_req = msdc_pre_req,
static const struct cqhci_host_ops msdc_cmdq_ops = {
.enable = msdc_cqe_enable,
.disable = msdc_cqe_disable,
+ .pre_enable = msdc_cqe_pre_enable,
+ .post_disable = msdc_cqe_post_disable,
};
static void msdc_of_property_parse(struct platform_device *pdev,
host->dev = &pdev->dev;
host->dev_comp = of_device_get_match_data(&pdev->dev);
- host->mmc = mmc;
host->src_clk_freq = clk_get_rate(host->src_clk);
/* Set host parameters to mmc */
mmc->ops = &mt_msdc_ops;
mmc_dev(mmc)->dma_mask = &host->dma_mask;
if (mmc->caps2 & MMC_CAP2_CQE) {
- host->cq_host = devm_kzalloc(host->mmc->parent,
+ host->cq_host = devm_kzalloc(mmc->parent,
sizeof(*host->cq_host),
GFP_KERNEL);
if (!host->cq_host) {
pm_runtime_get_sync(host->dev);
platform_set_drvdata(pdev, NULL);
- mmc_remove_host(host->mmc);
+ mmc_remove_host(mmc);
msdc_deinit_hw(host);
msdc_gate_clock(host);
dma_free_coherent(&pdev->dev, MAX_BD_NUM * sizeof(struct mt_bdma_desc),
host->dma.bd, host->dma.bd_addr);
- mmc_free_host(host->mmc);
+ mmc_free_host(mmc);
return 0;
}
static void msdc_restore_reg(struct msdc_host *host)
{
+ struct mmc_host *mmc = mmc_from_priv(host);
u32 tune_reg = host->dev_comp->pad_tune_reg;
writel(host->save_para.msdc_cfg, host->base + MSDC_CFG);
writel(host->save_para.pad_tune, host->base + tune_reg);
}
- if (sdio_irq_claimed(host->mmc))
+ if (sdio_irq_claimed(mmc))
__msdc_enable_sdio_irq(host, 1);
}
.remove = msdc_drv_remove,
.driver = {
.name = "mtk-msdc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = msdc_of_ids,
.pm = &msdc_dev_pm_ops,
},
.remove = mvsd_remove,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = mvsdio_dt_ids,
},
};
.id_table = mxcmci_devtype,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &mxcmci_pm_ops,
.of_match_table = mxcmci_of_match,
}
.id_table = mxs_ssp_ids,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &mxs_mmc_pm_ops,
.of_match_table = mxs_mmc_dt_ids,
},
.remove = mmc_omap_remove,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(mmc_omap_match),
},
};
int ret;
/* Disable the clocks */
- if (host->dbclk)
- clk_disable_unprepare(host->dbclk);
+ clk_disable_unprepare(host->dbclk);
/* Turn the power off */
ret = omap_hsmmc_set_power(host, 0);
/* Turn the power ON with given VDD 1.8 or 3.0v */
if (!ret)
ret = omap_hsmmc_set_power(host, 1);
- if (host->dbclk)
- clk_prepare_enable(host->dbclk);
+ clk_prepare_enable(host->dbclk);
if (ret != 0)
goto err;
pm_runtime_dont_use_autosuspend(host->dev);
pm_runtime_put_sync(host->dev);
pm_runtime_disable(host->dev);
- if (host->dbclk)
- clk_disable_unprepare(host->dbclk);
+ clk_disable_unprepare(host->dbclk);
err1:
mmc_free_host(mmc);
err:
pm_runtime_put_sync(host->dev);
pm_runtime_disable(host->dev);
device_init_wakeup(&pdev->dev, false);
- if (host->dbclk)
- clk_disable_unprepare(host->dbclk);
+ clk_disable_unprepare(host->dbclk);
mmc_free_host(host->mmc);
OMAP_HSMMC_READ(host->base, HCTL) & ~SDBP);
}
- if (host->dbclk)
- clk_disable_unprepare(host->dbclk);
+ clk_disable_unprepare(host->dbclk);
pm_runtime_put_sync(host->dev);
return 0;
pm_runtime_get_sync(host->dev);
- if (host->dbclk)
- clk_prepare_enable(host->dbclk);
+ clk_prepare_enable(host->dbclk);
if (!(host->mmc->pm_flags & MMC_PM_KEEP_POWER))
omap_hsmmc_conf_bus_power(host);
.remove = omap_hsmmc_remove,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &omap_hsmmc_dev_pm_ops,
.of_match_table = of_match_ptr(omap_mmc_of_match),
},
static struct platform_driver owl_mmc_driver = {
.driver = {
.name = "owl_mmc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = owl_mmc_of_match,
},
.probe = owl_mmc_probe,
.remove = pxamci_remove,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(pxa_mmc_dt_ids),
},
};
unsigned short max_segs;
};
+#define SDHI_CALIB_TABLE_MAX 32
+
struct renesas_sdhi_quirks {
bool hs400_disabled;
bool hs400_4taps;
u32 hs400_bad_taps;
+ const u8 (*hs400_calib_table)[SDHI_CALIB_TABLE_MAX];
};
struct tmio_mmc_dma {
void __iomem *scc_ctl;
u32 scc_tappos;
u32 scc_tappos_hs400;
- bool doing_tune;
+ const u8 *adjust_hs400_calib_table;
+ bool needs_adjust_hs400;
/* Tuning values: 1 for success, 0 for failure */
DECLARE_BITMAP(taps, BITS_PER_LONG);
#include <linux/platform_device.h>
#include <linux/pm_domain.h>
#include <linux/mmc/host.h>
+#include <linux/mmc/mmc.h>
#include <linux/mmc/slot-gpio.h>
#include <linux/mfd/tmio.h>
#include <linux/sh_dma.h>
#define SDHI_VER_GEN3_SD 0xcc10
#define SDHI_VER_GEN3_SDMMC 0xcd10
+#define SDHI_GEN3_MMC0_ADDR 0xee140000
+
static void renesas_sdhi_sdbuf_width(struct tmio_mmc_host *host, int width)
{
u32 val;
unsigned int freq, diff, best_freq = 0, diff_min = ~0;
int i;
- /* tested only on R-Car Gen2+ currently; may work for others */
- if (!(host->pdata->flags & TMIO_MMC_MIN_RCAR2))
+ /*
+ * We simply return the current rate if a) we are not on a R-Car Gen2+
+ * SoC (may work for others, but untested) or b) if the SCC needs its
+ * clock during tuning, so we don't change the external clock setup.
+ */
+ if (!(host->pdata->flags & TMIO_MMC_MIN_RCAR2) || mmc_doing_tune(host->mmc))
return clk_get_rate(priv->clk);
/*
#define SH_MOBILE_SDHI_SCC_RVSREQ 0x00A
#define SH_MOBILE_SDHI_SCC_SMPCMP 0x00C
#define SH_MOBILE_SDHI_SCC_TMPPORT2 0x00E
+#define SH_MOBILE_SDHI_SCC_TMPPORT3 0x014
+#define SH_MOBILE_SDHI_SCC_TMPPORT4 0x016
+#define SH_MOBILE_SDHI_SCC_TMPPORT5 0x018
+#define SH_MOBILE_SDHI_SCC_TMPPORT6 0x01A
+#define SH_MOBILE_SDHI_SCC_TMPPORT7 0x01C
#define SH_MOBILE_SDHI_SCC_DTCNTL_TAPEN BIT(0)
#define SH_MOBILE_SDHI_SCC_DTCNTL_TAPNUM_SHIFT 16
#define SH_MOBILE_SDHI_SCC_TMPPORT2_HS400OSEL BIT(4)
#define SH_MOBILE_SDHI_SCC_TMPPORT2_HS400EN BIT(31)
+/* Definitions for values the SH_MOBILE_SDHI_SCC_TMPPORT4 register */
+#define SH_MOBILE_SDHI_SCC_TMPPORT4_DLL_ACC_START BIT(0)
+
+/* Definitions for values the SH_MOBILE_SDHI_SCC_TMPPORT5 register */
+#define SH_MOBILE_SDHI_SCC_TMPPORT5_DLL_RW_SEL_R BIT(8)
+#define SH_MOBILE_SDHI_SCC_TMPPORT5_DLL_RW_SEL_W (0 << 8)
+#define SH_MOBILE_SDHI_SCC_TMPPORT5_DLL_ADR_MASK 0x3F
+
+/* Definitions for values the SH_MOBILE_SDHI_SCC register */
+#define SH_MOBILE_SDHI_SCC_TMPPORT_DISABLE_WP_CODE 0xa5000000
+#define SH_MOBILE_SDHI_SCC_TMPPORT_CALIB_CODE_MASK 0x1f
+#define SH_MOBILE_SDHI_SCC_TMPPORT_MANUAL_MODE BIT(7)
+
+static const u8 r8a7796_es13_calib_table[2][SDHI_CALIB_TABLE_MAX] = {
+ { 3, 3, 3, 3, 3, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 15,
+ 16, 16, 16, 16, 16, 16, 17, 18, 18, 19, 20, 21, 22, 23, 24, 25 },
+ { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 8, 11,
+ 12, 17, 18, 18, 18, 18, 18, 18, 18, 19, 20, 21, 22, 23, 25, 25 }
+};
+
+static const u8 r8a77965_calib_table[2][SDHI_CALIB_TABLE_MAX] = {
+ { 1, 2, 6, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 15, 15, 16,
+ 17, 18, 19, 20, 21, 22, 23, 24, 25, 25, 26, 27, 28, 29, 30, 31 },
+ { 2, 3, 4, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17,
+ 17, 17, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30, 31, 31, 31, 31 }
+};
+
+static const u8 r8a77990_calib_table[2][SDHI_CALIB_TABLE_MAX] = {
+ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+ { 0, 0, 0, 1, 2, 3, 3, 4, 4, 4, 5, 5, 6, 8, 9, 10,
+ 11, 12, 13, 15, 16, 17, 17, 18, 18, 19, 20, 22, 24, 25, 26, 26 }
+};
+
static inline u32 sd_scc_read32(struct tmio_mmc_host *host,
struct renesas_sdhi *priv, int addr)
{
sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, CLK_CTL_SCLKEN |
sd_ctrl_read16(host, CTL_SD_CARD_CLK_CTL));
+
+ if (priv->adjust_hs400_calib_table)
+ priv->needs_adjust_hs400 = true;
}
static void renesas_sdhi_reset_scc(struct tmio_mmc_host *host,
sd_ctrl_read16(host, CTL_SD_CARD_CLK_CTL));
}
+static u32 sd_scc_tmpport_read32(struct tmio_mmc_host *host,
+ struct renesas_sdhi *priv, u32 addr)
+{
+ /* read mode */
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT5,
+ SH_MOBILE_SDHI_SCC_TMPPORT5_DLL_RW_SEL_R |
+ (SH_MOBILE_SDHI_SCC_TMPPORT5_DLL_ADR_MASK & addr));
+
+ /* access start and stop */
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT4,
+ SH_MOBILE_SDHI_SCC_TMPPORT4_DLL_ACC_START);
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT4, 0);
+
+ return sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT7);
+}
+
+static void sd_scc_tmpport_write32(struct tmio_mmc_host *host,
+ struct renesas_sdhi *priv, u32 addr, u32 val)
+{
+ /* write mode */
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT5,
+ SH_MOBILE_SDHI_SCC_TMPPORT5_DLL_RW_SEL_W |
+ (SH_MOBILE_SDHI_SCC_TMPPORT5_DLL_ADR_MASK & addr));
+
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT6, val);
+
+ /* access start and stop */
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT4,
+ SH_MOBILE_SDHI_SCC_TMPPORT4_DLL_ACC_START);
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT4, 0);
+}
+
+static void renesas_sdhi_adjust_hs400_mode_enable(struct tmio_mmc_host *host)
+{
+ struct renesas_sdhi *priv = host_to_priv(host);
+ u32 calib_code;
+
+ /* disable write protect */
+ sd_scc_tmpport_write32(host, priv, 0x00,
+ SH_MOBILE_SDHI_SCC_TMPPORT_DISABLE_WP_CODE);
+ /* read calibration code and adjust */
+ calib_code = sd_scc_tmpport_read32(host, priv, 0x26);
+ calib_code &= SH_MOBILE_SDHI_SCC_TMPPORT_CALIB_CODE_MASK;
+
+ sd_scc_tmpport_write32(host, priv, 0x22,
+ SH_MOBILE_SDHI_SCC_TMPPORT_MANUAL_MODE |
+ priv->adjust_hs400_calib_table[calib_code]);
+
+ /* set offset value to TMPPORT3, hardcoded to OFFSET0 (= 0x3) for now */
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT3, 0x3);
+
+ /* adjustment done, clear flag */
+ priv->needs_adjust_hs400 = false;
+}
+
+static void renesas_sdhi_adjust_hs400_mode_disable(struct tmio_mmc_host *host)
+{
+ struct renesas_sdhi *priv = host_to_priv(host);
+
+ /* disable write protect */
+ sd_scc_tmpport_write32(host, priv, 0x00,
+ SH_MOBILE_SDHI_SCC_TMPPORT_DISABLE_WP_CODE);
+ /* disable manual calibration */
+ sd_scc_tmpport_write32(host, priv, 0x22, 0);
+ /* clear offset value of TMPPORT3 */
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT3, 0);
+}
+
static void renesas_sdhi_reset_hs400_mode(struct tmio_mmc_host *host,
struct renesas_sdhi *priv)
{
SH_MOBILE_SDHI_SCC_TMPPORT2_HS400OSEL) &
sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_TMPPORT2));
+ if (priv->adjust_hs400_calib_table)
+ renesas_sdhi_adjust_hs400_mode_disable(host);
+
sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, CLK_CTL_SCLKEN |
sd_ctrl_read16(host, CTL_SD_CARD_CLK_CTL));
}
return 0;
}
+static void renesas_sdhi_reset(struct tmio_mmc_host *host)
+{
+ struct renesas_sdhi *priv = host_to_priv(host);
+
+ renesas_sdhi_reset_scc(host, priv);
+ renesas_sdhi_reset_hs400_mode(host, priv);
+ priv->needs_adjust_hs400 = false;
+
+ sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, CLK_CTL_SCLKEN |
+ sd_ctrl_read16(host, CTL_SD_CARD_CLK_CTL));
+
+ sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL,
+ ~SH_MOBILE_SDHI_SCC_RVSCNTL_RVSEN &
+ sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL));
+
+ if (host->pdata->flags & TMIO_MMC_MIN_RCAR2)
+ sd_ctrl_write32_as_16_and_16(host, CTL_IRQ_MASK,
+ TMIO_MASK_INIT_RCAR2);
+}
+
+/*
+ * This is a temporary workaround! This driver used 'hw_reset' wrongly and the
+ * fix for that showed a regression. So, we mimic the old behaviour until the
+ * proper solution is found.
+ */
+static void renesas_sdhi_hw_reset(struct mmc_host *mmc)
+{
+ struct tmio_mmc_host *host = mmc_priv(mmc);
+ renesas_sdhi_reset(host);
+}
+
#define SH_MOBILE_SDHI_MIN_TAP_ROW 3
static int renesas_sdhi_select_tuning(struct tmio_mmc_host *host)
unsigned int taps_size = priv->tap_num * 2, min_tap_row;
unsigned long *bitmap;
- priv->doing_tune = false;
sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_RVSREQ, 0);
/*
return 0;
}
-static int renesas_sdhi_execute_tuning(struct tmio_mmc_host *host, u32 opcode)
+static int renesas_sdhi_execute_tuning(struct mmc_host *mmc, u32 opcode)
{
+ struct tmio_mmc_host *host = mmc_priv(mmc);
struct renesas_sdhi *priv = host_to_priv(host);
- int i;
+ int i, ret;
priv->tap_num = renesas_sdhi_init_tuning(host);
if (!priv->tap_num)
return -EINVAL;
}
- priv->doing_tune = true;
bitmap_zero(priv->taps, priv->tap_num * 2);
bitmap_zero(priv->smpcmp, priv->tap_num * 2);
/* Set sampling clock position */
sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_TAPSET, i % priv->tap_num);
- if (mmc_send_tuning(host->mmc, opcode, NULL) == 0)
+ if (mmc_send_tuning(mmc, opcode, NULL) == 0)
set_bit(i, priv->taps);
if (sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_SMPCMP) == 0)
set_bit(i, priv->smpcmp);
}
- return renesas_sdhi_select_tuning(host);
+ ret = renesas_sdhi_select_tuning(host);
+ if (ret < 0)
+ renesas_sdhi_reset(host);
+ return ret;
}
static bool renesas_sdhi_manual_correction(struct tmio_mmc_host *host, bool use_4tap)
!(host->mmc->ios.timing == MMC_TIMING_MMC_HS400 && !use_4tap))
return false;
- if (mmc_doing_retune(host->mmc) || priv->doing_tune)
+ if (mmc_doing_tune(host->mmc))
return false;
if (sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL) &
return renesas_sdhi_manual_correction(host, use_4tap);
}
-static void renesas_sdhi_hw_reset(struct tmio_mmc_host *host)
-{
- struct renesas_sdhi *priv;
-
- priv = host_to_priv(host);
-
- renesas_sdhi_reset_scc(host, priv);
- renesas_sdhi_reset_hs400_mode(host, priv);
-
- sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, CLK_CTL_SCLKEN |
- sd_ctrl_read16(host, CTL_SD_CARD_CLK_CTL));
-
- sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL,
- ~SH_MOBILE_SDHI_SCC_RVSCNTL_RVSEN &
- sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL));
-
- if (host->pdata->flags & TMIO_MMC_MIN_RCAR2)
- sd_ctrl_write32_as_16_and_16(host, CTL_IRQ_MASK,
- TMIO_MASK_INIT_RCAR2);
-}
-
static int renesas_sdhi_wait_idle(struct tmio_mmc_host *host, u32 bit)
{
int timeout = 1000;
return blk_size;
}
+static void renesas_sdhi_fixup_request(struct tmio_mmc_host *host, struct mmc_request *mrq)
+{
+ struct renesas_sdhi *priv = host_to_priv(host);
+
+ if (priv->needs_adjust_hs400 && mrq->cmd->opcode == MMC_SEND_STATUS)
+ renesas_sdhi_adjust_hs400_mode_enable(host);
+}
static void renesas_sdhi_enable_dma(struct tmio_mmc_host *host, bool enable)
{
/* Iff regs are 8 byte apart, sdbuf is 64 bit. Otherwise always 32. */
.hs400_bad_taps = BIT(2) | BIT(3) | BIT(6) | BIT(7),
};
+static const struct renesas_sdhi_quirks sdhi_quirks_r8a7796_es13 = {
+ .hs400_4taps = true,
+ .hs400_bad_taps = BIT(2) | BIT(3) | BIT(6) | BIT(7),
+ .hs400_calib_table = r8a7796_es13_calib_table,
+};
+
+static const struct renesas_sdhi_quirks sdhi_quirks_r8a77965 = {
+ .hs400_bad_taps = BIT(2) | BIT(3) | BIT(6) | BIT(7),
+ .hs400_calib_table = r8a77965_calib_table,
+};
+
+static const struct renesas_sdhi_quirks sdhi_quirks_r8a77990 = {
+ .hs400_calib_table = r8a77990_calib_table,
+};
+
/*
* Note for r8a7796 / r8a774a1: we can't distinguish ES1.1 and 1.2 as of now.
* So, we want to treat them equally and only have a match for ES1.2 to enforce
{ .soc_id = "r8a7795", .revision = "ES2.0", .data = &sdhi_quirks_4tap },
{ .soc_id = "r8a7795", .revision = "ES3.*", .data = &sdhi_quirks_bad_taps2367 },
{ .soc_id = "r8a7796", .revision = "ES1.[012]", .data = &sdhi_quirks_4tap_nohs400 },
- { .soc_id = "r8a7796", .revision = "ES1.*", .data = &sdhi_quirks_4tap },
+ { .soc_id = "r8a7796", .revision = "ES1.*", .data = &sdhi_quirks_r8a7796_es13 },
{ .soc_id = "r8a7796", .revision = "ES3.*", .data = &sdhi_quirks_bad_taps1357 },
- { .soc_id = "r8a77965", .data = &sdhi_quirks_bad_taps2367 },
+ { .soc_id = "r8a77965", .data = &sdhi_quirks_r8a77965 },
{ .soc_id = "r8a77980", .data = &sdhi_quirks_nohs400 },
+ { .soc_id = "r8a77990", .data = &sdhi_quirks_r8a77990 },
{ /* Sentinel. */ },
};
renesas_sdhi_start_signal_voltage_switch;
host->sdcard_irq_setbit_mask = TMIO_STAT_ALWAYS_SET_27;
- /* SDR and HS200/400 registers requires HW reset */
if (of_data && of_data->scc_offset) {
priv->scc_ctl = host->ctl + of_data->scc_offset;
+ host->reset = renesas_sdhi_reset;
+ host->ops.hw_reset = renesas_sdhi_hw_reset;
host->mmc->caps |= MMC_CAP_HW_RESET;
- host->hw_reset = renesas_sdhi_hw_reset;
}
}
if (ver == SDHI_VER_GEN2_SDR50)
mmc_data->flags &= ~TMIO_MMC_HAVE_CBSY;
+ if (ver == SDHI_VER_GEN3_SDMMC && quirks && quirks->hs400_calib_table) {
+ host->fixup_request = renesas_sdhi_fixup_request;
+ priv->adjust_hs400_calib_table = *(
+ res->start == SDHI_GEN3_MMC0_ADDR ?
+ quirks->hs400_calib_table :
+ quirks->hs400_calib_table + 1);
+ }
+
ret = tmio_mmc_host_probe(host);
if (ret < 0)
goto edisclk;
if (!hit)
dev_warn(&host->pdev->dev, "Unknown clock rate for tuning\n");
- host->execute_tuning = renesas_sdhi_execute_tuning;
host->check_retune = renesas_sdhi_check_scc_error;
+ host->ops.execute_tuning = renesas_sdhi_execute_tuning;
host->ops.prepare_hs400_tuning = renesas_sdhi_prepare_hs400_tuning;
host->ops.hs400_downgrade = renesas_sdhi_disable_scc;
host->ops.hs400_complete = renesas_sdhi_hs400_complete;
if (soc)
global_flags |= (unsigned long)soc->data;
- dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL);
- if (!dev->dma_parms)
- return -ENOMEM;
-
/* value is max of SD_SECCNT. Confirmed by HW engineers */
dma_set_max_seg_size(dev, 0xffffffff);
static struct platform_driver renesas_internal_dmac_sdhi_driver = {
.driver = {
.name = "renesas_sdhi_internal_dmac",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &renesas_sdhi_internal_dmac_dev_pm_ops,
.of_match_table = renesas_sdhi_internal_dmac_of_match,
},
static struct platform_driver renesas_sys_dmac_sdhi_driver = {
.driver = {
.name = "sh_mobile_sdhi",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &renesas_sdhi_sys_dmac_dev_pm_ops,
.of_match_table = renesas_sdhi_sys_dmac_of_match,
},
.id_table = rtsx_pci_sdmmc_ids,
.driver = {
.name = DRV_NAME_RTSX_PCI_SDMMC,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
},
};
module_platform_driver(rtsx_pci_sdmmc_driver);
static int sd_change_phase(struct rtsx_usb_sdmmc *host, u8 sample_point, int tx)
{
struct rtsx_ucr *ucr = host->ucr;
- int err;
dev_dbg(sdmmc_dev(host), "%s: %s sample_point = %d\n",
__func__, tx ? "TX" : "RX", sample_point);
rtsx_usb_add_cmd(ucr, WRITE_REG_CMD, CLK_DIV, CLK_CHANGE, 0);
rtsx_usb_add_cmd(ucr, WRITE_REG_CMD, SD_CFG1, SD_ASYNC_FIFO_RST, 0);
- err = rtsx_usb_send_cmd(ucr, MODE_C, 100);
- if (err)
- return err;
-
- return 0;
+ return rtsx_usb_send_cmd(ucr, MODE_C, 100);
}
static inline u32 get_phase_point(u32 phase_map, unsigned int idx)
.id_table = rtsx_usb_sdmmc_ids,
.driver = {
.name = "rtsx_usb_sdmmc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &rtsx_usb_sdmmc_dev_pm_ops,
},
};
static void dbg_dumpregs(struct s3cmci_host *host, char *prefix)
{
- u32 con, pre, cmdarg, cmdcon, cmdsta, r0, r1, r2, r3, timer, bsize;
- u32 datcon, datcnt, datsta, fsta, imask;
+ u32 con, pre, cmdarg, cmdcon, cmdsta, r0, r1, r2, r3, timer;
+ u32 datcon, datcnt, datsta, fsta;
con = readl(host->base + S3C2410_SDICON);
pre = readl(host->base + S3C2410_SDIPRE);
r2 = readl(host->base + S3C2410_SDIRSP2);
r3 = readl(host->base + S3C2410_SDIRSP3);
timer = readl(host->base + S3C2410_SDITIMER);
- bsize = readl(host->base + S3C2410_SDIBSIZE);
datcon = readl(host->base + S3C2410_SDIDCON);
datcnt = readl(host->base + S3C2410_SDIDCNT);
datsta = readl(host->base + S3C2410_SDIDSTA);
fsta = readl(host->base + S3C2410_SDIFSTA);
- imask = readl(host->base + host->sdiimsk);
dbg(host, dbg_debug, "%s CON:[%08x] PRE:[%08x] TMR:[%08x]\n",
prefix, con, pre, timer);
local_irq_restore(flags);
}
-/**
- *
- */
static void s3cmci_disable_irq(struct s3cmci_host *host, bool transfer)
{
unsigned long flags;
{
struct s3cmci_host *host = seq->private;
- seq_printf(seq, "Register base = 0x%08x\n", (u32)host->base);
+ seq_printf(seq, "Register base = 0x%p\n", host->base);
seq_printf(seq, "Clock rate = %ld\n", host->clk_rate);
seq_printf(seq, "Prescale = %d\n", host->prescaler);
seq_printf(seq, "is2440 = %d\n", host->is2440);
struct mmc_host *mmc = host->mmc;
int ret;
- host->is2440 = (int) of_device_get_match_data(&pdev->dev);
+ host->is2440 = (long) of_device_get_match_data(&pdev->dev);
ret = mmc_of_parse(mmc);
if (ret)
static struct platform_driver s3cmci_driver = {
.driver = {
.name = "s3c-sdi",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = s3cmci_dt_match,
},
.id_table = s3cmci_driver_ids,
(host->mmc->caps & MMC_CAP_1_8V_DDR))
host->mmc->caps2 = MMC_CAP2_HS400_1_8V;
+ /*
+ * There are two types of presets out in the wild:
+ * 1) Default/broken presets.
+ * These presets have two sets of problems:
+ * a) The clock divisor for SDR12, SDR25, and SDR50 is too small.
+ * This results in clock frequencies that are 2x higher than
+ * acceptable. i.e., SDR12 = 25 MHz, SDR25 = 50 MHz, SDR50 =
+ * 100 MHz.x
+ * b) The HS200 and HS400 driver strengths don't match.
+ * By default, the SDR104 preset register has a driver strength of
+ * A, but the (internal) HS400 preset register has a driver
+ * strength of B. As part of initializing HS400, HS200 tuning
+ * needs to be performed. Having different driver strengths
+ * between tuning and operation is wrong. It results in different
+ * rise/fall times that lead to incorrect sampling.
+ * 2) Firmware with properly initialized presets.
+ * These presets have proper clock divisors. i.e., SDR12 => 12MHz,
+ * SDR25 => 25 MHz, SDR50 => 50 MHz. Additionally the HS200 and
+ * HS400 preset driver strengths match.
+ *
+ * Enabling presets for HS400 doesn't work for the following reasons:
+ * 1) sdhci_set_ios has a hard coded list of timings that are used
+ * to determine if presets should be enabled.
+ * 2) sdhci_get_preset_value is using a non-standard register to
+ * read out HS400 presets. The AMD controller doesn't support this
+ * non-standard register. In fact, it doesn't expose the HS400
+ * preset register anywhere in the SDHCI memory map. This results
+ * in reading a garbage value and using the wrong presets.
+ *
+ * Since HS400 and HS200 presets must be identical, we could
+ * instead use the the SDR104 preset register.
+ *
+ * If the above issues are resolved we could remove this quirk for
+ * firmware that that has valid presets (i.e., SDR12 <= 12 MHz).
+ */
+ host->quirks2 |= SDHCI_QUIRK2_PRESET_VALUE_BROKEN;
+
host->mmc_host_ops.select_drive_strength = amd_select_drive_strength;
host->mmc_host_ops.set_ios = amd_set_ios;
host->mmc_host_ops.execute_tuning = amd_sdhci_execute_tuning;
static struct platform_driver sdhci_acpi_driver = {
.driver = {
.name = "sdhci-acpi",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.acpi_match_table = sdhci_acpi_ids,
.pm = &sdhci_acpi_pm_ops,
},
static struct platform_driver sdhci_bcm_kona_driver = {
.driver = {
.name = "sdhci-kona",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &sdhci_pltfm_pmops,
.of_match_table = sdhci_bcm_kona_of_match,
},
dev_dbg(&pdev->dev, "Probe found match for %s\n", match->compatible);
- clk = devm_clk_get(&pdev->dev, NULL);
- if (IS_ERR(clk)) {
- if (PTR_ERR(clk) == -EPROBE_DEFER)
- return -EPROBE_DEFER;
- dev_err(&pdev->dev, "Clock not found in Device Tree\n");
- clk = NULL;
- }
+ clk = devm_clk_get_optional(&pdev->dev, NULL);
+ if (IS_ERR(clk))
+ return dev_err_probe(&pdev->dev, PTR_ERR(clk),
+ "Failed to get clock from Device Tree\n");
+
res = clk_prepare_enable(clk);
if (res)
return res;
static struct platform_driver sdhci_brcmstb_driver = {
.driver = {
.name = "sdhci-brcmstb",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &sdhci_pltfm_pmops,
.of_match_table = of_match_ptr(sdhci_brcm_of_match),
},
static struct platform_driver sdhci_cdns_driver = {
.driver = {
.name = "sdhci-cdns",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &sdhci_cdns_pm_ops,
.of_match_table = sdhci_cdns_match,
},
static struct platform_driver sdhci_cns3xxx_driver = {
.driver = {
.name = "sdhci-cns3xxx",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &sdhci_pltfm_pmops,
},
.probe = sdhci_cns3xxx_probe,
static struct platform_driver sdhci_dove_driver = {
.driver = {
.name = "sdhci-dove",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &sdhci_pltfm_pmops,
.of_match_table = sdhci_dove_of_match_table,
},
static void esdhc_prepare_tuning(struct sdhci_host *host, u32 val)
{
u32 reg;
+ u8 sw_rst;
+ int ret;
/* FIXME: delay a bit for card to be ready for next tuning due to errors */
mdelay(1);
+ /* IC suggest to reset USDHC before every tuning command */
+ esdhc_clrset_le(host, 0xff, SDHCI_RESET_ALL, SDHCI_SOFTWARE_RESET);
+ ret = readb_poll_timeout(host->ioaddr + SDHCI_SOFTWARE_RESET, sw_rst,
+ !(sw_rst & SDHCI_RESET_ALL), 10, 100);
+ if (ret == -ETIMEDOUT)
+ dev_warn(mmc_dev(host->mmc),
+ "warning! RESET_ALL never complete before sending tuning command\n");
+
reg = readl(host->ioaddr + ESDHC_MIX_CTRL);
reg |= ESDHC_MIX_CTRL_EXE_TUNE | ESDHC_MIX_CTRL_SMPCLK_SEL |
ESDHC_MIX_CTRL_FBCLK_SEL;
* response, block the tuning procedure or the first command
* after the whole tuning procedure always can't get any response.
*/
- tmp |= ESDHC_TUNING_CMD_CRC_CHECK_DISABLE;
+ tmp |= ESDHC_TUNING_CMD_CRC_CHECK_DISABLE;
writel(tmp, host->ioaddr + ESDHC_TUNING_CTRL);
} else if (imx_data->socdata->flags & ESDHC_FLAG_MAN_TUNING) {
/*
goto disable_ipg_clk;
imx_data->pinctrl = devm_pinctrl_get(&pdev->dev);
- if (IS_ERR(imx_data->pinctrl)) {
- err = PTR_ERR(imx_data->pinctrl);
+ if (IS_ERR(imx_data->pinctrl))
dev_warn(mmc_dev(host->mmc), "could not get pinctrl\n");
- }
if (esdhc_is_usdhc(imx_data)) {
host->quirks2 |= SDHCI_QUIRK2_PRESET_VALUE_BROKEN;
static struct platform_driver sdhci_esdhc_imx_driver = {
.driver = {
.name = "sdhci-esdhc-imx",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = imx_esdhc_dt_ids,
.pm = &sdhci_esdhc_pmops,
},
static struct platform_driver sdhci_esdhc_mcf_driver = {
.driver = {
.name = "sdhci-esdhc-mcf",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
},
.probe = sdhci_esdhc_mcf_probe,
.remove = sdhci_esdhc_mcf_remove,
static const struct sdhci_iproc_data bcm2711_data = {
.pdata = &sdhci_bcm2711_pltfm_data,
+ .mmc_caps = MMC_CAP_3_3V_DDR,
};
static const struct of_device_id sdhci_iproc_of_match[] = {
static struct platform_driver sdhci_iproc_driver = {
.driver = {
.name = "sdhci-iproc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = sdhci_iproc_of_match,
.acpi_match_table = ACPI_PTR(sdhci_iproc_acpi_ids),
.pm = &sdhci_pltfm_pmops,
static struct platform_driver sdhci_milbeaut_driver = {
.driver = {
.name = "sdhci-milbeaut",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(mlb_dt_ids),
},
.probe = sdhci_milbeaut_probe,
unsigned long clk_rate;
struct mmc_host *mmc;
struct opp_table *opp_table;
- bool has_opp_table;
bool use_14lpp_dll_reset;
bool tuning_done;
bool calibration_done;
{.compatible = "qcom,sdhci-msm-v5", .data = &sdhci_msm_v5_var},
{.compatible = "qcom,sdm845-sdhci", .data = &sdm845_sdhci_var},
{.compatible = "qcom,sm8250-sdhci", .data = &sm8250_sdhci_var},
+ {.compatible = "qcom,sc7180-sdhci", .data = &sdm845_sdhci_var},
{},
};
/* OPP table is optional */
ret = dev_pm_opp_of_add_table(&pdev->dev);
- if (!ret) {
- msm_host->has_opp_table = true;
- } else if (ret != -ENODEV) {
+ if (ret && ret != -ENODEV) {
dev_err(&pdev->dev, "Invalid OPP table in Device tree\n");
- goto opp_cleanup;
+ goto opp_put_clkname;
}
/* Vote for maximum clock rate for maximum performance */
clk_bulk_disable_unprepare(ARRAY_SIZE(msm_host->bulk_clks),
msm_host->bulk_clks);
opp_cleanup:
- if (msm_host->has_opp_table)
- dev_pm_opp_of_remove_table(&pdev->dev);
+ dev_pm_opp_of_remove_table(&pdev->dev);
+opp_put_clkname:
dev_pm_opp_put_clkname(msm_host->opp_table);
bus_clk_disable:
if (!IS_ERR(msm_host->bus_clk))
sdhci_remove_host(host, dead);
- if (msm_host->has_opp_table)
- dev_pm_opp_of_remove_table(&pdev->dev);
+ dev_pm_opp_of_remove_table(&pdev->dev);
dev_pm_opp_put_clkname(msm_host->opp_table);
pm_runtime_get_sync(&pdev->dev);
pm_runtime_disable(&pdev->dev);
.name = "sdhci_msm",
.of_match_table = sdhci_msm_dt_match,
.pm = &sdhci_msm_pm_ops,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
},
};
of_node_put(node);
if (IS_ERR(sdhci_arasan->soc_ctl_base)) {
- ret = PTR_ERR(sdhci_arasan->soc_ctl_base);
- if (ret != -EPROBE_DEFER)
- dev_err(&pdev->dev, "Can't get syscon: %d\n",
- ret);
+ ret = dev_err_probe(&pdev->dev,
+ PTR_ERR(sdhci_arasan->soc_ctl_base),
+ "Can't get syscon\n");
goto err_pltfm_free;
}
}
static struct platform_driver sdhci_arasan_driver = {
.driver = {
.name = "sdhci-arasan",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = sdhci_arasan_of_match,
.pm = &sdhci_arasan_dev_pm_ops,
},
static struct platform_driver aspeed_sdhci_driver = {
.driver = {
.name = "sdhci-aspeed",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = aspeed_sdhci_of_match,
},
.probe = aspeed_sdhci_probe,
static struct platform_driver aspeed_sdc_driver = {
.driver = {
.name = "sd-controller-aspeed",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &sdhci_pltfm_pmops,
.of_match_table = aspeed_sdc_of_match,
},
static struct platform_driver sdhci_at91_driver = {
.driver = {
.name = "sdhci-at91",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = sdhci_at91_dt_match,
.pm = &sdhci_at91_dev_pm_ops,
},
static struct platform_driver sdhci_dwcmshc_driver = {
.driver = {
.name = "sdhci-dwcmshc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = sdhci_dwcmshc_dt_ids,
.pm = &dwcmshc_pmops,
},
clk_put(clk);
}
- if (esdhc->peripheral_clock) {
- esdhc_clock_enable(host, false);
- val = sdhci_readl(host, ESDHC_DMA_SYSCTL);
+ esdhc_clock_enable(host, false);
+ val = sdhci_readl(host, ESDHC_DMA_SYSCTL);
+ /*
+ * This bit is not able to be reset by SDHCI_RESET_ALL. Need to
+ * initialize it as 1 or 0 once, to override the different value
+ * which may be configured in bootloader.
+ */
+ if (esdhc->peripheral_clock)
val |= ESDHC_PERIPHERAL_CLK_SEL;
- sdhci_writel(host, val, ESDHC_DMA_SYSCTL);
- esdhc_clock_enable(host, true);
- }
+ else
+ val &= ~ESDHC_PERIPHERAL_CLK_SEL;
+ sdhci_writel(host, val, ESDHC_DMA_SYSCTL);
+ esdhc_clock_enable(host, true);
}
static int esdhc_hs400_prepare_ddr(struct mmc_host *mmc)
static struct platform_driver sdhci_esdhc_driver = {
.driver = {
.name = "sdhci-esdhc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = sdhci_esdhc_of_match,
.pm = &esdhc_of_dev_pm_ops,
},
static struct platform_driver sdhci_hlwd_driver = {
.driver = {
.name = "sdhci-hlwd",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = sdhci_hlwd_of_match,
.pm = &sdhci_pltfm_pmops,
},
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * drivers/mmc/host/sdhci-of-sparx5.c
+ *
+ * MCHP Sparx5 SoC Secure Digital Host Controller Interface.
+ *
+ * Copyright (c) 2019 Microchip Inc.
+ *
+ * Author: Lars Povlsen <lars.povlsen@microchip.com>
+ */
+
+#include <linux/sizes.h>
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/of_device.h>
+#include <linux/mfd/syscon.h>
+#include <linux/dma-mapping.h>
+
+#include "sdhci-pltfm.h"
+
+#define CPU_REGS_GENERAL_CTRL (0x22 * 4)
+#define MSHC_DLY_CC_MASK GENMASK(16, 13)
+#define MSHC_DLY_CC_SHIFT 13
+#define MSHC_DLY_CC_MAX 15
+
+#define CPU_REGS_PROC_CTRL (0x2C * 4)
+#define ACP_CACHE_FORCE_ENA BIT(4)
+#define ACP_AWCACHE BIT(3)
+#define ACP_ARCACHE BIT(2)
+#define ACP_CACHE_MASK (ACP_CACHE_FORCE_ENA|ACP_AWCACHE|ACP_ARCACHE)
+
+#define MSHC2_VERSION 0x500 /* Off 0x140, reg 0x0 */
+#define MSHC2_TYPE 0x504 /* Off 0x140, reg 0x1 */
+#define MSHC2_EMMC_CTRL 0x52c /* Off 0x140, reg 0xB */
+#define MSHC2_EMMC_CTRL_EMMC_RST_N BIT(2)
+#define MSHC2_EMMC_CTRL_IS_EMMC BIT(0)
+
+struct sdhci_sparx5_data {
+ struct sdhci_host *host;
+ struct regmap *cpu_ctrl;
+ int delay_clock;
+};
+
+#define BOUNDARY_OK(addr, len) \
+ ((addr | (SZ_128M - 1)) == ((addr + len - 1) | (SZ_128M - 1)))
+
+/*
+ * If DMA addr spans 128MB boundary, we split the DMA transfer into two
+ * so that each DMA transfer doesn't exceed the boundary.
+ */
+static void sdhci_sparx5_adma_write_desc(struct sdhci_host *host, void **desc,
+ dma_addr_t addr, int len,
+ unsigned int cmd)
+{
+ int tmplen, offset;
+
+ if (likely(!len || BOUNDARY_OK(addr, len))) {
+ sdhci_adma_write_desc(host, desc, addr, len, cmd);
+ return;
+ }
+
+ pr_debug("%s: write_desc: splitting dma len %d, offset %pad\n",
+ mmc_hostname(host->mmc), len, &addr);
+
+ offset = addr & (SZ_128M - 1);
+ tmplen = SZ_128M - offset;
+ sdhci_adma_write_desc(host, desc, addr, tmplen, cmd);
+
+ addr += tmplen;
+ len -= tmplen;
+ sdhci_adma_write_desc(host, desc, addr, len, cmd);
+}
+
+static void sparx5_set_cacheable(struct sdhci_host *host, u32 value)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_sparx5_data *sdhci_sparx5 = sdhci_pltfm_priv(pltfm_host);
+
+ pr_debug("%s: Set Cacheable = 0x%x\n", mmc_hostname(host->mmc), value);
+
+ /* Update ACP caching attributes in HW */
+ regmap_update_bits(sdhci_sparx5->cpu_ctrl,
+ CPU_REGS_PROC_CTRL, ACP_CACHE_MASK, value);
+}
+
+static void sparx5_set_delay(struct sdhci_host *host, u8 value)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_sparx5_data *sdhci_sparx5 = sdhci_pltfm_priv(pltfm_host);
+
+ pr_debug("%s: Set DLY_CC = %u\n", mmc_hostname(host->mmc), value);
+
+ /* Update DLY_CC in HW */
+ regmap_update_bits(sdhci_sparx5->cpu_ctrl,
+ CPU_REGS_GENERAL_CTRL,
+ MSHC_DLY_CC_MASK,
+ (value << MSHC_DLY_CC_SHIFT));
+}
+
+static void sdhci_sparx5_set_emmc(struct sdhci_host *host)
+{
+ if (!mmc_card_is_removable(host->mmc)) {
+ u8 value;
+
+ value = sdhci_readb(host, MSHC2_EMMC_CTRL);
+ if (!(value & MSHC2_EMMC_CTRL_IS_EMMC)) {
+ value |= MSHC2_EMMC_CTRL_IS_EMMC;
+ pr_debug("%s: Set EMMC_CTRL: 0x%08x\n",
+ mmc_hostname(host->mmc), value);
+ sdhci_writeb(host, value, MSHC2_EMMC_CTRL);
+ }
+ }
+}
+
+static void sdhci_sparx5_reset_emmc(struct sdhci_host *host)
+{
+ u8 value;
+
+ pr_debug("%s: Toggle EMMC_CTRL.EMMC_RST_N\n", mmc_hostname(host->mmc));
+ value = sdhci_readb(host, MSHC2_EMMC_CTRL) &
+ ~MSHC2_EMMC_CTRL_EMMC_RST_N;
+ sdhci_writeb(host, value, MSHC2_EMMC_CTRL);
+ /* For eMMC, minimum is 1us but give it 10us for good measure */
+ usleep_range(10, 20);
+ sdhci_writeb(host, value | MSHC2_EMMC_CTRL_EMMC_RST_N,
+ MSHC2_EMMC_CTRL);
+ /* For eMMC, minimum is 200us but give it 300us for good measure */
+ usleep_range(300, 400);
+}
+
+static void sdhci_sparx5_reset(struct sdhci_host *host, u8 mask)
+{
+ pr_debug("%s: *** RESET: mask %d\n", mmc_hostname(host->mmc), mask);
+
+ sdhci_reset(host, mask);
+
+ /* Be sure CARD_IS_EMMC stays set */
+ sdhci_sparx5_set_emmc(host);
+}
+
+static const struct sdhci_ops sdhci_sparx5_ops = {
+ .set_clock = sdhci_set_clock,
+ .set_bus_width = sdhci_set_bus_width,
+ .set_uhs_signaling = sdhci_set_uhs_signaling,
+ .get_max_clock = sdhci_pltfm_clk_get_max_clock,
+ .reset = sdhci_sparx5_reset,
+ .adma_write_desc = sdhci_sparx5_adma_write_desc,
+};
+
+static const struct sdhci_pltfm_data sdhci_sparx5_pdata = {
+ .quirks = 0,
+ .quirks2 = SDHCI_QUIRK2_HOST_NO_CMD23 | /* Controller issue */
+ SDHCI_QUIRK2_NO_1_8_V, /* No sdr104, ddr50, etc */
+ .ops = &sdhci_sparx5_ops,
+};
+
+static int sdhci_sparx5_probe(struct platform_device *pdev)
+{
+ int ret;
+ const char *syscon = "microchip,sparx5-cpu-syscon";
+ struct sdhci_host *host;
+ struct sdhci_pltfm_host *pltfm_host;
+ struct sdhci_sparx5_data *sdhci_sparx5;
+ struct device_node *np = pdev->dev.of_node;
+ u32 value;
+ u32 extra;
+
+ host = sdhci_pltfm_init(pdev, &sdhci_sparx5_pdata,
+ sizeof(*sdhci_sparx5));
+
+ if (IS_ERR(host))
+ return PTR_ERR(host);
+
+ /*
+ * extra adma table cnt for cross 128M boundary handling.
+ */
+ extra = DIV_ROUND_UP_ULL(dma_get_required_mask(&pdev->dev), SZ_128M);
+ if (extra > SDHCI_MAX_SEGS)
+ extra = SDHCI_MAX_SEGS;
+ host->adma_table_cnt += extra;
+
+ pltfm_host = sdhci_priv(host);
+ sdhci_sparx5 = sdhci_pltfm_priv(pltfm_host);
+ sdhci_sparx5->host = host;
+
+ pltfm_host->clk = devm_clk_get(&pdev->dev, "core");
+ if (IS_ERR(pltfm_host->clk)) {
+ ret = PTR_ERR(pltfm_host->clk);
+ dev_err(&pdev->dev, "failed to get core clk: %d\n", ret);
+ goto free_pltfm;
+ }
+ ret = clk_prepare_enable(pltfm_host->clk);
+ if (ret)
+ goto free_pltfm;
+
+ if (!of_property_read_u32(np, "microchip,clock-delay", &value) &&
+ (value > 0 && value <= MSHC_DLY_CC_MAX))
+ sdhci_sparx5->delay_clock = value;
+
+ sdhci_get_of_property(pdev);
+
+ ret = mmc_of_parse(host->mmc);
+ if (ret)
+ goto err_clk;
+
+ sdhci_sparx5->cpu_ctrl = syscon_regmap_lookup_by_compatible(syscon);
+ if (IS_ERR(sdhci_sparx5->cpu_ctrl)) {
+ dev_err(&pdev->dev, "No CPU syscon regmap !\n");
+ ret = PTR_ERR(sdhci_sparx5->cpu_ctrl);
+ goto err_clk;
+ }
+
+ if (sdhci_sparx5->delay_clock >= 0)
+ sparx5_set_delay(host, sdhci_sparx5->delay_clock);
+
+ if (!mmc_card_is_removable(host->mmc)) {
+ /* Do a HW reset of eMMC card */
+ sdhci_sparx5_reset_emmc(host);
+ /* Update EMMC_CTRL */
+ sdhci_sparx5_set_emmc(host);
+ /* If eMMC, disable SD and SDIO */
+ host->mmc->caps2 |= (MMC_CAP2_NO_SDIO|MMC_CAP2_NO_SD);
+ }
+
+ ret = sdhci_add_host(host);
+ if (ret)
+ goto err_clk;
+
+ /* Set AXI bus master to use un-cached access (for DMA) */
+ if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA) &&
+ IS_ENABLED(CONFIG_DMA_DECLARE_COHERENT))
+ sparx5_set_cacheable(host, ACP_CACHE_FORCE_ENA);
+
+ pr_debug("%s: SDHC version: 0x%08x\n",
+ mmc_hostname(host->mmc), sdhci_readl(host, MSHC2_VERSION));
+ pr_debug("%s: SDHC type: 0x%08x\n",
+ mmc_hostname(host->mmc), sdhci_readl(host, MSHC2_TYPE));
+
+ return ret;
+
+err_clk:
+ clk_disable_unprepare(pltfm_host->clk);
+free_pltfm:
+ sdhci_pltfm_free(pdev);
+ return ret;
+}
+
+static const struct of_device_id sdhci_sparx5_of_match[] = {
+ { .compatible = "microchip,dw-sparx5-sdhci" },
+ { }
+};
+MODULE_DEVICE_TABLE(of, sdhci_sparx5_of_match);
+
+static struct platform_driver sdhci_sparx5_driver = {
+ .driver = {
+ .name = "sdhci-sparx5",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
+ .of_match_table = sdhci_sparx5_of_match,
+ .pm = &sdhci_pltfm_pmops,
+ },
+ .probe = sdhci_sparx5_probe,
+ .remove = sdhci_pltfm_unregister,
+};
+
+module_platform_driver(sdhci_sparx5_driver);
+
+MODULE_DESCRIPTION("Sparx5 SDHCI OF driver");
+MODULE_AUTHOR("Lars Povlsen <lars.povlsen@microchip.com>");
+MODULE_LICENSE("GPL v2");
.remove = sdhci_omap_remove,
.driver = {
.name = "sdhci-omap",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &sdhci_omap_dev_pm_ops,
.of_match_table = omap_sdhci_match,
},
#include <linux/iopoll.h>
#include <linux/gpio.h>
#include <linux/pm_runtime.h>
+#include <linux/pm_qos.h>
+#include <linux/debugfs.h>
#include <linux/mmc/slot-gpio.h>
#include <linux/mmc/sdhci-pci-data.h>
#include <linux/acpi.h>
bool rpm_retune_ok;
u32 glk_rx_ctrl1;
u32 glk_tun_val;
+ u32 active_ltr;
+ u32 idle_ltr;
};
static const guid_t intel_dsm_guid =
return 0;
}
+#define INTEL_ACTIVELTR 0x804
+#define INTEL_IDLELTR 0x808
+
+#define INTEL_LTR_REQ BIT(15)
+#define INTEL_LTR_SCALE_MASK GENMASK(11, 10)
+#define INTEL_LTR_SCALE_1US (2 << 10)
+#define INTEL_LTR_SCALE_32US (3 << 10)
+#define INTEL_LTR_VALUE_MASK GENMASK(9, 0)
+
+static void intel_cache_ltr(struct sdhci_pci_slot *slot)
+{
+ struct intel_host *intel_host = sdhci_pci_priv(slot);
+ struct sdhci_host *host = slot->host;
+
+ intel_host->active_ltr = readl(host->ioaddr + INTEL_ACTIVELTR);
+ intel_host->idle_ltr = readl(host->ioaddr + INTEL_IDLELTR);
+}
+
+static void intel_ltr_set(struct device *dev, s32 val)
+{
+ struct sdhci_pci_chip *chip = dev_get_drvdata(dev);
+ struct sdhci_pci_slot *slot = chip->slots[0];
+ struct intel_host *intel_host = sdhci_pci_priv(slot);
+ struct sdhci_host *host = slot->host;
+ u32 ltr;
+
+ pm_runtime_get_sync(dev);
+
+ /*
+ * Program latency tolerance (LTR) accordingly what has been asked
+ * by the PM QoS layer or disable it in case we were passed
+ * negative value or PM_QOS_LATENCY_ANY.
+ */
+ ltr = readl(host->ioaddr + INTEL_ACTIVELTR);
+
+ if (val == PM_QOS_LATENCY_ANY || val < 0) {
+ ltr &= ~INTEL_LTR_REQ;
+ } else {
+ ltr |= INTEL_LTR_REQ;
+ ltr &= ~INTEL_LTR_SCALE_MASK;
+ ltr &= ~INTEL_LTR_VALUE_MASK;
+
+ if (val > INTEL_LTR_VALUE_MASK) {
+ val >>= 5;
+ if (val > INTEL_LTR_VALUE_MASK)
+ val = INTEL_LTR_VALUE_MASK;
+ ltr |= INTEL_LTR_SCALE_32US | val;
+ } else {
+ ltr |= INTEL_LTR_SCALE_1US | val;
+ }
+ }
+
+ if (ltr == intel_host->active_ltr)
+ goto out;
+
+ writel(ltr, host->ioaddr + INTEL_ACTIVELTR);
+ writel(ltr, host->ioaddr + INTEL_IDLELTR);
+
+ /* Cache the values into lpss structure */
+ intel_cache_ltr(slot);
+out:
+ pm_runtime_put_autosuspend(dev);
+}
+
+static bool intel_use_ltr(struct sdhci_pci_chip *chip)
+{
+ switch (chip->pdev->device) {
+ case PCI_DEVICE_ID_INTEL_BYT_EMMC:
+ case PCI_DEVICE_ID_INTEL_BYT_EMMC2:
+ case PCI_DEVICE_ID_INTEL_BYT_SDIO:
+ case PCI_DEVICE_ID_INTEL_BYT_SD:
+ case PCI_DEVICE_ID_INTEL_BSW_EMMC:
+ case PCI_DEVICE_ID_INTEL_BSW_SDIO:
+ case PCI_DEVICE_ID_INTEL_BSW_SD:
+ return false;
+ default:
+ return true;
+ }
+}
+
+static void intel_ltr_expose(struct sdhci_pci_chip *chip)
+{
+ struct device *dev = &chip->pdev->dev;
+
+ if (!intel_use_ltr(chip))
+ return;
+
+ dev->power.set_latency_tolerance = intel_ltr_set;
+ dev_pm_qos_expose_latency_tolerance(dev);
+}
+
+static void intel_ltr_hide(struct sdhci_pci_chip *chip)
+{
+ struct device *dev = &chip->pdev->dev;
+
+ if (!intel_use_ltr(chip))
+ return;
+
+ dev_pm_qos_hide_latency_tolerance(dev);
+ dev->power.set_latency_tolerance = NULL;
+}
+
static void byt_probe_slot(struct sdhci_pci_slot *slot)
{
struct mmc_host_ops *ops = &slot->host->mmc_host_ops;
ops->start_signal_voltage_switch = intel_start_signal_voltage_switch;
device_property_read_u32(dev, "max-frequency", &mmc->f_max);
+
+ if (!mmc->slotno) {
+ slot->chip->slots[mmc->slotno] = slot;
+ intel_ltr_expose(slot->chip);
+ }
+}
+
+static void byt_add_debugfs(struct sdhci_pci_slot *slot)
+{
+ struct intel_host *intel_host = sdhci_pci_priv(slot);
+ struct mmc_host *mmc = slot->host->mmc;
+ struct dentry *dir = mmc->debugfs_root;
+
+ if (!intel_use_ltr(slot->chip))
+ return;
+
+ debugfs_create_x32("active_ltr", 0444, dir, &intel_host->active_ltr);
+ debugfs_create_x32("idle_ltr", 0444, dir, &intel_host->idle_ltr);
+
+ intel_cache_ltr(slot);
+}
+
+static int byt_add_host(struct sdhci_pci_slot *slot)
+{
+ int ret = sdhci_add_host(slot->host);
+
+ if (!ret)
+ byt_add_debugfs(slot);
+ return ret;
+}
+
+static void byt_remove_slot(struct sdhci_pci_slot *slot, int dead)
+{
+ struct mmc_host *mmc = slot->host->mmc;
+
+ if (!mmc->slotno)
+ intel_ltr_hide(slot->chip);
}
static int byt_emmc_probe_slot(struct sdhci_pci_slot *slot)
static bool glk_broken_cqhci(struct sdhci_pci_slot *slot)
{
return slot->chip->pdev->device == PCI_DEVICE_ID_INTEL_GLK_EMMC &&
- dmi_match(DMI_BIOS_VENDOR, "LENOVO");
+ (dmi_match(DMI_BIOS_VENDOR, "LENOVO") ||
+ dmi_match(DMI_SYS_VENDOR, "IRBIS"));
}
static int glk_emmc_probe_slot(struct sdhci_pci_slot *slot)
if (ret)
goto cleanup;
+ byt_add_debugfs(slot);
+
return 0;
cleanup:
#endif
.allow_runtime_pm = true,
.probe_slot = byt_emmc_probe_slot,
+ .add_host = byt_add_host,
+ .remove_slot = byt_remove_slot,
.quirks = SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC |
SDHCI_QUIRK_NO_LED,
.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
.allow_runtime_pm = true,
.probe_slot = glk_emmc_probe_slot,
.add_host = glk_emmc_add_host,
+ .remove_slot = byt_remove_slot,
#ifdef CONFIG_PM_SLEEP
.suspend = sdhci_cqhci_suspend,
.resume = sdhci_cqhci_resume,
SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
.allow_runtime_pm = true,
.probe_slot = ni_byt_sdio_probe_slot,
+ .add_host = byt_add_host,
+ .remove_slot = byt_remove_slot,
.ops = &sdhci_intel_byt_ops,
.priv_size = sizeof(struct intel_host),
};
SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
.allow_runtime_pm = true,
.probe_slot = byt_sdio_probe_slot,
+ .add_host = byt_add_host,
+ .remove_slot = byt_remove_slot,
.ops = &sdhci_intel_byt_ops,
.priv_size = sizeof(struct intel_host),
};
.allow_runtime_pm = true,
.own_cd_for_runtime_pm = true,
.probe_slot = byt_sd_probe_slot,
+ .add_host = byt_add_host,
+ .remove_slot = byt_remove_slot,
.ops = &sdhci_intel_byt_ops,
.priv_size = sizeof(struct intel_host),
};
#include <linux/delay.h>
#include "sdhci.h"
#include "sdhci-pci.h"
+#include "cqhci.h"
/* Genesys Logic extra registers */
#define SDHCI_GLI_9750_WT 0x800
#define GLI_9763E_VHS_REV_R 0x0
#define GLI_9763E_VHS_REV_M 0x1
#define GLI_9763E_VHS_REV_W 0x2
+#define PCIE_GLI_9763E_MB 0x888
+#define GLI_9763E_MB_CMDQ_OFF BIT(19)
#define PCIE_GLI_9763E_SCR 0x8E0
#define GLI_9763E_SCR_AXI_REQ BIT(9)
+#define SDHCI_GLI_9763E_CQE_BASE_ADDR 0x200
+#define GLI_9763E_CQE_TRNS_MODE (SDHCI_TRNS_MULTI | \
+ SDHCI_TRNS_BLK_CNT_EN | \
+ SDHCI_TRNS_DMA)
+
#define PCI_GLI_9755_WT 0x800
#define PCI_GLI_9755_WT_EN BIT(0)
#define GLI_9755_WT_EN_ON 0x1
return sdhci_pci_resume_host(chip);
}
+
+static int sdhci_cqhci_gli_resume(struct sdhci_pci_chip *chip)
+{
+ struct sdhci_pci_slot *slot = chip->slots[0];
+ int ret;
+
+ ret = sdhci_pci_gli_resume(chip);
+ if (ret)
+ return ret;
+
+ return cqhci_resume(slot->host->mmc);
+}
+
+static int sdhci_cqhci_gli_suspend(struct sdhci_pci_chip *chip)
+{
+ struct sdhci_pci_slot *slot = chip->slots[0];
+ int ret;
+
+ ret = cqhci_suspend(slot->host->mmc);
+ if (ret)
+ return ret;
+
+ return sdhci_suspend_host(slot->host);
+}
#endif
static void gl9763e_hs400_enhanced_strobe(struct mmc_host *mmc,
sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
}
+static void sdhci_gl9763e_dumpregs(struct mmc_host *mmc)
+{
+ sdhci_dumpregs(mmc_priv(mmc));
+}
+
+static void sdhci_gl9763e_cqe_pre_enable(struct mmc_host *mmc)
+{
+ struct cqhci_host *cq_host = mmc->cqe_private;
+ u32 value;
+
+ value = cqhci_readl(cq_host, CQHCI_CFG);
+ value |= CQHCI_ENABLE;
+ cqhci_writel(cq_host, value, CQHCI_CFG);
+}
+
+static void sdhci_gl9763e_cqe_enable(struct mmc_host *mmc)
+{
+ struct sdhci_host *host = mmc_priv(mmc);
+
+ sdhci_writew(host, GLI_9763E_CQE_TRNS_MODE, SDHCI_TRANSFER_MODE);
+ sdhci_cqe_enable(mmc);
+}
+
+static u32 sdhci_gl9763e_cqhci_irq(struct sdhci_host *host, u32 intmask)
+{
+ int cmd_error = 0;
+ int data_error = 0;
+
+ if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
+ return intmask;
+
+ cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+
+ return 0;
+}
+
+static void sdhci_gl9763e_cqe_post_disable(struct mmc_host *mmc)
+{
+ struct sdhci_host *host = mmc_priv(mmc);
+ struct cqhci_host *cq_host = mmc->cqe_private;
+ u32 value;
+
+ value = cqhci_readl(cq_host, CQHCI_CFG);
+ value &= ~CQHCI_ENABLE;
+ cqhci_writel(cq_host, value, CQHCI_CFG);
+ sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE);
+}
+
+static const struct cqhci_host_ops sdhci_gl9763e_cqhci_ops = {
+ .enable = sdhci_gl9763e_cqe_enable,
+ .disable = sdhci_cqe_disable,
+ .dumpregs = sdhci_gl9763e_dumpregs,
+ .pre_enable = sdhci_gl9763e_cqe_pre_enable,
+ .post_disable = sdhci_gl9763e_cqe_post_disable,
+};
+
+static int gl9763e_add_host(struct sdhci_pci_slot *slot)
+{
+ struct device *dev = &slot->chip->pdev->dev;
+ struct sdhci_host *host = slot->host;
+ struct cqhci_host *cq_host;
+ bool dma64;
+ int ret;
+
+ ret = sdhci_setup_host(host);
+ if (ret)
+ return ret;
+
+ cq_host = devm_kzalloc(dev, sizeof(*cq_host), GFP_KERNEL);
+ if (!cq_host) {
+ ret = -ENOMEM;
+ goto cleanup;
+ }
+
+ cq_host->mmio = host->ioaddr + SDHCI_GLI_9763E_CQE_BASE_ADDR;
+ cq_host->ops = &sdhci_gl9763e_cqhci_ops;
+
+ dma64 = host->flags & SDHCI_USE_64_BIT_DMA;
+ if (dma64)
+ cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
+
+ ret = cqhci_init(cq_host, host->mmc, dma64);
+ if (ret)
+ goto cleanup;
+
+ ret = __sdhci_add_host(host);
+ if (ret)
+ goto cleanup;
+
+ return 0;
+
+cleanup:
+ sdhci_cleanup_host(host);
+ return ret;
+}
+
+static void sdhci_gl9763e_reset(struct sdhci_host *host, u8 mask)
+{
+ if ((host->mmc->caps2 & MMC_CAP2_CQE) && (mask & SDHCI_RESET_ALL) &&
+ host->mmc->cqe_private)
+ cqhci_deactivate(host->mmc);
+ sdhci_reset(host, mask);
+}
+
static void gli_set_gl9763e(struct sdhci_pci_slot *slot)
{
struct pci_dev *pdev = slot->chip->pdev;
static int gli_probe_slot_gl9763e(struct sdhci_pci_slot *slot)
{
+ struct pci_dev *pdev = slot->chip->pdev;
struct sdhci_host *host = slot->host;
+ u32 value;
host->mmc->caps |= MMC_CAP_8_BIT_DATA |
MMC_CAP_1_8V_DDR |
MMC_CAP2_HS400_ES |
MMC_CAP2_NO_SDIO |
MMC_CAP2_NO_SD;
+
+ pci_read_config_dword(pdev, PCIE_GLI_9763E_MB, &value);
+ if (!(value & GLI_9763E_MB_CMDQ_OFF))
+ host->mmc->caps2 |= MMC_CAP2_CQE | MMC_CAP2_CQE_DCMD;
+
gli_pcie_enable_msi(slot);
host->mmc_host_ops.hs400_enhanced_strobe =
gl9763e_hs400_enhanced_strobe;
.set_clock = sdhci_set_clock,
.enable_dma = sdhci_pci_enable_dma,
.set_bus_width = sdhci_set_bus_width,
- .reset = sdhci_reset,
+ .reset = sdhci_gl9763e_reset,
.set_uhs_signaling = sdhci_set_gl9763e_signaling,
.voltage_switch = sdhci_gli_voltage_switch,
+ .irq = sdhci_gl9763e_cqhci_irq,
};
const struct sdhci_pci_fixes sdhci_gl9763e = {
.probe_slot = gli_probe_slot_gl9763e,
.ops = &sdhci_gl9763e_ops,
#ifdef CONFIG_PM_SLEEP
- .resume = sdhci_pci_gli_resume,
+ .resume = sdhci_cqhci_gli_resume,
+ .suspend = sdhci_cqhci_gli_suspend,
#endif
+ .add_host = gl9763e_add_host,
};
static struct platform_driver pic32_sdhci_driver = {
.driver = {
.name = "pic32-sdhci",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(pic32_sdhci_id_table),
},
.probe = pic32_sdhci_probe,
static struct platform_driver sdhci_pxav2_driver = {
.driver = {
.name = "sdhci-pxav2",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(sdhci_pxav2_of_match),
.pm = &sdhci_pltfm_pmops,
},
static struct platform_driver sdhci_pxav3_driver = {
.driver = {
.name = "sdhci-pxav3",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(sdhci_pxav3_of_match),
.pm = &sdhci_pxav3_pmops,
},
}
#endif
+#ifdef CONFIG_OF
static const struct of_device_id sdhci_s3c_dt_match[];
+#endif
static inline struct sdhci_s3c_drv_data *sdhci_s3c_get_driver_data(
struct platform_device *pdev)
.id_table = sdhci_s3c_driver_ids,
.driver = {
.name = "s3c-sdhci",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(sdhci_s3c_dt_match),
.pm = &sdhci_s3c_pmops,
},
static struct platform_driver sdhci_sirf_driver = {
.driver = {
.name = "sdhci-sirf",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = sdhci_sirf_of_match,
.pm = &sdhci_pltfm_pmops,
},
static struct platform_driver sdhci_driver = {
.driver = {
.name = "sdhci",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &sdhci_pm_ops,
.of_match_table = of_match_ptr(sdhci_spear_id_table),
},
if (mmc_hsq_finalize_request(host->mmc, mrq))
return;
- mmc_request_done(host->mmc, mrq);
+ mmc_request_done(host->mmc, mrq);
}
static struct sdhci_ops sdhci_sprd_ops = {
}
static int sdhci_sprd_request_atomic(struct mmc_host *mmc,
- struct mmc_request *mrq)
+ struct mmc_request *mrq)
{
sdhci_sprd_check_auto_cmd23(mmc, mrq);
.remove = sdhci_sprd_remove,
.driver = {
.name = "sdhci_sprd_r11",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(sdhci_sprd_of_match),
.pm = &sdhci_sprd_pm_ops,
},
.remove = sdhci_st_remove,
.driver = {
.name = "sdhci-st",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &sdhci_st_pmops,
.of_match_table = of_match_ptr(st_sdhci_match),
},
clk = devm_clk_get(mmc_dev(host->mmc), NULL);
if (IS_ERR(clk)) {
- rc = PTR_ERR(clk);
-
- if (rc != -EPROBE_DEFER)
- dev_err(&pdev->dev, "failed to get clock: %d\n", rc);
-
+ rc = dev_err_probe(&pdev->dev, PTR_ERR(clk),
+ "failed to get clock\n");
goto err_clk_get;
}
clk_prepare_enable(clk);
static struct platform_driver sdhci_tegra_driver = {
.driver = {
.name = "sdhci-tegra",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = sdhci_tegra_dt_match,
.pm = &sdhci_tegra_dev_pm_ops,
},
static struct platform_driver sdhci_xenon_driver = {
.driver = {
.name = "xenon-sdhci",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = sdhci_xenon_dt_ids,
.pm = &sdhci_xenon_dev_pm_ops,
},
/*
* sdhci_am654.c - SDHCI driver for TI's AM654 SOCs
*
- * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
+ * Copyright (C) 2018 Texas Instruments Incorporated - https://www.ti.com
*
*/
#include <linux/clk.h>
+#include <linux/iopoll.h>
#include <linux/of.h>
#include <linux/module.h>
#include <linux/pm_runtime.h>
/* CTL_CFG Registers */
#define CTL_CFG_2 0x14
+#define CTL_CFG_3 0x18
#define SLOTTYPE_MASK GENMASK(31, 30)
#define SLOTTYPE_EMBEDDED BIT(30)
+#define TUNINGFORSDR50_MASK BIT(13)
/* PHY Registers */
#define PHY_CTRL1 0x100
#define RETRIM_MASK BIT(RETRIM_SHIFT)
#define SELDLYTXCLK_SHIFT 17
#define SELDLYTXCLK_MASK BIT(SELDLYTXCLK_SHIFT)
+#define SELDLYRXCLK_SHIFT 16
+#define SELDLYRXCLK_MASK BIT(SELDLYRXCLK_SHIFT)
+#define ITAPDLYSEL_SHIFT 0
+#define ITAPDLYSEL_MASK GENMASK(4, 0)
+#define ITAPDLYENA_SHIFT 8
+#define ITAPDLYENA_MASK BIT(ITAPDLYENA_SHIFT)
+#define ITAPCHGWIN_SHIFT 9
+#define ITAPCHGWIN_MASK BIT(ITAPCHGWIN_SHIFT)
#define DRIVER_STRENGTH_50_OHM 0x0
#define DRIVER_STRENGTH_33_OHM 0x1
#define DRIVER_STRENGTH_100_OHM 0x3
#define DRIVER_STRENGTH_40_OHM 0x4
-#define CLOCK_TOO_SLOW_HZ 400000
+#define CLOCK_TOO_SLOW_HZ 50000000
/* Command Queue Host Controller Interface Base address */
#define SDHCI_AM654_CQE_BASE_ADDR 0x200
.fast_io = true,
};
+struct timing_data {
+ const char *otap_binding;
+ const char *itap_binding;
+ u32 capability;
+};
+
+static const struct timing_data td[] = {
+ [MMC_TIMING_LEGACY] = {"ti,otap-del-sel-legacy",
+ "ti,itap-del-sel-legacy",
+ 0},
+ [MMC_TIMING_MMC_HS] = {"ti,otap-del-sel-mmc-hs",
+ "ti,itap-del-sel-mmc-hs",
+ MMC_CAP_MMC_HIGHSPEED},
+ [MMC_TIMING_SD_HS] = {"ti,otap-del-sel-sd-hs",
+ "ti,itap-del-sel-sd-hs",
+ MMC_CAP_SD_HIGHSPEED},
+ [MMC_TIMING_UHS_SDR12] = {"ti,otap-del-sel-sdr12",
+ "ti,itap-del-sel-sdr12",
+ MMC_CAP_UHS_SDR12},
+ [MMC_TIMING_UHS_SDR25] = {"ti,otap-del-sel-sdr25",
+ "ti,itap-del-sel-sdr25",
+ MMC_CAP_UHS_SDR25},
+ [MMC_TIMING_UHS_SDR50] = {"ti,otap-del-sel-sdr50",
+ NULL,
+ MMC_CAP_UHS_SDR50},
+ [MMC_TIMING_UHS_SDR104] = {"ti,otap-del-sel-sdr104",
+ NULL,
+ MMC_CAP_UHS_SDR104},
+ [MMC_TIMING_UHS_DDR50] = {"ti,otap-del-sel-ddr50",
+ NULL,
+ MMC_CAP_UHS_DDR50},
+ [MMC_TIMING_MMC_DDR52] = {"ti,otap-del-sel-ddr52",
+ "ti,itap-del-sel-ddr52",
+ MMC_CAP_DDR},
+ [MMC_TIMING_MMC_HS200] = {"ti,otap-del-sel-hs200",
+ NULL,
+ MMC_CAP2_HS200},
+ [MMC_TIMING_MMC_HS400] = {"ti,otap-del-sel-hs400",
+ NULL,
+ MMC_CAP2_HS400},
+};
+
struct sdhci_am654_data {
struct regmap *base;
bool legacy_otapdly;
- int otap_del_sel[11];
+ int otap_del_sel[ARRAY_SIZE(td)];
+ int itap_del_sel[ARRAY_SIZE(td)];
int clkbuf_sel;
int trm_icp;
int drv_strength;
- bool dll_on;
int strb_sel;
u32 flags;
};
#define DLL_CALIB (1 << 4)
};
-struct timing_data {
- const char *binding;
- u32 capability;
-};
-
-static const struct timing_data td[] = {
- [MMC_TIMING_LEGACY] = {"ti,otap-del-sel-legacy", 0},
- [MMC_TIMING_MMC_HS] = {"ti,otap-del-sel-mmc-hs", MMC_CAP_MMC_HIGHSPEED},
- [MMC_TIMING_SD_HS] = {"ti,otap-del-sel-sd-hs", MMC_CAP_SD_HIGHSPEED},
- [MMC_TIMING_UHS_SDR12] = {"ti,otap-del-sel-sdr12", MMC_CAP_UHS_SDR12},
- [MMC_TIMING_UHS_SDR25] = {"ti,otap-del-sel-sdr25", MMC_CAP_UHS_SDR25},
- [MMC_TIMING_UHS_SDR50] = {"ti,otap-del-sel-sdr50", MMC_CAP_UHS_SDR50},
- [MMC_TIMING_UHS_SDR104] = {"ti,otap-del-sel-sdr104",
- MMC_CAP_UHS_SDR104},
- [MMC_TIMING_UHS_DDR50] = {"ti,otap-del-sel-ddr50", MMC_CAP_UHS_DDR50},
- [MMC_TIMING_MMC_DDR52] = {"ti,otap-del-sel-ddr52", MMC_CAP_DDR},
- [MMC_TIMING_MMC_HS200] = {"ti,otap-del-sel-hs200", MMC_CAP2_HS200},
- [MMC_TIMING_MMC_HS400] = {"ti,otap-del-sel-hs400", MMC_CAP2_HS400},
-};
-
static void sdhci_am654_setup_dll(struct sdhci_host *host, unsigned int clock)
{
struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
u32 mask, val;
int ret;
+ /* Disable delay chain mode */
+ regmap_update_bits(sdhci_am654->base, PHY_CTRL5,
+ SELDLYTXCLK_MASK | SELDLYRXCLK_MASK, 0);
+
if (sdhci_am654->flags & FREQSEL_2_BIT) {
switch (clock) {
case 200000000:
dev_err(mmc_dev(host->mmc), "DLL failed to relock\n");
return;
}
+}
+
+static void sdhci_am654_write_itapdly(struct sdhci_am654_data *sdhci_am654,
+ u32 itapdly)
+{
+ /* Set ITAPCHGWIN before writing to ITAPDLY */
+ regmap_update_bits(sdhci_am654->base, PHY_CTRL4, ITAPCHGWIN_MASK,
+ 1 << ITAPCHGWIN_SHIFT);
+ regmap_update_bits(sdhci_am654->base, PHY_CTRL4, ITAPDLYSEL_MASK,
+ itapdly << ITAPDLYSEL_SHIFT);
+ regmap_update_bits(sdhci_am654->base, PHY_CTRL4, ITAPCHGWIN_MASK, 0);
+}
+
+static void sdhci_am654_setup_delay_chain(struct sdhci_am654_data *sdhci_am654,
+ unsigned char timing)
+{
+ u32 mask, val;
+
+ regmap_update_bits(sdhci_am654->base, PHY_CTRL1, ENDLL_MASK, 0);
+
+ val = 1 << SELDLYTXCLK_SHIFT | 1 << SELDLYRXCLK_SHIFT;
+ mask = SELDLYTXCLK_MASK | SELDLYRXCLK_MASK;
+ regmap_update_bits(sdhci_am654->base, PHY_CTRL5, mask, val);
- sdhci_am654->dll_on = true;
+ sdhci_am654_write_itapdly(sdhci_am654,
+ sdhci_am654->itap_del_sel[timing]);
}
static void sdhci_am654_set_clock(struct sdhci_host *host, unsigned int clock)
u32 otap_del_ena;
u32 mask, val;
- if (sdhci_am654->dll_on) {
- regmap_update_bits(sdhci_am654->base, PHY_CTRL1, ENDLL_MASK, 0);
-
- sdhci_am654->dll_on = false;
- }
+ regmap_update_bits(sdhci_am654->base, PHY_CTRL1, ENDLL_MASK, 0);
sdhci_set_clock(host, clock);
regmap_update_bits(sdhci_am654->base, PHY_CTRL4, mask, val);
- if (timing > MMC_TIMING_UHS_SDR25 && clock > CLOCK_TOO_SLOW_HZ) {
- regmap_update_bits(sdhci_am654->base, PHY_CTRL5,
- SELDLYTXCLK_MASK, 0);
+ if (timing > MMC_TIMING_UHS_SDR25 && clock >= CLOCK_TOO_SLOW_HZ)
sdhci_am654_setup_dll(host, clock);
- } else {
- regmap_update_bits(sdhci_am654->base, PHY_CTRL5,
- SELDLYTXCLK_MASK, 1 << SELDLYTXCLK_SHIFT);
- }
+ else
+ sdhci_am654_setup_delay_chain(sdhci_am654, timing);
regmap_update_bits(sdhci_am654->base, PHY_CTRL5, CLKBUFSEL_MASK,
sdhci_am654->clkbuf_sel);
sdhci_set_clock(host, clock);
}
+static u8 sdhci_am654_write_power_on(struct sdhci_host *host, u8 val, int reg)
+{
+ writeb(val, host->ioaddr + reg);
+ usleep_range(1000, 10000);
+ return readb(host->ioaddr + reg);
+}
+
+#define MAX_POWER_ON_TIMEOUT 1500000 /* us */
static void sdhci_am654_write_b(struct sdhci_host *host, u8 val, int reg)
{
unsigned char timing = host->mmc->ios.timing;
+ u8 pwr;
+ int ret;
if (reg == SDHCI_HOST_CONTROL) {
switch (timing) {
}
writeb(val, host->ioaddr + reg);
+ if (reg == SDHCI_POWER_CONTROL && (val & SDHCI_POWER_ON)) {
+ /*
+ * Power on will not happen until the card detect debounce
+ * timer expires. Wait at least 1.5 seconds for the power on
+ * bit to be set
+ */
+ ret = read_poll_timeout(sdhci_am654_write_power_on, pwr,
+ pwr & SDHCI_POWER_ON, 0,
+ MAX_POWER_ON_TIMEOUT, false, host, val,
+ reg);
+ if (ret)
+ dev_warn(mmc_dev(host->mmc), "Power on failed\n");
+ }
}
static int sdhci_am654_execute_tuning(struct mmc_host *mmc, u32 opcode)
return 0;
}
+#define ITAP_MAX 32
+static int sdhci_am654_platform_execute_tuning(struct sdhci_host *host,
+ u32 opcode)
+{
+ struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+ struct sdhci_am654_data *sdhci_am654 = sdhci_pltfm_priv(pltfm_host);
+ int cur_val, prev_val = 1, fail_len = 0, pass_window = 0, pass_len;
+ u32 itap;
+
+ /* Enable ITAPDLY */
+ regmap_update_bits(sdhci_am654->base, PHY_CTRL4, ITAPDLYENA_MASK,
+ 1 << ITAPDLYENA_SHIFT);
+
+ for (itap = 0; itap < ITAP_MAX; itap++) {
+ sdhci_am654_write_itapdly(sdhci_am654, itap);
+
+ cur_val = !mmc_send_tuning(host->mmc, opcode, NULL);
+ if (cur_val && !prev_val)
+ pass_window = itap;
+
+ if (!cur_val)
+ fail_len++;
+
+ prev_val = cur_val;
+ }
+ /*
+ * Having determined the length of the failing window and start of
+ * the passing window calculate the length of the passing window and
+ * set the final value halfway through it considering the range as a
+ * circular buffer
+ */
+ pass_len = ITAP_MAX - fail_len;
+ itap = (pass_window + (pass_len >> 1)) % ITAP_MAX;
+ sdhci_am654_write_itapdly(sdhci_am654, itap);
+
+ return 0;
+}
+
static struct sdhci_ops sdhci_am654_ops = {
+ .platform_execute_tuning = sdhci_am654_platform_execute_tuning,
.get_max_clock = sdhci_pltfm_clk_get_max_clock,
.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
.set_uhs_signaling = sdhci_set_uhs_signaling,
};
static struct sdhci_ops sdhci_j721e_8bit_ops = {
+ .platform_execute_tuning = sdhci_am654_platform_execute_tuning,
.get_max_clock = sdhci_pltfm_clk_get_max_clock,
.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
.set_uhs_signaling = sdhci_set_uhs_signaling,
};
static struct sdhci_ops sdhci_j721e_4bit_ops = {
+ .platform_execute_tuning = sdhci_am654_platform_execute_tuning,
.get_max_clock = sdhci_pltfm_clk_get_max_clock,
.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
.set_uhs_signaling = sdhci_set_uhs_signaling,
int i;
int ret;
- ret = device_property_read_u32(dev, td[MMC_TIMING_LEGACY].binding,
+ ret = device_property_read_u32(dev, td[MMC_TIMING_LEGACY].otap_binding,
&sdhci_am654->otap_del_sel[MMC_TIMING_LEGACY]);
if (ret) {
/*
for (i = MMC_TIMING_MMC_HS; i <= MMC_TIMING_MMC_HS400; i++) {
- ret = device_property_read_u32(dev, td[i].binding,
+ ret = device_property_read_u32(dev, td[i].otap_binding,
&sdhci_am654->otap_del_sel[i]);
if (ret) {
dev_dbg(dev, "Couldn't find %s\n",
- td[i].binding);
+ td[i].otap_binding);
/*
* Remove the corresponding capability
* if an otap-del-sel value is not found
else
host->mmc->caps2 &= ~td[i].capability;
}
+
+ if (td[i].itap_binding)
+ device_property_read_u32(dev, td[i].itap_binding,
+ &sdhci_am654->itap_del_sel[i]);
}
return 0;
regmap_update_bits(sdhci_am654->base, CTL_CFG_2, SLOTTYPE_MASK,
ctl_cfg_2);
+ /* Enable tuning for SDR50 */
+ regmap_update_bits(sdhci_am654->base, CTL_CFG_3, TUNINGFORSDR50_MASK,
+ TUNINGFORSDR50_MASK);
+
ret = sdhci_setup_host(host);
if (ret)
return ret;
},
{ /* sentinel */ }
};
+MODULE_DEVICE_TABLE(of, sdhci_am654_of_match);
static int sdhci_am654_probe(struct platform_device *pdev)
{
static struct platform_driver sdhci_am654_driver = {
.driver = {
.name = "sdhci-am654",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = sdhci_am654_of_match,
},
.probe = sdhci_am654_probe,
static struct platform_driver sdhci_f_sdh30_driver = {
.driver = {
.name = "f_sdh30",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(f_sdh30_dt_ids),
.acpi_match_table = ACPI_PTR(f_sdh30_acpi_ids),
.pm = &sdhci_pltfm_pmops,
.remove = sh_mmcif_remove,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &sh_mmcif_dev_pm_ops,
.of_match_table = sh_mmcif_of_match,
},
static struct platform_driver sunxi_mmc_driver = {
.driver = {
.name = "sunxi-mmc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = of_match_ptr(sunxi_mmc_of_match),
.pm = &sunxi_mmc_pm_ops,
},
static void tmio_mmc_reset(struct tmio_mmc_host *host)
{
- /* FIXME - should we set stop clock reg here */
- sd_ctrl_write16(host, CTL_RESET_SD, 0x0000);
sd_ctrl_write16(host, CTL_RESET_SDIO, 0x0000);
usleep_range(10000, 11000);
- sd_ctrl_write16(host, CTL_RESET_SD, 0x0001);
sd_ctrl_write16(host, CTL_RESET_SDIO, 0x0001);
usleep_range(10000, 11000);
-
- if (host->pdata->flags & TMIO_MMC_SDIO_IRQ) {
- sd_ctrl_write16(host, CTL_SDIO_IRQ_MASK, host->sdio_irq_mask);
- sd_ctrl_write16(host, CTL_TRANSACTION_CTL, 0x0001);
- }
}
#ifdef CONFIG_PM_SLEEP
static struct platform_driver tmio_mmc_driver = {
.driver = {
.name = "tmio-mmc",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = &tmio_mmc_dev_pm_ops,
},
.probe = tmio_mmc_probe,
unsigned int direction, int blk_size);
int (*write16_hook)(struct tmio_mmc_host *host, int addr);
void (*reset)(struct tmio_mmc_host *host);
- void (*hw_reset)(struct tmio_mmc_host *host);
bool (*check_retune)(struct tmio_mmc_host *host);
-
- /*
- * Mandatory callback for tuning to occur which is optional for SDR50
- * and mandatory for SDR104.
- */
- int (*execute_tuning)(struct tmio_mmc_host *host, u32 opcode);
+ void (*fixup_request)(struct tmio_mmc_host *host, struct mmc_request *mrq);
void (*prepare_hs400_tuning)(struct tmio_mmc_host *host);
void (*hs400_downgrade)(struct tmio_mmc_host *host);
sd_ctrl_write16(host, CTL_RESET_SD, 0x0001);
usleep_range(10000, 11000);
+ if (host->reset)
+ host->reset(host);
+
if (host->pdata->flags & TMIO_MMC_SDIO_IRQ) {
sd_ctrl_write16(host, CTL_SDIO_IRQ_MASK, host->sdio_irq_mask);
sd_ctrl_write16(host, CTL_TRANSACTION_CTL, 0x0001);
}
}
-static void tmio_mmc_hw_reset(struct mmc_host *mmc)
-{
- struct tmio_mmc_host *host = mmc_priv(mmc);
-
- host->reset(host);
-
- tmio_mmc_abort_dma(host);
-
- if (host->hw_reset)
- host->hw_reset(host);
-}
-
static void tmio_mmc_reset_work(struct work_struct *work)
{
struct tmio_mmc_host *host = container_of(work, struct tmio_mmc_host,
spin_unlock_irqrestore(&host->lock, flags);
- tmio_mmc_hw_reset(host->mmc);
+ tmio_mmc_reset(host);
/* Ready for new calls */
host->mrq = NULL;
+ tmio_mmc_abort_dma(host);
mmc_request_done(host->mmc, mrq);
}
return 0;
}
-static int tmio_mmc_execute_tuning(struct mmc_host *mmc, u32 opcode)
-{
- struct tmio_mmc_host *host = mmc_priv(mmc);
- int ret;
-
- if (!host->execute_tuning)
- return 0;
-
- ret = host->execute_tuning(host, opcode);
-
- if (ret < 0) {
- dev_warn(&host->pdev->dev, "Tuning procedure failed\n");
- tmio_mmc_hw_reset(mmc);
- }
-
- return ret;
-}
-
static void tmio_process_mrq(struct tmio_mmc_host *host,
struct mmc_request *mrq)
{
return;
}
+ if (host->fixup_request)
+ host->fixup_request(host, mrq);
+
mmc_request_done(host->mmc, mrq);
}
.get_cd = tmio_mmc_get_cd,
.enable_sdio_irq = tmio_mmc_enable_sdio_irq,
.multi_io_quirk = tmio_multi_io_quirk,
- .hw_reset = tmio_mmc_hw_reset,
- .execute_tuning = tmio_mmc_execute_tuning,
};
static int tmio_mmc_init_ocr(struct tmio_mmc_host *host)
mmc->caps & MMC_CAP_NEEDS_POLL ||
!mmc_card_is_removable(mmc));
- if (!_host->reset)
- _host->reset = tmio_mmc_reset;
-
/*
* On Gen2+, eMMC with NONREMOVABLE currently fails because native
* hotplug gets disabled. It seems RuntimePM related yet we need further
_host->sdio_irq_mask = TMIO_SDIO_MASK_ALL;
_host->set_clock(_host, 0);
- tmio_mmc_hw_reset(mmc);
+ tmio_mmc_reset(_host);
_host->sdcard_irq_mask = sd_ctrl_read16_and_16_as_32(_host, CTL_IRQ_MASK);
tmio_mmc_disable_mmc_irqs(_host, TMIO_MASK_ALL);
struct tmio_mmc_host *host = dev_get_drvdata(dev);
tmio_mmc_clk_enable(host);
- tmio_mmc_hw_reset(host->mmc);
+ tmio_mmc_reset(host);
if (host->clk_cache)
host->set_clock(host, host->clk_cache);
clk_disable_unprepare(priv->clk);
}
-static void uniphier_sd_hw_reset(struct tmio_mmc_host *host)
+static void uniphier_sd_hw_reset(struct mmc_host *mmc)
{
+ struct tmio_mmc_host *host = mmc_priv(mmc);
struct uniphier_sd_priv *priv = uniphier_sd_priv(host);
reset_control_assert(priv->rst_hw);
ret = PTR_ERR(priv->rst_hw);
goto free_host;
}
- host->hw_reset = uniphier_sd_hw_reset;
+ host->ops.hw_reset = uniphier_sd_hw_reset;
}
if (host->mmc->caps & MMC_CAP_UHS) {
.remove = uniphier_sd_remove,
.driver = {
.name = "uniphier-sd",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = uniphier_sd_match,
},
};
.remove = usdhi6_remove,
.driver = {
.name = "usdhi6rol0",
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.of_match_table = usdhi6_of_match,
},
};
static int __maybe_unused via_sd_suspend(struct device *dev)
{
struct via_crdr_mmc_host *host;
+ unsigned long flags;
host = dev_get_drvdata(dev);
+ spin_lock_irqsave(&host->lock, flags);
via_save_pcictrlreg(host);
via_save_sdcreg(host);
+ spin_unlock_irqrestore(&host->lock, flags);
device_wakeup_enable(dev);
.resume = wbsd_platform_resume,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
},
};
.remove = wmt_mci_remove,
.driver = {
.name = DRIVER_NAME,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
.pm = wmt_mci_pm_ops,
.of_match_table = wmt_mci_dt_ids,
},
bool strict_match;
};
-static int alb_upper_dev_walk(struct net_device *upper, void *_data)
+static int alb_upper_dev_walk(struct net_device *upper,
+ struct netdev_nested_priv *priv)
{
- struct alb_walk_data *data = _data;
+ struct alb_walk_data *data = (struct alb_walk_data *)priv->data;
bool strict_match = data->strict_match;
struct bonding *bond = data->bond;
struct slave *slave = data->slave;
bool strict_match)
{
struct bonding *bond = bond_get_bond_by_slave(slave);
+ struct netdev_nested_priv priv;
struct alb_walk_data data = {
.strict_match = strict_match,
.mac_addr = mac_addr,
.bond = bond,
};
+ priv.data = (void *)&data;
/* send untagged */
alb_send_lp_vid(slave, mac_addr, 0, 0);
* for that device.
*/
rcu_read_lock();
- netdev_walk_all_upper_dev_rcu(bond->dev, alb_upper_dev_walk, &data);
+ netdev_walk_all_upper_dev_rcu(bond->dev, alb_upper_dev_walk, &priv);
rcu_read_unlock();
}
bond_dev->type = slave_dev->type;
bond_dev->hard_header_len = slave_dev->hard_header_len;
+ bond_dev->needed_headroom = slave_dev->needed_headroom;
bond_dev->addr_len = slave_dev->addr_len;
memcpy(bond_dev->broadcast, slave_dev->broadcast,
}
}
-static int bond_upper_dev_walk(struct net_device *upper, void *data)
+static int bond_upper_dev_walk(struct net_device *upper,
+ struct netdev_nested_priv *priv)
{
- __be32 ip = *((__be32 *)data);
+ __be32 ip = *(__be32 *)priv->data;
return ip == bond_confirm_addr(upper, 0, ip);
}
static bool bond_has_this_ip(struct bonding *bond, __be32 ip)
{
+ struct netdev_nested_priv priv = {
+ .data = (void *)&ip,
+ };
bool ret = false;
if (ip == bond_confirm_addr(bond->dev, 0, ip))
return true;
rcu_read_lock();
- if (netdev_walk_all_upper_dev_rcu(bond->dev, bond_upper_dev_walk, &ip))
+ if (netdev_walk_all_upper_dev_rcu(bond->dev, bond_upper_dev_walk, &priv))
ret = true;
rcu_read_unlock();
int ksz_switch_register(struct ksz_device *dev,
const struct ksz_dev_ops *ops)
{
+ struct device_node *port, *ports;
phy_interface_t interface;
- struct device_node *port;
unsigned int port_num;
int ret;
ret = of_get_phy_mode(dev->dev->of_node, &interface);
if (ret == 0)
dev->compat_interface = interface;
- for_each_available_child_of_node(dev->dev->of_node, port) {
- if (of_property_read_u32(port, "reg", &port_num))
- continue;
- if (port_num >= dev->port_cnt)
- return -EINVAL;
- of_get_phy_mode(port, &dev->ports[port_num].interface);
- }
+ ports = of_get_child_by_name(dev->dev->of_node, "ports");
+ if (ports)
+ for_each_available_child_of_node(ports, port) {
+ if (of_property_read_u32(port, "reg",
+ &port_num))
+ continue;
+ if (port_num >= dev->port_cnt)
+ return -EINVAL;
+ of_get_phy_mode(port,
+ &dev->ports[port_num].interface);
+ }
dev->synclko_125 = of_property_read_bool(dev->dev->of_node,
"microchip,synclko-125");
}
[VCAP_IS2_ACT_POLICE_ENA] = { 9, 1},
[VCAP_IS2_ACT_POLICE_IDX] = { 10, 9},
[VCAP_IS2_ACT_POLICE_VCAP_ONLY] = { 19, 1},
- [VCAP_IS2_ACT_PORT_MASK] = { 20, 11},
- [VCAP_IS2_ACT_REW_OP] = { 31, 9},
- [VCAP_IS2_ACT_SMAC_REPLACE_ENA] = { 40, 1},
- [VCAP_IS2_ACT_RSV] = { 41, 2},
- [VCAP_IS2_ACT_ACL_ID] = { 43, 6},
- [VCAP_IS2_ACT_HIT_CNT] = { 49, 32},
+ [VCAP_IS2_ACT_PORT_MASK] = { 20, 6},
+ [VCAP_IS2_ACT_REW_OP] = { 26, 9},
+ [VCAP_IS2_ACT_SMAC_REPLACE_ENA] = { 35, 1},
+ [VCAP_IS2_ACT_RSV] = { 36, 2},
+ [VCAP_IS2_ACT_ACL_ID] = { 38, 6},
+ [VCAP_IS2_ACT_HIT_CNT] = { 44, 32},
};
static const struct vcap_props vsc9959_vcap_props[] = {
*/
static u16 vsc9959_wm_enc(u16 value)
{
+ WARN_ON(value >= 16 * BIT(8));
+
if (value >= BIT(8))
return BIT(8) | (value / 16);
static void vsc9959_sched_speed_set(struct ocelot *ocelot, int port,
u32 speed)
{
+ u8 tas_speed;
+
+ switch (speed) {
+ case SPEED_10:
+ tas_speed = OCELOT_SPEED_10;
+ break;
+ case SPEED_100:
+ tas_speed = OCELOT_SPEED_100;
+ break;
+ case SPEED_1000:
+ tas_speed = OCELOT_SPEED_1000;
+ break;
+ case SPEED_2500:
+ tas_speed = OCELOT_SPEED_2500;
+ break;
+ default:
+ tas_speed = OCELOT_SPEED_1000;
+ break;
+ }
+
ocelot_rmw_rix(ocelot,
- QSYS_TAG_CONFIG_LINK_SPEED(speed),
+ QSYS_TAG_CONFIG_LINK_SPEED(tas_speed),
QSYS_TAG_CONFIG_LINK_SPEED_M,
QSYS_TAG_CONFIG, port);
}
.action_type_width = 1,
.action_table = {
[IS2_ACTION_TYPE_NORMAL] = {
- .width = 44,
+ .width = 50, /* HIT_CNT not included */
.count = 2
},
[IS2_ACTION_TYPE_SMAC_SIP] = {
*/
static u16 vsc9953_wm_enc(u16 value)
{
+ WARN_ON(value >= 16 * BIT(9));
+
if (value >= BIT(9))
return BIT(9) | (value / 16);
* the checksum, we can do this once, at the end.
*/
csum = csum_fold(csum_partial_copy_nocheck(image_data,
- dpage, len,
- 0));
+ dpage, len));
iowrite32(len, ioaddr + TYPHOON_REG_BOOT_LENGTH);
iowrite32(le16_to_cpu((__force __le16)csum),
u32 lastWrite;
};
-/* The Typoon transmit ring -- same as a basic ring, plus:
+/* The Typhoon transmit ring -- same as a basic ring, plus:
* lastRead: where we're at in regard to cleaning up the ring
* writeRegister: register to use for writing (different for Hi & Lo rings)
*/
obj-$(CONFIG_AQTION) += atlantic.o
-ccflags-y += -I$(src)
+ccflags-y += -I$(srctree)/$(src)
atlantic-objs := aq_main.o \
aq_nic.o \
atlantic-$(CONFIG_MACSEC) += aq_macsec.o
-atlantic-$(CONFIG_PTP_1588_CLOCK) += aq_ptp.o
\ No newline at end of file
+atlantic-$(CONFIG_PTP_1588_CLOCK) += aq_ptp.o
#define CCM_REG_GR_ARB_TYPE 0xd015c
/* [RW 2] Load (FIC0) channel group priority. The lowest priority is 0; the
highest priority is 3. It is supposed; that the Store channel priority is
- the compliment to 4 of the rest priorities - Aggregation channel; Load
+ the complement to 4 of the rest priorities - Aggregation channel; Load
(FIC0) channel and Load (FIC1). */
#define CCM_REG_GR_LD0_PR 0xd0164
/* [RW 2] Load (FIC1) channel group priority. The lowest priority is 0; the
highest priority is 3. It is supposed; that the Store channel priority is
- the compliment to 4 of the rest priorities - Aggregation channel; Load
+ the complement to 4 of the rest priorities - Aggregation channel; Load
(FIC0) channel and Load (FIC1). */
#define CCM_REG_GR_LD1_PR 0xd0168
/* [RW 2] General flags index. */
#define TCM_REG_GR_ARB_TYPE 0x50114
/* [RW 2] Load (FIC0) channel group priority. The lowest priority is 0; the
highest priority is 3. It is supposed that the Store channel is the
- compliment of the other 3 groups. */
+ complement of the other 3 groups. */
#define TCM_REG_GR_LD0_PR 0x5011c
/* [RW 2] Load (FIC1) channel group priority. The lowest priority is 0; the
highest priority is 3. It is supposed that the Store channel is the
- compliment of the other 3 groups. */
+ complement of the other 3 groups. */
#define TCM_REG_GR_LD1_PR 0x50120
/* [RW 4] The number of double REG-pairs; loaded from the STORM context and
sent to STORM; for a specific connection type. The double REG-pairs are
#define UCM_REG_GR_ARB_TYPE 0xe0144
/* [RW 2] Load (FIC0) channel group priority. The lowest priority is 0; the
highest priority is 3. It is supposed that the Store channel group is
- compliment to the others. */
+ complement to the others. */
#define UCM_REG_GR_LD0_PR 0xe014c
/* [RW 2] Load (FIC1) channel group priority. The lowest priority is 0; the
highest priority is 3. It is supposed that the Store channel group is
- compliment to the others. */
+ complement to the others. */
#define UCM_REG_GR_LD1_PR 0xe0150
/* [RW 2] The queue index for invalidate counter flag decision. */
#define UCM_REG_INV_CFLG_Q 0xe00e4
#define XCM_REG_GR_ARB_TYPE 0x2020c
/* [RW 2] Load (FIC0) channel group priority. The lowest priority is 0; the
highest priority is 3. It is supposed that the Channel group is the
- compliment of the other 3 groups. */
+ complement of the other 3 groups. */
#define XCM_REG_GR_LD0_PR 0x20214
/* [RW 2] Load (FIC1) channel group priority. The lowest priority is 0; the
highest priority is 3. It is supposed that the Channel group is the
- compliment of the other 3 groups. */
+ complement of the other 3 groups. */
#define XCM_REG_GR_LD1_PR 0x20218
/* [RW 1] Input nig0 Interface enable. If 0 - the valid input is
disregarded; acknowledge output is deasserted; all other signals are
*/
if (netdev->phydev) {
netif_carrier_off(netdev);
- phy_start_aneg(netdev->phydev);
+ phy_start(netdev->phydev);
}
netif_wake_queue(netdev);
napi_disable(&p->napi);
netif_stop_queue(netdev);
- if (netdev->phydev)
+ if (netdev->phydev) {
+ phy_stop(netdev->phydev);
phy_disconnect(netdev->phydev);
+ }
netif_carrier_off(netdev);
#define DPNI_VER_MAJOR 7
#define DPNI_VER_MINOR 0
#define DPNI_CMD_BASE_VERSION 1
+#define DPNI_CMD_2ND_VERSION 2
#define DPNI_CMD_ID_OFFSET 4
#define DPNI_CMD(id) (((id) << DPNI_CMD_ID_OFFSET) | DPNI_CMD_BASE_VERSION)
+#define DPNI_CMD_V2(id) (((id) << DPNI_CMD_ID_OFFSET) | DPNI_CMD_2ND_VERSION)
#define DPNI_CMDID_OPEN DPNI_CMD(0x801)
#define DPNI_CMDID_CLOSE DPNI_CMD(0x800)
#define DPNI_CMDID_SET_MAX_FRAME_LENGTH DPNI_CMD(0x216)
#define DPNI_CMDID_GET_MAX_FRAME_LENGTH DPNI_CMD(0x217)
#define DPNI_CMDID_SET_LINK_CFG DPNI_CMD(0x21A)
-#define DPNI_CMDID_SET_TX_SHAPING DPNI_CMD(0x21B)
+#define DPNI_CMDID_SET_TX_SHAPING DPNI_CMD_V2(0x21B)
#define DPNI_CMDID_SET_MCAST_PROMISC DPNI_CMD(0x220)
#define DPNI_CMDID_GET_MCAST_PROMISC DPNI_CMD(0x221)
/* Return all Fs if nothing was there */
if ((xgmac_read32(®s->mdio_stat, endian) & MDIO_STAT_RD_ER) &&
!priv->has_a011043) {
- dev_err(&bus->dev,
+ dev_dbg(&bus->dev,
"Error while reading PHY%d reg at %d.%hhu\n",
phy_id, dev_addr, regnum);
return 0xffff;
config HINIC
tristate "Huawei Intelligent PCIE Network Interface Card"
depends on (PCI_MSI && (X86 || ARM64))
+ select NET_DEVLINK
help
This driver supports HiNIC PCIE Ethernet cards.
To compile this driver as part of the kernel, choose Y here.
sizeof(port_mac_cmd),
&port_mac_cmd, &out_size);
if (err || out_size != sizeof(port_mac_cmd) ||
- (port_mac_cmd.status &&
- port_mac_cmd.status != HINIC_PF_SET_VF_ALREADY &&
- port_mac_cmd.status != HINIC_MGMT_STATUS_EXIST)) {
+ (port_mac_cmd.status &&
+ (port_mac_cmd.status != HINIC_PF_SET_VF_ALREADY || !HINIC_IS_VF(hwif)) &&
+ port_mac_cmd.status != HINIC_MGMT_STATUS_EXIST)) {
dev_err(&pdev->dev, "Failed to change MAC, err: %d, status: 0x%x, out size: 0x%x\n",
err, port_mac_cmd.status, out_size);
return -EFAULT;
err = hinic_port_msg_cmd(hwdev, HINIC_PORT_CMD_SET_MAC, &mac_info,
sizeof(mac_info), &mac_info, &out_size);
if (err || out_size != sizeof(mac_info) ||
- (mac_info.status && mac_info.status != HINIC_PF_SET_VF_ALREADY &&
- mac_info.status != HINIC_MGMT_STATUS_EXIST)) {
+ (mac_info.status && mac_info.status != HINIC_MGMT_STATUS_EXIST)) {
dev_err(&hwdev->func_to_io.hwif->pdev->dev, "Failed to set MAC, err: %d, status: 0x%x, out size: 0x%x\n",
err, mac_info.status, out_size);
return -EIO;
static int hinic_check_mac_info(u8 status, u16 vlan_id)
{
- if ((status && status != HINIC_MGMT_STATUS_EXIST &&
- status != HINIC_PF_SET_VF_ALREADY) ||
+ if ((status && status != HINIC_MGMT_STATUS_EXIST) ||
(vlan_id & CHECK_IPSU_15BIT &&
status == HINIC_MGMT_STATUS_EXIST))
return -EINVAL;
return -EINVAL;
}
- if (mac_info.status == HINIC_PF_SET_VF_ALREADY) {
- dev_warn(&hwdev->hwif->pdev->dev,
- "PF has already set VF MAC. Ignore update operation\n");
- return HINIC_PF_SET_VF_ALREADY;
- }
-
if (mac_info.status == HINIC_MGMT_STATUS_EXIST)
dev_warn(&hwdev->hwif->pdev->dev, "MAC is repeated. Ignore update operation\n");
static int __maybe_unused iavf_resume(struct device *dev_d)
{
struct pci_dev *pdev = to_pci_dev(dev_d);
- struct iavf_adapter *adapter = pci_get_drvdata(pdev);
- struct net_device *netdev = adapter->netdev;
+ struct net_device *netdev = pci_get_drvdata(pdev);
+ struct iavf_adapter *adapter = netdev_priv(netdev);
u32 err;
pci_set_master(pdev);
{
struct ice_hw_func_caps *func_caps = &hw->func_caps;
struct ice_hw_dev_caps *dev_caps = &hw->dev_caps;
- u32 valid_func, rxq_first_id, txq_first_id;
- u32 msix_vector_first_id, max_mtu;
+ struct ice_hw_common_caps cached_caps;
u32 num_funcs;
/* cache some func_caps values that should be restored after memset */
- valid_func = func_caps->common_cap.valid_functions;
- txq_first_id = func_caps->common_cap.txq_first_id;
- rxq_first_id = func_caps->common_cap.rxq_first_id;
- msix_vector_first_id = func_caps->common_cap.msix_vector_first_id;
- max_mtu = func_caps->common_cap.max_mtu;
+ cached_caps = func_caps->common_cap;
/* unset func capabilities */
memset(func_caps, 0, sizeof(*func_caps));
+#define ICE_RESTORE_FUNC_CAP(name) \
+ func_caps->common_cap.name = cached_caps.name
+
/* restore cached values */
- func_caps->common_cap.valid_functions = valid_func;
- func_caps->common_cap.txq_first_id = txq_first_id;
- func_caps->common_cap.rxq_first_id = rxq_first_id;
- func_caps->common_cap.msix_vector_first_id = msix_vector_first_id;
- func_caps->common_cap.max_mtu = max_mtu;
+ ICE_RESTORE_FUNC_CAP(valid_functions);
+ ICE_RESTORE_FUNC_CAP(txq_first_id);
+ ICE_RESTORE_FUNC_CAP(rxq_first_id);
+ ICE_RESTORE_FUNC_CAP(msix_vector_first_id);
+ ICE_RESTORE_FUNC_CAP(max_mtu);
+ ICE_RESTORE_FUNC_CAP(nvm_unified_update);
+ ICE_RESTORE_FUNC_CAP(nvm_update_pending_nvm);
+ ICE_RESTORE_FUNC_CAP(nvm_update_pending_orom);
+ ICE_RESTORE_FUNC_CAP(nvm_update_pending_netlist);
/* one Tx and one Rx queue in safe mode */
func_caps->common_cap.num_rxq = 1;
func_caps->guar_num_vsi = 1;
/* cache some dev_caps values that should be restored after memset */
- valid_func = dev_caps->common_cap.valid_functions;
- txq_first_id = dev_caps->common_cap.txq_first_id;
- rxq_first_id = dev_caps->common_cap.rxq_first_id;
- msix_vector_first_id = dev_caps->common_cap.msix_vector_first_id;
- max_mtu = dev_caps->common_cap.max_mtu;
+ cached_caps = dev_caps->common_cap;
num_funcs = dev_caps->num_funcs;
/* unset dev capabilities */
memset(dev_caps, 0, sizeof(*dev_caps));
+#define ICE_RESTORE_DEV_CAP(name) \
+ dev_caps->common_cap.name = cached_caps.name
+
/* restore cached values */
- dev_caps->common_cap.valid_functions = valid_func;
- dev_caps->common_cap.txq_first_id = txq_first_id;
- dev_caps->common_cap.rxq_first_id = rxq_first_id;
- dev_caps->common_cap.msix_vector_first_id = msix_vector_first_id;
- dev_caps->common_cap.max_mtu = max_mtu;
+ ICE_RESTORE_DEV_CAP(valid_functions);
+ ICE_RESTORE_DEV_CAP(txq_first_id);
+ ICE_RESTORE_DEV_CAP(rxq_first_id);
+ ICE_RESTORE_DEV_CAP(msix_vector_first_id);
+ ICE_RESTORE_DEV_CAP(max_mtu);
+ ICE_RESTORE_DEV_CAP(nvm_unified_update);
+ ICE_RESTORE_DEV_CAP(nvm_update_pending_nvm);
+ ICE_RESTORE_DEV_CAP(nvm_update_pending_orom);
+ ICE_RESTORE_DEV_CAP(nvm_update_pending_netlist);
dev_caps->num_funcs = num_funcs;
/* one Tx and one Rx queue per function in safe mode */
return -EIO;
}
- err = ice_aq_wait_for_event(pf, ice_aqc_opc_nvm_write, HZ, &event);
+ /* In most cases, firmware reports a write completion within a few
+ * milliseconds. However, it has been observed that a completion might
+ * take more than a second to complete in some cases. The timeout here
+ * is conservative and is intended to prevent failure to update when
+ * firmware is slow to respond.
+ */
+ err = ice_aq_wait_for_event(pf, ice_aqc_opc_nvm_write, 15 * HZ, &event);
if (err) {
dev_err(dev, "Timed out waiting for firmware write completion for module 0x%02x, err %d\n",
module, err);
return -EIO;
}
- err = ice_aq_wait_for_event(pf, ice_aqc_opc_nvm_write_activate, HZ,
+ err = ice_aq_wait_for_event(pf, ice_aqc_opc_nvm_write_activate, 30 * HZ,
&event);
if (err) {
dev_err(dev, "Timed out waiting for firmware to switch active flash banks, err %d\n",
* ice_vsi_delete - delete a VSI from the switch
* @vsi: pointer to VSI being removed
*/
-void ice_vsi_delete(struct ice_vsi *vsi)
+static void ice_vsi_delete(struct ice_vsi *vsi)
{
struct ice_pf *pf = vsi->back;
struct ice_vsi_ctx *ctxt;
*
* Returns 0 on success, negative on failure
*/
-int ice_vsi_clear(struct ice_vsi *vsi)
+static int ice_vsi_clear(struct ice_vsi *vsi)
{
struct ice_pf *pf = NULL;
struct device *dev;
* ice_vsi_put_qs - Release queues from VSI to PF
* @vsi: the VSI that is going to release queues
*/
-void ice_vsi_put_qs(struct ice_vsi *vsi)
+static void ice_vsi_put_qs(struct ice_vsi *vsi)
{
struct ice_pf *pf = vsi->back;
int i;
{
int i;
+ /* Avoid stale references by clearing map from vector to ring */
+ if (vsi->q_vectors) {
+ ice_for_each_q_vector(vsi, i) {
+ struct ice_q_vector *q_vector = vsi->q_vectors[i];
+
+ if (q_vector) {
+ q_vector->tx.ring = NULL;
+ q_vector->rx.ring = NULL;
+ }
+ }
+ }
+
if (vsi->tx_rings) {
for (i = 0; i < vsi->alloc_txq; i++) {
if (vsi->tx_rings[i]) {
if (status) {
dev_err(dev, "VSI %d failed lan queue config, error %s\n",
vsi->vsi_num, ice_stat_str(status));
- goto unroll_vector_base;
+ goto unroll_clear_rings;
}
/* Add switch rule to drop all Tx Flow Control Frames, of look up
void ice_cfg_sw_lldp(struct ice_vsi *vsi, bool tx, bool create);
-void ice_vsi_delete(struct ice_vsi *vsi);
-
-int ice_vsi_clear(struct ice_vsi *vsi);
-
#ifdef CONFIG_DCB
int ice_vsi_cfg_tc(struct ice_vsi *vsi, u8 ena_tc);
#endif /* CONFIG_DCB */
void
ice_write_qrxflxp_cntxt(struct ice_hw *hw, u16 pf_q, u32 rxdid, u32 prio);
-void ice_vsi_put_qs(struct ice_vsi *vsi);
-
void ice_vsi_dis_irq(struct ice_vsi *vsi);
void ice_vsi_free_irq(struct ice_vsi *vsi);
return -EBUSY;
vsi = ice_pf_vsi_setup(pf, pf->hw.port_info);
- if (!vsi) {
- status = -ENOMEM;
- goto unroll_vsi_setup;
- }
+ if (!vsi)
+ return -ENOMEM;
status = ice_cfg_netdev(vsi);
if (status) {
}
unroll_vsi_setup:
- if (vsi) {
- ice_vsi_free_q_vectors(vsi);
- ice_vsi_delete(vsi);
- ice_vsi_put_qs(vsi);
- ice_vsi_clear(vsi);
- }
+ ice_vsi_release(vsi);
return status;
}
}
ice_clear_interrupt_scheme(pf);
+ pci_save_state(pdev);
pci_wake_from_d3(pdev, pf->wol_ena);
pci_set_power_state(pdev, PCI_D3hot);
return 0;
return err;
}
-static int ixgbe_macvlan_up(struct net_device *vdev, void *data)
+static int ixgbe_macvlan_up(struct net_device *vdev,
+ struct netdev_nested_priv *priv)
{
- struct ixgbe_adapter *adapter = data;
+ struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data;
struct ixgbe_fwd_adapter *accel;
if (!netif_is_macvlan(vdev))
static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter)
{
+ struct netdev_nested_priv priv = {
+ .data = (void *)adapter,
+ };
+
netdev_walk_all_upper_dev_rcu(adapter->netdev,
- ixgbe_macvlan_up, adapter);
+ ixgbe_macvlan_up, &priv);
}
static void ixgbe_configure(struct ixgbe_adapter *adapter)
}
#endif /* CONFIG_IXGBE_DCB */
-static int ixgbe_reassign_macvlan_pool(struct net_device *vdev, void *data)
+static int ixgbe_reassign_macvlan_pool(struct net_device *vdev,
+ struct netdev_nested_priv *priv)
{
- struct ixgbe_adapter *adapter = data;
+ struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data;
struct ixgbe_fwd_adapter *accel;
int pool;
static void ixgbe_defrag_macvlan_pools(struct net_device *dev)
{
struct ixgbe_adapter *adapter = netdev_priv(dev);
+ struct netdev_nested_priv priv = {
+ .data = (void *)adapter,
+ };
/* flush any stale bits out of the fwd bitmask */
bitmap_clear(adapter->fwd_bitmask, 1, 63);
/* walk through upper devices reassigning pools */
netdev_walk_all_upper_dev_rcu(dev, ixgbe_reassign_macvlan_pool,
- adapter);
+ &priv);
}
/**
u8 queue;
};
-static int get_macvlan_queue(struct net_device *upper, void *_data)
+static int get_macvlan_queue(struct net_device *upper,
+ struct netdev_nested_priv *priv)
{
if (netif_is_macvlan(upper)) {
struct ixgbe_fwd_adapter *vadapter = macvlan_accel_priv(upper);
- struct upper_walk_data *data = _data;
- struct ixgbe_adapter *adapter = data->adapter;
- int ifindex = data->ifindex;
+ struct ixgbe_adapter *adapter;
+ struct upper_walk_data *data;
+ int ifindex;
+ data = (struct upper_walk_data *)priv->data;
+ ifindex = data->ifindex;
+ adapter = data->adapter;
if (vadapter && upper->ifindex == ifindex) {
data->queue = adapter->rx_ring[vadapter->rx_base_queue]->reg_idx;
data->action = data->queue;
{
struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];
unsigned int num_vfs = adapter->num_vfs, vf;
+ struct netdev_nested_priv priv;
struct upper_walk_data data;
struct net_device *upper;
data.ifindex = ifindex;
data.action = 0;
data.queue = 0;
+ priv.data = (void *)&data;
if (netdev_walk_all_upper_dev_rcu(adapter->netdev,
- get_macvlan_queue, &data)) {
+ get_macvlan_queue, &priv)) {
*action = data.action;
*queue = data.queue;
int pkts = 0;
int bytes = 0;
+ netif_tx_lock(net_dev);
while (pkts < budget) {
struct ltq_dma_desc *desc = &ch->dma.desc_base[ch->tx_free];
net_dev->stats.tx_bytes += bytes;
netdev_completed_queue(ch->priv->net_dev, pkts, bytes);
+ netif_tx_unlock(net_dev);
if (netif_queue_stopped(net_dev))
netif_wake_queue(net_dev);
txq->last_desc = txq->size - 1;
txq->buf = kmalloc_array(txq->size, sizeof(*txq->buf), GFP_KERNEL);
- if (!txq->buf) {
- dma_free_coherent(pp->dev->dev.parent,
- txq->size * MVNETA_DESC_ALIGNED_SIZE,
- txq->descs, txq->descs_phys);
+ if (!txq->buf)
return -ENOMEM;
- }
/* Allocate DMA buffers for TSO MAC/IP/TCP headers */
txq->tso_hdrs = dma_alloc_coherent(pp->dev->dev.parent,
txq->size * TSO_HEADER_SIZE,
&txq->tso_hdrs_phys, GFP_KERNEL);
- if (!txq->tso_hdrs) {
- kfree(txq->buf);
- dma_free_coherent(pp->dev->dev.parent,
- txq->size * MVNETA_DESC_ALIGNED_SIZE,
- txq->descs, txq->descs_phys);
+ if (!txq->tso_hdrs)
return -ENOMEM;
- }
/* Setup XPS mapping */
if (txq_number > 1)
static const u16 msgs_offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
-void otx2_mbox_reset(struct otx2_mbox *mbox, int devid)
+void __otx2_mbox_reset(struct otx2_mbox *mbox, int devid)
{
void *hw_mbase = mbox->hwbase + (devid * MBOX_SIZE);
struct otx2_mbox_dev *mdev = &mbox->dev[devid];
tx_hdr = hw_mbase + mbox->tx_start;
rx_hdr = hw_mbase + mbox->rx_start;
- spin_lock(&mdev->mbox_lock);
mdev->msg_size = 0;
mdev->rsp_size = 0;
tx_hdr->num_msgs = 0;
tx_hdr->msg_size = 0;
rx_hdr->num_msgs = 0;
rx_hdr->msg_size = 0;
+}
+EXPORT_SYMBOL(__otx2_mbox_reset);
+
+void otx2_mbox_reset(struct otx2_mbox *mbox, int devid)
+{
+ struct otx2_mbox_dev *mdev = &mbox->dev[devid];
+
+ spin_lock(&mdev->mbox_lock);
+ __otx2_mbox_reset(mbox, devid);
spin_unlock(&mdev->mbox_lock);
}
EXPORT_SYMBOL(otx2_mbox_reset);
};
void otx2_mbox_reset(struct otx2_mbox *mbox, int devid);
+void __otx2_mbox_reset(struct otx2_mbox *mbox, int devid);
void otx2_mbox_destroy(struct otx2_mbox *mbox);
int otx2_mbox_init(struct otx2_mbox *mbox, void __force *hwbase,
struct pci_dev *pdev, void __force *reg_base,
int rvu_get_nixlf_count(struct rvu *rvu);
void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int npalf);
int nix_get_nixlf(struct rvu *rvu, u16 pcifunc, int *nixlf, int *nix_blkaddr);
+int nix_update_bcast_mce_list(struct rvu *rvu, u16 pcifunc, bool add);
/* NPC APIs */
int rvu_npc_init(struct rvu *rvu);
void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf);
void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc,
int nixlf, u64 chan);
-void rvu_npc_disable_bcast_entry(struct rvu *rvu, u16 pcifunc);
+void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, bool enable);
int rvu_npc_update_rxvlan(struct rvu *rvu, u16 pcifunc, int nixlf);
void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf);
void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf);
#include "npc.h"
#include "cgx.h"
-static int nix_update_bcast_mce_list(struct rvu *rvu, u16 pcifunc, bool add);
static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req,
int type, int chan_id);
return 0;
}
-static int nix_update_bcast_mce_list(struct rvu *rvu, u16 pcifunc, bool add)
+int nix_update_bcast_mce_list(struct rvu *rvu, u16 pcifunc, bool add)
{
int err = 0, idx, next_idx, last_idx;
struct nix_mce_list *mce_list;
/* Disable MCAM entry in NPC */
if (!mce_list->count) {
- rvu_npc_disable_bcast_entry(rvu, pcifunc);
+ rvu_npc_enable_bcast_entry(rvu, pcifunc, false);
goto end;
}
NIX_INTF_RX, &entry, true);
}
-void rvu_npc_disable_bcast_entry(struct rvu *rvu, u16 pcifunc)
+void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, bool enable)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
int blkaddr, index;
pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK;
index = npc_get_nixlf_mcam_index(mcam, pcifunc, 0, NIXLF_BCAST_ENTRY);
- npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false);
+ npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);
}
void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf,
nixlf, NIXLF_UCAST_ENTRY);
npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);
- /* For PF, ena/dis promisc and bcast MCAM match entries */
- if (pcifunc & RVU_PFVF_FUNC_MASK)
+ /* For PF, ena/dis promisc and bcast MCAM match entries.
+ * For VFs add/delete from bcast list when RX multicast
+ * feature is present.
+ */
+ if (pcifunc & RVU_PFVF_FUNC_MASK && !rvu->hw->cap.nix_rx_multicast)
return;
/* For bcast, enable/disable only if it's action is not
* packet replication, incase if action is replication
- * then this PF's nixlf is removed from bcast replication
+ * then this PF/VF's nixlf is removed from bcast replication
* list.
*/
- index = npc_get_nixlf_mcam_index(mcam, pcifunc,
+ index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK,
nixlf, NIXLF_BCAST_ENTRY);
bank = npc_get_bank(mcam, index);
*(u64 *)&action = rvu_read64(rvu, blkaddr,
NPC_AF_MCAMEX_BANKX_ACTION(index & (mcam->banksize - 1), bank));
- if (action.op != NIX_RX_ACTIONOP_MCAST)
+
+ /* VFs will not have BCAST entry */
+ if (action.op != NIX_RX_ACTIONOP_MCAST &&
+ !(pcifunc & RVU_PFVF_FUNC_MASK)) {
npc_enable_mcam_entry(rvu, mcam,
blkaddr, index, enable);
+ } else {
+ nix_update_bcast_mce_list(rvu, pcifunc, enable);
+ /* Enable PF's BCAST entry for packet replication */
+ rvu_npc_enable_bcast_entry(rvu, pcifunc, enable);
+ }
+
if (enable)
rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf);
else
dst_mbox = &pf->mbox;
dst_size = dst_mbox->mbox.tx_size -
ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN);
- /* Check if msgs fit into destination area */
- if (mbox_hdr->msg_size > dst_size)
+ /* Check if msgs fit into destination area and has valid size */
+ if (mbox_hdr->msg_size > dst_size || !mbox_hdr->msg_size)
return -EINVAL;
dst_mdev = &dst_mbox->mbox.dev[0];
end:
offset = mbox->rx_start + msg->next_msgoff;
+ if (mdev->msgs_acked == (vf_mbox->up_num_msgs - 1))
+ __otx2_mbox_reset(mbox, 0);
mdev->msgs_acked++;
}
-
- otx2_mbox_reset(mbox, vf_idx);
}
static irqreturn_t otx2_pfvf_mbox_intr_handler(int irq, void *pf_irq)
msg = (struct mbox_msghdr *)(mdev->mbase + offset);
otx2_process_pfaf_mbox_msg(pf, msg);
offset = mbox->rx_start + msg->next_msgoff;
+ if (mdev->msgs_acked == (af_mbox->num_msgs - 1))
+ __otx2_mbox_reset(mbox, 0);
mdev->msgs_acked++;
}
- otx2_mbox_reset(mbox, 0);
}
static void otx2_handle_link_event(struct otx2_nic *pf)
err = otx2_rxtx_enable(pf, true);
if (err)
- goto err_free_cints;
+ goto err_tx_stop_queues;
return 0;
+err_tx_stop_queues:
+ netif_tx_stop_all_queues(netdev);
+ netif_carrier_off(netdev);
err_free_cints:
otx2_free_cints(pf, qidx);
vec = pci_irq_vector(pf->pdev,
sqe_hdr->ol3type = NIX_SENDL3TYPE_IP4_CKSUM;
} else if (skb->protocol == htons(ETH_P_IPV6)) {
proto = ipv6_hdr(skb)->nexthdr;
+ sqe_hdr->ol3type = NIX_SENDL3TYPE_IP6;
}
if (proto == IPPROTO_TCP)
msg = (struct mbox_msghdr *)(mdev->mbase + offset);
otx2vf_process_vfaf_mbox_msg(af_mbox->pfvf, msg);
offset = mbox->rx_start + msg->next_msgoff;
+ if (mdev->msgs_acked == (af_mbox->num_msgs - 1))
+ __otx2_mbox_reset(mbox, 0);
mdev->msgs_acked++;
}
-
- otx2_mbox_reset(mbox, 0);
}
static int otx2vf_process_mbox_msg_up(struct otx2_nic *vf,
MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10,
};
-static struct mlx5_cmd_work_ent *alloc_cmd(struct mlx5_cmd *cmd,
- struct mlx5_cmd_msg *in,
- struct mlx5_cmd_msg *out,
- void *uout, int uout_size,
- mlx5_cmd_cbk_t cbk,
- void *context, int page_queue)
+static struct mlx5_cmd_work_ent *
+cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in,
+ struct mlx5_cmd_msg *out, void *uout, int uout_size,
+ mlx5_cmd_cbk_t cbk, void *context, int page_queue)
{
gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL;
struct mlx5_cmd_work_ent *ent;
if (!ent)
return ERR_PTR(-ENOMEM);
+ ent->idx = -EINVAL;
ent->in = in;
ent->out = out;
ent->uout = uout;
ent->context = context;
ent->cmd = cmd;
ent->page_queue = page_queue;
+ refcount_set(&ent->refcnt, 1);
return ent;
}
+static void cmd_free_ent(struct mlx5_cmd_work_ent *ent)
+{
+ kfree(ent);
+}
+
static u8 alloc_token(struct mlx5_cmd *cmd)
{
u8 token;
return token;
}
-static int alloc_ent(struct mlx5_cmd *cmd)
+static int cmd_alloc_index(struct mlx5_cmd *cmd)
{
unsigned long flags;
int ret;
return ret < cmd->max_reg_cmds ? ret : -ENOMEM;
}
-static void free_ent(struct mlx5_cmd *cmd, int idx)
+static void cmd_free_index(struct mlx5_cmd *cmd, int idx)
{
unsigned long flags;
spin_unlock_irqrestore(&cmd->alloc_lock, flags);
}
+static void cmd_ent_get(struct mlx5_cmd_work_ent *ent)
+{
+ refcount_inc(&ent->refcnt);
+}
+
+static void cmd_ent_put(struct mlx5_cmd_work_ent *ent)
+{
+ if (!refcount_dec_and_test(&ent->refcnt))
+ return;
+
+ if (ent->idx >= 0)
+ cmd_free_index(ent->cmd, ent->idx);
+
+ cmd_free_ent(ent);
+}
+
static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
{
return cmd->cmd_buf + (idx << cmd->log_stride);
ent->ret = -ETIMEDOUT;
}
-static void free_cmd(struct mlx5_cmd_work_ent *ent)
-{
- kfree(ent);
-}
-
static int verify_signature(struct mlx5_cmd_work_ent *ent)
{
struct mlx5_cmd_mailbox *next = ent->out->next;
struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev,
cmd);
+ mlx5_cmd_eq_recover(dev);
+
+ /* Maybe got handled by eq recover ? */
+ if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) {
+ mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx,
+ mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in));
+ goto out; /* phew, already handled */
+ }
+
ent->ret = -ETIMEDOUT;
- mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n",
- mlx5_command_str(msg_to_opcode(ent->in)),
- msg_to_opcode(ent->in));
+ mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n",
+ ent->idx, mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in));
mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true);
+
+out:
+ cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */
}
static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg);
return cmd->allowed_opcode == opcode;
}
+static int cmd_alloc_index_retry(struct mlx5_cmd *cmd)
+{
+ unsigned long alloc_end = jiffies + msecs_to_jiffies(1000);
+ int idx;
+
+retry:
+ idx = cmd_alloc_index(cmd);
+ if (idx < 0 && time_before(jiffies, alloc_end)) {
+ /* Index allocation can fail on heavy load of commands. This is a temporary
+ * situation as the current command already holds the semaphore, meaning that
+ * another command completion is being handled and it is expected to release
+ * the entry index soon.
+ */
+ cpu_relax();
+ goto retry;
+ }
+ return idx;
+}
+
+bool mlx5_cmd_is_down(struct mlx5_core_dev *dev)
+{
+ return pci_channel_offline(dev->pdev) ||
+ dev->cmd.state != MLX5_CMDIF_STATE_UP ||
+ dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR;
+}
+
static void cmd_work_handler(struct work_struct *work)
{
struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);
sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem;
down(sem);
if (!ent->page_queue) {
- alloc_ret = alloc_ent(cmd);
+ alloc_ret = cmd_alloc_index_retry(cmd);
if (alloc_ret < 0) {
mlx5_core_err_rl(dev, "failed to allocate command entry\n");
if (ent->callback) {
ent->callback(-EAGAIN, ent->context);
mlx5_free_cmd_msg(dev, ent->out);
free_msg(dev, ent->in);
- free_cmd(ent);
+ cmd_ent_put(ent);
} else {
ent->ret = -EAGAIN;
complete(&ent->done);
ent->ts1 = ktime_get_ns();
cmd_mode = cmd->mode;
- if (ent->callback)
- schedule_delayed_work(&ent->cb_timeout_work, cb_timeout);
+ if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, cb_timeout))
+ cmd_ent_get(ent);
set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state);
/* Skip sending command to fw if internal error */
- if (pci_channel_offline(dev->pdev) ||
- dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR ||
- cmd->state != MLX5_CMDIF_STATE_UP ||
- !opcode_allowed(&dev->cmd, ent->op)) {
+ if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) {
u8 status = 0;
u32 drv_synd;
MLX5_SET(mbox_out, ent->out, syndrome, drv_synd);
mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true);
- /* no doorbell, no need to keep the entry */
- free_ent(cmd, ent->idx);
- if (ent->callback)
- free_cmd(ent);
return;
}
+ cmd_ent_get(ent); /* for the _real_ FW event on completion */
/* ring doorbell after the descriptor is valid */
mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);
wmb();
}
}
+enum {
+ MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000,
+};
+
+static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev,
+ struct mlx5_cmd_work_ent *ent)
+{
+ unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC);
+
+ mlx5_cmd_eq_recover(dev);
+
+ /* Re-wait on the ent->done after executing the recovery flow. If the
+ * recovery flow (or any other recovery flow running simultaneously)
+ * has recovered an EQE, it should cause the entry to be completed by
+ * the command interface.
+ */
+ if (wait_for_completion_timeout(&ent->done, timeout)) {
+ mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx,
+ mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in));
+ return;
+ }
+
+ mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx,
+ mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in));
+
+ ent->ret = -ETIMEDOUT;
+ mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true);
+}
+
static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
{
unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC);
ent->ret = -ECANCELED;
goto out_err;
}
- if (cmd->mode == CMD_MODE_POLLING || ent->polling) {
+ if (cmd->mode == CMD_MODE_POLLING || ent->polling)
wait_for_completion(&ent->done);
- } else if (!wait_for_completion_timeout(&ent->done, timeout)) {
- ent->ret = -ETIMEDOUT;
- mlx5_cmd_comp_handler(dev, 1UL << ent->idx, true);
- }
+ else if (!wait_for_completion_timeout(&ent->done, timeout))
+ wait_func_handle_exec_timeout(dev, ent);
out_err:
err = ent->ret;
if (callback && page_queue)
return -EINVAL;
- ent = alloc_cmd(cmd, in, out, uout, uout_size, callback, context,
- page_queue);
+ ent = cmd_alloc_ent(cmd, in, out, uout, uout_size,
+ callback, context, page_queue);
if (IS_ERR(ent))
return PTR_ERR(ent);
+ /* put for this ent is when consumed, depending on the use case
+ * 1) (!callback) blocking flow: by caller after wait_func completes
+ * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled
+ */
+
ent->token = token;
ent->polling = force_polling;
}
if (callback)
- goto out;
+ goto out; /* mlx5_cmd_comp_handler() will put(ent) */
err = wait_func(dev, ent);
- if (err == -ETIMEDOUT)
- goto out;
- if (err == -ECANCELED)
+ if (err == -ETIMEDOUT || err == -ECANCELED)
goto out_free;
ds = ent->ts2 - ent->ts1;
*status = ent->status;
out_free:
- free_cmd(ent);
+ cmd_ent_put(ent);
out:
return err;
}
if (!forced) {
mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n",
ent->idx);
- free_ent(cmd, ent->idx);
- free_cmd(ent);
+ cmd_ent_put(ent);
}
continue;
}
- if (ent->callback)
- cancel_delayed_work(&ent->cb_timeout_work);
+ if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work))
+ cmd_ent_put(ent); /* timeout work was canceled */
+
+ if (!forced || /* Real FW completion */
+ pci_channel_offline(dev->pdev) || /* FW is inaccessible */
+ dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
+ cmd_ent_put(ent);
+
if (ent->page_queue)
sem = &cmd->pages_sem;
else
ent->ret, deliv_status_to_str(ent->status), ent->status);
}
- /* only real completion will free the entry slot */
- if (!forced)
- free_ent(cmd, ent->idx);
-
if (ent->callback) {
ds = ent->ts2 - ent->ts1;
if (ent->op < MLX5_CMD_OP_MAX) {
free_msg(dev, ent->in);
err = err ? err : ent->status;
- if (!forced)
- free_cmd(ent);
+ /* final consumer is done, release ent */
+ cmd_ent_put(ent);
callback(err, context);
} else {
+ /* release wait_func() so mlx5_cmd_invoke()
+ * can make the final ent_put()
+ */
complete(&ent->done);
}
up(sem);
void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev)
{
+ struct mlx5_cmd *cmd = &dev->cmd;
+ unsigned long bitmask;
unsigned long flags;
u64 vector;
+ int i;
/* wait for pending handlers to complete */
mlx5_eq_synchronize_cmd_irq(dev);
if (!vector)
goto no_trig;
+ bitmask = vector;
+ /* we must increment the allocated entries refcount before triggering the completions
+ * to guarantee pending commands will not get freed in the meanwhile.
+ * For that reason, it also has to be done inside the alloc_lock.
+ */
+ for_each_set_bit(i, &bitmask, (1 << cmd->log_sz))
+ cmd_ent_get(cmd->ent_arr[i]);
vector |= MLX5_TRIGGERED_CMD_COMP;
spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
mlx5_core_dbg(dev, "vector 0x%llx\n", vector);
mlx5_cmd_comp_handler(dev, vector, true);
+ for_each_set_bit(i, &bitmask, (1 << cmd->log_sz))
+ cmd_ent_put(cmd->ent_arr[i]);
return;
no_trig:
u8 token;
opcode = MLX5_GET(mbox_in, in, opcode);
- if (pci_channel_offline(dev->pdev) ||
- dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR ||
- dev->cmd.state != MLX5_CMDIF_STATE_UP ||
- !opcode_allowed(&dev->cmd, opcode)) {
+ if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode)) {
err = mlx5_internal_err_ret_value(dev, opcode, &drv_synd, &status);
MLX5_SET(mbox_out, out, status, status);
MLX5_SET(mbox_out, out, syndrome, drv_synd);
#define MLX5_MPWRQ_PAGES_PER_WQE BIT(MLX5_MPWRQ_WQE_PAGE_ORDER)
#define MLX5_MTT_OCTW(npages) (ALIGN(npages, 8) / 2)
-#define MLX5E_REQUIRED_WQE_MTTS (ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8))
+/* Add another page to MLX5E_REQUIRED_WQE_MTTS as a buffer between
+ * WQEs, This page will absorb write overflow by the hardware, when
+ * receiving packets larger than MTU. These oversize packets are
+ * dropped by the driver at a later stage.
+ */
+#define MLX5E_REQUIRED_WQE_MTTS (ALIGN(MLX5_MPWRQ_PAGES_PER_WQE + 1, 8))
#define MLX5E_LOG_ALIGNED_MPWQE_PPW (ilog2(MLX5E_REQUIRED_WQE_MTTS))
#define MLX5E_REQUIRED_MTTS(wqes) (wqes * MLX5E_REQUIRED_WQE_MTTS)
#define MLX5E_MAX_RQ_NUM_MTTS \
u32 rqn;
struct mlx5_core_dev *mdev;
struct mlx5_core_mkey umr_mkey;
+ struct mlx5e_dma_info wqe_overflow;
/* XDP read-mostly */
struct xdp_rxq_info xdp_rxq;
if (fec_policy >= (1 << MLX5E_FEC_LLRS_272_257_1) && !fec_50g_per_lane)
return -EOPNOTSUPP;
+ if (fec_policy && !mlx5e_fec_in_caps(dev, fec_policy))
+ return -EOPNOTSUPP;
+
MLX5_SET(pplm_reg, in, local_port, 1);
err = mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PPLM, 0, 0);
if (err)
rtnl_unlock();
}
+struct neigh_update_work {
+ struct work_struct work;
+ struct neighbour *n;
+ struct mlx5e_neigh_hash_entry *nhe;
+};
+
+static void mlx5e_release_neigh_update_work(struct neigh_update_work *update_work)
+{
+ neigh_release(update_work->n);
+ mlx5e_rep_neigh_entry_release(update_work->nhe);
+ kfree(update_work);
+}
+
static void mlx5e_rep_neigh_update(struct work_struct *work)
{
- struct mlx5e_neigh_hash_entry *nhe =
- container_of(work, struct mlx5e_neigh_hash_entry, neigh_update_work);
- struct neighbour *n = nhe->n;
+ struct neigh_update_work *update_work = container_of(work, struct neigh_update_work,
+ work);
+ struct mlx5e_neigh_hash_entry *nhe = update_work->nhe;
+ struct neighbour *n = update_work->n;
struct mlx5e_encap_entry *e;
unsigned char ha[ETH_ALEN];
struct mlx5e_priv *priv;
mlx5e_rep_update_flows(priv, e, neigh_connected, ha);
mlx5e_encap_put(priv, e);
}
- mlx5e_rep_neigh_entry_release(nhe);
rtnl_unlock();
- neigh_release(n);
+ mlx5e_release_neigh_update_work(update_work);
}
-static void mlx5e_rep_queue_neigh_update_work(struct mlx5e_priv *priv,
- struct mlx5e_neigh_hash_entry *nhe,
- struct neighbour *n)
+static struct neigh_update_work *mlx5e_alloc_neigh_update_work(struct mlx5e_priv *priv,
+ struct neighbour *n)
{
- /* Take a reference to ensure the neighbour and mlx5 encap
- * entry won't be destructed until we drop the reference in
- * delayed work.
- */
- neigh_hold(n);
+ struct neigh_update_work *update_work;
+ struct mlx5e_neigh_hash_entry *nhe;
+ struct mlx5e_neigh m_neigh = {};
- /* This assignment is valid as long as the the neigh reference
- * is taken
- */
- nhe->n = n;
+ update_work = kzalloc(sizeof(*update_work), GFP_ATOMIC);
+ if (WARN_ON(!update_work))
+ return NULL;
- if (!queue_work(priv->wq, &nhe->neigh_update_work)) {
- mlx5e_rep_neigh_entry_release(nhe);
- neigh_release(n);
+ m_neigh.dev = n->dev;
+ m_neigh.family = n->ops->family;
+ memcpy(&m_neigh.dst_ip, n->primary_key, n->tbl->key_len);
+
+ /* Obtain reference to nhe as last step in order not to release it in
+ * atomic context.
+ */
+ rcu_read_lock();
+ nhe = mlx5e_rep_neigh_entry_lookup(priv, &m_neigh);
+ rcu_read_unlock();
+ if (!nhe) {
+ kfree(update_work);
+ return NULL;
}
+
+ INIT_WORK(&update_work->work, mlx5e_rep_neigh_update);
+ neigh_hold(n);
+ update_work->n = n;
+ update_work->nhe = nhe;
+
+ return update_work;
}
static int mlx5e_rep_netevent_event(struct notifier_block *nb,
struct net_device *netdev = rpriv->netdev;
struct mlx5e_priv *priv = netdev_priv(netdev);
struct mlx5e_neigh_hash_entry *nhe = NULL;
- struct mlx5e_neigh m_neigh = {};
+ struct neigh_update_work *update_work;
struct neigh_parms *p;
struct neighbour *n;
bool found = false;
#endif
return NOTIFY_DONE;
- m_neigh.dev = n->dev;
- m_neigh.family = n->ops->family;
- memcpy(&m_neigh.dst_ip, n->primary_key, n->tbl->key_len);
-
- rcu_read_lock();
- nhe = mlx5e_rep_neigh_entry_lookup(priv, &m_neigh);
- rcu_read_unlock();
- if (!nhe)
+ update_work = mlx5e_alloc_neigh_update_work(priv, n);
+ if (!update_work)
return NOTIFY_DONE;
- mlx5e_rep_queue_neigh_update_work(priv, nhe, n);
+ queue_work(priv->wq, &update_work->work);
break;
case NETEVENT_DELAY_PROBE_TIME_UPDATE:
(*nhe)->priv = priv;
memcpy(&(*nhe)->m_neigh, &e->m_neigh, sizeof(e->m_neigh));
- INIT_WORK(&(*nhe)->neigh_update_work, mlx5e_rep_neigh_update);
spin_lock_init(&(*nhe)->encap_list_lock);
INIT_LIST_HEAD(&(*nhe)->encap_list);
refcount_set(&(*nhe)->refcnt, 1);
case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
ip6_offset = (offset - offsetof(struct ipv6hdr, saddr));
ip6_offset /= 4;
- if (ip6_offset < 8)
+ if (ip6_offset < 4)
tuple->ip.src_v6.s6_addr32[ip6_offset] = cpu_to_be32(val);
+ else if (ip6_offset < 8)
+ tuple->ip.dst_v6.s6_addr32[ip6_offset - 4] = cpu_to_be32(val);
else
return -EOPNOTSUPP;
break;
break;
}
+ if (WARN_ONCE(*rule_p, "VLAN rule already exists type %d", rule_type))
+ return 0;
+
*rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
if (IS_ERR(*rule_p)) {
for_each_set_bit(i, priv->fs.vlan.active_svlans, VLAN_N_VID)
mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, i);
- if (priv->fs.vlan.cvlan_filter_disabled &&
- !(priv->netdev->flags & IFF_PROMISC))
+ if (priv->fs.vlan.cvlan_filter_disabled)
mlx5e_add_any_vid_rules(priv);
}
for_each_set_bit(i, priv->fs.vlan.active_svlans, VLAN_N_VID)
mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, i);
- if (priv->fs.vlan.cvlan_filter_disabled &&
- !(priv->netdev->flags & IFF_PROMISC))
+ WARN_ON_ONCE(!(test_bit(MLX5E_STATE_DESTROYING, &priv->state)));
+
+ /* must be called after DESTROY bit is set and
+ * set_rx_mode is called and flushed
+ */
+ if (priv->fs.vlan.cvlan_filter_disabled)
mlx5e_del_any_vid_rules(priv);
}
static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
u64 npages, u8 page_shift,
- struct mlx5_core_mkey *umr_mkey)
+ struct mlx5_core_mkey *umr_mkey,
+ dma_addr_t filler_addr)
{
- int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
+ struct mlx5_mtt *mtt;
+ int inlen;
void *mkc;
u32 *in;
int err;
+ int i;
+
+ inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + sizeof(*mtt) * npages;
in = kvzalloc(inlen, GFP_KERNEL);
if (!in)
MLX5_SET(mkc, mkc, translations_octword_size,
MLX5_MTT_OCTW(npages));
MLX5_SET(mkc, mkc, log_page_size, page_shift);
+ MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
+ MLX5_MTT_OCTW(npages));
+
+ /* Initialize the mkey with all MTTs pointing to a default
+ * page (filler_addr). When the channels are activated, UMR
+ * WQEs will redirect the RX WQEs to the actual memory from
+ * the RQ's pool, while the gaps (wqe_overflow) remain mapped
+ * to the default page.
+ */
+ mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
+ for (i = 0 ; i < npages ; i++)
+ mtt[i].ptag = cpu_to_be64(filler_addr);
err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
{
u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->mpwqe.wq));
- return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey);
+ return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey,
+ rq->wqe_overflow.addr);
}
static inline u64 mlx5e_get_mpwqe_offset(struct mlx5e_rq *rq, u16 wqe_ix)
mlx5e_reporter_rq_cqe_err(rq);
}
+static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
+{
+ rq->wqe_overflow.page = alloc_page(GFP_KERNEL);
+ if (!rq->wqe_overflow.page)
+ return -ENOMEM;
+
+ rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0,
+ PAGE_SIZE, rq->buff.map_dir);
+ if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) {
+ __free_page(rq->wqe_overflow.page);
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
+{
+ dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE,
+ rq->buff.map_dir);
+ __free_page(rq->wqe_overflow.page);
+}
+
static int mlx5e_alloc_rq(struct mlx5e_channel *c,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
rq_xdp_ix += params->num_channels * MLX5E_RQ_GROUP_XSK;
err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix);
if (err < 0)
- goto err_rq_wq_destroy;
+ goto err_rq_xdp_prog;
rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
&rq->wq_ctrl);
+ if (err)
+ goto err_rq_xdp;
+
+ err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
if (err)
goto err_rq_wq_destroy;
err = mlx5e_create_rq_umr_mkey(mdev, rq);
if (err)
- goto err_rq_wq_destroy;
+ goto err_rq_drop_page;
rq->mkey_be = cpu_to_be32(rq->umr_mkey.key);
err = mlx5e_rq_alloc_mpwqe_info(rq, c);
if (err)
- goto err_free;
+ goto err_rq_mkey;
break;
default: /* MLX5_WQ_TYPE_CYCLIC */
err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
&rq->wq_ctrl);
if (err)
- goto err_rq_wq_destroy;
+ goto err_rq_xdp;
rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR];
GFP_KERNEL, cpu_to_node(c->cpu));
if (!rq->wqe.frags) {
err = -ENOMEM;
- goto err_free;
+ goto err_rq_wq_destroy;
}
err = mlx5e_init_di_list(rq, wq_sz, c->cpu);
if (err)
- goto err_free;
+ goto err_rq_frags;
rq->mkey_be = c->mkey_be;
}
err = mlx5e_rq_set_handlers(rq, params, xsk);
if (err)
- goto err_free;
+ goto err_free_by_rq_type;
if (xsk) {
err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
if (IS_ERR(rq->page_pool)) {
err = PTR_ERR(rq->page_pool);
rq->page_pool = NULL;
- goto err_free;
+ goto err_free_by_rq_type;
}
err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
MEM_TYPE_PAGE_POOL, rq->page_pool);
}
if (err)
- goto err_free;
+ goto err_free_by_rq_type;
for (i = 0; i < wq_sz; i++) {
if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
return 0;
-err_free:
+err_free_by_rq_type:
switch (rq->wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
kvfree(rq->mpwqe.info);
+err_rq_mkey:
mlx5_core_destroy_mkey(mdev, &rq->umr_mkey);
+err_rq_drop_page:
+ mlx5e_free_mpwqe_rq_drop_page(rq);
break;
default: /* MLX5_WQ_TYPE_CYCLIC */
- kvfree(rq->wqe.frags);
mlx5e_free_di_list(rq);
+err_rq_frags:
+ kvfree(rq->wqe.frags);
}
-
err_rq_wq_destroy:
+ mlx5_wq_destroy(&rq->wq_ctrl);
+err_rq_xdp:
+ xdp_rxq_info_unreg(&rq->xdp_rxq);
+err_rq_xdp_prog:
if (params->xdp_prog)
bpf_prog_put(params->xdp_prog);
- xdp_rxq_info_unreg(&rq->xdp_rxq);
- page_pool_destroy(rq->page_pool);
- mlx5_wq_destroy(&rq->wq_ctrl);
return err;
}
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
kvfree(rq->mpwqe.info);
mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey);
+ mlx5e_free_mpwqe_rq_drop_page(rq);
break;
default: /* MLX5_WQ_TYPE_CYCLIC */
kvfree(rq->wqe.frags);
}
#endif
+static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev,
+ struct sk_buff *skb)
+{
+ switch (skb->inner_protocol) {
+ case htons(ETH_P_IP):
+ case htons(ETH_P_IPV6):
+ case htons(ETH_P_TEB):
+ return true;
+ case htons(ETH_P_MPLS_UC):
+ case htons(ETH_P_MPLS_MC):
+ return MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre);
+ }
+ return false;
+}
+
static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv,
struct sk_buff *skb,
netdev_features_t features)
switch (proto) {
case IPPROTO_GRE:
- return features;
+ if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv->mdev, skb))
+ return features;
+ break;
case IPPROTO_IPIP:
case IPPROTO_IPV6:
if (mlx5e_tunnel_proto_supported(priv->mdev, IPPROTO_IPIP))
/* encap list sharing the same neigh */
struct list_head encap_list;
- /* valid only when the neigh reference is taken during
- * neigh_update_work workqueue callback.
- */
- struct neighbour *n;
- struct work_struct neigh_update_work;
-
/* neigh hash entry can be deleted only when the refcount is zero.
* refcount is needed to avoid neigh hash entry removal by TC, while
* it's used by the neigh notification call.
return count_eqe;
}
+static void mlx5_eq_async_int_lock(struct mlx5_eq_async *eq, unsigned long *flags)
+ __acquires(&eq->lock)
+{
+ if (in_irq())
+ spin_lock(&eq->lock);
+ else
+ spin_lock_irqsave(&eq->lock, *flags);
+}
+
+static void mlx5_eq_async_int_unlock(struct mlx5_eq_async *eq, unsigned long *flags)
+ __releases(&eq->lock)
+{
+ if (in_irq())
+ spin_unlock(&eq->lock);
+ else
+ spin_unlock_irqrestore(&eq->lock, *flags);
+}
+
+enum async_eq_nb_action {
+ ASYNC_EQ_IRQ_HANDLER = 0,
+ ASYNC_EQ_RECOVER = 1,
+};
+
static int mlx5_eq_async_int(struct notifier_block *nb,
unsigned long action, void *data)
{
struct mlx5_eq_table *eqt;
struct mlx5_core_dev *dev;
struct mlx5_eqe *eqe;
+ unsigned long flags;
int num_eqes = 0;
dev = eq->dev;
eqt = dev->priv.eq_table;
+ mlx5_eq_async_int_lock(eq_async, &flags);
+
eqe = next_eqe_sw(eq);
if (!eqe)
goto out;
out:
eq_update_ci(eq, 1);
+ mlx5_eq_async_int_unlock(eq_async, &flags);
- return 0;
+ return unlikely(action == ASYNC_EQ_RECOVER) ? num_eqes : 0;
+}
+
+void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev)
+{
+ struct mlx5_eq_async *eq = &dev->priv.eq_table->cmd_eq;
+ int eqes;
+
+ eqes = mlx5_eq_async_int(&eq->irq_nb, ASYNC_EQ_RECOVER, NULL);
+ if (eqes)
+ mlx5_core_warn(dev, "Recovered %d EQEs on cmd_eq\n", eqes);
}
static void init_eq_buf(struct mlx5_eq *eq)
int err;
eq->irq_nb.notifier_call = mlx5_eq_async_int;
+ spin_lock_init(&eq->lock);
err = create_async_eq(dev, &eq->core, param);
if (err) {
cleanup_async_eq(dev, &table->pages_eq, "pages");
cleanup_async_eq(dev, &table->async_eq, "async");
+ mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_DESTROY_EQ);
mlx5_cmd_use_polling(dev);
cleanup_async_eq(dev, &table->cmd_eq, "cmd");
+ mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
mlx5_eq_notifier_unregister(dev, &table->cq_err_nb);
}
struct mlx5_eq_async {
struct mlx5_eq core;
struct notifier_block irq_nb;
+ spinlock_t lock; /* To avoid irq EQ handle races with resiliency flows */
};
struct mlx5_eq_comp {
struct cpumask *mlx5_eq_comp_cpumask(struct mlx5_core_dev *dev, int ix);
u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq);
+void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev);
void mlx5_eq_synchronize_async_irq(struct mlx5_core_dev *dev);
void mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev *dev);
u32 npages;
u32 i = 0;
- if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR)
+ if (!mlx5_cmd_is_down(dev))
return mlx5_cmd_exec(dev, in, in_size, out, out_size);
/* No hard feelings, we want our pages back! */
return 0;
err_request_irq:
- for (; i >= 0; i--) {
+ while (i--) {
struct mlx5_irq *irq = mlx5_irq_get(dev, i);
int irqn = pci_irq_vector(dev->pdev, i);
return dev->netdev_ops == &mlxsw_sp_port_netdev_ops;
}
-static int mlxsw_sp_lower_dev_walk(struct net_device *lower_dev, void *data)
+static int mlxsw_sp_lower_dev_walk(struct net_device *lower_dev,
+ struct netdev_nested_priv *priv)
{
- struct mlxsw_sp_port **p_mlxsw_sp_port = data;
int ret = 0;
if (mlxsw_sp_port_dev_check(lower_dev)) {
- *p_mlxsw_sp_port = netdev_priv(lower_dev);
+ priv->data = (void *)netdev_priv(lower_dev);
ret = 1;
}
struct mlxsw_sp_port *mlxsw_sp_port_dev_lower_find(struct net_device *dev)
{
- struct mlxsw_sp_port *mlxsw_sp_port;
+ struct netdev_nested_priv priv = {
+ .data = NULL,
+ };
if (mlxsw_sp_port_dev_check(dev))
return netdev_priv(dev);
- mlxsw_sp_port = NULL;
- netdev_walk_all_lower_dev(dev, mlxsw_sp_lower_dev_walk, &mlxsw_sp_port);
+ netdev_walk_all_lower_dev(dev, mlxsw_sp_lower_dev_walk, &priv);
- return mlxsw_sp_port;
+ return (struct mlxsw_sp_port *)priv.data;
}
struct mlxsw_sp *mlxsw_sp_lower_get(struct net_device *dev)
struct mlxsw_sp_port *mlxsw_sp_port_dev_lower_find_rcu(struct net_device *dev)
{
- struct mlxsw_sp_port *mlxsw_sp_port;
+ struct netdev_nested_priv priv = {
+ .data = NULL,
+ };
if (mlxsw_sp_port_dev_check(dev))
return netdev_priv(dev);
- mlxsw_sp_port = NULL;
netdev_walk_all_lower_dev_rcu(dev, mlxsw_sp_lower_dev_walk,
- &mlxsw_sp_port);
+ &priv);
- return mlxsw_sp_port;
+ return (struct mlxsw_sp_port *)priv.data;
}
struct mlxsw_sp_port *mlxsw_sp_port_lower_dev_hold(struct net_device *dev)
int err;
group->tcam = tcam;
- mutex_init(&group->lock);
INIT_LIST_HEAD(&group->region_list);
err = mlxsw_sp_acl_tcam_group_id_get(tcam, &group->id);
if (err)
return err;
+ mutex_init(&group->lock);
+
return 0;
}
return err;
}
-static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, void *data)
+static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev,
+ struct netdev_nested_priv *priv)
{
- struct mlxsw_sp_rif *rif = data;
+ struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data;
if (!netif_is_macvlan(dev))
return 0;
static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
{
+ struct netdev_nested_priv priv = {
+ .data = (void *)rif,
+ };
+
if (!netif_is_macvlan_port(rif->dev))
return 0;
netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n");
return netdev_walk_all_upper_dev_rcu(rif->dev,
- __mlxsw_sp_rif_macvlan_flush, rif);
+ __mlxsw_sp_rif_macvlan_flush, &priv);
}
static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
}
static int mlxsw_sp_bridge_device_upper_rif_destroy(struct net_device *dev,
- void *data)
+ struct netdev_nested_priv *priv)
{
- struct mlxsw_sp *mlxsw_sp = data;
+ struct mlxsw_sp *mlxsw_sp = priv->data;
mlxsw_sp_rif_destroy_by_dev(mlxsw_sp, dev);
return 0;
static void mlxsw_sp_bridge_device_rifs_destroy(struct mlxsw_sp *mlxsw_sp,
struct net_device *dev)
{
+ struct netdev_nested_priv priv = {
+ .data = (void *)mlxsw_sp,
+ };
+
mlxsw_sp_rif_destroy_by_dev(mlxsw_sp, dev);
netdev_walk_all_upper_dev_rcu(dev,
mlxsw_sp_bridge_device_upper_rif_destroy,
- mlxsw_sp);
+ &priv);
}
static int mlxsw_sp_bridge_device_vxlan_init(struct mlxsw_sp_bridge *bridge,
struct ocelot_port *ocelot_port = ocelot->ports[port];
int maxlen = sdu + ETH_HLEN + ETH_FCS_LEN;
int pause_start, pause_stop;
- int atop_wm;
+ int atop, atop_tot;
if (port == ocelot->npi) {
maxlen += OCELOT_TAG_LEN;
ocelot_fields_write(ocelot, port, SYS_PAUSE_CFG_PAUSE_STOP,
pause_stop);
- /* Tail dropping watermark */
- atop_wm = (ocelot->shared_queue_sz - 9 * maxlen) /
+ /* Tail dropping watermarks */
+ atop_tot = (ocelot->shared_queue_sz - 9 * maxlen) /
OCELOT_BUFFER_CELL_SZ;
- ocelot_write_rix(ocelot, ocelot->ops->wm_enc(9 * maxlen),
- SYS_ATOP, port);
- ocelot_write(ocelot, ocelot->ops->wm_enc(atop_wm), SYS_ATOP_TOT_CFG);
+ atop = (9 * maxlen) / OCELOT_BUFFER_CELL_SZ;
+ ocelot_write_rix(ocelot, ocelot->ops->wm_enc(atop), SYS_ATOP, port);
+ ocelot_write(ocelot, ocelot->ops->wm_enc(atop_tot), SYS_ATOP_TOT_CFG);
}
EXPORT_SYMBOL(ocelot_port_set_maxlen);
*/
static u16 ocelot_wm_enc(u16 value)
{
+ WARN_ON(value >= 16 * BIT(8));
+
if (value >= BIT(8))
return BIT(8) | (value / 16);
void r8169_apply_firmware(struct rtl8169_private *tp)
{
+ int val;
+
/* TODO: release firmware if rtl_fw_write_firmware signals failure. */
if (tp->rtl_fw) {
rtl_fw_write_firmware(tp, tp->rtl_fw);
/* At least one firmware doesn't reset tp->ocp_base. */
tp->ocp_base = OCP_STD_PHY_BASE;
+
+ /* PHY soft reset may still be in progress */
+ phy_read_poll_timeout(tp->phydev, MII_BMCR, val,
+ !(val & BMCR_RESET),
+ 50000, 600000, true);
}
}
default:
break;
}
-
- clk_disable_unprepare(tp->clk);
}
static void rtl_pll_power_up(struct rtl8169_private *tp)
{
- clk_prepare_enable(tp->clk);
-
switch (tp->mac_version) {
case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_33:
case RTL_GIGA_MAC_VER_37:
{ 0x08, 0x0001, 0x0002 },
{ 0x09, 0x0000, 0x0080 },
{ 0x19, 0x0000, 0x0224 },
- { 0x00, 0x0000, 0x0004 },
+ { 0x00, 0x0000, 0x0008 },
{ 0x0c, 0x3df0, 0x0200 },
};
{ 0x06, 0x00c0, 0x0020 },
{ 0x0f, 0xffff, 0x5200 },
{ 0x19, 0x0000, 0x0224 },
- { 0x00, 0x0000, 0x0004 },
+ { 0x00, 0x0000, 0x0008 },
{ 0x0c, 0x3df0, 0x0200 },
};
#ifdef CONFIG_PM
+static int rtl8169_net_resume(struct rtl8169_private *tp)
+{
+ rtl_rar_set(tp, tp->dev->dev_addr);
+
+ if (tp->TxDescArray)
+ rtl8169_up(tp);
+
+ netif_device_attach(tp->dev);
+
+ return 0;
+}
+
static int __maybe_unused rtl8169_suspend(struct device *device)
{
struct rtl8169_private *tp = dev_get_drvdata(device);
rtnl_lock();
rtl8169_net_suspend(tp);
+ if (!device_may_wakeup(tp_to_dev(tp)))
+ clk_disable_unprepare(tp->clk);
rtnl_unlock();
return 0;
}
-static int rtl8169_resume(struct device *device)
+static int __maybe_unused rtl8169_resume(struct device *device)
{
struct rtl8169_private *tp = dev_get_drvdata(device);
- rtl_rar_set(tp, tp->dev->dev_addr);
+ if (!device_may_wakeup(tp_to_dev(tp)))
+ clk_prepare_enable(tp->clk);
- if (tp->TxDescArray)
- rtl8169_up(tp);
+ /* Reportedly at least Asus X453MA truncates packets otherwise */
+ if (tp->mac_version == RTL_GIGA_MAC_VER_37)
+ rtl_init_rxcfg(tp);
- netif_device_attach(tp->dev);
-
- return 0;
+ return rtl8169_net_resume(tp);
}
static int rtl8169_runtime_suspend(struct device *device)
__rtl8169_set_wol(tp, tp->saved_wolopts);
- return rtl8169_resume(device);
+ return rtl8169_net_resume(tp);
}
static int rtl8169_runtime_idle(struct device *device)
return error;
}
-/* MDIO bus init function */
-static int ravb_mdio_init(struct ravb_private *priv)
-{
- struct platform_device *pdev = priv->pdev;
- struct device *dev = &pdev->dev;
- int error;
-
- /* Bitbang init */
- priv->mdiobb.ops = &bb_ops;
-
- /* MII controller setting */
- priv->mii_bus = alloc_mdio_bitbang(&priv->mdiobb);
- if (!priv->mii_bus)
- return -ENOMEM;
-
- /* Hook up MII support for ethtool */
- priv->mii_bus->name = "ravb_mii";
- priv->mii_bus->parent = dev;
- snprintf(priv->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",
- pdev->name, pdev->id);
-
- /* Register MDIO bus */
- error = of_mdiobus_register(priv->mii_bus, dev->of_node);
- if (error)
- goto out_free_bus;
-
- return 0;
-
-out_free_bus:
- free_mdio_bitbang(priv->mii_bus);
- return error;
-}
-
-/* MDIO bus release function */
-static int ravb_mdio_release(struct ravb_private *priv)
-{
- /* Unregister mdio bus */
- mdiobus_unregister(priv->mii_bus);
-
- /* Free bitbang info */
- free_mdio_bitbang(priv->mii_bus);
-
- return 0;
-}
-
/* Network device open function for Ethernet AVB */
static int ravb_open(struct net_device *ndev)
{
struct device *dev = &pdev->dev;
int error;
- /* MDIO bus init */
- error = ravb_mdio_init(priv);
- if (error) {
- netdev_err(ndev, "failed to initialize MDIO\n");
- return error;
- }
-
napi_enable(&priv->napi[RAVB_BE]);
napi_enable(&priv->napi[RAVB_NC]);
out_napi_off:
napi_disable(&priv->napi[RAVB_NC]);
napi_disable(&priv->napi[RAVB_BE]);
- ravb_mdio_release(priv);
return error;
}
ravb_ring_free(ndev, RAVB_BE);
ravb_ring_free(ndev, RAVB_NC);
- ravb_mdio_release(priv);
-
return 0;
}
.ndo_set_features = ravb_set_features,
};
+/* MDIO bus init function */
+static int ravb_mdio_init(struct ravb_private *priv)
+{
+ struct platform_device *pdev = priv->pdev;
+ struct device *dev = &pdev->dev;
+ int error;
+
+ /* Bitbang init */
+ priv->mdiobb.ops = &bb_ops;
+
+ /* MII controller setting */
+ priv->mii_bus = alloc_mdio_bitbang(&priv->mdiobb);
+ if (!priv->mii_bus)
+ return -ENOMEM;
+
+ /* Hook up MII support for ethtool */
+ priv->mii_bus->name = "ravb_mii";
+ priv->mii_bus->parent = dev;
+ snprintf(priv->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",
+ pdev->name, pdev->id);
+
+ /* Register MDIO bus */
+ error = of_mdiobus_register(priv->mii_bus, dev->of_node);
+ if (error)
+ goto out_free_bus;
+
+ return 0;
+
+out_free_bus:
+ free_mdio_bitbang(priv->mii_bus);
+ return error;
+}
+
+/* MDIO bus release function */
+static int ravb_mdio_release(struct ravb_private *priv)
+{
+ /* Unregister mdio bus */
+ mdiobus_unregister(priv->mii_bus);
+
+ /* Free bitbang info */
+ free_mdio_bitbang(priv->mii_bus);
+
+ return 0;
+}
+
static const struct of_device_id ravb_match_table[] = {
{ .compatible = "renesas,etheravb-r8a7790", .data = (void *)RCAR_GEN2 },
{ .compatible = "renesas,etheravb-r8a7794", .data = (void *)RCAR_GEN2 },
eth_hw_addr_random(ndev);
}
+ /* MDIO bus init */
+ error = ravb_mdio_init(priv);
+ if (error) {
+ dev_err(&pdev->dev, "failed to initialize MDIO\n");
+ goto out_dma_free;
+ }
+
netif_napi_add(ndev, &priv->napi[RAVB_BE], ravb_poll, 64);
netif_napi_add(ndev, &priv->napi[RAVB_NC], ravb_poll, 64);
out_napi_del:
netif_napi_del(&priv->napi[RAVB_NC]);
netif_napi_del(&priv->napi[RAVB_BE]);
+ ravb_mdio_release(priv);
+out_dma_free:
dma_free_coherent(ndev->dev.parent, priv->desc_bat_size, priv->desc_bat,
priv->desc_bat_dma);
unregister_netdev(ndev);
netif_napi_del(&priv->napi[RAVB_NC]);
netif_napi_del(&priv->napi[RAVB_BE]);
+ ravb_mdio_release(priv);
pm_runtime_disable(&pdev->dev);
free_netdev(ndev);
platform_set_drvdata(pdev, NULL);
struct rocker_port *port;
};
-static int rocker_lower_dev_walk(struct net_device *lower_dev, void *_data)
+static int rocker_lower_dev_walk(struct net_device *lower_dev,
+ struct netdev_nested_priv *priv)
{
- struct rocker_walk_data *data = _data;
+ struct rocker_walk_data *data = (struct rocker_walk_data *)priv->data;
int ret = 0;
if (rocker_port_dev_check_under(lower_dev, data->rocker)) {
struct rocker_port *rocker_port_dev_lower_find(struct net_device *dev,
struct rocker *rocker)
{
+ struct netdev_nested_priv priv;
struct rocker_walk_data data;
if (rocker_port_dev_check_under(dev, rocker))
data.rocker = rocker;
data.port = NULL;
- netdev_walk_all_lower_dev(dev, rocker_lower_dev_walk, &data);
+ priv.data = (void *)&data;
+ netdev_walk_all_lower_dev(dev, rocker_lower_dev_walk, &priv);
return data.port;
}
pci_free_irq_vectors(pdev);
- clk_disable_unprepare(priv->plat->stmmac_clk);
clk_unregister_fixed_rate(priv->plat->stmmac_clk);
pcim_iounmap_regions(pdev, BIT(0));
int eee_enabled;
int eee_active;
int tx_lpi_timer;
+ int tx_lpi_enabled;
+ int eee_tw_timer;
unsigned int mode;
unsigned int chain_mode;
int extend_desc;
edata->eee_enabled = priv->eee_enabled;
edata->eee_active = priv->eee_active;
edata->tx_lpi_timer = priv->tx_lpi_timer;
+ edata->tx_lpi_enabled = priv->tx_lpi_enabled;
return phylink_ethtool_get_eee(priv->phylink, edata);
}
struct stmmac_priv *priv = netdev_priv(dev);
int ret;
- if (!edata->eee_enabled) {
+ if (!priv->dma_cap.eee)
+ return -EOPNOTSUPP;
+
+ if (priv->tx_lpi_enabled != edata->tx_lpi_enabled)
+ netdev_warn(priv->dev,
+ "Setting EEE tx-lpi is not supported\n");
+
+ if (!edata->eee_enabled)
stmmac_disable_eee_mode(priv);
- } else {
- /* We are asking for enabling the EEE but it is safe
- * to verify all by invoking the eee_init function.
- * In case of failure it will return an error.
- */
- edata->eee_enabled = stmmac_eee_init(priv);
- if (!edata->eee_enabled)
- return -EOPNOTSUPP;
- }
ret = phylink_ethtool_set_eee(priv->phylink, edata);
if (ret)
return ret;
- priv->eee_enabled = edata->eee_enabled;
- priv->tx_lpi_timer = edata->tx_lpi_timer;
+ if (edata->eee_enabled &&
+ priv->tx_lpi_timer != edata->tx_lpi_timer) {
+ priv->tx_lpi_timer = edata->tx_lpi_timer;
+ stmmac_eee_init(priv);
+ }
+
return 0;
}
static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
module_param(eee_timer, int, 0644);
MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
-#define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
+#define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x))
/* By default the driver will use the ring mode to manage tx and rx descriptors,
* but allow user to force to use the chain instead of the ring
struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
stmmac_enable_eee_mode(priv);
- mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
+ mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
}
/**
*/
bool stmmac_eee_init(struct stmmac_priv *priv)
{
- int tx_lpi_timer = priv->tx_lpi_timer;
+ int eee_tw_timer = priv->eee_tw_timer;
/* Using PCS we cannot dial with the phy registers at this stage
* so we do not support extra feature like EEE.
if (priv->eee_enabled) {
netdev_dbg(priv->dev, "disable EEE\n");
del_timer_sync(&priv->eee_ctrl_timer);
- stmmac_set_eee_timer(priv, priv->hw, 0, tx_lpi_timer);
+ stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer);
}
mutex_unlock(&priv->lock);
return false;
if (priv->eee_active && !priv->eee_enabled) {
timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);
- mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,
- tx_lpi_timer);
+ eee_tw_timer);
}
+ mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
+
mutex_unlock(&priv->lock);
netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
return true;
stmmac_mac_set(priv, priv->ioaddr, false);
priv->eee_active = false;
+ priv->tx_lpi_enabled = false;
stmmac_eee_init(priv);
stmmac_set_eee_pls(priv, priv->hw, false);
}
if (phy && priv->dma_cap.eee) {
priv->eee_active = phy_init_eee(phy, 1) >= 0;
priv->eee_enabled = stmmac_eee_init(priv);
+ priv->tx_lpi_enabled = priv->eee_enabled;
stmmac_set_eee_pls(priv, priv->hw, true);
}
}
if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
stmmac_enable_eee_mode(priv);
- mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
+ mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
}
/* We still have pending packets, let's call for a new scheduling */
netdev_warn(priv->dev, "PTP init failed\n");
}
- priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
+ priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS;
+
+ /* Convert the timer from msec to usec */
+ if (!priv->tx_lpi_timer)
+ priv->tx_lpi_timer = eee_timer * 1000;
if (priv->use_riwt) {
if (!priv->rx_riwt)
*(__sum16 *)(skb->data + offset) = 0;
csum = skb_copy_and_csum_bits(skb, start,
nskb->data + start,
- skb->len - start, 0);
+ skb->len - start);
/* add in the header checksums */
if (skb->protocol == htons(ETH_P_IP)) {
/*
Written 1998-2001 by Donald Becker.
- Current Maintainer: Roger Luethi <rl@hellgate.ch>
+ Current Maintainer: Kevin Brace <kevinbrace@bracecomputerlab.com>
This software may be used and distributed according to the terms of
the GNU General Public License (GPL), incorporated herein by reference.
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#define DRV_NAME "via-rhine"
-#define DRV_VERSION "1.5.1"
-#define DRV_RELDATE "2010-10-09"
#include <linux/types.h>
#include <linux/uaccess.h>
#include <linux/dmi.h>
-/* These identify the driver base version and may not be removed. */
-static const char version[] =
- "v1.10-LK" DRV_VERSION " " DRV_RELDATE " Written by Donald Becker";
-
MODULE_AUTHOR("Donald Becker <becker@scyld.com>");
MODULE_DESCRIPTION("VIA Rhine PCI Fast Ethernet driver");
MODULE_LICENSE("GPL");
VT8233 = 0x60, /* Integrated MAC */
VT8235 = 0x74, /* Integrated MAC */
VT8237 = 0x78, /* Integrated MAC */
- VTunknown1 = 0x7C,
+ VT8251 = 0x7C, /* Integrated MAC */
VT6105 = 0x80,
VT6105_B0 = 0x83,
VT6105L = 0x8A,
u32 quirks = 0;
#endif
-/* when built into the kernel, we only print version if device is found */
-#ifndef MODULE
- pr_info_once("%s\n", version);
-#endif
-
rc = pci_enable_device(pdev);
if (rc)
goto err_out;
goto out_free_ring;
alloc_tbufs(dev);
+ enable_mmio(rp->pioaddr, rp->quirks);
+ rhine_power_init(dev);
rhine_chip_reset(dev);
rhine_task_enable(rp);
init_registers(dev);
struct device *hwdev = dev->dev.parent;
strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
- strlcpy(info->version, DRV_VERSION, sizeof(info->version));
strlcpy(info->bus_info, dev_name(hwdev), sizeof(info->bus_info));
}
int ret_pci, ret_platform;
/* when a module, this is printed whether or not devices are found in probe */
-#ifdef MODULE
- pr_info("%s\n", version);
-#endif
if (dmi_check_system(rhine_dmi_table)) {
/* these BIOSes fail at PXE boot if chip is in D3 */
avoid_D3 = true;
struct macsec_rx_sa *rx_sa;
struct macsec_rxh_data *rxd;
struct macsec_dev *macsec;
+ unsigned int len;
sci_t sci;
u32 hdr_pn;
bool cbit;
macsec_rxsc_put(rx_sc);
skb_orphan(skb);
+ len = skb->len;
ret = gro_cells_receive(&macsec->gro_cells, skb);
if (ret == NET_RX_SUCCESS)
- count_rx(dev, skb->len);
+ count_rx(dev, len);
else
macsec->secy.netdev->stats.rx_dropped++;
depends on 64BIT
depends on PCI
select MDIO_CAVIUM
+ select MDIO_DEVRES
help
This driver supports the MDIO interfaces found on Cavium
ThunderX SoCs when the MDIO bus device appears as a PCI
// SPDX-License-Identifier: GPL-2.0+
-/*
- * drivers/net/phy/realtek.c
+/* drivers/net/phy/realtek.c
*
* Driver for Realtek PHYs
*
#define RTL8211F_TX_DELAY BIT(8)
#define RTL8211F_RX_DELAY BIT(3)
-#define RTL8211E_TX_DELAY BIT(1)
-#define RTL8211E_RX_DELAY BIT(2)
-#define RTL8211E_MODE_MII_GMII BIT(3)
+#define RTL8211E_CTRL_DELAY BIT(13)
+#define RTL8211E_TX_DELAY BIT(12)
+#define RTL8211E_RX_DELAY BIT(11)
#define RTL8201F_ISR 0x1e
#define RTL8201F_IER 0x13
/* enable TX/RX delay for rgmii-* modes, and disable them for rgmii. */
switch (phydev->interface) {
case PHY_INTERFACE_MODE_RGMII:
- val = 0;
+ val = RTL8211E_CTRL_DELAY | 0;
break;
case PHY_INTERFACE_MODE_RGMII_ID:
- val = RTL8211E_TX_DELAY | RTL8211E_RX_DELAY;
+ val = RTL8211E_CTRL_DELAY | RTL8211E_TX_DELAY | RTL8211E_RX_DELAY;
break;
case PHY_INTERFACE_MODE_RGMII_RXID:
- val = RTL8211E_RX_DELAY;
+ val = RTL8211E_CTRL_DELAY | RTL8211E_RX_DELAY;
break;
case PHY_INTERFACE_MODE_RGMII_TXID:
- val = RTL8211E_TX_DELAY;
+ val = RTL8211E_CTRL_DELAY | RTL8211E_TX_DELAY;
break;
default: /* the rest of the modes imply leaving delays as is. */
return 0;
/* According to a sample driver there is a 0x1c config register on the
* 0xa4 extension page (0x7) layout. It can be used to disable/enable
- * the RX/TX delays otherwise controlled by RXDLY/TXDLY pins. It can
- * also be used to customize the whole configuration register:
- * 8:6 = PHY Address, 5:4 = Auto-Negotiation, 3 = Interface Mode Select,
- * 2 = RX Delay, 1 = TX Delay, 0 = SELRGV (see original PHY datasheet
- * for details).
+ * the RX/TX delays otherwise controlled by RXDLY/TXDLY pins.
+ * The configuration register definition:
+ * 14 = reserved
+ * 13 = Force Tx RX Delay controlled by bit12 bit11,
+ * 12 = RX Delay, 11 = TX Delay
+ * 10:0 = Test && debug settings reserved by realtek
*/
oldpage = phy_select_page(phydev, 0x7);
if (oldpage < 0)
if (ret)
goto err_restore_page;
- ret = __phy_modify(phydev, 0x1c, RTL8211E_TX_DELAY | RTL8211E_RX_DELAY,
+ ret = __phy_modify(phydev, 0x1c, RTL8211E_CTRL_DELAY
+ | RTL8211E_TX_DELAY | RTL8211E_RX_DELAY,
val);
err_restore_page:
for (i--; i >= 0; i--)
__team_option_inst_del_option(team, dst_opts[i]);
- i = option_count - 1;
+ i = option_count;
alloc_rollback:
for (i--; i >= 0; i--)
kfree(dst_opts[i]);
dev->header_ops = port_dev->header_ops;
dev->type = port_dev->type;
dev->hard_header_len = port_dev->hard_header_len;
+ dev->needed_headroom = port_dev->needed_headroom;
dev->addr_len = port_dev->addr_len;
dev->mtu = port_dev->mtu;
memcpy(dev->broadcast, port_dev->broadcast, port_dev->addr_len);
.status = ax88179_status,
.link_reset = ax88179_link_reset,
.reset = ax88179_reset,
+ .stop = ax88179_stop,
+ .flags = FLAG_ETHER | FLAG_FRAMING_AX,
+ .rx_fixup = ax88179_rx_fixup,
+ .tx_fixup = ax88179_tx_fixup,
+};
+
+static const struct driver_info toshiba_info = {
+ .description = "Toshiba USB Ethernet Adapter",
+ .bind = ax88179_bind,
+ .unbind = ax88179_unbind,
+ .status = ax88179_status,
+ .link_reset = ax88179_link_reset,
+ .reset = ax88179_reset,
+ .stop = ax88179_stop,
+ .flags = FLAG_ETHER | FLAG_FRAMING_AX,
+ .rx_fixup = ax88179_rx_fixup,
+ .tx_fixup = ax88179_tx_fixup,
+};
+
+static const struct driver_info mct_info = {
+ .description = "MCT USB 3.0 Gigabit Ethernet Adapter",
+ .bind = ax88179_bind,
+ .unbind = ax88179_unbind,
+ .status = ax88179_status,
+ .link_reset = ax88179_link_reset,
+ .reset = ax88179_reset,
+ .stop = ax88179_stop,
.flags = FLAG_ETHER | FLAG_FRAMING_AX,
.rx_fixup = ax88179_rx_fixup,
.tx_fixup = ax88179_tx_fixup,
/* Belkin B2B128 USB 3.0 Hub + Gigabit Ethernet Adapter */
USB_DEVICE(0x050d, 0x0128),
.driver_info = (unsigned long)&belkin_info,
+}, {
+ /* Toshiba USB 3.0 GBit Ethernet Adapter */
+ USB_DEVICE(0x0930, 0x0a13),
+ .driver_info = (unsigned long)&toshiba_info,
+}, {
+ /* Magic Control Technology U3-A9003 USB 3.0 Gigabit Ethernet Adapter */
+ USB_DEVICE(0x0711, 0x0179),
+ .driver_info = (unsigned long)&mct_info,
},
{ },
};
}
#endif /* PEGASUS_WRITE_EEPROM */
-static inline void get_node_id(pegasus_t *pegasus, __u8 *id)
+static inline int get_node_id(pegasus_t *pegasus, u8 *id)
{
- int i;
- __u16 w16;
+ int i, ret;
+ u16 w16;
for (i = 0; i < 3; i++) {
- read_eprom_word(pegasus, i, &w16);
+ ret = read_eprom_word(pegasus, i, &w16);
+ if (ret < 0)
+ return ret;
((__le16 *) id)[i] = cpu_to_le16(w16);
}
+
+ return 0;
}
static void set_ethernet_addr(pegasus_t *pegasus)
{
- __u8 node_id[6];
+ int ret;
+ u8 node_id[6];
if (pegasus->features & PEGASUS_II) {
- get_registers(pegasus, 0x10, sizeof(node_id), node_id);
+ ret = get_registers(pegasus, 0x10, sizeof(node_id), node_id);
+ if (ret < 0)
+ goto err;
} else {
- get_node_id(pegasus, node_id);
- set_registers(pegasus, EthID, sizeof(node_id), node_id);
+ ret = get_node_id(pegasus, node_id);
+ if (ret < 0)
+ goto err;
+ ret = set_registers(pegasus, EthID, sizeof(node_id), node_id);
+ if (ret < 0)
+ goto err;
}
+
memcpy(pegasus->net->dev_addr, node_id, sizeof(node_id));
+
+ return;
+err:
+ eth_hw_addr_random(pegasus->net);
+ dev_info(&pegasus->intf->dev, "software assigned MAC address.\n");
+
+ return;
}
static inline int reset_mac(pegasus_t *pegasus)
{QMI_QUIRK_SET_DTR(0x2cb7, 0x0104, 4)}, /* Fibocom NL678 series */
{QMI_FIXED_INTF(0x0489, 0xe0b4, 0)}, /* Foxconn T77W968 LTE */
{QMI_FIXED_INTF(0x0489, 0xe0b5, 0)}, /* Foxconn T77W968 LTE with eSIM support*/
+ {QMI_FIXED_INTF(0x2692, 0x9025, 4)}, /* Cellient MPL200 (rebranded Qualcomm 05c6:9025) */
/* 4. Gobi 1000 devices */
{QMI_GOBI1K_DEVICE(0x05c6, 0x9212)}, /* Acer Gobi Modem Device */
return 1;
}
-static inline void set_ethernet_addr(rtl8150_t * dev)
+static void set_ethernet_addr(rtl8150_t *dev)
{
- u8 node_id[6];
+ u8 node_id[ETH_ALEN];
+ int ret;
+
+ ret = get_registers(dev, IDR, sizeof(node_id), node_id);
- get_registers(dev, IDR, sizeof(node_id), node_id);
- memcpy(dev->netdev->dev_addr, node_id, sizeof(node_id));
+ if (ret == sizeof(node_id)) {
+ ether_addr_copy(dev->netdev->dev_addr, node_id);
+ } else {
+ eth_hw_addr_random(dev->netdev);
+ netdev_notice(dev->netdev, "Assigned a random MAC address: %pM\n",
+ dev->netdev->dev_addr);
+ }
}
static int rtl8150_set_mac_address(struct net_device *netdev, void *p)
VIRTIO_NET_F_GUEST_CSUM
};
+#define GUEST_OFFLOAD_LRO_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
+ (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
+ (1ULL << VIRTIO_NET_F_GUEST_ECN) | \
+ (1ULL << VIRTIO_NET_F_GUEST_UFO))
+
struct virtnet_stat_desc {
char desc[ETH_GSTRING_LEN];
size_t offset;
if (features & NETIF_F_LRO)
offloads = vi->guest_offloads_capable;
else
- offloads = 0;
+ offloads = vi->guest_offloads_capable &
+ ~GUEST_OFFLOAD_LRO_MASK;
err = virtnet_set_guest_offloads(vi, offloads);
if (err)
/* Use temporary descriptor to avoid touching bits multiple times */
union Vmxnet3_GenericDesc tempTxDesc;
#endif
- struct udphdr *udph;
count = txd_estimate(skb);
gdesc->txd.om = VMXNET3_OM_ENCAP;
gdesc->txd.msscof = ctx.mss;
- udph = udp_hdr(skb);
- if (udph->check)
+ if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)
gdesc->txd.oco = 1;
} else {
gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size;
.ndo_change_mtu = vmxnet3_change_mtu,
.ndo_fix_features = vmxnet3_fix_features,
.ndo_set_features = vmxnet3_set_features,
+ .ndo_features_check = vmxnet3_features_check,
.ndo_get_stats64 = vmxnet3_get_stats64,
.ndo_tx_timeout = vmxnet3_tx_timeout,
.ndo_set_rx_mode = vmxnet3_set_mc,
return features;
}
+netdev_features_t vmxnet3_features_check(struct sk_buff *skb,
+ struct net_device *netdev,
+ netdev_features_t features)
+{
+ struct vmxnet3_adapter *adapter = netdev_priv(netdev);
+
+ /* Validate if the tunneled packet is being offloaded by the device */
+ if (VMXNET3_VERSION_GE_4(adapter) &&
+ skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL) {
+ u8 l4_proto = 0;
+
+ switch (vlan_get_protocol(skb)) {
+ case htons(ETH_P_IP):
+ l4_proto = ip_hdr(skb)->protocol;
+ break;
+ case htons(ETH_P_IPV6):
+ l4_proto = ipv6_hdr(skb)->nexthdr;
+ break;
+ default:
+ return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
+ }
+
+ if (l4_proto != IPPROTO_UDP)
+ return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
+ }
+ return features;
+}
+
static void vmxnet3_enable_encap_offloads(struct net_device *netdev)
{
struct vmxnet3_adapter *adapter = netdev_priv(netdev);
netdev_features_t
vmxnet3_fix_features(struct net_device *netdev, netdev_features_t features);
+netdev_features_t
+vmxnet3_features_check(struct sk_buff *skb,
+ struct net_device *netdev, netdev_features_t features);
+
int
vmxnet3_set_features(struct net_device *netdev, netdev_features_t features);
{
struct x25_asy *sl = netdev_priv(dev);
unsigned long len;
- int err;
if (sl->tty == NULL)
return -ENODEV;
sl->xleft = 0;
sl->flags &= (1 << SLF_INUSE); /* Clear ESCAPE & ERROR flags */
- netif_start_queue(dev);
-
- /*
- * Now attach LAPB
- */
- err = lapb_register(dev, &x25_asy_callbacks);
- if (err == LAPB_OK)
- return 0;
+ return 0;
/* Cleanup */
kfree(sl->xbuff);
if (sl->tty)
clear_bit(TTY_DO_WRITE_WAKEUP, &sl->tty->flags);
- netif_stop_queue(dev);
sl->rcount = 0;
sl->xleft = 0;
spin_unlock(&sl->lock);
static void x25_asy_close_tty(struct tty_struct *tty)
{
struct x25_asy *sl = tty->disc_data;
- int err;
/* First make sure we're connected. */
if (!sl || sl->magic != X25_ASY_MAGIC)
dev_close(sl->dev);
rtnl_unlock();
- err = lapb_unregister(sl->dev);
- if (err != LAPB_OK)
- pr_err("%s: lapb_unregister error: %d\n",
- __func__, err);
-
tty->disc_data = NULL;
sl->tty = NULL;
x25_asy_free(sl);
static int x25_asy_open_dev(struct net_device *dev)
{
+ int err;
struct x25_asy *sl = netdev_priv(dev);
if (sl->tty == NULL)
return -ENODEV;
+
+ err = lapb_register(dev, &x25_asy_callbacks);
+ if (err != LAPB_OK)
+ return -ENOMEM;
+
+ netif_start_queue(dev);
+
+ return 0;
+}
+
+static int x25_asy_close_dev(struct net_device *dev)
+{
+ int err;
+
+ netif_stop_queue(dev);
+
+ err = lapb_unregister(dev);
+ if (err != LAPB_OK)
+ pr_err("%s: lapb_unregister error: %d\n",
+ __func__, err);
+
+ x25_asy_close(dev);
+
return 0;
}
static const struct net_device_ops x25_asy_netdev_ops = {
.ndo_open = x25_asy_open_dev,
- .ndo_stop = x25_asy_close,
+ .ndo_stop = x25_asy_close_dev,
.ndo_start_xmit = x25_asy_xmit,
.ndo_tx_timeout = x25_asy_timeout,
.ndo_change_mtu = x25_asy_change_mtu,
select WIRELESS_EXT
select WEXT_SPY
select CRYPTO
- select CRYPTO_ARC4
- select CRYPTO_ECB
- select CRYPTO_AES
select CRYPTO_MICHAEL_MIC
- select CRYPTO_ECB
select CRC32
select LIB80211
select LIB80211_CRYPT_WEP
select WEXT_SPY
select WEXT_PRIV
select CRYPTO
- select CRYPTO_ARC4
- select CRYPTO_ECB
- select CRYPTO_AES
select CRYPTO_MICHAEL_MIC
- select CRYPTO_ECB
select CRC32
select LIB80211
select LIB80211_CRYPT_WEP
dev->mphy.sband_2g.sband.ht_cap.cap |= IEEE80211_HT_CAP_LDPC_CODING;
dev->mphy.sband_5g.sband.ht_cap.cap |= IEEE80211_HT_CAP_LDPC_CODING;
dev->mphy.sband_5g.sband.vht_cap.cap |=
- IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454 |
+ IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_7991 |
IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK;
mt7615_cap_dbdc_disable(dev);
dev->phy.dfs_state = -1;
return ndev->netdev_ops == &qtnf_netdev_ops;
}
-static int qtnf_check_br_ports(struct net_device *dev, void *data)
+static int qtnf_check_br_ports(struct net_device *dev,
+ struct netdev_nested_priv *priv)
{
- struct net_device *ndev = data;
+ struct net_device *ndev = (struct net_device *)priv->data;
if (dev != ndev && netdev_port_same_parent_id(dev, ndev))
return -ENOTSUPP;
{
struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
const struct netdev_notifier_changeupper_info *info;
+ struct netdev_nested_priv priv = {
+ .data = (void *)ndev,
+ };
struct net_device *brdev;
struct qtnf_vif *vif;
struct qtnf_bus *bus;
} else {
ret = netdev_walk_all_lower_dev(brdev,
qtnf_check_br_ports,
- ndev);
+ &priv);
}
break;
if (rw == READ) {
if (unlikely(is_bad_pmem(&nsio->bb, sector, sz_align)))
return -EIO;
- if (memcpy_mcsafe(buf, nsio->addr + offset, size) != 0)
+ if (copy_mc_to_kernel(buf, nsio->addr + offset, size) != 0)
return -EIO;
return 0;
}
while (len) {
mem = kmap_atomic(page);
chunk = min_t(unsigned int, len, PAGE_SIZE - off);
- rem = memcpy_mcsafe(mem + off, pmem_addr, chunk);
+ rem = copy_mc_to_kernel(mem + off, pmem_addr, chunk);
kunmap_atomic(mem);
if (rem)
return BLK_STS_IOERR;
/*
* Use the 'no check' versions of copy_from_iter_flushcache() and
- * copy_to_iter_mcsafe() to bypass HARDENED_USERCOPY overhead. Bounds
+ * copy_mc_to_iter() to bypass HARDENED_USERCOPY overhead. Bounds
* checking, both file offset and device offset, is handled by
* dax_iomap_actor()
*/
static size_t pmem_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,
void *addr, size_t bytes, struct iov_iter *i)
{
- return _copy_to_iter_mcsafe(addr, bytes, i);
+ return _copy_mc_to_iter(addr, bytes, i);
}
static const struct dax_operations pmem_dax_ops = {
if (!cel)
return -ENOMEM;
- ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CMD_EFFECTS, 0, csi,
+ ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
&cel->log, sizeof(cel->log), 0);
if (ret) {
kfree(cel);
if (ret < 0)
return ret;
- if (!ctrl->identified)
- nvme_hwmon_init(ctrl);
+ if (!ctrl->identified) {
+ ret = nvme_hwmon_init(ctrl);
+ if (ret < 0)
+ return ret;
+ }
ctrl->identified = true;
}
nvme_get_ctrl(ctrl);
- if (!try_module_get(ctrl->ops->module))
+ if (!try_module_get(ctrl->ops->module)) {
+ nvme_put_ctrl(ctrl);
return -EINVAL;
+ }
file->private_data = ctrl;
return 0;
spin_lock_irqsave(&nvme_fc_lock, flags);
list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
if (lport->localport.node_name != laddr.nn ||
- lport->localport.port_name != laddr.pn)
+ lport->localport.port_name != laddr.pn ||
+ lport->localport.port_state != FC_OBJSTATE_ONLINE)
continue;
list_for_each_entry(rport, &lport->endp_list, endp_list) {
if (rport->remoteport.node_name != raddr.nn ||
- rport->remoteport.port_name != raddr.pn)
+ rport->remoteport.port_name != raddr.pn ||
+ rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
continue;
/* if fail to get reference fall through. Will error */
static int nvme_hwmon_get_smart_log(struct nvme_hwmon_data *data)
{
- int ret;
-
- ret = nvme_get_log(data->ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
+ return nvme_get_log(data->ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0,
NVME_CSI_NVM, &data->log, sizeof(data->log), 0);
-
- return ret <= 0 ? ret : -EIO;
}
static int nvme_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
.info = nvme_hwmon_info,
};
-void nvme_hwmon_init(struct nvme_ctrl *ctrl)
+int nvme_hwmon_init(struct nvme_ctrl *ctrl)
{
struct device *dev = ctrl->dev;
struct nvme_hwmon_data *data;
data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
if (!data)
- return;
+ return 0;
data->ctrl = ctrl;
mutex_init(&data->read_lock);
dev_warn(ctrl->device,
"Failed to read smart log (error %d)\n", err);
devm_kfree(dev, data);
- return;
+ return err;
}
hwmon = devm_hwmon_device_register_with_info(dev, "nvme", data,
dev_warn(dev, "Failed to instantiate hwmon device\n");
devm_kfree(dev, data);
}
+
+ return 0;
}
}
#ifdef CONFIG_NVME_HWMON
-void nvme_hwmon_init(struct nvme_ctrl *ctrl);
+int nvme_hwmon_init(struct nvme_ctrl *ctrl);
#else
-static inline void nvme_hwmon_init(struct nvme_ctrl *ctrl) { }
+static inline int nvme_hwmon_init(struct nvme_ctrl *ctrl)
+{
+ return 0;
+}
#endif
u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
struct nvme_completion *cqe = &nvmeq->cqes[idx];
struct request *req;
- if (unlikely(cqe->command_id >= nvmeq->q_depth)) {
- dev_warn(nvmeq->dev->ctrl.device,
- "invalid id %d completed on queue %d\n",
- cqe->command_id, le16_to_cpu(cqe->sq_id));
- return;
- }
-
/*
* AEN requests are special as they don't time out and can
* survive any kind of queue freeze and often don't respond to
}
req = blk_mq_tag_to_rq(nvme_queue_tagset(nvmeq), cqe->command_id);
+ if (unlikely(!req)) {
+ dev_warn(nvmeq->dev->ctrl.device,
+ "invalid id %d completed on queue %d\n",
+ cqe->command_id, le16_to_cpu(cqe->sq_id));
+ return;
+ }
+
trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail);
if (!nvme_try_complete_req(req, cqe->status, cqe->result))
nvme_pci_complete_rq(req);
else
flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
- /* can't zcopy slab pages */
- if (unlikely(PageSlab(page))) {
- ret = sock_no_sendpage(queue->sock, page, offset, len,
+ if (sendpage_ok(page)) {
+ ret = kernel_sendpage(queue->sock, page, offset, len,
flags);
} else {
- ret = kernel_sendpage(queue->sock, page, offset, len,
+ ret = sock_no_sendpage(queue->sock, page, offset, len,
flags);
}
if (ret <= 0)
{
#ifdef CONFIG_MODULES
if (val != MODULE_STATE_COMING)
- return 0;
+ return NOTIFY_DONE;
/* FIXME: should we process all CPU buffers ? */
mutex_lock(&buffer_mutex);
add_event_entry(MODULE_LOADED_CODE);
mutex_unlock(&buffer_mutex);
#endif
- return 0;
+ return NOTIFY_OK;
}
depends on PCI_MSI
select GENERIC_MSI_IRQ_DOMAIN
+config PCI_MSI_ARCH_FALLBACKS
+ bool
+
config PCI_QUIRKS
default y
bool "Enable PCI quirk workarounds" if EXPERT
bool "NVIDIA Tegra PCIe controller"
depends on ARCH_TEGRA || COMPILE_TEST
depends on PCI_MSI_IRQ_DOMAIN
+ select PCI_MSI_ARCH_FALLBACKS
help
Say Y here if you want support for the PCIe host controller found
on NVIDIA Tegra SoCs.
bool "Renesas R-Car PCIe host controller"
depends on ARCH_RENESAS || COMPILE_TEST
depends on PCI_MSI_IRQ_DOMAIN
+ select PCI_MSI_ARCH_FALLBACKS
help
Say Y here if you want PCIe controller support on R-Car SoCs in host
mode.
config PCIE_XILINX
bool "Xilinx AXI PCIe host bridge support"
depends on OF || COMPILE_TEST
+ select PCI_MSI_ARCH_FALLBACKS
help
Say 'Y' here if you want kernel to support the Xilinx AXI PCIe
Host Bridge driver.
.irq_unmask = hv_irq_unmask,
};
-static irq_hw_number_t hv_msi_domain_ops_get_hwirq(struct msi_domain_info *info,
- msi_alloc_info_t *arg)
-{
- return arg->msi_hwirq;
-}
-
static struct msi_domain_ops hv_msi_ops = {
- .get_hwirq = hv_msi_domain_ops_get_hwirq,
.msi_prepare = pci_msi_prepare,
- .set_desc = pci_msi_set_desc,
.msi_free = hv_msi_free,
};
static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
struct pci_bus *bus, int dev)
{
- /* access only one slot on each root port */
- if (pci_is_root_bus(bus) && dev > 0)
- return 0;
-
/*
- * do not read more than one device on the bus directly attached
+ * Access only one slot on each root port.
+ * Do not read more than one device on the bus directly attached
* to RC's downstream side.
*/
- if (pci_is_root_bus(bus->parent) && dev > 0)
- return 0;
+ if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent))
+ return dev == 0;
return 1;
}
return -ENODEV;
vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info,
- x86_vector_domain);
+ NULL);
+
if (!vmd->irq_domain) {
irq_domain_free_fwnode(fn);
return -ENODEV;
}
+ /*
+ * Override the irq domain bus token so the domain can be distinguished
+ * from a regular PCI/MSI domain.
+ */
+ irq_domain_update_bus_token(vmd->irq_domain, DOMAIN_BUS_VMD_MSI);
+
pci_add_resource(&resources, &vmd->resources[0]);
pci_add_resource_offset(&resources, &vmd->resources[1], offset[0]);
pci_add_resource_offset(&resources, &vmd->resources[2], offset[1]);
#define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs
#endif
+#ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS
/* Arch hooks */
-
int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
{
struct msi_controller *chip = dev->bus->msi;
{
return default_teardown_msi_irqs(dev);
}
+#endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS */
static void default_restore_msi_irq(struct pci_dev *dev, int irq)
{
/**
* pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source
- * @dev: Pointer to the PCI device
* @desc: Pointer to the MSI descriptor
*
* The ID number is only used within the irqdomain.
*/
-irq_hw_number_t pci_msi_domain_calc_hwirq(struct pci_dev *dev,
- struct msi_desc *desc)
+static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc)
{
+ struct pci_dev *dev = msi_desc_to_pci_dev(desc);
+
return (irq_hw_number_t)desc->msi_attrib.entry_nr |
pci_dev_id(dev) << 11 |
(pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27;
return error;
}
-#ifdef GENERIC_MSI_DOMAIN_OPS
static void pci_msi_domain_set_desc(msi_alloc_info_t *arg,
struct msi_desc *desc)
{
arg->desc = desc;
- arg->hwirq = pci_msi_domain_calc_hwirq(msi_desc_to_pci_dev(desc),
- desc);
+ arg->hwirq = pci_msi_domain_calc_hwirq(desc);
}
-#else
-#define pci_msi_domain_set_desc NULL
-#endif
static struct msi_domain_ops pci_msi_domain_ops_default = {
.set_desc = pci_msi_domain_set_desc,
DOMAIN_BUS_PCI_MSI);
return dom;
}
+
+/**
+ * pci_dev_has_special_msi_domain - Check whether the device is handled by
+ * a non-standard PCI-MSI domain
+ * @pdev: The PCI device to check.
+ *
+ * Returns: True if the device irqdomain or the bus irqdomain is
+ * non-standard PCI/MSI.
+ */
+bool pci_dev_has_special_msi_domain(struct pci_dev *pdev)
+{
+ struct irq_domain *dom = dev_get_msi_domain(&pdev->dev);
+
+ if (!dom)
+ dom = dev_get_msi_domain(&pdev->bus->dev);
+
+ if (!dom)
+ return true;
+
+ return dom->bus_token != DOMAIN_BUS_PCI_MSI;
+}
+
#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */
PMU (perf) driver supporting the ARM CCN (Cache Coherent Network)
interconnect.
+config ARM_CMN
+ tristate "Arm CMN-600 PMU support"
+ depends on ARM64 || (COMPILE_TEST && 64BIT)
+ help
+ Support for PMU events monitoring on the Arm CMN-600 Coherent Mesh
+ Network interconnect.
+
config ARM_PMU
depends on ARM || ARM64
bool "ARM PMU framework"
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_ARM_CCI_PMU) += arm-cci.o
obj-$(CONFIG_ARM_CCN) += arm-ccn.o
+obj-$(CONFIG_ARM_CMN) += arm-cmn.o
obj-$(CONFIG_ARM_DSU_PMU) += arm_dsu_pmu.o
obj-$(CONFIG_ARM_PMU) += arm_pmu.o arm_pmu_platform.o
obj-$(CONFIG_ARM_PMU_ACPI) += arm_pmu_acpi.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2016-2020 Arm Limited
+// CMN-600 Coherent Mesh Network PMU driver
+
+#include <linux/acpi.h>
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/perf_event.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/sort.h>
+
+/* Common register stuff */
+#define CMN_NODE_INFO 0x0000
+#define CMN_NI_NODE_TYPE GENMASK_ULL(15, 0)
+#define CMN_NI_NODE_ID GENMASK_ULL(31, 16)
+#define CMN_NI_LOGICAL_ID GENMASK_ULL(47, 32)
+
+#define CMN_NODEID_DEVID(reg) ((reg) & 3)
+#define CMN_NODEID_PID(reg) (((reg) >> 2) & 1)
+#define CMN_NODEID_X(reg, bits) ((reg) >> (3 + (bits)))
+#define CMN_NODEID_Y(reg, bits) (((reg) >> 3) & ((1U << (bits)) - 1))
+
+#define CMN_CHILD_INFO 0x0080
+#define CMN_CI_CHILD_COUNT GENMASK_ULL(15, 0)
+#define CMN_CI_CHILD_PTR_OFFSET GENMASK_ULL(31, 16)
+
+#define CMN_CHILD_NODE_ADDR GENMASK(27,0)
+#define CMN_CHILD_NODE_EXTERNAL BIT(31)
+
+#define CMN_ADDR_NODE_PTR GENMASK(27, 14)
+
+#define CMN_NODE_PTR_DEVID(ptr) (((ptr) >> 2) & 3)
+#define CMN_NODE_PTR_PID(ptr) ((ptr) & 1)
+#define CMN_NODE_PTR_X(ptr, bits) ((ptr) >> (6 + (bits)))
+#define CMN_NODE_PTR_Y(ptr, bits) (((ptr) >> 6) & ((1U << (bits)) - 1))
+
+#define CMN_MAX_XPS (8 * 8)
+
+/* The CFG node has one other useful purpose */
+#define CMN_CFGM_PERIPH_ID_2 0x0010
+#define CMN_CFGM_PID2_REVISION GENMASK(7, 4)
+
+/* PMU registers occupy the 3rd 4KB page of each node's 16KB space */
+#define CMN_PMU_OFFSET 0x2000
+
+/* For most nodes, this is all there is */
+#define CMN_PMU_EVENT_SEL 0x000
+#define CMN_PMU_EVENTn_ID_SHIFT(n) ((n) * 8)
+
+/* DTMs live in the PMU space of XP registers */
+#define CMN_DTM_WPn(n) (0x1A0 + (n) * 0x18)
+#define CMN_DTM_WPn_CONFIG(n) (CMN_DTM_WPn(n) + 0x00)
+#define CMN_DTM_WPn_CONFIG_WP_COMBINE BIT(6)
+#define CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE BIT(5)
+#define CMN_DTM_WPn_CONFIG_WP_GRP BIT(4)
+#define CMN_DTM_WPn_CONFIG_WP_CHN_SEL GENMASK_ULL(3, 1)
+#define CMN_DTM_WPn_CONFIG_WP_DEV_SEL BIT(0)
+#define CMN_DTM_WPn_VAL(n) (CMN_DTM_WPn(n) + 0x08)
+#define CMN_DTM_WPn_MASK(n) (CMN_DTM_WPn(n) + 0x10)
+
+#define CMN_DTM_PMU_CONFIG 0x210
+#define CMN__PMEVCNT0_INPUT_SEL GENMASK_ULL(37, 32)
+#define CMN__PMEVCNT0_INPUT_SEL_WP 0x00
+#define CMN__PMEVCNT0_INPUT_SEL_XP 0x04
+#define CMN__PMEVCNT0_INPUT_SEL_DEV 0x10
+#define CMN__PMEVCNT0_GLOBAL_NUM GENMASK_ULL(18, 16)
+#define CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(n) ((n) * 4)
+#define CMN__PMEVCNT_PAIRED(n) BIT(4 + (n))
+#define CMN__PMEVCNT23_COMBINED BIT(2)
+#define CMN__PMEVCNT01_COMBINED BIT(1)
+#define CMN_DTM_PMU_CONFIG_PMU_EN BIT(0)
+
+#define CMN_DTM_PMEVCNT 0x220
+
+#define CMN_DTM_PMEVCNTSR 0x240
+
+#define CMN_DTM_NUM_COUNTERS 4
+
+/* The DTC node is where the magic happens */
+#define CMN_DT_DTC_CTL 0x0a00
+#define CMN_DT_DTC_CTL_DT_EN BIT(0)
+
+/* DTC counters are paired in 64-bit registers on a 16-byte stride. Yuck */
+#define _CMN_DT_CNT_REG(n) ((((n) / 2) * 4 + (n) % 2) * 4)
+#define CMN_DT_PMEVCNT(n) (CMN_PMU_OFFSET + _CMN_DT_CNT_REG(n))
+#define CMN_DT_PMCCNTR (CMN_PMU_OFFSET + 0x40)
+
+#define CMN_DT_PMEVCNTSR(n) (CMN_PMU_OFFSET + 0x50 + _CMN_DT_CNT_REG(n))
+#define CMN_DT_PMCCNTRSR (CMN_PMU_OFFSET + 0x90)
+
+#define CMN_DT_PMCR (CMN_PMU_OFFSET + 0x100)
+#define CMN_DT_PMCR_PMU_EN BIT(0)
+#define CMN_DT_PMCR_CNTR_RST BIT(5)
+#define CMN_DT_PMCR_OVFL_INTR_EN BIT(6)
+
+#define CMN_DT_PMOVSR (CMN_PMU_OFFSET + 0x118)
+#define CMN_DT_PMOVSR_CLR (CMN_PMU_OFFSET + 0x120)
+
+#define CMN_DT_PMSSR (CMN_PMU_OFFSET + 0x128)
+#define CMN_DT_PMSSR_SS_STATUS(n) BIT(n)
+
+#define CMN_DT_PMSRR (CMN_PMU_OFFSET + 0x130)
+#define CMN_DT_PMSRR_SS_REQ BIT(0)
+
+#define CMN_DT_NUM_COUNTERS 8
+#define CMN_MAX_DTCS 4
+
+/*
+ * Even in the worst case a DTC counter can't wrap in fewer than 2^42 cycles,
+ * so throwing away one bit to make overflow handling easy is no big deal.
+ */
+#define CMN_COUNTER_INIT 0x80000000
+/* Similarly for the 40-bit cycle counter */
+#define CMN_CC_INIT 0x8000000000ULL
+
+
+/* Event attributes */
+#define CMN_CONFIG_TYPE GENMASK(15, 0)
+#define CMN_CONFIG_EVENTID GENMASK(23, 16)
+#define CMN_CONFIG_OCCUPID GENMASK(27, 24)
+#define CMN_CONFIG_BYNODEID BIT(31)
+#define CMN_CONFIG_NODEID GENMASK(47, 32)
+
+#define CMN_EVENT_TYPE(event) FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config)
+#define CMN_EVENT_EVENTID(event) FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config)
+#define CMN_EVENT_OCCUPID(event) FIELD_GET(CMN_CONFIG_OCCUPID, (event)->attr.config)
+#define CMN_EVENT_BYNODEID(event) FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config)
+#define CMN_EVENT_NODEID(event) FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config)
+
+#define CMN_CONFIG_WP_COMBINE GENMASK(27, 24)
+#define CMN_CONFIG_WP_DEV_SEL BIT(48)
+#define CMN_CONFIG_WP_CHN_SEL GENMASK(50, 49)
+#define CMN_CONFIG_WP_GRP BIT(52)
+#define CMN_CONFIG_WP_EXCLUSIVE BIT(53)
+#define CMN_CONFIG1_WP_VAL GENMASK(63, 0)
+#define CMN_CONFIG2_WP_MASK GENMASK(63, 0)
+
+#define CMN_EVENT_WP_COMBINE(event) FIELD_GET(CMN_CONFIG_WP_COMBINE, (event)->attr.config)
+#define CMN_EVENT_WP_DEV_SEL(event) FIELD_GET(CMN_CONFIG_WP_DEV_SEL, (event)->attr.config)
+#define CMN_EVENT_WP_CHN_SEL(event) FIELD_GET(CMN_CONFIG_WP_CHN_SEL, (event)->attr.config)
+#define CMN_EVENT_WP_GRP(event) FIELD_GET(CMN_CONFIG_WP_GRP, (event)->attr.config)
+#define CMN_EVENT_WP_EXCLUSIVE(event) FIELD_GET(CMN_CONFIG_WP_EXCLUSIVE, (event)->attr.config)
+#define CMN_EVENT_WP_VAL(event) FIELD_GET(CMN_CONFIG1_WP_VAL, (event)->attr.config1)
+#define CMN_EVENT_WP_MASK(event) FIELD_GET(CMN_CONFIG2_WP_MASK, (event)->attr.config2)
+
+/* Made-up event IDs for watchpoint direction */
+#define CMN_WP_UP 0
+#define CMN_WP_DOWN 2
+
+
+/* r0px probably don't exist in silicon, thankfully */
+enum cmn_revision {
+ CMN600_R1P0,
+ CMN600_R1P1,
+ CMN600_R1P2,
+ CMN600_R1P3,
+ CMN600_R2P0,
+ CMN600_R3P0,
+};
+
+enum cmn_node_type {
+ CMN_TYPE_INVALID,
+ CMN_TYPE_DVM,
+ CMN_TYPE_CFG,
+ CMN_TYPE_DTC,
+ CMN_TYPE_HNI,
+ CMN_TYPE_HNF,
+ CMN_TYPE_XP,
+ CMN_TYPE_SBSX,
+ CMN_TYPE_RNI = 0xa,
+ CMN_TYPE_RND = 0xd,
+ CMN_TYPE_RNSAM = 0xf,
+ CMN_TYPE_CXRA = 0x100,
+ CMN_TYPE_CXHA = 0x101,
+ CMN_TYPE_CXLA = 0x102,
+ /* Not a real node type */
+ CMN_TYPE_WP = 0x7770
+};
+
+struct arm_cmn_node {
+ void __iomem *pmu_base;
+ u16 id, logid;
+ enum cmn_node_type type;
+
+ union {
+ /* Device node */
+ struct {
+ int to_xp;
+ /* DN/HN-F/CXHA */
+ unsigned int occupid_val;
+ unsigned int occupid_count;
+ };
+ /* XP */
+ struct {
+ int dtc;
+ u32 pmu_config_low;
+ union {
+ u8 input_sel[4];
+ __le32 pmu_config_high;
+ };
+ s8 wp_event[4];
+ };
+ };
+
+ union {
+ u8 event[4];
+ __le32 event_sel;
+ };
+};
+
+struct arm_cmn_dtc {
+ void __iomem *base;
+ int irq;
+ int irq_friend;
+ bool cc_active;
+
+ struct perf_event *counters[CMN_DT_NUM_COUNTERS];
+ struct perf_event *cycles;
+};
+
+#define CMN_STATE_DISABLED BIT(0)
+#define CMN_STATE_TXN BIT(1)
+
+struct arm_cmn {
+ struct device *dev;
+ void __iomem *base;
+
+ enum cmn_revision rev;
+ u8 mesh_x;
+ u8 mesh_y;
+ u16 num_xps;
+ u16 num_dns;
+ struct arm_cmn_node *xps;
+ struct arm_cmn_node *dns;
+
+ struct arm_cmn_dtc *dtc;
+ unsigned int num_dtcs;
+
+ int cpu;
+ struct hlist_node cpuhp_node;
+
+ unsigned int state;
+ struct pmu pmu;
+};
+
+#define to_cmn(p) container_of(p, struct arm_cmn, pmu)
+
+static int arm_cmn_hp_state;
+
+struct arm_cmn_hw_event {
+ struct arm_cmn_node *dn;
+ u64 dtm_idx[2];
+ unsigned int dtc_idx;
+ u8 dtcs_used;
+ u8 num_dns;
+};
+
+#define for_each_hw_dn(hw, dn, i) \
+ for (i = 0, dn = hw->dn; i < hw->num_dns; i++, dn++)
+
+static struct arm_cmn_hw_event *to_cmn_hw(struct perf_event *event)
+{
+ BUILD_BUG_ON(sizeof(struct arm_cmn_hw_event) > offsetof(struct hw_perf_event, target));
+ return (struct arm_cmn_hw_event *)&event->hw;
+}
+
+static void arm_cmn_set_index(u64 x[], unsigned int pos, unsigned int val)
+{
+ x[pos / 32] |= (u64)val << ((pos % 32) * 2);
+}
+
+static unsigned int arm_cmn_get_index(u64 x[], unsigned int pos)
+{
+ return (x[pos / 32] >> ((pos % 32) * 2)) & 3;
+}
+
+struct arm_cmn_event_attr {
+ struct device_attribute attr;
+ enum cmn_node_type type;
+ u8 eventid;
+ u8 occupid;
+};
+
+struct arm_cmn_format_attr {
+ struct device_attribute attr;
+ u64 field;
+ int config;
+};
+
+static int arm_cmn_xyidbits(const struct arm_cmn *cmn)
+{
+ return cmn->mesh_x > 4 || cmn->mesh_y > 4 ? 3 : 2;
+}
+
+static void arm_cmn_init_node_to_xp(const struct arm_cmn *cmn,
+ struct arm_cmn_node *dn)
+{
+ int bits = arm_cmn_xyidbits(cmn);
+ int x = CMN_NODEID_X(dn->id, bits);
+ int y = CMN_NODEID_Y(dn->id, bits);
+ int xp_idx = cmn->mesh_x * y + x;
+
+ dn->to_xp = (cmn->xps + xp_idx) - dn;
+}
+
+static struct arm_cmn_node *arm_cmn_node_to_xp(struct arm_cmn_node *dn)
+{
+ return dn->type == CMN_TYPE_XP ? dn : dn + dn->to_xp;
+}
+
+static struct arm_cmn_node *arm_cmn_node(const struct arm_cmn *cmn,
+ enum cmn_node_type type)
+{
+ int i;
+
+ for (i = 0; i < cmn->num_dns; i++)
+ if (cmn->dns[i].type == type)
+ return &cmn->dns[i];
+ return NULL;
+}
+
+#define CMN_EVENT_ATTR(_name, _type, _eventid, _occupid) \
+ (&((struct arm_cmn_event_attr[]) {{ \
+ .attr = __ATTR(_name, 0444, arm_cmn_event_show, NULL), \
+ .type = _type, \
+ .eventid = _eventid, \
+ .occupid = _occupid, \
+ }})[0].attr.attr)
+
+static bool arm_cmn_is_occup_event(enum cmn_node_type type, unsigned int id)
+{
+ return (type == CMN_TYPE_DVM && id == 0x05) ||
+ (type == CMN_TYPE_HNF && id == 0x0f);
+}
+
+static ssize_t arm_cmn_event_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct arm_cmn_event_attr *eattr;
+
+ eattr = container_of(attr, typeof(*eattr), attr);
+
+ if (eattr->type == CMN_TYPE_DTC)
+ return snprintf(buf, PAGE_SIZE, "type=0x%x\n", eattr->type);
+
+ if (eattr->type == CMN_TYPE_WP)
+ return snprintf(buf, PAGE_SIZE,
+ "type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n",
+ eattr->type, eattr->eventid);
+
+ if (arm_cmn_is_occup_event(eattr->type, eattr->eventid))
+ return snprintf(buf, PAGE_SIZE, "type=0x%x,eventid=0x%x,occupid=0x%x\n",
+ eattr->type, eattr->eventid, eattr->occupid);
+
+ return snprintf(buf, PAGE_SIZE, "type=0x%x,eventid=0x%x\n",
+ eattr->type, eattr->eventid);
+}
+
+static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj,
+ struct attribute *attr,
+ int unused)
+{
+ struct device *dev = kobj_to_dev(kobj);
+ struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev));
+ struct arm_cmn_event_attr *eattr;
+ enum cmn_node_type type;
+
+ eattr = container_of(attr, typeof(*eattr), attr.attr);
+ type = eattr->type;
+
+ /* Watchpoints aren't nodes */
+ if (type == CMN_TYPE_WP)
+ type = CMN_TYPE_XP;
+
+ /* Revision-specific differences */
+ if (cmn->rev < CMN600_R1P2) {
+ if (type == CMN_TYPE_HNF && eattr->eventid == 0x1b)
+ return 0;
+ }
+
+ if (!arm_cmn_node(cmn, type))
+ return 0;
+
+ return attr->mode;
+}
+
+#define _CMN_EVENT_DVM(_name, _event, _occup) \
+ CMN_EVENT_ATTR(dn_##_name, CMN_TYPE_DVM, _event, _occup)
+#define CMN_EVENT_DTC(_name) \
+ CMN_EVENT_ATTR(dtc_##_name, CMN_TYPE_DTC, 0, 0)
+#define _CMN_EVENT_HNF(_name, _event, _occup) \
+ CMN_EVENT_ATTR(hnf_##_name, CMN_TYPE_HNF, _event, _occup)
+#define CMN_EVENT_HNI(_name, _event) \
+ CMN_EVENT_ATTR(hni_##_name, CMN_TYPE_HNI, _event, 0)
+#define __CMN_EVENT_XP(_name, _event) \
+ CMN_EVENT_ATTR(mxp_##_name, CMN_TYPE_XP, _event, 0)
+#define CMN_EVENT_SBSX(_name, _event) \
+ CMN_EVENT_ATTR(sbsx_##_name, CMN_TYPE_SBSX, _event, 0)
+#define CMN_EVENT_RNID(_name, _event) \
+ CMN_EVENT_ATTR(rnid_##_name, CMN_TYPE_RNI, _event, 0)
+
+#define CMN_EVENT_DVM(_name, _event) \
+ _CMN_EVENT_DVM(_name, _event, 0)
+#define CMN_EVENT_HNF(_name, _event) \
+ _CMN_EVENT_HNF(_name, _event, 0)
+#define _CMN_EVENT_XP(_name, _event) \
+ __CMN_EVENT_XP(e_##_name, (_event) | (0 << 2)), \
+ __CMN_EVENT_XP(w_##_name, (_event) | (1 << 2)), \
+ __CMN_EVENT_XP(n_##_name, (_event) | (2 << 2)), \
+ __CMN_EVENT_XP(s_##_name, (_event) | (3 << 2)), \
+ __CMN_EVENT_XP(p0_##_name, (_event) | (4 << 2)), \
+ __CMN_EVENT_XP(p1_##_name, (_event) | (5 << 2))
+
+/* Good thing there are only 3 fundamental XP events... */
+#define CMN_EVENT_XP(_name, _event) \
+ _CMN_EVENT_XP(req_##_name, (_event) | (0 << 5)), \
+ _CMN_EVENT_XP(rsp_##_name, (_event) | (1 << 5)), \
+ _CMN_EVENT_XP(snp_##_name, (_event) | (2 << 5)), \
+ _CMN_EVENT_XP(dat_##_name, (_event) | (3 << 5))
+
+
+static struct attribute *arm_cmn_event_attrs[] = {
+ CMN_EVENT_DTC(cycles),
+
+ /*
+ * DVM node events conflict with HN-I events in the equivalent PMU
+ * slot, but our lazy short-cut of using the DTM counter index for
+ * the PMU index as well happens to avoid that by construction.
+ */
+ CMN_EVENT_DVM(rxreq_dvmop, 0x01),
+ CMN_EVENT_DVM(rxreq_dvmsync, 0x02),
+ CMN_EVENT_DVM(rxreq_dvmop_vmid_filtered, 0x03),
+ CMN_EVENT_DVM(rxreq_retried, 0x04),
+ _CMN_EVENT_DVM(rxreq_trk_occupancy_all, 0x05, 0),
+ _CMN_EVENT_DVM(rxreq_trk_occupancy_dvmop, 0x05, 1),
+ _CMN_EVENT_DVM(rxreq_trk_occupancy_dvmsync, 0x05, 2),
+
+ CMN_EVENT_HNF(cache_miss, 0x01),
+ CMN_EVENT_HNF(slc_sf_cache_access, 0x02),
+ CMN_EVENT_HNF(cache_fill, 0x03),
+ CMN_EVENT_HNF(pocq_retry, 0x04),
+ CMN_EVENT_HNF(pocq_reqs_recvd, 0x05),
+ CMN_EVENT_HNF(sf_hit, 0x06),
+ CMN_EVENT_HNF(sf_evictions, 0x07),
+ CMN_EVENT_HNF(dir_snoops_sent, 0x08),
+ CMN_EVENT_HNF(brd_snoops_sent, 0x09),
+ CMN_EVENT_HNF(slc_eviction, 0x0a),
+ CMN_EVENT_HNF(slc_fill_invalid_way, 0x0b),
+ CMN_EVENT_HNF(mc_retries, 0x0c),
+ CMN_EVENT_HNF(mc_reqs, 0x0d),
+ CMN_EVENT_HNF(qos_hh_retry, 0x0e),
+ _CMN_EVENT_HNF(qos_pocq_occupancy_all, 0x0f, 0),
+ _CMN_EVENT_HNF(qos_pocq_occupancy_read, 0x0f, 1),
+ _CMN_EVENT_HNF(qos_pocq_occupancy_write, 0x0f, 2),
+ _CMN_EVENT_HNF(qos_pocq_occupancy_atomic, 0x0f, 3),
+ _CMN_EVENT_HNF(qos_pocq_occupancy_stash, 0x0f, 4),
+ CMN_EVENT_HNF(pocq_addrhaz, 0x10),
+ CMN_EVENT_HNF(pocq_atomic_addrhaz, 0x11),
+ CMN_EVENT_HNF(ld_st_swp_adq_full, 0x12),
+ CMN_EVENT_HNF(cmp_adq_full, 0x13),
+ CMN_EVENT_HNF(txdat_stall, 0x14),
+ CMN_EVENT_HNF(txrsp_stall, 0x15),
+ CMN_EVENT_HNF(seq_full, 0x16),
+ CMN_EVENT_HNF(seq_hit, 0x17),
+ CMN_EVENT_HNF(snp_sent, 0x18),
+ CMN_EVENT_HNF(sfbi_dir_snp_sent, 0x19),
+ CMN_EVENT_HNF(sfbi_brd_snp_sent, 0x1a),
+ CMN_EVENT_HNF(snp_sent_untrk, 0x1b),
+ CMN_EVENT_HNF(intv_dirty, 0x1c),
+ CMN_EVENT_HNF(stash_snp_sent, 0x1d),
+ CMN_EVENT_HNF(stash_data_pull, 0x1e),
+ CMN_EVENT_HNF(snp_fwded, 0x1f),
+
+ CMN_EVENT_HNI(rrt_rd_occ_cnt_ovfl, 0x20),
+ CMN_EVENT_HNI(rrt_wr_occ_cnt_ovfl, 0x21),
+ CMN_EVENT_HNI(rdt_rd_occ_cnt_ovfl, 0x22),
+ CMN_EVENT_HNI(rdt_wr_occ_cnt_ovfl, 0x23),
+ CMN_EVENT_HNI(wdb_occ_cnt_ovfl, 0x24),
+ CMN_EVENT_HNI(rrt_rd_alloc, 0x25),
+ CMN_EVENT_HNI(rrt_wr_alloc, 0x26),
+ CMN_EVENT_HNI(rdt_rd_alloc, 0x27),
+ CMN_EVENT_HNI(rdt_wr_alloc, 0x28),
+ CMN_EVENT_HNI(wdb_alloc, 0x29),
+ CMN_EVENT_HNI(txrsp_retryack, 0x2a),
+ CMN_EVENT_HNI(arvalid_no_arready, 0x2b),
+ CMN_EVENT_HNI(arready_no_arvalid, 0x2c),
+ CMN_EVENT_HNI(awvalid_no_awready, 0x2d),
+ CMN_EVENT_HNI(awready_no_awvalid, 0x2e),
+ CMN_EVENT_HNI(wvalid_no_wready, 0x2f),
+ CMN_EVENT_HNI(txdat_stall, 0x30),
+ CMN_EVENT_HNI(nonpcie_serialization, 0x31),
+ CMN_EVENT_HNI(pcie_serialization, 0x32),
+
+ CMN_EVENT_XP(txflit_valid, 0x01),
+ CMN_EVENT_XP(txflit_stall, 0x02),
+ CMN_EVENT_XP(partial_dat_flit, 0x03),
+ /* We treat watchpoints as a special made-up class of XP events */
+ CMN_EVENT_ATTR(watchpoint_up, CMN_TYPE_WP, 0, 0),
+ CMN_EVENT_ATTR(watchpoint_down, CMN_TYPE_WP, 2, 0),
+
+ CMN_EVENT_SBSX(rd_req, 0x01),
+ CMN_EVENT_SBSX(wr_req, 0x02),
+ CMN_EVENT_SBSX(cmo_req, 0x03),
+ CMN_EVENT_SBSX(txrsp_retryack, 0x04),
+ CMN_EVENT_SBSX(txdat_flitv, 0x05),
+ CMN_EVENT_SBSX(txrsp_flitv, 0x06),
+ CMN_EVENT_SBSX(rd_req_trkr_occ_cnt_ovfl, 0x11),
+ CMN_EVENT_SBSX(wr_req_trkr_occ_cnt_ovfl, 0x12),
+ CMN_EVENT_SBSX(cmo_req_trkr_occ_cnt_ovfl, 0x13),
+ CMN_EVENT_SBSX(wdb_occ_cnt_ovfl, 0x14),
+ CMN_EVENT_SBSX(rd_axi_trkr_occ_cnt_ovfl, 0x15),
+ CMN_EVENT_SBSX(cmo_axi_trkr_occ_cnt_ovfl, 0x16),
+ CMN_EVENT_SBSX(arvalid_no_arready, 0x21),
+ CMN_EVENT_SBSX(awvalid_no_awready, 0x22),
+ CMN_EVENT_SBSX(wvalid_no_wready, 0x23),
+ CMN_EVENT_SBSX(txdat_stall, 0x24),
+ CMN_EVENT_SBSX(txrsp_stall, 0x25),
+
+ CMN_EVENT_RNID(s0_rdata_beats, 0x01),
+ CMN_EVENT_RNID(s1_rdata_beats, 0x02),
+ CMN_EVENT_RNID(s2_rdata_beats, 0x03),
+ CMN_EVENT_RNID(rxdat_flits, 0x04),
+ CMN_EVENT_RNID(txdat_flits, 0x05),
+ CMN_EVENT_RNID(txreq_flits_total, 0x06),
+ CMN_EVENT_RNID(txreq_flits_retried, 0x07),
+ CMN_EVENT_RNID(rrt_occ_ovfl, 0x08),
+ CMN_EVENT_RNID(wrt_occ_ovfl, 0x09),
+ CMN_EVENT_RNID(txreq_flits_replayed, 0x0a),
+ CMN_EVENT_RNID(wrcancel_sent, 0x0b),
+ CMN_EVENT_RNID(s0_wdata_beats, 0x0c),
+ CMN_EVENT_RNID(s1_wdata_beats, 0x0d),
+ CMN_EVENT_RNID(s2_wdata_beats, 0x0e),
+ CMN_EVENT_RNID(rrt_alloc, 0x0f),
+ CMN_EVENT_RNID(wrt_alloc, 0x10),
+ CMN_EVENT_RNID(rdb_unord, 0x11),
+ CMN_EVENT_RNID(rdb_replay, 0x12),
+ CMN_EVENT_RNID(rdb_hybrid, 0x13),
+ CMN_EVENT_RNID(rdb_ord, 0x14),
+
+ NULL
+};
+
+static const struct attribute_group arm_cmn_event_attrs_group = {
+ .name = "events",
+ .attrs = arm_cmn_event_attrs,
+ .is_visible = arm_cmn_event_attr_is_visible,
+};
+
+static ssize_t arm_cmn_format_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct arm_cmn_format_attr *fmt = container_of(attr, typeof(*fmt), attr);
+ int lo = __ffs(fmt->field), hi = __fls(fmt->field);
+
+ if (lo == hi)
+ return snprintf(buf, PAGE_SIZE, "config:%d\n", lo);
+
+ if (!fmt->config)
+ return snprintf(buf, PAGE_SIZE, "config:%d-%d\n", lo, hi);
+
+ return snprintf(buf, PAGE_SIZE, "config%d:%d-%d\n", fmt->config, lo, hi);
+}
+
+#define _CMN_FORMAT_ATTR(_name, _cfg, _fld) \
+ (&((struct arm_cmn_format_attr[]) {{ \
+ .attr = __ATTR(_name, 0444, arm_cmn_format_show, NULL), \
+ .config = _cfg, \
+ .field = _fld, \
+ }})[0].attr.attr)
+#define CMN_FORMAT_ATTR(_name, _fld) _CMN_FORMAT_ATTR(_name, 0, _fld)
+
+static struct attribute *arm_cmn_format_attrs[] = {
+ CMN_FORMAT_ATTR(type, CMN_CONFIG_TYPE),
+ CMN_FORMAT_ATTR(eventid, CMN_CONFIG_EVENTID),
+ CMN_FORMAT_ATTR(occupid, CMN_CONFIG_OCCUPID),
+ CMN_FORMAT_ATTR(bynodeid, CMN_CONFIG_BYNODEID),
+ CMN_FORMAT_ATTR(nodeid, CMN_CONFIG_NODEID),
+
+ CMN_FORMAT_ATTR(wp_dev_sel, CMN_CONFIG_WP_DEV_SEL),
+ CMN_FORMAT_ATTR(wp_chn_sel, CMN_CONFIG_WP_CHN_SEL),
+ CMN_FORMAT_ATTR(wp_grp, CMN_CONFIG_WP_GRP),
+ CMN_FORMAT_ATTR(wp_exclusive, CMN_CONFIG_WP_EXCLUSIVE),
+ CMN_FORMAT_ATTR(wp_combine, CMN_CONFIG_WP_COMBINE),
+
+ _CMN_FORMAT_ATTR(wp_val, 1, CMN_CONFIG1_WP_VAL),
+ _CMN_FORMAT_ATTR(wp_mask, 2, CMN_CONFIG2_WP_MASK),
+
+ NULL
+};
+
+static const struct attribute_group arm_cmn_format_attrs_group = {
+ .name = "format",
+ .attrs = arm_cmn_format_attrs,
+};
+
+static ssize_t arm_cmn_cpumask_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev));
+
+ return cpumap_print_to_pagebuf(true, buf, cpumask_of(cmn->cpu));
+}
+
+static struct device_attribute arm_cmn_cpumask_attr =
+ __ATTR(cpumask, 0444, arm_cmn_cpumask_show, NULL);
+
+static struct attribute *arm_cmn_cpumask_attrs[] = {
+ &arm_cmn_cpumask_attr.attr,
+ NULL,
+};
+
+static struct attribute_group arm_cmn_cpumask_attr_group = {
+ .attrs = arm_cmn_cpumask_attrs,
+};
+
+static const struct attribute_group *arm_cmn_attr_groups[] = {
+ &arm_cmn_event_attrs_group,
+ &arm_cmn_format_attrs_group,
+ &arm_cmn_cpumask_attr_group,
+ NULL
+};
+
+static int arm_cmn_wp_idx(struct perf_event *event)
+{
+ return CMN_EVENT_EVENTID(event) + CMN_EVENT_WP_GRP(event);
+}
+
+static u32 arm_cmn_wp_config(struct perf_event *event)
+{
+ u32 config;
+ u32 dev = CMN_EVENT_WP_DEV_SEL(event);
+ u32 chn = CMN_EVENT_WP_CHN_SEL(event);
+ u32 grp = CMN_EVENT_WP_GRP(event);
+ u32 exc = CMN_EVENT_WP_EXCLUSIVE(event);
+ u32 combine = CMN_EVENT_WP_COMBINE(event);
+
+ config = FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL, dev) |
+ FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_CHN_SEL, chn) |
+ FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_GRP, grp) |
+ FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE, exc);
+ if (combine && !grp)
+ config |= CMN_DTM_WPn_CONFIG_WP_COMBINE;
+
+ return config;
+}
+
+static void arm_cmn_set_state(struct arm_cmn *cmn, u32 state)
+{
+ if (!cmn->state)
+ writel_relaxed(0, cmn->dtc[0].base + CMN_DT_PMCR);
+ cmn->state |= state;
+}
+
+static void arm_cmn_clear_state(struct arm_cmn *cmn, u32 state)
+{
+ cmn->state &= ~state;
+ if (!cmn->state)
+ writel_relaxed(CMN_DT_PMCR_PMU_EN | CMN_DT_PMCR_OVFL_INTR_EN,
+ cmn->dtc[0].base + CMN_DT_PMCR);
+}
+
+static void arm_cmn_pmu_enable(struct pmu *pmu)
+{
+ arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_DISABLED);
+}
+
+static void arm_cmn_pmu_disable(struct pmu *pmu)
+{
+ arm_cmn_set_state(to_cmn(pmu), CMN_STATE_DISABLED);
+}
+
+static u64 arm_cmn_read_dtm(struct arm_cmn *cmn, struct arm_cmn_hw_event *hw,
+ bool snapshot)
+{
+ struct arm_cmn_node *dn;
+ unsigned int i, offset;
+ u64 count = 0;
+
+ offset = snapshot ? CMN_DTM_PMEVCNTSR : CMN_DTM_PMEVCNT;
+ for_each_hw_dn(hw, dn, i) {
+ struct arm_cmn_node *xp = arm_cmn_node_to_xp(dn);
+ int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
+ u64 reg = readq_relaxed(xp->pmu_base + offset);
+ u16 dtm_count = reg >> (dtm_idx * 16);
+
+ count += dtm_count;
+ }
+ return count;
+}
+
+static u64 arm_cmn_read_cc(struct arm_cmn_dtc *dtc)
+{
+ u64 val = readq_relaxed(dtc->base + CMN_DT_PMCCNTR);
+
+ writeq_relaxed(CMN_CC_INIT, dtc->base + CMN_DT_PMCCNTR);
+ return (val - CMN_CC_INIT) & ((CMN_CC_INIT << 1) - 1);
+}
+
+static u32 arm_cmn_read_counter(struct arm_cmn_dtc *dtc, int idx)
+{
+ u32 val, pmevcnt = CMN_DT_PMEVCNT(idx);
+
+ val = readl_relaxed(dtc->base + pmevcnt);
+ writel_relaxed(CMN_COUNTER_INIT, dtc->base + pmevcnt);
+ return val - CMN_COUNTER_INIT;
+}
+
+static void arm_cmn_init_counter(struct perf_event *event)
+{
+ struct arm_cmn *cmn = to_cmn(event->pmu);
+ struct arm_cmn_hw_event *hw = to_cmn_hw(event);
+ unsigned int i, pmevcnt = CMN_DT_PMEVCNT(hw->dtc_idx);
+ u64 count;
+
+ for (i = 0; hw->dtcs_used & (1U << i); i++) {
+ writel_relaxed(CMN_COUNTER_INIT, cmn->dtc[i].base + pmevcnt);
+ cmn->dtc[i].counters[hw->dtc_idx] = event;
+ }
+
+ count = arm_cmn_read_dtm(cmn, hw, false);
+ local64_set(&event->hw.prev_count, count);
+}
+
+static void arm_cmn_event_read(struct perf_event *event)
+{
+ struct arm_cmn *cmn = to_cmn(event->pmu);
+ struct arm_cmn_hw_event *hw = to_cmn_hw(event);
+ u64 delta, new, prev;
+ unsigned long flags;
+ unsigned int i;
+
+ if (hw->dtc_idx == CMN_DT_NUM_COUNTERS) {
+ i = __ffs(hw->dtcs_used);
+ delta = arm_cmn_read_cc(cmn->dtc + i);
+ local64_add(delta, &event->count);
+ return;
+ }
+ new = arm_cmn_read_dtm(cmn, hw, false);
+ prev = local64_xchg(&event->hw.prev_count, new);
+
+ delta = new - prev;
+
+ local_irq_save(flags);
+ for (i = 0; hw->dtcs_used & (1U << i); i++) {
+ new = arm_cmn_read_counter(cmn->dtc + i, hw->dtc_idx);
+ delta += new << 16;
+ }
+ local_irq_restore(flags);
+ local64_add(delta, &event->count);
+}
+
+static void arm_cmn_event_start(struct perf_event *event, int flags)
+{
+ struct arm_cmn *cmn = to_cmn(event->pmu);
+ struct arm_cmn_hw_event *hw = to_cmn_hw(event);
+ struct arm_cmn_node *dn;
+ enum cmn_node_type type = CMN_EVENT_TYPE(event);
+ int i;
+
+ if (type == CMN_TYPE_DTC) {
+ i = __ffs(hw->dtcs_used);
+ writeq_relaxed(CMN_CC_INIT, cmn->dtc[i].base + CMN_DT_PMCCNTR);
+ cmn->dtc[i].cc_active = true;
+ } else if (type == CMN_TYPE_WP) {
+ int wp_idx = arm_cmn_wp_idx(event);
+ u64 val = CMN_EVENT_WP_VAL(event);
+ u64 mask = CMN_EVENT_WP_MASK(event);
+
+ for_each_hw_dn(hw, dn, i) {
+ writeq_relaxed(val, dn->pmu_base + CMN_DTM_WPn_VAL(wp_idx));
+ writeq_relaxed(mask, dn->pmu_base + CMN_DTM_WPn_MASK(wp_idx));
+ }
+ } else for_each_hw_dn(hw, dn, i) {
+ int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
+
+ dn->event[dtm_idx] = CMN_EVENT_EVENTID(event);
+ writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL);
+ }
+}
+
+static void arm_cmn_event_stop(struct perf_event *event, int flags)
+{
+ struct arm_cmn *cmn = to_cmn(event->pmu);
+ struct arm_cmn_hw_event *hw = to_cmn_hw(event);
+ struct arm_cmn_node *dn;
+ enum cmn_node_type type = CMN_EVENT_TYPE(event);
+ int i;
+
+ if (type == CMN_TYPE_DTC) {
+ i = __ffs(hw->dtcs_used);
+ cmn->dtc[i].cc_active = false;
+ } else if (type == CMN_TYPE_WP) {
+ int wp_idx = arm_cmn_wp_idx(event);
+
+ for_each_hw_dn(hw, dn, i) {
+ writeq_relaxed(0, dn->pmu_base + CMN_DTM_WPn_MASK(wp_idx));
+ writeq_relaxed(~0ULL, dn->pmu_base + CMN_DTM_WPn_VAL(wp_idx));
+ }
+ } else for_each_hw_dn(hw, dn, i) {
+ int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
+
+ dn->event[dtm_idx] = 0;
+ writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL);
+ }
+
+ arm_cmn_event_read(event);
+}
+
+struct arm_cmn_val {
+ u8 dtm_count[CMN_MAX_XPS];
+ u8 occupid[CMN_MAX_XPS];
+ u8 wp[CMN_MAX_XPS][4];
+ int dtc_count;
+ bool cycles;
+};
+
+static void arm_cmn_val_add_event(struct arm_cmn_val *val, struct perf_event *event)
+{
+ struct arm_cmn_hw_event *hw = to_cmn_hw(event);
+ struct arm_cmn_node *dn;
+ enum cmn_node_type type;
+ int i;
+ u8 occupid;
+
+ if (is_software_event(event))
+ return;
+
+ type = CMN_EVENT_TYPE(event);
+ if (type == CMN_TYPE_DTC) {
+ val->cycles = true;
+ return;
+ }
+
+ val->dtc_count++;
+ if (arm_cmn_is_occup_event(type, CMN_EVENT_EVENTID(event)))
+ occupid = CMN_EVENT_OCCUPID(event) + 1;
+ else
+ occupid = 0;
+
+ for_each_hw_dn(hw, dn, i) {
+ int wp_idx, xp = arm_cmn_node_to_xp(dn)->logid;
+
+ val->dtm_count[xp]++;
+ val->occupid[xp] = occupid;
+
+ if (type != CMN_TYPE_WP)
+ continue;
+
+ wp_idx = arm_cmn_wp_idx(event);
+ val->wp[xp][wp_idx] = CMN_EVENT_WP_COMBINE(event) + 1;
+ }
+}
+
+static int arm_cmn_validate_group(struct perf_event *event)
+{
+ struct arm_cmn_hw_event *hw = to_cmn_hw(event);
+ struct arm_cmn_node *dn;
+ struct perf_event *sibling, *leader = event->group_leader;
+ enum cmn_node_type type;
+ struct arm_cmn_val val;
+ int i;
+ u8 occupid;
+
+ if (leader == event)
+ return 0;
+
+ if (event->pmu != leader->pmu && !is_software_event(leader))
+ return -EINVAL;
+
+ memset(&val, 0, sizeof(val));
+
+ arm_cmn_val_add_event(&val, leader);
+ for_each_sibling_event(sibling, leader)
+ arm_cmn_val_add_event(&val, sibling);
+
+ type = CMN_EVENT_TYPE(event);
+ if (type == CMN_TYPE_DTC)
+ return val.cycles ? -EINVAL : 0;
+
+ if (val.dtc_count == CMN_DT_NUM_COUNTERS)
+ return -EINVAL;
+
+ if (arm_cmn_is_occup_event(type, CMN_EVENT_EVENTID(event)))
+ occupid = CMN_EVENT_OCCUPID(event) + 1;
+ else
+ occupid = 0;
+
+ for_each_hw_dn(hw, dn, i) {
+ int wp_idx, wp_cmb, xp = arm_cmn_node_to_xp(dn)->logid;
+
+ if (val.dtm_count[xp] == CMN_DTM_NUM_COUNTERS)
+ return -EINVAL;
+
+ if (occupid && val.occupid[xp] && occupid != val.occupid[xp])
+ return -EINVAL;
+
+ if (type != CMN_TYPE_WP)
+ continue;
+
+ wp_idx = arm_cmn_wp_idx(event);
+ if (val.wp[xp][wp_idx])
+ return -EINVAL;
+
+ wp_cmb = val.wp[xp][wp_idx ^ 1];
+ if (wp_cmb && wp_cmb != CMN_EVENT_WP_COMBINE(event) + 1)
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int arm_cmn_event_init(struct perf_event *event)
+{
+ struct arm_cmn *cmn = to_cmn(event->pmu);
+ struct arm_cmn_hw_event *hw = to_cmn_hw(event);
+ enum cmn_node_type type;
+ unsigned int i;
+ bool bynodeid;
+ u16 nodeid, eventid;
+
+ if (event->attr.type != event->pmu->type)
+ return -ENOENT;
+
+ if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
+ return -EINVAL;
+
+ event->cpu = cmn->cpu;
+ if (event->cpu < 0)
+ return -EINVAL;
+
+ type = CMN_EVENT_TYPE(event);
+ /* DTC events (i.e. cycles) already have everything they need */
+ if (type == CMN_TYPE_DTC)
+ return 0;
+
+ /* For watchpoints we need the actual XP node here */
+ if (type == CMN_TYPE_WP) {
+ type = CMN_TYPE_XP;
+ /* ...and we need a "real" direction */
+ eventid = CMN_EVENT_EVENTID(event);
+ if (eventid != CMN_WP_UP && eventid != CMN_WP_DOWN)
+ return -EINVAL;
+ }
+
+ bynodeid = CMN_EVENT_BYNODEID(event);
+ nodeid = CMN_EVENT_NODEID(event);
+
+ hw->dn = arm_cmn_node(cmn, type);
+ for (i = hw->dn - cmn->dns; i < cmn->num_dns && cmn->dns[i].type == type; i++) {
+ if (!bynodeid) {
+ hw->num_dns++;
+ } else if (cmn->dns[i].id != nodeid) {
+ hw->dn++;
+ } else {
+ hw->num_dns = 1;
+ break;
+ }
+ }
+
+ if (!hw->num_dns) {
+ int bits = arm_cmn_xyidbits(cmn);
+
+ dev_dbg(cmn->dev, "invalid node 0x%x (%d,%d,%d,%d) type 0x%x\n",
+ nodeid, CMN_NODEID_X(nodeid, bits), CMN_NODEID_Y(nodeid, bits),
+ CMN_NODEID_PID(nodeid), CMN_NODEID_DEVID(nodeid), type);
+ return -EINVAL;
+ }
+ /*
+ * By assuming events count in all DTC domains, we cunningly avoid
+ * needing to know anything about how XPs are assigned to domains.
+ */
+ hw->dtcs_used = (1U << cmn->num_dtcs) - 1;
+
+ return arm_cmn_validate_group(event);
+}
+
+static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event,
+ int i)
+{
+ struct arm_cmn_hw_event *hw = to_cmn_hw(event);
+ enum cmn_node_type type = CMN_EVENT_TYPE(event);
+
+ while (i--) {
+ struct arm_cmn_node *xp = arm_cmn_node_to_xp(hw->dn + i);
+ unsigned int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
+
+ if (type == CMN_TYPE_WP)
+ hw->dn[i].wp_event[arm_cmn_wp_idx(event)] = -1;
+
+ if (arm_cmn_is_occup_event(type, CMN_EVENT_EVENTID(event)))
+ hw->dn[i].occupid_count--;
+
+ xp->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx);
+ writel_relaxed(xp->pmu_config_low, xp->pmu_base + CMN_DTM_PMU_CONFIG);
+ }
+ memset(hw->dtm_idx, 0, sizeof(hw->dtm_idx));
+
+ for (i = 0; hw->dtcs_used & (1U << i); i++)
+ cmn->dtc[i].counters[hw->dtc_idx] = NULL;
+}
+
+static int arm_cmn_event_add(struct perf_event *event, int flags)
+{
+ struct arm_cmn *cmn = to_cmn(event->pmu);
+ struct arm_cmn_hw_event *hw = to_cmn_hw(event);
+ struct arm_cmn_dtc *dtc = &cmn->dtc[0];
+ struct arm_cmn_node *dn;
+ enum cmn_node_type type = CMN_EVENT_TYPE(event);
+ unsigned int i, dtc_idx, input_sel;
+
+ if (type == CMN_TYPE_DTC) {
+ i = 0;
+ while (cmn->dtc[i].cycles)
+ if (++i == cmn->num_dtcs)
+ return -ENOSPC;
+
+ cmn->dtc[i].cycles = event;
+ hw->dtc_idx = CMN_DT_NUM_COUNTERS;
+ hw->dtcs_used = 1U << i;
+
+ if (flags & PERF_EF_START)
+ arm_cmn_event_start(event, 0);
+ return 0;
+ }
+
+ /* Grab a free global counter first... */
+ dtc_idx = 0;
+ while (dtc->counters[dtc_idx])
+ if (++dtc_idx == CMN_DT_NUM_COUNTERS)
+ return -ENOSPC;
+
+ hw->dtc_idx = dtc_idx;
+
+ /* ...then the local counters to feed it. */
+ for_each_hw_dn(hw, dn, i) {
+ struct arm_cmn_node *xp = arm_cmn_node_to_xp(dn);
+ unsigned int dtm_idx, shift;
+ u64 reg;
+
+ dtm_idx = 0;
+ while (xp->pmu_config_low & CMN__PMEVCNT_PAIRED(dtm_idx))
+ if (++dtm_idx == CMN_DTM_NUM_COUNTERS)
+ goto free_dtms;
+
+ if (type == CMN_TYPE_XP) {
+ input_sel = CMN__PMEVCNT0_INPUT_SEL_XP + dtm_idx;
+ } else if (type == CMN_TYPE_WP) {
+ int tmp, wp_idx = arm_cmn_wp_idx(event);
+ u32 cfg = arm_cmn_wp_config(event);
+
+ if (dn->wp_event[wp_idx] >= 0)
+ goto free_dtms;
+
+ tmp = dn->wp_event[wp_idx ^ 1];
+ if (tmp >= 0 && CMN_EVENT_WP_COMBINE(event) !=
+ CMN_EVENT_WP_COMBINE(dtc->counters[tmp]))
+ goto free_dtms;
+
+ input_sel = CMN__PMEVCNT0_INPUT_SEL_WP + wp_idx;
+ dn->wp_event[wp_idx] = dtc_idx;
+ writel_relaxed(cfg, dn->pmu_base + CMN_DTM_WPn_CONFIG(wp_idx));
+ } else {
+ unsigned int port = CMN_NODEID_PID(dn->id);
+ unsigned int dev = CMN_NODEID_DEVID(dn->id);
+
+ input_sel = CMN__PMEVCNT0_INPUT_SEL_DEV + dtm_idx +
+ (port << 4) + (dev << 2);
+
+ if (arm_cmn_is_occup_event(type, CMN_EVENT_EVENTID(event))) {
+ int occupid = CMN_EVENT_OCCUPID(event);
+
+ if (dn->occupid_count == 0) {
+ dn->occupid_val = occupid;
+ writel_relaxed(occupid,
+ dn->pmu_base + CMN_PMU_EVENT_SEL + 4);
+ } else if (dn->occupid_val != occupid) {
+ goto free_dtms;
+ }
+ dn->occupid_count++;
+ }
+ }
+
+ arm_cmn_set_index(hw->dtm_idx, i, dtm_idx);
+
+ xp->input_sel[dtm_idx] = input_sel;
+ shift = CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(dtm_idx);
+ xp->pmu_config_low &= ~(CMN__PMEVCNT0_GLOBAL_NUM << shift);
+ xp->pmu_config_low |= FIELD_PREP(CMN__PMEVCNT0_GLOBAL_NUM, dtc_idx) << shift;
+ xp->pmu_config_low |= CMN__PMEVCNT_PAIRED(dtm_idx);
+ reg = (u64)le32_to_cpu(xp->pmu_config_high) << 32 | xp->pmu_config_low;
+ writeq_relaxed(reg, xp->pmu_base + CMN_DTM_PMU_CONFIG);
+ }
+
+ /* Go go go! */
+ arm_cmn_init_counter(event);
+
+ if (flags & PERF_EF_START)
+ arm_cmn_event_start(event, 0);
+
+ return 0;
+
+free_dtms:
+ arm_cmn_event_clear(cmn, event, i);
+ return -ENOSPC;
+}
+
+static void arm_cmn_event_del(struct perf_event *event, int flags)
+{
+ struct arm_cmn *cmn = to_cmn(event->pmu);
+ struct arm_cmn_hw_event *hw = to_cmn_hw(event);
+ enum cmn_node_type type = CMN_EVENT_TYPE(event);
+
+ arm_cmn_event_stop(event, PERF_EF_UPDATE);
+
+ if (type == CMN_TYPE_DTC)
+ cmn->dtc[__ffs(hw->dtcs_used)].cycles = NULL;
+ else
+ arm_cmn_event_clear(cmn, event, hw->num_dns);
+}
+
+/*
+ * We stop the PMU for both add and read, to avoid skew across DTM counters.
+ * In theory we could use snapshots to read without stopping, but then it
+ * becomes a lot trickier to deal with overlow and racing against interrupts,
+ * plus it seems they don't work properly on some hardware anyway :(
+ */
+static void arm_cmn_start_txn(struct pmu *pmu, unsigned int flags)
+{
+ arm_cmn_set_state(to_cmn(pmu), CMN_STATE_TXN);
+}
+
+static void arm_cmn_end_txn(struct pmu *pmu)
+{
+ arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_TXN);
+}
+
+static int arm_cmn_commit_txn(struct pmu *pmu)
+{
+ arm_cmn_end_txn(pmu);
+ return 0;
+}
+
+static int arm_cmn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
+{
+ struct arm_cmn *cmn;
+ unsigned int target;
+
+ cmn = hlist_entry_safe(node, struct arm_cmn, cpuhp_node);
+ if (cpu != cmn->cpu)
+ return 0;
+
+ target = cpumask_any_but(cpu_online_mask, cpu);
+ if (target >= nr_cpu_ids)
+ return 0;
+
+ perf_pmu_migrate_context(&cmn->pmu, cpu, target);
+ cmn->cpu = target;
+ return 0;
+}
+
+static irqreturn_t arm_cmn_handle_irq(int irq, void *dev_id)
+{
+ struct arm_cmn_dtc *dtc = dev_id;
+ irqreturn_t ret = IRQ_NONE;
+
+ for (;;) {
+ u32 status = readl_relaxed(dtc->base + CMN_DT_PMOVSR);
+ u64 delta;
+ int i;
+
+ for (i = 0; i < CMN_DTM_NUM_COUNTERS; i++) {
+ if (status & (1U << i)) {
+ ret = IRQ_HANDLED;
+ if (WARN_ON(!dtc->counters[i]))
+ continue;
+ delta = (u64)arm_cmn_read_counter(dtc, i) << 16;
+ local64_add(delta, &dtc->counters[i]->count);
+ }
+ }
+
+ if (status & (1U << CMN_DT_NUM_COUNTERS)) {
+ ret = IRQ_HANDLED;
+ if (dtc->cc_active && !WARN_ON(!dtc->cycles)) {
+ delta = arm_cmn_read_cc(dtc);
+ local64_add(delta, &dtc->cycles->count);
+ }
+ }
+
+ writel_relaxed(status, dtc->base + CMN_DT_PMOVSR_CLR);
+
+ if (!dtc->irq_friend)
+ return ret;
+ dtc += dtc->irq_friend;
+ }
+}
+
+/* We can reasonably accommodate DTCs of the same CMN sharing IRQs */
+static int arm_cmn_init_irqs(struct arm_cmn *cmn)
+{
+ int i, j, irq, err;
+
+ for (i = 0; i < cmn->num_dtcs; i++) {
+ irq = cmn->dtc[i].irq;
+ for (j = i; j--; ) {
+ if (cmn->dtc[j].irq == irq) {
+ cmn->dtc[j].irq_friend = j - i;
+ goto next;
+ }
+ }
+ err = devm_request_irq(cmn->dev, irq, arm_cmn_handle_irq,
+ IRQF_NOBALANCING | IRQF_NO_THREAD,
+ dev_name(cmn->dev), &cmn->dtc[i]);
+ if (err)
+ return err;
+
+ err = irq_set_affinity_hint(irq, cpumask_of(cmn->cpu));
+ if (err)
+ return err;
+ next:
+ ; /* isn't C great? */
+ }
+ return 0;
+}
+
+static void arm_cmn_init_dtm(struct arm_cmn_node *xp)
+{
+ int i;
+
+ for (i = 0; i < 4; i++) {
+ xp->wp_event[i] = -1;
+ writeq_relaxed(0, xp->pmu_base + CMN_DTM_WPn_MASK(i));
+ writeq_relaxed(~0ULL, xp->pmu_base + CMN_DTM_WPn_VAL(i));
+ }
+ xp->pmu_config_low = CMN_DTM_PMU_CONFIG_PMU_EN;
+ xp->dtc = -1;
+}
+
+static int arm_cmn_init_dtc(struct arm_cmn *cmn, struct arm_cmn_node *dn, int idx)
+{
+ struct arm_cmn_dtc *dtc = cmn->dtc + idx;
+ struct arm_cmn_node *xp;
+
+ dtc->base = dn->pmu_base - CMN_PMU_OFFSET;
+ dtc->irq = platform_get_irq(to_platform_device(cmn->dev), idx);
+ if (dtc->irq < 0)
+ return dtc->irq;
+
+ writel_relaxed(0, dtc->base + CMN_DT_PMCR);
+ writel_relaxed(0x1ff, dtc->base + CMN_DT_PMOVSR_CLR);
+ writel_relaxed(CMN_DT_PMCR_OVFL_INTR_EN, dtc->base + CMN_DT_PMCR);
+
+ /* We do at least know that a DTC's XP must be in that DTC's domain */
+ xp = arm_cmn_node_to_xp(dn);
+ xp->dtc = idx;
+
+ return 0;
+}
+
+static int arm_cmn_node_cmp(const void *a, const void *b)
+{
+ const struct arm_cmn_node *dna = a, *dnb = b;
+ int cmp;
+
+ cmp = dna->type - dnb->type;
+ if (!cmp)
+ cmp = dna->logid - dnb->logid;
+ return cmp;
+}
+
+static int arm_cmn_init_dtcs(struct arm_cmn *cmn)
+{
+ struct arm_cmn_node *dn;
+ int dtc_idx = 0;
+
+ cmn->dtc = devm_kcalloc(cmn->dev, cmn->num_dtcs, sizeof(cmn->dtc[0]), GFP_KERNEL);
+ if (!cmn->dtc)
+ return -ENOMEM;
+
+ sort(cmn->dns, cmn->num_dns, sizeof(cmn->dns[0]), arm_cmn_node_cmp, NULL);
+
+ cmn->xps = arm_cmn_node(cmn, CMN_TYPE_XP);
+
+ for (dn = cmn->dns; dn < cmn->dns + cmn->num_dns; dn++) {
+ if (dn->type != CMN_TYPE_XP)
+ arm_cmn_init_node_to_xp(cmn, dn);
+ else if (cmn->num_dtcs == 1)
+ dn->dtc = 0;
+
+ if (dn->type == CMN_TYPE_DTC)
+ arm_cmn_init_dtc(cmn, dn, dtc_idx++);
+
+ /* To the PMU, RN-Ds don't add anything over RN-Is, so smoosh them together */
+ if (dn->type == CMN_TYPE_RND)
+ dn->type = CMN_TYPE_RNI;
+ }
+
+ writel_relaxed(CMN_DT_DTC_CTL_DT_EN, cmn->dtc[0].base + CMN_DT_DTC_CTL);
+
+ return 0;
+}
+
+static void arm_cmn_init_node_info(struct arm_cmn *cmn, u32 offset, struct arm_cmn_node *node)
+{
+ int level;
+ u64 reg = readq_relaxed(cmn->base + offset + CMN_NODE_INFO);
+
+ node->type = FIELD_GET(CMN_NI_NODE_TYPE, reg);
+ node->id = FIELD_GET(CMN_NI_NODE_ID, reg);
+ node->logid = FIELD_GET(CMN_NI_LOGICAL_ID, reg);
+
+ node->pmu_base = cmn->base + offset + CMN_PMU_OFFSET;
+
+ if (node->type == CMN_TYPE_CFG)
+ level = 0;
+ else if (node->type == CMN_TYPE_XP)
+ level = 1;
+ else
+ level = 2;
+
+ dev_dbg(cmn->dev, "node%*c%#06hx%*ctype:%-#6x id:%-4hd off:%#x\n",
+ (level * 2) + 1, ' ', node->id, 5 - (level * 2), ' ',
+ node->type, node->logid, offset);
+}
+
+static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset)
+{
+ void __iomem *cfg_region;
+ struct arm_cmn_node cfg, *dn;
+ u16 child_count, child_poff;
+ u32 xp_offset[CMN_MAX_XPS];
+ u64 reg;
+ int i, j;
+
+ cfg_region = cmn->base + rgn_offset;
+ reg = readl_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_2);
+ cmn->rev = FIELD_GET(CMN_CFGM_PID2_REVISION, reg);
+ dev_dbg(cmn->dev, "periph_id_2 revision: %d\n", cmn->rev);
+
+ arm_cmn_init_node_info(cmn, rgn_offset, &cfg);
+ if (cfg.type != CMN_TYPE_CFG)
+ return -ENODEV;
+
+ reg = readq_relaxed(cfg_region + CMN_CHILD_INFO);
+ child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg);
+ child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg);
+
+ cmn->num_xps = child_count;
+ cmn->num_dns = cmn->num_xps;
+
+ /* Pass 1: visit the XPs, enumerate their children */
+ for (i = 0; i < cmn->num_xps; i++) {
+ reg = readq_relaxed(cfg_region + child_poff + i * 8);
+ xp_offset[i] = reg & CMN_CHILD_NODE_ADDR;
+
+ reg = readq_relaxed(cmn->base + xp_offset[i] + CMN_CHILD_INFO);
+ cmn->num_dns += FIELD_GET(CMN_CI_CHILD_COUNT, reg);
+ }
+
+ /* Cheeky +1 to help terminate pointer-based iteration */
+ cmn->dns = devm_kcalloc(cmn->dev, cmn->num_dns + 1,
+ sizeof(*cmn->dns), GFP_KERNEL);
+ if (!cmn->dns)
+ return -ENOMEM;
+
+ /* Pass 2: now we can actually populate the nodes */
+ dn = cmn->dns;
+ for (i = 0; i < cmn->num_xps; i++) {
+ void __iomem *xp_region = cmn->base + xp_offset[i];
+ struct arm_cmn_node *xp = dn++;
+
+ arm_cmn_init_node_info(cmn, xp_offset[i], xp);
+ arm_cmn_init_dtm(xp);
+ /*
+ * Thanks to the order in which XP logical IDs seem to be
+ * assigned, we can handily infer the mesh X dimension by
+ * looking out for the XP at (0,1) without needing to know
+ * the exact node ID format, which we can later derive.
+ */
+ if (xp->id == (1 << 3))
+ cmn->mesh_x = xp->logid;
+
+ reg = readq_relaxed(xp_region + CMN_CHILD_INFO);
+ child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg);
+ child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg);
+
+ for (j = 0; j < child_count; j++) {
+ reg = readq_relaxed(xp_region + child_poff + j * 8);
+ /*
+ * Don't even try to touch anything external, since in general
+ * we haven't a clue how to power up arbitrary CHI requesters.
+ * As of CMN-600r1 these could only be RN-SAMs or CXLAs,
+ * neither of which have any PMU events anyway.
+ * (Actually, CXLAs do seem to have grown some events in r1p2,
+ * but they don't go to regular XP DTMs, and they depend on
+ * secure configuration which we can't easily deal with)
+ */
+ if (reg & CMN_CHILD_NODE_EXTERNAL) {
+ dev_dbg(cmn->dev, "ignoring external node %llx\n", reg);
+ continue;
+ }
+
+ arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, dn);
+
+ switch (dn->type) {
+ case CMN_TYPE_DTC:
+ cmn->num_dtcs++;
+ dn++;
+ break;
+ /* These guys have PMU events */
+ case CMN_TYPE_DVM:
+ case CMN_TYPE_HNI:
+ case CMN_TYPE_HNF:
+ case CMN_TYPE_SBSX:
+ case CMN_TYPE_RNI:
+ case CMN_TYPE_RND:
+ case CMN_TYPE_CXRA:
+ case CMN_TYPE_CXHA:
+ dn++;
+ break;
+ /* Nothing to see here */
+ case CMN_TYPE_RNSAM:
+ case CMN_TYPE_CXLA:
+ break;
+ /* Something has gone horribly wrong */
+ default:
+ dev_err(cmn->dev, "invalid device node type: 0x%x\n", dn->type);
+ return -ENODEV;
+ }
+ }
+ }
+
+ /* Correct for any nodes we skipped */
+ cmn->num_dns = dn - cmn->dns;
+
+ /*
+ * If mesh_x wasn't set during discovery then we never saw
+ * an XP at (0,1), thus we must have an Nx1 configuration.
+ */
+ if (!cmn->mesh_x)
+ cmn->mesh_x = cmn->num_xps;
+ cmn->mesh_y = cmn->num_xps / cmn->mesh_x;
+
+ dev_dbg(cmn->dev, "mesh %dx%d, ID width %d\n",
+ cmn->mesh_x, cmn->mesh_y, arm_cmn_xyidbits(cmn));
+
+ return 0;
+}
+
+static int arm_cmn_acpi_probe(struct platform_device *pdev, struct arm_cmn *cmn)
+{
+ struct resource *cfg, *root;
+
+ cfg = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!cfg)
+ return -EINVAL;
+
+ root = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ if (!root)
+ return -EINVAL;
+
+ if (!resource_contains(cfg, root))
+ swap(cfg, root);
+ /*
+ * Note that devm_ioremap_resource() is dumb and won't let the platform
+ * device claim cfg when the ACPI companion device has already claimed
+ * root within it. But since they *are* already both claimed in the
+ * appropriate name, we don't really need to do it again here anyway.
+ */
+ cmn->base = devm_ioremap(cmn->dev, cfg->start, resource_size(cfg));
+ if (!cmn->base)
+ return -ENOMEM;
+
+ return root->start - cfg->start;
+}
+
+static int arm_cmn_of_probe(struct platform_device *pdev, struct arm_cmn *cmn)
+{
+ struct device_node *np = pdev->dev.of_node;
+ u32 rootnode;
+ int ret;
+
+ cmn->base = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(cmn->base))
+ return PTR_ERR(cmn->base);
+
+ ret = of_property_read_u32(np, "arm,root-node", &rootnode);
+ if (ret)
+ return ret;
+
+ return rootnode;
+}
+
+static int arm_cmn_probe(struct platform_device *pdev)
+{
+ struct arm_cmn *cmn;
+ const char *name;
+ static atomic_t id;
+ int err, rootnode, this_id;
+
+ cmn = devm_kzalloc(&pdev->dev, sizeof(*cmn), GFP_KERNEL);
+ if (!cmn)
+ return -ENOMEM;
+
+ cmn->dev = &pdev->dev;
+ platform_set_drvdata(pdev, cmn);
+
+ if (has_acpi_companion(cmn->dev))
+ rootnode = arm_cmn_acpi_probe(pdev, cmn);
+ else
+ rootnode = arm_cmn_of_probe(pdev, cmn);
+ if (rootnode < 0)
+ return rootnode;
+
+ err = arm_cmn_discover(cmn, rootnode);
+ if (err)
+ return err;
+
+ err = arm_cmn_init_dtcs(cmn);
+ if (err)
+ return err;
+
+ err = arm_cmn_init_irqs(cmn);
+ if (err)
+ return err;
+
+ cmn->cpu = raw_smp_processor_id();
+ cmn->pmu = (struct pmu) {
+ .module = THIS_MODULE,
+ .attr_groups = arm_cmn_attr_groups,
+ .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
+ .task_ctx_nr = perf_invalid_context,
+ .pmu_enable = arm_cmn_pmu_enable,
+ .pmu_disable = arm_cmn_pmu_disable,
+ .event_init = arm_cmn_event_init,
+ .add = arm_cmn_event_add,
+ .del = arm_cmn_event_del,
+ .start = arm_cmn_event_start,
+ .stop = arm_cmn_event_stop,
+ .read = arm_cmn_event_read,
+ .start_txn = arm_cmn_start_txn,
+ .commit_txn = arm_cmn_commit_txn,
+ .cancel_txn = arm_cmn_end_txn,
+ };
+
+ this_id = atomic_fetch_inc(&id);
+ if (this_id == 0) {
+ name = "arm_cmn";
+ } else {
+ name = devm_kasprintf(cmn->dev, GFP_KERNEL, "arm_cmn_%d", this_id);
+ if (!name)
+ return -ENOMEM;
+ }
+
+ err = cpuhp_state_add_instance(arm_cmn_hp_state, &cmn->cpuhp_node);
+ if (err)
+ return err;
+
+ err = perf_pmu_register(&cmn->pmu, name, -1);
+ if (err)
+ cpuhp_state_remove_instance(arm_cmn_hp_state, &cmn->cpuhp_node);
+ return err;
+}
+
+static int arm_cmn_remove(struct platform_device *pdev)
+{
+ struct arm_cmn *cmn = platform_get_drvdata(pdev);
+ int i;
+
+ writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL);
+
+ perf_pmu_unregister(&cmn->pmu);
+ cpuhp_state_remove_instance(arm_cmn_hp_state, &cmn->cpuhp_node);
+
+ for (i = 0; i < cmn->num_dtcs; i++)
+ irq_set_affinity_hint(cmn->dtc[i].irq, NULL);
+
+ return 0;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id arm_cmn_of_match[] = {
+ { .compatible = "arm,cmn-600", },
+ {}
+};
+MODULE_DEVICE_TABLE(of, arm_cmn_of_match);
+#endif
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id arm_cmn_acpi_match[] = {
+ { "ARMHC600", },
+ {}
+};
+MODULE_DEVICE_TABLE(acpi, arm_cmn_acpi_match);
+#endif
+
+static struct platform_driver arm_cmn_driver = {
+ .driver = {
+ .name = "arm-cmn",
+ .of_match_table = of_match_ptr(arm_cmn_of_match),
+ .acpi_match_table = ACPI_PTR(arm_cmn_acpi_match),
+ },
+ .probe = arm_cmn_probe,
+ .remove = arm_cmn_remove,
+};
+
+static int __init arm_cmn_init(void)
+{
+ int ret;
+
+ ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
+ "perf/arm/cmn:online", NULL,
+ arm_cmn_pmu_offline_cpu);
+ if (ret < 0)
+ return ret;
+
+ arm_cmn_hp_state = ret;
+ ret = platform_driver_register(&arm_cmn_driver);
+ if (ret)
+ cpuhp_remove_multi_state(arm_cmn_hp_state);
+ return ret;
+}
+
+static void __exit arm_cmn_exit(void)
+{
+ platform_driver_unregister(&arm_cmn_driver);
+ cpuhp_remove_multi_state(arm_cmn_hp_state);
+}
+
+module_init(arm_cmn_init);
+module_exit(arm_cmn_exit);
+
+MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");
+MODULE_DESCRIPTION("Arm CMN-600 PMU driver");
+MODULE_LICENSE("GPL v2");
#define DRVNAME PMUNAME "_pmu"
#define pr_fmt(fmt) DRVNAME ": " fmt
+#include <linux/acpi.h>
#include <linux/bitmap.h>
#include <linux/bitops.h>
#include <linux/bug.h>
}
/**
- * dsu_pmu_dt_get_cpus: Get the list of CPUs in the cluster.
+ * dsu_pmu_dt_get_cpus: Get the list of CPUs in the cluster
+ * from device tree.
*/
-static int dsu_pmu_dt_get_cpus(struct device_node *dev, cpumask_t *mask)
+static int dsu_pmu_dt_get_cpus(struct device *dev, cpumask_t *mask)
{
int i = 0, n, cpu;
struct device_node *cpu_node;
- n = of_count_phandle_with_args(dev, "cpus", NULL);
+ n = of_count_phandle_with_args(dev->of_node, "cpus", NULL);
if (n <= 0)
return -ENODEV;
for (; i < n; i++) {
- cpu_node = of_parse_phandle(dev, "cpus", i);
+ cpu_node = of_parse_phandle(dev->of_node, "cpus", i);
if (!cpu_node)
break;
cpu = of_cpu_node_to_id(cpu_node);
return 0;
}
+/**
+ * dsu_pmu_acpi_get_cpus: Get the list of CPUs in the cluster
+ * from ACPI.
+ */
+static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask)
+{
+#ifdef CONFIG_ACPI
+ int cpu;
+
+ /*
+ * A dsu pmu node is inside a cluster parent node along with cpu nodes.
+ * We need to find out all cpus that have the same parent with this pmu.
+ */
+ for_each_possible_cpu(cpu) {
+ struct acpi_device *acpi_dev;
+ struct device *cpu_dev = get_cpu_device(cpu);
+
+ if (!cpu_dev)
+ continue;
+
+ acpi_dev = ACPI_COMPANION(cpu_dev);
+ if (acpi_dev &&
+ acpi_dev->parent == ACPI_COMPANION(dev)->parent)
+ cpumask_set_cpu(cpu, mask);
+ }
+#endif
+
+ return 0;
+}
+
/*
* dsu_pmu_probe_pmu: Probe the PMU details on a CPU in the cluster.
*/
{
int irq, rc;
struct dsu_pmu *dsu_pmu;
+ struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev);
char *name;
static atomic_t pmu_idx = ATOMIC_INIT(-1);
if (IS_ERR(dsu_pmu))
return PTR_ERR(dsu_pmu);
- rc = dsu_pmu_dt_get_cpus(pdev->dev.of_node, &dsu_pmu->associated_cpus);
+ if (IS_ERR_OR_NULL(fwnode))
+ return -ENOENT;
+
+ if (is_of_node(fwnode))
+ rc = dsu_pmu_dt_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);
+ else if (is_acpi_device_node(fwnode))
+ rc = dsu_pmu_acpi_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);
+ else
+ return -ENOENT;
+
if (rc) {
dev_warn(&pdev->dev, "Failed to parse the CPUs\n");
return rc;
{ .compatible = "arm,dsu-pmu", },
{},
};
+MODULE_DEVICE_TABLE(of, dsu_pmu_of_match);
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id dsu_pmu_acpi_match[] = {
+ { "ARMHD500", 0},
+ {},
+};
+MODULE_DEVICE_TABLE(acpi, dsu_pmu_acpi_match);
+#endif
static struct platform_driver dsu_pmu_driver = {
.driver = {
.name = DRVNAME,
.of_match_table = of_match_ptr(dsu_pmu_of_match),
+ .acpi_match_table = ACPI_PTR(dsu_pmu_acpi_match),
.suppress_bind_attrs = true,
},
.probe = dsu_pmu_device_probe,
module_init(dsu_pmu_init);
module_exit(dsu_pmu_exit);
-MODULE_DEVICE_TABLE(of, dsu_pmu_of_match);
MODULE_DESCRIPTION("Perf driver for ARM DynamIQ Shared Unit");
MODULE_AUTHOR("Suzuki K Poulose <suzuki.poulose@arm.com>");
MODULE_LICENSE("GPL v2");
#include <asm/irq_regs.h>
+static int armpmu_count_irq_users(const int irq);
+
+struct pmu_irq_ops {
+ void (*enable_pmuirq)(unsigned int irq);
+ void (*disable_pmuirq)(unsigned int irq);
+ void (*free_pmuirq)(unsigned int irq, int cpu, void __percpu *devid);
+};
+
+static void armpmu_free_pmuirq(unsigned int irq, int cpu, void __percpu *devid)
+{
+ free_irq(irq, per_cpu_ptr(devid, cpu));
+}
+
+static const struct pmu_irq_ops pmuirq_ops = {
+ .enable_pmuirq = enable_irq,
+ .disable_pmuirq = disable_irq_nosync,
+ .free_pmuirq = armpmu_free_pmuirq
+};
+
+static void armpmu_free_pmunmi(unsigned int irq, int cpu, void __percpu *devid)
+{
+ free_nmi(irq, per_cpu_ptr(devid, cpu));
+}
+
+static const struct pmu_irq_ops pmunmi_ops = {
+ .enable_pmuirq = enable_nmi,
+ .disable_pmuirq = disable_nmi_nosync,
+ .free_pmuirq = armpmu_free_pmunmi
+};
+
+static void armpmu_enable_percpu_pmuirq(unsigned int irq)
+{
+ enable_percpu_irq(irq, IRQ_TYPE_NONE);
+}
+
+static void armpmu_free_percpu_pmuirq(unsigned int irq, int cpu,
+ void __percpu *devid)
+{
+ if (armpmu_count_irq_users(irq) == 1)
+ free_percpu_irq(irq, devid);
+}
+
+static const struct pmu_irq_ops percpu_pmuirq_ops = {
+ .enable_pmuirq = armpmu_enable_percpu_pmuirq,
+ .disable_pmuirq = disable_percpu_irq,
+ .free_pmuirq = armpmu_free_percpu_pmuirq
+};
+
+static void armpmu_enable_percpu_pmunmi(unsigned int irq)
+{
+ if (!prepare_percpu_nmi(irq))
+ enable_percpu_nmi(irq, IRQ_TYPE_NONE);
+}
+
+static void armpmu_disable_percpu_pmunmi(unsigned int irq)
+{
+ disable_percpu_nmi(irq);
+ teardown_percpu_nmi(irq);
+}
+
+static void armpmu_free_percpu_pmunmi(unsigned int irq, int cpu,
+ void __percpu *devid)
+{
+ if (armpmu_count_irq_users(irq) == 1)
+ free_percpu_nmi(irq, devid);
+}
+
+static const struct pmu_irq_ops percpu_pmunmi_ops = {
+ .enable_pmuirq = armpmu_enable_percpu_pmunmi,
+ .disable_pmuirq = armpmu_disable_percpu_pmunmi,
+ .free_pmuirq = armpmu_free_percpu_pmunmi
+};
+
static DEFINE_PER_CPU(struct arm_pmu *, cpu_armpmu);
static DEFINE_PER_CPU(int, cpu_irq);
+static DEFINE_PER_CPU(const struct pmu_irq_ops *, cpu_irq_ops);
+
+static bool has_nmi;
static inline u64 arm_pmu_event_max_period(struct perf_event *event)
{
return count;
}
+static const struct pmu_irq_ops *armpmu_find_irq_ops(int irq)
+{
+ const struct pmu_irq_ops *ops = NULL;
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ if (per_cpu(cpu_irq, cpu) != irq)
+ continue;
+
+ ops = per_cpu(cpu_irq_ops, cpu);
+ if (ops)
+ break;
+ }
+
+ return ops;
+}
+
void armpmu_free_irq(int irq, int cpu)
{
if (per_cpu(cpu_irq, cpu) == 0)
if (WARN_ON(irq != per_cpu(cpu_irq, cpu)))
return;
- if (!irq_is_percpu_devid(irq))
- free_irq(irq, per_cpu_ptr(&cpu_armpmu, cpu));
- else if (armpmu_count_irq_users(irq) == 1)
- free_percpu_irq(irq, &cpu_armpmu);
+ per_cpu(cpu_irq_ops, cpu)->free_pmuirq(irq, cpu, &cpu_armpmu);
per_cpu(cpu_irq, cpu) = 0;
+ per_cpu(cpu_irq_ops, cpu) = NULL;
}
int armpmu_request_irq(int irq, int cpu)
{
int err = 0;
const irq_handler_t handler = armpmu_dispatch_irq;
+ const struct pmu_irq_ops *irq_ops;
+
if (!irq)
return 0;
IRQF_NO_THREAD;
irq_set_status_flags(irq, IRQ_NOAUTOEN);
- err = request_irq(irq, handler, irq_flags, "arm-pmu",
+
+ err = request_nmi(irq, handler, irq_flags, "arm-pmu",
per_cpu_ptr(&cpu_armpmu, cpu));
+
+ /* If cannot get an NMI, get a normal interrupt */
+ if (err) {
+ err = request_irq(irq, handler, irq_flags, "arm-pmu",
+ per_cpu_ptr(&cpu_armpmu, cpu));
+ irq_ops = &pmuirq_ops;
+ } else {
+ has_nmi = true;
+ irq_ops = &pmunmi_ops;
+ }
} else if (armpmu_count_irq_users(irq) == 0) {
- err = request_percpu_irq(irq, handler, "arm-pmu",
- &cpu_armpmu);
+ err = request_percpu_nmi(irq, handler, "arm-pmu", &cpu_armpmu);
+
+ /* If cannot get an NMI, get a normal interrupt */
+ if (err) {
+ err = request_percpu_irq(irq, handler, "arm-pmu",
+ &cpu_armpmu);
+ irq_ops = &percpu_pmuirq_ops;
+ } else {
+ has_nmi= true;
+ irq_ops = &percpu_pmunmi_ops;
+ }
+ } else {
+ /* Per cpudevid irq was already requested by another CPU */
+ irq_ops = armpmu_find_irq_ops(irq);
+
+ if (WARN_ON(!irq_ops))
+ err = -EINVAL;
}
if (err)
goto err_out;
per_cpu(cpu_irq, cpu) = irq;
+ per_cpu(cpu_irq_ops, cpu) = irq_ops;
return 0;
err_out:
per_cpu(cpu_armpmu, cpu) = pmu;
irq = armpmu_get_cpu_irq(pmu, cpu);
- if (irq) {
- if (irq_is_percpu_devid(irq))
- enable_percpu_irq(irq, IRQ_TYPE_NONE);
- else
- enable_irq(irq);
- }
+ if (irq)
+ per_cpu(cpu_irq_ops, cpu)->enable_pmuirq(irq);
return 0;
}
return 0;
irq = armpmu_get_cpu_irq(pmu, cpu);
- if (irq) {
- if (irq_is_percpu_devid(irq))
- disable_percpu_irq(irq);
- else
- disable_irq_nosync(irq);
- }
+ if (irq)
+ per_cpu(cpu_irq_ops, cpu)->disable_pmuirq(irq);
per_cpu(cpu_armpmu, cpu) = NULL;
if (!__oprofile_cpu_pmu)
__oprofile_cpu_pmu = pmu;
- pr_info("enabled with %s PMU driver, %d counters available\n",
- pmu->name, pmu->num_events);
+ pr_info("enabled with %s PMU driver, %d counters available%s\n",
+ pmu->name, pmu->num_events,
+ has_nmi ? ", using NMIs" : "");
return 0;
#include <linux/cpumask.h>
#include <linux/device.h>
#include <linux/kernel.h>
+#include <linux/module.h>
#include <linux/perf_event.h>
#include <linux/types.h>
list_for_each_entry(rentry, &list, node) {
if (resource_type(rentry->res) == IORESOURCE_MEM) {
res = *rentry->res;
+ rentry = NULL;
break;
}
}
+ acpi_dev_free_resource_list(&list);
- if (!rentry->res)
+ if (rentry) {
+ dev_err(dev, "PMU type %d: Fail to find resource\n", type);
return NULL;
+ }
- acpi_dev_free_resource_list(&list);
base = devm_ioremap_resource(dev, &res);
if (IS_ERR(base)) {
dev_err(dev, "PMU type %d: Fail to map resource\n", type);
}
#if defined(CONFIG_ACPI)
-static int acpi_pmu_dev_add_resource(struct acpi_resource *ares, void *data)
-{
- struct resource *res = data;
-
- if (ares->type == ACPI_RESOURCE_TYPE_FIXED_MEMORY32)
- acpi_dev_resource_memory(ares, res);
-
- /* Always tell the ACPI core to skip this resource */
- return 1;
-}
-
static struct
xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu,
struct acpi_device *adev, u32 type)
struct hw_pmu_info *inf;
void __iomem *dev_csr;
struct resource res;
+ struct resource_entry *rentry;
int enable_bit;
int rc;
return NULL;
INIT_LIST_HEAD(&resource_list);
- rc = acpi_dev_get_resources(adev, &resource_list,
- acpi_pmu_dev_add_resource, &res);
+ rc = acpi_dev_get_resources(adev, &resource_list, NULL, NULL);
+ if (rc <= 0) {
+ dev_err(dev, "PMU type %d: No resources found\n", type);
+ return NULL;
+ }
+
+ list_for_each_entry(rentry, &resource_list, node) {
+ if (resource_type(rentry->res) == IORESOURCE_MEM) {
+ res = *rentry->res;
+ rentry = NULL;
+ break;
+ }
+ }
acpi_dev_free_resource_list(&resource_list);
- if (rc < 0) {
- dev_err(dev, "PMU type %d: No resource address found\n", type);
+
+ if (rentry) {
+ dev_err(dev, "PMU type %d: No memory resource found\n", type);
return NULL;
}
pm_runtime_enable(dev);
phy = devm_phy_create(dev, NULL, &ops);
- if (IS_ERR(phy))
- return PTR_ERR(phy);
+ if (IS_ERR(phy)) {
+ ret = PTR_ERR(phy);
+ goto clk_err;
+ }
phy_set_drvdata(phy, am654_phy);
phy_provider = devm_of_phy_provider_register(dev, serdes_am654_xlate);
select PINMUX
select PINCONF
select GENERIC_PINCONF
+ select GPIOLIB
select GPIOLIB_IRQCHIP
default ARCH_BCM2835 || ARCH_BRCMSTB
help
#define CHV_PADCTRL1_CFGLOCK BIT(31)
#define CHV_PADCTRL1_INVRXTX_SHIFT 4
#define CHV_PADCTRL1_INVRXTX_MASK GENMASK(7, 4)
+#define CHV_PADCTRL1_INVRXTX_TXDATA BIT(7)
#define CHV_PADCTRL1_INVRXTX_RXDATA BIT(6)
#define CHV_PADCTRL1_INVRXTX_TXENABLE BIT(5)
#define CHV_PADCTRL1_ODEN BIT(3)
static void chv_gpio_clear_triggering(struct chv_pinctrl *pctrl,
unsigned int offset)
{
+ u32 invrxtx_mask = CHV_PADCTRL1_INVRXTX_MASK;
u32 value;
+ /*
+ * One some devices the GPIO should output the inverted value from what
+ * device-drivers / ACPI code expects (inverted external buffer?). The
+ * BIOS makes this work by setting the CHV_PADCTRL1_INVRXTX_TXDATA flag,
+ * preserve this flag if the pin is already setup as GPIO.
+ */
+ value = chv_readl(pctrl, offset, CHV_PADCTRL0);
+ if (value & CHV_PADCTRL0_GPIOEN)
+ invrxtx_mask &= ~CHV_PADCTRL1_INVRXTX_TXDATA;
+
value = chv_readl(pctrl, offset, CHV_PADCTRL1);
value &= ~CHV_PADCTRL1_INTWAKECFG_MASK;
- value &= ~CHV_PADCTRL1_INVRXTX_MASK;
+ value &= ~invrxtx_mask;
chv_writel(pctrl, offset, CHV_PADCTRL1, value);
}
desc = (const struct mtk_pin_desc *)&hw->soc->pins[gpio_n];
+ /* if the GPIO is not supported for eint mode */
+ if (desc->eint.eint_m == NO_EINT_SUPPORT)
+ return virt_gpio;
+
if (desc->funcs && !desc->funcs[desc->eint.eint_m].name)
virt_gpio = true;
MPP_VAR_FUNCTION(0x1, "i2c0", "sck", V_98DX3236_PLUS)),
MPP_MODE(15,
MPP_VAR_FUNCTION(0x0, "gpio", NULL, V_98DX3236_PLUS),
- MPP_VAR_FUNCTION(0x4, "i2c0", "sda", V_98DX3236_PLUS)),
+ MPP_VAR_FUNCTION(0x1, "i2c0", "sda", V_98DX3236_PLUS)),
MPP_MODE(16,
MPP_VAR_FUNCTION(0x0, "gpo", NULL, V_98DX3236_PLUS),
MPP_VAR_FUNCTION(0x4, "dev", "oe", V_98DX3236_PLUS)),
* when TLMM is powered on. To allow that, enable the GPIO
* summary line to be wakeup capable at GIC.
*/
- if (d->parent_data)
- irq_chip_set_wake_parent(d, on);
-
- irq_set_irq_wake(pctrl->irq, on);
+ if (d->parent_data && test_bit(d->hwirq, pctrl->skip_wake_irqs))
+ return irq_chip_set_wake_parent(d, on);
- return 0;
+ return irq_set_irq_wake(pctrl->irq, on);
}
static int msm_gpio_irq_reqres(struct irq_data *d)
pctrl->irq_chip.irq_release_resources = msm_gpio_irq_relres;
pctrl->irq_chip.irq_set_affinity = msm_gpio_irq_set_affinity;
pctrl->irq_chip.irq_set_vcpu_affinity = msm_gpio_irq_set_vcpu_affinity;
+ pctrl->irq_chip.flags = IRQCHIP_MASK_ON_SUSPEND |
+ IRQCHIP_SET_TYPE_MASKED |
+ IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND;
np = of_parse_phandle(pctrl->dev->of_node, "wakeup-parent", 0);
if (np) {
[178] = PINGROUP(178, WEST, _, _, _, _, _, _, _, _, _),
[179] = PINGROUP(179, WEST, _, _, _, _, _, _, _, _, _),
[180] = UFS_RESET(ufs_reset, 0xb8000),
- [181] = SDC_PINGROUP(sdc2_clk, 0x7000, 14, 6),
+ [181] = SDC_PINGROUP(sdc2_clk, 0xb7000, 14, 6),
[182] = SDC_PINGROUP(sdc2_cmd, 0xb7000, 11, 3),
[183] = SDC_PINGROUP(sdc2_data, 0xb7000, 9, 0),
};
&config);
if (IS_ERR(ec->dcon_rdev)) {
dev_err(&pdev->dev, "failed to register DCON regulator\n");
- return PTR_ERR(ec->dcon_rdev);
+ err = PTR_ERR(ec->dcon_rdev);
+ kfree(ec);
+ return err;
}
ec->dbgfs_dir = olpc_ec_setup_debugfs();
depends on BACKLIGHT_CLASS_DEVICE
depends on ACPI_VIDEO || ACPI_VIDEO = n
select INPUT_SPARSEKMAP
+ select NEW_LEDS
select LEDS_CLASS
help
This is a driver for laptops built by Fujitsu:
depends on ACPI_WMI
depends on INPUT
select INPUT_SPARSEKMAP
+ select NEW_LEDS
select LEDS_CLASS
help
This driver adds support for hotkeys as well as control of keyboard
.wmi_backlight_set_devstate = true,
};
+static struct quirk_entry quirk_asus_use_kbd_dock_devid = {
+ .use_kbd_dock_devid = true,
+};
+
static int dmi_matched(const struct dmi_system_id *dmi)
{
pr_info("Identified laptop model '%s'\n", dmi->ident);
},
.driver_data = &quirk_asus_vendor_backlight,
},
+ {
+ .callback = dmi_matched,
+ .ident = "Asus Transformer T100TA / T100HA / T100CHI",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+ /* Match *T100* */
+ DMI_MATCH(DMI_PRODUCT_NAME, "T100"),
+ },
+ .driver_data = &quirk_asus_use_kbd_dock_devid,
+ },
+ {
+ .callback = dmi_matched,
+ .ident = "Asus Transformer T101HA",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+ DMI_MATCH(DMI_PRODUCT_NAME, "T101HA"),
+ },
+ .driver_data = &quirk_asus_use_kbd_dock_devid,
+ },
+ {
+ .callback = dmi_matched,
+ .ident = "Asus Transformer T200TA",
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+ DMI_MATCH(DMI_PRODUCT_NAME, "T200TA"),
+ },
+ .driver_data = &quirk_asus_use_kbd_dock_devid,
+ },
{},
};
.detect_quirks = asus_nb_wmi_quirks,
};
-static const struct dmi_system_id asus_nb_wmi_blacklist[] __initconst = {
- {
- /*
- * asus-nb-wm adds no functionality. The T100TA has a detachable
- * USB kbd, so no hotkeys and it has no WMI rfkill; and loading
- * asus-nb-wm causes the camera LED to turn and _stay_ on.
- */
- .matches = {
- DMI_EXACT_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
- DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "T100TA"),
- },
- },
- {
- /* The Asus T200TA has the same issue as the T100TA */
- .matches = {
- DMI_EXACT_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
- DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "T200TA"),
- },
- },
- {} /* Terminating entry */
-};
static int __init asus_nb_wmi_init(void)
{
- if (dmi_check_system(asus_nb_wmi_blacklist))
- return -ENODEV;
-
return asus_wmi_register_driver(&asus_nb_wmi_driver);
}
if (err)
goto err_free_dev;
- result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_KBD_DOCK);
- if (result >= 0) {
- input_set_capability(asus->inputdev, EV_SW, SW_TABLET_MODE);
- input_report_switch(asus->inputdev, SW_TABLET_MODE, !result);
- } else if (result != -ENODEV) {
- pr_err("Error checking for keyboard-dock: %d\n", result);
+ if (asus->driver->quirks->use_kbd_dock_devid) {
+ result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_KBD_DOCK);
+ if (result >= 0) {
+ input_set_capability(asus->inputdev, EV_SW, SW_TABLET_MODE);
+ input_report_switch(asus->inputdev, SW_TABLET_MODE, !result);
+ } else if (result != -ENODEV) {
+ pr_err("Error checking for keyboard-dock: %d\n", result);
+ }
}
err = input_register_device(asus->inputdev);
*/
if (strcmp(battery->desc->name, "BAT0") != 0 &&
strcmp(battery->desc->name, "BAT1") != 0 &&
+ strcmp(battery->desc->name, "BATC") != 0 &&
strcmp(battery->desc->name, "BATT") != 0)
return -ENODEV;
return;
}
- if (code == NOTIFY_KBD_DOCK_CHANGE) {
+ if (asus->driver->quirks->use_kbd_dock_devid && code == NOTIFY_KBD_DOCK_CHANGE) {
result = asus_wmi_get_devstate_simple(asus,
ASUS_WMI_DEVID_KBD_DOCK);
if (result >= 0) {
bool wmi_backlight_native;
bool wmi_backlight_set_devstate;
bool wmi_force_als_set;
+ bool use_kbd_dock_devid;
int wapf;
/*
* For machines with AMD graphic chips, it will send out WMI event
return ACPI_SUCCESS(status);
}
+/*
+ * There are several laptops (non 2-in-1) models out there which support VGBS,
+ * but simply always return 0, which we translate to SW_TABLET_MODE=1. This in
+ * turn causes userspace (libinput) to suppress events from the builtin
+ * keyboard and touchpad, making the laptop essentially unusable.
+ *
+ * Since the problem of wrongly reporting SW_TABLET_MODE=1 in combination
+ * with libinput, leads to a non-usable system. Where as OTOH many people will
+ * not even notice when SW_TABLET_MODE is not being reported, a DMI based allow
+ * list is used here. This list mainly matches on the chassis-type of 2-in-1s.
+ *
+ * There are also some 2-in-1s which use the intel-vbtn ACPI interface to report
+ * SW_TABLET_MODE with a chassis-type of 8 ("Portable") or 10 ("Notebook"),
+ * these are matched on a per model basis, since many normal laptops with a
+ * possible broken VGBS ACPI-method also use these chassis-types.
+ */
+static const struct dmi_system_id dmi_switches_allow_list[] = {
+ {
+ .matches = {
+ DMI_EXACT_MATCH(DMI_CHASSIS_TYPE, "31" /* Convertible */),
+ },
+ },
+ {
+ .matches = {
+ DMI_EXACT_MATCH(DMI_CHASSIS_TYPE, "32" /* Detachable */),
+ },
+ },
+ {
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Venue 11 Pro 7130"),
+ },
+ },
+ {
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "HP Stream x360 Convertible PC 11"),
+ },
+ },
+ {} /* Array terminator */
+};
+
static bool intel_vbtn_has_switches(acpi_handle handle)
{
- const char *chassis_type = dmi_get_system_info(DMI_CHASSIS_TYPE);
unsigned long long vgbs;
acpi_status status;
- /*
- * Some normal laptops have a VGBS method despite being non-convertible
- * and their VGBS method always returns 0, causing detect_tablet_mode()
- * to report SW_TABLET_MODE=1 to userspace, which causes issues.
- * These laptops have a DMI chassis_type of 9 ("Laptop"), do not report
- * switches on any devices with a DMI chassis_type of 9.
- */
- if (chassis_type && strcmp(chassis_type, "9") == 0)
+ if (!dmi_check_system(dmi_switches_allow_list))
return false;
status = acpi_evaluate_integer(handle, "VGBS", NULL, &vgbs);
static void intel_pmc_core_release(struct device *dev)
{
- /* Nothing to do. */
+ kfree(dev);
}
-static struct platform_device pmc_core_device = {
- .name = "intel_pmc_core",
- .dev = {
- .release = intel_pmc_core_release,
- },
-};
+static struct platform_device *pmc_core_device;
/*
* intel_pmc_core_platform_ids is the list of platforms where we want to
static int __init pmc_core_platform_init(void)
{
+ int retval;
+
/* Skip creating the platform device if ACPI already has a device */
if (acpi_dev_present("INT33A1", NULL, -1))
return -ENODEV;
if (!x86_match_cpu(intel_pmc_core_platform_ids))
return -ENODEV;
- return platform_device_register(&pmc_core_device);
+ pmc_core_device = kzalloc(sizeof(*pmc_core_device), GFP_KERNEL);
+ if (!pmc_core_device)
+ return -ENOMEM;
+
+ pmc_core_device->name = "intel_pmc_core";
+ pmc_core_device->dev.release = intel_pmc_core_release;
+
+ retval = platform_device_register(pmc_core_device);
+ if (retval)
+ kfree(pmc_core_device);
+
+ return retval;
}
static void __exit pmc_core_platform_exit(void)
{
- platform_device_unregister(&pmc_core_device);
+ platform_device_unregister(pmc_core_device);
}
module_init(pmc_core_platform_init);
#define MLXPLAT_CPLD_NR_NONE -1
#define MLXPLAT_CPLD_PSU_DEFAULT_NR 10
#define MLXPLAT_CPLD_PSU_MSNXXXX_NR 4
-#define MLXPLAT_CPLD_PSU_MSNXXXX_NR2 3
#define MLXPLAT_CPLD_FAN1_DEFAULT_NR 11
#define MLXPLAT_CPLD_FAN2_DEFAULT_NR 12
#define MLXPLAT_CPLD_FAN3_DEFAULT_NR 13
},
};
+static struct i2c_board_info mlxplat_mlxcpld_ext_pwr[] = {
+ {
+ I2C_BOARD_INFO("dps460", 0x5b),
+ },
+ {
+ I2C_BOARD_INFO("dps460", 0x5a),
+ },
+};
+
static struct i2c_board_info mlxplat_mlxcpld_fan[] = {
{
I2C_BOARD_INFO("24c32", 0x50),
.label = "pwr3",
.reg = MLXPLAT_CPLD_LPC_REG_PWR_OFFSET,
.mask = BIT(2),
- .hpdev.brdinfo = &mlxplat_mlxcpld_pwr[0],
- .hpdev.nr = MLXPLAT_CPLD_PSU_MSNXXXX_NR2,
+ .hpdev.brdinfo = &mlxplat_mlxcpld_ext_pwr[0],
+ .hpdev.nr = MLXPLAT_CPLD_PSU_MSNXXXX_NR,
},
{
.label = "pwr4",
.reg = MLXPLAT_CPLD_LPC_REG_PWR_OFFSET,
.mask = BIT(3),
- .hpdev.brdinfo = &mlxplat_mlxcpld_pwr[1],
- .hpdev.nr = MLXPLAT_CPLD_PSU_MSNXXXX_NR2,
+ .hpdev.brdinfo = &mlxplat_mlxcpld_ext_pwr[1],
+ .hpdev.nr = MLXPLAT_CPLD_PSU_MSNXXXX_NR,
},
};
#define APU2_GPIO_REG_LED3 AMD_FCH_GPIO_REG_GPIO59_DEVSLP1
#define APU2_GPIO_REG_MODESW AMD_FCH_GPIO_REG_GPIO32_GE1
#define APU2_GPIO_REG_SIMSWAP AMD_FCH_GPIO_REG_GPIO33_GE2
-#define APU2_GPIO_REG_MPCIE2 AMD_FCH_GPIO_REG_GPIO59_DEVSLP0
+#define APU2_GPIO_REG_MPCIE2 AMD_FCH_GPIO_REG_GPIO55_DEVSLP0
#define APU2_GPIO_REG_MPCIE3 AMD_FCH_GPIO_REG_GPIO51
/* Order in which the GPIO lines are defined in the register list */
*/
static int hotkey_kthread(void *data)
{
- struct tp_nvram_state s[2];
+ struct tp_nvram_state s[2] = { 0 };
u32 poll_mask, event_mask;
unsigned int si, so;
unsigned long t;
list_for_each_entry(child, &device->children, node) {
acpi_status status = acpi_evaluate_object(child->handle, "_BCL",
NULL, &buffer);
- if (ACPI_FAILURE(status))
+ if (ACPI_FAILURE(status)) {
+ buffer.length = ACPI_ALLOCATE_BUFFER;
continue;
+ }
obj = (union acpi_object *)buffer.pointer;
if (!obj || (obj->type != ACPI_TYPE_PACKAGE)) {
.properties = jumper_ezpad_mini3_props,
};
+static const struct property_entry mpman_converter9_props[] = {
+ PROPERTY_ENTRY_U32("touchscreen-min-x", 8),
+ PROPERTY_ENTRY_U32("touchscreen-min-y", 8),
+ PROPERTY_ENTRY_U32("touchscreen-size-x", 1664),
+ PROPERTY_ENTRY_U32("touchscreen-size-y", 880),
+ PROPERTY_ENTRY_BOOL("touchscreen-inverted-y"),
+ PROPERTY_ENTRY_BOOL("touchscreen-swapped-x-y"),
+ PROPERTY_ENTRY_STRING("firmware-name", "gsl1680-mpman-converter9.fw"),
+ PROPERTY_ENTRY_U32("silead,max-fingers", 10),
+ { }
+};
+
+static const struct ts_dmi_data mpman_converter9_data = {
+ .acpi_name = "MSSL1680:00",
+ .properties = mpman_converter9_props,
+};
+
static const struct property_entry mpman_mpwin895cl_props[] = {
PROPERTY_ENTRY_U32("touchscreen-min-x", 3),
PROPERTY_ENTRY_U32("touchscreen-min-y", 9),
DMI_MATCH(DMI_PRODUCT_NAME, "FlexBook edge11 - M-FBE11"),
},
},
+ {
+ /* MP Man Converter 9 */
+ .driver_data = (void *)&mpman_converter9_data,
+ .matches = {
+ DMI_MATCH(DMI_SYS_VENDOR, "MPMAN"),
+ DMI_MATCH(DMI_PRODUCT_NAME, "Converter9"),
+ },
+ },
{
/* MP Man MPWIN895CL */
.driver_data = (void *)&mpman_mpwin895cl_data,
else
client->flags &= ~I2C_CLIENT_PEC;
+ if (of_device_is_compatible(client->dev.parent->of_node, "google,cros-ec-i2c-tunnel")
+ && client->flags & I2C_CLIENT_PEC) {
+ dev_info(&client->dev, "Disabling PEC because of broken Cros-EC implementation\n");
+ client->flags &= ~I2C_CLIENT_PEC;
+ }
+
dev_dbg(&client->dev, "PEC: %s\n", (client->flags & I2C_CLIENT_PEC) ?
"enabled" : "disabled");
static const char * const bins[] = { "00", "01", "10", "11" };
-static int array_dump(struct seq_file *m, void *v)
+static int array_show(struct seq_file *m, void *v)
{
struct ce_array *ca = &ce_arr;
int i;
return 0;
}
-static int array_open(struct inode *inode, struct file *filp)
-{
- return single_open(filp, array_dump, NULL);
-}
-
-static const struct file_operations array_ops = {
- .owner = THIS_MODULE,
- .open = array_open,
- .read = seq_read,
- .llseek = seq_lseek,
- .release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(array);
static int __init create_debugfs_nodes(void)
{
goto err;
}
- array = debugfs_create_file("array", S_IRUSR, d, NULL, &array_ops);
+ array = debugfs_create_file("array", S_IRUSR, d, NULL, &array_fops);
if (!array) {
pr_warn("Error creating array debugfs node!\n");
goto err;
.priority = MCE_PRIO_CEC,
};
-static void __init cec_init(void)
+static int __init cec_init(void)
{
if (ce_arr.disabled)
- return;
+ return -ENODEV;
ce_arr.array = (void *)get_zeroed_page(GFP_KERNEL);
if (!ce_arr.array) {
pr_err("Error allocating CE array page!\n");
- return;
+ return -ENOMEM;
}
if (create_debugfs_nodes()) {
free_page((unsigned long)ce_arr.array);
- return;
+ return -ENOMEM;
}
INIT_DELAYED_WORK(&cec_work, cec_work_fn);
mce_register_decode_chain(&cec_nb);
pr_info("Correctable Errors collector initialized.\n");
+ return 0;
}
late_initcall(cec_init);
return 0;
}
-static const struct of_device_id pg86x_dt_ids [] = {
+static const struct of_device_id __maybe_unused pg86x_dt_ids[] = {
{ .compatible = "marvell,88pg867" },
{ .compatible = "marvell,88pg868" },
{ }
This driver can also be built as a module. If so, the module
will be called bd9571mwv-regulator.
+config REGULATOR_BD957XMUF
+ tristate "ROHM BD9576MUF and BD9573MUF Regulators"
+ depends on MFD_ROHM_BD957XMUF
+ help
+ This driver supports voltage regulators on ROHM BD9576MUF and
+ BD9573MUF PMICs.
+
+ This driver can also be built as a module. If so, the module
+ will be called bd9576-regulator.
+
config REGULATOR_CPCAP
tristate "Motorola CPCAP regulator"
depends on MFD_CPCAP
config REGULATOR_MAX77620
tristate "Maxim 77620/MAX20024 voltage regulator"
- depends on MFD_MAX77620
+ depends on MFD_MAX77620 || COMPILE_TEST
help
This driver controls Maxim MAX77620 voltage output regulator
via I2C bus. The provided regulator is suitable for Tegra
config REGULATOR_MAX77650
tristate "Maxim MAX77650/77651 regulator support"
- depends on MFD_MAX77650
+ depends on MFD_MAX77650 || COMPILE_TEST
help
Regulator driver for MAX77650/77651 PMIC from Maxim
Semiconductor. This device has a SIMO with three independent
config REGULATOR_MAX8907
tristate "Maxim 8907 voltage regulator"
- depends on MFD_MAX8907
+ depends on MFD_MAX8907 || COMPILE_TEST
help
This driver controls a Maxim 8907 voltage output regulator
via I2C bus. The provided regulator is suitable for Tegra
config REGULATOR_MAX77686
tristate "Maxim 77686 regulator"
- depends on MFD_MAX77686
+ depends on MFD_MAX77686 || COMPILE_TEST
help
This driver controls a Maxim 77686 regulator
via I2C bus. The provided regulator is suitable for
config REGULATOR_MAX77693
tristate "Maxim 77693/77843 regulator"
- depends on (MFD_MAX77693 || MFD_MAX77843)
+ depends on MFD_MAX77693 || MFD_MAX77843 || COMPILE_TEST
help
This driver controls a Maxim 77693/77843 regulators via I2C bus.
The regulators include two LDOs, 'SAFEOUT1', 'SAFEOUT2'
config REGULATOR_MAX77802
tristate "Maxim 77802 regulator"
- depends on MFD_MAX77686
+ depends on MFD_MAX77686 || COMPILE_TEST
help
This driver controls a Maxim 77802 regulator
via I2C bus. The provided regulator is suitable for
This driver supports the control of different power rails of device
through regulator interface.
+config REGULATOR_MT6360
+ tristate "MT6360 SubPMIC Regulator"
+ depends on MFD_MT6360
+ help
+ Say Y here to enable MT6360 regulator support.
+ This is support MT6360 PMIC/LDO part include
+ 2-channel buck with Thermal Shutdown and Overload Protection
+ 6-channel High PSRR and Low Dropout LDO.
+
config REGULATOR_MT6380
tristate "MediaTek MT6380 PMIC"
depends on MTK_PMIC_WRAP
Say M here if you want to include support for enabling the VBUS output
as a module. The module will be named "qcom_usb_vbus_regulator".
+config REGULATOR_RASPBERRYPI_TOUCHSCREEN_ATTINY
+ tristate "Raspberry Pi 7-inch touchscreen panel ATTINY regulator"
+ depends on BACKLIGHT_CLASS_DEVICE
+ depends on I2C
+ select REGMAP_I2C
+ help
+ This driver supports ATTINY regulator on the Raspberry Pi 7-inch
+ touchscreen unit. The regulator is used to enable power to the
+ TC358762, display and to control backlight.
+
config REGULATOR_RC5T583
tristate "RICOH RC5T583 Power regulators"
depends on MFD_RC5T583
config REGULATOR_ROHM
tristate
+config REGULATOR_RT4801
+ tristate "Richtek RT4801 Regulators"
+ depends on I2C
+ select REGMAP_I2C
+ help
+ This adds support for voltage regulators in Richtek RT4801 Display Bias IC.
+ The device supports two regulators (DSVP/DSVN).
+
config REGULATOR_RT5033
tristate "Richtek RT5033 Regulators"
depends on MFD_RT5033
RT5033 PMIC. The device supports multiple regulators like
current source, LDO and Buck.
+config REGULATOR_RTMV20
+ tristate "RTMV20 Laser Diode Regulator"
+ depends on I2C
+ select REGMAP_I2C
+ help
+ This driver adds support for the load switch current regulator on
+ the Richtek RTMV20. It can support the load current up to 6A and
+ integrate strobe/vsync/fsin signal to synchronize the IR camera.
+
config REGULATOR_S2MPA01
tristate "Samsung S2MPA01 voltage regulator"
- depends on MFD_SEC_CORE
+ depends on MFD_SEC_CORE || COMPILE_TEST
help
This driver controls Samsung S2MPA01 voltage output regulator
via I2C bus. S2MPA01 has 10 Bucks and 26 LDO outputs.
config REGULATOR_S2MPS11
tristate "Samsung S2MPS11/13/14/15/S2MPU02 voltage regulator"
- depends on MFD_SEC_CORE
+ depends on MFD_SEC_CORE || COMPILE_TEST
help
This driver supports a Samsung S2MPS11/13/14/15/S2MPU02 voltage
output regulator via I2C bus. The chip is comprised of high efficient
config REGULATOR_S5M8767
tristate "Samsung S5M8767A voltage regulator"
- depends on MFD_SEC_CORE
+ depends on MFD_SEC_CORE || COMPILE_TEST
help
This driver supports a Samsung S5M8767A voltage output regulator
via I2C bus. S5M8767A have 9 Bucks and 28 LDOs output and
obj-$(CONFIG_REGULATOR_BD71828) += bd71828-regulator.o
obj-$(CONFIG_REGULATOR_BD718XX) += bd718x7-regulator.o
obj-$(CONFIG_REGULATOR_BD9571MWV) += bd9571mwv-regulator.o
+obj-$(CONFIG_REGULATOR_BD957XMUF) += bd9576-regulator.o
obj-$(CONFIG_REGULATOR_DA903X) += da903x-regulator.o
obj-$(CONFIG_REGULATOR_DA9052) += da9052-regulator.o
obj-$(CONFIG_REGULATOR_DA9055) += da9055-regulator.o
obj-$(CONFIG_REGULATOR_MT6311) += mt6311-regulator.o
obj-$(CONFIG_REGULATOR_MT6323) += mt6323-regulator.o
obj-$(CONFIG_REGULATOR_MT6358) += mt6358-regulator.o
+obj-$(CONFIG_REGULATOR_MT6360) += mt6360-regulator.o
obj-$(CONFIG_REGULATOR_MT6380) += mt6380-regulator.o
obj-$(CONFIG_REGULATOR_MT6397) += mt6397-regulator.o
obj-$(CONFIG_REGULATOR_QCOM_LABIBB) += qcom-labibb-regulator.o
obj-$(CONFIG_REGULATOR_PBIAS) += pbias-regulator.o
obj-$(CONFIG_REGULATOR_PCAP) += pcap-regulator.o
obj-$(CONFIG_REGULATOR_PCF50633) += pcf50633-regulator.o
+obj-$(CONFIG_REGULATOR_RASPBERRYPI_TOUCHSCREEN_ATTINY) += rpi-panel-attiny-regulator.o
obj-$(CONFIG_REGULATOR_RC5T583) += rc5t583-regulator.o
obj-$(CONFIG_REGULATOR_RK808) += rk808-regulator.o
obj-$(CONFIG_REGULATOR_RN5T618) += rn5t618-regulator.o
obj-$(CONFIG_REGULATOR_ROHM) += rohm-regulator.o
+obj-$(CONFIG_REGULATOR_RT4801) += rt4801-regulator.o
obj-$(CONFIG_REGULATOR_RT5033) += rt5033-regulator.o
+obj-$(CONFIG_REGULATOR_RTMV20) += rtmv20-regulator.o
obj-$(CONFIG_REGULATOR_S2MPA01) += s2mpa01.o
obj-$(CONFIG_REGULATOR_S2MPS11) += s2mps11.o
obj-$(CONFIG_REGULATOR_S5M8767) += s5m8767.o
#define AXP20X_DCDC2_V_OUT_MASK GENMASK(5, 0)
#define AXP20X_DCDC3_V_OUT_MASK GENMASK(7, 0)
-#define AXP20X_LDO24_V_OUT_MASK GENMASK(7, 4)
+#define AXP20X_LDO2_V_OUT_MASK GENMASK(7, 4)
#define AXP20X_LDO3_V_OUT_MASK GENMASK(6, 0)
+#define AXP20X_LDO4_V_OUT_MASK GENMASK(3, 0)
#define AXP20X_LDO5_V_OUT_MASK GENMASK(7, 4)
#define AXP20X_PWR_OUT_EXTEN_MASK BIT_MASK(0)
AXP20X_PWR_OUT_CTRL, AXP20X_PWR_OUT_DCDC3_MASK),
AXP_DESC_FIXED(AXP20X, LDO1, "ldo1", "acin", 1300),
AXP_DESC(AXP20X, LDO2, "ldo2", "ldo24in", 1800, 3300, 100,
- AXP20X_LDO24_V_OUT, AXP20X_LDO24_V_OUT_MASK,
+ AXP20X_LDO24_V_OUT, AXP20X_LDO2_V_OUT_MASK,
AXP20X_PWR_OUT_CTRL, AXP20X_PWR_OUT_LDO2_MASK),
AXP_DESC(AXP20X, LDO3, "ldo3", "ldo3in", 700, 3500, 25,
AXP20X_LDO3_V_OUT, AXP20X_LDO3_V_OUT_MASK,
AXP20X_PWR_OUT_CTRL, AXP20X_PWR_OUT_LDO3_MASK),
AXP_DESC_RANGES(AXP20X, LDO4, "ldo4", "ldo24in",
axp20x_ldo4_ranges, AXP20X_LDO4_V_OUT_NUM_VOLTAGES,
- AXP20X_LDO24_V_OUT, AXP20X_LDO24_V_OUT_MASK,
+ AXP20X_LDO24_V_OUT, AXP20X_LDO4_V_OUT_MASK,
AXP20X_PWR_OUT_CTRL, AXP20X_PWR_OUT_LDO4_MASK),
AXP_DESC_IO(AXP20X, LDO5, "ldo5", "ldo5in", 1800, 3300, 100,
AXP20X_LDO5_V_OUT, AXP20X_LDO5_V_OUT_MASK,
#include <linux/regulator/of_regulator.h>
#include <linux/slab.h>
+/*
+ * BD718(37/47/50) have two "enable control modes". ON/OFF can either be
+ * controlled by software - or by PMIC internal HW state machine. Whether
+ * regulator should be under SW or HW control can be defined from device-tree.
+ * Let's provide separate ops for regulators to use depending on the "enable
+ * control mode".
+ */
+#define BD718XX_HWOPNAME(swopname) swopname##_hwcontrol
+
+#define BD718XX_OPS(name, _list_voltage, _map_voltage, _set_voltage_sel, \
+ _get_voltage_sel, _set_voltage_time_sel, _set_ramp_delay) \
+static const struct regulator_ops name = { \
+ .enable = regulator_enable_regmap, \
+ .disable = regulator_disable_regmap, \
+ .is_enabled = regulator_is_enabled_regmap, \
+ .list_voltage = (_list_voltage), \
+ .map_voltage = (_map_voltage), \
+ .set_voltage_sel = (_set_voltage_sel), \
+ .get_voltage_sel = (_get_voltage_sel), \
+ .set_voltage_time_sel = (_set_voltage_time_sel), \
+ .set_ramp_delay = (_set_ramp_delay), \
+}; \
+ \
+static const struct regulator_ops BD718XX_HWOPNAME(name) = { \
+ .is_enabled = always_enabled_by_hwstate, \
+ .list_voltage = (_list_voltage), \
+ .map_voltage = (_map_voltage), \
+ .set_voltage_sel = (_set_voltage_sel), \
+ .get_voltage_sel = (_get_voltage_sel), \
+ .set_voltage_time_sel = (_set_voltage_time_sel), \
+ .set_ramp_delay = (_set_ramp_delay), \
+} \
+
/*
* BUCK1/2/3/4
* BUCK1RAMPRATE[1:0] BUCK1 DVS ramp rate setting
BUCK_RAMPRATE_MASK, ramp_value << 6);
}
+/* These functions are used when regulators are under HW state machine control.
+ * We assume PMIC is in RUN state because SW running and able to query the
+ * status. Most of the regulators have fixed ON or OFF state at RUN/IDLE so for
+ * them we just return a constant. BD71837 BUCK3 and BUCK4 are exceptions as
+ * they support configuring the ON/OFF state for RUN.
+ *
+ * Note for next hacker - these PMICs have a register where the HW state can be
+ * read. If assuming RUN appears to be false in your use-case - you can
+ * implement state reading (although that is not going to be atomic) before
+ * returning the enable state.
+ */
+static int always_enabled_by_hwstate(struct regulator_dev *rdev)
+{
+ return 1;
+}
+
+static int never_enabled_by_hwstate(struct regulator_dev *rdev)
+{
+ return 0;
+}
+
+static int bd71837_get_buck34_enable_hwctrl(struct regulator_dev *rdev)
+{
+ int ret;
+ unsigned int val;
+
+ ret = regmap_read(rdev->regmap, rdev->desc->enable_reg, &val);
+ if (ret)
+ return ret;
+
+ return !!(BD718XX_BUCK_RUN_ON & val);
+}
/*
* On BD71837 (not on BD71847, BD71850, ...)
* Bucks 1 to 4 support DVS. PWM mode is used when voltage is changed.
static int bd71837_set_voltage_sel_restricted(struct regulator_dev *rdev,
unsigned int sel)
{
- if (regulator_is_enabled_regmap(rdev))
+ if (rdev->desc->ops->is_enabled(rdev))
return -EBUSY;
return regulator_set_voltage_sel_regmap(rdev, sel);
int ret;
*mask = 0;
- if (regulator_is_enabled_regmap(rdev)) {
+ if (rdev->desc->ops->is_enabled(rdev)) {
int now, new;
now = rdev->desc->ops->get_voltage_sel(rdev);
static int bd71837_set_voltage_sel_pickable_restricted(
struct regulator_dev *rdev, unsigned int sel)
{
- if (regulator_is_enabled_regmap(rdev))
+ if (rdev->desc->ops->is_enabled(rdev))
return -EBUSY;
return regulator_set_voltage_sel_pickable_regmap(rdev, sel);
}
-static const struct regulator_ops bd718xx_pickable_range_ldo_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
+/*
+ * OPS common for BD71847 and BD71850
+ */
+BD718XX_OPS(bd718xx_pickable_range_ldo_ops,
+ regulator_list_voltage_pickable_linear_range, NULL,
+ bd718xx_set_voltage_sel_pickable_restricted,
+ regulator_get_voltage_sel_pickable_regmap, NULL, NULL);
+
+/* BD71847 and BD71850 LDO 5 is by default OFF at RUN state */
+static const struct regulator_ops bd718xx_ldo5_ops_hwstate = {
+ .is_enabled = never_enabled_by_hwstate,
.list_voltage = regulator_list_voltage_pickable_linear_range,
.set_voltage_sel = bd718xx_set_voltage_sel_pickable_restricted,
.get_voltage_sel = regulator_get_voltage_sel_pickable_regmap,
-
};
-static const struct regulator_ops bd71837_pickable_range_ldo_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
- .list_voltage = regulator_list_voltage_pickable_linear_range,
- .set_voltage_sel = bd71837_set_voltage_sel_pickable_restricted,
- .get_voltage_sel = regulator_get_voltage_sel_pickable_regmap,
-};
-
-static const struct regulator_ops bd718xx_pickable_range_buck_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
- .list_voltage = regulator_list_voltage_pickable_linear_range,
- .set_voltage_sel = regulator_set_voltage_sel_pickable_regmap,
- .get_voltage_sel = regulator_get_voltage_sel_pickable_regmap,
- .set_voltage_time_sel = regulator_set_voltage_time_sel,
-};
-
-static const struct regulator_ops bd71837_pickable_range_buck_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
- .list_voltage = regulator_list_voltage_pickable_linear_range,
- .set_voltage_sel = bd71837_set_voltage_sel_pickable_restricted,
- .get_voltage_sel = regulator_get_voltage_sel_pickable_regmap,
- .set_voltage_time_sel = regulator_set_voltage_time_sel,
-};
-
-static const struct regulator_ops bd71837_ldo_regulator_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
- .list_voltage = regulator_list_voltage_linear_range,
- .set_voltage_sel = bd71837_set_voltage_sel_restricted,
- .get_voltage_sel = regulator_get_voltage_sel_regmap,
-};
-
-static const struct regulator_ops bd718xx_ldo_regulator_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
- .list_voltage = regulator_list_voltage_linear_range,
- .set_voltage_sel = bd718xx_set_voltage_sel_restricted,
- .get_voltage_sel = regulator_get_voltage_sel_regmap,
-};
-
-static const struct regulator_ops bd71837_ldo_regulator_nolinear_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
- .list_voltage = regulator_list_voltage_table,
- .set_voltage_sel = bd71837_set_voltage_sel_restricted,
- .get_voltage_sel = regulator_get_voltage_sel_regmap,
-};
-
-static const struct regulator_ops bd718xx_ldo_regulator_nolinear_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
- .list_voltage = regulator_list_voltage_table,
- .set_voltage_sel = bd718xx_set_voltage_sel_restricted,
- .get_voltage_sel = regulator_get_voltage_sel_regmap,
-};
+BD718XX_OPS(bd718xx_pickable_range_buck_ops,
+ regulator_list_voltage_pickable_linear_range, NULL,
+ regulator_set_voltage_sel_pickable_regmap,
+ regulator_get_voltage_sel_pickable_regmap,
+ regulator_set_voltage_time_sel, NULL);
-static const struct regulator_ops bd718xx_buck_regulator_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
- .list_voltage = regulator_list_voltage_linear_range,
- .set_voltage_sel = regulator_set_voltage_sel_regmap,
- .get_voltage_sel = regulator_get_voltage_sel_regmap,
- .set_voltage_time_sel = regulator_set_voltage_time_sel,
-};
+BD718XX_OPS(bd718xx_ldo_regulator_ops, regulator_list_voltage_linear_range,
+ NULL, bd718xx_set_voltage_sel_restricted,
+ regulator_get_voltage_sel_regmap, NULL, NULL);
-static const struct regulator_ops bd71837_buck_regulator_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
- .list_voltage = regulator_list_voltage_linear_range,
- .set_voltage_sel = bd71837_set_voltage_sel_restricted,
- .get_voltage_sel = regulator_get_voltage_sel_regmap,
- .set_voltage_time_sel = regulator_set_voltage_time_sel,
-};
+BD718XX_OPS(bd718xx_ldo_regulator_nolinear_ops, regulator_list_voltage_table,
+ NULL, bd718xx_set_voltage_sel_restricted,
+ regulator_get_voltage_sel_regmap, NULL, NULL);
-static const struct regulator_ops bd718xx_buck_regulator_nolinear_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
- .list_voltage = regulator_list_voltage_table,
- .map_voltage = regulator_map_voltage_ascend,
- .set_voltage_sel = regulator_set_voltage_sel_regmap,
- .get_voltage_sel = regulator_get_voltage_sel_regmap,
- .set_voltage_time_sel = regulator_set_voltage_time_sel,
-};
+BD718XX_OPS(bd718xx_buck_regulator_ops, regulator_list_voltage_linear_range,
+ NULL, regulator_set_voltage_sel_regmap,
+ regulator_get_voltage_sel_regmap, regulator_set_voltage_time_sel,
+ NULL);
-static const struct regulator_ops bd71837_buck_regulator_nolinear_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
- .list_voltage = regulator_list_voltage_table,
- .map_voltage = regulator_map_voltage_ascend,
- .set_voltage_sel = bd718xx_set_voltage_sel_restricted,
- .get_voltage_sel = regulator_get_voltage_sel_regmap,
- .set_voltage_time_sel = regulator_set_voltage_time_sel,
-};
+BD718XX_OPS(bd718xx_buck_regulator_nolinear_ops, regulator_list_voltage_table,
+ regulator_map_voltage_ascend, regulator_set_voltage_sel_regmap,
+ regulator_get_voltage_sel_regmap, regulator_set_voltage_time_sel,
+ NULL);
-static const struct regulator_ops bd718xx_dvs_buck_regulator_ops = {
- .enable = regulator_enable_regmap,
- .disable = regulator_disable_regmap,
- .is_enabled = regulator_is_enabled_regmap,
+/*
+ * OPS for BD71837
+ */
+BD718XX_OPS(bd71837_pickable_range_ldo_ops,
+ regulator_list_voltage_pickable_linear_range, NULL,
+ bd71837_set_voltage_sel_pickable_restricted,
+ regulator_get_voltage_sel_pickable_regmap, NULL, NULL);
+
+BD718XX_OPS(bd71837_pickable_range_buck_ops,
+ regulator_list_voltage_pickable_linear_range, NULL,
+ bd71837_set_voltage_sel_pickable_restricted,
+ regulator_get_voltage_sel_pickable_regmap,
+ regulator_set_voltage_time_sel, NULL);
+
+BD718XX_OPS(bd71837_ldo_regulator_ops, regulator_list_voltage_linear_range,
+ NULL, bd71837_set_voltage_sel_restricted,
+ regulator_get_voltage_sel_regmap, NULL, NULL);
+
+BD718XX_OPS(bd71837_ldo_regulator_nolinear_ops, regulator_list_voltage_table,
+ NULL, bd71837_set_voltage_sel_restricted,
+ regulator_get_voltage_sel_regmap, NULL, NULL);
+
+BD718XX_OPS(bd71837_buck_regulator_ops, regulator_list_voltage_linear_range,
+ NULL, bd71837_set_voltage_sel_restricted,
+ regulator_get_voltage_sel_regmap, regulator_set_voltage_time_sel,
+ NULL);
+
+BD718XX_OPS(bd71837_buck_regulator_nolinear_ops, regulator_list_voltage_table,
+ regulator_map_voltage_ascend, bd718xx_set_voltage_sel_restricted,
+ regulator_get_voltage_sel_regmap, regulator_set_voltage_time_sel,
+ NULL);
+/*
+ * BD71837 bucks 3 and 4 support defining their enable/disable state also
+ * when buck enable state is under HW state machine control. In that case the
+ * bit [2] in CTRL register is used to indicate if regulator should be ON.
+ */
+static const struct regulator_ops bd71837_buck34_ops_hwctrl = {
+ .is_enabled = bd71837_get_buck34_enable_hwctrl,
.list_voltage = regulator_list_voltage_linear_range,
.set_voltage_sel = regulator_set_voltage_sel_regmap,
.get_voltage_sel = regulator_get_voltage_sel_regmap,
.set_ramp_delay = bd718xx_buck1234_set_ramp_delay,
};
+/*
+ * OPS for all of the ICs - BD718(37/47/50)
+ */
+BD718XX_OPS(bd718xx_dvs_buck_regulator_ops, regulator_list_voltage_linear_range,
+ NULL, regulator_set_voltage_sel_regmap,
+ regulator_get_voltage_sel_regmap, regulator_set_voltage_time_sel,
+ bd718xx_buck1234_set_ramp_delay);
+
/*
* BD71837 BUCK1/2/3/4
* BD71847 BUCK1/2
return rohm_regulator_set_dvs_levels(&data->dvs, np, desc, cfg->regmap);
}
-static const struct bd718xx_regulator_data bd71847_regulators[] = {
+static const struct regulator_ops *bd71847_swcontrol_ops[] = {
+ &bd718xx_dvs_buck_regulator_ops, &bd718xx_dvs_buck_regulator_ops,
+ &bd718xx_pickable_range_buck_ops, &bd718xx_pickable_range_buck_ops,
+ &bd718xx_buck_regulator_nolinear_ops, &bd718xx_buck_regulator_ops,
+ &bd718xx_pickable_range_ldo_ops, &bd718xx_ldo_regulator_nolinear_ops,
+ &bd718xx_ldo_regulator_ops, &bd718xx_ldo_regulator_ops,
+ &bd718xx_pickable_range_ldo_ops, &bd718xx_ldo_regulator_ops,
+};
+
+static const struct regulator_ops *bd71847_hwcontrol_ops[] = {
+ &BD718XX_HWOPNAME(bd718xx_dvs_buck_regulator_ops),
+ &BD718XX_HWOPNAME(bd718xx_dvs_buck_regulator_ops),
+ &BD718XX_HWOPNAME(bd718xx_pickable_range_buck_ops),
+ &BD718XX_HWOPNAME(bd718xx_pickable_range_buck_ops),
+ &BD718XX_HWOPNAME(bd718xx_buck_regulator_nolinear_ops),
+ &BD718XX_HWOPNAME(bd718xx_buck_regulator_ops),
+ &BD718XX_HWOPNAME(bd718xx_pickable_range_ldo_ops),
+ &BD718XX_HWOPNAME(bd718xx_ldo_regulator_nolinear_ops),
+ &BD718XX_HWOPNAME(bd718xx_ldo_regulator_ops),
+ &BD718XX_HWOPNAME(bd718xx_ldo_regulator_ops),
+ &bd718xx_ldo5_ops_hwstate,
+ &BD718XX_HWOPNAME(bd718xx_ldo_regulator_ops),
+};
+
+static struct bd718xx_regulator_data bd71847_regulators[] = {
{
.desc = {
.name = "buck1",
.of_match = of_match_ptr("BUCK1"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK1,
- .ops = &bd718xx_dvs_buck_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_DVS_BUCK_VOLTAGE_NUM,
.linear_ranges = bd718xx_dvs_buck_volts,
.of_match = of_match_ptr("BUCK2"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK2,
- .ops = &bd718xx_dvs_buck_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_DVS_BUCK_VOLTAGE_NUM,
.linear_ranges = bd718xx_dvs_buck_volts,
.of_match = of_match_ptr("BUCK3"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK3,
- .ops = &bd718xx_pickable_range_buck_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD71847_BUCK3_VOLTAGE_NUM,
.linear_ranges = bd71847_buck3_volts,
.of_match = of_match_ptr("BUCK4"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK4,
- .ops = &bd718xx_pickable_range_buck_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD71847_BUCK4_VOLTAGE_NUM,
.linear_ranges = bd71847_buck4_volts,
.of_match = of_match_ptr("BUCK5"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK5,
- .ops = &bd718xx_buck_regulator_nolinear_ops,
.type = REGULATOR_VOLTAGE,
.volt_table = &bd718xx_3rd_nodvs_buck_volts[0],
.n_voltages = ARRAY_SIZE(bd718xx_3rd_nodvs_buck_volts),
.of_match = of_match_ptr("BUCK6"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK6,
- .ops = &bd718xx_buck_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_4TH_NODVS_BUCK_VOLTAGE_NUM,
.linear_ranges = bd718xx_4th_nodvs_buck_volts,
.of_match = of_match_ptr("LDO1"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO1,
- .ops = &bd718xx_pickable_range_ldo_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_LDO1_VOLTAGE_NUM,
.linear_ranges = bd718xx_ldo1_volts,
.of_match = of_match_ptr("LDO2"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO2,
- .ops = &bd718xx_ldo_regulator_nolinear_ops,
.type = REGULATOR_VOLTAGE,
.volt_table = &ldo_2_volts[0],
.vsel_reg = BD718XX_REG_LDO2_VOLT,
.of_match = of_match_ptr("LDO3"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO3,
- .ops = &bd718xx_ldo_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_LDO3_VOLTAGE_NUM,
.linear_ranges = bd718xx_ldo3_volts,
.of_match = of_match_ptr("LDO4"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO4,
- .ops = &bd718xx_ldo_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_LDO4_VOLTAGE_NUM,
.linear_ranges = bd718xx_ldo4_volts,
.of_match = of_match_ptr("LDO5"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO5,
- .ops = &bd718xx_pickable_range_ldo_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD71847_LDO5_VOLTAGE_NUM,
.linear_ranges = bd71847_ldo5_volts,
.of_match = of_match_ptr("LDO6"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO6,
- .ops = &bd718xx_ldo_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_LDO6_VOLTAGE_NUM,
.linear_ranges = bd718xx_ldo6_volts,
},
};
-static const struct bd718xx_regulator_data bd71837_regulators[] = {
+static const struct regulator_ops *bd71837_swcontrol_ops[] = {
+ &bd718xx_dvs_buck_regulator_ops, &bd718xx_dvs_buck_regulator_ops,
+ &bd718xx_dvs_buck_regulator_ops, &bd718xx_dvs_buck_regulator_ops,
+ &bd71837_pickable_range_buck_ops, &bd71837_buck_regulator_ops,
+ &bd71837_buck_regulator_nolinear_ops, &bd71837_buck_regulator_ops,
+ &bd71837_pickable_range_ldo_ops, &bd71837_ldo_regulator_nolinear_ops,
+ &bd71837_ldo_regulator_ops, &bd71837_ldo_regulator_ops,
+ &bd71837_ldo_regulator_ops, &bd71837_ldo_regulator_ops,
+ &bd71837_ldo_regulator_ops,
+};
+
+static const struct regulator_ops *bd71837_hwcontrol_ops[] = {
+ &BD718XX_HWOPNAME(bd718xx_dvs_buck_regulator_ops),
+ &BD718XX_HWOPNAME(bd718xx_dvs_buck_regulator_ops),
+ &bd71837_buck34_ops_hwctrl, &bd71837_buck34_ops_hwctrl,
+ &BD718XX_HWOPNAME(bd71837_pickable_range_buck_ops),
+ &BD718XX_HWOPNAME(bd71837_buck_regulator_ops),
+ &BD718XX_HWOPNAME(bd71837_buck_regulator_nolinear_ops),
+ &BD718XX_HWOPNAME(bd71837_buck_regulator_ops),
+ &BD718XX_HWOPNAME(bd71837_pickable_range_ldo_ops),
+ &BD718XX_HWOPNAME(bd71837_ldo_regulator_nolinear_ops),
+ &BD718XX_HWOPNAME(bd71837_ldo_regulator_ops),
+ &BD718XX_HWOPNAME(bd71837_ldo_regulator_ops),
+ &BD718XX_HWOPNAME(bd71837_ldo_regulator_ops),
+ &BD718XX_HWOPNAME(bd71837_ldo_regulator_ops),
+ &BD718XX_HWOPNAME(bd71837_ldo_regulator_ops),
+};
+
+static struct bd718xx_regulator_data bd71837_regulators[] = {
{
.desc = {
.name = "buck1",
.of_match = of_match_ptr("BUCK1"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK1,
- .ops = &bd718xx_dvs_buck_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_DVS_BUCK_VOLTAGE_NUM,
.linear_ranges = bd718xx_dvs_buck_volts,
.of_match = of_match_ptr("BUCK2"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK2,
- .ops = &bd718xx_dvs_buck_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_DVS_BUCK_VOLTAGE_NUM,
.linear_ranges = bd718xx_dvs_buck_volts,
.of_match = of_match_ptr("BUCK3"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK3,
- .ops = &bd718xx_dvs_buck_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_DVS_BUCK_VOLTAGE_NUM,
.linear_ranges = bd718xx_dvs_buck_volts,
.of_match = of_match_ptr("BUCK4"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK4,
- .ops = &bd718xx_dvs_buck_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_DVS_BUCK_VOLTAGE_NUM,
.linear_ranges = bd718xx_dvs_buck_volts,
.of_match = of_match_ptr("BUCK5"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK5,
- .ops = &bd71837_pickable_range_buck_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD71837_BUCK5_VOLTAGE_NUM,
.linear_ranges = bd71837_buck5_volts,
.of_match = of_match_ptr("BUCK6"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK6,
- .ops = &bd71837_buck_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD71837_BUCK6_VOLTAGE_NUM,
.linear_ranges = bd71837_buck6_volts,
.of_match = of_match_ptr("BUCK7"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK7,
- .ops = &bd71837_buck_regulator_nolinear_ops,
.type = REGULATOR_VOLTAGE,
.volt_table = &bd718xx_3rd_nodvs_buck_volts[0],
.n_voltages = ARRAY_SIZE(bd718xx_3rd_nodvs_buck_volts),
.of_match = of_match_ptr("BUCK8"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_BUCK8,
- .ops = &bd71837_buck_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_4TH_NODVS_BUCK_VOLTAGE_NUM,
.linear_ranges = bd718xx_4th_nodvs_buck_volts,
.of_match = of_match_ptr("LDO1"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO1,
- .ops = &bd71837_pickable_range_ldo_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_LDO1_VOLTAGE_NUM,
.linear_ranges = bd718xx_ldo1_volts,
.of_match = of_match_ptr("LDO2"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO2,
- .ops = &bd71837_ldo_regulator_nolinear_ops,
.type = REGULATOR_VOLTAGE,
.volt_table = &ldo_2_volts[0],
.vsel_reg = BD718XX_REG_LDO2_VOLT,
.of_match = of_match_ptr("LDO3"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO3,
- .ops = &bd71837_ldo_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_LDO3_VOLTAGE_NUM,
.linear_ranges = bd718xx_ldo3_volts,
.of_match = of_match_ptr("LDO4"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO4,
- .ops = &bd71837_ldo_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_LDO4_VOLTAGE_NUM,
.linear_ranges = bd718xx_ldo4_volts,
.of_match = of_match_ptr("LDO5"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO5,
- .ops = &bd71837_ldo_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD71837_LDO5_VOLTAGE_NUM,
.linear_ranges = bd71837_ldo5_volts,
.of_match = of_match_ptr("LDO6"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO6,
- .ops = &bd71837_ldo_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD718XX_LDO6_VOLTAGE_NUM,
.linear_ranges = bd718xx_ldo6_volts,
.of_match = of_match_ptr("LDO7"),
.regulators_node = of_match_ptr("regulators"),
.id = BD718XX_LDO7,
- .ops = &bd71837_ldo_regulator_ops,
.type = REGULATOR_VOLTAGE,
.n_voltages = BD71837_LDO7_VOLTAGE_NUM,
.linear_ranges = bd71837_ldo7_volts,
},
};
+static void mark_hw_controlled(struct device *dev, struct device_node *np,
+ struct bd718xx_regulator_data *reg_data,
+ unsigned int num_reg_data, int *info)
+{
+ int i;
+
+ for (i = 1; i <= num_reg_data; i++) {
+ if (!of_node_name_eq(np, reg_data[i-1].desc.of_match))
+ continue;
+
+ *info |= 1 << (i - 1);
+ dev_dbg(dev, "regulator %d runlevel controlled\n", i);
+ return;
+ }
+ dev_warn(dev, "Bad regulator node\n");
+}
+
+static int get_hw_controlled_regulators(struct device *dev,
+ struct bd718xx_regulator_data *reg_data,
+ unsigned int num_reg_data, int *info)
+{
+ struct device_node *np;
+ struct device_node *nproot = dev->of_node;
+ const char *prop = "rohm,no-regulator-enable-control";
+
+ *info = 0;
+
+ nproot = of_get_child_by_name(nproot, "regulators");
+ if (!nproot) {
+ dev_err(dev, "failed to find regulators node\n");
+ return -ENODEV;
+ }
+ for_each_child_of_node(nproot, np)
+ if (of_property_read_bool(np, prop))
+ mark_hw_controlled(dev, np, reg_data, num_reg_data,
+ info);
+
+ of_node_put(nproot);
+ return 0;
+}
+
static int bd718xx_probe(struct platform_device *pdev)
{
struct bd718xx *mfd;
struct regulator_config config = { 0 };
- int i, j, err;
+ int i, j, err, omit_enable;
bool use_snvs;
- const struct bd718xx_regulator_data *reg_data;
+ struct bd718xx_regulator_data *reg_data;
unsigned int num_reg_data;
enum rohm_chip_type chip = platform_get_device_id(pdev)->driver_data;
+ const struct regulator_ops **swops, **hwops;
mfd = dev_get_drvdata(pdev->dev.parent);
if (!mfd) {
case ROHM_CHIP_TYPE_BD71837:
reg_data = bd71837_regulators;
num_reg_data = ARRAY_SIZE(bd71837_regulators);
+ swops = &bd71837_swcontrol_ops[0];
+ hwops = &bd71837_hwcontrol_ops[0];
break;
case ROHM_CHIP_TYPE_BD71847:
reg_data = bd71847_regulators;
num_reg_data = ARRAY_SIZE(bd71847_regulators);
+ swops = &bd71847_swcontrol_ops[0];
+ hwops = &bd71847_hwcontrol_ops[0];
break;
default:
dev_err(&pdev->dev, "Unsupported chip type\n");
}
}
+ config.dev = pdev->dev.parent;
+ config.regmap = mfd->chip.regmap;
+ /*
+ * There are cases when we want to leave the enable-control for
+ * the HW state machine and use this driver only for voltage control.
+ * One special case is when we use PMIC_STBY_REQ line from SoC to PMIC
+ * in order to set the system to SUSPEND state.
+ *
+ * If regulator is taken under SW control the regulator state will not
+ * be affected by PMIC state machine - Eg. regulator is likely to stay
+ * on even in SUSPEND
+ */
+ get_hw_controlled_regulators(pdev->dev.parent, reg_data, num_reg_data,
+ &omit_enable);
+
for (i = 0; i < num_reg_data; i++) {
- const struct regulator_desc *desc;
+ struct regulator_desc *desc;
struct regulator_dev *rdev;
- const struct bd718xx_regulator_data *r;
+ struct bd718xx_regulator_data *r;
+ int no_enable_control = omit_enable & (1 << i);
r = ®_data[i];
desc = &r->desc;
- config.dev = pdev->dev.parent;
- config.regmap = mfd->chip.regmap;
+ if (no_enable_control)
+ desc->ops = hwops[i];
+ else
+ desc->ops = swops[i];
rdev = devm_regulator_register(&pdev->dev, desc, &config);
if (IS_ERR(rdev)) {
* enable SW control for crucial regulators if snvs state is
* used
*/
- if (!use_snvs || !rdev->constraints->always_on ||
- !rdev->constraints->boot_on) {
+ if (!no_enable_control && (!use_snvs ||
+ !rdev->constraints->always_on ||
+ !rdev->constraints->boot_on)) {
err = regmap_update_bits(mfd->chip.regmap, r->init.reg,
r->init.mask, r->init.val);
if (err) {
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 ROHM Semiconductors
+// ROHM BD9576MUF/BD9573MUF regulator driver
+
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/gpio/consumer.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mfd/rohm-bd957x.h>
+#include <linux/mfd/rohm-generic.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+#include <linux/regulator/of_regulator.h>
+#include <linux/slab.h>
+
+#define BD957X_VOUTS1_VOLT 3300000
+#define BD957X_VOUTS4_BASE_VOLT 1030000
+#define BD957X_VOUTS34_NUM_VOLT 32
+
+static int vout1_volt_table[] = {5000000, 4900000, 4800000, 4700000, 4600000,
+ 4500000, 4500000, 4500000, 5000000, 5100000,
+ 5200000, 5300000, 5400000, 5500000, 5500000,
+ 5500000};
+
+static int vout2_volt_table[] = {1800000, 1780000, 1760000, 1740000, 1720000,
+ 1700000, 1680000, 1660000, 1800000, 1820000,
+ 1840000, 1860000, 1880000, 1900000, 1920000,
+ 1940000};
+
+static int voutl1_volt_table[] = {2500000, 2540000, 2580000, 2620000, 2660000,
+ 2700000, 2740000, 2780000, 2500000, 2460000,
+ 2420000, 2380000, 2340000, 2300000, 2260000,
+ 2220000};
+
+struct bd957x_regulator_data {
+ struct regulator_desc desc;
+ int base_voltage;
+};
+
+static int bd957x_vout34_list_voltage(struct regulator_dev *rdev,
+ unsigned int selector)
+{
+ const struct regulator_desc *desc = rdev->desc;
+ int multiplier = selector & desc->vsel_mask & 0x7f;
+ int tune;
+
+ /* VOUT3 and 4 has 10mV step */
+ tune = multiplier * 10000;
+
+ if (!(selector & 0x80))
+ return desc->fixed_uV - tune;
+
+ return desc->fixed_uV + tune;
+}
+
+static int bd957x_list_voltage(struct regulator_dev *rdev,
+ unsigned int selector)
+{
+ const struct regulator_desc *desc = rdev->desc;
+ int index = selector & desc->vsel_mask & 0x7f;
+
+ if (!(selector & 0x80))
+ index += desc->n_voltages/2;
+
+ if (index >= desc->n_voltages)
+ return -EINVAL;
+
+ return desc->volt_table[index];
+}
+
+static const struct regulator_ops bd957x_vout34_ops = {
+ .is_enabled = regulator_is_enabled_regmap,
+ .list_voltage = bd957x_vout34_list_voltage,
+ .get_voltage_sel = regulator_get_voltage_sel_regmap,
+};
+
+static const struct regulator_ops bd957X_vouts1_regulator_ops = {
+ .is_enabled = regulator_is_enabled_regmap,
+};
+
+static const struct regulator_ops bd957x_ops = {
+ .is_enabled = regulator_is_enabled_regmap,
+ .list_voltage = bd957x_list_voltage,
+ .get_voltage_sel = regulator_get_voltage_sel_regmap,
+};
+
+static struct bd957x_regulator_data bd9576_regulators[] = {
+ {
+ .desc = {
+ .name = "VD50",
+ .of_match = of_match_ptr("regulator-vd50"),
+ .regulators_node = of_match_ptr("regulators"),
+ .id = BD957X_VD50,
+ .type = REGULATOR_VOLTAGE,
+ .ops = &bd957x_ops,
+ .volt_table = &vout1_volt_table[0],
+ .n_voltages = ARRAY_SIZE(vout1_volt_table),
+ .vsel_reg = BD957X_REG_VOUT1_TUNE,
+ .vsel_mask = BD957X_MASK_VOUT1_TUNE,
+ .enable_reg = BD957X_REG_POW_TRIGGER1,
+ .enable_mask = BD957X_REGULATOR_EN_MASK,
+ .enable_val = BD957X_REGULATOR_DIS_VAL,
+ .enable_is_inverted = true,
+ .owner = THIS_MODULE,
+ },
+ },
+ {
+ .desc = {
+ .name = "VD18",
+ .of_match = of_match_ptr("regulator-vd18"),
+ .regulators_node = of_match_ptr("regulators"),
+ .id = BD957X_VD18,
+ .type = REGULATOR_VOLTAGE,
+ .ops = &bd957x_ops,
+ .volt_table = &vout2_volt_table[0],
+ .n_voltages = ARRAY_SIZE(vout2_volt_table),
+ .vsel_reg = BD957X_REG_VOUT2_TUNE,
+ .vsel_mask = BD957X_MASK_VOUT2_TUNE,
+ .enable_reg = BD957X_REG_POW_TRIGGER2,
+ .enable_mask = BD957X_REGULATOR_EN_MASK,
+ .enable_val = BD957X_REGULATOR_DIS_VAL,
+ .enable_is_inverted = true,
+ .owner = THIS_MODULE,
+ },
+ },
+ {
+ .desc = {
+ .name = "VDDDR",
+ .of_match = of_match_ptr("regulator-vdddr"),
+ .regulators_node = of_match_ptr("regulators"),
+ .id = BD957X_VDDDR,
+ .ops = &bd957x_vout34_ops,
+ .type = REGULATOR_VOLTAGE,
+ .n_voltages = BD957X_VOUTS34_NUM_VOLT,
+ .vsel_reg = BD957X_REG_VOUT3_TUNE,
+ .vsel_mask = BD957X_MASK_VOUT3_TUNE,
+ .enable_reg = BD957X_REG_POW_TRIGGER3,
+ .enable_mask = BD957X_REGULATOR_EN_MASK,
+ .enable_val = BD957X_REGULATOR_DIS_VAL,
+ .enable_is_inverted = true,
+ .owner = THIS_MODULE,
+ },
+ },
+ {
+ .desc = {
+ .name = "VD10",
+ .of_match = of_match_ptr("regulator-vd10"),
+ .regulators_node = of_match_ptr("regulators"),
+ .id = BD957X_VD10,
+ .ops = &bd957x_vout34_ops,
+ .type = REGULATOR_VOLTAGE,
+ .fixed_uV = BD957X_VOUTS4_BASE_VOLT,
+ .n_voltages = BD957X_VOUTS34_NUM_VOLT,
+ .vsel_reg = BD957X_REG_VOUT4_TUNE,
+ .vsel_mask = BD957X_MASK_VOUT4_TUNE,
+ .enable_reg = BD957X_REG_POW_TRIGGER4,
+ .enable_mask = BD957X_REGULATOR_EN_MASK,
+ .enable_val = BD957X_REGULATOR_DIS_VAL,
+ .enable_is_inverted = true,
+ .owner = THIS_MODULE,
+ },
+ },
+ {
+ .desc = {
+ .name = "VOUTL1",
+ .of_match = of_match_ptr("regulator-voutl1"),
+ .regulators_node = of_match_ptr("regulators"),
+ .id = BD957X_VOUTL1,
+ .ops = &bd957x_ops,
+ .type = REGULATOR_VOLTAGE,
+ .volt_table = &voutl1_volt_table[0],
+ .n_voltages = ARRAY_SIZE(voutl1_volt_table),
+ .vsel_reg = BD957X_REG_VOUTL1_TUNE,
+ .vsel_mask = BD957X_MASK_VOUTL1_TUNE,
+ .enable_reg = BD957X_REG_POW_TRIGGERL1,
+ .enable_mask = BD957X_REGULATOR_EN_MASK,
+ .enable_val = BD957X_REGULATOR_DIS_VAL,
+ .enable_is_inverted = true,
+ .owner = THIS_MODULE,
+ },
+ },
+ {
+ .desc = {
+ .name = "VOUTS1",
+ .of_match = of_match_ptr("regulator-vouts1"),
+ .regulators_node = of_match_ptr("regulators"),
+ .id = BD957X_VOUTS1,
+ .ops = &bd957X_vouts1_regulator_ops,
+ .type = REGULATOR_VOLTAGE,
+ .n_voltages = 1,
+ .fixed_uV = BD957X_VOUTS1_VOLT,
+ .enable_reg = BD957X_REG_POW_TRIGGERS1,
+ .enable_mask = BD957X_REGULATOR_EN_MASK,
+ .enable_val = BD957X_REGULATOR_DIS_VAL,
+ .enable_is_inverted = true,
+ .owner = THIS_MODULE,
+ },
+ },
+};
+
+static int bd957x_probe(struct platform_device *pdev)
+{
+ struct regmap *regmap;
+ struct regulator_config config = { 0 };
+ int i, err;
+ bool vout_mode, ddr_sel;
+ const struct bd957x_regulator_data *reg_data = &bd9576_regulators[0];
+ unsigned int num_reg_data = ARRAY_SIZE(bd9576_regulators);
+ enum rohm_chip_type chip = platform_get_device_id(pdev)->driver_data;
+
+ regmap = dev_get_regmap(pdev->dev.parent, NULL);
+ if (!regmap) {
+ dev_err(&pdev->dev, "No regmap\n");
+ return -EINVAL;
+ }
+ vout_mode = of_property_read_bool(pdev->dev.parent->of_node,
+ "rohm,vout1-en-low");
+ if (vout_mode) {
+ struct gpio_desc *en;
+
+ dev_dbg(&pdev->dev, "GPIO controlled mode\n");
+
+ /* VOUT1 enable state judged by VOUT1_EN pin */
+ /* See if we have GPIO defined */
+ en = devm_gpiod_get_from_of_node(&pdev->dev,
+ pdev->dev.parent->of_node,
+ "rohm,vout1-en-gpios", 0,
+ GPIOD_OUT_LOW, "vout1-en");
+ if (!IS_ERR(en)) {
+ /* VOUT1_OPS gpio ctrl */
+ /*
+ * Regulator core prioritizes the ena_gpio over
+ * enable/disable/is_enabled callbacks so no need to
+ * clear them. We can still use same ops
+ */
+ config.ena_gpiod = en;
+ } else {
+ /*
+ * In theory it is possible someone wants to set
+ * vout1-en LOW during OTP loading and set VOUT1 to be
+ * controlled by GPIO - but control the GPIO from some
+ * where else than this driver. For that to work we
+ * should unset the is_enabled callback here.
+ *
+ * I believe such case where rohm,vout1-en-low is set
+ * and vout1-en-gpios is not is likely to be a
+ * misconfiguration. So let's just err out for now.
+ */
+ dev_err(&pdev->dev,
+ "Failed to get VOUT1 control GPIO\n");
+ return PTR_ERR(en);
+ }
+ }
+
+ /*
+ * If more than one PMIC needs to be controlled by same processor then
+ * allocate the regulator data array here and use bd9576_regulators as
+ * template. At the moment I see no such use-case so I spare some
+ * bytes and use bd9576_regulators directly for non-constant configs
+ * like DDR voltage selection.
+ */
+ ddr_sel = of_property_read_bool(pdev->dev.parent->of_node,
+ "rohm,ddr-sel-low");
+ if (ddr_sel)
+ bd9576_regulators[2].desc.fixed_uV = 1350000;
+ else
+ bd9576_regulators[2].desc.fixed_uV = 1500000;
+
+ switch (chip) {
+ case ROHM_CHIP_TYPE_BD9576:
+ dev_dbg(&pdev->dev, "Found BD9576MUF\n");
+ break;
+ case ROHM_CHIP_TYPE_BD9573:
+ dev_dbg(&pdev->dev, "Found BD9573MUF\n");
+ break;
+ default:
+ dev_err(&pdev->dev, "Unsupported chip type\n");
+ err = -EINVAL;
+ goto err;
+ }
+
+ config.dev = pdev->dev.parent;
+ config.regmap = regmap;
+
+ for (i = 0; i < num_reg_data; i++) {
+
+ const struct regulator_desc *desc;
+ struct regulator_dev *rdev;
+ const struct bd957x_regulator_data *r;
+
+ r = ®_data[i];
+ desc = &r->desc;
+
+ rdev = devm_regulator_register(&pdev->dev, desc, &config);
+ if (IS_ERR(rdev)) {
+ dev_err(&pdev->dev,
+ "failed to register %s regulator\n",
+ desc->name);
+ err = PTR_ERR(rdev);
+ goto err;
+ }
+ /*
+ * Clear the VOUT1 GPIO setting - rest of the regulators do not
+ * support GPIO control
+ */
+ config.ena_gpiod = NULL;
+ }
+
+err:
+ return err;
+}
+
+static const struct platform_device_id bd957x_pmic_id[] = {
+ { "bd9573-pmic", ROHM_CHIP_TYPE_BD9573 },
+ { "bd9576-pmic", ROHM_CHIP_TYPE_BD9576 },
+ { },
+};
+MODULE_DEVICE_TABLE(platform, bd957x_pmic_id);
+
+static struct platform_driver bd957x_regulator = {
+ .driver = {
+ .name = "bd957x-pmic",
+ },
+ .probe = bd957x_probe,
+ .id_table = bd957x_pmic_id,
+};
+
+module_platform_driver(bd957x_regulator);
+
+MODULE_AUTHOR("Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com>");
+MODULE_DESCRIPTION("ROHM BD9576/BD9573 voltage regulator driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:bd957x-pmic");
* than the one, which initially locked the mutex, it will
* wait on mutex.
*/
-void regulator_lock(struct regulator_dev *rdev)
+static void regulator_lock(struct regulator_dev *rdev)
{
regulator_lock_nested(rdev, NULL);
}
-EXPORT_SYMBOL_GPL(regulator_lock);
/**
* regulator_unlock - unlock a single regulator
* This function unlocks the mutex when the
* reference counter reaches 0.
*/
-void regulator_unlock(struct regulator_dev *rdev)
+static void regulator_unlock(struct regulator_dev *rdev)
{
mutex_lock(®ulator_nesting_mutex);
mutex_unlock(®ulator_nesting_mutex);
}
-EXPORT_SYMBOL_GPL(regulator_unlock);
static bool regulator_supply_is_couple(struct regulator_dev *rdev)
{
static struct device_node *of_get_regulator(struct device *dev, const char *supply)
{
struct device_node *regnode = NULL;
- char prop_name[32]; /* 32 is max size of property name */
+ char prop_name[64]; /* 64 is max size of property name */
dev_dbg(dev, "Looking up %s-supply from device tree\n", supply);
- snprintf(prop_name, 32, "%s-supply", supply);
+ snprintf(prop_name, 64, "%s-supply", supply);
regnode = of_parse_phandle(dev->of_node, prop_name, 0);
if (!regnode) {
}
}
+static const struct regulator_state *
+regulator_get_suspend_state_check(struct regulator_dev *rdev, suspend_state_t state)
+{
+ const struct regulator_state *rstate;
+
+ rstate = regulator_get_suspend_state(rdev, state);
+ if (rstate == NULL)
+ return NULL;
+
+ /* If we have no suspend mode configuration don't set anything;
+ * only warn if the driver implements set_suspend_voltage or
+ * set_suspend_mode callback.
+ */
+ if (rstate->enabled != ENABLE_IN_SUSPEND &&
+ rstate->enabled != DISABLE_IN_SUSPEND) {
+ if (rdev->desc->ops->set_suspend_voltage ||
+ rdev->desc->ops->set_suspend_mode)
+ rdev_warn(rdev, "No configuration\n");
+ return NULL;
+ }
+
+ return rstate;
+}
+
static ssize_t regulator_uV_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
/* set the optimum mode for our new total regulator load */
err = rdev->desc->ops->set_load(rdev, current_uA);
if (err < 0)
- rdev_err(rdev, "failed to set load %d\n", current_uA);
+ rdev_err(rdev, "failed to set load %d: %pe\n",
+ current_uA, ERR_PTR(err));
} else {
/* get output voltage */
output_uV = regulator_get_voltage_rdev(rdev);
/* check the new mode is allowed */
err = regulator_mode_constrain(rdev, &mode);
if (err < 0) {
- rdev_err(rdev, "failed to get optimum mode @ %d uA %d -> %d uV\n",
- current_uA, input_uV, output_uV);
+ rdev_err(rdev, "failed to get optimum mode @ %d uA %d -> %d uV: %pe\n",
+ current_uA, input_uV, output_uV, ERR_PTR(err));
return err;
}
err = rdev->desc->ops->set_mode(rdev, mode);
if (err < 0)
- rdev_err(rdev, "failed to set optimum mode %x\n", mode);
+ rdev_err(rdev, "failed to set optimum mode %x: %pe\n",
+ mode, ERR_PTR(err));
}
return err;
}
-static int suspend_set_state(struct regulator_dev *rdev,
- suspend_state_t state)
+static int __suspend_set_state(struct regulator_dev *rdev,
+ const struct regulator_state *rstate)
{
int ret = 0;
- struct regulator_state *rstate;
-
- rstate = regulator_get_suspend_state(rdev, state);
- if (rstate == NULL)
- return 0;
-
- /* If we have no suspend mode configuration don't set anything;
- * only warn if the driver implements set_suspend_voltage or
- * set_suspend_mode callback.
- */
- if (rstate->enabled != ENABLE_IN_SUSPEND &&
- rstate->enabled != DISABLE_IN_SUSPEND) {
- if (rdev->desc->ops->set_suspend_voltage ||
- rdev->desc->ops->set_suspend_mode)
- rdev_warn(rdev, "No configuration\n");
- return 0;
- }
if (rstate->enabled == ENABLE_IN_SUSPEND &&
rdev->desc->ops->set_suspend_enable)
ret = 0;
if (ret < 0) {
- rdev_err(rdev, "failed to enabled/disable\n");
+ rdev_err(rdev, "failed to enabled/disable: %pe\n", ERR_PTR(ret));
return ret;
}
if (rdev->desc->ops->set_suspend_voltage && rstate->uV > 0) {
ret = rdev->desc->ops->set_suspend_voltage(rdev, rstate->uV);
if (ret < 0) {
- rdev_err(rdev, "failed to set voltage\n");
+ rdev_err(rdev, "failed to set voltage: %pe\n", ERR_PTR(ret));
return ret;
}
}
if (rdev->desc->ops->set_suspend_mode && rstate->mode > 0) {
ret = rdev->desc->ops->set_suspend_mode(rdev, rstate->mode);
if (ret < 0) {
- rdev_err(rdev, "failed to set mode\n");
+ rdev_err(rdev, "failed to set mode: %pe\n", ERR_PTR(ret));
return ret;
}
}
return ret;
}
-static void print_constraints(struct regulator_dev *rdev)
+static int suspend_set_initial_state(struct regulator_dev *rdev)
+{
+ const struct regulator_state *rstate;
+
+ rstate = regulator_get_suspend_state_check(rdev,
+ rdev->constraints->initial_state);
+ if (!rstate)
+ return 0;
+
+ return __suspend_set_state(rdev, rstate);
+}
+
+#if defined(DEBUG) || defined(CONFIG_DYNAMIC_DEBUG)
+static void print_constraints_debug(struct regulator_dev *rdev)
{
struct regulation_constraints *constraints = rdev->constraints;
char buf[160] = "";
if (constraints->valid_modes_mask & REGULATOR_MODE_IDLE)
count += scnprintf(buf + count, len - count, "idle ");
if (constraints->valid_modes_mask & REGULATOR_MODE_STANDBY)
- count += scnprintf(buf + count, len - count, "standby");
+ count += scnprintf(buf + count, len - count, "standby ");
if (!count)
- scnprintf(buf, len, "no parameters");
+ count = scnprintf(buf, len, "no parameters");
+ else
+ --count;
+
+ count += scnprintf(buf + count, len - count, ", %s",
+ _regulator_is_enabled(rdev) ? "enabled" : "disabled");
rdev_dbg(rdev, "%s\n", buf);
+}
+#else /* !DEBUG && !CONFIG_DYNAMIC_DEBUG */
+static inline void print_constraints_debug(struct regulator_dev *rdev) {}
+#endif /* !DEBUG && !CONFIG_DYNAMIC_DEBUG */
+
+static void print_constraints(struct regulator_dev *rdev)
+{
+ struct regulation_constraints *constraints = rdev->constraints;
+
+ print_constraints_debug(rdev);
if ((constraints->min_uV != constraints->max_uV) &&
!regulator_ops_is_valid(rdev, REGULATOR_CHANGE_VOLTAGE))
if (current_uV < 0) {
rdev_err(rdev,
- "failed to get the current voltage(%d)\n",
- current_uV);
+ "failed to get the current voltage: %pe\n",
+ ERR_PTR(current_uV));
return current_uV;
}
rdev, target_min, target_max);
if (ret < 0) {
rdev_err(rdev,
- "failed to apply %d-%duV constraint(%d)\n",
- target_min, target_max, ret);
+ "failed to apply %d-%duV constraint: %pe\n",
+ target_min, target_max, ERR_PTR(ret));
return ret;
}
}
ret = ops->set_input_current_limit(rdev,
rdev->constraints->ilim_uA);
if (ret < 0) {
- rdev_err(rdev, "failed to set input limit\n");
+ rdev_err(rdev, "failed to set input limit: %pe\n", ERR_PTR(ret));
return ret;
}
}
/* do we need to setup our suspend state */
if (rdev->constraints->initial_state) {
- ret = suspend_set_state(rdev, rdev->constraints->initial_state);
+ ret = suspend_set_initial_state(rdev);
if (ret < 0) {
- rdev_err(rdev, "failed to set suspend state\n");
+ rdev_err(rdev, "failed to set suspend state: %pe\n", ERR_PTR(ret));
return ret;
}
}
ret = ops->set_mode(rdev, rdev->constraints->initial_mode);
if (ret < 0) {
- rdev_err(rdev, "failed to set initial mode: %d\n", ret);
+ rdev_err(rdev, "failed to set initial mode: %pe\n", ERR_PTR(ret));
return ret;
}
} else if (rdev->constraints->system_load) {
&& ops->set_ramp_delay) {
ret = ops->set_ramp_delay(rdev, rdev->constraints->ramp_delay);
if (ret < 0) {
- rdev_err(rdev, "failed to set ramp_delay\n");
+ rdev_err(rdev, "failed to set ramp_delay: %pe\n", ERR_PTR(ret));
return ret;
}
}
if (rdev->constraints->pull_down && ops->set_pull_down) {
ret = ops->set_pull_down(rdev);
if (ret < 0) {
- rdev_err(rdev, "failed to set pull down\n");
+ rdev_err(rdev, "failed to set pull down: %pe\n", ERR_PTR(ret));
return ret;
}
}
if (rdev->constraints->soft_start && ops->set_soft_start) {
ret = ops->set_soft_start(rdev);
if (ret < 0) {
- rdev_err(rdev, "failed to set soft start\n");
+ rdev_err(rdev, "failed to set soft start: %pe\n", ERR_PTR(ret));
return ret;
}
}
&& ops->set_over_current_protection) {
ret = ops->set_over_current_protection(rdev);
if (ret < 0) {
- rdev_err(rdev, "failed to set over current protection\n");
+ rdev_err(rdev, "failed to set over current protection: %pe\n",
+ ERR_PTR(ret));
return ret;
}
}
ret = ops->set_active_discharge(rdev, ad_state);
if (ret < 0) {
- rdev_err(rdev, "failed to set active discharge\n");
+ rdev_err(rdev, "failed to set active discharge: %pe\n", ERR_PTR(ret));
return ret;
}
}
ret = _regulator_do_enable(rdev);
if (ret < 0 && ret != -EINVAL) {
- rdev_err(rdev, "failed to enable\n");
+ rdev_err(rdev, "failed to enable: %pe\n", ERR_PTR(ret));
return ret;
}
err = sysfs_create_link_nowarn(&rdev->dev.kobj, &dev->kobj,
supply_name);
if (err) {
- rdev_dbg(rdev, "could not add device link %s err %d\n",
- dev->kobj.name, err);
+ rdev_dbg(rdev, "could not add device link %s: %pe\n",
+ dev->kobj.name, ERR_PTR(err));
/* non-fatal */
}
}
if (ret >= 0) {
delay = ret;
} else {
- rdev_warn(rdev, "enable_time() failed: %d\n", ret);
+ rdev_warn(rdev, "enable_time() failed: %pe\n", ERR_PTR(ret));
delay = 0;
}
_notifier_call_chain(rdev, REGULATOR_EVENT_ENABLE,
NULL);
} else if (ret < 0) {
- rdev_err(rdev, "is_enabled() failed: %d\n", ret);
+ rdev_err(rdev, "is_enabled() failed: %pe\n", ERR_PTR(ret));
goto err_consumer_disable;
}
/* Fallthrough on positive return values - already enabled */
ret = _regulator_do_disable(rdev);
if (ret < 0) {
- rdev_err(rdev, "failed to disable\n");
+ rdev_err(rdev, "failed to disable: %pe\n", ERR_PTR(ret));
_notifier_call_chain(rdev,
REGULATOR_EVENT_ABORT_DISABLE,
NULL);
ret = _regulator_do_disable(rdev);
if (ret < 0) {
- rdev_err(rdev, "failed to force disable\n");
+ rdev_err(rdev, "failed to force disable: %pe\n", ERR_PTR(ret));
_notifier_call_chain(rdev, REGULATOR_EVENT_FORCE_DISABLE |
REGULATOR_EVENT_ABORT_DISABLE, NULL);
return ret;
for (i = 0; i < count; i++) {
ret = _regulator_disable(regulator);
if (ret != 0)
- rdev_err(rdev, "Deferred disable failed: %d\n", ret);
+ rdev_err(rdev, "Deferred disable failed: %pe\n",
+ ERR_PTR(ret));
}
}
WARN_ON(!total_count);
*vsel_reg = rdev->desc->vsel_reg;
*vsel_mask = rdev->desc->vsel_mask;
- return 0;
+ return 0;
}
EXPORT_SYMBOL_GPL(regulator_get_hardware_vsel_register);
}
if (delay < 0) {
- rdev_warn(rdev, "failed to get delay: %d\n", delay);
+ rdev_warn(rdev, "failed to get delay: %pe\n", ERR_PTR(delay));
delay = 0;
}
ret = regulator_set_voltage_unlocked(rdev->supply,
best_supply_uV, INT_MAX, state);
if (ret) {
- dev_err(&rdev->dev, "Failed to increase supply voltage: %d\n",
- ret);
+ dev_err(&rdev->dev, "Failed to increase supply voltage: %pe\n",
+ ERR_PTR(ret));
goto out;
}
}
ret = regulator_set_voltage_unlocked(rdev->supply,
best_supply_uV, INT_MAX, state);
if (ret)
- dev_warn(&rdev->dev, "Failed to decrease supply voltage: %d\n",
- ret);
+ dev_warn(&rdev->dev, "Failed to decrease supply voltage: %pe\n",
+ ERR_PTR(ret));
/* No need to fail here */
ret = 0;
}
err:
if (ret != -EPROBE_DEFER)
- dev_err(dev, "Failed to get supply '%s': %d\n",
- consumers[i].supply, ret);
+ dev_err(dev, "Failed to get supply '%s': %pe\n",
+ consumers[i].supply, ERR_PTR(ret));
else
dev_dbg(dev, "Failed to get supply '%s', deferring\n",
consumers[i].supply);
err:
for (i = 0; i < num_consumers; i++) {
if (consumers[i].ret < 0)
- pr_err("Failed to enable %s: %d\n", consumers[i].supply,
- consumers[i].ret);
+ pr_err("Failed to enable %s: %pe\n", consumers[i].supply,
+ ERR_PTR(consumers[i].ret));
else
regulator_disable(consumers[i].consumer);
}
return 0;
err:
- pr_err("Failed to disable %s: %d\n", consumers[i].supply, ret);
+ pr_err("Failed to disable %s: %pe\n", consumers[i].supply, ERR_PTR(ret));
for (++i; i < num_consumers; ++i) {
r = regulator_enable(consumers[i].consumer);
if (r != 0)
- pr_err("Failed to re-enable %s: %d\n",
- consumers[i].supply, r);
+ pr_err("Failed to re-enable %s: %pe\n",
+ consumers[i].supply, ERR_PTR(r));
}
return ret;
* @data: callback-specific data.
*
* Called by regulator drivers to notify clients a regulator event has
- * occurred. We also notify regulator clients downstream.
- * Note lock must be held by caller.
+ * occurred.
*/
int regulator_notifier_call_chain(struct regulator_dev *rdev,
unsigned long event, void *data)
{
- lockdep_assert_held_once(&rdev->mutex.base);
-
_notifier_call_chain(rdev, event, data);
return NOTIFY_DONE;
if (coupler && coupler->detach_regulator) {
err = coupler->detach_regulator(coupler, rdev);
if (err)
- rdev_err(rdev, "failed to detach from coupler: %d\n",
- err);
+ rdev_err(rdev, "failed to detach from coupler: %pe\n",
+ ERR_PTR(err));
}
kfree(rdev->coupling_desc.coupled_rdevs);
static int regulator_init_coupling(struct regulator_dev *rdev)
{
+ struct regulator_dev **coupled;
int err, n_phandles;
- size_t alloc_size;
if (!IS_ENABLED(CONFIG_OF))
n_phandles = 0;
else
n_phandles = of_get_n_coupled(rdev);
- alloc_size = sizeof(*rdev) * (n_phandles + 1);
-
- rdev->coupling_desc.coupled_rdevs = kzalloc(alloc_size, GFP_KERNEL);
- if (!rdev->coupling_desc.coupled_rdevs)
+ coupled = kcalloc(n_phandles + 1, sizeof(*coupled), GFP_KERNEL);
+ if (!coupled)
return -ENOMEM;
+ rdev->coupling_desc.coupled_rdevs = coupled;
+
/*
* Every regulator should always have coupling descriptor filled with
* at least pointer to itself.
if (IS_ERR(rdev->coupling_desc.coupler)) {
err = PTR_ERR(rdev->coupling_desc.coupler);
- rdev_err(rdev, "failed to get coupler: %d\n", err);
+ rdev_err(rdev, "failed to get coupler: %pe\n", ERR_PTR(err));
return err;
}
if (config->ena_gpiod) {
ret = regulator_ena_gpio_request(rdev, config);
if (ret != 0) {
- rdev_err(rdev, "Failed to request enable GPIO: %d\n",
- ret);
+ rdev_err(rdev, "Failed to request enable GPIO: %pe\n",
+ ERR_PTR(ret));
goto clean;
}
/* The regulator core took over the GPIO descriptor */
else if (regulator_desc->supply_name)
rdev->supply_name = regulator_desc->supply_name;
- /*
- * Attempt to resolve the regulator supply, if specified,
- * but don't return an error if we fail because we will try
- * to resolve it again later as more regulators are added.
- */
- if (regulator_resolve_supply(rdev))
- rdev_dbg(rdev, "unable to resolve supply\n");
-
ret = set_machine_constraints(rdev, constraints);
+ if (ret == -EPROBE_DEFER) {
+ /* Regulator might be in bypass mode and so needs its supply
+ * to set the constraints */
+ /* FIXME: this currently triggers a chicken-and-egg problem
+ * when creating -SUPPLY symlink in sysfs to a regulator
+ * that is just being created */
+ ret = regulator_resolve_supply(rdev);
+ if (!ret)
+ ret = set_machine_constraints(rdev, constraints);
+ else
+ rdev_dbg(rdev, "unable to resolve supply early: %pe\n",
+ ERR_PTR(ret));
+ }
if (ret < 0)
goto wash;
struct regulator_dev *rdev = dev_to_rdev(dev);
suspend_state_t state = pm_suspend_target_state;
int ret;
+ const struct regulator_state *rstate;
+
+ rstate = regulator_get_suspend_state_check(rdev, state);
+ if (!rstate)
+ return 0;
regulator_lock(rdev);
- ret = suspend_set_state(rdev, state);
+ ret = __suspend_set_state(rdev, rstate);
regulator_unlock(rdev);
return ret;
if (rstate == NULL)
return 0;
+ /* Avoid grabbing the lock if we don't need to */
+ if (!rdev->desc->ops->resume)
+ return 0;
+
regulator_lock(rdev);
- if (rdev->desc->ops->resume &&
- (rstate->enabled == ENABLE_IN_SUSPEND ||
- rstate->enabled == DISABLE_IN_SUSPEND))
+ if (rstate->enabled == ENABLE_IN_SUSPEND ||
+ rstate->enabled == DISABLE_IN_SUSPEND)
ret = rdev->desc->ops->resume(rdev);
regulator_unlock(rdev);
rdev_info(rdev, "disabling\n");
ret = _regulator_do_disable(rdev);
if (ret != 0)
- rdev_err(rdev, "couldn't disable: %d\n", ret);
+ rdev_err(rdev, "couldn't disable: %pe\n", ERR_PTR(ret));
} else {
/* The intention is that in future we will
* assume that full constraints are provided
{
struct da9055_regulator *regulator = data;
- regulator_lock(regulator->rdev);
regulator_notifier_call_chain(regulator->rdev,
REGULATOR_EVENT_OVER_CURRENT, NULL);
- regulator_unlock(regulator->rdev);
return IRQ_HANDLED;
}
continue;
if (BIT(regl->info->oc_event.lsb) & bits) {
- regulator_lock(regl->rdev);
regulator_notifier_call_chain(regl->rdev,
REGULATOR_EVENT_OVER_CURRENT, NULL);
- regulator_unlock(regl->rdev);
handled = IRQ_HANDLED;
}
}
continue;
if (BIT(regl->info->oc_event.lsb) & bits) {
- regulator_lock(regl->rdev);
regulator_notifier_call_chain(regl->rdev,
REGULATOR_EVENT_OVER_CURRENT, NULL);
- regulator_unlock(regl->rdev);
}
}
if (error < 0)
goto error_i2c;
- regulator_lock(chip->rdev);
-
if (val & DA9210_E_OVCURR) {
regulator_notifier_call_chain(chip->rdev,
REGULATOR_EVENT_OVER_CURRENT,
handled |= DA9210_E_VMAX;
}
- regulator_unlock(chip->rdev);
-
if (handled) {
/* Clear handled events */
error = regmap_write(chip->regmap, DA9210_REG_EVENT_B, handled);
* I2C driver interface functions
*/
-static const struct of_device_id da9210_dt_ids[] = {
+static const struct of_device_id __maybe_unused da9210_dt_ids[] = {
{ .compatible = "dlg,da9210", },
{ }
};
},
};
+static bool da9211_volatile_reg(struct device *dev, unsigned int reg)
+{
+ switch (reg) {
+ case DA9211_REG_STATUS_A:
+ case DA9211_REG_STATUS_B:
+ case DA9211_REG_EVENT_A:
+ case DA9211_REG_EVENT_B:
+ return true;
+ }
+ return false;
+}
+
static const struct regmap_config da9211_regmap_config = {
.reg_bits = 8,
.val_bits = 8,
.max_register = 5 * 128,
+ .volatile_reg = da9211_volatile_reg,
+ .cache_type = REGCACHE_RBTREE,
.ranges = da9211_regmap_range,
.num_ranges = ARRAY_SIZE(da9211_regmap_range),
};
goto error_i2c;
if (reg_val & DA9211_E_OV_CURR_A) {
- regulator_lock(chip->rdev[0]);
regulator_notifier_call_chain(chip->rdev[0],
REGULATOR_EVENT_OVER_CURRENT, NULL);
- regulator_unlock(chip->rdev[0]);
err = regmap_write(chip->regmap, DA9211_REG_EVENT_B,
DA9211_E_OV_CURR_A);
}
if (reg_val & DA9211_E_OV_CURR_B) {
- regulator_lock(chip->rdev[1]);
regulator_notifier_call_chain(chip->rdev[1],
REGULATOR_EVENT_OVER_CURRENT, NULL);
- regulator_unlock(chip->rdev[1]);
err = regmap_write(chip->regmap, DA9211_REG_EVENT_B,
DA9211_E_OV_CURR_B);
static struct ux500_regulator_debug {
struct dentry *dir;
- struct dentry *status_file;
- struct dentry *power_state_cnt_file;
struct dbx500_regulator_info *regulator_array;
int num_regulators;
u8 *state_before_suspend;
{
/* create directory */
rdebug.dir = debugfs_create_dir("ux500-regulator", NULL);
- if (!rdebug.dir)
- goto exit_no_debugfs;
/* create "status" file */
- rdebug.status_file = debugfs_create_file("status",
- S_IRUGO, rdebug.dir, &pdev->dev,
- &ux500_regulator_status_fops);
- if (!rdebug.status_file)
- goto exit_destroy_dir;
+ debugfs_create_file("status", S_IRUGO, rdebug.dir, &pdev->dev,
+ &ux500_regulator_status_fops);
/* create "power-state-count" file */
- rdebug.power_state_cnt_file = debugfs_create_file("power-state-count",
- S_IRUGO, rdebug.dir, &pdev->dev,
- &ux500_regulator_power_state_cnt_fops);
- if (!rdebug.power_state_cnt_file)
- goto exit_destroy_status;
+ debugfs_create_file("power-state-count", S_IRUGO, rdebug.dir,
+ &pdev->dev, &ux500_regulator_power_state_cnt_fops);
rdebug.regulator_array = regulator_info;
rdebug.num_regulators = num_regulators;
exit_free:
kfree(rdebug.state_before_suspend);
exit_destroy_power_state:
- debugfs_remove(rdebug.power_state_cnt_file);
-exit_destroy_status:
- debugfs_remove(rdebug.status_file);
-exit_destroy_dir:
- debugfs_remove(rdebug.dir);
-exit_no_debugfs:
- dev_err(&pdev->dev, "failed to create debugfs entries.\n");
+ debugfs_remove_recursive(rdebug.dir);
return -ENOMEM;
}
struct regulator_dev *dummy_regulator_rdev;
-static struct regulator_init_data dummy_initdata = {
+static const struct regulator_init_data dummy_initdata = {
.constraints = {
.always_on = 1,
},
};
-static struct regulator_ops dummy_ops;
+static const struct regulator_ops dummy_ops;
static const struct regulator_desc dummy_desc = {
.name = "regulator-dummy",
return pdata;
}
-static const struct of_device_id fan53555_dt_ids[] = {
+static const struct of_device_id __maybe_unused fan53555_dt_ids[] = {
{
.compatible = "fcs,fan53526",
.data = (void *)FAN53526_VENDOR_FAIRCHILD,
bool has_enable_clock;
};
-static const struct fixed_dev_type fixed_voltage_data = {
- .has_enable_clock = false,
-};
-
-static const struct fixed_dev_type fixed_clkenable_data = {
- .has_enable_clock = true,
-};
-
static int reg_clock_enable(struct regulator_dev *rdev)
{
struct fixed_voltage_data *priv = rdev_get_drvdata(rdev);
return config;
}
-static struct regulator_ops fixed_voltage_ops = {
+static const struct regulator_ops fixed_voltage_ops = {
};
-static struct regulator_ops fixed_voltage_clkenabled_ops = {
+static const struct regulator_ops fixed_voltage_clkenabled_ops = {
.enable = reg_clock_enable,
.disable = reg_clock_disable,
.is_enabled = reg_clock_is_enabled,
}
#if defined(CONFIG_OF)
+static const struct fixed_dev_type fixed_voltage_data = {
+ .has_enable_clock = false,
+};
+
+static const struct fixed_dev_type fixed_clkenable_data = {
+ .has_enable_clock = true,
+};
+
static const struct of_device_id fixed_of_match[] = {
{
.compatible = "regulator-fixed",
};
static const struct linear_range lochnagar_vddcore_ranges[] = {
+ REGULATOR_LINEAR_RANGE(600000, 0, 0x7, 0),
REGULATOR_LINEAR_RANGE(600000, 0x8, 0x41, 12500),
};
struct regulator_dev *rdev[LP8755_BUCK_MAX];
};
-/**
- *lp8755_read : read a single register value from lp8755.
- *@pchip : device to read from
- *@reg : register to read from
- *@val : pointer to store read value
- */
-static int lp8755_read(struct lp8755_chip *pchip, unsigned int reg,
- unsigned int *val)
-{
- return regmap_read(pchip->regmap, reg, val);
-}
-
-/**
- *lp8755_write : write a single register value to lp8755.
- *@pchip : device to write to
- *@reg : register to write to
- *@val : value to be written
- */
-static int lp8755_write(struct lp8755_chip *pchip, unsigned int reg,
- unsigned int val)
-{
- return regmap_write(pchip->regmap, reg, val);
-}
-
-/**
- *lp8755_update_bits : set the values of bit fields in lp8755 register.
- *@pchip : device to read from
- *@reg : register to update
- *@mask : bitmask to be changed
- *@val : value for bitmask
- */
-static int lp8755_update_bits(struct lp8755_chip *pchip, unsigned int reg,
- unsigned int mask, unsigned int val)
-{
- return regmap_update_bits(pchip->regmap, reg, mask, val);
-}
-
static int lp8755_buck_enable_time(struct regulator_dev *rdev)
{
int ret;
unsigned int regval;
enum lp8755_bucks id = rdev_get_id(rdev);
- struct lp8755_chip *pchip = rdev_get_drvdata(rdev);
- ret = lp8755_read(pchip, 0x12 + id, ®val);
+ ret = regmap_read(rdev->regmap, 0x12 + id, ®val);
if (ret < 0) {
- dev_err(pchip->dev, "i2c access error %s\n", __func__);
+ dev_err(&rdev->dev, "i2c access error %s\n", __func__);
return ret;
}
return (regval & 0xff) * 100;
break;
case REGULATOR_MODE_NORMAL:
/* enable automatic pwm/pfm mode */
- ret = lp8755_update_bits(pchip, 0x08 + id, 0x20, 0x00);
+ ret = regmap_update_bits(rdev->regmap, 0x08 + id, 0x20, 0x00);
if (ret < 0)
goto err_i2c;
break;
case REGULATOR_MODE_IDLE:
/* enable automatic pwm/pfm/lppfm mode */
- ret = lp8755_update_bits(pchip, 0x08 + id, 0x20, 0x20);
+ ret = regmap_update_bits(rdev->regmap, 0x08 + id, 0x20, 0x20);
if (ret < 0)
goto err_i2c;
- ret = lp8755_update_bits(pchip, 0x10, 0x01, 0x01);
+ ret = regmap_update_bits(rdev->regmap, 0x10, 0x01, 0x01);
if (ret < 0)
goto err_i2c;
break;
regbval = (0x01 << id);
}
- ret = lp8755_update_bits(pchip, 0x06, 0x01 << id, regbval);
+ ret = regmap_update_bits(rdev->regmap, 0x06, 0x01 << id, regbval);
if (ret < 0)
goto err_i2c;
return ret;
err_i2c:
- dev_err(pchip->dev, "i2c access error %s\n", __func__);
+ dev_err(&rdev->dev, "i2c access error %s\n", __func__);
return ret;
}
int ret;
unsigned int regval;
enum lp8755_bucks id = rdev_get_id(rdev);
- struct lp8755_chip *pchip = rdev_get_drvdata(rdev);
- ret = lp8755_read(pchip, 0x06, ®val);
+ ret = regmap_read(rdev->regmap, 0x06, ®val);
if (ret < 0)
goto err_i2c;
if (regval & (0x01 << id))
return REGULATOR_MODE_FAST;
- ret = lp8755_read(pchip, 0x08 + id, ®val);
+ ret = regmap_read(rdev->regmap, 0x08 + id, ®val);
if (ret < 0)
goto err_i2c;
return REGULATOR_MODE_NORMAL;
err_i2c:
- dev_err(pchip->dev, "i2c access error %s\n", __func__);
+ dev_err(&rdev->dev, "i2c access error %s\n", __func__);
return 0;
}
int ret;
unsigned int regval = 0x00;
enum lp8755_bucks id = rdev_get_id(rdev);
- struct lp8755_chip *pchip = rdev_get_drvdata(rdev);
/* uV/us */
switch (ramp) {
regval = 0x00;
break;
default:
- dev_err(pchip->dev,
+ dev_err(&rdev->dev,
"Not supported ramp value %d %s\n", ramp, __func__);
return -EINVAL;
}
- ret = lp8755_update_bits(pchip, 0x07 + id, 0x07, regval);
+ ret = regmap_update_bits(rdev->regmap, 0x07 + id, 0x07, regval);
if (ret < 0)
goto err_i2c;
return ret;
err_i2c:
- dev_err(pchip->dev, "i2c access error %s\n", __func__);
+ dev_err(&rdev->dev, "i2c access error %s\n", __func__);
return ret;
}
struct lp8755_platform_data *pdata = pchip->pdata;
/* read back muti-phase configuration */
- ret = lp8755_read(pchip, 0x3D, ®val);
+ ret = regmap_read(pchip->regmap, 0x3D, ®val);
if (ret < 0)
goto out_i2c_error;
pchip->mphase = regval & 0x0F;
struct lp8755_chip *pchip = data;
/* read flag0 register */
- ret = lp8755_read(pchip, 0x0D, &flag0);
+ ret = regmap_read(pchip->regmap, 0x0D, &flag0);
if (ret < 0)
goto err_i2c;
/* clear flag register to pull up int. pin */
- ret = lp8755_write(pchip, 0x0D, 0x00);
+ ret = regmap_write(pchip->regmap, 0x0D, 0x00);
if (ret < 0)
goto err_i2c;
if ((flag0 & (0x4 << icnt))
&& (pchip->irqmask & (0x04 << icnt))
&& (pchip->rdev[icnt] != NULL)) {
- regulator_lock(pchip->rdev[icnt]);
regulator_notifier_call_chain(pchip->rdev[icnt],
LP8755_EVENT_PWR_FAULT,
NULL);
- regulator_unlock(pchip->rdev[icnt]);
}
/* read flag1 register */
- ret = lp8755_read(pchip, 0x0E, &flag1);
+ ret = regmap_read(pchip->regmap, 0x0E, &flag1);
if (ret < 0)
goto err_i2c;
/* clear flag register to pull up int. pin */
- ret = lp8755_write(pchip, 0x0E, 0x00);
+ ret = regmap_write(pchip->regmap, 0x0E, 0x00);
if (ret < 0)
goto err_i2c;
if ((flag1 & 0x01) && (pchip->irqmask & 0x01))
for (icnt = 0; icnt < LP8755_BUCK_MAX; icnt++)
if (pchip->rdev[icnt] != NULL) {
- regulator_lock(pchip->rdev[icnt]);
regulator_notifier_call_chain(pchip->rdev[icnt],
LP8755_EVENT_OCP,
NULL);
- regulator_unlock(pchip->rdev[icnt]);
}
/* send OVP event to all regulator devices */
if ((flag1 & 0x02) && (pchip->irqmask & 0x02))
for (icnt = 0; icnt < LP8755_BUCK_MAX; icnt++)
if (pchip->rdev[icnt] != NULL) {
- regulator_lock(pchip->rdev[icnt]);
regulator_notifier_call_chain(pchip->rdev[icnt],
LP8755_EVENT_OVP,
NULL);
- regulator_unlock(pchip->rdev[icnt]);
}
return IRQ_HANDLED;
return 0;
}
- ret = lp8755_read(pchip, 0x0F, ®val);
+ ret = regmap_read(pchip->regmap, 0x0F, ®val);
if (ret < 0) {
dev_err(pchip->dev, "i2c access error %s\n", __func__);
return ret;
err:
/* output disable */
for (icnt = 0; icnt < LP8755_BUCK_MAX; icnt++)
- lp8755_write(pchip, icnt, 0x00);
+ regmap_write(pchip->regmap, icnt, 0x00);
return ret;
}
struct lp8755_chip *pchip = i2c_get_clientdata(client);
for (icnt = 0; icnt < LP8755_BUCK_MAX; icnt++)
- lp8755_write(pchip, icnt, 0x00);
+ regmap_write(pchip->regmap, icnt, 0x00);
return 0;
}
if (irqstat & LTC3589_IRQSTAT_THERMAL_WARN) {
event = REGULATOR_EVENT_OVER_TEMP;
- for (i = 0; i < LTC3589_NUM_REGULATORS; i++) {
- regulator_lock(ltc3589->regulators[i]);
+ for (i = 0; i < LTC3589_NUM_REGULATORS; i++)
regulator_notifier_call_chain(ltc3589->regulators[i],
event, NULL);
- regulator_unlock(ltc3589->regulators[i]);
- }
}
if (irqstat & LTC3589_IRQSTAT_UNDERVOLT_WARN) {
event = REGULATOR_EVENT_UNDER_VOLTAGE;
- for (i = 0; i < LTC3589_NUM_REGULATORS; i++) {
- regulator_lock(ltc3589->regulators[i]);
+ for (i = 0; i < LTC3589_NUM_REGULATORS; i++)
regulator_notifier_call_chain(ltc3589->regulators[i],
event, NULL);
- regulator_unlock(ltc3589->regulators[i]);
- }
}
/* Clear warning condition */
};
MODULE_DEVICE_TABLE(i2c, ltc3589_i2c_id);
-static const struct of_device_id ltc3589_of_match[] = {
+static const struct of_device_id __maybe_unused ltc3589_of_match[] = {
{
.compatible = "lltc,ltc3589",
.data = (void *)LTC3589,
if (irqstat & LTC3676_IRQSTAT_THERMAL_WARN) {
dev_warn(dev, "Over-temperature Warning\n");
event = REGULATOR_EVENT_OVER_TEMP;
- for (i = 0; i < LTC3676_NUM_REGULATORS; i++) {
- regulator_lock(ltc3676->regulators[i]);
+ for (i = 0; i < LTC3676_NUM_REGULATORS; i++)
regulator_notifier_call_chain(ltc3676->regulators[i],
event, NULL);
- regulator_unlock(ltc3676->regulators[i]);
- }
}
if (irqstat & LTC3676_IRQSTAT_UNDERVOLT_WARN) {
dev_info(dev, "Undervoltage Warning\n");
event = REGULATOR_EVENT_UNDER_VOLTAGE;
- for (i = 0; i < LTC3676_NUM_REGULATORS; i++) {
- regulator_lock(ltc3676->regulators[i]);
+ for (i = 0; i < LTC3676_NUM_REGULATORS; i++)
regulator_notifier_call_chain(ltc3676->regulators[i],
event, NULL);
- regulator_unlock(ltc3676->regulators[i]);
- }
}
/* Clear warning condition */
};
MODULE_DEVICE_TABLE(i2c, ltc3676_i2c_id);
-static const struct of_device_id ltc3676_of_match[] = {
+static const struct of_device_id __maybe_unused ltc3676_of_match[] = {
{ .compatible = "lltc,ltc3676" },
{ },
};
return 0;
}
-static const struct of_device_id max1586_of_match[] = {
+static const struct of_device_id __maybe_unused max1586_of_match[] = {
{ .compatible = "maxim,max1586", },
{},
};
return max77826_read_device_id(regmap, dev);
}
-static const struct of_device_id max77826_of_match[] = {
+static const struct of_device_id __maybe_unused max77826_of_match[] = {
{ .compatible = "maxim,max77826" },
{ /* sentinel */ }
};
#define MP886X_V_BOOT (1 << 7)
#define MP886X_SYSCNTLREG1 0x01
#define MP886X_MODE (1 << 0)
+#define MP886X_SLEW_SHIFT 3
+#define MP886X_SLEW_MASK (0x7 << MP886X_SLEW_SHIFT)
#define MP886X_GO (1 << 6)
#define MP886X_EN (1 << 7)
+#define MP8869_SYSCNTLREG2 0x02
+
+struct mp886x_cfg_info {
+ const struct regulator_ops *rops;
+ const int slew_rates[8];
+ const int switch_freq[4];
+ const u8 fs_reg;
+ const u8 fs_shift;
+};
struct mp886x_device_info {
struct device *dev;
struct regulator_desc desc;
struct regulator_init_data *regulator;
struct gpio_desc *en_gpio;
+ const struct mp886x_cfg_info *ci;
u32 r[2];
unsigned int sel;
};
+static int mp886x_set_ramp(struct regulator_dev *rdev, int ramp)
+{
+ struct mp886x_device_info *di = rdev_get_drvdata(rdev);
+ const struct mp886x_cfg_info *ci = di->ci;
+ int reg = -1, i;
+
+ for (i = 0; i < ARRAY_SIZE(ci->slew_rates); i++) {
+ if (ramp <= ci->slew_rates[i])
+ reg = i;
+ else
+ break;
+ }
+
+ if (reg < 0) {
+ dev_err(di->dev, "unsupported ramp value %d\n", ramp);
+ return -EINVAL;
+ }
+
+ return regmap_update_bits(rdev->regmap, MP886X_SYSCNTLREG1,
+ MP886X_SLEW_MASK, reg << MP886X_SLEW_SHIFT);
+}
+
+static void mp886x_set_switch_freq(struct mp886x_device_info *di,
+ struct regmap *regmap,
+ u32 freq)
+{
+ const struct mp886x_cfg_info *ci = di->ci;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(ci->switch_freq); i++) {
+ if (freq == ci->switch_freq[i]) {
+ regmap_update_bits(regmap, ci->fs_reg,
+ 0x3 << ci->fs_shift, i << ci->fs_shift);
+ return;
+ }
+ }
+
+ dev_err(di->dev, "invalid frequency %d\n", freq);
+}
+
static int mp886x_set_mode(struct regulator_dev *rdev, unsigned int mode)
{
switch (mode) {
.is_enabled = regulator_is_enabled_regmap,
.set_mode = mp886x_set_mode,
.get_mode = mp886x_get_mode,
+ .set_ramp_delay = mp886x_set_ramp,
+};
+
+static const struct mp886x_cfg_info mp8869_ci = {
+ .rops = &mp8869_regulator_ops,
+ .slew_rates = {
+ 40000,
+ 30000,
+ 20000,
+ 10000,
+ 5000,
+ 2500,
+ 1250,
+ 625,
+ },
+ .switch_freq = {
+ 500000,
+ 750000,
+ 1000000,
+ 1250000,
+ },
+ .fs_reg = MP8869_SYSCNTLREG2,
+ .fs_shift = 4,
};
static int mp8867_set_voltage_sel(struct regulator_dev *rdev, unsigned int sel)
.is_enabled = regulator_is_enabled_regmap,
.set_mode = mp886x_set_mode,
.get_mode = mp886x_get_mode,
+ .set_ramp_delay = mp886x_set_ramp,
+};
+
+static const struct mp886x_cfg_info mp8867_ci = {
+ .rops = &mp8867_regulator_ops,
+ .slew_rates = {
+ 64000,
+ 32000,
+ 16000,
+ 8000,
+ 4000,
+ 2000,
+ 1000,
+ 500,
+ },
+ .switch_freq = {
+ 500000,
+ 750000,
+ 1000000,
+ 1500000,
+ },
+ .fs_reg = MP886X_SYSCNTLREG1,
+ .fs_shift = 1,
};
static int mp886x_regulator_register(struct mp886x_device_info *di,
rdesc->name = "mp886x-reg";
rdesc->supply_name = "vin";
- rdesc->ops = of_device_get_match_data(di->dev);
+ rdesc->ops = di->ci->rops;
rdesc->type = REGULATOR_VOLTAGE;
rdesc->n_voltages = 128;
rdesc->enable_reg = MP886X_SYSCNTLREG1;
struct mp886x_device_info *di;
struct regulator_config config = { };
struct regmap *regmap;
+ u32 freq;
int ret;
di = devm_kzalloc(dev, sizeof(struct mp886x_device_info), GFP_KERNEL);
if (IS_ERR(di->en_gpio))
return PTR_ERR(di->en_gpio);
+ di->ci = of_device_get_match_data(dev);
di->dev = dev;
regmap = devm_regmap_init_i2c(client, &mp886x_regmap_config);
config.driver_data = di;
config.of_node = np;
+ if (!of_property_read_u32(np, "mps,switch-frequency-hz", &freq))
+ mp886x_set_switch_freq(di, regmap, freq);
+
ret = mp886x_regulator_register(di, &config);
if (ret < 0)
dev_err(dev, "Failed to register regulator!\n");
static const struct of_device_id mp886x_dt_ids[] = {
{
.compatible = "mps,mp8867",
- .data = &mp8867_regulator_ops
+ .data = &mp8867_ci
},
{
.compatible = "mps,mp8869",
- .data = &mp8869_regulator_ops
+ .data = &mp8869_ci
},
{ }
};
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// Copyright (C) 2020 MediaTek Inc.
+//
+// Author: Gene Chen <gene_chen@richtek.com>
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+
+#include <dt-bindings/regulator/mediatek,mt6360-regulator.h>
+
+enum {
+ MT6360_REGULATOR_BUCK1 = 0,
+ MT6360_REGULATOR_BUCK2,
+ MT6360_REGULATOR_LDO6,
+ MT6360_REGULATOR_LDO7,
+ MT6360_REGULATOR_LDO1,
+ MT6360_REGULATOR_LDO2,
+ MT6360_REGULATOR_LDO3,
+ MT6360_REGULATOR_LDO5,
+ MT6360_REGULATOR_MAX,
+};
+
+struct mt6360_irq_mapping {
+ const char *name;
+ irq_handler_t handler;
+};
+
+struct mt6360_regulator_desc {
+ const struct regulator_desc desc;
+ unsigned int mode_reg;
+ unsigned int mode_mask;
+ unsigned int state_reg;
+ unsigned int state_mask;
+ const struct mt6360_irq_mapping *irq_tables;
+ int irq_table_size;
+};
+
+struct mt6360_regulator_data {
+ struct device *dev;
+ struct regmap *regmap;
+};
+
+static irqreturn_t mt6360_pgb_event_handler(int irq, void *data)
+{
+ struct regulator_dev *rdev = data;
+
+ regulator_notifier_call_chain(rdev, REGULATOR_EVENT_FAIL, NULL);
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t mt6360_oc_event_handler(int irq, void *data)
+{
+ struct regulator_dev *rdev = data;
+
+ regulator_notifier_call_chain(rdev, REGULATOR_EVENT_OVER_CURRENT, NULL);
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t mt6360_ov_event_handler(int irq, void *data)
+{
+ struct regulator_dev *rdev = data;
+
+ regulator_notifier_call_chain(rdev, REGULATOR_EVENT_REGULATION_OUT, NULL);
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t mt6360_uv_event_handler(int irq, void *data)
+{
+ struct regulator_dev *rdev = data;
+
+ regulator_notifier_call_chain(rdev, REGULATOR_EVENT_UNDER_VOLTAGE, NULL);
+ return IRQ_HANDLED;
+}
+
+static const struct mt6360_irq_mapping buck1_irq_tbls[] = {
+ { "buck1_pgb_evt", mt6360_pgb_event_handler },
+ { "buck1_oc_evt", mt6360_oc_event_handler },
+ { "buck1_ov_evt", mt6360_ov_event_handler },
+ { "buck1_uv_evt", mt6360_uv_event_handler },
+};
+
+static const struct mt6360_irq_mapping buck2_irq_tbls[] = {
+ { "buck2_pgb_evt", mt6360_pgb_event_handler },
+ { "buck2_oc_evt", mt6360_oc_event_handler },
+ { "buck2_ov_evt", mt6360_ov_event_handler },
+ { "buck2_uv_evt", mt6360_uv_event_handler },
+};
+
+static const struct mt6360_irq_mapping ldo6_irq_tbls[] = {
+ { "ldo6_pgb_evt", mt6360_pgb_event_handler },
+ { "ldo6_oc_evt", mt6360_oc_event_handler },
+};
+
+static const struct mt6360_irq_mapping ldo7_irq_tbls[] = {
+ { "ldo7_pgb_evt", mt6360_pgb_event_handler },
+ { "ldo7_oc_evt", mt6360_oc_event_handler },
+};
+
+static const struct mt6360_irq_mapping ldo1_irq_tbls[] = {
+ { "ldo1_pgb_evt", mt6360_pgb_event_handler },
+ { "ldo1_oc_evt", mt6360_oc_event_handler },
+};
+
+static const struct mt6360_irq_mapping ldo2_irq_tbls[] = {
+ { "ldo2_pgb_evt", mt6360_pgb_event_handler },
+ { "ldo2_oc_evt", mt6360_oc_event_handler },
+};
+
+static const struct mt6360_irq_mapping ldo3_irq_tbls[] = {
+ { "ldo3_pgb_evt", mt6360_pgb_event_handler },
+ { "ldo3_oc_evt", mt6360_oc_event_handler },
+};
+
+static const struct mt6360_irq_mapping ldo5_irq_tbls[] = {
+ { "ldo5_pgb_evt", mt6360_pgb_event_handler },
+ { "ldo5_oc_evt", mt6360_oc_event_handler },
+};
+
+static const struct linear_range buck_vout_ranges[] = {
+ REGULATOR_LINEAR_RANGE(300000, 0x00, 0xc7, 5000),
+ REGULATOR_LINEAR_RANGE(1300000, 0xc8, 0xff, 0),
+};
+
+static const struct linear_range ldo_vout_ranges1[] = {
+ REGULATOR_LINEAR_RANGE(500000, 0x00, 0x09, 10000),
+ REGULATOR_LINEAR_RANGE(600000, 0x0a, 0x10, 0),
+ REGULATOR_LINEAR_RANGE(610000, 0x11, 0x19, 10000),
+ REGULATOR_LINEAR_RANGE(700000, 0x1a, 0x20, 0),
+ REGULATOR_LINEAR_RANGE(710000, 0x21, 0x29, 10000),
+ REGULATOR_LINEAR_RANGE(800000, 0x2a, 0x30, 0),
+ REGULATOR_LINEAR_RANGE(810000, 0x31, 0x39, 10000),
+ REGULATOR_LINEAR_RANGE(900000, 0x3a, 0x40, 0),
+ REGULATOR_LINEAR_RANGE(910000, 0x41, 0x49, 10000),
+ REGULATOR_LINEAR_RANGE(1000000, 0x4a, 0x50, 0),
+ REGULATOR_LINEAR_RANGE(1010000, 0x51, 0x59, 10000),
+ REGULATOR_LINEAR_RANGE(1100000, 0x5a, 0x60, 0),
+ REGULATOR_LINEAR_RANGE(1110000, 0x61, 0x69, 10000),
+ REGULATOR_LINEAR_RANGE(1200000, 0x6a, 0x70, 0),
+ REGULATOR_LINEAR_RANGE(1210000, 0x71, 0x79, 10000),
+ REGULATOR_LINEAR_RANGE(1300000, 0x7a, 0x80, 0),
+ REGULATOR_LINEAR_RANGE(1310000, 0x81, 0x89, 10000),
+ REGULATOR_LINEAR_RANGE(1400000, 0x8a, 0x90, 0),
+ REGULATOR_LINEAR_RANGE(1410000, 0x91, 0x99, 10000),
+ REGULATOR_LINEAR_RANGE(1500000, 0x9a, 0xa0, 0),
+ REGULATOR_LINEAR_RANGE(1510000, 0xa1, 0xa9, 10000),
+ REGULATOR_LINEAR_RANGE(1600000, 0xaa, 0xb0, 0),
+ REGULATOR_LINEAR_RANGE(1610000, 0xb1, 0xb9, 10000),
+ REGULATOR_LINEAR_RANGE(1700000, 0xba, 0xc0, 0),
+ REGULATOR_LINEAR_RANGE(1710000, 0xc1, 0xc9, 10000),
+ REGULATOR_LINEAR_RANGE(1800000, 0xca, 0xd0, 0),
+ REGULATOR_LINEAR_RANGE(1810000, 0xd1, 0xd9, 10000),
+ REGULATOR_LINEAR_RANGE(1900000, 0xda, 0xe0, 0),
+ REGULATOR_LINEAR_RANGE(1910000, 0xe1, 0xe9, 10000),
+ REGULATOR_LINEAR_RANGE(2000000, 0xea, 0xf0, 0),
+ REGULATOR_LINEAR_RANGE(2010000, 0xf1, 0xf9, 10000),
+ REGULATOR_LINEAR_RANGE(2100000, 0xfa, 0xff, 0),
+};
+
+static const struct linear_range ldo_vout_ranges2[] = {
+ REGULATOR_LINEAR_RANGE(1200000, 0x00, 0x09, 10000),
+ REGULATOR_LINEAR_RANGE(1300000, 0x0a, 0x10, 0),
+ REGULATOR_LINEAR_RANGE(1310000, 0x11, 0x19, 10000),
+ REGULATOR_LINEAR_RANGE(1400000, 0x1a, 0x1f, 0),
+ REGULATOR_LINEAR_RANGE(1500000, 0x20, 0x29, 10000),
+ REGULATOR_LINEAR_RANGE(1600000, 0x2a, 0x2f, 0),
+ REGULATOR_LINEAR_RANGE(1700000, 0x30, 0x39, 10000),
+ REGULATOR_LINEAR_RANGE(1800000, 0x3a, 0x40, 0),
+ REGULATOR_LINEAR_RANGE(1810000, 0x41, 0x49, 10000),
+ REGULATOR_LINEAR_RANGE(1900000, 0x4a, 0x4f, 0),
+ REGULATOR_LINEAR_RANGE(2000000, 0x50, 0x59, 10000),
+ REGULATOR_LINEAR_RANGE(2100000, 0x5a, 0x60, 0),
+ REGULATOR_LINEAR_RANGE(2110000, 0x61, 0x69, 10000),
+ REGULATOR_LINEAR_RANGE(2200000, 0x6a, 0x6f, 0),
+ REGULATOR_LINEAR_RANGE(2500000, 0x70, 0x79, 10000),
+ REGULATOR_LINEAR_RANGE(2600000, 0x7a, 0x7f, 0),
+ REGULATOR_LINEAR_RANGE(2700000, 0x80, 0x89, 10000),
+ REGULATOR_LINEAR_RANGE(2800000, 0x8a, 0x90, 0),
+ REGULATOR_LINEAR_RANGE(2810000, 0x91, 0x99, 10000),
+ REGULATOR_LINEAR_RANGE(2900000, 0x9a, 0xa0, 0),
+ REGULATOR_LINEAR_RANGE(2910000, 0xa1, 0xa9, 10000),
+ REGULATOR_LINEAR_RANGE(3000000, 0xaa, 0xb0, 0),
+ REGULATOR_LINEAR_RANGE(3010000, 0xb1, 0xb9, 10000),
+ REGULATOR_LINEAR_RANGE(3100000, 0xba, 0xc0, 0),
+ REGULATOR_LINEAR_RANGE(3110000, 0xc1, 0xc9, 10000),
+ REGULATOR_LINEAR_RANGE(3200000, 0xca, 0xcf, 0),
+ REGULATOR_LINEAR_RANGE(3300000, 0xd0, 0xd9, 10000),
+ REGULATOR_LINEAR_RANGE(3400000, 0xda, 0xe0, 0),
+ REGULATOR_LINEAR_RANGE(3410000, 0xe1, 0xe9, 10000),
+ REGULATOR_LINEAR_RANGE(3500000, 0xea, 0xf0, 0),
+ REGULATOR_LINEAR_RANGE(3510000, 0xf1, 0xf9, 10000),
+ REGULATOR_LINEAR_RANGE(3600000, 0xfa, 0xff, 0),
+};
+
+static const struct linear_range ldo_vout_ranges3[] = {
+ REGULATOR_LINEAR_RANGE(2700000, 0x00, 0x09, 10000),
+ REGULATOR_LINEAR_RANGE(2800000, 0x0a, 0x10, 0),
+ REGULATOR_LINEAR_RANGE(2810000, 0x11, 0x19, 10000),
+ REGULATOR_LINEAR_RANGE(2900000, 0x1a, 0x20, 0),
+ REGULATOR_LINEAR_RANGE(2910000, 0x21, 0x29, 10000),
+ REGULATOR_LINEAR_RANGE(3000000, 0x2a, 0x30, 0),
+ REGULATOR_LINEAR_RANGE(3010000, 0x31, 0x39, 10000),
+ REGULATOR_LINEAR_RANGE(3100000, 0x3a, 0x40, 0),
+ REGULATOR_LINEAR_RANGE(3110000, 0x41, 0x49, 10000),
+ REGULATOR_LINEAR_RANGE(3200000, 0x4a, 0x4f, 0),
+ REGULATOR_LINEAR_RANGE(3300000, 0x50, 0x59, 10000),
+ REGULATOR_LINEAR_RANGE(3400000, 0x5a, 0x60, 0),
+ REGULATOR_LINEAR_RANGE(3410000, 0x61, 0x69, 10000),
+ REGULATOR_LINEAR_RANGE(3500000, 0x6a, 0x70, 0),
+ REGULATOR_LINEAR_RANGE(3510000, 0x71, 0x79, 10000),
+ REGULATOR_LINEAR_RANGE(3600000, 0x7a, 0x7f, 0),
+};
+
+static int mt6360_regulator_set_mode(struct regulator_dev *rdev,
+ unsigned int mode)
+{
+ const struct mt6360_regulator_desc *rdesc = (struct mt6360_regulator_desc *)rdev->desc;
+ struct regmap *regmap = rdev_get_regmap(rdev);
+ int shift = ffs(rdesc->mode_mask) - 1;
+ unsigned int val;
+ int ret;
+
+ switch (mode) {
+ case REGULATOR_MODE_NORMAL:
+ val = MT6360_OPMODE_NORMAL;
+ break;
+ case REGULATOR_MODE_STANDBY:
+ val = MT6360_OPMODE_ULP;
+ break;
+ case REGULATOR_MODE_IDLE:
+ val = MT6360_OPMODE_LP;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ ret = regmap_update_bits(regmap, rdesc->mode_reg, rdesc->mode_mask, val << shift);
+ if (ret) {
+ dev_err(&rdev->dev, "%s: fail (%d)\n", __func__, ret);
+ return ret;
+ }
+
+ return 0;
+}
+
+static unsigned int mt6360_regulator_get_mode(struct regulator_dev *rdev)
+{
+ const struct mt6360_regulator_desc *rdesc = (struct mt6360_regulator_desc *)rdev->desc;
+ struct regmap *regmap = rdev_get_regmap(rdev);
+ int shift = ffs(rdesc->mode_mask) - 1;
+ unsigned int val;
+ int ret;
+
+ ret = regmap_read(regmap, rdesc->mode_reg, &val);
+ if (ret)
+ return ret;
+
+ val &= rdesc->mode_mask;
+ val >>= shift;
+
+ switch (val) {
+ case MT6360_OPMODE_LP:
+ return REGULATOR_MODE_IDLE;
+ case MT6360_OPMODE_ULP:
+ return REGULATOR_MODE_STANDBY;
+ case MT6360_OPMODE_NORMAL:
+ return REGULATOR_MODE_NORMAL;
+ default:
+ return -EINVAL;
+ }
+}
+
+static int mt6360_regulator_get_status(struct regulator_dev *rdev)
+{
+ const struct mt6360_regulator_desc *rdesc = (struct mt6360_regulator_desc *)rdev->desc;
+ struct regmap *regmap = rdev_get_regmap(rdev);
+ unsigned int val;
+ int ret;
+
+ ret = regmap_read(regmap, rdesc->state_reg, &val);
+ if (ret)
+ return ret;
+
+ if (val & rdesc->state_mask)
+ return REGULATOR_STATUS_ON;
+
+ return REGULATOR_STATUS_OFF;
+}
+
+static const struct regulator_ops mt6360_regulator_ops = {
+ .list_voltage = regulator_list_voltage_linear_range,
+ .enable = regulator_enable_regmap,
+ .disable = regulator_disable_regmap,
+ .is_enabled = regulator_is_enabled_regmap,
+ .set_voltage_sel = regulator_set_voltage_sel_regmap,
+ .get_voltage_sel = regulator_get_voltage_sel_regmap,
+ .set_mode = mt6360_regulator_set_mode,
+ .get_mode = mt6360_regulator_get_mode,
+ .get_status = mt6360_regulator_get_status,
+};
+
+static unsigned int mt6360_regulator_of_map_mode(unsigned int hw_mode)
+{
+ switch (hw_mode) {
+ case MT6360_OPMODE_NORMAL:
+ return REGULATOR_MODE_NORMAL;
+ case MT6360_OPMODE_LP:
+ return REGULATOR_MODE_IDLE;
+ case MT6360_OPMODE_ULP:
+ return REGULATOR_MODE_STANDBY;
+ default:
+ return REGULATOR_MODE_INVALID;
+ }
+}
+
+#define MT6360_REGULATOR_DESC(_name, _sname, ereg, emask, vreg, vmask, \
+ mreg, mmask, streg, stmask, vranges, \
+ vcnts, offon_delay, irq_tbls) \
+{ \
+ .desc = { \
+ .name = #_name, \
+ .supply_name = #_sname, \
+ .id = MT6360_REGULATOR_##_name, \
+ .of_match = of_match_ptr(#_name), \
+ .regulators_node = of_match_ptr("regulator"), \
+ .of_map_mode = mt6360_regulator_of_map_mode, \
+ .owner = THIS_MODULE, \
+ .ops = &mt6360_regulator_ops, \
+ .type = REGULATOR_VOLTAGE, \
+ .vsel_reg = vreg, \
+ .vsel_mask = vmask, \
+ .enable_reg = ereg, \
+ .enable_mask = emask, \
+ .linear_ranges = vranges, \
+ .n_linear_ranges = ARRAY_SIZE(vranges), \
+ .n_voltages = vcnts, \
+ .off_on_delay = offon_delay, \
+ }, \
+ .mode_reg = mreg, \
+ .mode_mask = mmask, \
+ .state_reg = streg, \
+ .state_mask = stmask, \
+ .irq_tables = irq_tbls, \
+ .irq_table_size = ARRAY_SIZE(irq_tbls), \
+}
+
+static const struct mt6360_regulator_desc mt6360_regulator_descs[] = {
+ MT6360_REGULATOR_DESC(BUCK1, BUCK1_VIN, 0x117, 0x40, 0x110, 0xff, 0x117, 0x30, 0x117, 0x04,
+ buck_vout_ranges, 256, 0, buck1_irq_tbls),
+ MT6360_REGULATOR_DESC(BUCK2, BUCK2_VIN, 0x127, 0x40, 0x120, 0xff, 0x127, 0x30, 0x127, 0x04,
+ buck_vout_ranges, 256, 0, buck2_irq_tbls),
+ MT6360_REGULATOR_DESC(LDO6, LDO_VIN3, 0x137, 0x40, 0x13B, 0xff, 0x137, 0x30, 0x137, 0x04,
+ ldo_vout_ranges1, 256, 0, ldo6_irq_tbls),
+ MT6360_REGULATOR_DESC(LDO7, LDO_VIN3, 0x131, 0x40, 0x135, 0xff, 0x131, 0x30, 0x131, 0x04,
+ ldo_vout_ranges1, 256, 0, ldo7_irq_tbls),
+ MT6360_REGULATOR_DESC(LDO1, LDO_VIN1, 0x217, 0x40, 0x21B, 0xff, 0x217, 0x30, 0x217, 0x04,
+ ldo_vout_ranges2, 256, 0, ldo1_irq_tbls),
+ MT6360_REGULATOR_DESC(LDO2, LDO_VIN1, 0x211, 0x40, 0x215, 0xff, 0x211, 0x30, 0x211, 0x04,
+ ldo_vout_ranges2, 256, 0, ldo2_irq_tbls),
+ MT6360_REGULATOR_DESC(LDO3, LDO_VIN1, 0x205, 0x40, 0x209, 0xff, 0x205, 0x30, 0x205, 0x04,
+ ldo_vout_ranges2, 256, 100, ldo3_irq_tbls),
+ MT6360_REGULATOR_DESC(LDO5, LDO_VIN2, 0x20B, 0x40, 0x20F, 0x7f, 0x20B, 0x30, 0x20B, 0x04,
+ ldo_vout_ranges3, 128, 100, ldo5_irq_tbls),
+};
+
+static int mt6360_regulator_irq_register(struct platform_device *pdev,
+ struct regulator_dev *rdev,
+ const struct mt6360_irq_mapping *tbls,
+ int tbl_size)
+{
+ int i, irq, ret;
+
+ for (i = 0; i < tbl_size; i++) {
+ const struct mt6360_irq_mapping *irq_desc = tbls + i;
+
+ irq = platform_get_irq_byname(pdev, irq_desc->name);
+ if (irq < 0) {
+ dev_err(&pdev->dev, "Fail to get %s irq\n", irq_desc->name);
+ return irq;
+ }
+
+ ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, irq_desc->handler, 0,
+ irq_desc->name, rdev);
+ if (ret) {
+ dev_err(&pdev->dev, "Fail to request %s irq\n", irq_desc->name);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+static int mt6360_regulator_probe(struct platform_device *pdev)
+{
+ struct mt6360_regulator_data *mrd;
+ struct regulator_config config = {};
+ int i, ret;
+
+ mrd = devm_kzalloc(&pdev->dev, sizeof(*mrd), GFP_KERNEL);
+ if (!mrd)
+ return -ENOMEM;
+
+ mrd->dev = &pdev->dev;
+
+ mrd->regmap = dev_get_regmap(pdev->dev.parent, NULL);
+ if (!mrd->regmap) {
+ dev_err(&pdev->dev, "Failed to get parent regmap\n");
+ return -ENODEV;
+ }
+
+ config.dev = pdev->dev.parent;
+ config.driver_data = mrd;
+ config.regmap = mrd->regmap;
+
+ for (i = 0; i < ARRAY_SIZE(mt6360_regulator_descs); i++) {
+ const struct mt6360_regulator_desc *rdesc = mt6360_regulator_descs + i;
+ struct regulator_dev *rdev;
+
+ rdev = devm_regulator_register(&pdev->dev, &rdesc->desc, &config);
+ if (IS_ERR(rdev)) {
+ dev_err(&pdev->dev, "Failed to register %d regulator\n", i);
+ return PTR_ERR(rdev);
+ }
+
+ ret = mt6360_regulator_irq_register(pdev, rdev, rdesc->irq_tables,
+ rdesc->irq_table_size);
+ if (ret) {
+ dev_err(&pdev->dev, "Failed to register %d regulator irqs\n", i);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+static const struct platform_device_id mt6360_regulator_id_table[] = {
+ { "mt6360-regulator", 0 },
+ {},
+};
+MODULE_DEVICE_TABLE(platform, mt6360_regulator_id_table);
+
+static struct platform_driver mt6360_regulator_driver = {
+ .driver = {
+ .name = "mt6360-regulator",
+ },
+ .probe = mt6360_regulator_probe,
+ .id_table = mt6360_regulator_id_table,
+};
+module_platform_driver(mt6360_regulator_driver);
+
+MODULE_AUTHOR("Gene Chen <gene_chen@richtek.com>");
+MODULE_DESCRIPTION("MT6360 Regulator Driver");
+MODULE_LICENSE("GPL v2");
BUCK1_RAMP_MASK, ramp_value << 6);
}
-static struct regulator_ops pca9450_dvs_buck_regulator_ops = {
+static const struct regulator_ops pca9450_dvs_buck_regulator_ops = {
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.is_enabled = regulator_is_enabled_regmap,
.set_ramp_delay = pca9450_dvs_set_ramp_delay,
};
-static struct regulator_ops pca9450_buck_regulator_ops = {
+static const struct regulator_ops pca9450_buck_regulator_ops = {
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.is_enabled = regulator_is_enabled_regmap,
.set_voltage_time_sel = regulator_set_voltage_time_sel,
};
-static struct regulator_ops pca9450_ldo_regulator_ops = {
+static const struct regulator_ops pca9450_ldo_regulator_ops = {
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.is_enabled = regulator_is_enabled_regmap,
if (reg_val & PV88060_E_VDD_FLT) {
for (i = 0; i < PV88060_MAX_REGULATORS; i++) {
- if (chip->rdev[i] != NULL) {
- regulator_lock(chip->rdev[i]);
+ if (chip->rdev[i] != NULL)
regulator_notifier_call_chain(chip->rdev[i],
REGULATOR_EVENT_UNDER_VOLTAGE,
NULL);
- regulator_unlock(chip->rdev[i]);
- }
}
err = regmap_write(chip->regmap, PV88060_REG_EVENT_A,
if (reg_val & PV88060_E_OVER_TEMP) {
for (i = 0; i < PV88060_MAX_REGULATORS; i++) {
- if (chip->rdev[i] != NULL) {
- regulator_lock(chip->rdev[i]);
+ if (chip->rdev[i] != NULL)
regulator_notifier_call_chain(chip->rdev[i],
REGULATOR_EVENT_OVER_TEMP,
NULL);
- regulator_unlock(chip->rdev[i]);
- }
}
err = regmap_write(chip->regmap, PV88060_REG_EVENT_A,
if (reg_val & PV88080_E_VDD_FLT) {
for (i = 0; i < PV88080_MAX_REGULATORS; i++) {
- if (chip->rdev[i] != NULL) {
- regulator_lock(chip->rdev[i]);
+ if (chip->rdev[i] != NULL)
regulator_notifier_call_chain(chip->rdev[i],
REGULATOR_EVENT_UNDER_VOLTAGE,
NULL);
- regulator_unlock(chip->rdev[i]);
- }
}
err = regmap_write(chip->regmap, PV88080_REG_EVENT_A,
if (reg_val & PV88080_E_OVER_TEMP) {
for (i = 0; i < PV88080_MAX_REGULATORS; i++) {
- if (chip->rdev[i] != NULL) {
- regulator_lock(chip->rdev[i]);
+ if (chip->rdev[i] != NULL)
regulator_notifier_call_chain(chip->rdev[i],
REGULATOR_EVENT_OVER_TEMP,
NULL);
- regulator_unlock(chip->rdev[i]);
- }
}
err = regmap_write(chip->regmap, PV88080_REG_EVENT_A,
if (reg_val & PV88090_E_VDD_FLT) {
for (i = 0; i < PV88090_MAX_REGULATORS; i++) {
- if (chip->rdev[i] != NULL) {
- regulator_lock(chip->rdev[i]);
+ if (chip->rdev[i] != NULL)
regulator_notifier_call_chain(chip->rdev[i],
REGULATOR_EVENT_UNDER_VOLTAGE,
NULL);
- regulator_unlock(chip->rdev[i]);
- }
}
err = regmap_write(chip->regmap, PV88090_REG_EVENT_A,
if (reg_val & PV88090_E_OVER_TEMP) {
for (i = 0; i < PV88090_MAX_REGULATORS; i++) {
- if (chip->rdev[i] != NULL) {
- regulator_lock(chip->rdev[i]);
+ if (chip->rdev[i] != NULL)
regulator_notifier_call_chain(chip->rdev[i],
REGULATOR_EVENT_OVER_TEMP,
NULL);
- regulator_unlock(chip->rdev[i]);
- }
}
err = regmap_write(chip->regmap, PV88090_REG_EVENT_A,
return 0;
}
-static const struct of_device_id pwm_of_match[] = {
+static const struct of_device_id __maybe_unused pwm_of_match[] = {
{ .compatible = "pwm-regulator" },
{ },
};
const char *name;
u8 type;
u16 base;
- struct regulator_desc *desc;
+ const struct regulator_desc *desc;
};
-static struct regulator_ops qcom_labibb_ops = {
+static const struct regulator_ops qcom_labibb_ops = {
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.is_enabled = regulator_is_enabled_regmap,
};
-static struct regulator_desc pmi8998_lab_desc = {
+static const struct regulator_desc pmi8998_lab_desc = {
.enable_mask = LAB_ENABLE_CTL_MASK,
.enable_reg = (PMI8998_LAB_REG_BASE + REG_LABIBB_ENABLE_CTL),
.enable_val = LABIBB_CONTROL_ENABLE,
.ops = &qcom_labibb_ops,
};
-static struct regulator_desc pmi8998_ibb_desc = {
+static const struct regulator_desc pmi8998_ibb_desc = {
.enable_mask = IBB_ENABLE_CTL_MASK,
.enable_reg = (PMI8998_IBB_REG_BASE + REG_LABIBB_ENABLE_CTL),
.enable_val = LABIBB_CONTROL_ENABLE,
return 0;
}
-static const struct of_device_id rpmh_regulator_match_table[] = {
+static const struct of_device_id __maybe_unused rpmh_regulator_match_table[] = {
{
.compatible = "qcom,pm8005-rpmh-regulators",
.data = pm8005_vreg_data,
.ops = &rpm_smps_ldo_ops,
};
+static const struct regulator_desc pm8953_lnldo = {
+ .linear_ranges = (struct linear_range[]) {
+ REGULATOR_LINEAR_RANGE(1380000, 8, 15, 120000),
+ REGULATOR_LINEAR_RANGE(690000, 0, 7, 60000),
+ },
+ .n_linear_ranges = 2,
+ .n_voltages = 16,
+ .ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pm8953_ult_nldo = {
+ .linear_ranges = (struct linear_range[]) {
+ REGULATOR_LINEAR_RANGE(375000, 0, 93, 12500),
+ },
+ .n_linear_ranges = 1,
+ .n_voltages = 94,
+ .ops = &rpm_smps_ldo_ops,
+};
static const struct regulator_desc pm8994_hfsmps = {
.linear_ranges = (struct linear_range[]) {
.ops = &rpm_bob_ops,
};
+static const struct regulator_desc pm660_ftsmps = {
+ .linear_ranges = (struct linear_range[]) {
+ REGULATOR_LINEAR_RANGE(355000, 0, 199, 5000),
+ },
+ .n_linear_ranges = 1,
+ .n_voltages = 200,
+ .ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pm660_hfsmps = {
+ .linear_ranges = (struct linear_range[]) {
+ REGULATOR_LINEAR_RANGE(320000, 0, 216, 8000),
+ },
+ .n_linear_ranges = 1,
+ .n_voltages = 217,
+ .ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pm660_ht_nldo = {
+ .linear_ranges = (struct linear_range[]) {
+ REGULATOR_LINEAR_RANGE(312000, 0, 124, 8000),
+ },
+ .n_linear_ranges = 1,
+ .n_voltages = 125,
+ .ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pm660_ht_lvpldo = {
+ .linear_ranges = (struct linear_range[]) {
+ REGULATOR_LINEAR_RANGE(1504000, 0, 62, 8000),
+ },
+ .n_linear_ranges = 1,
+ .n_voltages = 63,
+ .ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pm660_nldo660 = {
+ .linear_ranges = (struct linear_range[]) {
+ REGULATOR_LINEAR_RANGE(320000, 0, 123, 8000),
+ },
+ .n_linear_ranges = 1,
+ .n_voltages = 124,
+ .ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pm660_pldo660 = {
+ .linear_ranges = (struct linear_range[]) {
+ REGULATOR_LINEAR_RANGE(1504000, 0, 255, 8000),
+ },
+ .n_linear_ranges = 1,
+ .n_voltages = 256,
+ .ops = &rpm_smps_ldo_ops,
+};
+
+static const struct regulator_desc pm660l_bob = {
+ .linear_ranges = (struct linear_range[]) {
+ REGULATOR_LINEAR_RANGE(1800000, 0, 84, 32000),
+ },
+ .n_linear_ranges = 1,
+ .n_voltages = 85,
+ .ops = &rpm_bob_ops,
+};
+
static const struct regulator_desc pms405_hfsmps3 = {
.linear_ranges = (struct linear_range[]) {
REGULATOR_LINEAR_RANGE(320000, 0, 215, 8000),
{}
};
+static const struct rpm_regulator_data rpm_pm8953_regulators[] = {
+ { "s1", QCOM_SMD_RPM_SMPA, 1, &pm8998_hfsmps, "vdd_s1" },
+ { "s2", QCOM_SMD_RPM_SMPA, 2, &pm8998_hfsmps, "vdd_s2" },
+ { "s3", QCOM_SMD_RPM_SMPA, 3, &pm8998_hfsmps, "vdd_s3" },
+ { "s4", QCOM_SMD_RPM_SMPA, 4, &pm8998_hfsmps, "vdd_s4" },
+ { "s5", QCOM_SMD_RPM_SMPA, 5, &pm8950_ftsmps2p5, "vdd_s5" },
+ { "s6", QCOM_SMD_RPM_SMPA, 6, &pm8950_ftsmps2p5, "vdd_s6" },
+ { "s7", QCOM_SMD_RPM_SMPA, 7, &pm8998_hfsmps, "vdd_s7" },
+
+ { "l1", QCOM_SMD_RPM_LDOA, 1, &pm8953_ult_nldo, "vdd_l1" },
+ { "l2", QCOM_SMD_RPM_LDOA, 2, &pm8953_ult_nldo, "vdd_l2_l3" },
+ { "l3", QCOM_SMD_RPM_LDOA, 3, &pm8953_ult_nldo, "vdd_l2_l3" },
+ { "l4", QCOM_SMD_RPM_LDOA, 4, &pm8950_ult_pldo, "vdd_l4_l5_l6_l7_l16_l19" },
+ { "l5", QCOM_SMD_RPM_LDOA, 5, &pm8950_ult_pldo, "vdd_l4_l5_l6_l7_l16_l19" },
+ { "l6", QCOM_SMD_RPM_LDOA, 6, &pm8950_ult_pldo, "vdd_l4_l5_l6_l7_l16_l19" },
+ { "l7", QCOM_SMD_RPM_LDOA, 7, &pm8950_ult_pldo, "vdd_l4_l5_l6_l7_l16_l19" },
+ { "l8", QCOM_SMD_RPM_LDOA, 8, &pm8950_ult_pldo, "vdd_l8_l11_l12_l13_l14_l15" },
+ { "l9", QCOM_SMD_RPM_LDOA, 9, &pm8950_ult_pldo, "vdd_l9_l10_l17_l18_l22" },
+ { "l10", QCOM_SMD_RPM_LDOA, 10, &pm8950_ult_pldo, "vdd_l9_l10_l17_l18_l22" },
+ { "l11", QCOM_SMD_RPM_LDOA, 11, &pm8950_ult_pldo, "vdd_l8_l11_l12_l13_l14_l15" },
+ { "l12", QCOM_SMD_RPM_LDOA, 12, &pm8950_ult_pldo, "vdd_l8_l11_l12_l13_l14_l15" },
+ { "l13", QCOM_SMD_RPM_LDOA, 13, &pm8950_ult_pldo, "vdd_l8_l11_l12_l13_l14_l15" },
+ { "l14", QCOM_SMD_RPM_LDOA, 14, &pm8950_ult_pldo, "vdd_l8_l11_l12_l13_l14_l15" },
+ { "l15", QCOM_SMD_RPM_LDOA, 15, &pm8950_ult_pldo, "vdd_l8_l11_l12_l13_l14_l15" },
+ { "l16", QCOM_SMD_RPM_LDOA, 16, &pm8950_ult_pldo, "vdd_l4_l5_l6_l7_l16_l19" },
+ { "l17", QCOM_SMD_RPM_LDOA, 17, &pm8950_ult_pldo, "vdd_l9_l10_l17_l18_l22" },
+ { "l18", QCOM_SMD_RPM_LDOA, 18, &pm8950_ult_pldo, "vdd_l9_l10_l17_l18_l22" },
+ { "l19", QCOM_SMD_RPM_LDOA, 19, &pm8953_ult_nldo, "vdd_l4_l5_l6_l7_l16_l19" },
+ { "l20", QCOM_SMD_RPM_LDOA, 20, &pm8953_lnldo, "vdd_l20" },
+ { "l21", QCOM_SMD_RPM_LDOA, 21, &pm8953_lnldo, "vdd_l21" },
+ { "l22", QCOM_SMD_RPM_LDOA, 22, &pm8950_ult_pldo, "vdd_l9_l10_l17_l18_l22" },
+ { "l23", QCOM_SMD_RPM_LDOA, 23, &pm8953_ult_nldo, "vdd_l23" },
+ {}
+};
+
static const struct rpm_regulator_data rpm_pm8994_regulators[] = {
{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm8994_ftsmps, "vdd_s1" },
{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm8994_ftsmps, "vdd_s2" },
{}
};
+static const struct rpm_regulator_data rpm_pm660_regulators[] = {
+ { "s1", QCOM_SMD_RPM_SMPA, 1, &pm660_ftsmps, "vdd_s1" },
+ { "s2", QCOM_SMD_RPM_SMPA, 2, &pm660_ftsmps, "vdd_s2" },
+ { "s3", QCOM_SMD_RPM_SMPA, 3, &pm660_ftsmps, "vdd_s3" },
+ { "s4", QCOM_SMD_RPM_SMPA, 4, &pm660_hfsmps, "vdd_s4" },
+ { "s5", QCOM_SMD_RPM_SMPA, 5, &pm660_hfsmps, "vdd_s5" },
+ { "s6", QCOM_SMD_RPM_SMPA, 6, &pm660_hfsmps, "vdd_s6" },
+ { "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd_l1_l6_l7" },
+ { "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_ht_nldo, "vdd_l2_l3" },
+ { "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd_l2_l3" },
+ /* l4 is unaccessible on PM660 */
+ { "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_ht_nldo, "vdd_l5" },
+ { "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_ht_nldo, "vdd_l1_l6_l7" },
+ { "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_ht_nldo, "vdd_l1_l6_l7" },
+ { "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+ { "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+ { "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+ { "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+ { "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+ { "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+ { "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+ { "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
+ { "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
+ { "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
+ { "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
+ { "l19", QCOM_SMD_RPM_LDOA, 19, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
+ { }
+};
+
+static const struct rpm_regulator_data rpm_pm660l_regulators[] = {
+ { "s1", QCOM_SMD_RPM_SMPB, 1, &pm660_ftsmps, "vdd_s1" },
+ { "s2", QCOM_SMD_RPM_SMPB, 2, &pm660_ftsmps, "vdd_s2" },
+ { "s3", QCOM_SMD_RPM_RWCX, 0, &pm660_ftsmps, "vdd_s3_s4" },
+ { "s5", QCOM_SMD_RPM_RWMX, 0, &pm660_ftsmps, "vdd_s5" },
+ { "l1", QCOM_SMD_RPM_LDOB, 1, &pm660_nldo660, "vdd_l1_l9_l10" },
+ { "l2", QCOM_SMD_RPM_LDOB, 2, &pm660_pldo660, "vdd_l2" },
+ { "l3", QCOM_SMD_RPM_LDOB, 3, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
+ { "l4", QCOM_SMD_RPM_LDOB, 4, &pm660_pldo660, "vdd_l4_l6" },
+ { "l5", QCOM_SMD_RPM_LDOB, 5, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
+ { "l6", QCOM_SMD_RPM_LDOB, 6, &pm660_pldo660, "vdd_l4_l6" },
+ { "l7", QCOM_SMD_RPM_LDOB, 7, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
+ { "l8", QCOM_SMD_RPM_LDOB, 8, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
+ { "l9", QCOM_SMD_RPM_RWLC, 0, &pm660_ht_nldo, "vdd_l1_l9_l10" },
+ { "l10", QCOM_SMD_RPM_RWLM, 0, &pm660_ht_nldo, "vdd_l1_l9_l10" },
+ { "bob", QCOM_SMD_RPM_BOBB, 1, &pm660l_bob, "vdd_bob", },
+ { }
+};
+
static const struct rpm_regulator_data rpm_pms405_regulators[] = {
{ "s1", QCOM_SMD_RPM_SMPA, 1, &pms405_hfsmps3, "vdd_s1" },
{ "s2", QCOM_SMD_RPM_SMPA, 2, &pms405_hfsmps3, "vdd_s2" },
{ .compatible = "qcom,rpm-pm8916-regulators", .data = &rpm_pm8916_regulators },
{ .compatible = "qcom,rpm-pm8941-regulators", .data = &rpm_pm8941_regulators },
{ .compatible = "qcom,rpm-pm8950-regulators", .data = &rpm_pm8950_regulators },
+ { .compatible = "qcom,rpm-pm8953-regulators", .data = &rpm_pm8953_regulators },
{ .compatible = "qcom,rpm-pm8994-regulators", .data = &rpm_pm8994_regulators },
{ .compatible = "qcom,rpm-pm8998-regulators", .data = &rpm_pm8998_regulators },
+ { .compatible = "qcom,rpm-pm660-regulators", .data = &rpm_pm660_regulators },
+ { .compatible = "qcom,rpm-pm660l-regulators", .data = &rpm_pm660l_regulators },
{ .compatible = "qcom,rpm-pma8084-regulators", .data = &rpm_pma8084_regulators },
{ .compatible = "qcom,rpm-pmi8994-regulators", .data = &rpm_pmi8994_regulators },
{ .compatible = "qcom,rpm-pmi8998-regulators", .data = &rpm_pmi8998_regulators },
SPMI_REGULATOR_SUBTYPE_LV_P600 = 0x2b,
SPMI_REGULATOR_SUBTYPE_LV_P1200 = 0x2c,
SPMI_REGULATOR_SUBTYPE_LV_P450 = 0x2d,
+ SPMI_REGULATOR_SUBTYPE_HT_N300_ST = 0x30,
+ SPMI_REGULATOR_SUBTYPE_HT_N600_ST = 0x31,
+ SPMI_REGULATOR_SUBTYPE_HT_N1200_ST = 0x32,
+ SPMI_REGULATOR_SUBTYPE_HT_LVP150 = 0x3b,
+ SPMI_REGULATOR_SUBTYPE_HT_LVP300 = 0x3c,
+ SPMI_REGULATOR_SUBTYPE_L660_N300_ST = 0x42,
+ SPMI_REGULATOR_SUBTYPE_L660_N600_ST = 0x43,
+ SPMI_REGULATOR_SUBTYPE_L660_P50 = 0x46,
+ SPMI_REGULATOR_SUBTYPE_L660_P150 = 0x47,
+ SPMI_REGULATOR_SUBTYPE_L660_P600 = 0x49,
+ SPMI_REGULATOR_SUBTYPE_L660_LVP150 = 0x4d,
+ SPMI_REGULATOR_SUBTYPE_L660_LVP600 = 0x4f,
SPMI_REGULATOR_SUBTYPE_LV100 = 0x01,
SPMI_REGULATOR_SUBTYPE_LV300 = 0x02,
SPMI_REGULATOR_SUBTYPE_MV300 = 0x08,
SPMI_VOLTAGE_RANGE(0, 1750000, 1750000, 3337500, 3337500, 12500),
};
+static struct spmi_voltage_range pldo660_ranges[] = {
+ SPMI_VOLTAGE_RANGE(0, 1504000, 1504000, 3544000, 3544000, 8000),
+};
+
+static struct spmi_voltage_range nldo660_ranges[] = {
+ SPMI_VOLTAGE_RANGE(0, 320000, 320000, 1304000, 1304000, 8000),
+};
+
+static struct spmi_voltage_range ht_lvpldo_ranges[] = {
+ SPMI_VOLTAGE_RANGE(0, 1504000, 1504000, 2000000, 2000000, 8000),
+};
+
+static struct spmi_voltage_range ht_nldo_ranges[] = {
+ SPMI_VOLTAGE_RANGE(0, 312000, 312000, 1304000, 1304000, 8000),
+};
+
static struct spmi_voltage_range hfs430_ranges[] = {
SPMI_VOLTAGE_RANGE(0, 320000, 320000, 2040000, 2040000, 8000),
};
static DEFINE_SPMI_SET_POINTS(ult_ho_smps);
static DEFINE_SPMI_SET_POINTS(ult_nldo);
static DEFINE_SPMI_SET_POINTS(ult_pldo);
+static DEFINE_SPMI_SET_POINTS(pldo660);
+static DEFINE_SPMI_SET_POINTS(nldo660);
+static DEFINE_SPMI_SET_POINTS(ht_lvpldo);
+static DEFINE_SPMI_SET_POINTS(ht_nldo);
static DEFINE_SPMI_SET_POINTS(hfs430);
static inline int spmi_vreg_read(struct spmi_regulator *vreg, u16 addr, u8 *buf,
SPMI_VREG(LDO, LV_P300, 0, INF, LDO, ldo, pldo, 10000),
SPMI_VREG(LDO, LV_P600, 0, INF, LDO, ldo, pldo, 10000),
SPMI_VREG(LDO, LV_P1200, 0, INF, LDO, ldo, pldo, 10000),
+ SPMI_VREG(LDO, HT_N300_ST, 0, INF, FTSMPS426, ftsmps426,
+ ht_nldo, 30000),
+ SPMI_VREG(LDO, HT_N600_ST, 0, INF, FTSMPS426, ftsmps426,
+ ht_nldo, 30000),
+ SPMI_VREG(LDO, HT_N1200_ST, 0, INF, FTSMPS426, ftsmps426,
+ ht_nldo, 30000),
+ SPMI_VREG(LDO, HT_LVP150, 0, INF, FTSMPS426, ftsmps426,
+ ht_lvpldo, 10000),
+ SPMI_VREG(LDO, HT_LVP300, 0, INF, FTSMPS426, ftsmps426,
+ ht_lvpldo, 10000),
+ SPMI_VREG(LDO, L660_N300_ST, 0, INF, FTSMPS426, ftsmps426,
+ nldo660, 10000),
+ SPMI_VREG(LDO, L660_N600_ST, 0, INF, FTSMPS426, ftsmps426,
+ nldo660, 10000),
+ SPMI_VREG(LDO, L660_P50, 0, INF, FTSMPS426, ftsmps426,
+ pldo660, 10000),
+ SPMI_VREG(LDO, L660_P150, 0, INF, FTSMPS426, ftsmps426,
+ pldo660, 10000),
+ SPMI_VREG(LDO, L660_P600, 0, INF, FTSMPS426, ftsmps426,
+ pldo660, 10000),
+ SPMI_VREG(LDO, L660_LVP150, 0, INF, FTSMPS426, ftsmps426,
+ ht_lvpldo, 10000),
+ SPMI_VREG(LDO, L660_LVP600, 0, INF, FTSMPS426, ftsmps426,
+ ht_lvpldo, 10000),
SPMI_VREG_VS(LV100, 0, INF),
SPMI_VREG_VS(LV300, 0, INF),
SPMI_VREG_VS(MV300, 0, INF),
return ret;
/* Set up enable pin control. */
- if ((type == SPMI_REGULATOR_LOGICAL_TYPE_SMPS
- || type == SPMI_REGULATOR_LOGICAL_TYPE_LDO
- || type == SPMI_REGULATOR_LOGICAL_TYPE_VS)
- && !(data->pin_ctrl_enable
- & SPMI_REGULATOR_PIN_CTRL_ENABLE_HW_DEFAULT)) {
- ctrl_reg[SPMI_COMMON_IDX_ENABLE] &=
- ~SPMI_COMMON_ENABLE_FOLLOW_ALL_MASK;
- ctrl_reg[SPMI_COMMON_IDX_ENABLE] |=
- data->pin_ctrl_enable & SPMI_COMMON_ENABLE_FOLLOW_ALL_MASK;
+ if (!(data->pin_ctrl_enable & SPMI_REGULATOR_PIN_CTRL_ENABLE_HW_DEFAULT)) {
+ switch (type) {
+ case SPMI_REGULATOR_LOGICAL_TYPE_SMPS:
+ case SPMI_REGULATOR_LOGICAL_TYPE_LDO:
+ case SPMI_REGULATOR_LOGICAL_TYPE_VS:
+ ctrl_reg[SPMI_COMMON_IDX_ENABLE] &=
+ ~SPMI_COMMON_ENABLE_FOLLOW_ALL_MASK;
+ ctrl_reg[SPMI_COMMON_IDX_ENABLE] |=
+ data->pin_ctrl_enable & SPMI_COMMON_ENABLE_FOLLOW_ALL_MASK;
+ break;
+ default:
+ break;
+ }
}
/* Set up mode pin control. */
- if ((type == SPMI_REGULATOR_LOGICAL_TYPE_SMPS
- || type == SPMI_REGULATOR_LOGICAL_TYPE_LDO)
- && !(data->pin_ctrl_hpm
- & SPMI_REGULATOR_PIN_CTRL_HPM_HW_DEFAULT)) {
- ctrl_reg[SPMI_COMMON_IDX_MODE] &=
- ~SPMI_COMMON_MODE_FOLLOW_ALL_MASK;
- ctrl_reg[SPMI_COMMON_IDX_MODE] |=
- data->pin_ctrl_hpm & SPMI_COMMON_MODE_FOLLOW_ALL_MASK;
- }
-
- if (type == SPMI_REGULATOR_LOGICAL_TYPE_VS
- && !(data->pin_ctrl_hpm & SPMI_REGULATOR_PIN_CTRL_HPM_HW_DEFAULT)) {
- ctrl_reg[SPMI_COMMON_IDX_MODE] &=
- ~SPMI_COMMON_MODE_FOLLOW_AWAKE_MASK;
- ctrl_reg[SPMI_COMMON_IDX_MODE] |=
- data->pin_ctrl_hpm & SPMI_COMMON_MODE_FOLLOW_AWAKE_MASK;
- }
-
- if ((type == SPMI_REGULATOR_LOGICAL_TYPE_ULT_LO_SMPS
- || type == SPMI_REGULATOR_LOGICAL_TYPE_ULT_HO_SMPS
- || type == SPMI_REGULATOR_LOGICAL_TYPE_ULT_LDO)
- && !(data->pin_ctrl_hpm
- & SPMI_REGULATOR_PIN_CTRL_HPM_HW_DEFAULT)) {
- ctrl_reg[SPMI_COMMON_IDX_MODE] &=
- ~SPMI_COMMON_MODE_FOLLOW_AWAKE_MASK;
- ctrl_reg[SPMI_COMMON_IDX_MODE] |=
- data->pin_ctrl_hpm & SPMI_COMMON_MODE_FOLLOW_AWAKE_MASK;
+ if (!(data->pin_ctrl_hpm & SPMI_REGULATOR_PIN_CTRL_HPM_HW_DEFAULT)) {
+ switch (type) {
+ case SPMI_REGULATOR_LOGICAL_TYPE_SMPS:
+ case SPMI_REGULATOR_LOGICAL_TYPE_LDO:
+ ctrl_reg[SPMI_COMMON_IDX_MODE] &=
+ ~SPMI_COMMON_MODE_FOLLOW_ALL_MASK;
+ ctrl_reg[SPMI_COMMON_IDX_MODE] |=
+ data->pin_ctrl_hpm & SPMI_COMMON_MODE_FOLLOW_ALL_MASK;
+ break;
+ case SPMI_REGULATOR_LOGICAL_TYPE_VS:
+ case SPMI_REGULATOR_LOGICAL_TYPE_ULT_LO_SMPS:
+ case SPMI_REGULATOR_LOGICAL_TYPE_ULT_HO_SMPS:
+ case SPMI_REGULATOR_LOGICAL_TYPE_ULT_LDO:
+ ctrl_reg[SPMI_COMMON_IDX_MODE] &=
+ ~SPMI_COMMON_MODE_FOLLOW_AWAKE_MASK;
+ ctrl_reg[SPMI_COMMON_IDX_MODE] |=
+ data->pin_ctrl_hpm & SPMI_COMMON_MODE_FOLLOW_AWAKE_MASK;
+ break;
+ default:
+ break;
+ }
}
/* Write back any control register values that were modified. */
{ }
};
+static const struct spmi_regulator_data pm660_regulators[] = {
+ { "s1", 0x1400, "vdd_s1", },
+ { "s2", 0x1700, "vdd_s2", },
+ { "s3", 0x1a00, "vdd_s3", },
+ { "s4", 0x1d00, "vdd_s3", },
+ { "s5", 0x2000, "vdd_s5", },
+ { "s6", 0x2300, "vdd_s6", },
+ { "l1", 0x4000, "vdd_l1_l6_l7", },
+ { "l2", 0x4100, "vdd_l2_l3", },
+ { "l3", 0x4200, "vdd_l2_l3", },
+ /* l4 is unaccessible on PM660 */
+ { "l5", 0x4400, "vdd_l5", },
+ { "l6", 0x4500, "vdd_l1_l6_l7", },
+ { "l7", 0x4600, "vdd_l1_l6_l7", },
+ { "l8", 0x4700, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+ { "l9", 0x4800, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+ { "l10", 0x4900, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+ { "l11", 0x4a00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+ { "l12", 0x4b00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+ { "l13", 0x4c00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+ { "l14", 0x4d00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+ { "l15", 0x4e00, "vdd_l15_l16_l17_l18_l19", },
+ { "l16", 0x4f00, "vdd_l15_l16_l17_l18_l19", },
+ { "l17", 0x5000, "vdd_l15_l16_l17_l18_l19", },
+ { "l18", 0x5100, "vdd_l15_l16_l17_l18_l19", },
+ { "l19", 0x5200, "vdd_l15_l16_l17_l18_l19", },
+ { }
+};
+
+static const struct spmi_regulator_data pm660l_regulators[] = {
+ { "s1", 0x1400, "vdd_s1", },
+ { "s2", 0x1700, "vdd_s2", },
+ { "s3", 0x1a00, "vdd_s3", },
+ { "s4", 0x1d00, "vdd_s4", },
+ { "s5", 0x2000, "vdd_s5", },
+ { "l1", 0x4000, "vdd_l1_l9_l10", },
+ { "l2", 0x4100, "vdd_l2", },
+ { "l3", 0x4200, "vdd_l3_l5_l7_l8", },
+ { "l4", 0x4300, "vdd_l4_l6", },
+ { "l5", 0x4400, "vdd_l3_l5_l7_l8", },
+ { "l6", 0x4500, "vdd_l4_l6", },
+ { "l7", 0x4600, "vdd_l3_l5_l7_l8", },
+ { "l8", 0x4700, "vdd_l3_l5_l7_l8", },
+ { "l9", 0x4800, "vdd_l1_l9_l10", },
+ { "l10", 0x4900, "vdd_l1_l9_l10", },
+ { }
+};
+
+
static const struct spmi_regulator_data pm8004_regulators[] = {
{ "s2", 0x1700, "vdd_s2", },
{ "s5", 0x2000, "vdd_s5", },
{ .compatible = "qcom,pm8950-regulators", .data = &pm8950_regulators },
{ .compatible = "qcom,pm8994-regulators", .data = &pm8994_regulators },
{ .compatible = "qcom,pmi8994-regulators", .data = &pmi8994_regulators },
+ { .compatible = "qcom,pm660-regulators", .data = &pm660_regulators },
+ { .compatible = "qcom,pm660l-regulators", .data = &pm660l_regulators },
{ .compatible = "qcom,pms405-regulators", .data = &pms405_regulators },
{ }
};
qcom_usb_vbus_rdesc.enable_mask = OTG_EN;
config.dev = dev;
config.init_data = init_data;
+ config.of_node = dev->of_node;
config.regmap = regmap;
rdev = devm_regulator_register(dev, &qcom_usb_vbus_rdesc, &config);
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Marek Vasut <marex@denx.de>
+ *
+ * Based on rpi_touchscreen.c by Eric Anholt <eric@anholt.net>
+ */
+
+#include <linux/backlight.h>
+#include <linux/err.h>
+#include <linux/gpio.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+#include <linux/regulator/of_regulator.h>
+#include <linux/slab.h>
+
+/* I2C registers of the Atmel microcontroller. */
+#define REG_ID 0x80
+#define REG_PORTA 0x81
+#define REG_PORTA_HF BIT(2)
+#define REG_PORTA_VF BIT(3)
+#define REG_PORTB 0x82
+#define REG_POWERON 0x85
+#define REG_PWM 0x86
+
+static const struct regmap_config attiny_regmap_config = {
+ .reg_bits = 8,
+ .val_bits = 8,
+ .max_register = REG_PWM,
+ .cache_type = REGCACHE_NONE,
+};
+
+static int attiny_lcd_power_enable(struct regulator_dev *rdev)
+{
+ unsigned int data;
+
+ regmap_write(rdev->regmap, REG_POWERON, 1);
+ /* Wait for nPWRDWN to go low to indicate poweron is done. */
+ regmap_read_poll_timeout(rdev->regmap, REG_PORTB, data,
+ data & BIT(0), 10, 1000000);
+
+ /* Default to the same orientation as the closed source
+ * firmware used for the panel. Runtime rotation
+ * configuration will be supported using VC4's plane
+ * orientation bits.
+ */
+ regmap_write(rdev->regmap, REG_PORTA, BIT(2));
+
+ return 0;
+}
+
+static int attiny_lcd_power_disable(struct regulator_dev *rdev)
+{
+ regmap_write(rdev->regmap, REG_PWM, 0);
+ regmap_write(rdev->regmap, REG_POWERON, 0);
+ udelay(1);
+ return 0;
+}
+
+static int attiny_lcd_power_is_enabled(struct regulator_dev *rdev)
+{
+ unsigned int data;
+ int ret;
+
+ ret = regmap_read(rdev->regmap, REG_POWERON, &data);
+ if (ret < 0)
+ return ret;
+
+ if (!(data & BIT(0)))
+ return 0;
+
+ ret = regmap_read(rdev->regmap, REG_PORTB, &data);
+ if (ret < 0)
+ return ret;
+
+ return data & BIT(0);
+}
+
+static const struct regulator_init_data attiny_regulator_default = {
+ .constraints = {
+ .valid_ops_mask = REGULATOR_CHANGE_STATUS,
+ },
+};
+
+static const struct regulator_ops attiny_regulator_ops = {
+ .enable = attiny_lcd_power_enable,
+ .disable = attiny_lcd_power_disable,
+ .is_enabled = attiny_lcd_power_is_enabled,
+};
+
+static const struct regulator_desc attiny_regulator = {
+ .name = "tc358762-power",
+ .ops = &attiny_regulator_ops,
+ .type = REGULATOR_VOLTAGE,
+ .owner = THIS_MODULE,
+};
+
+static int attiny_update_status(struct backlight_device *bl)
+{
+ struct regmap *regmap = bl_get_data(bl);
+ int brightness = bl->props.brightness;
+
+ if (bl->props.power != FB_BLANK_UNBLANK ||
+ bl->props.fb_blank != FB_BLANK_UNBLANK)
+ brightness = 0;
+
+ return regmap_write(regmap, REG_PWM, brightness);
+}
+
+static int attiny_get_brightness(struct backlight_device *bl)
+{
+ struct regmap *regmap = bl_get_data(bl);
+ int ret, brightness;
+
+ ret = regmap_read(regmap, REG_PWM, &brightness);
+ if (ret)
+ return ret;
+
+ return brightness;
+}
+
+static const struct backlight_ops attiny_bl = {
+ .update_status = attiny_update_status,
+ .get_brightness = attiny_get_brightness,
+};
+
+/*
+ * I2C driver interface functions
+ */
+static int attiny_i2c_probe(struct i2c_client *i2c,
+ const struct i2c_device_id *id)
+{
+ struct backlight_properties props = { };
+ struct regulator_config config = { };
+ struct backlight_device *bl;
+ struct regulator_dev *rdev;
+ struct regmap *regmap;
+ unsigned int data;
+ int ret;
+
+ regmap = devm_regmap_init_i2c(i2c, &attiny_regmap_config);
+ if (IS_ERR(regmap)) {
+ ret = PTR_ERR(regmap);
+ dev_err(&i2c->dev, "Failed to allocate register map: %d\n",
+ ret);
+ return ret;
+ }
+
+ ret = regmap_read(regmap, REG_ID, &data);
+ if (ret < 0) {
+ dev_err(&i2c->dev, "Failed to read REG_ID reg: %d\n", ret);
+ return ret;
+ }
+
+ switch (data) {
+ case 0xde: /* ver 1 */
+ case 0xc3: /* ver 2 */
+ break;
+ default:
+ dev_err(&i2c->dev, "Unknown Atmel firmware revision: 0x%02x\n", data);
+ return -ENODEV;
+ }
+
+ regmap_write(regmap, REG_POWERON, 0);
+ mdelay(1);
+
+ config.dev = &i2c->dev;
+ config.regmap = regmap;
+ config.of_node = i2c->dev.of_node;
+ config.init_data = &attiny_regulator_default;
+
+ rdev = devm_regulator_register(&i2c->dev, &attiny_regulator, &config);
+ if (IS_ERR(rdev)) {
+ dev_err(&i2c->dev, "Failed to register ATTINY regulator\n");
+ return PTR_ERR(rdev);
+ }
+
+ props.type = BACKLIGHT_RAW;
+ props.max_brightness = 0xff;
+ bl = devm_backlight_device_register(&i2c->dev,
+ "7inch-touchscreen-panel-bl",
+ &i2c->dev, regmap, &attiny_bl,
+ &props);
+ if (IS_ERR(bl))
+ return PTR_ERR(bl);
+
+ bl->props.brightness = 0xff;
+
+ return 0;
+}
+
+static const struct of_device_id attiny_dt_ids[] = {
+ { .compatible = "raspberrypi,7inch-touchscreen-panel-regulator" },
+ {},
+};
+MODULE_DEVICE_TABLE(of, attiny_dt_ids);
+
+static struct i2c_driver attiny_regulator_driver = {
+ .driver = {
+ .name = "rpi_touchscreen_attiny",
+ .of_match_table = of_match_ptr(attiny_dt_ids),
+ },
+ .probe = attiny_i2c_probe,
+};
+
+module_i2c_driver(attiny_regulator_driver);
+
+MODULE_AUTHOR("Marek Vasut <marex@denx.de>");
+MODULE_DESCRIPTION("Regulator device driver for Raspberry Pi 7-inch touchscreen");
+MODULE_LICENSE("GPL v2");
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+
+#define RT4801_REG_VOP 0x00
+#define RT4801_REG_VON 0x01
+#define RT4801_REG_APPS 0x03
+
+#define VOUT_MASK 0x1F
+
+#define MIN_UV 4000000
+#define STEP_UV 100000
+#define MAX_UV 6000000
+#define N_VOLTAGES ((MAX_UV - MIN_UV) / STEP_UV + 1)
+
+#define DSV_OUT_POS 0
+#define DSV_OUT_NEG 1
+#define DSV_OUT_MAX 2
+
+#define DSVP_ENABLE BIT(0)
+#define DSVN_ENABLE BIT(1)
+#define DSVALL_ENABLE (DSVP_ENABLE | DSVN_ENABLE)
+
+struct rt4801_priv {
+ struct device *dev;
+ struct gpio_descs *enable_gpios;
+ unsigned int enable_flag;
+ unsigned int volt_sel[DSV_OUT_MAX];
+};
+
+static int rt4801_set_voltage_sel(struct regulator_dev *rdev, unsigned int selector)
+{
+ struct rt4801_priv *priv = rdev_get_drvdata(rdev);
+ int id = rdev_get_id(rdev), ret;
+
+ if (priv->enable_flag & BIT(id)) {
+ ret = regulator_set_voltage_sel_regmap(rdev, selector);
+ if (ret)
+ return ret;
+ }
+
+ priv->volt_sel[id] = selector;
+ return 0;
+}
+
+static int rt4801_get_voltage_sel(struct regulator_dev *rdev)
+{
+ struct rt4801_priv *priv = rdev_get_drvdata(rdev);
+ int id = rdev_get_id(rdev);
+
+ if (priv->enable_flag & BIT(id))
+ return regulator_get_voltage_sel_regmap(rdev);
+
+ return priv->volt_sel[id];
+}
+
+static int rt4801_enable(struct regulator_dev *rdev)
+{
+ struct rt4801_priv *priv = rdev_get_drvdata(rdev);
+ struct gpio_descs *gpios = priv->enable_gpios;
+ int id = rdev_get_id(rdev), ret;
+
+ if (gpios->ndescs <= id) {
+ dev_warn(&rdev->dev, "no dedicated gpio can control\n");
+ goto bypass_gpio;
+ }
+
+ gpiod_set_value(gpios->desc[id], 1);
+
+bypass_gpio:
+ ret = regmap_write(rdev->regmap, rdev->desc->vsel_reg, priv->volt_sel[id]);
+ if (ret)
+ return ret;
+
+ priv->enable_flag |= BIT(id);
+ return 0;
+}
+
+static int rt4801_disable(struct regulator_dev *rdev)
+{
+ struct rt4801_priv *priv = rdev_get_drvdata(rdev);
+ struct gpio_descs *gpios = priv->enable_gpios;
+ int id = rdev_get_id(rdev);
+
+ if (gpios->ndescs <= id) {
+ dev_warn(&rdev->dev, "no dedicated gpio can control\n");
+ goto bypass_gpio;
+ }
+
+ gpiod_set_value(gpios->desc[id], 0);
+
+bypass_gpio:
+ priv->enable_flag &= ~BIT(id);
+ return 0;
+}
+
+static int rt4801_is_enabled(struct regulator_dev *rdev)
+{
+ struct rt4801_priv *priv = rdev_get_drvdata(rdev);
+ int id = rdev_get_id(rdev);
+
+ return !!(priv->enable_flag & BIT(id));
+}
+
+static const struct regulator_ops rt4801_regulator_ops = {
+ .list_voltage = regulator_list_voltage_linear,
+ .set_voltage_sel = rt4801_set_voltage_sel,
+ .get_voltage_sel = rt4801_get_voltage_sel,
+ .enable = rt4801_enable,
+ .disable = rt4801_disable,
+ .is_enabled = rt4801_is_enabled,
+};
+
+static const struct regulator_desc rt4801_regulator_descs[] = {
+ {
+ .name = "DSVP",
+ .ops = &rt4801_regulator_ops,
+ .of_match = of_match_ptr("DSVP"),
+ .type = REGULATOR_VOLTAGE,
+ .id = DSV_OUT_POS,
+ .min_uV = MIN_UV,
+ .uV_step = STEP_UV,
+ .n_voltages = N_VOLTAGES,
+ .owner = THIS_MODULE,
+ .vsel_reg = RT4801_REG_VOP,
+ .vsel_mask = VOUT_MASK,
+ },
+ {
+ .name = "DSVN",
+ .ops = &rt4801_regulator_ops,
+ .of_match = of_match_ptr("DSVN"),
+ .type = REGULATOR_VOLTAGE,
+ .id = DSV_OUT_NEG,
+ .min_uV = MIN_UV,
+ .uV_step = STEP_UV,
+ .n_voltages = N_VOLTAGES,
+ .owner = THIS_MODULE,
+ .vsel_reg = RT4801_REG_VON,
+ .vsel_mask = VOUT_MASK,
+ },
+};
+
+static const struct regmap_config rt4801_regmap_config = {
+ .reg_bits = 8,
+ .val_bits = 8,
+ .max_register = RT4801_REG_APPS,
+};
+
+static int rt4801_probe(struct i2c_client *i2c)
+{
+ struct rt4801_priv *priv;
+ struct regmap *regmap;
+ int i;
+
+ priv = devm_kzalloc(&i2c->dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->dev = &i2c->dev;
+ /* bootloader will on, driver only reconfigure enable to all output high */
+ priv->enable_flag = DSVALL_ENABLE;
+
+ regmap = devm_regmap_init_i2c(i2c, &rt4801_regmap_config);
+ if (IS_ERR(regmap)) {
+ dev_err(&i2c->dev, "Failed to init regmap\n");
+ return PTR_ERR(regmap);
+ }
+
+ priv->enable_gpios = devm_gpiod_get_array_optional(&i2c->dev, "enable", GPIOD_OUT_HIGH);
+ if (IS_ERR(priv->enable_gpios)) {
+ dev_err(&i2c->dev, "Failed to get gpios\n");
+ return PTR_ERR(priv->enable_gpios);
+ }
+
+ for (i = 0; i < DSV_OUT_MAX; i++) {
+ const struct regulator_desc *desc = rt4801_regulator_descs + i;
+ struct regulator_config config = { .dev = &i2c->dev, .driver_data = priv,
+ .regmap = regmap, };
+ struct regulator_dev *rdev;
+ unsigned int val;
+ int ret;
+
+ /* initialize volt_sel variable */
+ ret = regmap_read(regmap, desc->vsel_reg, &val);
+ if (ret)
+ return ret;
+
+ priv->volt_sel[i] = val & desc->vsel_mask;
+
+ rdev = devm_regulator_register(&i2c->dev, desc, &config);
+ if (IS_ERR(rdev)) {
+ dev_err(&i2c->dev, "Failed to register [%d] regulator\n", i);
+ return PTR_ERR(rdev);
+ }
+ }
+
+ return 0;
+}
+
+static const struct of_device_id __maybe_unused rt4801_of_id[] = {
+ { .compatible = "richtek,rt4801", },
+ { },
+};
+MODULE_DEVICE_TABLE(of, rt4801_of_id);
+
+static struct i2c_driver rt4801_driver = {
+ .driver = {
+ .name = "rt4801",
+ .of_match_table = of_match_ptr(rt4801_of_id),
+ },
+ .probe_new = rt4801_probe,
+};
+module_i2c_driver(rt4801_driver);
+
+MODULE_AUTHOR("ChiYuan Hwang <cy_huang@richtek.com>");
+MODULE_DESCRIPTION("Richtek RT4801 Display Bias Driver");
+MODULE_LICENSE("GPL v2");
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/driver.h>
+
+#define RTMV20_REG_DEVINFO 0x00
+#define RTMV20_REG_PULSEDELAY 0x01
+#define RTMV20_REG_PULSEWIDTH 0x03
+#define RTMV20_REG_LDCTRL1 0x05
+#define RTMV20_REG_ESPULSEWIDTH 0x06
+#define RTMV20_REG_ESLDCTRL1 0x08
+#define RTMV20_REG_LBP 0x0A
+#define RTMV20_REG_LDCTRL2 0x0B
+#define RTMV20_REG_FSIN1CTRL1 0x0D
+#define RTMV20_REG_FSIN1CTRL3 0x0F
+#define RTMV20_REG_FSIN2CTRL1 0x10
+#define RTMV20_REG_FSIN2CTRL3 0x12
+#define RTMV20_REG_ENCTRL 0x13
+#define RTMV20_REG_STRBVSYNDLYL 0x29
+#define RTMV20_REG_LDIRQ 0x30
+#define RTMV20_REG_LDSTAT 0x40
+#define RTMV20_REG_LDMASK 0x50
+
+#define RTMV20_VID_MASK GENMASK(7, 4)
+#define RICHTEK_VID 0x80
+#define RTMV20_LDCURR_MASK GENMASK(7, 0)
+#define RTMV20_DELAY_MASK GENMASK(9, 0)
+#define RTMV20_WIDTH_MASK GENMASK(13, 0)
+#define RTMV20_WIDTH2_MASK GENMASK(7, 0)
+#define RTMV20_LBPLVL_MASK GENMASK(3, 0)
+#define RTMV20_LBPEN_MASK BIT(7)
+#define RTMV20_STROBEPOL_MASK BIT(1)
+#define RTMV20_VSYNPOL_MASK BIT(1)
+#define RTMV20_FSINEN_MASK BIT(7)
+#define RTMV20_ESEN_MASK BIT(6)
+#define RTMV20_FSINOUT_MASK BIT(2)
+#define LDENABLE_MASK (BIT(3) | BIT(0))
+
+#define OTPEVT_MASK BIT(4)
+#define SHORTEVT_MASK BIT(3)
+#define OPENEVT_MASK BIT(2)
+#define LBPEVT_MASK BIT(1)
+#define OCPEVT_MASK BIT(0)
+#define FAILEVT_MASK (SHORTEVT_MASK | OPENEVT_MASK | LBPEVT_MASK)
+
+#define RTMV20_LSW_MINUA 0
+#define RTMV20_LSW_MAXUA 6000000
+#define RTMV20_LSW_STEPUA 30000
+
+#define RTMV20_LSW_DEFAULTUA 3000000
+
+#define RTMV20_I2CRDY_TIMEUS 200
+#define RTMV20_CSRDY_TIMEUS 2000
+
+struct rtmv20_priv {
+ struct device *dev;
+ struct regmap *regmap;
+ struct gpio_desc *enable_gpio;
+ struct regulator_dev *rdev;
+};
+
+static int rtmv20_lsw_enable(struct regulator_dev *rdev)
+{
+ struct rtmv20_priv *priv = rdev_get_drvdata(rdev);
+ int ret;
+
+ gpiod_set_value(priv->enable_gpio, 1);
+
+ /* Wait for I2C can be accessed */
+ usleep_range(RTMV20_I2CRDY_TIMEUS, RTMV20_I2CRDY_TIMEUS + 100);
+
+ /* HW re-enable, disable cache only and sync regcache here */
+ regcache_cache_only(priv->regmap, false);
+ ret = regcache_sync(priv->regmap);
+ if (ret)
+ return ret;
+
+ return regulator_enable_regmap(rdev);
+}
+
+static int rtmv20_lsw_disable(struct regulator_dev *rdev)
+{
+ struct rtmv20_priv *priv = rdev_get_drvdata(rdev);
+ int ret;
+
+ ret = regulator_disable_regmap(rdev);
+ if (ret)
+ return ret;
+
+ /* Mark the regcache as dirty and cache only before HW disabled */
+ regcache_cache_only(priv->regmap, true);
+ regcache_mark_dirty(priv->regmap);
+
+ gpiod_set_value(priv->enable_gpio, 0);
+
+ return 0;
+}
+
+static const struct regulator_ops rtmv20_regulator_ops = {
+ .set_current_limit = regulator_set_current_limit_regmap,
+ .get_current_limit = regulator_get_current_limit_regmap,
+ .enable = rtmv20_lsw_enable,
+ .disable = rtmv20_lsw_disable,
+ .is_enabled = regulator_is_enabled_regmap,
+};
+
+static const struct regulator_desc rtmv20_lsw_desc = {
+ .name = "rtmv20,lsw",
+ .of_match = of_match_ptr("lsw"),
+ .type = REGULATOR_CURRENT,
+ .owner = THIS_MODULE,
+ .ops = &rtmv20_regulator_ops,
+ .csel_reg = RTMV20_REG_LDCTRL1,
+ .csel_mask = RTMV20_LDCURR_MASK,
+ .enable_reg = RTMV20_REG_ENCTRL,
+ .enable_mask = LDENABLE_MASK,
+ .enable_time = RTMV20_CSRDY_TIMEUS,
+};
+
+static irqreturn_t rtmv20_irq_handler(int irq, void *data)
+{
+ struct rtmv20_priv *priv = data;
+ unsigned int val;
+ int ret;
+
+ ret = regmap_read(priv->regmap, RTMV20_REG_LDIRQ, &val);
+ if (ret) {
+ dev_err(priv->dev, "Failed to get irq flags\n");
+ return IRQ_NONE;
+ }
+
+ if (val & OTPEVT_MASK)
+ regulator_notifier_call_chain(priv->rdev, REGULATOR_EVENT_OVER_TEMP, NULL);
+
+ if (val & OCPEVT_MASK)
+ regulator_notifier_call_chain(priv->rdev, REGULATOR_EVENT_OVER_CURRENT, NULL);
+
+ if (val & FAILEVT_MASK)
+ regulator_notifier_call_chain(priv->rdev, REGULATOR_EVENT_FAIL, NULL);
+
+ return IRQ_HANDLED;
+}
+
+static u32 clamp_to_selector(u32 val, u32 min, u32 max, u32 step)
+{
+ u32 retval = clamp_val(val, min, max);
+
+ return (retval - min) / step;
+}
+
+static int rtmv20_properties_init(struct rtmv20_priv *priv)
+{
+ const struct {
+ const char *name;
+ u32 def;
+ u32 min;
+ u32 max;
+ u32 step;
+ u32 addr;
+ u32 mask;
+ } props[] = {
+ { "richtek,ld-pulse-delay-us", 0, 0, 100000, 100, RTMV20_REG_PULSEDELAY,
+ RTMV20_DELAY_MASK },
+ { "richtek,ld-pulse-width-us", 1200, 0, 10000, 1, RTMV20_REG_PULSEWIDTH,
+ RTMV20_WIDTH_MASK },
+ { "richtek,fsin1-delay-us", 23000, 0, 100000, 100, RTMV20_REG_FSIN1CTRL1,
+ RTMV20_DELAY_MASK },
+ { "richtek,fsin1-width-us", 160, 40, 10000, 40, RTMV20_REG_FSIN1CTRL3,
+ RTMV20_WIDTH2_MASK },
+ { "richtek,fsin2-delay-us", 23000, 0, 100000, 100, RTMV20_REG_FSIN2CTRL1,
+ RTMV20_DELAY_MASK },
+ { "richtek,fsin2-width-us", 160, 40, 10000, 40, RTMV20_REG_FSIN2CTRL3,
+ RTMV20_WIDTH2_MASK },
+ { "richtek,es-pulse-width-us", 1200, 0, 10000, 1, RTMV20_REG_ESPULSEWIDTH,
+ RTMV20_WIDTH_MASK },
+ { "richtek,es-ld-current-microamp", 3000000, 0, 6000000, 30000,
+ RTMV20_REG_ESLDCTRL1, RTMV20_LDCURR_MASK },
+ { "richtek,lbp-level-microvolt", 2700000, 2400000, 3700000, 100000, RTMV20_REG_LBP,
+ RTMV20_LBPLVL_MASK },
+ { "richtek,lbp-enable", 0, 0, 1, 1, RTMV20_REG_LBP, RTMV20_LBPEN_MASK },
+ { "richtek,strobe-polarity-high", 1, 0, 1, 1, RTMV20_REG_LDCTRL2,
+ RTMV20_STROBEPOL_MASK },
+ { "richtek,vsync-polarity-high", 1, 0, 1, 1, RTMV20_REG_LDCTRL2,
+ RTMV20_VSYNPOL_MASK },
+ { "richtek,fsin-enable", 0, 0, 1, 1, RTMV20_REG_ENCTRL, RTMV20_FSINEN_MASK },
+ { "richtek,fsin-output", 0, 0, 1, 1, RTMV20_REG_ENCTRL, RTMV20_FSINOUT_MASK },
+ { "richtek,es-enable", 0, 0, 1, 1, RTMV20_REG_ENCTRL, RTMV20_ESEN_MASK },
+ };
+ int i, ret;
+
+ for (i = 0; i < ARRAY_SIZE(props); i++) {
+ __be16 bval16;
+ u16 val16;
+ u32 temp;
+ int significant_bit = fls(props[i].mask);
+ int shift = ffs(props[i].mask) - 1;
+
+ if (props[i].max > 1) {
+ ret = device_property_read_u32(priv->dev, props[i].name, &temp);
+ if (ret)
+ temp = props[i].def;
+ } else
+ temp = device_property_read_bool(priv->dev, props[i].name);
+
+ temp = clamp_to_selector(temp, props[i].min, props[i].max, props[i].step);
+
+ /* If significant bit is over 8, two byte access, others one */
+ if (significant_bit > 8) {
+ ret = regmap_raw_read(priv->regmap, props[i].addr, &bval16, sizeof(bval16));
+ if (ret)
+ return ret;
+
+ val16 = be16_to_cpu(bval16);
+ val16 &= ~props[i].mask;
+ val16 |= (temp << shift);
+ bval16 = cpu_to_be16(val16);
+
+ ret = regmap_raw_write(priv->regmap, props[i].addr, &bval16,
+ sizeof(bval16));
+ } else {
+ ret = regmap_update_bits(priv->regmap, props[i].addr, props[i].mask,
+ temp << shift);
+ }
+
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static int rtmv20_check_chip_exist(struct rtmv20_priv *priv)
+{
+ unsigned int val;
+ int ret;
+
+ ret = regmap_read(priv->regmap, RTMV20_REG_DEVINFO, &val);
+ if (ret)
+ return ret;
+
+ if ((val & RTMV20_VID_MASK) != RICHTEK_VID)
+ return -ENODEV;
+
+ return 0;
+}
+
+static bool rtmv20_is_accessible_reg(struct device *dev, unsigned int reg)
+{
+ switch (reg) {
+ case RTMV20_REG_DEVINFO ... RTMV20_REG_STRBVSYNDLYL:
+ case RTMV20_REG_LDIRQ:
+ case RTMV20_REG_LDSTAT:
+ case RTMV20_REG_LDMASK:
+ return true;
+ }
+ return false;
+}
+
+static bool rtmv20_is_volatile_reg(struct device *dev, unsigned int reg)
+{
+ if (reg == RTMV20_REG_LDIRQ || reg == RTMV20_REG_LDSTAT)
+ return true;
+ return false;
+}
+
+static const struct regmap_config rtmv20_regmap_config = {
+ .reg_bits = 8,
+ .val_bits = 8,
+ .cache_type = REGCACHE_RBTREE,
+ .max_register = RTMV20_REG_LDMASK,
+
+ .writeable_reg = rtmv20_is_accessible_reg,
+ .readable_reg = rtmv20_is_accessible_reg,
+ .volatile_reg = rtmv20_is_volatile_reg,
+};
+
+static int rtmv20_probe(struct i2c_client *i2c)
+{
+ struct rtmv20_priv *priv;
+ struct regulator_config config = {};
+ int ret;
+
+ priv = devm_kzalloc(&i2c->dev, sizeof(*priv), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->dev = &i2c->dev;
+
+ /* Before regmap register, configure HW enable to make I2C accessible */
+ priv->enable_gpio = devm_gpiod_get(&i2c->dev, "enable", GPIOD_OUT_HIGH);
+ if (IS_ERR(priv->enable_gpio)) {
+ dev_err(&i2c->dev, "Failed to get enable gpio\n");
+ return PTR_ERR(priv->enable_gpio);
+ }
+
+ /* Wait for I2C can be accessed */
+ usleep_range(RTMV20_I2CRDY_TIMEUS, RTMV20_I2CRDY_TIMEUS + 100);
+
+ priv->regmap = devm_regmap_init_i2c(i2c, &rtmv20_regmap_config);
+ if (IS_ERR(priv->regmap)) {
+ dev_err(&i2c->dev, "Failed to allocate register map\n");
+ return PTR_ERR(priv->regmap);
+ }
+
+ ret = rtmv20_check_chip_exist(priv);
+ if (ret) {
+ dev_err(&i2c->dev, "Chip vendor info is not matched\n");
+ return ret;
+ }
+
+ ret = rtmv20_properties_init(priv);
+ if (ret) {
+ dev_err(&i2c->dev, "Failed to init properties\n");
+ return ret;
+ }
+
+ /*
+ * keep in shutdown mode to minimize the current consumption
+ * and also mark regcache as dirty
+ */
+ regcache_cache_only(priv->regmap, true);
+ regcache_mark_dirty(priv->regmap);
+ gpiod_set_value(priv->enable_gpio, 0);
+
+ config.dev = &i2c->dev;
+ config.regmap = priv->regmap;
+ config.driver_data = priv;
+ priv->rdev = devm_regulator_register(&i2c->dev, &rtmv20_lsw_desc, &config);
+ if (IS_ERR(priv->rdev)) {
+ dev_err(&i2c->dev, "Failed to register regulator\n");
+ return PTR_ERR(priv->rdev);
+ }
+
+ /* Unmask all events before IRQ registered */
+ ret = regmap_write(priv->regmap, RTMV20_REG_LDMASK, 0);
+ if (ret)
+ return ret;
+
+ return devm_request_threaded_irq(&i2c->dev, i2c->irq, NULL, rtmv20_irq_handler,
+ IRQF_ONESHOT, dev_name(&i2c->dev), priv);
+}
+
+static int __maybe_unused rtmv20_suspend(struct device *dev)
+{
+ struct i2c_client *i2c = to_i2c_client(dev);
+
+ /*
+ * When system suspend, disable irq to prevent interrupt trigger
+ * during I2C bus suspend
+ */
+ disable_irq(i2c->irq);
+ if (device_may_wakeup(dev))
+ enable_irq_wake(i2c->irq);
+
+ return 0;
+}
+
+static int __maybe_unused rtmv20_resume(struct device *dev)
+{
+ struct i2c_client *i2c = to_i2c_client(dev);
+
+ /* Enable irq after I2C bus already resume */
+ enable_irq(i2c->irq);
+ if (device_may_wakeup(dev))
+ disable_irq_wake(i2c->irq);
+
+ return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(rtmv20_pm, rtmv20_suspend, rtmv20_resume);
+
+static const struct of_device_id __maybe_unused rtmv20_of_id[] = {
+ { .compatible = "richtek,rtmv20", },
+ {}
+};
+MODULE_DEVICE_TABLE(of, rtmv20_of_id);
+
+static struct i2c_driver rtmv20_driver = {
+ .driver = {
+ .name = "rtmv20",
+ .of_match_table = of_match_ptr(rtmv20_of_id),
+ .pm = &rtmv20_pm,
+ },
+ .probe_new = rtmv20_probe,
+};
+module_i2c_driver(rtmv20_driver);
+
+MODULE_AUTHOR("ChiYuan Huang <cy_huang@richtek.com>");
+MODULE_DESCRIPTION("Richtek RTMV20 Regulator Driver");
+MODULE_LICENSE("GPL v2");
.probe = s5m8767_pmic_probe,
.id_table = s5m8767_pmic_id,
};
-
-static int __init s5m8767_pmic_init(void)
-{
- return platform_driver_register(&s5m8767_pmic_driver);
-}
-subsys_initcall(s5m8767_pmic_init);
-
-static void __exit s5m8767_pmic_exit(void)
-{
- platform_driver_unregister(&s5m8767_pmic_driver);
-}
-module_exit(s5m8767_pmic_exit);
+module_platform_driver(s5m8767_pmic_driver);
/* Module information */
MODULE_AUTHOR("Sangbeom Kim <sbkim73@samsung.com>");
for (i = 0; i < SLG51000_MAX_REGULATORS; i++) {
if (!(evt[i][R2] & SLG51000_IRQ_ILIM_FLAG_MASK) &&
(evt[i][R0] & SLG51000_EVT_ILIM_FLAG_MASK)) {
- regulator_lock(chip->rdev[i]);
regulator_notifier_call_chain(chip->rdev[i],
REGULATOR_EVENT_OVER_CURRENT, NULL);
- regulator_unlock(chip->rdev[i]);
if (evt[i][R1] & SLG51000_STA_ILIM_FLAG_MASK)
dev_warn(chip->dev,
for (i = 0; i < SLG51000_MAX_REGULATORS; i++) {
if (!(evt[i][R1] & SLG51000_STA_ILIM_FLAG_MASK) &&
(evt[i][R1] & SLG51000_STA_VOUT_OK_FLAG_MASK)) {
- regulator_lock(chip->rdev[i]);
regulator_notifier_call_chain(chip->rdev[i],
REGULATOR_EVENT_OVER_TEMP, NULL);
- regulator_unlock(chip->rdev[i]);
}
}
handled = IRQ_HANDLED;
return 0;
}
-static const struct of_device_id stm32_booster_of_match[] = {
+static const struct of_device_id __maybe_unused stm32_booster_of_match[] = {
{
.compatible = "st,stm32h7-booster",
.data = (void *)&stm32h7_booster_desc
return ret;
}
-static const struct of_device_id stm32_pwr_of_match[] = {
+static const struct of_device_id __maybe_unused stm32_pwr_of_match[] = {
{ .compatible = "st,stm32mp1,pwr-reg", },
{},
};
NULL)
};
-static const struct of_device_id stm32_vrefbuf_of_match[] = {
+static const struct of_device_id __maybe_unused stm32_vrefbuf_of_match[] = {
{ .compatible = "st,stm32-vrefbuf", },
{},
};
{
struct regulator_dev *rdev = (struct regulator_dev *)data;
- regulator_lock(rdev);
-
/* Send an overcurrent notification */
regulator_notifier_call_chain(rdev,
REGULATOR_EVENT_OVER_CURRENT,
NULL);
- regulator_unlock(rdev);
-
return IRQ_HANDLED;
}
3300000,
};
-static struct regulator_ops stw481x_vmmc_ops = {
+static const struct regulator_ops stw481x_vmmc_ops = {
.list_voltage = regulator_list_voltage_table,
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.set_voltage_sel = regulator_set_voltage_sel_regmap,
};
-static struct regulator_desc vmmc_regulator = {
+static const struct regulator_desc vmmc_regulator = {
.name = "VMMC",
.id = 0,
.ops = &stw481x_vmmc_ops,
return 0;
}
-static const struct of_device_id sy8106a_i2c_of_match[] = {
+static const struct of_device_id __maybe_unused sy8106a_i2c_of_match[] = {
{ .compatible = "silergy,sy8106a" },
{ },
};
return ret;
}
+#ifdef CONFIG_OF
static const struct of_device_id sy8827n_dt_ids[] = {
{
.compatible = "silergy,sy8827n",
{ }
};
MODULE_DEVICE_TABLE(of, sy8827n_dt_ids);
+#endif
static const struct i2c_device_id sy8827n_id[] = {
{ "sy8827n", },
return 0;
}
-static struct regulator_ops ti_abb_reg_ops = {
+static const struct regulator_ops ti_abb_reg_ops = {
.list_voltage = regulator_list_voltage_table,
.set_voltage_sel = ti_abb_set_voltage_sel,
return ret;
}
-static struct regulator_ops tps51632_dcdc_ops = {
+static const struct regulator_ops tps51632_dcdc_ops = {
.get_voltage_sel = regulator_get_voltage_sel_regmap,
.set_voltage_sel = regulator_set_voltage_sel_regmap,
.list_voltage = regulator_list_voltage_linear,
5000000, /* There is an additional 5V */
};
-static struct regulator_ops tps6105x_regulator_ops = {
+static const struct regulator_ops tps6105x_regulator_ops = {
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.is_enabled = regulator_is_enabled_regmap,
REGULATOR_MODE_FAST : REGULATOR_MODE_NORMAL;
}
-static struct regulator_ops tps62360_dcdc_ops = {
+static const struct regulator_ops tps62360_dcdc_ops = {
.get_voltage_sel = tps62360_dcdc_get_voltage_sel,
.set_voltage_sel = tps62360_dcdc_set_voltage_sel,
.list_voltage = regulator_list_voltage_linear,
return 0;
}
-static const struct of_device_id tps65023_of_match[] = {
+static const struct of_device_id __maybe_unused tps65023_of_match[] = {
{ .compatible = "ti,tps65020", .data = &tps65020_drv_data},
{ .compatible = "ti,tps65021", .data = &tps65021_drv_data},
{ .compatible = "ti,tps65023", .data = &tps65023_drv_data},
};
/* Operations permitted on regulators */
-static struct regulator_ops reg_ops = {
+static const struct regulator_ops reg_ops = {
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.is_enabled = regulator_is_enabled_regmap,
};
/* Operations permitted on load switches */
-static struct regulator_ops switch_ops = {
+static const struct regulator_ops switch_ops = {
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.is_enabled = regulator_is_enabled_regmap,
int overcurrent_wait;
};
-static struct regulator_ops tps65090_ext_control_ops = {
+static const struct regulator_ops tps65090_ext_control_ops = {
};
/**
return ret;
}
-static struct regulator_ops tps65090_reg_control_ops = {
+static const struct regulator_ops tps65090_reg_control_ops = {
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.is_enabled = regulator_is_enabled_regmap,
};
-static struct regulator_ops tps65090_fet_control_ops = {
+static const struct regulator_ops tps65090_fet_control_ops = {
.enable = tps65090_fet_enable,
.disable = regulator_disable_regmap,
.is_enabled = regulator_is_enabled_regmap,
};
-static struct regulator_ops tps65090_ldo_ops = {
+static const struct regulator_ops tps65090_ldo_ops = {
};
#define tps65090_REG_DESC(_id, _sname, _en_reg, _en_bits, _nvolt, _volt, _ops) \
int enable_reg[2];
};
-static struct regulator_ops tps6586x_rw_regulator_ops = {
+static const struct regulator_ops tps6586x_rw_regulator_ops = {
.list_voltage = regulator_list_voltage_table,
.map_voltage = regulator_map_voltage_ascend,
.get_voltage_sel = regulator_get_voltage_sel_regmap,
.disable = regulator_disable_regmap,
};
-static struct regulator_ops tps6586x_rw_linear_regulator_ops = {
+static const struct regulator_ops tps6586x_rw_linear_regulator_ops = {
.list_voltage = regulator_list_voltage_linear,
.get_voltage_sel = regulator_get_voltage_sel_regmap,
.set_voltage_sel = regulator_set_voltage_sel_regmap,
.disable = regulator_disable_regmap,
};
-static struct regulator_ops tps6586x_ro_regulator_ops = {
+static const struct regulator_ops tps6586x_ro_regulator_ops = {
.list_voltage = regulator_list_voltage_table,
.map_voltage = regulator_map_voltage_ascend,
.get_voltage_sel = regulator_get_voltage_sel_regmap,
.disable = regulator_disable_regmap,
};
-static struct regulator_ops tps6586x_sys_regulator_ops = {
+static const struct regulator_ops tps6586x_sys_regulator_ops = {
};
static const unsigned int tps6586x_ldo0_voltages[] = {
static int tps65910_set_mode(struct regulator_dev *dev, unsigned int mode)
{
struct tps65910_reg *pmic = rdev_get_drvdata(dev);
- struct tps65910 *mfd = pmic->mfd;
- int reg, value, id = rdev_get_id(dev);
+ struct regmap *regmap = rdev_get_regmap(dev);
+ int reg, id = rdev_get_id(dev);
reg = pmic->get_ctrl_reg(id);
if (reg < 0)
switch (mode) {
case REGULATOR_MODE_NORMAL:
- return tps65910_reg_update_bits(pmic->mfd, reg,
- LDO_ST_MODE_BIT | LDO_ST_ON_BIT,
- LDO_ST_ON_BIT);
+ return regmap_update_bits(regmap, reg,
+ LDO_ST_MODE_BIT | LDO_ST_ON_BIT,
+ LDO_ST_ON_BIT);
case REGULATOR_MODE_IDLE:
- value = LDO_ST_ON_BIT | LDO_ST_MODE_BIT;
- return tps65910_reg_set_bits(mfd, reg, value);
+ return regmap_set_bits(regmap, reg,
+ LDO_ST_ON_BIT | LDO_ST_MODE_BIT);
case REGULATOR_MODE_STANDBY:
- return tps65910_reg_clear_bits(mfd, reg, LDO_ST_ON_BIT);
+ return regmap_clear_bits(regmap, reg, LDO_ST_ON_BIT);
}
return -EINVAL;
static unsigned int tps65910_get_mode(struct regulator_dev *dev)
{
struct tps65910_reg *pmic = rdev_get_drvdata(dev);
+ struct regmap *regmap = rdev_get_regmap(dev);
int ret, reg, value, id = rdev_get_id(dev);
reg = pmic->get_ctrl_reg(id);
if (reg < 0)
return reg;
- ret = tps65910_reg_read(pmic->mfd, reg, &value);
+ ret = regmap_read(regmap, reg, &value);
if (ret < 0)
return ret;
static int tps65910_get_voltage_dcdc_sel(struct regulator_dev *dev)
{
- struct tps65910_reg *pmic = rdev_get_drvdata(dev);
+ struct regmap *regmap = rdev_get_regmap(dev);
int ret, id = rdev_get_id(dev);
int opvsel = 0, srvsel = 0, vselmax = 0, mult = 0, sr = 0;
switch (id) {
case TPS65910_REG_VDD1:
- ret = tps65910_reg_read(pmic->mfd, TPS65910_VDD1_OP, &opvsel);
+ ret = regmap_read(regmap, TPS65910_VDD1_OP, &opvsel);
if (ret < 0)
return ret;
- ret = tps65910_reg_read(pmic->mfd, TPS65910_VDD1, &mult);
+ ret = regmap_read(regmap, TPS65910_VDD1, &mult);
if (ret < 0)
return ret;
mult = (mult & VDD1_VGAIN_SEL_MASK) >> VDD1_VGAIN_SEL_SHIFT;
- ret = tps65910_reg_read(pmic->mfd, TPS65910_VDD1_SR, &srvsel);
+ ret = regmap_read(regmap, TPS65910_VDD1_SR, &srvsel);
if (ret < 0)
return ret;
sr = opvsel & VDD1_OP_CMD_MASK;
vselmax = 75;
break;
case TPS65910_REG_VDD2:
- ret = tps65910_reg_read(pmic->mfd, TPS65910_VDD2_OP, &opvsel);
+ ret = regmap_read(regmap, TPS65910_VDD2_OP, &opvsel);
if (ret < 0)
return ret;
- ret = tps65910_reg_read(pmic->mfd, TPS65910_VDD2, &mult);
+ ret = regmap_read(regmap, TPS65910_VDD2, &mult);
if (ret < 0)
return ret;
mult = (mult & VDD2_VGAIN_SEL_MASK) >> VDD2_VGAIN_SEL_SHIFT;
- ret = tps65910_reg_read(pmic->mfd, TPS65910_VDD2_SR, &srvsel);
+ ret = regmap_read(regmap, TPS65910_VDD2_SR, &srvsel);
if (ret < 0)
return ret;
sr = opvsel & VDD2_OP_CMD_MASK;
vselmax = 75;
break;
case TPS65911_REG_VDDCTRL:
- ret = tps65910_reg_read(pmic->mfd, TPS65911_VDDCTRL_OP,
- &opvsel);
+ ret = regmap_read(regmap, TPS65911_VDDCTRL_OP, &opvsel);
if (ret < 0)
return ret;
- ret = tps65910_reg_read(pmic->mfd, TPS65911_VDDCTRL_SR,
- &srvsel);
+ ret = regmap_read(regmap, TPS65911_VDDCTRL_SR, &srvsel);
if (ret < 0)
return ret;
sr = opvsel & VDDCTRL_OP_CMD_MASK;
static int tps65910_get_voltage_sel(struct regulator_dev *dev)
{
struct tps65910_reg *pmic = rdev_get_drvdata(dev);
+ struct regmap *regmap = rdev_get_regmap(dev);
int ret, reg, value, id = rdev_get_id(dev);
reg = pmic->get_ctrl_reg(id);
if (reg < 0)
return reg;
- ret = tps65910_reg_read(pmic->mfd, reg, &value);
+ ret = regmap_read(regmap, reg, &value);
if (ret < 0)
return ret;
static int tps65911_get_voltage_sel(struct regulator_dev *dev)
{
struct tps65910_reg *pmic = rdev_get_drvdata(dev);
+ struct regmap *regmap = rdev_get_regmap(dev);
int ret, id = rdev_get_id(dev);
unsigned int value, reg;
reg = pmic->get_ctrl_reg(id);
- ret = tps65910_reg_read(pmic->mfd, reg, &value);
+ ret = regmap_read(regmap, reg, &value);
if (ret < 0)
return ret;
static int tps65910_set_voltage_dcdc_sel(struct regulator_dev *dev,
unsigned selector)
{
- struct tps65910_reg *pmic = rdev_get_drvdata(dev);
+ struct regmap *regmap = rdev_get_regmap(dev);
int id = rdev_get_id(dev), vsel;
int dcdc_mult = 0;
dcdc_mult--;
vsel = (selector % VDD1_2_NUM_VOLT_FINE) + 3;
- tps65910_reg_update_bits(pmic->mfd, TPS65910_VDD1,
- VDD1_VGAIN_SEL_MASK,
- dcdc_mult << VDD1_VGAIN_SEL_SHIFT);
- tps65910_reg_write(pmic->mfd, TPS65910_VDD1_OP, vsel);
+ regmap_update_bits(regmap, TPS65910_VDD1, VDD1_VGAIN_SEL_MASK,
+ dcdc_mult << VDD1_VGAIN_SEL_SHIFT);
+ regmap_write(regmap, TPS65910_VDD1_OP, vsel);
break;
case TPS65910_REG_VDD2:
dcdc_mult = (selector / VDD1_2_NUM_VOLT_FINE) + 1;
dcdc_mult--;
vsel = (selector % VDD1_2_NUM_VOLT_FINE) + 3;
- tps65910_reg_update_bits(pmic->mfd, TPS65910_VDD2,
- VDD1_VGAIN_SEL_MASK,
- dcdc_mult << VDD2_VGAIN_SEL_SHIFT);
- tps65910_reg_write(pmic->mfd, TPS65910_VDD2_OP, vsel);
+ regmap_update_bits(regmap, TPS65910_VDD2, VDD1_VGAIN_SEL_MASK,
+ dcdc_mult << VDD2_VGAIN_SEL_SHIFT);
+ regmap_write(regmap, TPS65910_VDD2_OP, vsel);
break;
case TPS65911_REG_VDDCTRL:
vsel = selector + 3;
- tps65910_reg_write(pmic->mfd, TPS65911_VDDCTRL_OP, vsel);
+ regmap_write(regmap, TPS65911_VDDCTRL_OP, vsel);
+ break;
}
return 0;
unsigned selector)
{
struct tps65910_reg *pmic = rdev_get_drvdata(dev);
+ struct regmap *regmap = rdev_get_regmap(dev);
int reg, id = rdev_get_id(dev);
reg = pmic->get_ctrl_reg(id);
case TPS65910_REG_VAUX2:
case TPS65910_REG_VAUX33:
case TPS65910_REG_VMMC:
- return tps65910_reg_update_bits(pmic->mfd, reg, LDO_SEL_MASK,
- selector << LDO_SEL_SHIFT);
+ return regmap_update_bits(regmap, reg, LDO_SEL_MASK,
+ selector << LDO_SEL_SHIFT);
case TPS65910_REG_VBB:
- return tps65910_reg_update_bits(pmic->mfd, reg, BBCH_BBSEL_MASK,
- selector << BBCH_BBSEL_SHIFT);
+ return regmap_update_bits(regmap, reg, BBCH_BBSEL_MASK,
+ selector << BBCH_BBSEL_SHIFT);
}
return -EINVAL;
unsigned selector)
{
struct tps65910_reg *pmic = rdev_get_drvdata(dev);
+ struct regmap *regmap = rdev_get_regmap(dev);
int reg, id = rdev_get_id(dev);
reg = pmic->get_ctrl_reg(id);
case TPS65911_REG_LDO1:
case TPS65911_REG_LDO2:
case TPS65911_REG_LDO4:
- return tps65910_reg_update_bits(pmic->mfd, reg, LDO1_SEL_MASK,
- selector << LDO_SEL_SHIFT);
+ return regmap_update_bits(regmap, reg, LDO1_SEL_MASK,
+ selector << LDO_SEL_SHIFT);
case TPS65911_REG_LDO3:
case TPS65911_REG_LDO5:
case TPS65911_REG_LDO6:
case TPS65911_REG_LDO7:
case TPS65911_REG_LDO8:
- return tps65910_reg_update_bits(pmic->mfd, reg, LDO3_SEL_MASK,
- selector << LDO_SEL_SHIFT);
+ return regmap_update_bits(regmap, reg, LDO3_SEL_MASK,
+ selector << LDO_SEL_SHIFT);
case TPS65910_REG_VIO:
- return tps65910_reg_update_bits(pmic->mfd, reg, LDO_SEL_MASK,
- selector << LDO_SEL_SHIFT);
+ return regmap_update_bits(regmap, reg, LDO_SEL_MASK,
+ selector << LDO_SEL_SHIFT);
case TPS65910_REG_VBB:
- return tps65910_reg_update_bits(pmic->mfd, reg, BBCH_BBSEL_MASK,
- selector << BBCH_BBSEL_SHIFT);
+ return regmap_update_bits(regmap, reg, BBCH_BBSEL_MASK,
+ selector << BBCH_BBSEL_SHIFT);
}
return -EINVAL;
}
/* Regulator ops (except VRTC) */
-static struct regulator_ops tps65910_ops_dcdc = {
+static const struct regulator_ops tps65910_ops_dcdc = {
.is_enabled = regulator_is_enabled_regmap,
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.map_voltage = regulator_map_voltage_ascend,
};
-static struct regulator_ops tps65910_ops_vdd3 = {
+static const struct regulator_ops tps65910_ops_vdd3 = {
.is_enabled = regulator_is_enabled_regmap,
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.map_voltage = regulator_map_voltage_ascend,
};
-static struct regulator_ops tps65910_ops_vbb = {
+static const struct regulator_ops tps65910_ops_vbb = {
.is_enabled = regulator_is_enabled_regmap,
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.map_voltage = regulator_map_voltage_iterate,
};
-static struct regulator_ops tps65910_ops = {
+static const struct regulator_ops tps65910_ops = {
.is_enabled = regulator_is_enabled_regmap,
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
.map_voltage = regulator_map_voltage_ascend,
};
-static struct regulator_ops tps65911_ops = {
+static const struct regulator_ops tps65911_ops = {
.is_enabled = regulator_is_enabled_regmap,
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
/* External EN1 control */
if (ext_sleep_config & TPS65910_SLEEP_CONTROL_EXT_INPUT_EN1)
- ret = tps65910_reg_set_bits(mfd,
+ ret = regmap_set_bits(mfd->regmap,
TPS65910_EN1_LDO_ASS + regoffs, bit_pos);
else
- ret = tps65910_reg_clear_bits(mfd,
+ ret = regmap_clear_bits(mfd->regmap,
TPS65910_EN1_LDO_ASS + regoffs, bit_pos);
if (ret < 0) {
dev_err(mfd->dev,
/* External EN2 control */
if (ext_sleep_config & TPS65910_SLEEP_CONTROL_EXT_INPUT_EN2)
- ret = tps65910_reg_set_bits(mfd,
+ ret = regmap_set_bits(mfd->regmap,
TPS65910_EN2_LDO_ASS + regoffs, bit_pos);
else
- ret = tps65910_reg_clear_bits(mfd,
+ ret = regmap_clear_bits(mfd->regmap,
TPS65910_EN2_LDO_ASS + regoffs, bit_pos);
if (ret < 0) {
dev_err(mfd->dev,
if ((tps65910_chip_id(mfd) == TPS65910) &&
(id >= TPS65910_REG_VDIG1)) {
if (ext_sleep_config & TPS65910_SLEEP_CONTROL_EXT_INPUT_EN3)
- ret = tps65910_reg_set_bits(mfd,
+ ret = regmap_set_bits(mfd->regmap,
TPS65910_EN3_LDO_ASS + regoffs, bit_pos);
else
- ret = tps65910_reg_clear_bits(mfd,
+ ret = regmap_clear_bits(mfd->regmap,
TPS65910_EN3_LDO_ASS + regoffs, bit_pos);
if (ret < 0) {
dev_err(mfd->dev,
/* Return if no external control is selected */
if (!(ext_sleep_config & EXT_SLEEP_CONTROL)) {
/* Clear all sleep controls */
- ret = tps65910_reg_clear_bits(mfd,
+ ret = regmap_clear_bits(mfd->regmap,
TPS65910_SLEEP_KEEP_LDO_ON + regoffs, bit_pos);
if (!ret)
- ret = tps65910_reg_clear_bits(mfd,
+ ret = regmap_clear_bits(mfd->regmap,
TPS65910_SLEEP_SET_LDO_OFF + regoffs, bit_pos);
if (ret < 0)
dev_err(mfd->dev,
int sr_reg_add = pmic->get_ctrl_reg(id) + 2;
int opvsel, srvsel;
- ret = tps65910_reg_read(pmic->mfd, op_reg_add, &opvsel);
+ ret = regmap_read(mfd->regmap, op_reg_add, &opvsel);
if (ret < 0)
return ret;
- ret = tps65910_reg_read(pmic->mfd, sr_reg_add, &srvsel);
+ ret = regmap_read(mfd->regmap, sr_reg_add, &srvsel);
if (ret < 0)
return ret;
if (opvsel & VDD1_OP_CMD_MASK) {
u8 reg_val = srvsel & VDD1_OP_SEL_MASK;
- ret = tps65910_reg_write(pmic->mfd, op_reg_add,
- reg_val);
+ ret = regmap_write(mfd->regmap, op_reg_add, reg_val);
if (ret < 0) {
dev_err(mfd->dev,
"Error in configuring op register\n");
return ret;
}
}
- ret = tps65910_reg_write(pmic->mfd, sr_reg_add, 0);
+ ret = regmap_write(mfd->regmap, sr_reg_add, 0);
if (ret < 0) {
dev_err(mfd->dev, "Error in setting sr register\n");
return ret;
}
}
- ret = tps65910_reg_clear_bits(mfd,
+ ret = regmap_clear_bits(mfd->regmap,
TPS65910_SLEEP_KEEP_LDO_ON + regoffs, bit_pos);
if (!ret) {
if (ext_sleep_config & TPS65911_SLEEP_CONTROL_EXT_INPUT_SLEEP)
- ret = tps65910_reg_set_bits(mfd,
+ ret = regmap_set_bits(mfd->regmap,
TPS65910_SLEEP_SET_LDO_OFF + regoffs, bit_pos);
else
- ret = tps65910_reg_clear_bits(mfd,
+ ret = regmap_clear_bits(mfd->regmap,
TPS65910_SLEEP_SET_LDO_OFF + regoffs, bit_pos);
}
if (ret < 0)
platform_set_drvdata(pdev, pmic);
/* Give control of all register to control port */
- err = tps65910_reg_set_bits(pmic->mfd, TPS65910_DEVCTRL,
+ err = regmap_set_bits(pmic->mfd->regmap, TPS65910_DEVCTRL,
DEVCTRL_SR_CTL_I2C_SEL_MASK);
if (err < 0)
return err;
* voltage level can go higher than expected or crash
* Workaround: use no synchronization of DCDC clocks
*/
- tps65910_reg_clear_bits(pmic->mfd, TPS65910_DCDCCTRL,
+ regmap_clear_bits(pmic->mfd->regmap, TPS65910_DCDCCTRL,
DCDCCTRL_DCDCCKSYNC_MASK);
break;
case TPS65911:
};
/* Operations permitted on DCDCx */
-static struct regulator_ops tps65912_ops_dcdc = {
+static const struct regulator_ops tps65912_ops_dcdc = {
.is_enabled = regulator_is_enabled_regmap,
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
};
/* Operations permitted on LDOx */
-static struct regulator_ops tps65912_ops_ldo = {
+static const struct regulator_ops tps65912_ops_ldo = {
.is_enabled = regulator_is_enabled_regmap,
.enable = regulator_enable_regmap,
.disable = regulator_disable_regmap,
{
struct wm831x_dcdc *dcdc = data;
- regulator_lock(dcdc->regulator);
regulator_notifier_call_chain(dcdc->regulator,
REGULATOR_EVENT_UNDER_VOLTAGE,
NULL);
- regulator_unlock(dcdc->regulator);
return IRQ_HANDLED;
}
{
struct wm831x_dcdc *dcdc = data;
- regulator_lock(dcdc->regulator);
regulator_notifier_call_chain(dcdc->regulator,
REGULATOR_EVENT_OVER_CURRENT,
NULL);
- regulator_unlock(dcdc->regulator);
return IRQ_HANDLED;
}
{
struct wm831x_isink *isink = data;
- regulator_lock(isink->regulator);
regulator_notifier_call_chain(isink->regulator,
REGULATOR_EVENT_OVER_CURRENT,
NULL);
- regulator_unlock(isink->regulator);
return IRQ_HANDLED;
}
{
struct wm831x_ldo *ldo = data;
- regulator_lock(ldo->regulator);
regulator_notifier_call_chain(ldo->regulator,
REGULATOR_EVENT_UNDER_VOLTAGE,
NULL);
- regulator_unlock(ldo->regulator);
return IRQ_HANDLED;
}
{
struct regulator_dev *rdev = (struct regulator_dev *)data;
- regulator_lock(rdev);
if (irq == WM8350_IRQ_CS1 || irq == WM8350_IRQ_CS2)
regulator_notifier_call_chain(rdev,
REGULATOR_EVENT_REGULATION_OUT,
regulator_notifier_call_chain(rdev,
REGULATOR_EVENT_UNDER_VOLTAGE,
NULL);
- regulator_unlock(rdev);
return IRQ_HANDLED;
}
config MTK_SCP
tristate "Mediatek SCP support"
- depends on ARCH_MEDIATEK
+ depends on ARCH_MEDIATEK || COMPILE_TEST
select RPMSG_MTK_SCP
help
Say y here to support Mediatek's System Companion Processor (SCP) via
dasd_put_device(base);
return -EFAULT;
}
- if (bdev != bdev->bd_contains) {
+ if (bdev_is_partition(bdev)) {
pr_warn("%s: The specified DASD is a partition and cannot be formatted\n",
dev_name(&base->cdev->dev));
dasd_put_device(base);
base = dasd_device_from_gendisk(bdev->bd_disk);
if (!base)
return -ENODEV;
- if (bdev != bdev->bd_contains) {
+ if (bdev_is_partition(bdev)) {
pr_warn("%s: The specified DASD is a partition and cannot be checked\n",
dev_name(&base->cdev->dev));
rc = -EINVAL;
rc = -EROFS;
goto out_err;
}
- if (bdev != bdev->bd_contains) {
+ if (bdev_is_partition(bdev)) {
pr_warn("%s: The specified DASD is a partition and tracks cannot be released\n",
dev_name(&base->cdev->dev));
rc = -EINVAL;
if (!capable(CAP_SYS_ADMIN))
return -EACCES;
- if (bdev != bdev->bd_contains)
+ if (bdev_is_partition(bdev))
// ro setting is not allowed for partitions
return -EINVAL;
if (get_user(intval, (int __user *)argp))
if (!reqcnt)
return -ENOMEM;
zcrypt_perdev_reqcnt(reqcnt, AP_DEVICES);
- if (copy_to_user((int __user *) arg, reqcnt, sizeof(reqcnt)))
+ if (copy_to_user((int __user *) arg, reqcnt,
+ sizeof(u32) * AP_DEVICES))
rc = -EFAULT;
kfree(reqcnt);
return rc;
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
struct sockaddr_in6 addr;
+ struct socket *sock;
int rc;
switch(param) {
spin_unlock_bh(&conn->session->frwd_lock);
return -ENOTCONN;
}
+ sock = tcp_sw_conn->sock;
+ sock_hold(sock->sk);
+ spin_unlock_bh(&conn->session->frwd_lock);
+
if (param == ISCSI_PARAM_LOCAL_PORT)
- rc = kernel_getsockname(tcp_sw_conn->sock,
+ rc = kernel_getsockname(sock,
(struct sockaddr *)&addr);
else
- rc = kernel_getpeername(tcp_sw_conn->sock,
+ rc = kernel_getpeername(sock,
(struct sockaddr *)&addr);
- spin_unlock_bh(&conn->session->frwd_lock);
+ sock_put(sock->sk);
if (rc < 0)
return rc;
struct iscsi_tcp_conn *tcp_conn;
struct iscsi_sw_tcp_conn *tcp_sw_conn;
struct sockaddr_in6 addr;
+ struct socket *sock;
int rc;
switch (param) {
return -ENOTCONN;
}
tcp_conn = conn->dd_data;
-
tcp_sw_conn = tcp_conn->dd_data;
- if (!tcp_sw_conn->sock) {
+ sock = tcp_sw_conn->sock;
+ if (!sock) {
spin_unlock_bh(&session->frwd_lock);
return -ENOTCONN;
}
+ sock_hold(sock->sk);
+ spin_unlock_bh(&session->frwd_lock);
- rc = kernel_getsockname(tcp_sw_conn->sock,
+ rc = kernel_getsockname(sock,
(struct sockaddr *)&addr);
- spin_unlock_bh(&session->frwd_lock);
+ sock_put(sock->sk);
if (rc < 0)
return rc;
* coalescing neighboring slab objects into a single frag which
* triggers one of hardened usercopy checks.
*/
- if (!recv && page_count(sg_page(sg)) >= 1 && !PageSlab(sg_page(sg)))
+ if (!recv && sendpage_ok(sg_page(sg)))
return;
if (recv) {
static void lpfc_disc_flush_list(struct lpfc_vport *vport);
static void lpfc_unregister_fcfi_cmpl(struct lpfc_hba *, LPFC_MBOXQ_t *);
static int lpfc_fcf_inuse(struct lpfc_hba *);
+static void lpfc_mbx_cmpl_read_sparam(struct lpfc_hba *, LPFC_MBOXQ_t *);
void
lpfc_terminate_rport_io(struct fc_rport *rport)
return;
}
-
void
lpfc_mbx_cmpl_local_config_link(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
{
struct lpfc_vport *vport = pmb->vport;
+ LPFC_MBOXQ_t *sparam_mb;
+ struct lpfc_dmabuf *sparam_mp;
+ int rc;
if (pmb->u.mb.mbxStatus)
goto out;
}
/* Start discovery by sending a FLOGI. port_state is identically
- * LPFC_FLOGI while waiting for FLOGI cmpl. Check if sending
- * the FLOGI is being deferred till after MBX_READ_SPARAM completes.
+ * LPFC_FLOGI while waiting for FLOGI cmpl.
*/
if (vport->port_state != LPFC_FLOGI) {
- if (!(phba->hba_flag & HBA_DEFER_FLOGI))
+ /* Issue MBX_READ_SPARAM to update CSPs before FLOGI if
+ * bb-credit recovery is in place.
+ */
+ if (phba->bbcredit_support && phba->cfg_enable_bbcr &&
+ !(phba->link_flag & LS_LOOPBACK_MODE)) {
+ sparam_mb = mempool_alloc(phba->mbox_mem_pool,
+ GFP_KERNEL);
+ if (!sparam_mb)
+ goto sparam_out;
+
+ rc = lpfc_read_sparam(phba, sparam_mb, 0);
+ if (rc) {
+ mempool_free(sparam_mb, phba->mbox_mem_pool);
+ goto sparam_out;
+ }
+ sparam_mb->vport = vport;
+ sparam_mb->mbox_cmpl = lpfc_mbx_cmpl_read_sparam;
+ rc = lpfc_sli_issue_mbox(phba, sparam_mb, MBX_NOWAIT);
+ if (rc == MBX_NOT_FINISHED) {
+ sparam_mp = (struct lpfc_dmabuf *)
+ sparam_mb->ctx_buf;
+ lpfc_mbuf_free(phba, sparam_mp->virt,
+ sparam_mp->phys);
+ kfree(sparam_mp);
+ sparam_mb->ctx_buf = NULL;
+ mempool_free(sparam_mb, phba->mbox_mem_pool);
+ goto sparam_out;
+ }
+
+ phba->hba_flag |= HBA_DEFER_FLOGI;
+ } else {
lpfc_initial_flogi(vport);
+ }
} else {
if (vport->fc_flag & FC_PT2PT)
lpfc_disc_start(vport);
"0306 CONFIG_LINK mbxStatus error x%x "
"HBA state x%x\n",
pmb->u.mb.mbxStatus, vport->port_state);
+sparam_out:
mempool_free(pmb, phba->mbox_mem_pool);
lpfc_linkdown(phba);
lpfc_linkup(phba);
sparam_mbox = NULL;
- if (!(phba->hba_flag & HBA_FCOE_MODE)) {
- cfglink_mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
- if (!cfglink_mbox)
- goto out;
- vport->port_state = LPFC_LOCAL_CFG_LINK;
- lpfc_config_link(phba, cfglink_mbox);
- cfglink_mbox->vport = vport;
- cfglink_mbox->mbox_cmpl = lpfc_mbx_cmpl_local_config_link;
- rc = lpfc_sli_issue_mbox(phba, cfglink_mbox, MBX_NOWAIT);
- if (rc == MBX_NOT_FINISHED) {
- mempool_free(cfglink_mbox, phba->mbox_mem_pool);
- goto out;
- }
- }
-
sparam_mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
if (!sparam_mbox)
goto out;
goto out;
}
- if (phba->hba_flag & HBA_FCOE_MODE) {
+ if (!(phba->hba_flag & HBA_FCOE_MODE)) {
+ cfglink_mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
+ if (!cfglink_mbox)
+ goto out;
+ vport->port_state = LPFC_LOCAL_CFG_LINK;
+ lpfc_config_link(phba, cfglink_mbox);
+ cfglink_mbox->vport = vport;
+ cfglink_mbox->mbox_cmpl = lpfc_mbx_cmpl_local_config_link;
+ rc = lpfc_sli_issue_mbox(phba, cfglink_mbox, MBX_NOWAIT);
+ if (rc == MBX_NOT_FINISHED) {
+ mempool_free(cfglink_mbox, phba->mbox_mem_pool);
+ goto out;
+ }
+ } else {
vport->port_state = LPFC_VPORT_UNKNOWN;
/*
* Add the driver's default FCF record at FCF index 0 now. This
}
/* Reset FCF roundrobin bmask for new discovery */
lpfc_sli4_clear_fcf_rr_bmask(phba);
- } else {
- if (phba->bbcredit_support && phba->cfg_enable_bbcr &&
- !(phba->link_flag & LS_LOOPBACK_MODE))
- phba->hba_flag |= HBA_DEFER_FLOGI;
}
/* Prepare for LINK up registrations */
if (sdkp->device->type == TYPE_ZBC) {
/* Host-managed */
- q->limits.zoned = BLK_ZONED_HM;
+ blk_queue_set_zoned(sdkp->disk, BLK_ZONED_HM);
} else {
sdkp->zoned = (buffer[8] >> 4) & 3;
- if (sdkp->zoned == 1 && !disk_has_partitions(sdkp->disk)) {
+ if (sdkp->zoned == 1) {
/* Host-aware */
- q->limits.zoned = BLK_ZONED_HA;
+ blk_queue_set_zoned(sdkp->disk, BLK_ZONED_HA);
} else {
- /*
- * Treat drive-managed devices and host-aware devices
- * with partitions as regular block devices.
- */
- q->limits.zoned = BLK_ZONED_NONE;
- if (sdkp->zoned == 2 && sdkp->first_scan)
- sd_printk(KERN_NOTICE, sdkp,
- "Drive-managed SMR disk\n");
+ /* Regular disk or drive managed disk */
+ blk_queue_set_zoned(sdkp->disk, BLK_ZONED_NONE);
}
}
- if (blk_queue_is_zoned(q) && sdkp->first_scan)
+
+ if (!sdkp->first_scan)
+ goto out;
+
+ if (blk_queue_is_zoned(q)) {
sd_printk(KERN_NOTICE, sdkp, "Host-%s zoned block device\n",
q->limits.zoned == BLK_ZONED_HM ? "managed" : "aware");
+ } else {
+ if (sdkp->zoned == 1)
+ sd_printk(KERN_NOTICE, sdkp,
+ "Host-aware SMR disk used as regular disk\n");
+ else if (sdkp->zoned == 2)
+ sd_printk(KERN_NOTICE, sdkp,
+ "Drive-managed SMR disk\n");
+ }
out:
kfree(buffer);
sdkp->first_scan = 1;
sdkp->max_medium_access_timeouts = SD_MAX_MEDIUM_TIMEOUTS;
- error = sd_zbc_init_disk(sdkp);
- if (error)
- goto out_free_index;
-
sd_revalidate_disk(gd);
gd->flags = GENHD_FL_EXT_DEVT;
#ifdef CONFIG_BLK_DEV_ZONED
-int sd_zbc_init_disk(struct scsi_disk *sdkp);
void sd_zbc_release_disk(struct scsi_disk *sdkp);
int sd_zbc_read_zones(struct scsi_disk *sdkp, unsigned char *buffer);
int sd_zbc_revalidate_zones(struct scsi_disk *sdkp);
#else /* CONFIG_BLK_DEV_ZONED */
-static inline int sd_zbc_init_disk(struct scsi_disk *sdkp)
-{
- return 0;
-}
-
static inline void sd_zbc_release_disk(struct scsi_disk *sdkp) {}
static inline int sd_zbc_read_zones(struct scsi_disk *sdkp,
static inline unsigned int sd_zbc_complete(struct scsi_cmnd *cmd,
unsigned int good_bytes, struct scsi_sense_hdr *sshdr)
{
- return 0;
+ return good_bytes;
}
static inline blk_status_t sd_zbc_prepare_zone_append(struct scsi_cmnd *cmd,
sdkp->zone_blocks);
}
+static int sd_zbc_init_disk(struct scsi_disk *sdkp)
+{
+ sdkp->zones_wp_offset = NULL;
+ spin_lock_init(&sdkp->zones_wp_offset_lock);
+ sdkp->rev_wp_offset = NULL;
+ mutex_init(&sdkp->rev_mutex);
+ INIT_WORK(&sdkp->zone_wp_offset_work, sd_zbc_update_wp_offset_workfn);
+ sdkp->zone_wp_update_buf = kzalloc(SD_BUF_SIZE, GFP_KERNEL);
+ if (!sdkp->zone_wp_update_buf)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void sd_zbc_release_disk(struct scsi_disk *sdkp)
+{
+ kvfree(sdkp->zones_wp_offset);
+ sdkp->zones_wp_offset = NULL;
+ kfree(sdkp->zone_wp_update_buf);
+ sdkp->zone_wp_update_buf = NULL;
+}
+
static void sd_zbc_revalidate_zones_cb(struct gendisk *disk)
{
struct scsi_disk *sdkp = scsi_disk(disk);
u32 max_append;
int ret = 0;
- if (!sd_is_zoned(sdkp))
+ /*
+ * For all zoned disks, initialize zone append emulation data if not
+ * already done. This is necessary also for host-aware disks used as
+ * regular disks due to the presence of partitions as these partitions
+ * may be deleted and the disk zoned model changed back from
+ * BLK_ZONED_NONE to BLK_ZONED_HA.
+ */
+ if (sd_is_zoned(sdkp) && !sdkp->zone_wp_update_buf) {
+ ret = sd_zbc_init_disk(sdkp);
+ if (ret)
+ return ret;
+ }
+
+ /*
+ * There is nothing to do for regular disks, including host-aware disks
+ * that have partitions.
+ */
+ if (!blk_queue_is_zoned(q))
return 0;
/*
return ret;
}
-
-int sd_zbc_init_disk(struct scsi_disk *sdkp)
-{
- if (!sd_is_zoned(sdkp))
- return 0;
-
- sdkp->zones_wp_offset = NULL;
- spin_lock_init(&sdkp->zones_wp_offset_lock);
- sdkp->rev_wp_offset = NULL;
- mutex_init(&sdkp->rev_mutex);
- INIT_WORK(&sdkp->zone_wp_offset_work, sd_zbc_update_wp_offset_workfn);
- sdkp->zone_wp_update_buf = kzalloc(SD_BUF_SIZE, GFP_KERNEL);
- if (!sdkp->zone_wp_update_buf)
- return -ENOMEM;
-
- return 0;
-}
-
-void sd_zbc_release_disk(struct scsi_disk *sdkp)
-{
- kvfree(sdkp->zones_wp_offset);
- sdkp->zones_wp_offset = NULL;
- kfree(sdkp->zone_wp_update_buf);
- sdkp->zone_wp_update_buf = NULL;
-}
struct iovec *iov = NULL;
struct iov_iter i;
-#ifdef CONFIG_COMPAT
- if (in_compat_syscall())
- res = compat_import_iovec(rw, hp->dxferp, iov_count,
- 0, &iov, &i);
- else
-#endif
- res = import_iovec(rw, hp->dxferp, iov_count,
- 0, &iov, &i);
+ res = import_iovec(rw, hp->dxferp, iov_count, 0, &iov, &i);
if (res < 0)
return res;
event->id,
&pmc->irq, pmc);
- /*
- * GPIOs don't have an equivalent interrupt in the
- * parent controller (GIC). However some code, such
- * as the one in irq_get_irqchip_state(), require a
- * valid IRQ chip to be set. Make sure that's the
- * case by passing NULL here, which will install a
- * dummy IRQ chip for the interrupt in the parent
- * domain.
- */
- if (domain->parent)
- irq_domain_set_hwirq_and_chip(domain->parent,
- virq, 0, NULL,
- NULL);
-
+ /* GPIO hierarchies stop at the PMC level */
+ if (!err && domain->parent)
+ err = irq_domain_disconnect_hierarchy(domain->parent,
+ virq);
break;
}
}
- /*
- * For interrupts that don't have associated wake events, assign a
- * dummy hardware IRQ number. This is used in the ->irq_set_type()
- * and ->irq_set_wake() callbacks to return early for these IRQs.
- */
- if (i == soc->num_wake_events) {
- err = irq_domain_set_hwirq_and_chip(domain, virq, ULONG_MAX,
- &pmc->irq, pmc);
-
- /*
- * Interrupts without a wake event don't have a corresponding
- * interrupt in the parent controller (GIC). Pass NULL for the
- * chip here, which causes a dummy IRQ chip to be installed
- * for the interrupt in the parent domain, to make this
- * explicit.
- */
- if (domain->parent)
- irq_domain_set_hwirq_and_chip(domain->parent, virq, 0,
- NULL, NULL);
- }
+ /* If there is no wake-up event, there is no PMC mapping */
+ if (i == soc->num_wake_events)
+ err = irq_domain_disconnect_hierarchy(domain, virq);
return err;
}
unsigned int offset, bit;
u32 value;
- if (data->hwirq == ULONG_MAX)
- return 0;
-
offset = data->hwirq / 32;
bit = data->hwirq % 32;
unsigned int offset, bit;
u32 value;
- if (data->hwirq == ULONG_MAX)
- return 0;
-
offset = data->hwirq / 32;
bit = data->hwirq % 32;
unsigned int offset, bit;
u32 value;
- /* nothing to do if there's no associated wake event */
- if (WARN_ON(data->hwirq == ULONG_MAX))
- return 0;
-
offset = data->hwirq / 32;
bit = data->hwirq % 32;
struct tegra_pmc *pmc = irq_data_get_irq_chip_data(data);
u32 value;
- /* nothing to do if there's no associated wake event */
- if (data->hwirq == ULONG_MAX)
- return 0;
-
value = readl(pmc->wake + WAKE_AOWAKE_CNTRL(data->hwirq));
switch (type) {
return 0;
}
+static void tegra_irq_mask_parent(struct irq_data *data)
+{
+ if (data->parent_data)
+ irq_chip_mask_parent(data);
+}
+
+static void tegra_irq_unmask_parent(struct irq_data *data)
+{
+ if (data->parent_data)
+ irq_chip_unmask_parent(data);
+}
+
+static void tegra_irq_eoi_parent(struct irq_data *data)
+{
+ if (data->parent_data)
+ irq_chip_eoi_parent(data);
+}
+
+static int tegra_irq_set_affinity_parent(struct irq_data *data,
+ const struct cpumask *dest,
+ bool force)
+{
+ if (data->parent_data)
+ return irq_chip_set_affinity_parent(data, dest, force);
+
+ return -EINVAL;
+}
+
static int tegra_pmc_irq_init(struct tegra_pmc *pmc)
{
struct irq_domain *parent = NULL;
return 0;
pmc->irq.name = dev_name(pmc->dev);
- pmc->irq.irq_mask = irq_chip_mask_parent;
- pmc->irq.irq_unmask = irq_chip_unmask_parent;
- pmc->irq.irq_eoi = irq_chip_eoi_parent;
- pmc->irq.irq_set_affinity = irq_chip_set_affinity_parent;
+ pmc->irq.irq_mask = tegra_irq_mask_parent;
+ pmc->irq.irq_unmask = tegra_irq_unmask_parent;
+ pmc->irq.irq_eoi = tegra_irq_eoi_parent;
+ pmc->irq.irq_set_affinity = tegra_irq_set_affinity_parent;
pmc->irq.irq_set_type = pmc->soc->irq_set_type;
pmc->irq.irq_set_wake = pmc->soc->irq_set_wake;
config SPI_DESIGNWARE
tristate "DesignWare SPI controller core support"
+ imply SPI_MEM
help
general driver for SPI controller core from DesignWare
tristate "Memory-mapped io interface driver for DW SPI core"
depends on HAS_IOMEM
+config SPI_DW_BT1
+ tristate "Baikal-T1 SPI driver for DW SPI core"
+ depends on MIPS_BAIKAL_T1 || COMPILE_TEST
+ help
+ Baikal-T1 SoC is equipped with three DW APB SSI-based MMIO SPI
+ controllers. Two of them are pretty much normal: with IRQ, DMA,
+ FIFOs of 64 words depth, 4x CSs, but the third one as being a
+ part of the Baikal-T1 System Boot Controller has got a very
+ limited resources: no IRQ, no DMA, only a single native
+ chip-select and Tx/Rx FIFO with just 8 words depth available.
+ The later one is normally connected to an external SPI-nor flash
+ of 128Mb (in general can be of bigger size).
+
+config SPI_DW_BT1_DIRMAP
+ bool "Directly mapped Baikal-T1 Boot SPI flash support"
+ depends on SPI_DW_BT1
+ select MULTIPLEXER
+ select MUX_MMIO
+ help
+ Directly mapped SPI flash memory is an interface specific to the
+ Baikal-T1 System Boot Controller. It is a 16MB MMIO region, which
+ can be used to access a peripheral memory device just by
+ reading/writing data from/to it. Note that the system APB bus
+ will stall during each IO from/to the dirmap region until the
+ operation is finished. So try not to use it concurrently with
+ time-critical tasks (like the SPI memory operations implemented
+ in this driver).
+
endif
config SPI_DLN2
config SPI_QUP
tristate "Qualcomm SPI controller with QUP interface"
- depends on ARCH_QCOM || (ARM && COMPILE_TEST)
+ depends on ARCH_QCOM || COMPILE_TEST
help
Qualcomm Universal Peripheral (QUP) core is an AHB slave that
provides a common data path (an output FIFO and an input FIFO)
obj-$(CONFIG_SPI_DESIGNWARE) += spi-dw.o
spi-dw-y := spi-dw-core.o
spi-dw-$(CONFIG_SPI_DW_DMA) += spi-dw-dma.o
+obj-$(CONFIG_SPI_DW_BT1) += spi-dw-bt1.o
obj-$(CONFIG_SPI_DW_MMIO) += spi-dw-mmio.o
obj-$(CONFIG_SPI_DW_PCI) += spi-dw-pci.o
obj-$(CONFIG_SPI_EFM32) += spi-efm32.o
platform_set_drvdata(pdev, master);
spi = spi_master_get_devdata(master);
- memset(spi, 0, sizeof(struct a3700_spi));
spi->master = master;
master->dma_tx = dma_request_chan(dev, "tx");
if (IS_ERR(master->dma_tx)) {
- err = PTR_ERR(master->dma_tx);
- if (err != -EPROBE_DEFER)
- dev_err(dev, "No TX DMA channel, DMA is disabled\n");
+ err = dev_err_probe(dev, PTR_ERR(master->dma_tx),
+ "No TX DMA channel, DMA is disabled\n");
goto error_clear;
}
csr = spi_readl(as, CSR0 + 4 * chip_select);
csr = SPI_BFINS(SCBR, scbr, csr);
spi_writel(as, CSR0 + 4 * chip_select, csr);
+ xfer->effective_speed_hz = bus_hz / scbr;
return 0;
}
static const struct of_device_id bcm_qspi_of_match[] = {
{
- .compatible = "brcm,spi-bcm7425-qspi",
- .data = &bcm_qspi_no_rev_data,
- },
- {
- .compatible = "brcm,spi-bcm7429-qspi",
- .data = &bcm_qspi_no_rev_data,
- },
- {
- .compatible = "brcm,spi-bcm7435-qspi",
- .data = &bcm_qspi_no_rev_data,
+ .compatible = "brcm,spi-bcm7445-qspi",
+ .data = &bcm_qspi_rev_data,
+
},
{
.compatible = "brcm,spi-bcm-qspi",
- .data = &bcm_qspi_rev_data,
+ .data = &bcm_qspi_no_rev_data,
},
{
.compatible = "brcm,spi-bcm7216-qspi",
#define DRV_NAME "spi-bcm2835"
/* define polling limits */
-unsigned int polling_limit_us = 30;
+static unsigned int polling_limit_us = 30;
module_param(polling_limit_us, uint, 0664);
MODULE_PARM_DESC(polling_limit_us,
"time in us to run a transfer in polling mode\n");
bs->clk = devm_clk_get(&pdev->dev, NULL);
if (IS_ERR(bs->clk)) {
- err = PTR_ERR(bs->clk);
- if (err == -EPROBE_DEFER)
- dev_dbg(&pdev->dev, "could not get clk: %d\n", err);
- else
- dev_err(&pdev->dev, "could not get clk: %d\n", err);
+ err = dev_err_probe(&pdev->dev, PTR_ERR(bs->clk),
+ "could not get clk\n");
goto out_controller_put;
}
cqspi->rx_chan = dma_request_chan_by_mask(&mask);
if (IS_ERR(cqspi->rx_chan)) {
int ret = PTR_ERR(cqspi->rx_chan);
-
- if (ret != -EPROBE_DEFER)
- dev_err(&cqspi->pdev->dev, "No Rx DMA available\n");
cqspi->rx_chan = NULL;
- return ret;
+ return dev_err_probe(&cqspi->pdev->dev, ret, "No Rx DMA available\n");
}
init_completion(&cqspi->rx_dma_complete);
xspi->rx_bytes = transfer->len;
cdns_spi_setup_transfer(spi, transfer);
-
cdns_spi_fill_tx_fifo(xspi);
+ spi_transfer_delay_exec(transfer);
cdns_spi_write(xspi, CDNS_SPI_IER, CDNS_SPI_IXR_DEFAULT);
return transfer->len;
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// Copyright (C) 2020 BAIKAL ELECTRONICS, JSC
+//
+// Authors:
+// Ramil Zaripov <Ramil.Zaripov@baikalelectronics.ru>
+// Serge Semin <Sergey.Semin@baikalelectronics.ru>
+//
+// Baikal-T1 DW APB SPI and System Boot SPI driver
+//
+
+#include <linux/clk.h>
+#include <linux/cpumask.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mux/consumer.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/property.h>
+#include <linux/slab.h>
+#include <linux/spi/spi-mem.h>
+#include <linux/spi/spi.h>
+
+#include "spi-dw.h"
+
+#define BT1_BOOT_DIRMAP 0
+#define BT1_BOOT_REGS 1
+
+struct dw_spi_bt1 {
+ struct dw_spi dws;
+ struct clk *clk;
+ struct mux_control *mux;
+
+#ifdef CONFIG_SPI_DW_BT1_DIRMAP
+ void __iomem *map;
+ resource_size_t map_len;
+#endif
+};
+#define to_dw_spi_bt1(_ctlr) \
+ container_of(spi_controller_get_devdata(_ctlr), struct dw_spi_bt1, dws)
+
+typedef int (*dw_spi_bt1_init_cb)(struct platform_device *pdev,
+ struct dw_spi_bt1 *dwsbt1);
+
+#ifdef CONFIG_SPI_DW_BT1_DIRMAP
+
+static int dw_spi_bt1_dirmap_create(struct spi_mem_dirmap_desc *desc)
+{
+ struct dw_spi_bt1 *dwsbt1 = to_dw_spi_bt1(desc->mem->spi->controller);
+
+ if (!dwsbt1->map ||
+ !dwsbt1->dws.mem_ops.supports_op(desc->mem, &desc->info.op_tmpl))
+ return -EOPNOTSUPP;
+
+ /*
+ * Make sure the requested region doesn't go out of the physically
+ * mapped flash memory bounds and the operation is read-only.
+ */
+ if (desc->info.offset + desc->info.length > dwsbt1->map_len ||
+ desc->info.op_tmpl.data.dir != SPI_MEM_DATA_IN)
+ return -EOPNOTSUPP;
+
+ return 0;
+}
+
+/*
+ * Directly mapped SPI memory region is only accessible in the dword chunks.
+ * That's why we have to create a dedicated read-method to copy data from there
+ * to the passed buffer.
+ */
+static void dw_spi_bt1_dirmap_copy_from_map(void *to, void __iomem *from, size_t len)
+{
+ size_t shift, chunk;
+ u32 data;
+
+ /*
+ * We split the copying up into the next three stages: unaligned head,
+ * aligned body, unaligned tail.
+ */
+ shift = (size_t)from & 0x3;
+ if (shift) {
+ chunk = min_t(size_t, 4 - shift, len);
+ data = readl_relaxed(from - shift);
+ memcpy(to, &data + shift, chunk);
+ from += chunk;
+ to += chunk;
+ len -= chunk;
+ }
+
+ while (len >= 4) {
+ data = readl_relaxed(from);
+ memcpy(to, &data, 4);
+ from += 4;
+ to += 4;
+ len -= 4;
+ }
+
+ if (len) {
+ data = readl_relaxed(from);
+ memcpy(to, &data, len);
+ }
+}
+
+static ssize_t dw_spi_bt1_dirmap_read(struct spi_mem_dirmap_desc *desc,
+ u64 offs, size_t len, void *buf)
+{
+ struct dw_spi_bt1 *dwsbt1 = to_dw_spi_bt1(desc->mem->spi->controller);
+ struct dw_spi *dws = &dwsbt1->dws;
+ struct spi_mem *mem = desc->mem;
+ struct dw_spi_cfg cfg;
+ int ret;
+
+ /*
+ * Make sure the requested operation length is valid. Truncate the
+ * length if it's greater than the length of the MMIO region.
+ */
+ if (offs >= dwsbt1->map_len || !len)
+ return 0;
+
+ len = min_t(size_t, len, dwsbt1->map_len - offs);
+
+ /* Collect the controller configuration required by the operation */
+ cfg.tmode = SPI_TMOD_EPROMREAD;
+ cfg.dfs = 8;
+ cfg.ndf = 4;
+ cfg.freq = mem->spi->max_speed_hz;
+
+ /* Make sure the corresponding CS is de-asserted on transmission */
+ dw_spi_set_cs(mem->spi, false);
+
+ spi_enable_chip(dws, 0);
+
+ dw_spi_update_config(dws, mem->spi, &cfg);
+
+ spi_umask_intr(dws, SPI_INT_RXFI);
+
+ spi_enable_chip(dws, 1);
+
+ /*
+ * Enable the transparent mode of the System Boot Controller.
+ * The SPI core IO should have been locked before calling this method
+ * so noone would be touching the controller' registers during the
+ * dirmap operation.
+ */
+ ret = mux_control_select(dwsbt1->mux, BT1_BOOT_DIRMAP);
+ if (ret)
+ return ret;
+
+ dw_spi_bt1_dirmap_copy_from_map(buf, dwsbt1->map + offs, len);
+
+ mux_control_deselect(dwsbt1->mux);
+
+ dw_spi_set_cs(mem->spi, true);
+
+ ret = dw_spi_check_status(dws, true);
+
+ return ret ?: len;
+}
+
+#endif /* CONFIG_SPI_DW_BT1_DIRMAP */
+
+static int dw_spi_bt1_std_init(struct platform_device *pdev,
+ struct dw_spi_bt1 *dwsbt1)
+{
+ struct dw_spi *dws = &dwsbt1->dws;
+
+ dws->irq = platform_get_irq(pdev, 0);
+ if (dws->irq < 0)
+ return dws->irq;
+
+ dws->num_cs = 4;
+
+ /*
+ * Baikal-T1 Normal SPI Controllers don't always keep up with full SPI
+ * bus speed especially when it comes to the concurrent access to the
+ * APB bus resources. Thus we have no choice but to set a constraint on
+ * the SPI bus frequency for the memory operations which require to
+ * read/write data as fast as possible.
+ */
+ dws->max_mem_freq = 20000000U;
+
+ dw_spi_dma_setup_generic(dws);
+
+ return 0;
+}
+
+static int dw_spi_bt1_sys_init(struct platform_device *pdev,
+ struct dw_spi_bt1 *dwsbt1)
+{
+ struct resource *mem __maybe_unused;
+ struct dw_spi *dws = &dwsbt1->dws;
+
+ /*
+ * Baikal-T1 System Boot Controller is equipped with a mux, which
+ * switches between the directly mapped SPI flash access mode and
+ * IO access to the DW APB SSI registers. Note the mux controller
+ * must be setup to preserve the registers being accessible by default
+ * (on idle-state).
+ */
+ dwsbt1->mux = devm_mux_control_get(&pdev->dev, NULL);
+ if (IS_ERR(dwsbt1->mux))
+ return PTR_ERR(dwsbt1->mux);
+
+ /*
+ * Directly mapped SPI flash memory is a 16MB MMIO region, which can be
+ * used to access a peripheral memory device just by reading/writing
+ * data from/to it. Note the system APB bus will stall during each IO
+ * from/to the dirmap region until the operation is finished. So don't
+ * use it concurrently with time-critical tasks (like the SPI memory
+ * operations implemented in the DW APB SSI driver).
+ */
+#ifdef CONFIG_SPI_DW_BT1_DIRMAP
+ mem = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ if (mem) {
+ dwsbt1->map = devm_ioremap_resource(&pdev->dev, mem);
+ if (!IS_ERR(dwsbt1->map)) {
+ dwsbt1->map_len = (mem->end - mem->start + 1);
+ dws->mem_ops.dirmap_create = dw_spi_bt1_dirmap_create;
+ dws->mem_ops.dirmap_read = dw_spi_bt1_dirmap_read;
+ } else {
+ dwsbt1->map = NULL;
+ }
+ }
+#endif /* CONFIG_SPI_DW_BT1_DIRMAP */
+
+ /*
+ * There is no IRQ, no DMA and just one CS available on the System Boot
+ * SPI controller.
+ */
+ dws->irq = IRQ_NOTCONNECTED;
+ dws->num_cs = 1;
+
+ /*
+ * Baikal-T1 System Boot SPI Controller doesn't keep up with the full
+ * SPI bus speed due to relatively slow APB bus and races for it'
+ * resources from different CPUs. The situation is worsen by a small
+ * FIFOs depth (just 8 words). It works better in a single CPU mode
+ * though, but still tends to be not fast enough at low CPU
+ * frequencies.
+ */
+ if (num_possible_cpus() > 1)
+ dws->max_mem_freq = 10000000U;
+ else
+ dws->max_mem_freq = 20000000U;
+
+ return 0;
+}
+
+static int dw_spi_bt1_probe(struct platform_device *pdev)
+{
+ dw_spi_bt1_init_cb init_func;
+ struct dw_spi_bt1 *dwsbt1;
+ struct resource *mem;
+ struct dw_spi *dws;
+ int ret;
+
+ dwsbt1 = devm_kzalloc(&pdev->dev, sizeof(struct dw_spi_bt1), GFP_KERNEL);
+ if (!dwsbt1)
+ return -ENOMEM;
+
+ dws = &dwsbt1->dws;
+
+ dws->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &mem);
+ if (IS_ERR(dws->regs))
+ return PTR_ERR(dws->regs);
+
+ dws->paddr = mem->start;
+
+ dwsbt1->clk = devm_clk_get(&pdev->dev, NULL);
+ if (IS_ERR(dwsbt1->clk))
+ return PTR_ERR(dwsbt1->clk);
+
+ ret = clk_prepare_enable(dwsbt1->clk);
+ if (ret)
+ return ret;
+
+ dws->bus_num = pdev->id;
+ dws->reg_io_width = 4;
+ dws->max_freq = clk_get_rate(dwsbt1->clk);
+ if (!dws->max_freq)
+ goto err_disable_clk;
+
+ init_func = device_get_match_data(&pdev->dev);
+ ret = init_func(pdev, dwsbt1);
+ if (ret)
+ goto err_disable_clk;
+
+ pm_runtime_enable(&pdev->dev);
+
+ ret = dw_spi_add_host(&pdev->dev, dws);
+ if (ret)
+ goto err_disable_clk;
+
+ platform_set_drvdata(pdev, dwsbt1);
+
+ return 0;
+
+err_disable_clk:
+ clk_disable_unprepare(dwsbt1->clk);
+
+ return ret;
+}
+
+static int dw_spi_bt1_remove(struct platform_device *pdev)
+{
+ struct dw_spi_bt1 *dwsbt1 = platform_get_drvdata(pdev);
+
+ dw_spi_remove_host(&dwsbt1->dws);
+
+ pm_runtime_disable(&pdev->dev);
+
+ clk_disable_unprepare(dwsbt1->clk);
+
+ return 0;
+}
+
+static const struct of_device_id dw_spi_bt1_of_match[] = {
+ { .compatible = "baikal,bt1-ssi", .data = dw_spi_bt1_std_init},
+ { .compatible = "baikal,bt1-sys-ssi", .data = dw_spi_bt1_sys_init},
+ { }
+};
+MODULE_DEVICE_TABLE(of, dw_spi_bt1_of_match);
+
+static struct platform_driver dw_spi_bt1_driver = {
+ .probe = dw_spi_bt1_probe,
+ .remove = dw_spi_bt1_remove,
+ .driver = {
+ .name = "bt1-sys-ssi",
+ .of_match_table = dw_spi_bt1_of_match,
+ },
+};
+module_platform_driver(dw_spi_bt1_driver);
+
+MODULE_AUTHOR("Serge Semin <Sergey.Semin@baikalelectronics.ru>");
+MODULE_DESCRIPTION("Baikal-T1 System Boot SPI Controller driver");
+MODULE_LICENSE("GPL v2");
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/module.h>
+#include <linux/preempt.h>
#include <linux/highmem.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
+#include <linux/string.h>
+#include <linux/of.h>
#include "spi-dw.h"
#include <linux/debugfs.h>
#endif
-/* Slave spi_dev related */
+/* Slave spi_device related */
struct chip_data {
- u8 tmode; /* TR/TO/RO/EEPROM */
- u8 type; /* SPI/SSP/MicroWire */
-
- u16 clk_div; /* baud rate divider */
- u32 speed_hz; /* baud rate */
+ u32 cr0;
+ u32 rx_sample_dly; /* RX sample delay */
};
#ifdef CONFIG_DEBUG_FS
DW_SPI_DBGFS_REG("DMACR", DW_SPI_DMACR),
DW_SPI_DBGFS_REG("DMATDLR", DW_SPI_DMATDLR),
DW_SPI_DBGFS_REG("DMARDLR", DW_SPI_DMARDLR),
+ DW_SPI_DBGFS_REG("RX_SAMPLE_DLY", DW_SPI_RX_SAMPLE_DLY),
};
static int dw_spi_debugfs_init(struct dw_spi *dws)
*/
if (cs_high == enable)
dw_writel(dws, DW_SPI_SER, BIT(spi->chip_select));
- else if (dws->cs_override)
+ else
dw_writel(dws, DW_SPI_SER, 0);
}
EXPORT_SYMBOL_GPL(dw_spi_set_cs);
/* Return the max entries we can fill into tx fifo */
static inline u32 tx_max(struct dw_spi *dws)
{
- u32 tx_left, tx_room, rxtx_gap;
+ u32 tx_room, rxtx_gap;
- tx_left = (dws->tx_end - dws->tx) / dws->n_bytes;
tx_room = dws->fifo_len - dw_readl(dws, DW_SPI_TXFLR);
/*
* shift registers. So a control from sw point of
* view is taken.
*/
- rxtx_gap = ((dws->rx_end - dws->rx) - (dws->tx_end - dws->tx))
- / dws->n_bytes;
+ rxtx_gap = dws->fifo_len - (dws->rx_len - dws->tx_len);
- return min3(tx_left, tx_room, (u32) (dws->fifo_len - rxtx_gap));
+ return min3((u32)dws->tx_len, tx_room, rxtx_gap);
}
/* Return the max entries we should read out of rx fifo */
static inline u32 rx_max(struct dw_spi *dws)
{
- u32 rx_left = (dws->rx_end - dws->rx) / dws->n_bytes;
-
- return min_t(u32, rx_left, dw_readl(dws, DW_SPI_RXFLR));
+ return min_t(u32, dws->rx_len, dw_readl(dws, DW_SPI_RXFLR));
}
static void dw_writer(struct dw_spi *dws)
{
- u32 max;
+ u32 max = tx_max(dws);
u16 txw = 0;
- spin_lock(&dws->buf_lock);
- max = tx_max(dws);
while (max--) {
- /* Set the tx word if the transfer's original "tx" is not null */
- if (dws->tx_end - dws->len) {
+ if (dws->tx) {
if (dws->n_bytes == 1)
txw = *(u8 *)(dws->tx);
else
txw = *(u16 *)(dws->tx);
+
+ dws->tx += dws->n_bytes;
}
dw_write_io_reg(dws, DW_SPI_DR, txw);
- dws->tx += dws->n_bytes;
+ --dws->tx_len;
}
- spin_unlock(&dws->buf_lock);
}
static void dw_reader(struct dw_spi *dws)
{
- u32 max;
+ u32 max = rx_max(dws);
u16 rxw;
- spin_lock(&dws->buf_lock);
- max = rx_max(dws);
while (max--) {
rxw = dw_read_io_reg(dws, DW_SPI_DR);
- /* Care rx only if the transfer's original "rx" is not null */
- if (dws->rx_end - dws->len) {
+ if (dws->rx) {
if (dws->n_bytes == 1)
*(u8 *)(dws->rx) = rxw;
else
*(u16 *)(dws->rx) = rxw;
+
+ dws->rx += dws->n_bytes;
}
- dws->rx += dws->n_bytes;
+ --dws->rx_len;
}
- spin_unlock(&dws->buf_lock);
}
-static void int_error_stop(struct dw_spi *dws, const char *msg)
+int dw_spi_check_status(struct dw_spi *dws, bool raw)
{
- spi_reset_chip(dws);
+ u32 irq_status;
+ int ret = 0;
+
+ if (raw)
+ irq_status = dw_readl(dws, DW_SPI_RISR);
+ else
+ irq_status = dw_readl(dws, DW_SPI_ISR);
+
+ if (irq_status & SPI_INT_RXOI) {
+ dev_err(&dws->master->dev, "RX FIFO overflow detected\n");
+ ret = -EIO;
+ }
+
+ if (irq_status & SPI_INT_RXUI) {
+ dev_err(&dws->master->dev, "RX FIFO underflow detected\n");
+ ret = -EIO;
+ }
+
+ if (irq_status & SPI_INT_TXOI) {
+ dev_err(&dws->master->dev, "TX FIFO overflow detected\n");
+ ret = -EIO;
+ }
- dev_err(&dws->master->dev, "%s\n", msg);
- dws->master->cur_msg->status = -EIO;
- spi_finalize_current_transfer(dws->master);
+ /* Generically handle the erroneous situation */
+ if (ret) {
+ spi_reset_chip(dws);
+ if (dws->master->cur_msg)
+ dws->master->cur_msg->status = ret;
+ }
+
+ return ret;
}
+EXPORT_SYMBOL_GPL(dw_spi_check_status);
-static irqreturn_t interrupt_transfer(struct dw_spi *dws)
+static irqreturn_t dw_spi_transfer_handler(struct dw_spi *dws)
{
u16 irq_status = dw_readl(dws, DW_SPI_ISR);
- /* Error handling */
- if (irq_status & (SPI_INT_TXOI | SPI_INT_RXOI | SPI_INT_RXUI)) {
- dw_readl(dws, DW_SPI_ICR);
- int_error_stop(dws, "interrupt_transfer: fifo overrun/underrun");
+ if (dw_spi_check_status(dws, false)) {
+ spi_finalize_current_transfer(dws->master);
return IRQ_HANDLED;
}
+ /*
+ * Read data from the Rx FIFO every time we've got a chance executing
+ * this method. If there is nothing left to receive, terminate the
+ * procedure. Otherwise adjust the Rx FIFO Threshold level if it's a
+ * final stage of the transfer. By doing so we'll get the next IRQ
+ * right when the leftover incoming data is received.
+ */
dw_reader(dws);
- if (dws->rx_end == dws->rx) {
- spi_mask_intr(dws, SPI_INT_TXEI);
+ if (!dws->rx_len) {
+ spi_mask_intr(dws, 0xff);
spi_finalize_current_transfer(dws->master);
- return IRQ_HANDLED;
+ } else if (dws->rx_len <= dw_readl(dws, DW_SPI_RXFTLR)) {
+ dw_writel(dws, DW_SPI_RXFTLR, dws->rx_len - 1);
}
+
+ /*
+ * Send data out if Tx FIFO Empty IRQ is received. The IRQ will be
+ * disabled after the data transmission is finished so not to
+ * have the TXE IRQ flood at the final stage of the transfer.
+ */
if (irq_status & SPI_INT_TXEI) {
- spi_mask_intr(dws, SPI_INT_TXEI);
dw_writer(dws);
- /* Enable TX irq always, it will be disabled when RX finished */
- spi_umask_intr(dws, SPI_INT_TXEI);
+ if (!dws->tx_len)
+ spi_mask_intr(dws, SPI_INT_TXEI);
}
return IRQ_HANDLED;
return IRQ_NONE;
if (!master->cur_msg) {
- spi_mask_intr(dws, SPI_INT_TXEI);
+ spi_mask_intr(dws, 0xff);
return IRQ_HANDLED;
}
return dws->transfer_handler(dws);
}
-/* Configure CTRLR0 for DW_apb_ssi */
-u32 dw_spi_update_cr0(struct spi_controller *master, struct spi_device *spi,
- struct spi_transfer *transfer)
+static u32 dw_spi_prepare_cr0(struct dw_spi *dws, struct spi_device *spi)
{
- struct chip_data *chip = spi_get_ctldata(spi);
- u32 cr0;
-
- /* Default SPI mode is SCPOL = 0, SCPH = 0 */
- cr0 = (transfer->bits_per_word - 1)
- | (chip->type << SPI_FRF_OFFSET)
- | ((((spi->mode & SPI_CPOL) ? 1 : 0) << SPI_SCOL_OFFSET) |
- (((spi->mode & SPI_CPHA) ? 1 : 0) << SPI_SCPH_OFFSET) |
- (((spi->mode & SPI_LOOP) ? 1 : 0) << SPI_SRL_OFFSET))
- | (chip->tmode << SPI_TMOD_OFFSET);
+ u32 cr0 = 0;
+
+ if (!(dws->caps & DW_SPI_CAP_DWC_SSI)) {
+ /* CTRLR0[ 5: 4] Frame Format */
+ cr0 |= SSI_MOTO_SPI << SPI_FRF_OFFSET;
+
+ /*
+ * SPI mode (SCPOL|SCPH)
+ * CTRLR0[ 6] Serial Clock Phase
+ * CTRLR0[ 7] Serial Clock Polarity
+ */
+ cr0 |= ((spi->mode & SPI_CPOL) ? 1 : 0) << SPI_SCOL_OFFSET;
+ cr0 |= ((spi->mode & SPI_CPHA) ? 1 : 0) << SPI_SCPH_OFFSET;
+
+ /* CTRLR0[11] Shift Register Loop */
+ cr0 |= ((spi->mode & SPI_LOOP) ? 1 : 0) << SPI_SRL_OFFSET;
+ } else {
+ /* CTRLR0[ 7: 6] Frame Format */
+ cr0 |= SSI_MOTO_SPI << DWC_SSI_CTRLR0_FRF_OFFSET;
+
+ /*
+ * SPI mode (SCPOL|SCPH)
+ * CTRLR0[ 8] Serial Clock Phase
+ * CTRLR0[ 9] Serial Clock Polarity
+ */
+ cr0 |= ((spi->mode & SPI_CPOL) ? 1 : 0) << DWC_SSI_CTRLR0_SCPOL_OFFSET;
+ cr0 |= ((spi->mode & SPI_CPHA) ? 1 : 0) << DWC_SSI_CTRLR0_SCPH_OFFSET;
+
+ /* CTRLR0[13] Shift Register Loop */
+ cr0 |= ((spi->mode & SPI_LOOP) ? 1 : 0) << DWC_SSI_CTRLR0_SRL_OFFSET;
+
+ if (dws->caps & DW_SPI_CAP_KEEMBAY_MST)
+ cr0 |= DWC_SSI_CTRLR0_KEEMBAY_MST;
+ }
return cr0;
}
-EXPORT_SYMBOL_GPL(dw_spi_update_cr0);
-/* Configure CTRLR0 for DWC_ssi */
-u32 dw_spi_update_cr0_v1_01a(struct spi_controller *master,
- struct spi_device *spi,
- struct spi_transfer *transfer)
+void dw_spi_update_config(struct dw_spi *dws, struct spi_device *spi,
+ struct dw_spi_cfg *cfg)
{
struct chip_data *chip = spi_get_ctldata(spi);
- u32 cr0;
+ u32 cr0 = chip->cr0;
+ u32 speed_hz;
+ u16 clk_div;
+
+ /* CTRLR0[ 4/3: 0] Data Frame Size */
+ cr0 |= (cfg->dfs - 1);
+
+ if (!(dws->caps & DW_SPI_CAP_DWC_SSI))
+ /* CTRLR0[ 9:8] Transfer Mode */
+ cr0 |= cfg->tmode << SPI_TMOD_OFFSET;
+ else
+ /* CTRLR0[11:10] Transfer Mode */
+ cr0 |= cfg->tmode << DWC_SSI_CTRLR0_TMOD_OFFSET;
+
+ dw_writel(dws, DW_SPI_CTRLR0, cr0);
+
+ if (cfg->tmode == SPI_TMOD_EPROMREAD || cfg->tmode == SPI_TMOD_RO)
+ dw_writel(dws, DW_SPI_CTRLR1, cfg->ndf ? cfg->ndf - 1 : 0);
+
+ /* Note DW APB SSI clock divider doesn't support odd numbers */
+ clk_div = (DIV_ROUND_UP(dws->max_freq, cfg->freq) + 1) & 0xfffe;
+ speed_hz = dws->max_freq / clk_div;
+
+ if (dws->current_freq != speed_hz) {
+ spi_set_clk(dws, clk_div);
+ dws->current_freq = speed_hz;
+ }
- /* CTRLR0[ 4: 0] Data Frame Size */
- cr0 = (transfer->bits_per_word - 1);
+ /* Update RX sample delay if required */
+ if (dws->cur_rx_sample_dly != chip->rx_sample_dly) {
+ dw_writel(dws, DW_SPI_RX_SAMPLE_DLY, chip->rx_sample_dly);
+ dws->cur_rx_sample_dly = chip->rx_sample_dly;
+ }
+}
+EXPORT_SYMBOL_GPL(dw_spi_update_config);
- /* CTRLR0[ 7: 6] Frame Format */
- cr0 |= chip->type << DWC_SSI_CTRLR0_FRF_OFFSET;
+static void dw_spi_irq_setup(struct dw_spi *dws)
+{
+ u16 level;
+ u8 imask;
/*
- * SPI mode (SCPOL|SCPH)
- * CTRLR0[ 8] Serial Clock Phase
- * CTRLR0[ 9] Serial Clock Polarity
+ * Originally Tx and Rx data lengths match. Rx FIFO Threshold level
+ * will be adjusted at the final stage of the IRQ-based SPI transfer
+ * execution so not to lose the leftover of the incoming data.
*/
- cr0 |= ((spi->mode & SPI_CPOL) ? 1 : 0) << DWC_SSI_CTRLR0_SCPOL_OFFSET;
- cr0 |= ((spi->mode & SPI_CPHA) ? 1 : 0) << DWC_SSI_CTRLR0_SCPH_OFFSET;
+ level = min_t(u16, dws->fifo_len / 2, dws->tx_len);
+ dw_writel(dws, DW_SPI_TXFTLR, level);
+ dw_writel(dws, DW_SPI_RXFTLR, level - 1);
- /* CTRLR0[11:10] Transfer Mode */
- cr0 |= chip->tmode << DWC_SSI_CTRLR0_TMOD_OFFSET;
+ imask = SPI_INT_TXEI | SPI_INT_TXOI | SPI_INT_RXUI | SPI_INT_RXOI |
+ SPI_INT_RXFI;
+ spi_umask_intr(dws, imask);
- /* CTRLR0[13] Shift Register Loop */
- cr0 |= ((spi->mode & SPI_LOOP) ? 1 : 0) << DWC_SSI_CTRLR0_SRL_OFFSET;
+ dws->transfer_handler = dw_spi_transfer_handler;
+}
- return cr0;
+/*
+ * The iterative procedure of the poll-based transfer is simple: write as much
+ * as possible to the Tx FIFO, wait until the pending to receive data is ready
+ * to be read, read it from the Rx FIFO and check whether the performed
+ * procedure has been successful.
+ *
+ * Note this method the same way as the IRQ-based transfer won't work well for
+ * the SPI devices connected to the controller with native CS due to the
+ * automatic CS assertion/de-assertion.
+ */
+static int dw_spi_poll_transfer(struct dw_spi *dws,
+ struct spi_transfer *transfer)
+{
+ struct spi_delay delay;
+ u16 nbits;
+ int ret;
+
+ delay.unit = SPI_DELAY_UNIT_SCK;
+ nbits = dws->n_bytes * BITS_PER_BYTE;
+
+ do {
+ dw_writer(dws);
+
+ delay.value = nbits * (dws->rx_len - dws->tx_len);
+ spi_delay_exec(&delay, transfer);
+
+ dw_reader(dws);
+
+ ret = dw_spi_check_status(dws, true);
+ if (ret)
+ return ret;
+ } while (dws->rx_len);
+
+ return 0;
}
-EXPORT_SYMBOL_GPL(dw_spi_update_cr0_v1_01a);
static int dw_spi_transfer_one(struct spi_controller *master,
struct spi_device *spi, struct spi_transfer *transfer)
{
struct dw_spi *dws = spi_controller_get_devdata(master);
- struct chip_data *chip = spi_get_ctldata(spi);
- unsigned long flags;
- u8 imask = 0;
- u16 txlevel = 0;
- u32 cr0;
+ struct dw_spi_cfg cfg = {
+ .tmode = SPI_TMOD_TR,
+ .dfs = transfer->bits_per_word,
+ .freq = transfer->speed_hz,
+ };
int ret;
dws->dma_mapped = 0;
- spin_lock_irqsave(&dws->buf_lock, flags);
+ dws->n_bytes = DIV_ROUND_UP(transfer->bits_per_word, BITS_PER_BYTE);
dws->tx = (void *)transfer->tx_buf;
- dws->tx_end = dws->tx + transfer->len;
+ dws->tx_len = transfer->len / dws->n_bytes;
dws->rx = transfer->rx_buf;
- dws->rx_end = dws->rx + transfer->len;
- dws->len = transfer->len;
- spin_unlock_irqrestore(&dws->buf_lock, flags);
+ dws->rx_len = dws->tx_len;
- /* Ensure dw->rx and dw->rx_end are visible */
+ /* Ensure the data above is visible for all CPUs */
smp_mb();
spi_enable_chip(dws, 0);
- /* Handle per transfer options for bpw and speed */
- if (transfer->speed_hz != dws->current_freq) {
- if (transfer->speed_hz != chip->speed_hz) {
- /* clk_div doesn't support odd number */
- chip->clk_div = (DIV_ROUND_UP(dws->max_freq, transfer->speed_hz) + 1) & 0xfffe;
- chip->speed_hz = transfer->speed_hz;
- }
- dws->current_freq = transfer->speed_hz;
- spi_set_clk(dws, chip->clk_div);
- }
-
- transfer->effective_speed_hz = dws->max_freq / chip->clk_div;
- dws->n_bytes = DIV_ROUND_UP(transfer->bits_per_word, BITS_PER_BYTE);
+ dw_spi_update_config(dws, spi, &cfg);
- cr0 = dws->update_cr0(master, spi, transfer);
- dw_writel(dws, DW_SPI_CTRLR0, cr0);
+ transfer->effective_speed_hz = dws->current_freq;
/* Check if current transfer is a DMA transaction */
if (master->can_dma && master->can_dma(master, spi, transfer))
/* For poll mode just disable all interrupts */
spi_mask_intr(dws, 0xff);
- /*
- * Interrupt mode
- * we only need set the TXEI IRQ, as TX/RX always happen syncronizely
- */
if (dws->dma_mapped) {
ret = dws->dma_ops->dma_setup(dws, transfer);
- if (ret < 0) {
- spi_enable_chip(dws, 1);
+ if (ret)
return ret;
- }
- } else {
- txlevel = min_t(u16, dws->fifo_len / 2, dws->len / dws->n_bytes);
- dw_writel(dws, DW_SPI_TXFTLR, txlevel);
-
- /* Set the interrupt mask */
- imask |= SPI_INT_TXEI | SPI_INT_TXOI |
- SPI_INT_RXUI | SPI_INT_RXOI;
- spi_umask_intr(dws, imask);
-
- dws->transfer_handler = interrupt_transfer;
}
spi_enable_chip(dws, 1);
if (dws->dma_mapped)
return dws->dma_ops->dma_transfer(dws, transfer);
+ else if (dws->irq == IRQ_NOTCONNECTED)
+ return dw_spi_poll_transfer(dws, transfer);
+
+ dw_spi_irq_setup(dws);
return 1;
}
spi_reset_chip(dws);
}
+static int dw_spi_adjust_mem_op_size(struct spi_mem *mem, struct spi_mem_op *op)
+{
+ if (op->data.dir == SPI_MEM_DATA_IN)
+ op->data.nbytes = clamp_val(op->data.nbytes, 0, SPI_NDF_MASK + 1);
+
+ return 0;
+}
+
+static bool dw_spi_supports_mem_op(struct spi_mem *mem,
+ const struct spi_mem_op *op)
+{
+ if (op->data.buswidth > 1 || op->addr.buswidth > 1 ||
+ op->dummy.buswidth > 1 || op->cmd.buswidth > 1)
+ return false;
+
+ return spi_mem_default_supports_op(mem, op);
+}
+
+static int dw_spi_init_mem_buf(struct dw_spi *dws, const struct spi_mem_op *op)
+{
+ unsigned int i, j, len;
+ u8 *out;
+
+ /*
+ * Calculate the total length of the EEPROM command transfer and
+ * either use the pre-allocated buffer or create a temporary one.
+ */
+ len = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes;
+ if (op->data.dir == SPI_MEM_DATA_OUT)
+ len += op->data.nbytes;
+
+ if (len <= SPI_BUF_SIZE) {
+ out = dws->buf;
+ } else {
+ out = kzalloc(len, GFP_KERNEL);
+ if (!out)
+ return -ENOMEM;
+ }
+
+ /*
+ * Collect the operation code, address and dummy bytes into the single
+ * buffer. If it's a transfer with data to be sent, also copy it into the
+ * single buffer in order to speed the data transmission up.
+ */
+ for (i = 0; i < op->cmd.nbytes; ++i)
+ out[i] = SPI_GET_BYTE(op->cmd.opcode, op->cmd.nbytes - i - 1);
+ for (j = 0; j < op->addr.nbytes; ++i, ++j)
+ out[i] = SPI_GET_BYTE(op->addr.val, op->addr.nbytes - j - 1);
+ for (j = 0; j < op->dummy.nbytes; ++i, ++j)
+ out[i] = 0x0;
+
+ if (op->data.dir == SPI_MEM_DATA_OUT)
+ memcpy(&out[i], op->data.buf.out, op->data.nbytes);
+
+ dws->n_bytes = 1;
+ dws->tx = out;
+ dws->tx_len = len;
+ if (op->data.dir == SPI_MEM_DATA_IN) {
+ dws->rx = op->data.buf.in;
+ dws->rx_len = op->data.nbytes;
+ } else {
+ dws->rx = NULL;
+ dws->rx_len = 0;
+ }
+
+ return 0;
+}
+
+static void dw_spi_free_mem_buf(struct dw_spi *dws)
+{
+ if (dws->tx != dws->buf)
+ kfree(dws->tx);
+}
+
+static int dw_spi_write_then_read(struct dw_spi *dws, struct spi_device *spi)
+{
+ u32 room, entries, sts;
+ unsigned int len;
+ u8 *buf;
+
+ /*
+ * At initial stage we just pre-fill the Tx FIFO in with no rush,
+ * since native CS hasn't been enabled yet and the automatic data
+ * transmission won't start til we do that.
+ */
+ len = min(dws->fifo_len, dws->tx_len);
+ buf = dws->tx;
+ while (len--)
+ dw_write_io_reg(dws, DW_SPI_DR, *buf++);
+
+ /*
+ * After setting any bit in the SER register the transmission will
+ * start automatically. We have to keep up with that procedure
+ * otherwise the CS de-assertion will happen whereupon the memory
+ * operation will be pre-terminated.
+ */
+ len = dws->tx_len - ((void *)buf - dws->tx);
+ dw_spi_set_cs(spi, false);
+ while (len) {
+ entries = readl_relaxed(dws->regs + DW_SPI_TXFLR);
+ if (!entries) {
+ dev_err(&dws->master->dev, "CS de-assertion on Tx\n");
+ return -EIO;
+ }
+ room = min(dws->fifo_len - entries, len);
+ for (; room; --room, --len)
+ dw_write_io_reg(dws, DW_SPI_DR, *buf++);
+ }
+
+ /*
+ * Data fetching will start automatically if the EEPROM-read mode is
+ * activated. We have to keep up with the incoming data pace to
+ * prevent the Rx FIFO overflow causing the inbound data loss.
+ */
+ len = dws->rx_len;
+ buf = dws->rx;
+ while (len) {
+ entries = readl_relaxed(dws->regs + DW_SPI_RXFLR);
+ if (!entries) {
+ sts = readl_relaxed(dws->regs + DW_SPI_RISR);
+ if (sts & SPI_INT_RXOI) {
+ dev_err(&dws->master->dev, "FIFO overflow on Rx\n");
+ return -EIO;
+ }
+ continue;
+ }
+ entries = min(entries, len);
+ for (; entries; --entries, --len)
+ *buf++ = dw_read_io_reg(dws, DW_SPI_DR);
+ }
+
+ return 0;
+}
+
+static inline bool dw_spi_ctlr_busy(struct dw_spi *dws)
+{
+ return dw_readl(dws, DW_SPI_SR) & SR_BUSY;
+}
+
+static int dw_spi_wait_mem_op_done(struct dw_spi *dws)
+{
+ int retry = SPI_WAIT_RETRIES;
+ struct spi_delay delay;
+ unsigned long ns, us;
+ u32 nents;
+
+ nents = dw_readl(dws, DW_SPI_TXFLR);
+ ns = NSEC_PER_SEC / dws->current_freq * nents;
+ ns *= dws->n_bytes * BITS_PER_BYTE;
+ if (ns <= NSEC_PER_USEC) {
+ delay.unit = SPI_DELAY_UNIT_NSECS;
+ delay.value = ns;
+ } else {
+ us = DIV_ROUND_UP(ns, NSEC_PER_USEC);
+ delay.unit = SPI_DELAY_UNIT_USECS;
+ delay.value = clamp_val(us, 0, USHRT_MAX);
+ }
+
+ while (dw_spi_ctlr_busy(dws) && retry--)
+ spi_delay_exec(&delay, NULL);
+
+ if (retry < 0) {
+ dev_err(&dws->master->dev, "Mem op hanged up\n");
+ return -EIO;
+ }
+
+ return 0;
+}
+
+static void dw_spi_stop_mem_op(struct dw_spi *dws, struct spi_device *spi)
+{
+ spi_enable_chip(dws, 0);
+ dw_spi_set_cs(spi, true);
+ spi_enable_chip(dws, 1);
+}
+
+/*
+ * The SPI memory operation implementation below is the best choice for the
+ * devices, which are selected by the native chip-select lane. It's
+ * specifically developed to workaround the problem with automatic chip-select
+ * lane toggle when there is no data in the Tx FIFO buffer. Luckily the current
+ * SPI-mem core calls exec_op() callback only if the GPIO-based CS is
+ * unavailable.
+ */
+static int dw_spi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
+{
+ struct dw_spi *dws = spi_controller_get_devdata(mem->spi->controller);
+ struct dw_spi_cfg cfg;
+ unsigned long flags;
+ int ret;
+
+ /*
+ * Collect the outbound data into a single buffer to speed the
+ * transmission up at least on the initial stage.
+ */
+ ret = dw_spi_init_mem_buf(dws, op);
+ if (ret)
+ return ret;
+
+ /*
+ * DW SPI EEPROM-read mode is required only for the SPI memory Data-IN
+ * operation. Transmit-only mode is suitable for the rest of them.
+ */
+ cfg.dfs = 8;
+ cfg.freq = clamp(mem->spi->max_speed_hz, 0U, dws->max_mem_freq);
+ if (op->data.dir == SPI_MEM_DATA_IN) {
+ cfg.tmode = SPI_TMOD_EPROMREAD;
+ cfg.ndf = op->data.nbytes;
+ } else {
+ cfg.tmode = SPI_TMOD_TO;
+ }
+
+ spi_enable_chip(dws, 0);
+
+ dw_spi_update_config(dws, mem->spi, &cfg);
+
+ spi_mask_intr(dws, 0xff);
+
+ spi_enable_chip(dws, 1);
+
+ /*
+ * DW APB SSI controller has very nasty peculiarities. First originally
+ * (without any vendor-specific modifications) it doesn't provide a
+ * direct way to set and clear the native chip-select signal. Instead
+ * the controller asserts the CS lane if Tx FIFO isn't empty and a
+ * transmission is going on, and automatically de-asserts it back to
+ * the high level if the Tx FIFO doesn't have anything to be pushed
+ * out. Due to that a multi-tasking or heavy IRQs activity might be
+ * fatal, since the transfer procedure preemption may cause the Tx FIFO
+ * getting empty and sudden CS de-assertion, which in the middle of the
+ * transfer will most likely cause the data loss. Secondly the
+ * EEPROM-read or Read-only DW SPI transfer modes imply the incoming
+ * data being automatically pulled in into the Rx FIFO. So if the
+ * driver software is late in fetching the data from the FIFO before
+ * it's overflown, new incoming data will be lost. In order to make
+ * sure the executed memory operations are CS-atomic and to prevent the
+ * Rx FIFO overflow we have to disable the local interrupts so to block
+ * any preemption during the subsequent IO operations.
+ *
+ * Note. At some circumstances disabling IRQs may not help to prevent
+ * the problems described above. The CS de-assertion and Rx FIFO
+ * overflow may still happen due to the relatively slow system bus or
+ * CPU not working fast enough, so the write-then-read algo implemented
+ * here just won't keep up with the SPI bus data transfer. Such
+ * situation is highly platform specific and is supposed to be fixed by
+ * manually restricting the SPI bus frequency using the
+ * dws->max_mem_freq parameter.
+ */
+ local_irq_save(flags);
+ preempt_disable();
+
+ ret = dw_spi_write_then_read(dws, mem->spi);
+
+ local_irq_restore(flags);
+ preempt_enable();
+
+ /*
+ * Wait for the operation being finished and check the controller
+ * status only if there hasn't been any run-time error detected. In the
+ * former case it's just pointless. In the later one to prevent an
+ * additional error message printing since any hw error flag being set
+ * would be due to an error detected on the data transfer.
+ */
+ if (!ret) {
+ ret = dw_spi_wait_mem_op_done(dws);
+ if (!ret)
+ ret = dw_spi_check_status(dws, true);
+ }
+
+ dw_spi_stop_mem_op(dws, mem->spi);
+
+ dw_spi_free_mem_buf(dws);
+
+ return ret;
+}
+
+/*
+ * Initialize the default memory operations if a glue layer hasn't specified
+ * custom ones. Direct mapping operations will be preserved anyway since DW SPI
+ * controller doesn't have an embedded dirmap interface. Note the memory
+ * operations implemented in this driver is the best choice only for the DW APB
+ * SSI controller with standard native CS functionality. If a hardware vendor
+ * has fixed the automatic CS assertion/de-assertion peculiarity, then it will
+ * be safer to use the normal SPI-messages-based transfers implementation.
+ */
+static void dw_spi_init_mem_ops(struct dw_spi *dws)
+{
+ if (!dws->mem_ops.exec_op && !(dws->caps & DW_SPI_CAP_CS_OVERRIDE) &&
+ !dws->set_cs) {
+ dws->mem_ops.adjust_op_size = dw_spi_adjust_mem_op_size;
+ dws->mem_ops.supports_op = dw_spi_supports_mem_op;
+ dws->mem_ops.exec_op = dw_spi_exec_mem_op;
+ if (!dws->max_mem_freq)
+ dws->max_mem_freq = dws->max_freq;
+ }
+}
+
/* This may be called twice for each spi dev */
static int dw_spi_setup(struct spi_device *spi)
{
+ struct dw_spi *dws = spi_controller_get_devdata(spi->controller);
struct chip_data *chip;
/* Only alloc on first setup */
chip = spi_get_ctldata(spi);
if (!chip) {
+ struct dw_spi *dws = spi_controller_get_devdata(spi->controller);
+ u32 rx_sample_dly_ns;
+
chip = kzalloc(sizeof(struct chip_data), GFP_KERNEL);
if (!chip)
return -ENOMEM;
spi_set_ctldata(spi, chip);
+ /* Get specific / default rx-sample-delay */
+ if (device_property_read_u32(&spi->dev,
+ "rx-sample-delay-ns",
+ &rx_sample_dly_ns) != 0)
+ /* Use default controller value */
+ rx_sample_dly_ns = dws->def_rx_sample_dly_ns;
+ chip->rx_sample_dly = DIV_ROUND_CLOSEST(rx_sample_dly_ns,
+ NSEC_PER_SEC /
+ dws->max_freq);
}
- chip->tmode = SPI_TMOD_TR;
+ /*
+ * Update CR0 data each time the setup callback is invoked since
+ * the device parameters could have been changed, for instance, by
+ * the MMC SPI driver or something else.
+ */
+ chip->cr0 = dw_spi_prepare_cr0(dws, spi);
return 0;
}
}
/* enable HW fixup for explicit CS deselect for Amazon's alpine chip */
- if (dws->cs_override)
+ if (dws->caps & DW_SPI_CAP_CS_OVERRIDE)
dw_writel(dws, DW_SPI_CS_OVERRIDE, 0xF);
}
return -ENOMEM;
dws->master = master;
- dws->type = SSI_MOTO_SPI;
dws->dma_addr = (dma_addr_t)(dws->paddr + DW_SPI_DR);
- spin_lock_init(&dws->buf_lock);
spi_controller_set_devdata(master, dws);
+ /* Basic HW init */
+ spi_hw_init(dev, dws);
+
ret = request_irq(dws->irq, dw_spi_irq, IRQF_SHARED, dev_name(dev),
master);
- if (ret < 0) {
+ if (ret < 0 && ret != -ENOTCONN) {
dev_err(dev, "can not get IRQ\n");
goto err_free_master;
}
+ dw_spi_init_mem_ops(dws);
+
master->use_gpio_descriptors = true;
master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LOOP;
master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16);
master->num_chipselect = dws->num_cs;
master->setup = dw_spi_setup;
master->cleanup = dw_spi_cleanup;
- master->set_cs = dw_spi_set_cs;
+ if (dws->set_cs)
+ master->set_cs = dws->set_cs;
+ else
+ master->set_cs = dw_spi_set_cs;
master->transfer_one = dw_spi_transfer_one;
master->handle_err = dw_spi_handle_err;
+ master->mem_ops = &dws->mem_ops;
master->max_speed_hz = dws->max_freq;
master->dev.of_node = dev->of_node;
master->dev.fwnode = dev->fwnode;
master->flags = SPI_MASTER_GPIO_SS;
master->auto_runtime_pm = true;
- if (dws->set_cs)
- master->set_cs = dws->set_cs;
-
- /* Basic HW init */
- spi_hw_init(dev, dws);
+ /* Get default rx sample delay */
+ device_property_read_u32(dev, "rx-sample-delay-ns",
+ &dws->def_rx_sample_dly_ns);
if (dws->dma_ops && dws->dma_ops->dma_init) {
ret = dws->dma_ops->dma_init(dev, dws);
#include "spi-dw.h"
-#define WAIT_RETRIES 5
#define RX_BUSY 0
#define RX_BURST_LEVEL 16
#define TX_BUSY 1
max_burst = RX_BURST_LEVEL;
dws->rxburst = min(max_burst, def_burst);
+ dw_writel(dws, DW_SPI_DMARDLR, dws->rxburst - 1);
ret = dma_get_slave_caps(dws->txchan, &caps);
if (!ret && caps.max_burst)
else
max_burst = TX_BURST_LEVEL;
+ /*
+ * Having a Rx DMA channel serviced with higher priority than a Tx DMA
+ * channel might not be enough to provide a well balanced DMA-based
+ * SPI transfer interface. There might still be moments when the Tx DMA
+ * channel is occasionally handled faster than the Rx DMA channel.
+ * That in its turn will eventually cause the SPI Rx FIFO overflow if
+ * SPI bus speed is high enough to fill the SPI Rx FIFO in before it's
+ * cleared by the Rx DMA channel. In order to fix the problem the Tx
+ * DMA activity is intentionally slowed down by limiting the SPI Tx
+ * FIFO depth with a value twice bigger than the Tx burst length.
+ */
dws->txburst = min(max_burst, def_burst);
+ dw_writel(dws, DW_SPI_DMATDLR, dws->txburst);
+}
+
+static void dw_spi_dma_sg_burst_init(struct dw_spi *dws)
+{
+ struct dma_slave_caps tx = {0}, rx = {0};
+
+ dma_get_slave_caps(dws->txchan, &tx);
+ dma_get_slave_caps(dws->rxchan, &rx);
+
+ if (tx.max_sg_burst > 0 && rx.max_sg_burst > 0)
+ dws->dma_sg_burst = min(tx.max_sg_burst, rx.max_sg_burst);
+ else if (tx.max_sg_burst > 0)
+ dws->dma_sg_burst = tx.max_sg_burst;
+ else if (rx.max_sg_burst > 0)
+ dws->dma_sg_burst = rx.max_sg_burst;
+ else
+ dws->dma_sg_burst = 0;
}
static int dw_spi_dma_init_mfld(struct device *dev, struct dw_spi *dws)
dw_spi_dma_maxburst_init(dws);
+ dw_spi_dma_sg_burst_init(dws);
+
return 0;
free_rxchan:
dw_spi_dma_maxburst_init(dws);
+ dw_spi_dma_sg_burst_init(dws);
+
return 0;
}
dmaengine_terminate_sync(dws->rxchan);
dma_release_channel(dws->rxchan);
}
-
- dw_writel(dws, DW_SPI_DMACR, 0);
}
static irqreturn_t dw_spi_dma_transfer_handler(struct dw_spi *dws)
{
- u16 irq_status = dw_readl(dws, DW_SPI_ISR);
-
- if (!irq_status)
- return IRQ_NONE;
-
- dw_readl(dws, DW_SPI_ICR);
- spi_reset_chip(dws);
+ dw_spi_check_status(dws, false);
- dev_err(&dws->master->dev, "%s: FIFO overrun/underrun\n", __func__);
- dws->master->cur_msg->status = -EIO;
complete(&dws->dma_completion);
+
return IRQ_HANDLED;
}
return DMA_SLAVE_BUSWIDTH_UNDEFINED;
}
-static int dw_spi_dma_wait(struct dw_spi *dws, struct spi_transfer *xfer)
+static int dw_spi_dma_wait(struct dw_spi *dws, unsigned int len, u32 speed)
{
unsigned long long ms;
- ms = xfer->len * MSEC_PER_SEC * BITS_PER_BYTE;
- do_div(ms, xfer->effective_speed_hz);
+ ms = len * MSEC_PER_SEC * BITS_PER_BYTE;
+ do_div(ms, speed);
ms += ms + 200;
if (ms > UINT_MAX)
static int dw_spi_dma_wait_tx_done(struct dw_spi *dws,
struct spi_transfer *xfer)
{
- int retry = WAIT_RETRIES;
+ int retry = SPI_WAIT_RETRIES;
struct spi_delay delay;
u32 nents;
if (test_bit(RX_BUSY, &dws->dma_chan_busy))
return;
- dw_writel(dws, DW_SPI_DMACR, 0);
complete(&dws->dma_completion);
}
-static struct dma_async_tx_descriptor *
-dw_spi_dma_prepare_tx(struct dw_spi *dws, struct spi_transfer *xfer)
+static int dw_spi_dma_config_tx(struct dw_spi *dws)
{
struct dma_slave_config txconf;
- struct dma_async_tx_descriptor *txdesc;
-
- if (!xfer->tx_buf)
- return NULL;
memset(&txconf, 0, sizeof(txconf));
txconf.direction = DMA_MEM_TO_DEV;
txconf.dst_addr_width = dw_spi_dma_convert_width(dws->n_bytes);
txconf.device_fc = false;
- dmaengine_slave_config(dws->txchan, &txconf);
+ return dmaengine_slave_config(dws->txchan, &txconf);
+}
- txdesc = dmaengine_prep_slave_sg(dws->txchan,
- xfer->tx_sg.sgl,
- xfer->tx_sg.nents,
- DMA_MEM_TO_DEV,
- DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+static int dw_spi_dma_submit_tx(struct dw_spi *dws, struct scatterlist *sgl,
+ unsigned int nents)
+{
+ struct dma_async_tx_descriptor *txdesc;
+ dma_cookie_t cookie;
+ int ret;
+
+ txdesc = dmaengine_prep_slave_sg(dws->txchan, sgl, nents,
+ DMA_MEM_TO_DEV,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!txdesc)
- return NULL;
+ return -ENOMEM;
txdesc->callback = dw_spi_dma_tx_done;
txdesc->callback_param = dws;
- return txdesc;
+ cookie = dmaengine_submit(txdesc);
+ ret = dma_submit_error(cookie);
+ if (ret) {
+ dmaengine_terminate_sync(dws->txchan);
+ return ret;
+ }
+
+ set_bit(TX_BUSY, &dws->dma_chan_busy);
+
+ return 0;
}
static inline bool dw_spi_dma_rx_busy(struct dw_spi *dws)
static int dw_spi_dma_wait_rx_done(struct dw_spi *dws)
{
- int retry = WAIT_RETRIES;
+ int retry = SPI_WAIT_RETRIES;
struct spi_delay delay;
unsigned long ns, us;
u32 nents;
if (test_bit(TX_BUSY, &dws->dma_chan_busy))
return;
- dw_writel(dws, DW_SPI_DMACR, 0);
complete(&dws->dma_completion);
}
-static struct dma_async_tx_descriptor *dw_spi_dma_prepare_rx(struct dw_spi *dws,
- struct spi_transfer *xfer)
+static int dw_spi_dma_config_rx(struct dw_spi *dws)
{
struct dma_slave_config rxconf;
- struct dma_async_tx_descriptor *rxdesc;
-
- if (!xfer->rx_buf)
- return NULL;
memset(&rxconf, 0, sizeof(rxconf));
rxconf.direction = DMA_DEV_TO_MEM;
rxconf.src_addr_width = dw_spi_dma_convert_width(dws->n_bytes);
rxconf.device_fc = false;
- dmaengine_slave_config(dws->rxchan, &rxconf);
+ return dmaengine_slave_config(dws->rxchan, &rxconf);
+}
+
+static int dw_spi_dma_submit_rx(struct dw_spi *dws, struct scatterlist *sgl,
+ unsigned int nents)
+{
+ struct dma_async_tx_descriptor *rxdesc;
+ dma_cookie_t cookie;
+ int ret;
- rxdesc = dmaengine_prep_slave_sg(dws->rxchan,
- xfer->rx_sg.sgl,
- xfer->rx_sg.nents,
- DMA_DEV_TO_MEM,
- DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ rxdesc = dmaengine_prep_slave_sg(dws->rxchan, sgl, nents,
+ DMA_DEV_TO_MEM,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
if (!rxdesc)
- return NULL;
+ return -ENOMEM;
rxdesc->callback = dw_spi_dma_rx_done;
rxdesc->callback_param = dws;
- return rxdesc;
+ cookie = dmaengine_submit(rxdesc);
+ ret = dma_submit_error(cookie);
+ if (ret) {
+ dmaengine_terminate_sync(dws->rxchan);
+ return ret;
+ }
+
+ set_bit(RX_BUSY, &dws->dma_chan_busy);
+
+ return 0;
}
static int dw_spi_dma_setup(struct dw_spi *dws, struct spi_transfer *xfer)
{
- u16 imr = 0, dma_ctrl = 0;
+ u16 imr, dma_ctrl;
+ int ret;
- /*
- * Having a Rx DMA channel serviced with higher priority than a Tx DMA
- * channel might not be enough to provide a well balanced DMA-based
- * SPI transfer interface. There might still be moments when the Tx DMA
- * channel is occasionally handled faster than the Rx DMA channel.
- * That in its turn will eventually cause the SPI Rx FIFO overflow if
- * SPI bus speed is high enough to fill the SPI Rx FIFO in before it's
- * cleared by the Rx DMA channel. In order to fix the problem the Tx
- * DMA activity is intentionally slowed down by limiting the SPI Tx
- * FIFO depth with a value twice bigger than the Tx burst length
- * calculated earlier by the dw_spi_dma_maxburst_init() method.
- */
- dw_writel(dws, DW_SPI_DMARDLR, dws->rxburst - 1);
- dw_writel(dws, DW_SPI_DMATDLR, dws->txburst);
+ if (!xfer->tx_buf)
+ return -EINVAL;
+
+ /* Setup DMA channels */
+ ret = dw_spi_dma_config_tx(dws);
+ if (ret)
+ return ret;
- if (xfer->tx_buf)
- dma_ctrl |= SPI_DMA_TDMAE;
+ if (xfer->rx_buf) {
+ ret = dw_spi_dma_config_rx(dws);
+ if (ret)
+ return ret;
+ }
+
+ /* Set the DMA handshaking interface */
+ dma_ctrl = SPI_DMA_TDMAE;
if (xfer->rx_buf)
dma_ctrl |= SPI_DMA_RDMAE;
dw_writel(dws, DW_SPI_DMACR, dma_ctrl);
/* Set the interrupt mask */
- if (xfer->tx_buf)
- imr |= SPI_INT_TXOI;
+ imr = SPI_INT_TXOI;
if (xfer->rx_buf)
imr |= SPI_INT_RXUI | SPI_INT_RXOI;
spi_umask_intr(dws, imr);
return 0;
}
-static int dw_spi_dma_transfer(struct dw_spi *dws, struct spi_transfer *xfer)
+static int dw_spi_dma_transfer_all(struct dw_spi *dws,
+ struct spi_transfer *xfer)
{
- struct dma_async_tx_descriptor *txdesc, *rxdesc;
int ret;
- /* Prepare the TX dma transfer */
- txdesc = dw_spi_dma_prepare_tx(dws, xfer);
+ /* Submit the DMA Tx transfer */
+ ret = dw_spi_dma_submit_tx(dws, xfer->tx_sg.sgl, xfer->tx_sg.nents);
+ if (ret)
+ goto err_clear_dmac;
- /* Prepare the RX dma transfer */
- rxdesc = dw_spi_dma_prepare_rx(dws, xfer);
+ /* Submit the DMA Rx transfer if required */
+ if (xfer->rx_buf) {
+ ret = dw_spi_dma_submit_rx(dws, xfer->rx_sg.sgl,
+ xfer->rx_sg.nents);
+ if (ret)
+ goto err_clear_dmac;
- /* rx must be started before tx due to spi instinct */
- if (rxdesc) {
- set_bit(RX_BUSY, &dws->dma_chan_busy);
- dmaengine_submit(rxdesc);
+ /* rx must be started before tx due to spi instinct */
dma_async_issue_pending(dws->rxchan);
}
- if (txdesc) {
- set_bit(TX_BUSY, &dws->dma_chan_busy);
- dmaengine_submit(txdesc);
+ dma_async_issue_pending(dws->txchan);
+
+ ret = dw_spi_dma_wait(dws, xfer->len, xfer->effective_speed_hz);
+
+err_clear_dmac:
+ dw_writel(dws, DW_SPI_DMACR, 0);
+
+ return ret;
+}
+
+/*
+ * In case if at least one of the requested DMA channels doesn't support the
+ * hardware accelerated SG list entries traverse, the DMA driver will most
+ * likely work that around by performing the IRQ-based SG list entries
+ * resubmission. That might and will cause a problem if the DMA Tx channel is
+ * recharged and re-executed before the Rx DMA channel. Due to
+ * non-deterministic IRQ-handler execution latency the DMA Tx channel will
+ * start pushing data to the SPI bus before the Rx DMA channel is even
+ * reinitialized with the next inbound SG list entry. By doing so the DMA Tx
+ * channel will implicitly start filling the DW APB SSI Rx FIFO up, which while
+ * the DMA Rx channel being recharged and re-executed will eventually be
+ * overflown.
+ *
+ * In order to solve the problem we have to feed the DMA engine with SG list
+ * entries one-by-one. It shall keep the DW APB SSI Tx and Rx FIFOs
+ * synchronized and prevent the Rx FIFO overflow. Since in general the tx_sg
+ * and rx_sg lists may have different number of entries of different lengths
+ * (though total length should match) let's virtually split the SG-lists to the
+ * set of DMA transfers, which length is a minimum of the ordered SG-entries
+ * lengths. An ASCII-sketch of the implemented algo is following:
+ * xfer->len
+ * |___________|
+ * tx_sg list: |___|____|__|
+ * rx_sg list: |_|____|____|
+ * DMA transfers: |_|_|__|_|__|
+ *
+ * Note in order to have this workaround solving the denoted problem the DMA
+ * engine driver should properly initialize the max_sg_burst capability and set
+ * the DMA device max segment size parameter with maximum data block size the
+ * DMA engine supports.
+ */
+
+static int dw_spi_dma_transfer_one(struct dw_spi *dws,
+ struct spi_transfer *xfer)
+{
+ struct scatterlist *tx_sg = NULL, *rx_sg = NULL, tx_tmp, rx_tmp;
+ unsigned int tx_len = 0, rx_len = 0;
+ unsigned int base, len;
+ int ret;
+
+ sg_init_table(&tx_tmp, 1);
+ sg_init_table(&rx_tmp, 1);
+
+ for (base = 0, len = 0; base < xfer->len; base += len) {
+ /* Fetch next Tx DMA data chunk */
+ if (!tx_len) {
+ tx_sg = !tx_sg ? &xfer->tx_sg.sgl[0] : sg_next(tx_sg);
+ sg_dma_address(&tx_tmp) = sg_dma_address(tx_sg);
+ tx_len = sg_dma_len(tx_sg);
+ }
+
+ /* Fetch next Rx DMA data chunk */
+ if (!rx_len) {
+ rx_sg = !rx_sg ? &xfer->rx_sg.sgl[0] : sg_next(rx_sg);
+ sg_dma_address(&rx_tmp) = sg_dma_address(rx_sg);
+ rx_len = sg_dma_len(rx_sg);
+ }
+
+ len = min(tx_len, rx_len);
+
+ sg_dma_len(&tx_tmp) = len;
+ sg_dma_len(&rx_tmp) = len;
+
+ /* Submit DMA Tx transfer */
+ ret = dw_spi_dma_submit_tx(dws, &tx_tmp, 1);
+ if (ret)
+ break;
+
+ /* Submit DMA Rx transfer */
+ ret = dw_spi_dma_submit_rx(dws, &rx_tmp, 1);
+ if (ret)
+ break;
+
+ /* Rx must be started before Tx due to SPI instinct */
+ dma_async_issue_pending(dws->rxchan);
+
dma_async_issue_pending(dws->txchan);
+
+ /*
+ * Here we only need to wait for the DMA transfer to be
+ * finished since SPI controller is kept enabled during the
+ * procedure this loop implements and there is no risk to lose
+ * data left in the Tx/Rx FIFOs.
+ */
+ ret = dw_spi_dma_wait(dws, len, xfer->effective_speed_hz);
+ if (ret)
+ break;
+
+ reinit_completion(&dws->dma_completion);
+
+ sg_dma_address(&tx_tmp) += len;
+ sg_dma_address(&rx_tmp) += len;
+ tx_len -= len;
+ rx_len -= len;
}
- ret = dw_spi_dma_wait(dws, xfer);
+ dw_writel(dws, DW_SPI_DMACR, 0);
+
+ return ret;
+}
+
+static int dw_spi_dma_transfer(struct dw_spi *dws, struct spi_transfer *xfer)
+{
+ unsigned int nents;
+ int ret;
+
+ nents = max(xfer->tx_sg.nents, xfer->rx_sg.nents);
+
+ /*
+ * Execute normal DMA-based transfer (which submits the Rx and Tx SG
+ * lists directly to the DMA engine at once) if either full hardware
+ * accelerated SG list traverse is supported by both channels, or the
+ * Tx-only SPI transfer is requested, or the DMA engine is capable to
+ * handle both SG lists on hardware accelerated basis.
+ */
+ if (!dws->dma_sg_burst || !xfer->rx_buf || nents <= dws->dma_sg_burst)
+ ret = dw_spi_dma_transfer_all(dws, xfer);
+ else
+ ret = dw_spi_dma_transfer_one(dws, xfer);
if (ret)
return ret;
- if (txdesc && dws->master->cur_msg->status == -EINPROGRESS) {
+ if (dws->master->cur_msg->status == -EINPROGRESS) {
ret = dw_spi_dma_wait_tx_done(dws, xfer);
if (ret)
return ret;
}
- if (rxdesc && dws->master->cur_msg->status == -EINPROGRESS)
+ if (xfer->rx_buf && dws->master->cur_msg->status == -EINPROGRESS)
ret = dw_spi_dma_wait_rx_done(dws);
return ret;
dmaengine_terminate_sync(dws->rxchan);
clear_bit(RX_BUSY, &dws->dma_chan_busy);
}
-
- dw_writel(dws, DW_SPI_DMACR, 0);
}
static const struct dw_spi_dma_ops dw_spi_dma_mfld_ops = {
#define MSCC_SPI_MST_SW_MODE_SW_PIN_CTRL_MODE BIT(13)
#define MSCC_SPI_MST_SW_MODE_SW_SPI_CS(x) (x << 5)
-/*
- * For Keem Bay, CTRLR0[31] is used to select controller mode.
- * 0: SSI is slave
- * 1: SSI is master
- */
-#define KEEMBAY_CTRLR0_SSIC_IS_MST BIT(31)
+#define SPARX5_FORCE_ENA 0xa4
+#define SPARX5_FORCE_VAL 0xa8
struct dw_spi_mscc {
struct regmap *syscon;
- void __iomem *spi_mst;
+ void __iomem *spi_mst; /* Not sparx5 */
};
/*
dwsmmio->dws.set_cs = dw_spi_mscc_set_cs;
dwsmmio->priv = dwsmscc;
- /* Register hook to configure CTRLR0 */
- dwsmmio->dws.update_cr0 = dw_spi_update_cr0;
-
return 0;
}
JAGUAR2_IF_SI_OWNER_OFFSET);
}
+/*
+ * The Designware SPI controller (referred to as master in the
+ * documentation) automatically deasserts chip select when the tx fifo
+ * is empty. The chip selects then needs to be driven by a CS override
+ * register. enable is an active low signal.
+ */
+static void dw_spi_sparx5_set_cs(struct spi_device *spi, bool enable)
+{
+ struct dw_spi *dws = spi_master_get_devdata(spi->master);
+ struct dw_spi_mmio *dwsmmio = container_of(dws, struct dw_spi_mmio, dws);
+ struct dw_spi_mscc *dwsmscc = dwsmmio->priv;
+ u8 cs = spi->chip_select;
+
+ if (!enable) {
+ /* CS override drive enable */
+ regmap_write(dwsmscc->syscon, SPARX5_FORCE_ENA, 1);
+ /* Now set CSx enabled */
+ regmap_write(dwsmscc->syscon, SPARX5_FORCE_VAL, ~BIT(cs));
+ /* Allow settle */
+ usleep_range(1, 5);
+ } else {
+ /* CS value */
+ regmap_write(dwsmscc->syscon, SPARX5_FORCE_VAL, ~0);
+ /* Allow settle */
+ usleep_range(1, 5);
+ /* CS override drive disable */
+ regmap_write(dwsmscc->syscon, SPARX5_FORCE_ENA, 0);
+ }
+
+ dw_spi_set_cs(spi, enable);
+}
+
+static int dw_spi_mscc_sparx5_init(struct platform_device *pdev,
+ struct dw_spi_mmio *dwsmmio)
+{
+ const char *syscon_name = "microchip,sparx5-cpu-syscon";
+ struct device *dev = &pdev->dev;
+ struct dw_spi_mscc *dwsmscc;
+
+ if (!IS_ENABLED(CONFIG_SPI_MUX)) {
+ dev_err(dev, "This driver needs CONFIG_SPI_MUX\n");
+ return -EOPNOTSUPP;
+ }
+
+ dwsmscc = devm_kzalloc(dev, sizeof(*dwsmscc), GFP_KERNEL);
+ if (!dwsmscc)
+ return -ENOMEM;
+
+ dwsmscc->syscon =
+ syscon_regmap_lookup_by_compatible(syscon_name);
+ if (IS_ERR(dwsmscc->syscon)) {
+ dev_err(dev, "No syscon map %s\n", syscon_name);
+ return PTR_ERR(dwsmscc->syscon);
+ }
+
+ dwsmmio->dws.set_cs = dw_spi_sparx5_set_cs;
+ dwsmmio->priv = dwsmscc;
+
+ return 0;
+}
+
static int dw_spi_alpine_init(struct platform_device *pdev,
struct dw_spi_mmio *dwsmmio)
{
- dwsmmio->dws.cs_override = 1;
-
- /* Register hook to configure CTRLR0 */
- dwsmmio->dws.update_cr0 = dw_spi_update_cr0;
+ dwsmmio->dws.caps = DW_SPI_CAP_CS_OVERRIDE;
return 0;
}
static int dw_spi_dw_apb_init(struct platform_device *pdev,
struct dw_spi_mmio *dwsmmio)
{
- /* Register hook to configure CTRLR0 */
- dwsmmio->dws.update_cr0 = dw_spi_update_cr0;
-
dw_spi_dma_setup_generic(&dwsmmio->dws);
return 0;
static int dw_spi_dwc_ssi_init(struct platform_device *pdev,
struct dw_spi_mmio *dwsmmio)
{
- /* Register hook to configure CTRLR0 */
- dwsmmio->dws.update_cr0 = dw_spi_update_cr0_v1_01a;
+ dwsmmio->dws.caps = DW_SPI_CAP_DWC_SSI;
dw_spi_dma_setup_generic(&dwsmmio->dws);
return 0;
}
-static u32 dw_spi_update_cr0_keembay(struct spi_controller *master,
- struct spi_device *spi,
- struct spi_transfer *transfer)
-{
- u32 cr0 = dw_spi_update_cr0_v1_01a(master, spi, transfer);
-
- return cr0 | KEEMBAY_CTRLR0_SSIC_IS_MST;
-}
-
static int dw_spi_keembay_init(struct platform_device *pdev,
struct dw_spi_mmio *dwsmmio)
{
- /* Register hook to configure CTRLR0 */
- dwsmmio->dws.update_cr0 = dw_spi_update_cr0_keembay;
+ dwsmmio->dws.caps = DW_SPI_CAP_KEEMBAY_MST | DW_SPI_CAP_DWC_SSI;
return 0;
}
{ .compatible = "renesas,rzn1-spi", .data = dw_spi_dw_apb_init},
{ .compatible = "snps,dwc-ssi-1.01a", .data = dw_spi_dwc_ssi_init},
{ .compatible = "intel,keembay-ssi", .data = dw_spi_keembay_init},
+ { .compatible = "microchip,sparx5-spi", dw_spi_mscc_sparx5_init},
{ /* end of table */}
};
MODULE_DEVICE_TABLE(of, dw_spi_mmio_of_match);
iounmap(clk_reg);
- /* Register hook to configure CTRLR0 */
- dws->update_cr0 = dw_spi_update_cr0;
-
dw_spi_dma_setup_mfld(dws);
return 0;
static int spi_generic_init(struct dw_spi *dws)
{
- /* Register hook to configure CTRLR0 */
- dws->update_cr0 = dw_spi_update_cr0;
-
dw_spi_dma_setup_generic(dws);
return 0;
if (desc->setup) {
ret = desc->setup(dws);
if (ret)
- return ret;
+ goto err_free_irq_vectors;
}
} else {
- pci_free_irq_vectors(pdev);
- return -ENODEV;
+ ret = -ENODEV;
+ goto err_free_irq_vectors;
}
ret = dw_spi_add_host(&pdev->dev, dws);
- if (ret) {
- pci_free_irq_vectors(pdev);
- return ret;
- }
+ if (ret)
+ goto err_free_irq_vectors;
/* PCI hook and SPI hook use the same drv data */
pci_set_drvdata(pdev, dws);
pm_runtime_allow(&pdev->dev);
return 0;
+
+err_free_irq_vectors:
+ pci_free_irq_vectors(pdev);
+ return ret;
}
static void spi_pci_remove(struct pci_dev *pdev)
#ifndef DW_SPI_HEADER_H
#define DW_SPI_HEADER_H
+#include <linux/bits.h>
#include <linux/completion.h>
#include <linux/debugfs.h>
#include <linux/irqreturn.h>
#include <linux/io.h>
#include <linux/scatterlist.h>
+#include <linux/spi/spi-mem.h>
/* Register offsets */
#define DW_SPI_CTRLR0 0x00
#define DW_SPI_IDR 0x58
#define DW_SPI_VERSION 0x5c
#define DW_SPI_DR 0x60
+#define DW_SPI_RX_SAMPLE_DLY 0xf0
#define DW_SPI_CS_OVERRIDE 0xf4
/* Bit fields in CTRLR0 */
#define DWC_SSI_CTRLR0_FRF_OFFSET 6
#define DWC_SSI_CTRLR0_DFS_OFFSET 0
+/*
+ * For Keem Bay, CTRLR0[31] is used to select controller mode.
+ * 0: SSI is slave
+ * 1: SSI is master
+ */
+#define DWC_SSI_CTRLR0_KEEMBAY_MST BIT(31)
+
+/* Bit fields in CTRLR1 */
+#define SPI_NDF_MASK GENMASK(15, 0)
+
/* Bit fields in SR, 7 bits */
#define SR_MASK 0x7f /* cover 7 bits */
#define SR_BUSY (1 << 0)
#define SPI_DMA_RDMAE (1 << 0)
#define SPI_DMA_TDMAE (1 << 1)
-/* TX RX interrupt level threshold, max can be 256 */
-#define SPI_INT_THRESHOLD 32
+#define SPI_WAIT_RETRIES 5
+#define SPI_BUF_SIZE \
+ (sizeof_field(struct spi_mem_op, cmd.opcode) + \
+ sizeof_field(struct spi_mem_op, addr.val) + 256)
+#define SPI_GET_BYTE(_val, _idx) \
+ ((_val) >> (BITS_PER_BYTE * (_idx)) & 0xff)
enum dw_ssi_type {
SSI_MOTO_SPI = 0,
SSI_NS_MICROWIRE,
};
+/* DW SPI capabilities */
+#define DW_SPI_CAP_CS_OVERRIDE BIT(0)
+#define DW_SPI_CAP_KEEMBAY_MST BIT(1)
+#define DW_SPI_CAP_DWC_SSI BIT(2)
+
+/* Slave spi_transfer/spi_mem_op related */
+struct dw_spi_cfg {
+ u8 tmode;
+ u8 dfs;
+ u32 ndf;
+ u32 freq;
+};
+
struct dw_spi;
struct dw_spi_dma_ops {
int (*dma_init)(struct device *dev, struct dw_spi *dws);
struct dw_spi {
struct spi_controller *master;
- enum dw_ssi_type type;
void __iomem *regs;
unsigned long paddr;
int irq;
u32 fifo_len; /* depth of the FIFO buffer */
+ u32 max_mem_freq; /* max mem-ops bus freq */
u32 max_freq; /* max bus freq supported */
- int cs_override;
+ u32 caps; /* DW SPI capabilities */
+
u32 reg_io_width; /* DR I/O width in bytes */
u16 bus_num;
u16 num_cs; /* supported slave numbers */
void (*set_cs)(struct spi_device *spi, bool enable);
- u32 (*update_cr0)(struct spi_controller *master, struct spi_device *spi,
- struct spi_transfer *transfer);
/* Current message transfer state info */
- size_t len;
void *tx;
- void *tx_end;
- spinlock_t buf_lock;
+ unsigned int tx_len;
void *rx;
- void *rx_end;
+ unsigned int rx_len;
+ u8 buf[SPI_BUF_SIZE];
int dma_mapped;
u8 n_bytes; /* current is a 1/2 bytes op */
irqreturn_t (*transfer_handler)(struct dw_spi *dws);
u32 current_freq; /* frequency in hz */
+ u32 cur_rx_sample_dly;
+ u32 def_rx_sample_dly_ns;
+
+ /* Custom memory operations */
+ struct spi_controller_mem_ops mem_ops;
/* DMA info */
struct dma_chan *txchan;
u32 txburst;
struct dma_chan *rxchan;
u32 rxburst;
+ u32 dma_sg_burst;
unsigned long dma_chan_busy;
dma_addr_t dma_addr; /* phy address of the Data register */
const struct dw_spi_dma_ops *dma_ops;
return __raw_readl(dws->regs + offset);
}
-static inline u16 dw_readw(struct dw_spi *dws, u32 offset)
-{
- return __raw_readw(dws->regs + offset);
-}
-
static inline void dw_writel(struct dw_spi *dws, u32 offset, u32 val)
{
__raw_writel(val, dws->regs + offset);
}
-static inline void dw_writew(struct dw_spi *dws, u32 offset, u16 val)
-{
- __raw_writew(val, dws->regs + offset);
-}
-
static inline u32 dw_read_io_reg(struct dw_spi *dws, u32 offset)
{
switch (dws->reg_io_width) {
case 2:
- return dw_readw(dws, offset);
+ return readw_relaxed(dws->regs + offset);
case 4:
default:
- return dw_readl(dws, offset);
+ return readl_relaxed(dws->regs + offset);
}
}
{
switch (dws->reg_io_width) {
case 2:
- dw_writew(dws, offset, val);
+ writew_relaxed(val, dws->regs + offset);
break;
case 4:
default:
- dw_writel(dws, offset, val);
+ writel_relaxed(val, dws->regs + offset);
break;
}
}
}
/*
- * This does disable the SPI controller, interrupts, and re-enable the
- * controller back. Transmit and receive FIFO buffers are cleared when the
- * device is disabled.
+ * This disables the SPI controller, interrupts, clears the interrupts status
+ * and CS, then re-enables the controller back. Transmit and receive FIFO
+ * buffers are cleared when the device is disabled.
*/
static inline void spi_reset_chip(struct dw_spi *dws)
{
spi_enable_chip(dws, 0);
spi_mask_intr(dws, 0xff);
+ dw_readl(dws, DW_SPI_ICR);
+ dw_writel(dws, DW_SPI_SER, 0);
spi_enable_chip(dws, 1);
}
}
extern void dw_spi_set_cs(struct spi_device *spi, bool enable);
+extern void dw_spi_update_config(struct dw_spi *dws, struct spi_device *spi,
+ struct dw_spi_cfg *cfg);
+extern int dw_spi_check_status(struct dw_spi *dws, bool raw);
extern int dw_spi_add_host(struct device *dev, struct dw_spi *dws);
extern void dw_spi_remove_host(struct dw_spi *dws);
extern int dw_spi_suspend_host(struct dw_spi *dws);
extern int dw_spi_resume_host(struct dw_spi *dws);
-extern u32 dw_spi_update_cr0(struct spi_controller *master,
- struct spi_device *spi,
- struct spi_transfer *transfer);
-extern u32 dw_spi_update_cr0_v1_01a(struct spi_controller *master,
- struct spi_device *spi,
- struct spi_transfer *transfer);
#ifdef CONFIG_SPI_DW_DMA
#define FSI_ENGID_SPI 0x23
#define FSI_MBOX_ROOT_CTRL_8 0x2860
+#define FSI_MBOX_ROOT_CTRL_8_SPI_MUX 0xf0000000
#define FSI2SPI_DATA0 0x00
#define FSI2SPI_DATA1 0x04
#define SPI_FSI_BASE 0x70000
#define SPI_FSI_INIT_TIMEOUT_MS 1000
-#define SPI_FSI_MAX_TRANSFER_SIZE 2048
+#define SPI_FSI_MAX_XFR_SIZE 2048
+#define SPI_FSI_MAX_XFR_SIZE_RESTRICTED 32
#define SPI_FSI_ERROR 0x0
#define SPI_FSI_COUNTER_CFG 0x1
#define SPI_FSI_COUNTER_CFG_LOOPS(x) (((u64)(x) & 0xffULL) << 32)
+#define SPI_FSI_COUNTER_CFG_N2_RX BIT_ULL(8)
+#define SPI_FSI_COUNTER_CFG_N2_TX BIT_ULL(9)
+#define SPI_FSI_COUNTER_CFG_N2_IMPLICIT BIT_ULL(10)
+#define SPI_FSI_COUNTER_CFG_N2_RELOAD BIT_ULL(11)
#define SPI_FSI_CFG1 0x2
#define SPI_FSI_CLOCK_CFG 0x3
#define SPI_FSI_CLOCK_CFG_MM_ENABLE BIT_ULL(32)
#define SPI_FSI_STATUS_RDR_OVERRUN BIT_ULL(62)
#define SPI_FSI_STATUS_RDR_FULL BIT_ULL(63)
#define SPI_FSI_STATUS_ANY_ERROR \
- (SPI_FSI_STATUS_ERROR | SPI_FSI_STATUS_TDR_UNDERRUN | \
+ (SPI_FSI_STATUS_ERROR | \
SPI_FSI_STATUS_TDR_OVERRUN | SPI_FSI_STATUS_RDR_UNDERRUN | \
SPI_FSI_STATUS_RDR_OVERRUN)
#define SPI_FSI_PORT_CTRL 0x9
struct device *dev; /* SPI controller device */
struct fsi_device *fsi; /* FSI2SPI CFAM engine device */
u32 base;
+ size_t max_xfr_size;
+ bool restricted;
};
struct fsi_spi_sequence {
u64 data;
};
+static int fsi_spi_check_mux(struct fsi_device *fsi, struct device *dev)
+{
+ int rc;
+ u32 root_ctrl_8;
+ __be32 root_ctrl_8_be;
+
+ rc = fsi_slave_read(fsi->slave, FSI_MBOX_ROOT_CTRL_8, &root_ctrl_8_be,
+ sizeof(root_ctrl_8_be));
+ if (rc)
+ return rc;
+
+ root_ctrl_8 = be32_to_cpu(root_ctrl_8_be);
+ dev_dbg(dev, "Root control register 8: %08x\n", root_ctrl_8);
+ if ((root_ctrl_8 & FSI_MBOX_ROOT_CTRL_8_SPI_MUX) ==
+ FSI_MBOX_ROOT_CTRL_8_SPI_MUX)
+ return 0;
+
+ return -ENOLINK;
+}
+
static int fsi_spi_check_status(struct fsi_spi *ctx)
{
int rc;
if (rc)
return rc;
- return fsi_spi_write_reg(ctx, SPI_FSI_CLOCK_CFG,
- SPI_FSI_CLOCK_CFG_RESET2);
+ rc = fsi_spi_write_reg(ctx, SPI_FSI_CLOCK_CFG,
+ SPI_FSI_CLOCK_CFG_RESET2);
+ if (rc)
+ return rc;
+
+ return fsi_spi_write_reg(ctx, SPI_FSI_STATUS, 0ULL);
}
static int fsi_spi_sequence_add(struct fsi_spi_sequence *seq, u8 val)
/*
* Add the next byte of instruction to the 8-byte sequence register.
* Then decrement the counter so that the next instruction will go in
- * the right place. Return the number of "slots" left in the sequence
- * register.
+ * the right place. Return the index of the slot we just filled in the
+ * sequence register.
*/
seq->data |= (u64)val << seq->bit;
seq->bit -= 8;
struct fsi_spi_sequence *seq,
struct spi_transfer *transfer)
{
+ bool docfg = false;
int loops;
int idx;
int rc;
+ u8 val = 0;
u8 len = min(transfer->len, 8U);
u8 rem = transfer->len % len;
+ u64 cfg = 0ULL;
loops = transfer->len / len;
if (transfer->tx_buf) {
- idx = fsi_spi_sequence_add(seq,
- SPI_FSI_SEQUENCE_SHIFT_OUT(len));
+ val = SPI_FSI_SEQUENCE_SHIFT_OUT(len);
+ idx = fsi_spi_sequence_add(seq, val);
+
if (rem)
rem = SPI_FSI_SEQUENCE_SHIFT_OUT(rem);
} else if (transfer->rx_buf) {
- idx = fsi_spi_sequence_add(seq,
- SPI_FSI_SEQUENCE_SHIFT_IN(len));
+ val = SPI_FSI_SEQUENCE_SHIFT_IN(len);
+ idx = fsi_spi_sequence_add(seq, val);
+
if (rem)
rem = SPI_FSI_SEQUENCE_SHIFT_IN(rem);
} else {
return -EINVAL;
}
+ if (ctx->restricted) {
+ const int eidx = rem ? 5 : 6;
+
+ while (loops > 1 && idx <= eidx) {
+ idx = fsi_spi_sequence_add(seq, val);
+ loops--;
+ docfg = true;
+ }
+
+ if (loops > 1) {
+ dev_warn(ctx->dev, "No sequencer slots; aborting.\n");
+ return -EINVAL;
+ }
+ }
+
if (loops > 1) {
fsi_spi_sequence_add(seq, SPI_FSI_SEQUENCE_BRANCH(idx));
+ docfg = true;
+ }
- if (rem)
- fsi_spi_sequence_add(seq, rem);
+ if (docfg) {
+ cfg = SPI_FSI_COUNTER_CFG_LOOPS(loops - 1);
+ if (transfer->rx_buf)
+ cfg |= SPI_FSI_COUNTER_CFG_N2_RX |
+ SPI_FSI_COUNTER_CFG_N2_TX |
+ SPI_FSI_COUNTER_CFG_N2_IMPLICIT |
+ SPI_FSI_COUNTER_CFG_N2_RELOAD;
- rc = fsi_spi_write_reg(ctx, SPI_FSI_COUNTER_CFG,
- SPI_FSI_COUNTER_CFG_LOOPS(loops - 1));
+ rc = fsi_spi_write_reg(ctx, SPI_FSI_COUNTER_CFG, cfg);
if (rc)
return rc;
+ } else {
+ fsi_spi_write_reg(ctx, SPI_FSI_COUNTER_CFG, 0ULL);
}
+ if (rem)
+ fsi_spi_sequence_add(seq, rem);
+
return 0;
}
{
int rc = 0;
u64 status = 0ULL;
+ u64 cfg = 0ULL;
if (transfer->tx_buf) {
int nb;
u64 in = 0ULL;
u8 *rx = transfer->rx_buf;
+ rc = fsi_spi_read_reg(ctx, SPI_FSI_COUNTER_CFG, &cfg);
+ if (rc)
+ return rc;
+
+ if (cfg & SPI_FSI_COUNTER_CFG_N2_IMPLICIT) {
+ rc = fsi_spi_write_reg(ctx, SPI_FSI_DATA_TX, 0);
+ if (rc)
+ return rc;
+ }
+
while (transfer->len > recv) {
do {
rc = fsi_spi_read_reg(ctx, SPI_FSI_STATUS,
u64 status = 0ULL;
u64 wanted_clock_cfg = SPI_FSI_CLOCK_CFG_ECC_DISABLE |
SPI_FSI_CLOCK_CFG_SCK_NO_DEL |
- FIELD_PREP(SPI_FSI_CLOCK_CFG_SCK_DIV, 4);
+ FIELD_PREP(SPI_FSI_CLOCK_CFG_SCK_DIV, 19);
end = jiffies + msecs_to_jiffies(SPI_FSI_INIT_TIMEOUT_MS);
do {
static int fsi_spi_transfer_one_message(struct spi_controller *ctlr,
struct spi_message *mesg)
{
- int rc = 0;
+ int rc;
u8 seq_slave = SPI_FSI_SEQUENCE_SEL_SLAVE(mesg->spi->chip_select + 1);
struct spi_transfer *transfer;
struct fsi_spi *ctx = spi_controller_get_devdata(ctlr);
+ rc = fsi_spi_check_mux(ctx->fsi, ctx->dev);
+ if (rc)
+ return rc;
+
list_for_each_entry(transfer, &mesg->transfers, transfer_list) {
struct fsi_spi_sequence seq;
struct spi_transfer *next = NULL;
/* Sequencer must do shift out (tx) first. */
if (!transfer->tx_buf ||
- transfer->len > SPI_FSI_MAX_TRANSFER_SIZE) {
+ transfer->len > (ctx->max_xfr_size + 8)) {
rc = -EINVAL;
goto error;
}
/* Sequencer can only do shift in (rx) after tx. */
if (next->rx_buf) {
- if (next->len > SPI_FSI_MAX_TRANSFER_SIZE) {
+ if (next->len > ctx->max_xfr_size) {
rc = -EINVAL;
goto error;
}
static size_t fsi_spi_max_transfer_size(struct spi_device *spi)
{
- return SPI_FSI_MAX_TRANSFER_SIZE;
+ struct fsi_spi *ctx = spi_controller_get_devdata(spi->controller);
+
+ return ctx->max_xfr_size;
}
static int fsi_spi_probe(struct device *dev)
{
int rc;
- u32 root_ctrl_8;
struct device_node *np;
int num_controllers_registered = 0;
struct fsi_device *fsi = to_fsi_dev(dev);
- /*
- * Check the SPI mux before attempting to probe. If the mux isn't set
- * then the SPI controllers can't access their slave devices.
- */
- rc = fsi_slave_read(fsi->slave, FSI_MBOX_ROOT_CTRL_8, &root_ctrl_8,
- sizeof(root_ctrl_8));
+ rc = fsi_spi_check_mux(fsi, dev);
if (rc)
- return rc;
-
- if (!root_ctrl_8) {
- dev_dbg(dev, "SPI mux not set, aborting probe.\n");
return -ENODEV;
- }
for_each_available_child_of_node(dev->of_node, np) {
u32 base;
ctx->fsi = fsi;
ctx->base = base + SPI_FSI_BASE;
+ if (of_device_is_compatible(np, "ibm,fsi2spi-restricted")) {
+ ctx->restricted = true;
+ ctx->max_xfr_size = SPI_FSI_MAX_XFR_SIZE_RESTRICTED;
+ } else {
+ ctx->restricted = false;
+ ctx->max_xfr_size = SPI_FSI_MAX_XFR_SIZE;
+ }
+
rc = devm_spi_register_controller(dev, ctlr);
if (rc)
spi_controller_put(ctlr);
#define SPI_SR 0x2c
#define SPI_SR_TCFQF BIT(31)
-#define SPI_SR_EOQF BIT(28)
#define SPI_SR_TFUF BIT(27)
#define SPI_SR_TFFF BIT(25)
#define SPI_SR_CMDTCF BIT(23)
#define SPI_SR_TFIWF BIT(18)
#define SPI_SR_RFDF BIT(17)
#define SPI_SR_CMDFFF BIT(16)
-#define SPI_SR_CLEAR (SPI_SR_TCFQF | SPI_SR_EOQF | \
+#define SPI_SR_CLEAR (SPI_SR_TCFQF | \
SPI_SR_TFUF | SPI_SR_TFFF | \
SPI_SR_CMDTCF | SPI_SR_SPEF | \
SPI_SR_RFOF | SPI_SR_TFIWF | \
#define SPI_RSER 0x30
#define SPI_RSER_TCFQE BIT(31)
-#define SPI_RSER_EOQFE BIT(28)
#define SPI_RSER_CMDTCFE BIT(23)
#define SPI_PUSHR 0x34
};
enum dspi_trans_mode {
- DSPI_EOQ_MODE = 0,
DSPI_XSPI_MODE,
DSPI_DMA_MODE,
};
.fifo_size = 16,
},
[LS2080A] = {
- .trans_mode = DSPI_DMA_MODE,
+ .trans_mode = DSPI_XSPI_MODE,
.max_clock_factor = 8,
.fifo_size = 4,
},
[LS2085A] = {
- .trans_mode = DSPI_DMA_MODE,
+ .trans_mode = DSPI_XSPI_MODE,
.max_clock_factor = 8,
.fifo_size = 4,
},
[LX2160A] = {
- .trans_mode = DSPI_DMA_MODE,
+ .trans_mode = DSPI_XSPI_MODE,
.max_clock_factor = 8,
.fifo_size = 4,
},
[MCF5441X] = {
- .trans_mode = DSPI_EOQ_MODE,
+ .trans_mode = DSPI_DMA_MODE,
.max_clock_factor = 8,
.fifo_size = 16,
},
}
}
-static void dspi_pushr_write(struct fsl_dspi *dspi)
-{
- regmap_write(dspi->regmap, SPI_PUSHR, dspi_pop_tx_pushr(dspi));
-}
-
static void dspi_pushr_cmd_write(struct fsl_dspi *dspi, u16 cmd)
{
/*
}
}
-static void dspi_eoq_fifo_write(struct fsl_dspi *dspi, int num_words)
-{
- u16 xfer_cmd = dspi->tx_cmd;
-
- /* Fill TX FIFO with as many transfers as possible */
- while (num_words--) {
- dspi->tx_cmd = xfer_cmd;
- /* Request EOQF for last transfer in FIFO */
- if (num_words == 0)
- dspi->tx_cmd |= SPI_PUSHR_CMD_EOQ;
- /* Write combined TX FIFO and CMD FIFO entry */
- dspi_pushr_write(dspi);
- }
-}
-
static u32 dspi_popr_read(struct fsl_dspi *dspi)
{
u32 rxdata = 0;
dspi->oper_word_size = DIV_ROUND_UP(dspi->oper_bits_per_word, 8);
/*
- * Update CTAR here (code is common for EOQ, XSPI and DMA modes).
+ * Update CTAR here (code is common for XSPI and DMA modes).
* We will update CTARE in the portion specific to XSPI, when we
* also know the preload value (DTCP).
*/
spi_take_timestamp_pre(dspi->ctlr, xfer, dspi->progress, !dspi->irq);
- if (dspi->devtype_data->trans_mode == DSPI_EOQ_MODE)
- dspi_eoq_fifo_write(dspi, num_words);
- else
- dspi_xspi_fifo_write(dspi, num_words);
+ dspi_xspi_fifo_write(dspi, num_words);
/*
* Everything after this point is in a potential race with the next
* interrupt, so we must never use dspi->words_in_flight again since it
regmap_read(dspi->regmap, SPI_SR, &spi_sr);
regmap_write(dspi->regmap, SPI_SR, spi_sr);
- if (spi_sr & (SPI_SR_EOQF | SPI_SR_CMDTCF))
+ if (spi_sr & SPI_SR_CMDTCF)
break;
} while (--tries);
regmap_read(dspi->regmap, SPI_SR, &spi_sr);
regmap_write(dspi->regmap, SPI_SR, spi_sr);
- if (!(spi_sr & (SPI_SR_EOQF | SPI_SR_CMDTCF)))
+ if (!(spi_sr & SPI_SR_CMDTCF))
return IRQ_NONE;
if (dspi_rxtx(dspi) == 0)
regmap_write(dspi->regmap, SPI_SR, SPI_SR_CLEAR);
switch (dspi->devtype_data->trans_mode) {
- case DSPI_EOQ_MODE:
- regmap_write(dspi->regmap, SPI_RSER, SPI_RSER_EOQFE);
- break;
case DSPI_XSPI_MODE:
regmap_write(dspi->regmap, SPI_RSER, SPI_RSER_CMDTCFE);
break;
return 0;
}
-/*
- * EOQ mode will inevitably deassert its PCS signal on last word in a queue
- * (hardware limitation), so we need to inform the spi_device that larger
- * buffers than the FIFO size are going to have the chip select randomly
- * toggling, so it has a chance to adapt its message sizes.
- */
-static size_t dspi_max_message_size(struct spi_device *spi)
-{
- struct fsl_dspi *dspi = spi_controller_get_devdata(spi->controller);
-
- if (dspi->devtype_data->trans_mode == DSPI_EOQ_MODE)
- return dspi->devtype_data->fifo_size;
-
- return SIZE_MAX;
-}
-
static int dspi_probe(struct platform_device *pdev)
{
struct device_node *np = pdev->dev.of_node;
void __iomem *base;
bool big_endian;
- ctlr = spi_alloc_master(&pdev->dev, sizeof(struct fsl_dspi));
+ dspi = devm_kzalloc(&pdev->dev, sizeof(*dspi), GFP_KERNEL);
+ if (!dspi)
+ return -ENOMEM;
+
+ ctlr = spi_alloc_master(&pdev->dev, 0);
if (!ctlr)
return -ENOMEM;
- dspi = spi_controller_get_devdata(ctlr);
+ spi_controller_set_devdata(ctlr, dspi);
+ platform_set_drvdata(pdev, dspi);
+
dspi->pdev = pdev;
dspi->ctlr = ctlr;
ctlr->setup = dspi_setup;
ctlr->transfer_one_message = dspi_transfer_one_message;
- ctlr->max_message_size = dspi_max_message_size;
ctlr->dev.of_node = pdev->dev.of_node;
ctlr->cleanup = dspi_cleanup;
if (dspi->devtype_data->trans_mode != DSPI_DMA_MODE)
ctlr->ptp_sts_supported = true;
- platform_set_drvdata(pdev, ctlr);
-
ret = spi_register_controller(ctlr);
if (ret != 0) {
dev_err(&pdev->dev, "Problem registering DSPI ctlr\n");
static int dspi_remove(struct platform_device *pdev)
{
- struct spi_controller *ctlr = platform_get_drvdata(pdev);
- struct fsl_dspi *dspi = spi_controller_get_devdata(ctlr);
+ struct fsl_dspi *dspi = platform_get_drvdata(pdev);
/* Disconnect from the SPI framework */
spi_unregister_controller(dspi->ctlr);
static irqreturn_t fsl_espi_irq(s32 irq, void *context_data)
{
struct fsl_espi *espi = context_data;
- u32 events;
+ u32 events, mask;
spin_lock(&espi->lock);
/* Get interrupt events(tx/rx) */
events = fsl_espi_read_reg(espi, ESPI_SPIE);
- if (!events) {
+ mask = fsl_espi_read_reg(espi, ESPI_SPIM);
+ if (!(events & mask)) {
spin_unlock(&espi->lock);
return IRQ_NONE;
}
if (ret < 0)
goto err_pm;
- dev_info(dev, "at 0x%p (irq = %u)\n", espi->reg_base, irq);
+ dev_info(dev, "irq = %u\n", irq);
pm_runtime_mark_last_busy(dev);
pm_runtime_put_autosuspend(dev);
return 0;
}
-#ifdef CONFIG_PM_SLEEP
-static int fsl_lpspi_suspend(struct device *dev)
+static int __maybe_unused fsl_lpspi_suspend(struct device *dev)
{
int ret;
return ret;
}
-static int fsl_lpspi_resume(struct device *dev)
+static int __maybe_unused fsl_lpspi_resume(struct device *dev)
{
int ret;
return 0;
}
-#endif /* CONFIG_PM_SLEEP */
static const struct dev_pm_ops fsl_lpspi_pm_ops = {
SET_RUNTIME_PM_OPS(fsl_lpspi_runtime_suspend,
{
struct geni_se *se = &mas->se;
unsigned int proto, major, minor, ver;
+ u32 spi_tx_cfg;
pm_runtime_get_sync(mas->dev);
* Hardware programming guide suggests to configure
* RX FIFO RFR level to fifo_depth-2.
*/
- geni_se_init(se, mas->tx_fifo_depth / 2, mas->tx_fifo_depth - 2);
+ geni_se_init(se, mas->tx_fifo_depth - 3, mas->tx_fifo_depth - 2);
/* Transmit an entire FIFO worth of data per IRQ */
mas->tx_wm = 1;
ver = geni_se_get_qup_hw_version(se);
geni_se_select_mode(se, GENI_SE_FIFO);
+ /* We always control CS manually */
+ spi_tx_cfg = readl(se->base + SE_SPI_TRANS_CFG);
+ spi_tx_cfg &= ~CS_TOGGLE;
+ writel(spi_tx_cfg, se->base + SE_SPI_TRANS_CFG);
+
pm_runtime_put(mas->dev);
return 0;
}
+static unsigned int geni_byte_per_fifo_word(struct spi_geni_master *mas)
+{
+ /*
+ * Calculate how many bytes we'll put in each FIFO word. If the
+ * transfer words don't pack cleanly into a FIFO word we'll just put
+ * one transfer word in each FIFO word. If they do pack we'll pack 'em.
+ */
+ if (mas->fifo_width_bits % mas->cur_bits_per_word)
+ return roundup_pow_of_two(DIV_ROUND_UP(mas->cur_bits_per_word,
+ BITS_PER_BYTE));
+
+ return mas->fifo_width_bits / BITS_PER_BYTE;
+}
+
+static bool geni_spi_handle_tx(struct spi_geni_master *mas)
+{
+ struct geni_se *se = &mas->se;
+ unsigned int max_bytes;
+ const u8 *tx_buf;
+ unsigned int bytes_per_fifo_word = geni_byte_per_fifo_word(mas);
+ unsigned int i = 0;
+
+ max_bytes = (mas->tx_fifo_depth - mas->tx_wm) * bytes_per_fifo_word;
+ if (mas->tx_rem_bytes < max_bytes)
+ max_bytes = mas->tx_rem_bytes;
+
+ tx_buf = mas->cur_xfer->tx_buf + mas->cur_xfer->len - mas->tx_rem_bytes;
+ while (i < max_bytes) {
+ unsigned int j;
+ unsigned int bytes_to_write;
+ u32 fifo_word = 0;
+ u8 *fifo_byte = (u8 *)&fifo_word;
+
+ bytes_to_write = min(bytes_per_fifo_word, max_bytes - i);
+ for (j = 0; j < bytes_to_write; j++)
+ fifo_byte[j] = tx_buf[i++];
+ iowrite32_rep(se->base + SE_GENI_TX_FIFOn, &fifo_word, 1);
+ }
+ mas->tx_rem_bytes -= max_bytes;
+ if (!mas->tx_rem_bytes) {
+ writel(0, se->base + SE_GENI_TX_WATERMARK_REG);
+ return false;
+ }
+ return true;
+}
+
+static void geni_spi_handle_rx(struct spi_geni_master *mas)
+{
+ struct geni_se *se = &mas->se;
+ u32 rx_fifo_status;
+ unsigned int rx_bytes;
+ unsigned int rx_last_byte_valid;
+ u8 *rx_buf;
+ unsigned int bytes_per_fifo_word = geni_byte_per_fifo_word(mas);
+ unsigned int i = 0;
+
+ rx_fifo_status = readl(se->base + SE_GENI_RX_FIFO_STATUS);
+ rx_bytes = (rx_fifo_status & RX_FIFO_WC_MSK) * bytes_per_fifo_word;
+ if (rx_fifo_status & RX_LAST) {
+ rx_last_byte_valid = rx_fifo_status & RX_LAST_BYTE_VALID_MSK;
+ rx_last_byte_valid >>= RX_LAST_BYTE_VALID_SHFT;
+ if (rx_last_byte_valid && rx_last_byte_valid < 4)
+ rx_bytes -= bytes_per_fifo_word - rx_last_byte_valid;
+ }
+ if (mas->rx_rem_bytes < rx_bytes)
+ rx_bytes = mas->rx_rem_bytes;
+
+ rx_buf = mas->cur_xfer->rx_buf + mas->cur_xfer->len - mas->rx_rem_bytes;
+ while (i < rx_bytes) {
+ u32 fifo_word = 0;
+ u8 *fifo_byte = (u8 *)&fifo_word;
+ unsigned int bytes_to_read;
+ unsigned int j;
+
+ bytes_to_read = min(bytes_per_fifo_word, rx_bytes - i);
+ ioread32_rep(se->base + SE_GENI_RX_FIFOn, &fifo_word, 1);
+ for (j = 0; j < bytes_to_read; j++)
+ rx_buf[i++] = fifo_byte[j];
+ }
+ mas->rx_rem_bytes -= rx_bytes;
+}
+
static void setup_fifo_xfer(struct spi_transfer *xfer,
struct spi_geni_master *mas,
u16 mode, struct spi_master *spi)
{
u32 m_cmd = 0;
- u32 spi_tx_cfg, len;
+ u32 len;
struct geni_se *se = &mas->se;
int ret;
spin_lock_irq(&mas->lock);
spin_unlock_irq(&mas->lock);
- spi_tx_cfg = readl(se->base + SE_SPI_TRANS_CFG);
if (xfer->bits_per_word != mas->cur_bits_per_word) {
spi_setup_word_len(mas, mode, xfer->bits_per_word);
mas->cur_bits_per_word = xfer->bits_per_word;
mas->tx_rem_bytes = 0;
mas->rx_rem_bytes = 0;
- spi_tx_cfg &= ~CS_TOGGLE;
-
if (!(mas->cur_bits_per_word % MIN_WORD_LEN))
len = xfer->len * BITS_PER_BYTE / mas->cur_bits_per_word;
else
writel(len, se->base + SE_SPI_RX_TRANS_LEN);
mas->rx_rem_bytes = xfer->len;
}
- writel(spi_tx_cfg, se->base + SE_SPI_TRANS_CFG);
/*
* Lock around right before we start the transfer since our
* setting up GENI SE engine, as driver starts data transfer
* for the watermark interrupt.
*/
- if (m_cmd & SPI_TX_ONLY)
- writel(mas->tx_wm, se->base + SE_GENI_TX_WATERMARK_REG);
+ if (m_cmd & SPI_TX_ONLY) {
+ if (geni_spi_handle_tx(mas))
+ writel(mas->tx_wm, se->base + SE_GENI_TX_WATERMARK_REG);
+ }
spin_unlock_irq(&mas->lock);
}
return 1;
}
-static unsigned int geni_byte_per_fifo_word(struct spi_geni_master *mas)
-{
- /*
- * Calculate how many bytes we'll put in each FIFO word. If the
- * transfer words don't pack cleanly into a FIFO word we'll just put
- * one transfer word in each FIFO word. If they do pack we'll pack 'em.
- */
- if (mas->fifo_width_bits % mas->cur_bits_per_word)
- return roundup_pow_of_two(DIV_ROUND_UP(mas->cur_bits_per_word,
- BITS_PER_BYTE));
-
- return mas->fifo_width_bits / BITS_PER_BYTE;
-}
-
-static void geni_spi_handle_tx(struct spi_geni_master *mas)
-{
- struct geni_se *se = &mas->se;
- unsigned int max_bytes;
- const u8 *tx_buf;
- unsigned int bytes_per_fifo_word = geni_byte_per_fifo_word(mas);
- unsigned int i = 0;
-
- max_bytes = (mas->tx_fifo_depth - mas->tx_wm) * bytes_per_fifo_word;
- if (mas->tx_rem_bytes < max_bytes)
- max_bytes = mas->tx_rem_bytes;
-
- tx_buf = mas->cur_xfer->tx_buf + mas->cur_xfer->len - mas->tx_rem_bytes;
- while (i < max_bytes) {
- unsigned int j;
- unsigned int bytes_to_write;
- u32 fifo_word = 0;
- u8 *fifo_byte = (u8 *)&fifo_word;
-
- bytes_to_write = min(bytes_per_fifo_word, max_bytes - i);
- for (j = 0; j < bytes_to_write; j++)
- fifo_byte[j] = tx_buf[i++];
- iowrite32_rep(se->base + SE_GENI_TX_FIFOn, &fifo_word, 1);
- }
- mas->tx_rem_bytes -= max_bytes;
- if (!mas->tx_rem_bytes)
- writel(0, se->base + SE_GENI_TX_WATERMARK_REG);
-}
-
-static void geni_spi_handle_rx(struct spi_geni_master *mas)
-{
- struct geni_se *se = &mas->se;
- u32 rx_fifo_status;
- unsigned int rx_bytes;
- unsigned int rx_last_byte_valid;
- u8 *rx_buf;
- unsigned int bytes_per_fifo_word = geni_byte_per_fifo_word(mas);
- unsigned int i = 0;
-
- rx_fifo_status = readl(se->base + SE_GENI_RX_FIFO_STATUS);
- rx_bytes = (rx_fifo_status & RX_FIFO_WC_MSK) * bytes_per_fifo_word;
- if (rx_fifo_status & RX_LAST) {
- rx_last_byte_valid = rx_fifo_status & RX_LAST_BYTE_VALID_MSK;
- rx_last_byte_valid >>= RX_LAST_BYTE_VALID_SHFT;
- if (rx_last_byte_valid && rx_last_byte_valid < 4)
- rx_bytes -= bytes_per_fifo_word - rx_last_byte_valid;
- }
- if (mas->rx_rem_bytes < rx_bytes)
- rx_bytes = mas->rx_rem_bytes;
-
- rx_buf = mas->cur_xfer->rx_buf + mas->cur_xfer->len - mas->rx_rem_bytes;
- while (i < rx_bytes) {
- u32 fifo_word = 0;
- u8 *fifo_byte = (u8 *)&fifo_word;
- unsigned int bytes_to_read;
- unsigned int j;
-
- bytes_to_read = min(bytes_per_fifo_word, rx_bytes - i);
- ioread32_rep(se->base + SE_GENI_RX_FIFOn, &fifo_word, 1);
- for (j = 0; j < bytes_to_read; j++)
- rx_buf[i++] = fifo_byte[j];
- }
- mas->rx_rem_bytes -= rx_bytes;
-}
-
static irqreturn_t geni_spi_isr(int irq, void *data)
{
struct spi_master *spi = data;
return PTR_ERR(mas->se.opp_table);
/* OPP table is optional */
ret = dev_pm_opp_of_add_table(&pdev->dev);
- if (!ret) {
- mas->se.has_opp_table = true;
- } else if (ret != -ENODEV) {
+ if (ret && ret != -ENODEV) {
dev_err(&pdev->dev, "invalid OPP table in device tree\n");
- return ret;
+ goto put_clkname;
}
spi->bus_num = -1;
spi_geni_probe_runtime_disable:
pm_runtime_disable(dev);
spi_master_put(spi);
- if (mas->se.has_opp_table)
- dev_pm_opp_of_remove_table(&pdev->dev);
+ dev_pm_opp_of_remove_table(&pdev->dev);
+put_clkname:
dev_pm_opp_put_clkname(mas->se.opp_table);
return ret;
}
free_irq(mas->irq, spi);
pm_runtime_disable(&pdev->dev);
- if (mas->se.has_opp_table)
- dev_pm_opp_of_remove_table(&pdev->dev);
+ dev_pm_opp_of_remove_table(&pdev->dev);
dev_pm_opp_put_clkname(mas->se.opp_table);
return 0;
}
#include <linux/acpi.h>
#include <linux/bitops.h>
+#include <linux/completion.h>
#include <linux/dmi.h>
+#include <linux/interrupt.h>
#include <linux/iopoll.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#define HISI_SFC_V3XX_VERSION (0x1f8)
-#define HISI_SFC_V3XX_INT_STAT (0x120)
-#define HISI_SFC_V3XX_INT_STAT_PP_ERR BIT(2)
-#define HISI_SFC_V3XX_INT_STAT_ADDR_IACCES BIT(5)
+#define HISI_SFC_V3XX_RAW_INT_STAT (0x120)
+#define HISI_SFC_V3XX_INT_STAT (0x124)
+#define HISI_SFC_V3XX_INT_MASK (0x128)
#define HISI_SFC_V3XX_INT_CLR (0x12c)
-#define HISI_SFC_V3XX_INT_CLR_CLEAR (0xff)
#define HISI_SFC_V3XX_CMD_CFG (0x300)
-#define HISI_SFC_V3XX_CMD_CFG_DUAL_IN_DUAL_OUT (1 << 17)
-#define HISI_SFC_V3XX_CMD_CFG_DUAL_IO (2 << 17)
-#define HISI_SFC_V3XX_CMD_CFG_FULL_DIO (3 << 17)
-#define HISI_SFC_V3XX_CMD_CFG_QUAD_IN_QUAD_OUT (5 << 17)
-#define HISI_SFC_V3XX_CMD_CFG_QUAD_IO (6 << 17)
-#define HISI_SFC_V3XX_CMD_CFG_FULL_QIO (7 << 17)
#define HISI_SFC_V3XX_CMD_CFG_DATA_CNT_OFF 9
#define HISI_SFC_V3XX_CMD_CFG_RW_MSK BIT(8)
#define HISI_SFC_V3XX_CMD_CFG_DATA_EN_MSK BIT(7)
#define HISI_SFC_V3XX_CMD_ADDR (0x30c)
#define HISI_SFC_V3XX_CMD_DATABUF0 (0x400)
+/* Common definition of interrupt bit masks */
+#define HISI_SFC_V3XX_INT_MASK_ALL (0x1ff) /* all the masks */
+#define HISI_SFC_V3XX_INT_MASK_CPLT BIT(0) /* command execution complete */
+#define HISI_SFC_V3XX_INT_MASK_PP_ERR BIT(2) /* page progrom error */
+#define HISI_SFC_V3XX_INT_MASK_IACCES BIT(5) /* error visiting inaccessible/
+ * protected address
+ */
+
+/* IO Mode definition in HISI_SFC_V3XX_CMD_CFG */
+#define HISI_SFC_V3XX_STD (0 << 17)
+#define HISI_SFC_V3XX_DIDO (1 << 17)
+#define HISI_SFC_V3XX_DIO (2 << 17)
+#define HISI_SFC_V3XX_FULL_DIO (3 << 17)
+#define HISI_SFC_V3XX_QIQO (5 << 17)
+#define HISI_SFC_V3XX_QIO (6 << 17)
+#define HISI_SFC_V3XX_FULL_QIO (7 << 17)
+
+/*
+ * The IO modes lookup table. hisi_sfc_v3xx_io_modes[(z - 1) / 2][y / 2][x / 2]
+ * stands for x-y-z mode, as described in SFDP terminology. -EIO indicates
+ * an invalid mode.
+ */
+static const int hisi_sfc_v3xx_io_modes[2][3][3] = {
+ {
+ { HISI_SFC_V3XX_DIDO, HISI_SFC_V3XX_DIDO, HISI_SFC_V3XX_DIDO },
+ { HISI_SFC_V3XX_DIO, HISI_SFC_V3XX_FULL_DIO, -EIO },
+ { -EIO, -EIO, -EIO },
+ },
+ {
+ { HISI_SFC_V3XX_QIQO, HISI_SFC_V3XX_QIQO, HISI_SFC_V3XX_QIQO },
+ { -EIO, -EIO, -EIO },
+ { HISI_SFC_V3XX_QIO, -EIO, HISI_SFC_V3XX_FULL_QIO },
+ },
+};
+
struct hisi_sfc_v3xx_host {
struct device *dev;
void __iomem *regbase;
int max_cmd_dword;
+ struct completion *completion;
+ int irq;
};
+static void hisi_sfc_v3xx_disable_int(struct hisi_sfc_v3xx_host *host)
+{
+ writel(0, host->regbase + HISI_SFC_V3XX_INT_MASK);
+}
+
+static void hisi_sfc_v3xx_enable_int(struct hisi_sfc_v3xx_host *host)
+{
+ writel(HISI_SFC_V3XX_INT_MASK_ALL, host->regbase + HISI_SFC_V3XX_INT_MASK);
+}
+
+static void hisi_sfc_v3xx_clear_int(struct hisi_sfc_v3xx_host *host)
+{
+ writel(HISI_SFC_V3XX_INT_MASK_ALL, host->regbase + HISI_SFC_V3XX_INT_CLR);
+}
+
+/*
+ * The interrupt status register indicates whether an error occurs
+ * after per operation. Check it, and clear the interrupts for
+ * next time judgement.
+ */
+static int hisi_sfc_v3xx_handle_completion(struct hisi_sfc_v3xx_host *host)
+{
+ u32 reg;
+
+ reg = readl(host->regbase + HISI_SFC_V3XX_RAW_INT_STAT);
+ hisi_sfc_v3xx_clear_int(host);
+
+ if (reg & HISI_SFC_V3XX_INT_MASK_IACCES) {
+ dev_err(host->dev, "fail to access protected address\n");
+ return -EIO;
+ }
+
+ if (reg & HISI_SFC_V3XX_INT_MASK_PP_ERR) {
+ dev_err(host->dev, "page program operation failed\n");
+ return -EIO;
+ }
+
+ /*
+ * The other bits of the interrupt registers is not currently
+ * used and probably not be triggered in this driver. When it
+ * happens, we regard it as an unsupported error here.
+ */
+ if (!(reg & HISI_SFC_V3XX_INT_MASK_CPLT)) {
+ dev_err(host->dev, "unsupported error occurred, status=0x%x\n", reg);
+ return -EIO;
+ }
+
+ return 0;
+}
+
#define HISI_SFC_V3XX_WAIT_TIMEOUT_US 1000000
#define HISI_SFC_V3XX_WAIT_POLL_INTERVAL_US 10
return 0;
}
+/*
+ * The controller only supports Standard SPI mode, Duall mode and
+ * Quad mode. Double sanitize the ops here to avoid OOB access.
+ */
+static bool hisi_sfc_v3xx_supports_op(struct spi_mem *mem,
+ const struct spi_mem_op *op)
+{
+ if (op->data.buswidth > 4 || op->dummy.buswidth > 4 ||
+ op->addr.buswidth > 4 || op->cmd.buswidth > 4)
+ return false;
+
+ return spi_mem_default_supports_op(mem, op);
+}
+
/*
* memcpy_{to,from}io doesn't gurantee 32b accesses - which we require for the
* DATABUF registers -so use __io{read,write}32_copy when possible. For
}
}
-static int hisi_sfc_v3xx_generic_exec_op(struct hisi_sfc_v3xx_host *host,
- const struct spi_mem_op *op,
- u8 chip_select)
+static int hisi_sfc_v3xx_start_bus(struct hisi_sfc_v3xx_host *host,
+ const struct spi_mem_op *op,
+ u8 chip_select)
{
- int ret, len = op->data.nbytes;
- u32 int_stat, config = 0;
+ int len = op->data.nbytes, buswidth_mode;
+ u32 config = 0;
if (op->addr.nbytes)
config |= HISI_SFC_V3XX_CMD_CFG_ADDR_EN_MSK;
- switch (op->data.buswidth) {
- case 0 ... 1:
- break;
- case 2:
- if (op->addr.buswidth <= 1) {
- config |= HISI_SFC_V3XX_CMD_CFG_DUAL_IN_DUAL_OUT;
- } else if (op->addr.buswidth == 2) {
- if (op->cmd.buswidth <= 1) {
- config |= HISI_SFC_V3XX_CMD_CFG_DUAL_IO;
- } else if (op->cmd.buswidth == 2) {
- config |= HISI_SFC_V3XX_CMD_CFG_FULL_DIO;
- } else {
- return -EIO;
- }
- } else {
- return -EIO;
- }
- break;
- case 4:
- if (op->addr.buswidth <= 1) {
- config |= HISI_SFC_V3XX_CMD_CFG_QUAD_IN_QUAD_OUT;
- } else if (op->addr.buswidth == 4) {
- if (op->cmd.buswidth <= 1) {
- config |= HISI_SFC_V3XX_CMD_CFG_QUAD_IO;
- } else if (op->cmd.buswidth == 4) {
- config |= HISI_SFC_V3XX_CMD_CFG_FULL_QIO;
- } else {
- return -EIO;
- }
- } else {
- return -EIO;
- }
- break;
- default:
- return -EOPNOTSUPP;
+ if (op->data.buswidth == 0 || op->data.buswidth == 1) {
+ buswidth_mode = HISI_SFC_V3XX_STD;
+ } else {
+ int data_idx, addr_idx, cmd_idx;
+
+ data_idx = (op->data.buswidth - 1) / 2;
+ addr_idx = op->addr.buswidth / 2;
+ cmd_idx = op->cmd.buswidth / 2;
+ buswidth_mode = hisi_sfc_v3xx_io_modes[data_idx][addr_idx][cmd_idx];
}
+ if (buswidth_mode < 0)
+ return buswidth_mode;
+ config |= buswidth_mode;
if (op->data.dir != SPI_MEM_NO_DATA) {
config |= (len - 1) << HISI_SFC_V3XX_CMD_CFG_DATA_CNT_OFF;
config |= HISI_SFC_V3XX_CMD_CFG_DATA_EN_MSK;
}
- if (op->data.dir == SPI_MEM_DATA_OUT)
- hisi_sfc_v3xx_write_databuf(host, op->data.buf.out, len);
- else if (op->data.dir == SPI_MEM_DATA_IN)
+ if (op->data.dir == SPI_MEM_DATA_IN)
config |= HISI_SFC_V3XX_CMD_CFG_RW_MSK;
config |= op->dummy.nbytes << HISI_SFC_V3XX_CMD_CFG_DUMMY_CNT_OFF |
writel(config, host->regbase + HISI_SFC_V3XX_CMD_CFG);
- ret = hisi_sfc_v3xx_wait_cmd_idle(host);
+ return 0;
+}
+
+static int hisi_sfc_v3xx_generic_exec_op(struct hisi_sfc_v3xx_host *host,
+ const struct spi_mem_op *op,
+ u8 chip_select)
+{
+ DECLARE_COMPLETION_ONSTACK(done);
+ int ret;
+
+ if (host->irq) {
+ host->completion = &done;
+ hisi_sfc_v3xx_enable_int(host);
+ }
+
+ if (op->data.dir == SPI_MEM_DATA_OUT)
+ hisi_sfc_v3xx_write_databuf(host, op->data.buf.out, op->data.nbytes);
+
+ ret = hisi_sfc_v3xx_start_bus(host, op, chip_select);
if (ret)
return ret;
- /*
- * The interrupt status register indicates whether an error occurs
- * after per operation. Check it, and clear the interrupts for
- * next time judgement.
- */
- int_stat = readl(host->regbase + HISI_SFC_V3XX_INT_STAT);
- writel(HISI_SFC_V3XX_INT_CLR_CLEAR,
- host->regbase + HISI_SFC_V3XX_INT_CLR);
+ if (host->irq) {
+ ret = wait_for_completion_timeout(host->completion,
+ usecs_to_jiffies(HISI_SFC_V3XX_WAIT_TIMEOUT_US));
+ if (!ret)
+ ret = -ETIMEDOUT;
+ else
+ ret = 0;
- if (int_stat & HISI_SFC_V3XX_INT_STAT_ADDR_IACCES) {
- dev_err(host->dev, "fail to access protected address\n");
- return -EIO;
+ hisi_sfc_v3xx_disable_int(host);
+ host->completion = NULL;
+ } else {
+ ret = hisi_sfc_v3xx_wait_cmd_idle(host);
}
-
- if (int_stat & HISI_SFC_V3XX_INT_STAT_PP_ERR) {
- dev_err(host->dev, "page program operation failed\n");
+ if (hisi_sfc_v3xx_handle_completion(host) || ret)
return -EIO;
- }
if (op->data.dir == SPI_MEM_DATA_IN)
- hisi_sfc_v3xx_read_databuf(host, op->data.buf.in, len);
+ hisi_sfc_v3xx_read_databuf(host, op->data.buf.in, op->data.nbytes);
return 0;
}
static const struct spi_controller_mem_ops hisi_sfc_v3xx_mem_ops = {
.adjust_op_size = hisi_sfc_v3xx_adjust_op_size,
+ .supports_op = hisi_sfc_v3xx_supports_op,
.exec_op = hisi_sfc_v3xx_exec_op,
};
+static irqreturn_t hisi_sfc_v3xx_isr(int irq, void *data)
+{
+ struct hisi_sfc_v3xx_host *host = data;
+
+ hisi_sfc_v3xx_disable_int(host);
+
+ complete(host->completion);
+
+ return IRQ_HANDLED;
+}
+
static int hisi_sfc_v3xx_buswidth_override_bits;
/*
goto err_put_master;
}
+ host->irq = platform_get_irq_optional(pdev, 0);
+ if (host->irq == -EPROBE_DEFER) {
+ ret = -EPROBE_DEFER;
+ goto err_put_master;
+ }
+
+ hisi_sfc_v3xx_disable_int(host);
+
+ if (host->irq > 0) {
+ ret = devm_request_irq(dev, host->irq, hisi_sfc_v3xx_isr, 0,
+ "hisi-sfc-v3xx", host);
+
+ if (ret) {
+ dev_err(dev, "failed to request irq%d, ret = %d\n", host->irq, ret);
+ host->irq = 0;
+ }
+ } else {
+ host->irq = 0;
+ }
+
ctlr->bus_num = -1;
ctlr->num_chipselect = 1;
ctlr->mem_ops = &hisi_sfc_v3xx_mem_ops;
if (ret)
goto err_put_master;
- dev_info(&pdev->dev, "hw version 0x%x\n", version);
+ dev_info(&pdev->dev, "hw version 0x%x, %s mode.\n",
+ version, host->irq ? "irq" : "polling");
return 0;
{
struct spi_imx_data *spi_imx = spi_master_get_devdata(spi->master);
+ transfer->effective_speed_hz = spi_imx->spi_bus_clk;
+
/* flush rxfifo before transfer */
while (spi_imx->devtype_data->rx_available(spi_imx))
readl(spi_imx->base + MXC_CSPIRXDATA);
goto out_runtime_pm_put;
if (ret < 0)
- dev_err(&pdev->dev, "dma setup error %d, use pio\n",
+ dev_dbg(&pdev->dev, "dma setup error %d, use pio\n",
ret);
}
ret = spi_bitbang_start(&spi_imx->bitbang);
if (ret) {
dev_err(&pdev->dev, "bitbang start failed with %d\n", ret);
- goto out_runtime_pm_put;
+ goto out_bitbang_start;
}
- dev_info(&pdev->dev, "probed\n");
-
pm_runtime_mark_last_busy(spi_imx->dev);
pm_runtime_put_autosuspend(spi_imx->dev);
return ret;
+out_bitbang_start:
+ if (spi_imx->devtype_data->has_dmamode)
+ spi_imx_sdma_exit(spi_imx);
out_runtime_pm_put:
pm_runtime_dont_use_autosuspend(spi_imx->dev);
pm_runtime_put_sync(spi_imx->dev);
struct lantiq_ssc_spi *spi = data;
const struct lantiq_ssc_hwcfg *hwcfg = spi->hwcfg;
u32 val = lantiq_ssc_readl(spi, hwcfg->irncr);
- unsigned long flags;
- spin_lock_irqsave(&spi->lock, flags);
+ spin_lock(&spi->lock);
if (hwcfg->irq_ack)
lantiq_ssc_writel(spi, val, hwcfg->irncr);
}
}
- spin_unlock_irqrestore(&spi->lock, flags);
+ spin_unlock(&spi->lock);
return IRQ_HANDLED;
completed:
queue_work(spi->wq, &spi->work);
- spin_unlock_irqrestore(&spi->lock, flags);
+ spin_unlock(&spi->lock);
return IRQ_HANDLED;
}
const struct lantiq_ssc_hwcfg *hwcfg = spi->hwcfg;
u32 stat = lantiq_ssc_readl(spi, LTQ_SPI_STAT);
u32 val = lantiq_ssc_readl(spi, hwcfg->irncr);
- unsigned long flags;
if (!(stat & LTQ_SPI_STAT_ERRORS))
return IRQ_NONE;
- spin_lock_irqsave(&spi->lock, flags);
+ spin_lock(&spi->lock);
if (hwcfg->irq_ack)
lantiq_ssc_writel(spi, val, hwcfg->irncr);
if (spi->master->cur_msg)
spi->master->cur_msg->status = -EIO;
queue_work(spi->wq, &spi->work);
- spin_unlock_irqrestore(&spi->lock, flags);
+ spin_unlock(&spi->lock);
return IRQ_HANDLED;
}
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/of_device.h>
+#include <linux/pm_runtime.h>
#include <linux/spi/spi.h>
#include <linux/spi/spi-mem.h>
#include <linux/string.h>
#define MTK_NOR_CMD_MASK GENMASK(5, 0)
#define MTK_NOR_REG_PRG_CNT 0x04
+#define MTK_NOR_PRG_CNT_MAX 56
#define MTK_NOR_REG_RDATA 0x0c
#define MTK_NOR_REG_RADR0 0x10
#define MTK_NOR_REG_DMA_FADR 0x71c
#define MTK_NOR_REG_DMA_DADR 0x720
#define MTK_NOR_REG_DMA_END_DADR 0x724
+#define MTK_NOR_REG_DMA_DADR_HB 0x738
+#define MTK_NOR_REG_DMA_END_DADR_HB 0x73c
#define MTK_NOR_PRG_MAX_SIZE 6
// Reading DMA src/dst addresses have to be 16-byte aligned
// Buffered page program can do one 128-byte transfer
#define MTK_NOR_PP_SIZE 128
-#define CLK_TO_US(sp, clkcnt) ((clkcnt) * 1000000 / sp->spi_freq)
+#define CLK_TO_US(sp, clkcnt) DIV_ROUND_UP(clkcnt, sp->spi_freq / 1000000)
struct mtk_nor {
struct spi_controller *ctlr;
struct device *dev;
void __iomem *base;
u8 *buffer;
+ dma_addr_t buffer_dma;
struct clk *spi_clk;
struct clk *ctlr_clk;
unsigned int spi_freq;
bool wbuf_en;
bool has_irq;
+ bool high_dma;
struct completion op_done;
};
}
}
+static bool need_bounce(struct mtk_nor *sp, const struct spi_mem_op *op)
+{
+ return ((uintptr_t)op->data.buf.in & MTK_NOR_DMA_ALIGN_MASK);
+}
+
static bool mtk_nor_match_read(const struct spi_mem_op *op)
{
int dummy = 0;
return false;
}
+static bool mtk_nor_match_prg(const struct spi_mem_op *op)
+{
+ int tx_len, rx_len, prg_len, prg_left;
+
+ // prg mode is spi-only.
+ if ((op->cmd.buswidth > 1) || (op->addr.buswidth > 1) ||
+ (op->dummy.buswidth > 1) || (op->data.buswidth > 1))
+ return false;
+
+ tx_len = op->cmd.nbytes + op->addr.nbytes;
+
+ if (op->data.dir == SPI_MEM_DATA_OUT) {
+ // count dummy bytes only if we need to write data after it
+ tx_len += op->dummy.nbytes;
+
+ // leave at least one byte for data
+ if (tx_len > MTK_NOR_REG_PRGDATA_MAX)
+ return false;
+
+ // if there's no addr, meaning adjust_op_size is impossible,
+ // check data length as well.
+ if ((!op->addr.nbytes) &&
+ (tx_len + op->data.nbytes > MTK_NOR_REG_PRGDATA_MAX + 1))
+ return false;
+ } else if (op->data.dir == SPI_MEM_DATA_IN) {
+ if (tx_len > MTK_NOR_REG_PRGDATA_MAX + 1)
+ return false;
+
+ rx_len = op->data.nbytes;
+ prg_left = MTK_NOR_PRG_CNT_MAX / 8 - tx_len - op->dummy.nbytes;
+ if (prg_left > MTK_NOR_REG_SHIFT_MAX + 1)
+ prg_left = MTK_NOR_REG_SHIFT_MAX + 1;
+ if (rx_len > prg_left) {
+ if (!op->addr.nbytes)
+ return false;
+ rx_len = prg_left;
+ }
+
+ prg_len = tx_len + op->dummy.nbytes + rx_len;
+ if (prg_len > MTK_NOR_PRG_CNT_MAX / 8)
+ return false;
+ } else {
+ prg_len = tx_len + op->dummy.nbytes;
+ if (prg_len > MTK_NOR_PRG_CNT_MAX / 8)
+ return false;
+ }
+ return true;
+}
+
+static void mtk_nor_adj_prg_size(struct spi_mem_op *op)
+{
+ int tx_len, tx_left, prg_left;
+
+ tx_len = op->cmd.nbytes + op->addr.nbytes;
+ if (op->data.dir == SPI_MEM_DATA_OUT) {
+ tx_len += op->dummy.nbytes;
+ tx_left = MTK_NOR_REG_PRGDATA_MAX + 1 - tx_len;
+ if (op->data.nbytes > tx_left)
+ op->data.nbytes = tx_left;
+ } else if (op->data.dir == SPI_MEM_DATA_IN) {
+ prg_left = MTK_NOR_PRG_CNT_MAX / 8 - tx_len - op->dummy.nbytes;
+ if (prg_left > MTK_NOR_REG_SHIFT_MAX + 1)
+ prg_left = MTK_NOR_REG_SHIFT_MAX + 1;
+ if (op->data.nbytes > prg_left)
+ op->data.nbytes = prg_left;
+ }
+}
+
static int mtk_nor_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
{
- size_t len;
+ struct mtk_nor *sp = spi_controller_get_devdata(mem->spi->master);
if (!op->data.nbytes)
return 0;
if ((op->addr.nbytes == 3) || (op->addr.nbytes == 4)) {
if ((op->data.dir == SPI_MEM_DATA_IN) &&
mtk_nor_match_read(op)) {
+ // limit size to prevent timeout calculation overflow
+ if (op->data.nbytes > 0x400000)
+ op->data.nbytes = 0x400000;
+
if ((op->addr.val & MTK_NOR_DMA_ALIGN_MASK) ||
(op->data.nbytes < MTK_NOR_DMA_ALIGN))
op->data.nbytes = 1;
- else if (!((ulong)(op->data.buf.in) &
- MTK_NOR_DMA_ALIGN_MASK))
+ else if (!need_bounce(sp, op))
op->data.nbytes &= ~MTK_NOR_DMA_ALIGN_MASK;
else if (op->data.nbytes > MTK_NOR_BOUNCE_BUF_SIZE)
op->data.nbytes = MTK_NOR_BOUNCE_BUF_SIZE;
}
}
- len = MTK_NOR_PRG_MAX_SIZE - op->cmd.nbytes - op->addr.nbytes -
- op->dummy.nbytes;
- if (op->data.nbytes > len)
- op->data.nbytes = len;
-
+ mtk_nor_adj_prg_size(op);
return 0;
}
static bool mtk_nor_supports_op(struct spi_mem *mem,
const struct spi_mem_op *op)
{
- size_t len;
-
- if (op->cmd.buswidth != 1)
+ if (!spi_mem_default_supports_op(mem, op))
return false;
- /* DTR ops not supported. */
- if (op->cmd.dtr || op->addr.dtr || op->dummy.dtr || op->data.dtr)
- return false;
- if (op->cmd.nbytes != 1)
+ if (op->cmd.buswidth != 1)
return false;
if ((op->addr.nbytes == 3) || (op->addr.nbytes == 4)) {
- if ((op->data.dir == SPI_MEM_DATA_IN) && mtk_nor_match_read(op))
- return true;
- else if (op->data.dir == SPI_MEM_DATA_OUT)
- return (op->addr.buswidth == 1) &&
- (op->dummy.buswidth == 0) &&
- (op->data.buswidth == 1);
+ switch(op->data.dir) {
+ case SPI_MEM_DATA_IN:
+ if (mtk_nor_match_read(op))
+ return true;
+ break;
+ case SPI_MEM_DATA_OUT:
+ if ((op->addr.buswidth == 1) &&
+ (op->dummy.nbytes == 0) &&
+ (op->data.buswidth == 1))
+ return true;
+ break;
+ default:
+ break;
+ }
}
- len = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes;
- if ((len > MTK_NOR_PRG_MAX_SIZE) ||
- ((op->data.nbytes) && (len == MTK_NOR_PRG_MAX_SIZE)))
- return false;
- return true;
+
+ return mtk_nor_match_prg(op);
}
static void mtk_nor_setup_bus(struct mtk_nor *sp, const struct spi_mem_op *op)
mtk_nor_rmw(sp, MTK_NOR_REG_BUSCFG, reg, MTK_NOR_BUS_MODE_MASK);
}
-static int mtk_nor_read_dma(struct mtk_nor *sp, u32 from, unsigned int length,
- u8 *buffer)
+static int mtk_nor_dma_exec(struct mtk_nor *sp, u32 from, unsigned int length,
+ dma_addr_t dma_addr)
{
int ret = 0;
ulong delay;
u32 reg;
- dma_addr_t dma_addr;
-
- dma_addr = dma_map_single(sp->dev, buffer, length, DMA_FROM_DEVICE);
- if (dma_mapping_error(sp->dev, dma_addr)) {
- dev_err(sp->dev, "failed to map dma buffer.\n");
- return -EINVAL;
- }
writel(from, sp->base + MTK_NOR_REG_DMA_FADR);
writel(dma_addr, sp->base + MTK_NOR_REG_DMA_DADR);
writel(dma_addr + length, sp->base + MTK_NOR_REG_DMA_END_DADR);
+ if (sp->high_dma) {
+ writel(upper_32_bits(dma_addr),
+ sp->base + MTK_NOR_REG_DMA_DADR_HB);
+ writel(upper_32_bits(dma_addr + length),
+ sp->base + MTK_NOR_REG_DMA_END_DADR_HB);
+ }
+
if (sp->has_irq) {
reinit_completion(&sp->op_done);
mtk_nor_rmw(sp, MTK_NOR_REG_IRQ_EN, MTK_NOR_IRQ_DMA, 0);
(delay + 1) * 100);
}
- dma_unmap_single(sp->dev, dma_addr, length, DMA_FROM_DEVICE);
if (ret < 0)
dev_err(sp->dev, "dma read timeout.\n");
return ret;
}
-static int mtk_nor_read_bounce(struct mtk_nor *sp, u32 from,
- unsigned int length, u8 *buffer)
+static int mtk_nor_read_bounce(struct mtk_nor *sp, const struct spi_mem_op *op)
{
unsigned int rdlen;
int ret;
- if (length & MTK_NOR_DMA_ALIGN_MASK)
- rdlen = (length + MTK_NOR_DMA_ALIGN) & ~MTK_NOR_DMA_ALIGN_MASK;
+ if (op->data.nbytes & MTK_NOR_DMA_ALIGN_MASK)
+ rdlen = (op->data.nbytes + MTK_NOR_DMA_ALIGN) & ~MTK_NOR_DMA_ALIGN_MASK;
else
- rdlen = length;
+ rdlen = op->data.nbytes;
- ret = mtk_nor_read_dma(sp, from, rdlen, sp->buffer);
- if (ret)
- return ret;
+ ret = mtk_nor_dma_exec(sp, op->addr.val, rdlen, sp->buffer_dma);
- memcpy(buffer, sp->buffer, length);
- return 0;
+ if (!ret)
+ memcpy(op->data.buf.in, sp->buffer, op->data.nbytes);
+
+ return ret;
+}
+
+static int mtk_nor_read_dma(struct mtk_nor *sp, const struct spi_mem_op *op)
+{
+ int ret;
+ dma_addr_t dma_addr;
+
+ if (need_bounce(sp, op))
+ return mtk_nor_read_bounce(sp, op);
+
+ dma_addr = dma_map_single(sp->dev, op->data.buf.in,
+ op->data.nbytes, DMA_FROM_DEVICE);
+
+ if (dma_mapping_error(sp->dev, dma_addr))
+ return -EINVAL;
+
+ ret = mtk_nor_dma_exec(sp, op->addr.val, op->data.nbytes, dma_addr);
+
+ dma_unmap_single(sp->dev, dma_addr, op->data.nbytes, DMA_FROM_DEVICE);
+
+ return ret;
}
static int mtk_nor_read_pio(struct mtk_nor *sp, const struct spi_mem_op *op)
return mtk_nor_cmd_exec(sp, MTK_NOR_CMD_WRITE, 6 * BITS_PER_BYTE);
}
+static int mtk_nor_spi_mem_prg(struct mtk_nor *sp, const struct spi_mem_op *op)
+{
+ int rx_len = 0;
+ int reg_offset = MTK_NOR_REG_PRGDATA_MAX;
+ int tx_len, prg_len;
+ int i, ret;
+ void __iomem *reg;
+ u8 bufbyte;
+
+ tx_len = op->cmd.nbytes + op->addr.nbytes;
+
+ // count dummy bytes only if we need to write data after it
+ if (op->data.dir == SPI_MEM_DATA_OUT)
+ tx_len += op->dummy.nbytes + op->data.nbytes;
+ else if (op->data.dir == SPI_MEM_DATA_IN)
+ rx_len = op->data.nbytes;
+
+ prg_len = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes +
+ op->data.nbytes;
+
+ // an invalid op may reach here if the caller calls exec_op without
+ // adjust_op_size. return -EINVAL instead of -ENOTSUPP so that
+ // spi-mem won't try this op again with generic spi transfers.
+ if ((tx_len > MTK_NOR_REG_PRGDATA_MAX + 1) ||
+ (rx_len > MTK_NOR_REG_SHIFT_MAX + 1) ||
+ (prg_len > MTK_NOR_PRG_CNT_MAX / 8))
+ return -EINVAL;
+
+ // fill tx data
+ for (i = op->cmd.nbytes; i > 0; i--, reg_offset--) {
+ reg = sp->base + MTK_NOR_REG_PRGDATA(reg_offset);
+ bufbyte = (op->cmd.opcode >> ((i - 1) * BITS_PER_BYTE)) & 0xff;
+ writeb(bufbyte, reg);
+ }
+
+ for (i = op->addr.nbytes; i > 0; i--, reg_offset--) {
+ reg = sp->base + MTK_NOR_REG_PRGDATA(reg_offset);
+ bufbyte = (op->addr.val >> ((i - 1) * BITS_PER_BYTE)) & 0xff;
+ writeb(bufbyte, reg);
+ }
+
+ if (op->data.dir == SPI_MEM_DATA_OUT) {
+ for (i = 0; i < op->dummy.nbytes; i++, reg_offset--) {
+ reg = sp->base + MTK_NOR_REG_PRGDATA(reg_offset);
+ writeb(0, reg);
+ }
+
+ for (i = 0; i < op->data.nbytes; i++, reg_offset--) {
+ reg = sp->base + MTK_NOR_REG_PRGDATA(reg_offset);
+ writeb(((const u8 *)(op->data.buf.out))[i], reg);
+ }
+ }
+
+ for (; reg_offset >= 0; reg_offset--) {
+ reg = sp->base + MTK_NOR_REG_PRGDATA(reg_offset);
+ writeb(0, reg);
+ }
+
+ // trigger op
+ writel(prg_len * BITS_PER_BYTE, sp->base + MTK_NOR_REG_PRG_CNT);
+ ret = mtk_nor_cmd_exec(sp, MTK_NOR_CMD_PROGRAM,
+ prg_len * BITS_PER_BYTE);
+ if (ret)
+ return ret;
+
+ // fetch read data
+ reg_offset = 0;
+ if (op->data.dir == SPI_MEM_DATA_IN) {
+ for (i = op->data.nbytes - 1; i >= 0; i--, reg_offset++) {
+ reg = sp->base + MTK_NOR_REG_SHIFT(reg_offset);
+ ((u8 *)(op->data.buf.in))[i] = readb(reg);
+ }
+ }
+
+ return 0;
+}
+
static int mtk_nor_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
{
struct mtk_nor *sp = spi_controller_get_devdata(mem->spi->master);
if ((op->data.nbytes == 0) ||
((op->addr.nbytes != 3) && (op->addr.nbytes != 4)))
- return -ENOTSUPP;
+ return mtk_nor_spi_mem_prg(sp, op);
if (op->data.dir == SPI_MEM_DATA_OUT) {
mtk_nor_set_addr(sp, op);
if (op->data.nbytes == 1) {
mtk_nor_set_addr(sp, op);
return mtk_nor_read_pio(sp, op);
- } else if (((ulong)(op->data.buf.in) &
- MTK_NOR_DMA_ALIGN_MASK)) {
- return mtk_nor_read_bounce(sp, op->addr.val,
- op->data.nbytes,
- op->data.buf.in);
} else {
- return mtk_nor_read_dma(sp, op->addr.val,
- op->data.nbytes,
- op->data.buf.in);
+ return mtk_nor_read_dma(sp, op);
}
}
- return -ENOTSUPP;
+ return mtk_nor_spi_mem_prg(sp, op);
}
static int mtk_nor_setup(struct spi_device *spi)
return 0;
}
-static int mtk_nor_init(struct mtk_nor *sp)
+static void mtk_nor_init(struct mtk_nor *sp)
{
- int ret;
-
- ret = mtk_nor_enable_clk(sp);
- if (ret)
- return ret;
-
- sp->spi_freq = clk_get_rate(sp->spi_clk);
+ writel(0, sp->base + MTK_NOR_REG_IRQ_EN);
+ writel(MTK_NOR_IRQ_MASK, sp->base + MTK_NOR_REG_IRQ_STAT);
writel(MTK_NOR_ENABLE_SF_CMD, sp->base + MTK_NOR_REG_WP);
mtk_nor_rmw(sp, MTK_NOR_REG_CFG2, MTK_NOR_WR_CUSTOM_OP_EN, 0);
mtk_nor_rmw(sp, MTK_NOR_REG_CFG3,
MTK_NOR_DISABLE_WREN | MTK_NOR_DISABLE_SR_POLL, 0);
-
- return ret;
}
static irqreturn_t mtk_nor_irq_handler(int irq, void *data)
};
static const struct of_device_id mtk_nor_match[] = {
- { .compatible = "mediatek,mt8173-nor" },
+ { .compatible = "mediatek,mt8192-nor", .data = (void *)36 },
+ { .compatible = "mediatek,mt8173-nor", .data = (void *)32 },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, mtk_nor_match);
struct spi_controller *ctlr;
struct mtk_nor *sp;
void __iomem *base;
- u8 *buffer;
struct clk *spi_clk, *ctlr_clk;
int ret, irq;
+ unsigned long dma_bits;
base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(base))
if (IS_ERR(ctlr_clk))
return PTR_ERR(ctlr_clk);
- buffer = devm_kmalloc(&pdev->dev,
- MTK_NOR_BOUNCE_BUF_SIZE + MTK_NOR_DMA_ALIGN,
- GFP_KERNEL);
- if (!buffer)
- return -ENOMEM;
-
- if ((ulong)buffer & MTK_NOR_DMA_ALIGN_MASK)
- buffer = (u8 *)(((ulong)buffer + MTK_NOR_DMA_ALIGN) &
- ~MTK_NOR_DMA_ALIGN_MASK);
+ dma_bits = (unsigned long)of_device_get_match_data(&pdev->dev);
+ if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(dma_bits))) {
+ dev_err(&pdev->dev, "failed to set dma mask(%lu)\n", dma_bits);
+ return -EINVAL;
+ }
ctlr = spi_alloc_master(&pdev->dev, sizeof(*sp));
if (!ctlr) {
ctlr->num_chipselect = 1;
ctlr->setup = mtk_nor_setup;
ctlr->transfer_one_message = mtk_nor_transfer_one_message;
+ ctlr->auto_runtime_pm = true;
dev_set_drvdata(&pdev->dev, ctlr);
sp = spi_controller_get_devdata(ctlr);
sp->base = base;
- sp->buffer = buffer;
sp->has_irq = false;
sp->wbuf_en = false;
sp->ctlr = ctlr;
sp->dev = &pdev->dev;
sp->spi_clk = spi_clk;
sp->ctlr_clk = ctlr_clk;
+ sp->high_dma = (dma_bits > 32);
+ sp->buffer = dmam_alloc_coherent(&pdev->dev,
+ MTK_NOR_BOUNCE_BUF_SIZE + MTK_NOR_DMA_ALIGN,
+ &sp->buffer_dma, GFP_KERNEL);
+ if (!sp->buffer)
+ return -ENOMEM;
+
+ if ((uintptr_t)sp->buffer & MTK_NOR_DMA_ALIGN_MASK) {
+ dev_err(sp->dev, "misaligned allocation of internal buffer.\n");
+ return -ENOMEM;
+ }
+
+ ret = mtk_nor_enable_clk(sp);
+ if (ret < 0)
+ return ret;
+
+ sp->spi_freq = clk_get_rate(sp->spi_clk);
+
+ mtk_nor_init(sp);
irq = platform_get_irq_optional(pdev, 0);
+
if (irq < 0) {
dev_warn(sp->dev, "IRQ not available.");
} else {
- writel(MTK_NOR_IRQ_MASK, base + MTK_NOR_REG_IRQ_STAT);
- writel(0, base + MTK_NOR_REG_IRQ_EN);
ret = devm_request_irq(sp->dev, irq, mtk_nor_irq_handler, 0,
pdev->name, sp);
if (ret < 0) {
}
}
- ret = mtk_nor_init(sp);
- if (ret < 0) {
- kfree(ctlr);
- return ret;
- }
+ pm_runtime_set_autosuspend_delay(&pdev->dev, -1);
+ pm_runtime_use_autosuspend(&pdev->dev);
+ pm_runtime_set_active(&pdev->dev);
+ pm_runtime_enable(&pdev->dev);
+ pm_runtime_get_noresume(&pdev->dev);
+
+ ret = devm_spi_register_controller(&pdev->dev, ctlr);
+ if (ret < 0)
+ goto err_probe;
+
+ pm_runtime_mark_last_busy(&pdev->dev);
+ pm_runtime_put_autosuspend(&pdev->dev);
dev_info(&pdev->dev, "spi frequency: %d Hz\n", sp->spi_freq);
- return devm_spi_register_controller(&pdev->dev, ctlr);
+ return 0;
+
+err_probe:
+ pm_runtime_disable(&pdev->dev);
+ pm_runtime_set_suspended(&pdev->dev);
+ pm_runtime_dont_use_autosuspend(&pdev->dev);
+
+ mtk_nor_disable_clk(sp);
+
+ return ret;
}
static int mtk_nor_remove(struct platform_device *pdev)
{
- struct spi_controller *ctlr;
- struct mtk_nor *sp;
+ struct spi_controller *ctlr = dev_get_drvdata(&pdev->dev);
+ struct mtk_nor *sp = spi_controller_get_devdata(ctlr);
- ctlr = dev_get_drvdata(&pdev->dev);
- sp = spi_controller_get_devdata(ctlr);
+ pm_runtime_disable(&pdev->dev);
+ pm_runtime_set_suspended(&pdev->dev);
+ pm_runtime_dont_use_autosuspend(&pdev->dev);
mtk_nor_disable_clk(sp);
return 0;
}
+static int __maybe_unused mtk_nor_runtime_suspend(struct device *dev)
+{
+ struct spi_controller *ctlr = dev_get_drvdata(dev);
+ struct mtk_nor *sp = spi_controller_get_devdata(ctlr);
+
+ mtk_nor_disable_clk(sp);
+
+ return 0;
+}
+
+static int __maybe_unused mtk_nor_runtime_resume(struct device *dev)
+{
+ struct spi_controller *ctlr = dev_get_drvdata(dev);
+ struct mtk_nor *sp = spi_controller_get_devdata(ctlr);
+
+ return mtk_nor_enable_clk(sp);
+}
+
+static int __maybe_unused mtk_nor_suspend(struct device *dev)
+{
+ return pm_runtime_force_suspend(dev);
+}
+
+static int __maybe_unused mtk_nor_resume(struct device *dev)
+{
+ return pm_runtime_force_resume(dev);
+}
+
+static const struct dev_pm_ops mtk_nor_pm_ops = {
+ SET_RUNTIME_PM_OPS(mtk_nor_runtime_suspend,
+ mtk_nor_runtime_resume, NULL)
+ SET_SYSTEM_SLEEP_PM_OPS(mtk_nor_suspend, mtk_nor_resume)
+};
+
static struct platform_driver mtk_nor_driver = {
.driver = {
.name = DRIVER_NAME,
.of_match_table = mtk_nor_match,
+ .pm = &mtk_nor_pm_ops,
},
.probe = mtk_nor_probe,
.remove = mtk_nor_remove,
priv->mux = devm_mux_control_get(&spi->dev, NULL);
if (IS_ERR(priv->mux)) {
- ret = PTR_ERR(priv->mux);
- if (ret != -EPROBE_DEFER)
- dev_err(&spi->dev, "failed to get control-mux\n");
+ ret = dev_err_probe(&spi->dev, PTR_ERR(priv->mux),
+ "failed to get control-mux\n");
goto err_put_ctlr;
}
struct npcm_fiu_spi *fiu;
void __iomem *regbase;
struct resource *res;
- int ret;
int id;
ctrl = spi_alloc_master(dev, sizeof(*fiu));
ctrl->num_chipselect = fiu->info->max_cs;
ctrl->dev.of_node = dev->of_node;
- ret = devm_spi_register_master(dev, ctrl);
- if (ret)
- return ret;
-
- return 0;
+ return devm_spi_register_master(dev, ctrl);
}
static int npcm_fiu_remove(struct platform_device *pdev)
/*
* NXP FlexSPI(FSPI) controller driver.
*
- * Copyright 2019 NXP.
+ * Copyright 2019-2020 NXP
+ * Copyright 2020 Puresoftware Ltd.
*
* FlexSPI is a flexsible SPI host controller which supports two SPI
* channels and up to 4 external devices. Each channel supports
* Frieder Schrempf <frieder.schrempf@kontron.de>
*/
+#include <linux/acpi.h>
#include <linux/bitops.h>
#include <linux/clk.h>
#include <linux/completion.h>
{
int ret;
+ if (is_acpi_node(f->dev->fwnode))
+ return 0;
+
ret = clk_prepare_enable(f->clk_en);
if (ret)
return ret;
return 0;
}
-static void nxp_fspi_clk_disable_unprep(struct nxp_fspi *f)
+static int nxp_fspi_clk_disable_unprep(struct nxp_fspi *f)
{
+ if (is_acpi_node(f->dev->fwnode))
+ return 0;
+
clk_disable_unprepare(f->clk);
clk_disable_unprepare(f->clk_en);
+
+ return 0;
}
/*
f = spi_controller_get_devdata(ctlr);
f->dev = dev;
- f->devtype_data = of_device_get_match_data(dev);
+ f->devtype_data = device_get_match_data(dev);
if (!f->devtype_data) {
ret = -ENODEV;
goto err_put_ctrl;
platform_set_drvdata(pdev, f);
/* find the resources - configuration register address space */
- res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fspi_base");
+ if (is_acpi_node(f->dev->fwnode))
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ else
+ res = platform_get_resource_byname(pdev,
+ IORESOURCE_MEM, "fspi_base");
+
f->iobase = devm_ioremap_resource(dev, res);
if (IS_ERR(f->iobase)) {
ret = PTR_ERR(f->iobase);
}
/* find the resources - controller memory mapped space */
- res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fspi_mmap");
+ if (is_acpi_node(f->dev->fwnode))
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ else
+ res = platform_get_resource_byname(pdev,
+ IORESOURCE_MEM, "fspi_mmap");
+
if (!res) {
ret = -ENODEV;
goto err_put_ctrl;
f->memmap_phy_size = resource_size(res);
/* find the clocks */
- f->clk_en = devm_clk_get(dev, "fspi_en");
- if (IS_ERR(f->clk_en)) {
- ret = PTR_ERR(f->clk_en);
- goto err_put_ctrl;
- }
+ if (dev_of_node(&pdev->dev)) {
+ f->clk_en = devm_clk_get(dev, "fspi_en");
+ if (IS_ERR(f->clk_en)) {
+ ret = PTR_ERR(f->clk_en);
+ goto err_put_ctrl;
+ }
- f->clk = devm_clk_get(dev, "fspi");
- if (IS_ERR(f->clk)) {
- ret = PTR_ERR(f->clk);
- goto err_put_ctrl;
- }
+ f->clk = devm_clk_get(dev, "fspi");
+ if (IS_ERR(f->clk)) {
+ ret = PTR_ERR(f->clk);
+ goto err_put_ctrl;
+ }
- ret = nxp_fspi_clk_prep_enable(f);
- if (ret) {
- dev_err(dev, "can not enable the clock\n");
- goto err_put_ctrl;
+ ret = nxp_fspi_clk_prep_enable(f);
+ if (ret) {
+ dev_err(dev, "can not enable the clock\n");
+ goto err_put_ctrl;
+ }
}
/* find the irq */
};
MODULE_DEVICE_TABLE(of, nxp_fspi_dt_ids);
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id nxp_fspi_acpi_ids[] = {
+ { "NXP0009", .driver_data = (kernel_ulong_t)&lx2160a_data, },
+ {}
+};
+MODULE_DEVICE_TABLE(acpi, nxp_fspi_acpi_ids);
+#endif
+
static const struct dev_pm_ops nxp_fspi_pm_ops = {
.suspend = nxp_fspi_suspend,
.resume = nxp_fspi_resume,
.driver = {
.name = "nxp-fspi",
.of_match_table = nxp_fspi_dt_ids,
+ .acpi_match_table = ACPI_PTR(nxp_fspi_acpi_ids),
.pm = &nxp_fspi_pm_ops,
},
.probe = nxp_fspi_probe,
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/gcd.h>
-#include <linux/iopoll.h>
#include <linux/spi/spi.h>
static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit)
{
- u32 val;
-
- return readl_poll_timeout(reg, val, val & bit, 1, MSEC_PER_SEC);
+ unsigned long timeout;
+
+ timeout = jiffies + msecs_to_jiffies(1000);
+ while (!(readl_relaxed(reg) & bit)) {
+ if (time_after(jiffies, timeout)) {
+ if (!(readl_relaxed(reg) & bit))
+ return -ETIMEDOUT;
+ else
+ return 0;
+ }
+ cpu_relax();
+ }
+ return 0;
}
static int mcspi_wait_for_completion(struct omap2_mcspi *mcspi,
struct qspi_xfer xfer;
struct icc_path *icc_path_cpu_to_qspi;
struct opp_table *opp_table;
- bool has_opp_table;
unsigned long last_speed;
/* Lock to protect data accessed by IRQs */
spinlock_t lock;
u32 int_status;
struct qcom_qspi *ctrl = dev_id;
irqreturn_t ret = IRQ_NONE;
- unsigned long flags;
- spin_lock_irqsave(&ctrl->lock, flags);
+ spin_lock(&ctrl->lock);
int_status = readl(ctrl->base + MSTR_INT_STATUS);
writel(int_status, ctrl->base + MSTR_INT_STATUS);
spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));
}
- spin_unlock_irqrestore(&ctrl->lock, flags);
+ spin_unlock(&ctrl->lock);
return ret;
}
ctrl->icc_path_cpu_to_qspi = devm_of_icc_get(dev, "qspi-config");
if (IS_ERR(ctrl->icc_path_cpu_to_qspi)) {
- ret = PTR_ERR(ctrl->icc_path_cpu_to_qspi);
- if (ret != -EPROBE_DEFER)
- dev_err(dev, "Failed to get cpu path: %d\n", ret);
+ ret = dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi),
+ "Failed to get cpu path\n");
goto exit_probe_master_put;
}
/* Set BW vote for register access */
}
/* OPP table is optional */
ret = dev_pm_opp_of_add_table(&pdev->dev);
- if (!ret) {
- ctrl->has_opp_table = true;
- } else if (ret != -ENODEV) {
+ if (ret && ret != -ENODEV) {
dev_err(&pdev->dev, "invalid OPP table in device tree\n");
- goto exit_probe_master_put;
+ goto exit_probe_put_clkname;
}
pm_runtime_use_autosuspend(dev);
return 0;
pm_runtime_disable(dev);
- if (ctrl->has_opp_table)
- dev_pm_opp_of_remove_table(&pdev->dev);
+ dev_pm_opp_of_remove_table(&pdev->dev);
+
+exit_probe_put_clkname:
dev_pm_opp_put_clkname(ctrl->opp_table);
exit_probe_master_put:
spi_unregister_master(master);
pm_runtime_disable(&pdev->dev);
- if (ctrl->has_opp_table)
- dev_pm_opp_of_remove_table(&pdev->dev);
+ dev_pm_opp_of_remove_table(&pdev->dev);
dev_pm_opp_put_clkname(ctrl->opp_table);
return 0;
{
struct spi_qup *controller = spi_master_get_devdata(master);
unsigned long timeout, flags;
- int ret = -EIO;
+ int ret;
ret = spi_qup_io_prep(spi, xfer);
if (ret)
#define SPCMD_SPRW 0x0010 /* SPI Read/Write Access (Dual/Quad) */
#define SPCMD_SSLA(i) ((i) << 4) /* SSL Assert Signal Setting */
#define SPCMD_BRDV_MASK 0x000c /* Bit Rate Division Setting */
+#define SPCMD_BRDV(brdv) ((brdv) << 2)
#define SPCMD_CPOL 0x0002 /* Clock Polarity Setting */
#define SPCMD_CPHA 0x0001 /* Clock Phase Setting */
int (*transfer_one)(struct spi_controller *ctlr,
struct spi_device *spi, struct spi_transfer *xfer);
u16 extra_mode_bits;
+ u16 min_div;
+ u16 max_div;
u16 flags;
u16 fifo_size;
u8 num_hw_ss;
};
+static void rspi_set_rate(struct rspi_data *rspi)
+{
+ unsigned long clksrc;
+ int brdv = 0, spbr;
+
+ clksrc = clk_get_rate(rspi->clk);
+ spbr = DIV_ROUND_UP(clksrc, 2 * rspi->speed_hz) - 1;
+ while (spbr > 255 && brdv < 3) {
+ brdv++;
+ spbr = DIV_ROUND_UP(spbr + 1, 2) - 1;
+ }
+
+ rspi_write8(rspi, clamp(spbr, 0, 255), RSPI_SPBR);
+ rspi->spcmd |= SPCMD_BRDV(brdv);
+ rspi->speed_hz = DIV_ROUND_UP(clksrc, (2U << brdv) * (spbr + 1));
+}
+
/*
* functions for RSPI on legacy SH
*/
static int rspi_set_config_register(struct rspi_data *rspi, int access_size)
{
- int spbr;
-
/* Sets output mode, MOSI signal, and (optionally) loopback */
rspi_write8(rspi, rspi->sppcr, RSPI_SPPCR);
/* Sets transfer bit rate */
- spbr = DIV_ROUND_UP(clk_get_rate(rspi->clk), 2 * rspi->speed_hz) - 1;
- rspi_write8(rspi, clamp(spbr, 0, 255), RSPI_SPBR);
+ rspi_set_rate(rspi);
/* Disable dummy transmission, set 16-bit word access, 1 frame */
rspi_write8(rspi, 0, RSPI_SPDCR);
*/
static int rspi_rz_set_config_register(struct rspi_data *rspi, int access_size)
{
- int spbr;
- int div = 0;
- unsigned long clksrc;
-
/* Sets output mode, MOSI signal, and (optionally) loopback */
rspi_write8(rspi, rspi->sppcr, RSPI_SPPCR);
- clksrc = clk_get_rate(rspi->clk);
- while (div < 3) {
- if (rspi->speed_hz >= clksrc/4) /* 4=(CLK/2)/2 */
- break;
- div++;
- clksrc /= 2;
- }
-
/* Sets transfer bit rate */
- spbr = DIV_ROUND_UP(clksrc, 2 * rspi->speed_hz) - 1;
- rspi_write8(rspi, clamp(spbr, 0, 255), RSPI_SPBR);
- rspi->spcmd |= div << 2;
+ rspi_set_rate(rspi);
/* Disable dummy transmission, set byte access */
rspi_write8(rspi, SPDCR_SPLBYTE, RSPI_SPDCR);
*/
static int qspi_set_config_register(struct rspi_data *rspi, int access_size)
{
- int spbr;
+ unsigned long clksrc;
+ int brdv = 0, spbr;
/* Sets output mode, MOSI signal, and (optionally) loopback */
rspi_write8(rspi, rspi->sppcr, RSPI_SPPCR);
/* Sets transfer bit rate */
- spbr = DIV_ROUND_UP(clk_get_rate(rspi->clk), 2 * rspi->speed_hz);
- rspi_write8(rspi, clamp(spbr, 0, 255), RSPI_SPBR);
+ clksrc = clk_get_rate(rspi->clk);
+ if (rspi->speed_hz >= clksrc) {
+ spbr = 0;
+ rspi->speed_hz = clksrc;
+ } else {
+ spbr = DIV_ROUND_UP(clksrc, 2 * rspi->speed_hz);
+ while (spbr > 255 && brdv < 3) {
+ brdv++;
+ spbr = DIV_ROUND_UP(spbr, 2);
+ }
+ spbr = clamp(spbr, 0, 255);
+ rspi->speed_hz = DIV_ROUND_UP(clksrc, (2U << brdv) * spbr);
+ }
+ rspi_write8(rspi, spbr, RSPI_SPBR);
+ rspi->spcmd |= SPCMD_BRDV(brdv);
/* Disable dummy transmission, set byte access */
rspi_write8(rspi, 0, RSPI_SPDCR);
{
int ret;
+ xfer->effective_speed_hz = rspi->speed_hz;
+
ret = rspi_dma_check_then_transfer(rspi, xfer);
if (ret != -EAGAIN)
return ret;
{
struct rspi_data *rspi = spi_controller_get_devdata(ctlr);
+ xfer->effective_speed_hz = rspi->speed_hz;
if (spi->mode & SPI_LOOP) {
return qspi_transfer_out_in(rspi, xfer);
} else if (xfer->tx_nbits > SPI_NBITS_SINGLE) {
static const struct spi_ops rspi_ops = {
.set_config_register = rspi_set_config_register,
.transfer_one = rspi_transfer_one,
+ .min_div = 2,
+ .max_div = 4096,
.flags = SPI_CONTROLLER_MUST_TX,
.fifo_size = 8,
.num_hw_ss = 2,
static const struct spi_ops rspi_rz_ops = {
.set_config_register = rspi_rz_set_config_register,
.transfer_one = rspi_rz_transfer_one,
+ .min_div = 2,
+ .max_div = 4096,
.flags = SPI_CONTROLLER_MUST_RX | SPI_CONTROLLER_MUST_TX,
.fifo_size = 8, /* 8 for TX, 32 for RX */
.num_hw_ss = 1,
.transfer_one = qspi_transfer_one,
.extra_mode_bits = SPI_TX_DUAL | SPI_TX_QUAD |
SPI_RX_DUAL | SPI_RX_QUAD,
+ .min_div = 1,
+ .max_div = 4080,
.flags = SPI_CONTROLLER_MUST_RX | SPI_CONTROLLER_MUST_TX,
.fifo_size = 32,
.num_hw_ss = 1,
int ret;
const struct rspi_plat_data *rspi_pd;
const struct spi_ops *ops;
+ unsigned long clksrc;
ctlr = spi_alloc_master(&pdev->dev, sizeof(struct rspi_data));
if (ctlr == NULL)
ctlr->num_chipselect = 2; /* default */
}
- /* ops parameter check */
- if (!ops->set_config_register) {
- dev_err(&pdev->dev, "there is no set_config_register\n");
- ret = -ENODEV;
- goto error1;
- }
-
rspi = spi_controller_get_devdata(ctlr);
platform_set_drvdata(pdev, rspi);
rspi->ops = ops;
ctlr->unprepare_message = rspi_unprepare_message;
ctlr->mode_bits = SPI_CPHA | SPI_CPOL | SPI_CS_HIGH | SPI_LSB_FIRST |
SPI_LOOP | ops->extra_mode_bits;
+ clksrc = clk_get_rate(rspi->clk);
+ ctlr->min_speed_hz = DIV_ROUND_UP(clksrc, ops->max_div);
+ ctlr->max_speed_hz = DIV_ROUND_UP(clksrc, ops->min_div);
ctlr->flags = ops->flags;
ctlr->dev.of_node = pdev->dev.of_node;
ctlr->use_gpio_descriptors = true;
#include "spi-s3c24xx-fiq.h"
/**
- * s3c24xx_spi_devstate - per device data
+ * struct s3c24xx_spi_devstate - per device data
* @hz: Last frequency calculated for @sppre field.
* @mode: Last mode setting for the @spcon field.
* @spcon: Value to write to the SPCON register.
#define S3C64XX_SPI_CH_CFG 0x00
#define S3C64XX_SPI_CLK_CFG 0x04
#define S3C64XX_SPI_MODE_CFG 0x08
-#define S3C64XX_SPI_SLAVE_SEL 0x0C
+#define S3C64XX_SPI_CS_REG 0x0C
#define S3C64XX_SPI_INT_EN 0x10
#define S3C64XX_SPI_STATUS 0x14
#define S3C64XX_SPI_TX_DATA 0x18
#define S3C64XX_SPI_MODE_TXDMA_ON (1<<1)
#define S3C64XX_SPI_MODE_4BURST (1<<0)
-#define S3C64XX_SPI_SLAVE_AUTO (1<<1)
-#define S3C64XX_SPI_SLAVE_SIG_INACT (1<<0)
-#define S3C64XX_SPI_SLAVE_NSC_CNT_2 (2<<4)
+#define S3C64XX_SPI_CS_NSC_CNT_2 (2<<4)
+#define S3C64XX_SPI_CS_AUTO (1<<1)
+#define S3C64XX_SPI_CS_SIG_INACT (1<<0)
#define S3C64XX_SPI_INT_TRAILING_EN (1<<6)
#define S3C64XX_SPI_INT_RX_OVERRUN_EN (1<<5)
struct s3c64xx_spi_dma_data {
struct dma_chan *ch;
+ dma_cookie_t cookie;
enum dma_transfer_direction direction;
};
* @cntrlr_info: Platform specific data for the controller this driver manages.
* @lock: Controller specific lock.
* @state: Set of FLAGS to indicate status.
- * @rx_dmach: Controller's DMA channel for Rx.
- * @tx_dmach: Controller's DMA channel for Tx.
* @sfr_start: BUS address of SPI controller regs.
* @regs: Pointer to ioremap'ed controller registers.
- * @irq: interrupt
* @xfer_completion: To indicate completion of xfer task.
* @cur_mode: Stores the active configuration of the controller.
* @cur_bpw: Stores the active bits per word settings.
struct clk *ioclk;
struct platform_device *pdev;
struct spi_master *master;
- struct s3c64xx_spi_info *cntrlr_info;
+ struct s3c64xx_spi_info *cntrlr_info;
spinlock_t lock;
unsigned long sfr_start;
struct completion xfer_completion;
spin_unlock_irqrestore(&sdd->lock, flags);
}
-static void prepare_dma(struct s3c64xx_spi_dma_data *dma,
+static int prepare_dma(struct s3c64xx_spi_dma_data *dma,
struct sg_table *sgt)
{
struct s3c64xx_spi_driver_data *sdd;
struct dma_slave_config config;
struct dma_async_tx_descriptor *desc;
+ int ret;
memset(&config, 0, sizeof(config));
desc = dmaengine_prep_slave_sg(dma->ch, sgt->sgl, sgt->nents,
dma->direction, DMA_PREP_INTERRUPT);
+ if (!desc) {
+ dev_err(&sdd->pdev->dev, "unable to prepare %s scatterlist",
+ dma->direction == DMA_DEV_TO_MEM ? "rx" : "tx");
+ return -ENOMEM;
+ }
desc->callback = s3c64xx_spi_dmacb;
desc->callback_param = dma;
- dmaengine_submit(desc);
+ dma->cookie = dmaengine_submit(desc);
+ ret = dma_submit_error(dma->cookie);
+ if (ret) {
+ dev_err(&sdd->pdev->dev, "DMA submission failed");
+ return -EIO;
+ }
+
dma_async_issue_pending(dma->ch);
+ return 0;
}
static void s3c64xx_spi_set_cs(struct spi_device *spi, bool enable)
if (enable) {
if (!(sdd->port_conf->quirks & S3C64XX_SPI_QUIRK_CS_AUTO)) {
- writel(0, sdd->regs + S3C64XX_SPI_SLAVE_SEL);
+ writel(0, sdd->regs + S3C64XX_SPI_CS_REG);
} else {
- u32 ssel = readl(sdd->regs + S3C64XX_SPI_SLAVE_SEL);
+ u32 ssel = readl(sdd->regs + S3C64XX_SPI_CS_REG);
- ssel |= (S3C64XX_SPI_SLAVE_AUTO |
- S3C64XX_SPI_SLAVE_NSC_CNT_2);
- writel(ssel, sdd->regs + S3C64XX_SPI_SLAVE_SEL);
+ ssel |= (S3C64XX_SPI_CS_AUTO |
+ S3C64XX_SPI_CS_NSC_CNT_2);
+ writel(ssel, sdd->regs + S3C64XX_SPI_CS_REG);
}
} else {
if (!(sdd->port_conf->quirks & S3C64XX_SPI_QUIRK_CS_AUTO))
- writel(S3C64XX_SPI_SLAVE_SIG_INACT,
- sdd->regs + S3C64XX_SPI_SLAVE_SEL);
+ writel(S3C64XX_SPI_CS_SIG_INACT,
+ sdd->regs + S3C64XX_SPI_CS_REG);
}
}
return xfer->len > (FIFO_LVL_MASK(sdd) >> 1) + 1;
}
-static void s3c64xx_enable_datapath(struct s3c64xx_spi_driver_data *sdd,
+static int s3c64xx_enable_datapath(struct s3c64xx_spi_driver_data *sdd,
struct spi_transfer *xfer, int dma_mode)
{
void __iomem *regs = sdd->regs;
u32 modecfg, chcfg;
+ int ret = 0;
modecfg = readl(regs + S3C64XX_SPI_MODE_CFG);
modecfg &= ~(S3C64XX_SPI_MODE_TXDMA_ON | S3C64XX_SPI_MODE_RXDMA_ON);
chcfg |= S3C64XX_SPI_CH_TXCH_ON;
if (dma_mode) {
modecfg |= S3C64XX_SPI_MODE_TXDMA_ON;
- prepare_dma(&sdd->tx_dma, &xfer->tx_sg);
+ ret = prepare_dma(&sdd->tx_dma, &xfer->tx_sg);
} else {
switch (sdd->cur_bpw) {
case 32:
writel(((xfer->len * 8 / sdd->cur_bpw) & 0xffff)
| S3C64XX_SPI_PACKET_CNT_EN,
regs + S3C64XX_SPI_PACKET_CNT);
- prepare_dma(&sdd->rx_dma, &xfer->rx_sg);
+ ret = prepare_dma(&sdd->rx_dma, &xfer->rx_sg);
}
}
+ if (ret)
+ return ret;
+
writel(modecfg, regs + S3C64XX_SPI_MODE_CFG);
writel(chcfg, regs + S3C64XX_SPI_CH_CFG);
+
+ return 0;
}
static u32 s3c64xx_spi_wait_for_timeout(struct s3c64xx_spi_driver_data *sdd,
/* millisecs to xfer 'len' bytes @ 'cur_speed' */
ms = xfer->len * 8 * 1000 / sdd->cur_speed;
- ms += 10; /* some tolerance */
+ ms += 30; /* some tolerance */
+ ms = max(ms, 100); /* minimum timeout */
val = msecs_to_jiffies(ms) + 10;
val = wait_for_completion_timeout(&sdd->xfer_completion, val);
return 0;
}
-static void s3c64xx_spi_config(struct s3c64xx_spi_driver_data *sdd)
+static int s3c64xx_spi_config(struct s3c64xx_spi_driver_data *sdd)
{
void __iomem *regs = sdd->regs;
+ int ret;
u32 val;
/* Disable Clock */
if (sdd->port_conf->clk_from_cmu) {
/* The src_clk clock is divided internally by 2 */
- clk_set_rate(sdd->src_clk, sdd->cur_speed * 2);
+ ret = clk_set_rate(sdd->src_clk, sdd->cur_speed * 2);
+ if (ret)
+ return ret;
+ sdd->cur_speed = clk_get_rate(sdd->src_clk) / 2;
} else {
/* Configure Clock */
val = readl(regs + S3C64XX_SPI_CLK_CFG);
val |= S3C64XX_SPI_ENCLK_ENABLE;
writel(val, regs + S3C64XX_SPI_CLK_CFG);
}
+
+ return 0;
}
#define XFER_DMAADDR_INVALID DMA_BIT_MASK(32)
sdd->cur_bpw = bpw;
sdd->cur_speed = speed;
sdd->cur_mode = spi->mode;
- s3c64xx_spi_config(sdd);
+ status = s3c64xx_spi_config(sdd);
+ if (status)
+ return status;
}
if (!is_polling(sdd) && (xfer->len > fifo_len) &&
sdd->state &= ~RXBUSY;
sdd->state &= ~TXBUSY;
- s3c64xx_enable_datapath(sdd, xfer, use_dma);
-
/* Start the signals */
s3c64xx_spi_set_cs(spi, true);
+ status = s3c64xx_enable_datapath(sdd, xfer, use_dma);
+
spin_unlock_irqrestore(&sdd->lock, flags);
+ if (status) {
+ dev_err(&spi->dev, "failed to enable data path for transfer: %d\n", status);
+ break;
+ }
+
if (use_dma)
status = s3c64xx_wait_for_dma(sdd, xfer);
else
if (status) {
dev_err(&spi->dev,
- "I/O Error: rx-%d tx-%d res:rx-%c tx-%c len-%d\n",
+ "I/O Error: rx-%d tx-%d rx-%c tx-%c len-%d dma-%d res-(%d)\n",
xfer->rx_buf ? 1 : 0, xfer->tx_buf ? 1 : 0,
(sdd->state & RXBUSY) ? 'f' : 'p',
(sdd->state & TXBUSY) ? 'f' : 'p',
- xfer->len);
+ xfer->len, use_dma ? 1 : 0, status);
if (use_dma) {
- if (xfer->tx_buf && (sdd->state & TXBUSY))
+ struct dma_tx_state s;
+
+ if (xfer->tx_buf && (sdd->state & TXBUSY)) {
+ dmaengine_pause(sdd->tx_dma.ch);
+ dmaengine_tx_status(sdd->tx_dma.ch, sdd->tx_dma.cookie, &s);
dmaengine_terminate_all(sdd->tx_dma.ch);
- if (xfer->rx_buf && (sdd->state & RXBUSY))
+ dev_err(&spi->dev, "TX residue: %d\n", s.residue);
+
+ }
+ if (xfer->rx_buf && (sdd->state & RXBUSY)) {
+ dmaengine_pause(sdd->rx_dma.ch);
+ dmaengine_tx_status(sdd->rx_dma.ch, sdd->rx_dma.cookie, &s);
dmaengine_terminate_all(sdd->rx_dma.ch);
+ dev_err(&spi->dev, "RX residue: %d\n", s.residue);
+ }
}
} else {
s3c64xx_flush_fifo(sdd);
sdd->cur_speed = 0;
if (sci->no_cs)
- writel(0, sdd->regs + S3C64XX_SPI_SLAVE_SEL);
+ writel(0, sdd->regs + S3C64XX_SPI_CS_REG);
else if (!(sdd->port_conf->quirks & S3C64XX_SPI_QUIRK_CS_AUTO))
- writel(S3C64XX_SPI_SLAVE_SIG_INACT, sdd->regs + S3C64XX_SPI_SLAVE_SEL);
+ writel(S3C64XX_SPI_CS_SIG_INACT, sdd->regs + S3C64XX_SPI_CS_REG);
/* Disable Interrupts - we use Polling if not DMA mode */
writel(0, regs + S3C64XX_SPI_INT_EN);
s3c64xx_spi_hwinit(sdd);
+ writel(S3C64XX_SPI_INT_RX_OVERRUN_EN | S3C64XX_SPI_INT_RX_UNDERRUN_EN |
+ S3C64XX_SPI_INT_TX_OVERRUN_EN | S3C64XX_SPI_INT_TX_UNDERRUN_EN,
+ sdd->regs + S3C64XX_SPI_INT_EN);
+
return 0;
err_disable_src_clk:
.tx_st_done = 25,
.high_speed = true,
.clk_from_cmu = true,
+ .quirks = S3C64XX_SPI_QUIRK_CS_AUTO,
};
static struct s3c64xx_spi_port_config exynos7_spi_port_config = {
dev_info(&pdev->dev, "no hardware spinlock supplied\n");
break;
default:
- dev_err(&pdev->dev,
- "failed to find hwlock id, %d\n", ret);
- fallthrough;
- case -EPROBE_DEFER:
+ dev_err_probe(&pdev->dev, ret, "failed to find hwlock id\n");
goto put_ctlr;
}
}
static int sprd_spi_dma_request(struct sprd_spi *ss)
{
ss->dma.dma_chan[SPRD_SPI_RX] = dma_request_chan(ss->dev, "rx_chn");
- if (IS_ERR_OR_NULL(ss->dma.dma_chan[SPRD_SPI_RX])) {
- if (PTR_ERR(ss->dma.dma_chan[SPRD_SPI_RX]) == -EPROBE_DEFER)
- return PTR_ERR(ss->dma.dma_chan[SPRD_SPI_RX]);
-
- dev_err(ss->dev, "request RX DMA channel failed!\n");
- return PTR_ERR(ss->dma.dma_chan[SPRD_SPI_RX]);
- }
+ if (IS_ERR_OR_NULL(ss->dma.dma_chan[SPRD_SPI_RX]))
+ return dev_err_probe(ss->dev, PTR_ERR(ss->dma.dma_chan[SPRD_SPI_RX]),
+ "request RX DMA channel failed!\n");
ss->dma.dma_chan[SPRD_SPI_TX] = dma_request_chan(ss->dev, "tx_chn");
if (IS_ERR_OR_NULL(ss->dma.dma_chan[SPRD_SPI_TX])) {
- if (PTR_ERR(ss->dma.dma_chan[SPRD_SPI_TX]) == -EPROBE_DEFER)
- return PTR_ERR(ss->dma.dma_chan[SPRD_SPI_TX]);
-
- dev_err(ss->dev, "request TX DMA channel failed!\n");
dma_release_channel(ss->dma.dma_chan[SPRD_SPI_RX]);
- return PTR_ERR(ss->dma.dma_chan[SPRD_SPI_TX]);
+ return dev_err_probe(ss->dev, PTR_ERR(ss->dma.dma_chan[SPRD_SPI_TX]),
+ "request TX DMA channel failed!\n");
}
return 0;
struct spi_master *master = dev_id;
struct stm32_spi *spi = spi_master_get_devdata(master);
u32 sr, mask = 0;
- unsigned long flags;
bool end = false;
- spin_lock_irqsave(&spi->lock, flags);
+ spin_lock(&spi->lock);
sr = readl_relaxed(spi->base + STM32F4_SPI_SR);
/*
if (!(sr & mask)) {
dev_dbg(spi->dev, "spurious IT (sr=0x%08x)\n", sr);
- spin_unlock_irqrestore(&spi->lock, flags);
+ spin_unlock(&spi->lock);
return IRQ_NONE;
}
STM32F4_SPI_CR2_TXEIE |
STM32F4_SPI_CR2_RXNEIE |
STM32F4_SPI_CR2_ERRIE);
- spin_unlock_irqrestore(&spi->lock, flags);
+ spin_unlock(&spi->lock);
return IRQ_WAKE_THREAD;
}
- spin_unlock_irqrestore(&spi->lock, flags);
+ spin_unlock(&spi->lock);
return IRQ_HANDLED;
}
spi->irq = platform_get_irq(pdev, 0);
if (spi->irq <= 0) {
- ret = spi->irq;
- if (ret != -EPROBE_DEFER)
- dev_err(&pdev->dev, "failed to get irq: %d\n", ret);
+ ret = dev_err_probe(&pdev->dev, spi->irq, "failed to get irq\n");
goto err_master_put;
}
ret = devm_request_threaded_irq(&pdev->dev, spi->irq,
}
if (IS_ERR(sspi->clk)) {
- if (!(PTR_ERR(sspi->clk) == -EPROBE_DEFER))
- dev_err(&pdev->dev, "clock not found\n");
- ret = PTR_ERR(sspi->clk);
+ ret = dev_err_probe(&pdev->dev, PTR_ERR(sspi->clk),
+ "clock not found\n");
goto put_spi;
}
struct dma_chan *dma_chan;
u32 *dma_buf;
dma_addr_t dma_phys;
- int ret;
dma_chan = dma_request_chan(tspi->dev, dma_to_memory ? "rx" : "tx");
- if (IS_ERR(dma_chan)) {
- ret = PTR_ERR(dma_chan);
- if (ret != -EPROBE_DEFER)
- dev_err(tspi->dev,
- "Dma channel is not available: %d\n", ret);
- return ret;
- }
+ if (IS_ERR(dma_chan))
+ return dev_err_probe(tspi->dev, PTR_ERR(dma_chan),
+ "Dma channel is not available\n");
dma_buf = dma_alloc_coherent(tspi->dev, tspi->dma_buf_size,
&dma_phys, GFP_KERNEL);
static irqreturn_t handle_cpu_based_xfer(struct tegra_sflash_data *tsd)
{
struct spi_transfer *t = tsd->curr_xfer;
- unsigned long flags;
- spin_lock_irqsave(&tsd->lock, flags);
+ spin_lock(&tsd->lock);
if (tsd->tx_status || tsd->rx_status || (tsd->status_reg & SPI_BSY)) {
dev_err(tsd->dev,
"CpuXfer ERROR bit set 0x%x\n", tsd->status_reg);
tegra_sflash_calculate_curr_xfer_param(tsd->cur_spi, tsd, t);
tegra_sflash_start_cpu_based_transfer(tsd, t);
exit:
- spin_unlock_irqrestore(&tsd->lock, flags);
+ spin_unlock(&tsd->lock);
return IRQ_HANDLED;
}
struct dma_slave_config dma_sconfig;
dma_chan = dma_request_chan(tspi->dev, dma_to_memory ? "rx" : "tx");
- if (IS_ERR(dma_chan)) {
- ret = PTR_ERR(dma_chan);
- if (ret != -EPROBE_DEFER)
- dev_err(tspi->dev,
- "Dma channel is not available: %d\n", ret);
- return ret;
- }
+ if (IS_ERR(dma_chan))
+ return dev_err_probe(tspi->dev, PTR_ERR(dma_chan),
+ "Dma channel is not available\n");
dma_buf = dma_alloc_coherent(tspi->dev, tspi->dma_buf_size,
&dma_phys, GFP_KERNEL);
spin_unlock_irqrestore(&data->lock, flags);
/* RX */
- dma->sg_rx_p = kcalloc(num, sizeof(*dma->sg_rx_p), GFP_ATOMIC);
+ dma->sg_rx_p = kmalloc_array(num, sizeof(*dma->sg_rx_p), GFP_ATOMIC);
if (!dma->sg_rx_p)
return;
head = 0;
}
- dma->sg_tx_p = kcalloc(num, sizeof(*dma->sg_tx_p), GFP_ATOMIC);
+ dma->sg_tx_p = kmalloc_array(num, sizeof(*dma->sg_tx_p), GFP_ATOMIC);
if (!dma->sg_tx_p)
return;
goto put_master;
}
- dev_info(&pdev->dev, "at 0x%08llX mapped to 0x%p, irq=%d\n",
- (unsigned long long)res->start, xspi->regs, xspi->irq);
+ dev_info(&pdev->dev, "at %pR, irq=%d\n", res, xspi->irq);
if (pdata) {
for (i = 0; i < pdata->num_devices; i++)
#include <linux/spi/spi.h>
#include <linux/spinlock.h>
#include <linux/workqueue.h>
+#include <linux/spi/spi-mem.h>
/* Generic QSPI register offsets */
#define GQSPI_CONFIG_OFST 0x00000100
* @dma_addr: DMA address after mapping the kernel buffer
* @genfifoentry: Used for storing the genfifoentry instruction.
* @mode: Defines the mode in which QSPI is operating
+ * @data_completion: completion structure
*/
struct zynqmp_qspi {
void __iomem *regs;
dma_addr_t dma_addr;
u32 genfifoentry;
enum mode_type mode;
+ struct completion data_completion;
};
/**
- * zynqmp_gqspi_read: For GQSPI controller read operation
+ * zynqmp_gqspi_read - For GQSPI controller read operation
* @xqspi: Pointer to the zynqmp_qspi structure
* @offset: Offset from where to read
+ * Return: Value at the offset
*/
static u32 zynqmp_gqspi_read(struct zynqmp_qspi *xqspi, u32 offset)
{
}
/**
- * zynqmp_gqspi_write: For GQSPI controller write operation
+ * zynqmp_gqspi_write - For GQSPI controller write operation
* @xqspi: Pointer to the zynqmp_qspi structure
* @offset: Offset where to write
* @val: Value to be written
}
/**
- * zynqmp_gqspi_selectslave: For selection of slave device
+ * zynqmp_gqspi_selectslave - For selection of slave device
* @instanceptr: Pointer to the zynqmp_qspi structure
* @slavecs: For chip select
* @slavebus: To check which bus is selected- upper or lower
}
/**
- * zynqmp_qspi_init_hw: Initialize the hardware
+ * zynqmp_qspi_init_hw - Initialize the hardware
* @xqspi: Pointer to the zynqmp_qspi structure
*
* The default settings of the QSPI controller's configurable parameters on
GQSPI_SELECT_FLASH_BUS_LOWER);
/* Initialize DMA */
zynqmp_gqspi_write(xqspi,
- GQSPI_QSPIDMA_DST_CTRL_OFST,
- GQSPI_QSPIDMA_DST_CTRL_RESET_VAL);
+ GQSPI_QSPIDMA_DST_CTRL_OFST,
+ GQSPI_QSPIDMA_DST_CTRL_RESET_VAL);
/* Enable the GQSPI */
zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, GQSPI_EN_MASK);
}
/**
- * zynqmp_qspi_copy_read_data: Copy data to RX buffer
+ * zynqmp_qspi_copy_read_data - Copy data to RX buffer
* @xqspi: Pointer to the zynqmp_qspi structure
* @data: The variable where data is stored
* @size: Number of bytes to be copied from data to RX buffer
}
/**
- * zynqmp_prepare_transfer_hardware: Prepares hardware for transfer.
- * @master: Pointer to the spi_master structure which provides
- * information about the controller.
- *
- * This function enables SPI master controller.
- *
- * Return: 0 on success; error value otherwise
- */
-static int zynqmp_prepare_transfer_hardware(struct spi_master *master)
-{
- struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
-
- zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, GQSPI_EN_MASK);
- return 0;
-}
-
-/**
- * zynqmp_unprepare_transfer_hardware: Relaxes hardware after transfer
- * @master: Pointer to the spi_master structure which provides
- * information about the controller.
- *
- * This function disables the SPI master controller.
- *
- * Return: Always 0
- */
-static int zynqmp_unprepare_transfer_hardware(struct spi_master *master)
-{
- struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
-
- zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, 0x0);
- return 0;
-}
-
-/**
- * zynqmp_qspi_chipselect: Select or deselect the chip select line
+ * zynqmp_qspi_chipselect - Select or deselect the chip select line
* @qspi: Pointer to the spi_device structure
* @is_high: Select(0) or deselect (1) the chip select line
*/
{
struct zynqmp_qspi *xqspi = spi_master_get_devdata(qspi->master);
ulong timeout;
- u32 genfifoentry = 0x0, statusreg;
+ u32 genfifoentry = 0, statusreg;
genfifoentry |= GQSPI_GENFIFO_MODE_SPI;
- genfifoentry |= xqspi->genfifobus;
if (!is_high) {
+ xqspi->genfifobus = GQSPI_GENFIFO_BUS_LOWER;
+ xqspi->genfifocs = GQSPI_GENFIFO_CS_LOWER;
+ genfifoentry |= xqspi->genfifobus;
genfifoentry |= xqspi->genfifocs;
genfifoentry |= GQSPI_GENFIFO_CS_SETUP;
} else {
/* Manually start the generic FIFO command */
zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST,
- zynqmp_gqspi_read(xqspi, GQSPI_CONFIG_OFST) |
- GQSPI_CFG_START_GEN_FIFO_MASK);
+ zynqmp_gqspi_read(xqspi, GQSPI_CONFIG_OFST) |
+ GQSPI_CFG_START_GEN_FIFO_MASK);
timeout = jiffies + msecs_to_jiffies(1000);
statusreg = zynqmp_gqspi_read(xqspi, GQSPI_ISR_OFST);
if ((statusreg & GQSPI_ISR_GENFIFOEMPTY_MASK) &&
- (statusreg & GQSPI_ISR_TXEMPTY_MASK))
+ (statusreg & GQSPI_ISR_TXEMPTY_MASK))
break;
- else
- cpu_relax();
+ cpu_relax();
} while (!time_after_eq(jiffies, timeout));
if (time_after_eq(jiffies, timeout))
}
/**
- * zynqmp_qspi_setup_transfer: Configure QSPI controller for specified
+ * zynqmp_qspi_selectspimode - Selects SPI mode - x1 or x2 or x4.
+ * @xqspi: xqspi is a pointer to the GQSPI instance
+ * @spimode: spimode - SPI or DUAL or QUAD.
+ * Return: Mask to set desired SPI mode in GENFIFO entry.
+ */
+static inline u32 zynqmp_qspi_selectspimode(struct zynqmp_qspi *xqspi,
+ u8 spimode)
+{
+ u32 mask = 0;
+
+ switch (spimode) {
+ case GQSPI_SELECT_MODE_DUALSPI:
+ mask = GQSPI_GENFIFO_MODE_DUALSPI;
+ break;
+ case GQSPI_SELECT_MODE_QUADSPI:
+ mask = GQSPI_GENFIFO_MODE_QUADSPI;
+ break;
+ case GQSPI_SELECT_MODE_SPI:
+ mask = GQSPI_GENFIFO_MODE_SPI;
+ break;
+ default:
+ dev_warn(xqspi->dev, "Invalid SPI mode\n");
+ }
+
+ return mask;
+}
+
+/**
+ * zynqmp_qspi_config_op - Configure QSPI controller for specified
* transfer
+ * @xqspi: Pointer to the zynqmp_qspi structure
* @qspi: Pointer to the spi_device structure
- * @transfer: Pointer to the spi_transfer structure which provides
- * information about next transfer setup parameters
*
* Sets the operational mode of QSPI controller for the next QSPI transfer and
* sets the requested clock frequency.
* by the QSPI controller the driver will set the highest or lowest
* frequency supported by controller.
*/
-static int zynqmp_qspi_setup_transfer(struct spi_device *qspi,
- struct spi_transfer *transfer)
+static int zynqmp_qspi_config_op(struct zynqmp_qspi *xqspi,
+ struct spi_device *qspi)
{
- struct zynqmp_qspi *xqspi = spi_master_get_devdata(qspi->master);
ulong clk_rate;
- u32 config_reg, req_hz, baud_rate_val = 0;
-
- if (transfer)
- req_hz = transfer->speed_hz;
- else
- req_hz = qspi->max_speed_hz;
+ u32 config_reg, baud_rate_val = 0;
/* Set the clock frequency */
/* If req_hz == 0, default to lowest speed */
while ((baud_rate_val < GQSPI_BAUD_DIV_MAX) &&
(clk_rate /
- (GQSPI_BAUD_DIV_SHIFT << baud_rate_val)) > req_hz)
+ (GQSPI_BAUD_DIV_SHIFT << baud_rate_val)) > qspi->max_speed_hz)
baud_rate_val++;
config_reg = zynqmp_gqspi_read(xqspi, GQSPI_CONFIG_OFST);
}
/**
- * zynqmp_qspi_setup: Configure the QSPI controller
+ * zynqmp_qspi_setup_op - Configure the QSPI controller
* @qspi: Pointer to the spi_device structure
*
* Sets the operational mode of QSPI controller for the next QSPI transfer,
*
* Return: 0 on success; error value otherwise.
*/
-static int zynqmp_qspi_setup(struct spi_device *qspi)
+static int zynqmp_qspi_setup_op(struct spi_device *qspi)
{
- if (qspi->master->busy)
+ struct spi_controller *ctlr = qspi->master;
+ struct zynqmp_qspi *xqspi = spi_controller_get_devdata(ctlr);
+ struct device *dev = &ctlr->dev;
+ int ret;
+
+ if (ctlr->busy)
return -EBUSY;
+
+ ret = clk_enable(xqspi->refclk);
+ if (ret) {
+ dev_err(dev, "Cannot enable device clock.\n");
+ return ret;
+ }
+
+ ret = clk_enable(xqspi->pclk);
+ if (ret) {
+ dev_err(dev, "Cannot enable APB clock.\n");
+ clk_disable(xqspi->refclk);
+ return ret;
+ }
+ zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, GQSPI_EN_MASK);
+
return 0;
}
/**
- * zynqmp_qspi_filltxfifo: Fills the TX FIFO as long as there is room in
+ * zynqmp_qspi_filltxfifo - Fills the TX FIFO as long as there is room in
* the FIFO or the bytes required to be
* transmitted.
* @xqspi: Pointer to the zynqmp_qspi structure
}
/**
- * zynqmp_qspi_readrxfifo: Fills the RX FIFO as long as there is room in
+ * zynqmp_qspi_readrxfifo - Fills the RX FIFO as long as there is room in
* the FIFO.
* @xqspi: Pointer to the zynqmp_qspi structure
* @size: Number of bytes to be copied from RX buffer to RX FIFO
while ((count < size) && (xqspi->bytes_to_receive > 0)) {
if (xqspi->bytes_to_receive >= 4) {
- (*(u32 *) xqspi->rxbuf) =
+ (*(u32 *)xqspi->rxbuf) =
zynqmp_gqspi_read(xqspi, GQSPI_RXD_OFST);
xqspi->rxbuf += 4;
xqspi->bytes_to_receive -= 4;
}
/**
- * zynqmp_process_dma_irq: Handler for DMA done interrupt of QSPI
+ * zynqmp_qspi_fillgenfifo - Fills the GENFIFO.
+ * @xqspi: Pointer to the zynqmp_qspi structure
+ * @nbits: Transfer/Receive buswidth.
+ * @genfifoentry: Variable in which GENFIFO mask is saved
+ */
+static void zynqmp_qspi_fillgenfifo(struct zynqmp_qspi *xqspi, u8 nbits,
+ u32 genfifoentry)
+{
+ u32 transfer_len = 0;
+
+ if (xqspi->txbuf) {
+ genfifoentry &= ~GQSPI_GENFIFO_RX;
+ genfifoentry |= GQSPI_GENFIFO_DATA_XFER;
+ genfifoentry |= GQSPI_GENFIFO_TX;
+ transfer_len = xqspi->bytes_to_transfer;
+ } else {
+ genfifoentry &= ~GQSPI_GENFIFO_TX;
+ genfifoentry |= GQSPI_GENFIFO_DATA_XFER;
+ genfifoentry |= GQSPI_GENFIFO_RX;
+ if (xqspi->mode == GQSPI_MODE_DMA)
+ transfer_len = xqspi->dma_rx_bytes;
+ else
+ transfer_len = xqspi->bytes_to_receive;
+ }
+ genfifoentry |= zynqmp_qspi_selectspimode(xqspi, nbits);
+ xqspi->genfifoentry = genfifoentry;
+
+ if ((transfer_len) < GQSPI_GENFIFO_IMM_DATA_MASK) {
+ genfifoentry &= ~GQSPI_GENFIFO_IMM_DATA_MASK;
+ genfifoentry |= transfer_len;
+ zynqmp_gqspi_write(xqspi, GQSPI_GEN_FIFO_OFST, genfifoentry);
+ } else {
+ int tempcount = transfer_len;
+ u32 exponent = 8; /* 2^8 = 256 */
+ u8 imm_data = tempcount & 0xFF;
+
+ tempcount &= ~(tempcount & 0xFF);
+ /* Immediate entry */
+ if (tempcount != 0) {
+ /* Exponent entries */
+ genfifoentry |= GQSPI_GENFIFO_EXP;
+ while (tempcount != 0) {
+ if (tempcount & GQSPI_GENFIFO_EXP_START) {
+ genfifoentry &=
+ ~GQSPI_GENFIFO_IMM_DATA_MASK;
+ genfifoentry |= exponent;
+ zynqmp_gqspi_write(xqspi,
+ GQSPI_GEN_FIFO_OFST,
+ genfifoentry);
+ }
+ tempcount = tempcount >> 1;
+ exponent++;
+ }
+ }
+ if (imm_data != 0) {
+ genfifoentry &= ~GQSPI_GENFIFO_EXP;
+ genfifoentry &= ~GQSPI_GENFIFO_IMM_DATA_MASK;
+ genfifoentry |= (u8)(imm_data & 0xFF);
+ zynqmp_gqspi_write(xqspi, GQSPI_GEN_FIFO_OFST,
+ genfifoentry);
+ }
+ }
+ if (xqspi->mode == GQSPI_MODE_IO && xqspi->rxbuf) {
+ /* Dummy generic FIFO entry */
+ zynqmp_gqspi_write(xqspi, GQSPI_GEN_FIFO_OFST, 0x0);
+ }
+}
+
+/**
+ * zynqmp_process_dma_irq - Handler for DMA done interrupt of QSPI
* controller
* @xqspi: zynqmp_qspi instance pointer
*
u32 config_reg, genfifoentry;
dma_unmap_single(xqspi->dev, xqspi->dma_addr,
- xqspi->dma_rx_bytes, DMA_FROM_DEVICE);
+ xqspi->dma_rx_bytes, DMA_FROM_DEVICE);
xqspi->rxbuf += xqspi->dma_rx_bytes;
xqspi->bytes_to_receive -= xqspi->dma_rx_bytes;
xqspi->dma_rx_bytes = 0;
/* Disabling the DMA interrupts */
zynqmp_gqspi_write(xqspi, GQSPI_QSPIDMA_DST_I_DIS_OFST,
- GQSPI_QSPIDMA_DST_I_EN_DONE_MASK);
+ GQSPI_QSPIDMA_DST_I_EN_DONE_MASK);
if (xqspi->bytes_to_receive > 0) {
/* Switch to IO mode,for remaining bytes to receive */
/* Manual start */
zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST,
- (zynqmp_gqspi_read(xqspi, GQSPI_CONFIG_OFST) |
- GQSPI_CFG_START_GEN_FIFO_MASK));
+ (zynqmp_gqspi_read(xqspi,
+ GQSPI_CONFIG_OFST) |
+ GQSPI_CFG_START_GEN_FIFO_MASK));
/* Enable the RX interrupts for IO mode */
zynqmp_gqspi_write(xqspi, GQSPI_IER_OFST,
- GQSPI_IER_GENFIFOEMPTY_MASK |
- GQSPI_IER_RXNEMPTY_MASK |
- GQSPI_IER_RXEMPTY_MASK);
+ GQSPI_IER_GENFIFOEMPTY_MASK |
+ GQSPI_IER_RXNEMPTY_MASK |
+ GQSPI_IER_RXEMPTY_MASK);
}
}
/**
- * zynqmp_qspi_irq: Interrupt service routine of the QSPI controller
+ * zynqmp_qspi_irq - Interrupt service routine of the QSPI controller
* @irq: IRQ number
* @dev_id: Pointer to the xqspi structure
*
*/
static irqreturn_t zynqmp_qspi_irq(int irq, void *dev_id)
{
- struct spi_master *master = dev_id;
- struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
- int ret = IRQ_NONE;
+ struct zynqmp_qspi *xqspi = (struct zynqmp_qspi *)dev_id;
+ irqreturn_t ret = IRQ_NONE;
u32 status, mask, dma_status = 0;
status = zynqmp_gqspi_read(xqspi, GQSPI_ISR_OFST);
dma_status =
zynqmp_gqspi_read(xqspi, GQSPI_QSPIDMA_DST_I_STS_OFST);
zynqmp_gqspi_write(xqspi, GQSPI_QSPIDMA_DST_I_STS_OFST,
- dma_status);
+ dma_status);
}
if (mask & GQSPI_ISR_TXNOT_FULL_MASK) {
ret = IRQ_HANDLED;
}
- if ((xqspi->bytes_to_receive == 0) && (xqspi->bytes_to_transfer == 0)
- && ((status & GQSPI_IRQ_MASK) == GQSPI_IRQ_MASK)) {
+ if (xqspi->bytes_to_receive == 0 && xqspi->bytes_to_transfer == 0 &&
+ ((status & GQSPI_IRQ_MASK) == GQSPI_IRQ_MASK)) {
zynqmp_gqspi_write(xqspi, GQSPI_IDR_OFST, GQSPI_ISR_IDR_MASK);
- spi_finalize_current_transfer(master);
+ complete(&xqspi->data_completion);
ret = IRQ_HANDLED;
}
return ret;
}
/**
- * zynqmp_qspi_selectspimode: Selects SPI mode - x1 or x2 or x4.
- * @xqspi: xqspi is a pointer to the GQSPI instance
- * @spimode: spimode - SPI or DUAL or QUAD.
- * Return: Mask to set desired SPI mode in GENFIFO entry.
- */
-static inline u32 zynqmp_qspi_selectspimode(struct zynqmp_qspi *xqspi,
- u8 spimode)
-{
- u32 mask = 0;
-
- switch (spimode) {
- case GQSPI_SELECT_MODE_DUALSPI:
- mask = GQSPI_GENFIFO_MODE_DUALSPI;
- break;
- case GQSPI_SELECT_MODE_QUADSPI:
- mask = GQSPI_GENFIFO_MODE_QUADSPI;
- break;
- case GQSPI_SELECT_MODE_SPI:
- mask = GQSPI_GENFIFO_MODE_SPI;
- break;
- default:
- dev_warn(xqspi->dev, "Invalid SPI mode\n");
- }
-
- return mask;
-}
-
-/**
- * zynq_qspi_setuprxdma: This function sets up the RX DMA operation
+ * zynqmp_qspi_setuprxdma - This function sets up the RX DMA operation
* @xqspi: xqspi is a pointer to the GQSPI instance.
*/
-static void zynq_qspi_setuprxdma(struct zynqmp_qspi *xqspi)
+static void zynqmp_qspi_setuprxdma(struct zynqmp_qspi *xqspi)
{
u32 rx_bytes, rx_rem, config_reg;
dma_addr_t addr;
u64 dma_align = (u64)(uintptr_t)xqspi->rxbuf;
- if ((xqspi->bytes_to_receive < 8) ||
- ((dma_align & GQSPI_DMA_UNALIGN) != 0x0)) {
+ if (xqspi->bytes_to_receive < 8 ||
+ ((dma_align & GQSPI_DMA_UNALIGN) != 0x0)) {
/* Setting to IO mode */
config_reg = zynqmp_gqspi_read(xqspi, GQSPI_CONFIG_OFST);
config_reg &= ~GQSPI_CFG_MODE_EN_MASK;
rx_bytes = (xqspi->bytes_to_receive - rx_rem);
addr = dma_map_single(xqspi->dev, (void *)xqspi->rxbuf,
- rx_bytes, DMA_FROM_DEVICE);
+ rx_bytes, DMA_FROM_DEVICE);
if (dma_mapping_error(xqspi->dev, addr))
dev_err(xqspi->dev, "ERR:rxdma:memory not mapped\n");
xqspi->dma_rx_bytes = rx_bytes;
xqspi->dma_addr = addr;
zynqmp_gqspi_write(xqspi, GQSPI_QSPIDMA_DST_ADDR_OFST,
- (u32)(addr & 0xffffffff));
+ (u32)(addr & 0xffffffff));
addr = ((addr >> 16) >> 16);
zynqmp_gqspi_write(xqspi, GQSPI_QSPIDMA_DST_ADDR_MSB_OFST,
- ((u32)addr) & 0xfff);
+ ((u32)addr) & 0xfff);
/* Enabling the DMA mode */
config_reg = zynqmp_gqspi_read(xqspi, GQSPI_CONFIG_OFST);
}
/**
- * zynqmp_qspi_txrxsetup: This function checks the TX/RX buffers in
- * the transfer and sets up the GENFIFO entries,
- * TX FIFO as required.
- * @xqspi: xqspi is a pointer to the GQSPI instance.
- * @transfer: It is a pointer to the structure containing transfer data.
- * @genfifoentry: genfifoentry is pointer to the variable in which
- * GENFIFO mask is returned to calling function
+ * zynqmp_qspi_write_op - This function sets up the GENFIFO entries,
+ * TX FIFO, and fills the TX FIFO with as many
+ * bytes as possible.
+ * @xqspi: Pointer to the GQSPI instance.
+ * @tx_nbits: Transfer buswidth.
+ * @genfifoentry: Variable in which GENFIFO mask is returned
+ * to calling function
*/
-static void zynqmp_qspi_txrxsetup(struct zynqmp_qspi *xqspi,
- struct spi_transfer *transfer,
- u32 *genfifoentry)
+static void zynqmp_qspi_write_op(struct zynqmp_qspi *xqspi, u8 tx_nbits,
+ u32 genfifoentry)
{
u32 config_reg;
- /* Transmit */
- if ((xqspi->txbuf != NULL) && (xqspi->rxbuf == NULL)) {
- /* Setup data to be TXed */
- *genfifoentry &= ~GQSPI_GENFIFO_RX;
- *genfifoentry |= GQSPI_GENFIFO_DATA_XFER;
- *genfifoentry |= GQSPI_GENFIFO_TX;
- *genfifoentry |=
- zynqmp_qspi_selectspimode(xqspi, transfer->tx_nbits);
- xqspi->bytes_to_transfer = transfer->len;
- if (xqspi->mode == GQSPI_MODE_DMA) {
- config_reg = zynqmp_gqspi_read(xqspi,
- GQSPI_CONFIG_OFST);
- config_reg &= ~GQSPI_CFG_MODE_EN_MASK;
- zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST,
- config_reg);
- xqspi->mode = GQSPI_MODE_IO;
- }
- zynqmp_qspi_filltxfifo(xqspi, GQSPI_TXD_DEPTH);
- /* Discard RX data */
- xqspi->bytes_to_receive = 0;
- } else if ((xqspi->txbuf == NULL) && (xqspi->rxbuf != NULL)) {
- /* Receive */
-
- /* TX auto fill */
- *genfifoentry &= ~GQSPI_GENFIFO_TX;
- /* Setup RX */
- *genfifoentry |= GQSPI_GENFIFO_DATA_XFER;
- *genfifoentry |= GQSPI_GENFIFO_RX;
- *genfifoentry |=
- zynqmp_qspi_selectspimode(xqspi, transfer->rx_nbits);
- xqspi->bytes_to_transfer = 0;
- xqspi->bytes_to_receive = transfer->len;
- zynq_qspi_setuprxdma(xqspi);
+ zynqmp_qspi_fillgenfifo(xqspi, tx_nbits, genfifoentry);
+ zynqmp_qspi_filltxfifo(xqspi, GQSPI_TXD_DEPTH);
+ if (xqspi->mode == GQSPI_MODE_DMA) {
+ config_reg = zynqmp_gqspi_read(xqspi,
+ GQSPI_CONFIG_OFST);
+ config_reg &= ~GQSPI_CFG_MODE_EN_MASK;
+ zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST,
+ config_reg);
+ xqspi->mode = GQSPI_MODE_IO;
}
}
/**
- * zynqmp_qspi_start_transfer: Initiates the QSPI transfer
- * @master: Pointer to the spi_master structure which provides
- * information about the controller.
- * @qspi: Pointer to the spi_device structure
- * @transfer: Pointer to the spi_transfer structure which provide information
- * about next transfer parameters
- *
- * This function fills the TX FIFO, starts the QSPI transfer, and waits for the
- * transfer to be completed.
- *
- * Return: Number of bytes transferred in the last transfer
+ * zynqmp_qspi_read_op - This function sets up the GENFIFO entries and
+ * RX DMA operation.
+ * @xqspi: xqspi is a pointer to the GQSPI instance.
+ * @rx_nbits: Receive buswidth.
+ * @genfifoentry: genfifoentry is pointer to the variable in which
+ * GENFIFO mask is returned to calling function
*/
-static int zynqmp_qspi_start_transfer(struct spi_master *master,
- struct spi_device *qspi,
- struct spi_transfer *transfer)
+static void zynqmp_qspi_read_op(struct zynqmp_qspi *xqspi, u8 rx_nbits,
+ u32 genfifoentry)
{
- struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
- u32 genfifoentry = 0x0, transfer_len;
-
- xqspi->txbuf = transfer->tx_buf;
- xqspi->rxbuf = transfer->rx_buf;
-
- zynqmp_qspi_setup_transfer(qspi, transfer);
-
- genfifoentry |= xqspi->genfifocs;
- genfifoentry |= xqspi->genfifobus;
-
- zynqmp_qspi_txrxsetup(xqspi, transfer, &genfifoentry);
-
- if (xqspi->mode == GQSPI_MODE_DMA)
- transfer_len = xqspi->dma_rx_bytes;
- else
- transfer_len = transfer->len;
-
- xqspi->genfifoentry = genfifoentry;
- if ((transfer_len) < GQSPI_GENFIFO_IMM_DATA_MASK) {
- genfifoentry &= ~GQSPI_GENFIFO_IMM_DATA_MASK;
- genfifoentry |= transfer_len;
- zynqmp_gqspi_write(xqspi, GQSPI_GEN_FIFO_OFST, genfifoentry);
- } else {
- int tempcount = transfer_len;
- u32 exponent = 8; /* 2^8 = 256 */
- u8 imm_data = tempcount & 0xFF;
-
- tempcount &= ~(tempcount & 0xFF);
- /* Immediate entry */
- if (tempcount != 0) {
- /* Exponent entries */
- genfifoentry |= GQSPI_GENFIFO_EXP;
- while (tempcount != 0) {
- if (tempcount & GQSPI_GENFIFO_EXP_START) {
- genfifoentry &=
- ~GQSPI_GENFIFO_IMM_DATA_MASK;
- genfifoentry |= exponent;
- zynqmp_gqspi_write(xqspi,
- GQSPI_GEN_FIFO_OFST,
- genfifoentry);
- }
- tempcount = tempcount >> 1;
- exponent++;
- }
- }
- if (imm_data != 0) {
- genfifoentry &= ~GQSPI_GENFIFO_EXP;
- genfifoentry &= ~GQSPI_GENFIFO_IMM_DATA_MASK;
- genfifoentry |= (u8) (imm_data & 0xFF);
- zynqmp_gqspi_write(xqspi,
- GQSPI_GEN_FIFO_OFST, genfifoentry);
- }
- }
-
- if ((xqspi->mode == GQSPI_MODE_IO) &&
- (xqspi->rxbuf != NULL)) {
- /* Dummy generic FIFO entry */
- zynqmp_gqspi_write(xqspi, GQSPI_GEN_FIFO_OFST, 0x0);
- }
-
- /* Since we are using manual mode */
- zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST,
- zynqmp_gqspi_read(xqspi, GQSPI_CONFIG_OFST) |
- GQSPI_CFG_START_GEN_FIFO_MASK);
-
- if (xqspi->txbuf != NULL)
- /* Enable interrupts for TX */
- zynqmp_gqspi_write(xqspi, GQSPI_IER_OFST,
- GQSPI_IER_TXEMPTY_MASK |
- GQSPI_IER_GENFIFOEMPTY_MASK |
- GQSPI_IER_TXNOT_FULL_MASK);
-
- if (xqspi->rxbuf != NULL) {
- /* Enable interrupts for RX */
- if (xqspi->mode == GQSPI_MODE_DMA) {
- /* Enable DMA interrupts */
- zynqmp_gqspi_write(xqspi,
- GQSPI_QSPIDMA_DST_I_EN_OFST,
- GQSPI_QSPIDMA_DST_I_EN_DONE_MASK);
- } else {
- zynqmp_gqspi_write(xqspi, GQSPI_IER_OFST,
- GQSPI_IER_GENFIFOEMPTY_MASK |
- GQSPI_IER_RXNEMPTY_MASK |
- GQSPI_IER_RXEMPTY_MASK);
- }
- }
-
- return transfer->len;
+ zynqmp_qspi_fillgenfifo(xqspi, rx_nbits, genfifoentry);
+ zynqmp_qspi_setuprxdma(xqspi);
}
/**
- * zynqmp_qspi_suspend: Suspend method for the QSPI driver
+ * zynqmp_qspi_suspend - Suspend method for the QSPI driver
* @dev: Address of the platform_device structure
*
* This function stops the QSPI driver queue and disables the QSPI controller
*/
static int __maybe_unused zynqmp_qspi_suspend(struct device *dev)
{
- struct spi_master *master = dev_get_drvdata(dev);
+ struct spi_controller *ctlr = dev_get_drvdata(dev);
+ struct zynqmp_qspi *xqspi = spi_controller_get_devdata(ctlr);
- spi_master_suspend(master);
+ spi_controller_suspend(ctlr);
- zynqmp_unprepare_transfer_hardware(master);
+ zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, 0x0);
return 0;
}
/**
- * zynqmp_qspi_resume: Resume method for the QSPI driver
+ * zynqmp_qspi_resume - Resume method for the QSPI driver
* @dev: Address of the platform_device structure
*
* The function starts the QSPI driver queue and initializes the QSPI
*/
static int __maybe_unused zynqmp_qspi_resume(struct device *dev)
{
- struct spi_master *master = dev_get_drvdata(dev);
- struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
+ struct spi_controller *ctlr = dev_get_drvdata(dev);
+ struct zynqmp_qspi *xqspi = spi_controller_get_devdata(ctlr);
int ret = 0;
ret = clk_enable(xqspi->pclk);
return ret;
}
- spi_master_resume(master);
+ spi_controller_resume(ctlr);
clk_disable(xqspi->refclk);
clk_disable(xqspi->pclk);
*/
static int __maybe_unused zynqmp_runtime_suspend(struct device *dev)
{
- struct spi_master *master = dev_get_drvdata(dev);
- struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
+ struct zynqmp_qspi *xqspi = (struct zynqmp_qspi *)dev_get_drvdata(dev);
clk_disable(xqspi->refclk);
clk_disable(xqspi->pclk);
*/
static int __maybe_unused zynqmp_runtime_resume(struct device *dev)
{
- struct spi_master *master = dev_get_drvdata(dev);
- struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
+ struct zynqmp_qspi *xqspi = (struct zynqmp_qspi *)dev_get_drvdata(dev);
int ret;
ret = clk_enable(xqspi->pclk);
return 0;
}
+/**
+ * zynqmp_qspi_exec_op() - Initiates the QSPI transfer
+ * @mem: The SPI memory
+ * @op: The memory operation to execute
+ *
+ * Executes a memory operation.
+ *
+ * This function first selects the chip and starts the memory operation.
+ *
+ * Return: 0 in case of success, a negative error code otherwise.
+ */
+static int zynqmp_qspi_exec_op(struct spi_mem *mem,
+ const struct spi_mem_op *op)
+{
+ struct zynqmp_qspi *xqspi = spi_controller_get_devdata
+ (mem->spi->master);
+ int err = 0, i;
+ u8 *tmpbuf;
+ u32 genfifoentry = 0;
+
+ dev_dbg(xqspi->dev, "cmd:%#x mode:%d.%d.%d.%d\n",
+ op->cmd.opcode, op->cmd.buswidth, op->addr.buswidth,
+ op->dummy.buswidth, op->data.buswidth);
+
+ zynqmp_qspi_config_op(xqspi, mem->spi);
+ zynqmp_qspi_chipselect(mem->spi, false);
+ genfifoentry |= xqspi->genfifocs;
+ genfifoentry |= xqspi->genfifobus;
+
+ if (op->cmd.opcode) {
+ tmpbuf = kzalloc(op->cmd.nbytes, GFP_KERNEL | GFP_DMA);
+ if (!tmpbuf)
+ return -ENOMEM;
+ tmpbuf[0] = op->cmd.opcode;
+ reinit_completion(&xqspi->data_completion);
+ xqspi->txbuf = tmpbuf;
+ xqspi->rxbuf = NULL;
+ xqspi->bytes_to_transfer = op->cmd.nbytes;
+ xqspi->bytes_to_receive = 0;
+ zynqmp_qspi_write_op(xqspi, op->cmd.buswidth, genfifoentry);
+ zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST,
+ zynqmp_gqspi_read(xqspi, GQSPI_CONFIG_OFST) |
+ GQSPI_CFG_START_GEN_FIFO_MASK);
+ zynqmp_gqspi_write(xqspi, GQSPI_IER_OFST,
+ GQSPI_IER_GENFIFOEMPTY_MASK |
+ GQSPI_IER_TXNOT_FULL_MASK);
+ if (!wait_for_completion_interruptible_timeout
+ (&xqspi->data_completion, msecs_to_jiffies(1000))) {
+ err = -ETIMEDOUT;
+ kfree(tmpbuf);
+ goto return_err;
+ }
+ kfree(tmpbuf);
+ }
+
+ if (op->addr.nbytes) {
+ for (i = 0; i < op->addr.nbytes; i++) {
+ *(((u8 *)xqspi->txbuf) + i) = op->addr.val >>
+ (8 * (op->addr.nbytes - i - 1));
+ }
+
+ reinit_completion(&xqspi->data_completion);
+ xqspi->rxbuf = NULL;
+ xqspi->bytes_to_transfer = op->addr.nbytes;
+ xqspi->bytes_to_receive = 0;
+ zynqmp_qspi_write_op(xqspi, op->addr.buswidth, genfifoentry);
+ zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST,
+ zynqmp_gqspi_read(xqspi,
+ GQSPI_CONFIG_OFST) |
+ GQSPI_CFG_START_GEN_FIFO_MASK);
+ zynqmp_gqspi_write(xqspi, GQSPI_IER_OFST,
+ GQSPI_IER_TXEMPTY_MASK |
+ GQSPI_IER_GENFIFOEMPTY_MASK |
+ GQSPI_IER_TXNOT_FULL_MASK);
+ if (!wait_for_completion_interruptible_timeout
+ (&xqspi->data_completion, msecs_to_jiffies(1000))) {
+ err = -ETIMEDOUT;
+ goto return_err;
+ }
+ }
+
+ if (op->dummy.nbytes) {
+ tmpbuf = kzalloc(op->dummy.nbytes, GFP_KERNEL | GFP_DMA);
+ if (!tmpbuf)
+ return -ENOMEM;
+ memset(tmpbuf, 0xff, op->dummy.nbytes);
+ reinit_completion(&xqspi->data_completion);
+ xqspi->txbuf = tmpbuf;
+ xqspi->rxbuf = NULL;
+ xqspi->bytes_to_transfer = op->dummy.nbytes;
+ xqspi->bytes_to_receive = 0;
+ zynqmp_qspi_write_op(xqspi, op->dummy.buswidth,
+ genfifoentry);
+ zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST,
+ zynqmp_gqspi_read(xqspi, GQSPI_CONFIG_OFST) |
+ GQSPI_CFG_START_GEN_FIFO_MASK);
+ zynqmp_gqspi_write(xqspi, GQSPI_IER_OFST,
+ GQSPI_IER_TXEMPTY_MASK |
+ GQSPI_IER_GENFIFOEMPTY_MASK |
+ GQSPI_IER_TXNOT_FULL_MASK);
+ if (!wait_for_completion_interruptible_timeout
+ (&xqspi->data_completion, msecs_to_jiffies(1000))) {
+ err = -ETIMEDOUT;
+ kfree(tmpbuf);
+ goto return_err;
+ }
+
+ kfree(tmpbuf);
+ }
+
+ if (op->data.nbytes) {
+ reinit_completion(&xqspi->data_completion);
+ if (op->data.dir == SPI_MEM_DATA_OUT) {
+ xqspi->txbuf = (u8 *)op->data.buf.out;
+ xqspi->rxbuf = NULL;
+ xqspi->bytes_to_transfer = op->data.nbytes;
+ xqspi->bytes_to_receive = 0;
+ zynqmp_qspi_write_op(xqspi, op->data.buswidth,
+ genfifoentry);
+ zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST,
+ zynqmp_gqspi_read
+ (xqspi, GQSPI_CONFIG_OFST) |
+ GQSPI_CFG_START_GEN_FIFO_MASK);
+ zynqmp_gqspi_write(xqspi, GQSPI_IER_OFST,
+ GQSPI_IER_TXEMPTY_MASK |
+ GQSPI_IER_GENFIFOEMPTY_MASK |
+ GQSPI_IER_TXNOT_FULL_MASK);
+ } else {
+ xqspi->txbuf = NULL;
+ xqspi->rxbuf = (u8 *)op->data.buf.in;
+ xqspi->bytes_to_receive = op->data.nbytes;
+ xqspi->bytes_to_transfer = 0;
+ zynqmp_qspi_read_op(xqspi, op->data.buswidth,
+ genfifoentry);
+ zynqmp_gqspi_write(xqspi, GQSPI_CONFIG_OFST,
+ zynqmp_gqspi_read
+ (xqspi, GQSPI_CONFIG_OFST) |
+ GQSPI_CFG_START_GEN_FIFO_MASK);
+ if (xqspi->mode == GQSPI_MODE_DMA) {
+ zynqmp_gqspi_write
+ (xqspi, GQSPI_QSPIDMA_DST_I_EN_OFST,
+ GQSPI_QSPIDMA_DST_I_EN_DONE_MASK);
+ } else {
+ zynqmp_gqspi_write(xqspi, GQSPI_IER_OFST,
+ GQSPI_IER_GENFIFOEMPTY_MASK |
+ GQSPI_IER_RXNEMPTY_MASK |
+ GQSPI_IER_RXEMPTY_MASK);
+ }
+ }
+ if (!wait_for_completion_interruptible_timeout
+ (&xqspi->data_completion, msecs_to_jiffies(1000)))
+ err = -ETIMEDOUT;
+ }
+
+return_err:
+
+ zynqmp_qspi_chipselect(mem->spi, true);
+
+ return err;
+}
+
static const struct dev_pm_ops zynqmp_qspi_dev_pm_ops = {
SET_RUNTIME_PM_OPS(zynqmp_runtime_suspend,
zynqmp_runtime_resume, NULL)
SET_SYSTEM_SLEEP_PM_OPS(zynqmp_qspi_suspend, zynqmp_qspi_resume)
};
+static const struct spi_controller_mem_ops zynqmp_qspi_mem_ops = {
+ .exec_op = zynqmp_qspi_exec_op,
+};
+
/**
- * zynqmp_qspi_probe: Probe method for the QSPI driver
+ * zynqmp_qspi_probe - Probe method for the QSPI driver
* @pdev: Pointer to the platform_device structure
*
* This function initializes the driver data structures and the hardware.
static int zynqmp_qspi_probe(struct platform_device *pdev)
{
int ret = 0;
- struct spi_master *master;
+ struct spi_controller *ctlr;
struct zynqmp_qspi *xqspi;
struct device *dev = &pdev->dev;
+ struct device_node *np = dev->of_node;
- master = spi_alloc_master(&pdev->dev, sizeof(*xqspi));
- if (!master)
+ ctlr = spi_alloc_master(&pdev->dev, sizeof(*xqspi));
+ if (!ctlr)
return -ENOMEM;
- xqspi = spi_master_get_devdata(master);
- master->dev.of_node = pdev->dev.of_node;
- platform_set_drvdata(pdev, master);
+ xqspi = spi_controller_get_devdata(ctlr);
+ xqspi->dev = dev;
+ platform_set_drvdata(pdev, xqspi);
xqspi->regs = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(xqspi->regs)) {
goto remove_master;
}
- xqspi->dev = dev;
xqspi->pclk = devm_clk_get(&pdev->dev, "pclk");
if (IS_ERR(xqspi->pclk)) {
dev_err(dev, "pclk clock not found.\n");
goto remove_master;
}
- ret = clk_prepare_enable(xqspi->pclk);
- if (ret) {
- dev_err(dev, "Unable to enable APB clock.\n");
- goto remove_master;
- }
+ init_completion(&xqspi->data_completion);
xqspi->refclk = devm_clk_get(&pdev->dev, "ref_clk");
if (IS_ERR(xqspi->refclk)) {
goto clk_dis_pclk;
}
+ ret = clk_prepare_enable(xqspi->pclk);
+ if (ret) {
+ dev_err(dev, "Unable to enable APB clock.\n");
+ goto remove_master;
+ }
+
ret = clk_prepare_enable(xqspi->refclk);
if (ret) {
dev_err(dev, "Unable to enable device clock.\n");
goto clk_dis_all;
}
ret = devm_request_irq(&pdev->dev, xqspi->irq, zynqmp_qspi_irq,
- 0, pdev->name, master);
+ 0, pdev->name, xqspi);
if (ret != 0) {
ret = -ENXIO;
dev_err(dev, "request_irq failed\n");
goto clk_dis_all;
}
- master->num_chipselect = GQSPI_DEFAULT_NUM_CS;
-
- master->setup = zynqmp_qspi_setup;
- master->set_cs = zynqmp_qspi_chipselect;
- master->transfer_one = zynqmp_qspi_start_transfer;
- master->prepare_transfer_hardware = zynqmp_prepare_transfer_hardware;
- master->unprepare_transfer_hardware =
- zynqmp_unprepare_transfer_hardware;
- master->max_speed_hz = clk_get_rate(xqspi->refclk) / 2;
- master->bits_per_word_mask = SPI_BPW_MASK(8);
- master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_RX_DUAL | SPI_RX_QUAD |
+ ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
+ ctlr->num_chipselect = GQSPI_DEFAULT_NUM_CS;
+ ctlr->mem_ops = &zynqmp_qspi_mem_ops;
+ ctlr->setup = zynqmp_qspi_setup_op;
+ ctlr->max_speed_hz = clk_get_rate(xqspi->refclk) / 2;
+ ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
+ ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_RX_DUAL | SPI_RX_QUAD |
SPI_TX_DUAL | SPI_TX_QUAD;
+ ctlr->dev.of_node = np;
- if (master->dev.parent == NULL)
- master->dev.parent = &master->dev;
-
- ret = spi_register_master(master);
- if (ret)
+ ret = devm_spi_register_controller(&pdev->dev, ctlr);
+ if (ret) {
+ dev_err(&pdev->dev, "spi_register_controller failed\n");
goto clk_dis_all;
+ }
return 0;
clk_dis_pclk:
clk_disable_unprepare(xqspi->pclk);
remove_master:
- spi_master_put(master);
+ spi_controller_put(ctlr);
return ret;
}
/**
- * zynqmp_qspi_remove: Remove method for the QSPI driver
+ * zynqmp_qspi_remove - Remove method for the QSPI driver
* @pdev: Pointer to the platform_device structure
*
* This function is called if a device is physically removed from the system or
*/
static int zynqmp_qspi_remove(struct platform_device *pdev)
{
- struct spi_master *master = platform_get_drvdata(pdev);
- struct zynqmp_qspi *xqspi = spi_master_get_devdata(master);
+ struct zynqmp_qspi *xqspi = platform_get_drvdata(pdev);
zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, 0x0);
clk_disable_unprepare(xqspi->refclk);
pm_runtime_set_suspended(&pdev->dev);
pm_runtime_disable(&pdev->dev);
- spi_unregister_master(master);
-
return 0;
}
spidev_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos)
{
struct spidev_data *spidev;
- ssize_t status = 0;
+ ssize_t status;
/* chipselect only toggles at start or end of operation */
if (count > bufsiz)
size_t count, loff_t *f_pos)
{
struct spidev_data *spidev;
- ssize_t status = 0;
+ ssize_t status;
unsigned long missing;
/* chipselect only toggles at start or end of operation */
source "drivers/staging/media/tegra-vde/Kconfig"
+source "drivers/staging/media/zoran/Kconfig"
+
source "drivers/staging/media/tegra-video/Kconfig"
source "drivers/staging/media/ipu3/Kconfig"
source "drivers/staging/media/rkisp1/Kconfig"
-if MEDIA_ANALOG_TV_SUPPORT
-source "drivers/staging/media/usbvision/Kconfig"
-endif
-
endif
obj-$(CONFIG_VIDEO_IPU3_IMGU) += ipu3/
obj-$(CONFIG_PHY_ROCKCHIP_DPHY_RX0) += phy-rockchip-dphy-rx0/
obj-$(CONFIG_VIDEO_ROCKCHIP_ISP1) += rkisp1/
-obj-$(CONFIG_VIDEO_USBVISION) += usbvision/
+obj-$(CONFIG_VIDEO_ZORAN) += zoran/
-I$(atomisp)/pci/runtime/queue/src/ \
-I$(atomisp)/pci/runtime/rmgr/interface/ \
-I$(atomisp)/pci/runtime/spctrl/interface/ \
- -I$(atomisp)/pci/runtime/tagger/interface/
-
-INCLUDES_byt += \
+ -I$(atomisp)/pci/runtime/tagger/interface/ \
-I$(atomisp)/pci/css_2400_system/hive/ \
-
-INCLUDES_cht += \
-I$(atomisp)/pci/css_2401_system/ \
-I$(atomisp)/pci/css_2401_system/host/ \
-I$(atomisp)/pci/css_2401_system/hive/ \
- -I$(atomisp)/pci/css_2401_system/hrt/ \
-
-# -I$(atomisp)/pci/css_2401_system/hive_isp_css_2401_system_generated/ \
+ -I$(atomisp)/pci/css_2401_system/hrt/
DEFINES := -DHRT_HW -DHRT_ISP_CSS_CUSTOM_HOST -DHRT_USE_VIR_ADDRS -D__HOST__
#DEFINES += -DUSE_DYNAMIC_BIN
ifeq ($(CONFIG_VIDEO_ATOMISP_ISP2401),y)
atomisp-objs += $(obj-cht)
-INCLUDES += $(INCLUDES_cht)
DEFINES += -DISP2401 -DISP2401_NEW_INPUT_SYSTEM -DSYSTEM_hive_isp_css_2401_system
else
atomisp-objs += $(obj-byt)
-INCLUDES += $(INCLUDES_byt)
DEFINES += -DISP2400 -DSYSTEM_hive_isp_css_2400_system
endif
# Kconfig for sensor drivers
#
-source "drivers/staging/media/atomisp/i2c/ov5693/Kconfig"
-
config VIDEO_ATOMISP_OV2722
- tristate "OVT ov2722 sensor support"
+ tristate "OVT ov2722 sensor support"
depends on ACPI
- depends on I2C && VIDEO_V4L2
+ depends on I2C && VIDEO_V4L2
help
- This is a Video4Linux2 sensor-level driver for the OVT
- OV2722 raw camera.
+ This is a Video4Linux2 sensor-level driver for the OVT
+ OV2722 raw camera.
- OVT is a 2M raw sensor.
+ OVT is a 2M raw sensor.
- It currently only works with the atomisp driver.
+ It currently only works with the atomisp driver.
config VIDEO_ATOMISP_GC2235
- tristate "Galaxy gc2235 sensor support"
+ tristate "Galaxy gc2235 sensor support"
depends on ACPI
- depends on I2C && VIDEO_V4L2
+ depends on I2C && VIDEO_V4L2
help
- This is a Video4Linux2 sensor-level driver for the OVT
- GC2235 raw camera.
+ This is a Video4Linux2 sensor-level driver for the OVT
+ GC2235 raw camera.
- GC2235 is a 2M raw sensor.
+ GC2235 is a 2M raw sensor.
- It currently only works with the atomisp driver.
+ It currently only works with the atomisp driver.
config VIDEO_ATOMISP_MSRLIST_HELPER
- tristate "Helper library to load, parse and apply large register lists."
- depends on I2C
+ tristate "Helper library to load, parse and apply large register lists."
+ depends on I2C
help
- This is a helper library to be used from a sensor driver to load, parse
- and apply large register lists.
+ This is a helper library to be used from a sensor driver to load, parse
+ and apply large register lists.
- To compile this driver as a module, choose M here: the
- module will be called libmsrlisthelper.
+ To compile this driver as a module, choose M here: the
+ module will be called libmsrlisthelper.
config VIDEO_ATOMISP_MT9M114
- tristate "Aptina mt9m114 sensor support"
+ tristate "Aptina mt9m114 sensor support"
depends on ACPI
- depends on I2C && VIDEO_V4L2
+ depends on I2C && VIDEO_V4L2
help
- This is a Video4Linux2 sensor-level driver for the Micron
- mt9m114 1.3 Mpixel camera.
+ This is a Video4Linux2 sensor-level driver for the Micron
+ mt9m114 1.3 Mpixel camera.
- mt9m114 is video camera sensor.
+ mt9m114 is video camera sensor.
- It currently only works with the atomisp driver.
+ It currently only works with the atomisp driver.
config VIDEO_ATOMISP_GC0310
tristate "GC0310 sensor support"
GC0310 0.3MP sensor.
config VIDEO_ATOMISP_OV2680
- tristate "Omnivision OV2680 sensor support"
+ tristate "Omnivision OV2680 sensor support"
+ depends on ACPI
+ depends on I2C && VIDEO_V4L2
+ help
+ This is a Video4Linux2 sensor-level driver for the Omnivision
+ OV2680 raw camera.
+
+ ov2680 is a 2M raw sensor.
+
+ It currently only works with the atomisp driver.
+
+config VIDEO_ATOMISP_OV5693
+ tristate "Omnivision ov5693 sensor support"
depends on ACPI
- depends on I2C && VIDEO_V4L2
+ depends on I2C && VIDEO_V4L2
help
- This is a Video4Linux2 sensor-level driver for the Omnivision
- OV2680 raw camera.
+ This is a Video4Linux2 sensor-level driver for the Micron
+ ov5693 5 Mpixel camera.
- ov2680 is a 2M raw sensor.
+ ov5693 is video camera sensor.
- It currently only works with the atomisp driver.
+ It currently only works with the atomisp driver.
#
# Kconfig for flash drivers
err = mt9m114_read_reg(client, data_length, reg, &val);
if (err) {
- v4l2_err(client, "misensor_rmw_reg error exit, read failed\n");
+ v4l2_err(client, "%s error exit, read failed\n", __func__);
return -EINVAL;
}
err = mt9m114_write_reg(client, data_length, reg, val);
if (err) {
- v4l2_err(client, "misensor_rmw_reg error exit, write failed\n");
+ v4l2_err(client, "%s error exit, write failed\n", __func__);
return -EINVAL;
}
+++ /dev/null
-# SPDX-License-Identifier: GPL-2.0
-config VIDEO_ATOMISP_OV5693
- tristate "Omnivision ov5693 sensor support"
- depends on ACPI
- depends on I2C && VIDEO_V4L2
- help
- This is a Video4Linux2 sensor-level driver for the Micron
- ov5693 5 Mpixel camera.
-
- ov5693 is video camera sensor.
-
- It currently only works with the atomisp driver.
return asd->video_out_capture.buffers_in_css ||
asd->video_out_vf.buffers_in_css ||
asd->video_out_preview.buffers_in_css ||
- asd->video_out_video_capture.buffers_in_css ?
- true : false;
+ asd->video_out_video_capture.buffers_in_css;
}
/* ISP2401 */
enum atomisp_metadata_type
atomisp_get_metadata_type(struct atomisp_sub_device *asd,
- enum ia_css_pipe_id pipe_id) {
+ enum ia_css_pipe_id pipe_id)
+{
if (!asd->continuous_mode->val)
return ATOMISP_MAIN_METADATA;
default:
break;
}
- if (vb)
- {
+ if (vb) {
vb->ts = ktime_get_ns();
vb->field_count = atomic_read(&asd->sequence) << 1;
/*mark videobuffer done for dequeue*/
* Requeue should only be done for 3a and dis buffers.
* Queue/dequeue order will change if driver recycles image buffers.
*/
- if (requeue)
- {
+ if (requeue) {
err = atomisp_css_queue_buffer(asd,
stream_id, css_pipe_id,
buf_type, &buffer);
* Get internal fmt according to V4L2 fmt
*/
static enum ia_css_frame_format
-v4l2_fmt_to_sh_fmt(u32 fmt) {
- switch (fmt)
- {
+v4l2_fmt_to_sh_fmt(u32 fmt)
+{
+ switch (fmt) {
case V4L2_PIX_FMT_YUV420:
return IA_CSS_FRAME_FORMAT_YUV420;
case V4L2_PIX_FMT_YVU420:
struct atomisp_metadata *md)
{
struct atomisp_device *isp = asd->isp;
- struct ia_css_stream_config *stream_config;
struct ia_css_stream_info *stream_info;
struct camera_mipi_info *mipi_info;
struct atomisp_metadata_buf *md_buf;
if (flag != 0)
return -EINVAL;
- stream_config = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
- stream_config;
stream_info = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
stream_info;
struct atomisp_metadata_with_type *md)
{
struct atomisp_device *isp = asd->isp;
- struct ia_css_stream_config *stream_config;
struct ia_css_stream_info *stream_info;
struct camera_mipi_info *mipi_info;
struct atomisp_metadata_buf *md_buf;
if (flag != 0)
return -EINVAL;
- stream_config = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
- stream_config;
stream_info = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].
stream_info;
case ATOMISP_CAMERA_PORT_SECONDARY:
return MIPI_PORT1_ID;
case ATOMISP_CAMERA_PORT_TERTIARY:
- if (MIPI_PORT1_ID + 1 != N_MIPI_PORT_ID) {
+ if (MIPI_PORT1_ID + 1 != N_MIPI_PORT_ID)
return MIPI_PORT1_ID + 1;
- }
fallthrough;
default:
dev_err(isp->dev, "unsupported port: %d\n", port);
if (!enable)
return -EINVAL;
- value = *enable > 0 ? true : false;
+ value = *enable > 0;
atomisp_en_dz_capt_pipe(asd, value);
spin_unlock_irqrestore(&mmio_lock, flags);
}
-static int atomisp_css2_dbg_print(const char *fmt, va_list args)
-{
- vprintk(fmt, args);
- return 0;
-}
-
-static int atomisp_css2_dbg_ftrace_print(const char *fmt, va_list args)
+static int __printf(1, 0) atomisp_css2_dbg_ftrace_print(const char *fmt,
+ va_list args)
{
ftrace_vprintk(fmt, args);
return 0;
}
-static int atomisp_css2_err_print(const char *fmt, va_list args)
+static int __printf(1, 0) atomisp_vprintk(const char *fmt, va_list args)
{
vprintk(fmt, args);
return 0;
return true;
return false;
- fallthrough;
case ATOMISP_RUN_MODE_VIDEO:
if (!asd->continuous_mode->val) {
if (pipe_id == IA_CSS_PIPE_ID_VIDEO ||
isp->css_env.isp_css_env.print_env.debug_print =
atomisp_css2_dbg_ftrace_print;
else if (opt == 2)
- isp->css_env.isp_css_env.print_env.debug_print =
- atomisp_css2_dbg_print;
+ isp->css_env.isp_css_env.print_env.debug_print = atomisp_vprintk;
else
ret = -EINVAL;
__set_css_print_env(isp, dbg_func);
- isp->css_env.isp_css_env.print_env.error_print = atomisp_css2_err_print;
+ isp->css_env.isp_css_env.print_env.error_print = atomisp_vprintk;
/* load isp fw into ISP memory */
err = ia_css_load_firmware(isp->dev, &isp->css_env.isp_css_env,
#include "atomisp_ioctl.h"
#include "atomisp_compat_ioctl32.h"
-/* Macro borrowed from v4l2-compat-ioctl32.c */
-/* Use the same argument order as copy_in_user */
-#define assign_in_user(to, from) \
-({ \
- typeof(*from) __assign_tmp; \
- \
- get_user(__assign_tmp, from) || put_user(__assign_tmp, to); \
+/* Macros borrowed from v4l2-compat-ioctl32.c */
+
+#define get_user_cast(__x, __ptr) \
+({ \
+ get_user(__x, (typeof(*__ptr) __user *)(__ptr)); \
})
+#define put_user_force(__x, __ptr) \
+({ \
+ put_user((typeof(*__x) __force *)(__x), __ptr); \
+})
+
+/* Use the same argument order as copy_in_user */
+#define assign_in_user(to, from) \
+({ \
+ typeof(*from) __assign_tmp; \
+ \
+ get_user_cast(__assign_tmp, from) || put_user(__assign_tmp, to);\
+})
static int get_atomisp_histogram32(struct atomisp_histogram __user *kp,
struct atomisp_histogram32 __user *up)
}
static int get_v4l2_framebuffer32(struct v4l2_framebuffer __user *kp,
- struct v4l2_framebuffer32 __user *up)
+ struct v4l2_framebuffer32 __user *up)
{
compat_uptr_t tmp;
if (!access_ok(up, sizeof(struct v4l2_framebuffer32)) ||
get_user(tmp, &up->base) ||
- put_user(compat_ptr(tmp), &kp->base) ||
+ put_user_force(compat_ptr(tmp), &kp->base) ||
assign_in_user(&kp->capability, &up->capability) ||
assign_in_user(&kp->flags, &up->flags) ||
copy_in_user(&kp->fmt, &up->fmt, sizeof(kp->fmt)))
get_user(ycoords_y, &up->ycoords_y) ||
get_user(xcoords_uv, &up->xcoords_uv) ||
get_user(ycoords_uv, &up->ycoords_uv) ||
- put_user(compat_ptr(xcoords_y), &kp->xcoords_y) ||
- put_user(compat_ptr(ycoords_y), &kp->ycoords_y) ||
- put_user(compat_ptr(xcoords_uv), &kp->xcoords_uv) ||
- put_user(compat_ptr(ycoords_uv), &kp->ycoords_uv))
+ put_user_force(compat_ptr(xcoords_y), &kp->xcoords_y) ||
+ put_user_force(compat_ptr(ycoords_y), &kp->ycoords_y) ||
+ put_user_force(compat_ptr(xcoords_uv), &kp->xcoords_uv) ||
+ put_user_force(compat_ptr(ycoords_uv), &kp->ycoords_uv))
return -EFAULT;
return 0;
void __user *rgby_data;
if (!access_ok(up, sizeof(struct atomisp_3a_statistics32)) ||
- copy_to_user(up, kp, sizeof(struct atomisp_grid_info)) ||
+ copy_in_user(up, kp, sizeof(struct atomisp_grid_info)) ||
get_user(rgby_data, &kp->rgby_data) ||
put_user(ptr_to_compat(rgby_data), &up->rgby_data) ||
get_user(data, &kp->data) ||
assign_in_user(&kp->stride, &up->stride) ||
assign_in_user(&kp->exp_id, &up->exp_id) ||
get_user(effective_width, &up->effective_width) ||
- put_user(compat_ptr(effective_width), &kp->effective_width))
+ put_user_force(compat_ptr(effective_width), &kp->effective_width))
return -EFAULT;
return 0;
struct atomisp_metadata32 __user *up)
{
void __user *data;
- void __user *effective_width;
+ void *effective_width;
if (!access_ok(up, sizeof(struct atomisp_metadata32)) ||
get_user(data, &kp->data) ||
assign_in_user(&up->stride, &kp->stride) ||
assign_in_user(&up->exp_id, &kp->exp_id) ||
get_user(effective_width, &kp->effective_width) ||
- put_user(ptr_to_compat(effective_width), &up->effective_width))
+ put_user(ptr_to_compat((void __user *)effective_width),
+ &up->effective_width))
return -EFAULT;
return 0;
struct atomisp_metadata_with_type32 __user *up)
{
void __user *data;
- void __user *effective_width;
+ u32 *effective_width;
if (!access_ok(up, sizeof(struct atomisp_metadata_with_type32)) ||
get_user(data, &kp->data) ||
assign_in_user(&up->stride, &kp->stride) ||
assign_in_user(&up->exp_id, &kp->exp_id) ||
get_user(effective_width, &kp->effective_width) ||
- put_user(ptr_to_compat(effective_width),
+ put_user(ptr_to_compat((void __user *)effective_width),
&up->effective_width) ||
assign_in_user(&up->type, &kp->type))
return -EFAULT;
assign_in_user(&kp->stride, &up->stride) ||
assign_in_user(&kp->exp_id, &up->exp_id) ||
get_user(effective_width, &up->effective_width) ||
- put_user(compat_ptr(effective_width), &kp->effective_width) ||
+ put_user_force(compat_ptr(effective_width), &kp->effective_width) ||
assign_in_user(&kp->type, &up->type))
return -EFAULT;
if (!access_ok(up, sizeof(struct atomisp_overlay32)) ||
get_user(frame, &up->frame) ||
- put_user(compat_ptr(frame), &kp->frame) ||
+ put_user_force(compat_ptr(frame), &kp->frame) ||
assign_in_user(&kp->bg_y, &up->bg_y) ||
assign_in_user(&kp->bg_u, &up->bg_u) ||
assign_in_user(&kp->bg_v, &up->bg_v) ||
static int put_atomisp_overlay32(struct atomisp_overlay __user *kp,
struct atomisp_overlay32 __user *up)
{
- void __user *frame;
+ void *frame;
if (!access_ok(up, sizeof(struct atomisp_overlay32)) ||
get_user(frame, &kp->frame) ||
- put_user(ptr_to_compat(frame), &up->frame) ||
+ put_user(ptr_to_compat((void __user *)frame), &up->frame) ||
assign_in_user(&up->bg_y, &kp->bg_y) ||
assign_in_user(&up->bg_u, &kp->bg_u) ||
assign_in_user(&up->bg_v, &kp->bg_v) ||
assign_in_user(&kp->size, &up->size) ||
assign_in_user(&kp->type, &up->type) ||
get_user(calb_grp_values, &up->calb_grp_values) ||
- put_user(compat_ptr(calb_grp_values), &kp->calb_grp_values))
+ put_user_force(compat_ptr(calb_grp_values), &kp->calb_grp_values))
return -EFAULT;
return 0;
put_atomisp_calibration_group32(struct atomisp_calibration_group __user *kp,
struct atomisp_calibration_group32 __user *up)
{
- void __user *calb_grp_values;
+ void *calb_grp_values;
if (!access_ok(up, sizeof(struct atomisp_calibration_group32)) ||
assign_in_user(&up->size, &kp->size) ||
assign_in_user(&up->type, &kp->type) ||
get_user(calb_grp_values, &kp->calb_grp_values) ||
- put_user(ptr_to_compat(calb_grp_values), &up->calb_grp_values))
+ put_user(ptr_to_compat((void __user *)calb_grp_values),
+ &up->calb_grp_values))
return -EFAULT;
return 0;
if (!access_ok(up, sizeof(struct atomisp_acc_fw_load32)) ||
assign_in_user(&kp->size, &up->size) ||
assign_in_user(&kp->fw_handle, &up->fw_handle) ||
- get_user(data, &up->data) ||
+ get_user_cast(data, &up->data) ||
put_user(compat_ptr(data), &kp->data))
return -EFAULT;
compat_uptr_t tmp;
if (get_user(tmp, &up->data[n]) ||
- put_user(compat_ptr(tmp), &kp->data[n]))
+ put_user_force(compat_ptr(tmp), &kp->data[n]))
return -EFAULT;
}
return 0;
struct atomisp_morph_table morph_table;
struct atomisp_dis_coefficients dvs2_coefs;
struct atomisp_dvs_6axis_config dvs_6axis_config;
- } __user *karg = (void *)(kp + 1);
+ } __user *karg = (void __user *)(kp + 1);
if (!access_ok(up, sizeof(struct atomisp_parameters32)))
return -EFAULT;
while (n >= 0) {
- compat_uptr_t *src = (compat_uptr_t *)up + n;
+ compat_uptr_t __user *src = (compat_uptr_t __user *)up + n;
void * __user *dst = (void * __user *)kp + n;
compat_uptr_t tmp;
- if (get_user(tmp, src) || put_user(compat_ptr(tmp), dst))
+ if (get_user_cast(tmp, src) || put_user_force(compat_ptr(tmp), dst))
return -EFAULT;
n--;
}
/* handle shading table */
if (stp && (get_atomisp_shading_table32(&karg->shading_table,
compat_ptr(stp)) ||
- put_user(&karg->shading_table, &kp->shading_table)))
+ put_user_force(&karg->shading_table, &kp->shading_table)))
return -EFAULT;
/* handle morph table */
if (mtp && (get_atomisp_morph_table32(&karg->morph_table,
compat_ptr(mtp)) ||
- put_user(&karg->morph_table, &kp->morph_table)))
+ put_user_force(&karg->morph_table, &kp->morph_table)))
return -EFAULT;
/* handle dvs2 coefficients */
if (dcp && (get_atomisp_dis_coefficients32(&karg->dvs2_coefs,
compat_ptr(dcp)) ||
- put_user(&karg->dvs2_coefs, &kp->dvs2_coefs)))
+ put_user_force(&karg->dvs2_coefs, &kp->dvs2_coefs)))
return -EFAULT;
/* handle dvs 6axis configuration */
if (dscp &&
(get_atomisp_dvs_6axis_config32(&karg->dvs_6axis_config,
compat_ptr(dscp)) ||
- put_user(&karg->dvs_6axis_config, &kp->dvs_6axis_config)))
+ put_user_force(&karg->dvs_6axis_config, &kp->dvs_6axis_config)))
return -EFAULT;
return 0;
if (!access_ok(up, sizeof(struct atomisp_sensor_ae_bracketing_lut32)) ||
assign_in_user(&kp->lut_size, &up->lut_size) ||
get_user(lut, &up->lut) ||
- put_user(compat_ptr(lut), &kp->lut))
+ put_user_force(compat_ptr(lut), &kp->lut))
return -EFAULT;
return 0;
int ret;
int value;
+ if (!gs || gs->v1p8_on == on)
+ return 0;
+
if (gs->v1p8_gpio >= 0) {
pr_info("atomisp_gmin_platform: 1.8v power on GPIO %d\n",
gs->v1p8_gpio);
pr_err("V1P8 GPIO initialization failed\n");
}
- if (!gs || gs->v1p8_on == on)
- return 0;
gs->v1p8_on = on;
if (gs->v1p8_gpio >= 0)
int ret;
int value;
+ if (WARN_ON(!gs))
+ return -ENODEV;
+
if (gs->v2p8_gpio >= 0) {
pr_info("atomisp_gmin_platform: 2.8v power on GPIO %d\n",
gs->v2p8_gpio);
pr_err("V2P8 GPIO initialization failed\n");
}
- if (!gs || gs->v2p8_on == on)
+ if (gs->v2p8_on == on)
return 0;
gs->v2p8_on = on;
#endif
};
-const struct atomisp_format_bridge *atomisp_get_format_bridge(
- unsigned int pixelformat)
+const struct atomisp_format_bridge *
+atomisp_get_format_bridge(unsigned int pixelformat)
{
unsigned int i;
return NULL;
}
-const struct atomisp_format_bridge *atomisp_get_format_bridge_from_mbus(
- u32 mbus_code)
+const struct atomisp_format_bridge *
+atomisp_get_format_bridge_from_mbus(u32 mbus_code)
{
unsigned int i;
return 0;
}
-static unsigned int atomisp_subdev_streaming_count(
- struct atomisp_sub_device *asd)
+static unsigned int
+atomisp_subdev_streaming_count(struct atomisp_sub_device *asd)
{
return asd->video_out_preview.capq.streaming
+ asd->video_out_capture.capq.streaming
continue;
}
- strlcpy(f->description, format->description,
+ strscpy(f->description, format->description,
sizeof(f->description));
f->pixelformat = format->pixelformat;
return 0;
}
}
- /* Workaround: Due to the design of HALv3,
+ /*
+ * Workaround: Due to the design of HALv3,
* sometimes in ZSL or SDV mode HAL needs to
* capture multiple images within one streaming cycle.
* But the capture number cannot be determined by HAL.
* So HAL only sets the capture number to be 1 and queue multiple
* buffers. Atomisp driver needs to check this case and re-trigger
- * CSS to do capture when new buffer is queued. */
+ * CSS to do capture when new buffer is queued.
+ */
if (asd->continuous_mode->val &&
atomisp_subdev_source_pad(vdev)
== ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE &&
/*
* set freq to max when streaming count > 1 which indicate
* dual camera would run
- */
+ */
if (atomisp_streaming_count(isp) > 1) {
if (atomisp_freq_scaling(isp,
ATOMISP_DFS_MODE_MAX, false) < 0)
dev_err(isp->dev, "master slave sensor stream on failed!\n");
goto out;
}
- if (!IS_ISP2401) {
+ if (!IS_ISP2401)
__wdt_on_master_slave_sensor(isp, wdt_duration);
- } else {
+ else
__wdt_on_master_slave_sensor_pipe(pipe, wdt_duration, true);
- }
goto start_delay_wq;
} else if (asd->depth_mode->val && (atomisp_streaming_count(isp) <
ATOMISP_DEPTH_SENSOR_STREAMON_COUNT)) {
struct v4l2_control ctrl;
int i, ret = 0;
- /* input_lock is not need for the Camera related IOCTLs
- * The input_lock downgrade the FPS of 3A*/
+ /*
+ * input_lock is not need for the Camera related IOCTLs
+ * The input_lock downgrade the FPS of 3A
+ */
ret = atomisp_camera_g_ext_ctrls(file, fh, c);
if (ret != -EINVAL)
return ret;
ret =
v4l2_s_ctrl(NULL, isp->flash->ctrl_handler,
&ctrl);
- /* When flash mode is changed we need to reset
- * flash state */
+ /*
+ * When flash mode is changed we need to reset
+ * flash state
+ */
if (ctrl.id == V4L2_CID_FLASH_MODE) {
asd->params.flash_state =
ATOMISP_FLASH_IDLE;
struct v4l2_control ctrl;
int i, ret = 0;
- /* input_lock is not need for the Camera related IOCTLs
- * The input_lock downgrade the FPS of 3A*/
+ /*
+ * input_lock is not need for the Camera related IOCTLs
+ * The input_lock downgrade the FPS of 3A
+ */
ret = atomisp_camera_s_ext_ctrls(file, fh, c);
if (ret != -EINVAL)
return ret;
struct atomisp_device *isp = video_get_drvdata(vdev);
if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
- dev_err(isp->dev, "unsupport v4l2 buf type\n");
+ dev_err(isp->dev, "unsupported v4l2 buf type\n");
return -EINVAL;
}
int fps;
if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
- dev_err(isp->dev, "unsupport v4l2 buf type\n");
+ dev_err(isp->dev, "unsupported v4l2 buf type\n");
return -EINVAL;
}
struct atomisp_device *isp = video_get_drvdata(vdev);
if (parm->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) {
- dev_err(isp->dev, "unsupport v4l2 buf type for output\n");
+ dev_err(isp->dev, "unsupported v4l2 buf type for output\n");
return -EINVAL;
}
if (atomisp_subdev_format_conversion(isp_sd,
isp_sd->capture_pad)
- && crop[pad]->width && crop[pad]->height)
- crop[pad]->width -= padding_w, crop[pad]->height -= padding_h;
+ && crop[pad]->width && crop[pad]->height) {
+ crop[pad]->width -= padding_w;
+ crop[pad]->height -= padding_h;
+ }
/* if subdev type is SOC camera,we do not need to set DVS */
if (isp->inputs[isp_sd->input_curr].type == SOC_CAMERA)
*/
static bool is_valid_device(struct pci_dev *pdev, const struct pci_device_id *id)
{
- unsigned int a0_max_id = 0;
const char *name;
const char *product;
switch (id->device & ATOMISP_PCI_DEVICE_SOC_MASK) {
case ATOMISP_PCI_DEVICE_SOC_MRFLD:
- a0_max_id = ATOMISP_PCI_REV_MRFLD_A0_MAX;
name = "Merrifield";
break;
case ATOMISP_PCI_DEVICE_SOC_BYT:
- a0_max_id = ATOMISP_PCI_REV_BYT_A0_MAX;
name = "Baytrail";
break;
case ATOMISP_PCI_DEVICE_SOC_ANN:
pci_set_master(pdev);
- err = pci_enable_msi(pdev);
- if (err) {
+ err = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
+ if (err < 0) {
dev_err(&pdev->dev, "Failed to enable msi (%d)\n", err);
goto enable_msi_fail;
}
initialize_modules_fail:
cpu_latency_qos_remove_request(&isp->pm_qos);
atomisp_msi_irq_uninit(isp);
- pci_disable_msi(pdev);
+ pci_free_irq_vectors(pdev);
enable_msi_fail:
fw_validation_fail:
release_firmware(isp->firmware);
*/
#include "system_global.h"
+#include "csi_rx_global.h"
const u32 N_SHORT_PACKET_LUT_ENTRIES[N_CSI_RX_BACKEND_ID] = {
4, /* 4 entries at CSI_RX_BACKEND0_ID*/
#include <type_support.h>
#include "system_global.h"
+#include "ibuf_ctrl_global.h"
const u32 N_IBUF_CTRL_PROCS[N_IBUF_CTRL_ID] = {
8, /* IBUF_CTRL0_ID supports at most 8 processes */
#define __IBUF_CTRL_LOCAL_H_INCLUDED__
#include "ibuf_ctrl_global.h"
+#include "ibuf_ctrl_local.h"
typedef struct ibuf_ctrl_proc_state_s ibuf_ctrl_proc_state_t;
typedef struct ibuf_ctrl_state_s ibuf_ctrl_state_t;
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IBUF_CTRL_PRIVATE_H_INCLUDED__
-#define __IBUF_CTRL_PRIVATE_H_INCLUDED__
-
-#include "ibuf_ctrl_public.h"
-
-#include "device_access.h" /* ia_css_device_load_uint32 */
-
-#include "assert_support.h" /* assert */
-#include "print_support.h" /* print */
-
-/*****************************************************
- *
- * Native command interface (NCI).
- *
- *****************************************************/
-/**
- * @brief Get the ibuf-controller state.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-STORAGE_CLASS_IBUF_CTRL_C void ibuf_ctrl_get_state(
- const ibuf_ctrl_ID_t ID,
- ibuf_ctrl_state_t *state)
-{
- u32 i;
-
- state->recalc_words =
- ibuf_ctrl_reg_load(ID, _IBUF_CNTRL_RECALC_WORDS_STATUS);
- state->arbiters =
- ibuf_ctrl_reg_load(ID, _IBUF_CNTRL_ARBITERS_STATUS);
-
- /*
- * Get the values of the register-set per
- * ibuf-controller process.
- */
- for (i = 0; i < N_IBUF_CTRL_PROCS[ID]; i++) {
- ibuf_ctrl_get_proc_state(
- ID,
- i,
- &state->proc_state[i]);
- }
-}
-
-/**
- * @brief Get the state of the ibuf-controller process.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-STORAGE_CLASS_IBUF_CTRL_C void ibuf_ctrl_get_proc_state(
- const ibuf_ctrl_ID_t ID,
- const u32 proc_id,
- ibuf_ctrl_proc_state_t *state)
-{
- hrt_address reg_bank_offset;
-
- reg_bank_offset =
- _IBUF_CNTRL_PROC_REG_ALIGN * (1 + proc_id);
-
- state->num_items =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_NUM_ITEMS_PER_STORE);
-
- state->num_stores =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_NUM_STORES_PER_FRAME);
-
- state->dma_channel =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DMA_CHANNEL);
-
- state->dma_command =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DMA_CMD);
-
- state->ibuf_st_addr =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_BUFFER_START_ADDRESS);
-
- state->ibuf_stride =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_BUFFER_STRIDE);
-
- state->ibuf_end_addr =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_BUFFER_END_ADDRESS);
-
- state->dest_st_addr =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DEST_START_ADDRESS);
-
- state->dest_stride =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DEST_STRIDE);
-
- state->dest_end_addr =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DEST_END_ADDRESS);
-
- state->sync_frame =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_SYNC_FRAME);
-
- state->sync_command =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_STR2MMIO_SYNC_CMD);
-
- state->store_command =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_STR2MMIO_STORE_CMD);
-
- state->shift_returned_items =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_SHIFT_ITEMS);
-
- state->elems_ibuf =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_ELEMS_P_WORD_IBUF);
-
- state->elems_dest =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_ELEMS_P_WORD_DEST);
-
- state->cur_stores =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_STORES);
-
- state->cur_acks =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_ACKS);
-
- state->cur_s2m_ibuf_addr =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_S2M_IBUF_ADDR);
-
- state->cur_dma_ibuf_addr =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_DMA_IBUF_ADDR);
-
- state->cur_dma_dest_addr =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_DMA_DEST_ADDR);
-
- state->cur_isp_dest_addr =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_ISP_DEST_ADDR);
-
- state->dma_cmds_send =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_NR_DMA_CMDS_SEND);
-
- state->main_cntrl_state =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_MAIN_CNTRL_STATE);
-
- state->dma_sync_state =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DMA_SYNC_STATE);
-
- state->isp_sync_state =
- ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_ISP_SYNC_STATE);
-}
-
-/**
- * @brief Dump the ibuf-controller state.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-STORAGE_CLASS_IBUF_CTRL_C void ibuf_ctrl_dump_state(
- const ibuf_ctrl_ID_t ID,
- ibuf_ctrl_state_t *state)
-{
- u32 i;
-
- ia_css_print("IBUF controller ID %d recalculate words 0x%x\n", ID,
- state->recalc_words);
- ia_css_print("IBUF controller ID %d arbiters 0x%x\n", ID, state->arbiters);
-
- /*
- * Dump the values of the register-set per
- * ibuf-controller process.
- */
- for (i = 0; i < N_IBUF_CTRL_PROCS[ID]; i++) {
- ia_css_print("IBUF controller ID %d Process ID %d num_items 0x%x\n", ID, i,
- state->proc_state[i].num_items);
- ia_css_print("IBUF controller ID %d Process ID %d num_stores 0x%x\n", ID, i,
- state->proc_state[i].num_stores);
- ia_css_print("IBUF controller ID %d Process ID %d dma_channel 0x%x\n", ID, i,
- state->proc_state[i].dma_channel);
- ia_css_print("IBUF controller ID %d Process ID %d dma_command 0x%x\n", ID, i,
- state->proc_state[i].dma_command);
- ia_css_print("IBUF controller ID %d Process ID %d ibuf_st_addr 0x%x\n", ID, i,
- state->proc_state[i].ibuf_st_addr);
- ia_css_print("IBUF controller ID %d Process ID %d ibuf_stride 0x%x\n", ID, i,
- state->proc_state[i].ibuf_stride);
- ia_css_print("IBUF controller ID %d Process ID %d ibuf_end_addr 0x%x\n", ID, i,
- state->proc_state[i].ibuf_end_addr);
- ia_css_print("IBUF controller ID %d Process ID %d dest_st_addr 0x%x\n", ID, i,
- state->proc_state[i].dest_st_addr);
- ia_css_print("IBUF controller ID %d Process ID %d dest_stride 0x%x\n", ID, i,
- state->proc_state[i].dest_stride);
- ia_css_print("IBUF controller ID %d Process ID %d dest_end_addr 0x%x\n", ID, i,
- state->proc_state[i].dest_end_addr);
- ia_css_print("IBUF controller ID %d Process ID %d sync_frame 0x%x\n", ID, i,
- state->proc_state[i].sync_frame);
- ia_css_print("IBUF controller ID %d Process ID %d sync_command 0x%x\n", ID, i,
- state->proc_state[i].sync_command);
- ia_css_print("IBUF controller ID %d Process ID %d store_command 0x%x\n", ID, i,
- state->proc_state[i].store_command);
- ia_css_print("IBUF controller ID %d Process ID %d shift_returned_items 0x%x\n",
- ID, i,
- state->proc_state[i].shift_returned_items);
- ia_css_print("IBUF controller ID %d Process ID %d elems_ibuf 0x%x\n", ID, i,
- state->proc_state[i].elems_ibuf);
- ia_css_print("IBUF controller ID %d Process ID %d elems_dest 0x%x\n", ID, i,
- state->proc_state[i].elems_dest);
- ia_css_print("IBUF controller ID %d Process ID %d cur_stores 0x%x\n", ID, i,
- state->proc_state[i].cur_stores);
- ia_css_print("IBUF controller ID %d Process ID %d cur_acks 0x%x\n", ID, i,
- state->proc_state[i].cur_acks);
- ia_css_print("IBUF controller ID %d Process ID %d cur_s2m_ibuf_addr 0x%x\n", ID,
- i,
- state->proc_state[i].cur_s2m_ibuf_addr);
- ia_css_print("IBUF controller ID %d Process ID %d cur_dma_ibuf_addr 0x%x\n", ID,
- i,
- state->proc_state[i].cur_dma_ibuf_addr);
- ia_css_print("IBUF controller ID %d Process ID %d cur_dma_dest_addr 0x%x\n", ID,
- i,
- state->proc_state[i].cur_dma_dest_addr);
- ia_css_print("IBUF controller ID %d Process ID %d cur_isp_dest_addr 0x%x\n", ID,
- i,
- state->proc_state[i].cur_isp_dest_addr);
- ia_css_print("IBUF controller ID %d Process ID %d dma_cmds_send 0x%x\n", ID, i,
- state->proc_state[i].dma_cmds_send);
- ia_css_print("IBUF controller ID %d Process ID %d main_cntrl_state 0x%x\n", ID,
- i,
- state->proc_state[i].main_cntrl_state);
- ia_css_print("IBUF controller ID %d Process ID %d dma_sync_state 0x%x\n", ID, i,
- state->proc_state[i].dma_sync_state);
- ia_css_print("IBUF controller ID %d Process ID %d isp_sync_state 0x%x\n", ID, i,
- state->proc_state[i].isp_sync_state);
- }
-}
-
-/* end of NCI */
-
-/*****************************************************
- *
- * Device level interface (DLI).
- *
- *****************************************************/
-/**
- * @brief Load the register value.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-STORAGE_CLASS_IBUF_CTRL_C hrt_data ibuf_ctrl_reg_load(
- const ibuf_ctrl_ID_t ID,
- const hrt_address reg)
-{
- assert(ID < N_IBUF_CTRL_ID);
- assert(IBUF_CTRL_BASE[ID] != (hrt_address)-1);
- return ia_css_device_load_uint32(IBUF_CTRL_BASE[ID] + reg * sizeof(hrt_data));
-}
-
-/**
- * @brief Store a value to the register.
- * Refer to "ibuf_ctrl_public.h" for details.
- */
-STORAGE_CLASS_IBUF_CTRL_C void ibuf_ctrl_reg_store(
- const ibuf_ctrl_ID_t ID,
- const hrt_address reg,
- const hrt_data value)
-{
- assert(ID < N_IBUF_CTRL_ID);
- assert(IBUF_CTRL_BASE[ID] != (hrt_address)-1);
-
- ia_css_device_store_uint32(IBUF_CTRL_BASE[ID] + reg * sizeof(hrt_data), value);
-}
-
-/* end of DLI */
-
-#endif /* __IBUF_CTRL_PRIVATE_H_INCLUDED__ */
* more details.
*/
-#include "isys_dma.h"
+#include "system_local.h"
+#include "isys_dma_global.h"
#include "assert_support.h"
-
-#ifndef __INLINE_ISYS2401_DMA__
-/*
- * Include definitions for isys dma register access functions. isys_dma.h
- * includes declarations of these functions by including isys_dma_public.h.
- */
#include "isys_dma_private.h"
-#endif
const isys2401_dma_channel N_ISYS2401_DMA_CHANNEL_PROCS[N_ISYS2401_DMA_ID] = {
N_ISYS2401_DMA_CHANNEL
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_DMA_LOCAL_H_INCLUDED__
-#define __ISYS_DMA_LOCAL_H_INCLUDED__
-
-#include "isys_dma_global.h"
-
-#endif /* __ISYS_DMA_LOCAL_H_INCLUDED__ */
#include "dma_v2_defs.h"
#include "print_support.h"
-STORAGE_CLASS_ISYS2401_DMA_C void isys2401_dma_reg_store(
- const isys2401_dma_ID_t dma_id,
- const unsigned int reg,
- const hrt_data value)
+void isys2401_dma_reg_store(const isys2401_dma_ID_t dma_id,
+ const unsigned int reg,
+ const hrt_data value)
{
unsigned int reg_loc;
ia_css_device_store_uint32(reg_loc, value);
}
-STORAGE_CLASS_ISYS2401_DMA_C hrt_data isys2401_dma_reg_load(
- const isys2401_dma_ID_t dma_id,
- const unsigned int reg)
+hrt_data isys2401_dma_reg_load(const isys2401_dma_ID_t dma_id,
+ const unsigned int reg)
{
unsigned int reg_loc;
hrt_data value;
#endif
/* Public interface */
-STORAGE_CLASS_ISYS2401_IRQ_C void isys_irqc_status_enable(
- const isys_irq_ID_t isys_irqc_id)
+void isys_irqc_status_enable(const isys_irq_ID_t isys_irqc_id)
{
assert(isys_irqc_id < N_ISYS_IRQ_ID);
#include <type_support.h>
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
typedef struct isys_irqc_state_s isys_irqc_state_t;
/*hrt_data clear; */ /* write-only register */
};
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2401) */
+#endif /* defined(ISP2401) */
#endif /* __ISYS_IRQ_LOCAL_H__ */
#include "isys_irq_global.h"
#include "isys_irq_local.h"
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
/* -------------------------------------------------------+
| Native command interface (NCI) |
* @brief Get the isys irq status.
* Refer to "isys_irq.h" for details.
*/
-STORAGE_CLASS_ISYS2401_IRQ_C void isys_irqc_state_get(
+void isys_irqc_state_get(
const isys_irq_ID_t isys_irqc_id,
isys_irqc_state_t *state)
{
* @brief Dump the isys irq status.
* Refer to "isys_irq.h" for details.
*/
-STORAGE_CLASS_ISYS2401_IRQ_C void isys_irqc_state_dump(
+void isys_irqc_state_dump(
const isys_irq_ID_t isys_irqc_id,
const isys_irqc_state_t *state)
{
+ -------------------------------------------------------*/
/* Support functions */
-STORAGE_CLASS_ISYS2401_IRQ_C void isys_irqc_reg_store(
+void isys_irqc_reg_store(
const isys_irq_ID_t isys_irqc_id,
const unsigned int reg_idx,
const hrt_data value)
ia_css_device_store_uint32(reg_addr, value);
}
-STORAGE_CLASS_ISYS2401_IRQ_C hrt_data isys_irqc_reg_load(
+hrt_data isys_irqc_reg_load(
const isys_irq_ID_t isys_irqc_id,
const unsigned int reg_idx)
{
/* end of DLI */
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2401) */
+#endif /* defined(ISP2401) */
#endif /* __ISYS_IRQ_PRIVATE_H__ */
#include "device_access.h" /* ia_css_device_load_uint32 */
#include "assert_support.h" /* assert */
+/*****************************************************
+ *
+ * Device level interface (DLI).
+ *
+ *****************************************************/
+/**
+ * @brief Load the register value.
+ * Refer to "pixelgen_public.h" for details.
+ */
+STORAGE_CLASS_PIXELGEN_C hrt_data pixelgen_ctrl_reg_load(
+ const pixelgen_ID_t ID,
+ const hrt_address reg)
+{
+ assert(ID < N_PIXELGEN_ID);
+ assert(PIXELGEN_CTRL_BASE[ID] != (hrt_address) - 1);
+ return ia_css_device_load_uint32(PIXELGEN_CTRL_BASE[ID] + reg * sizeof(
+ hrt_data));
+}
+
+/**
+ * @brief Store a value to the register.
+ * Refer to "pixelgen_ctrl_public.h" for details.
+ */
+STORAGE_CLASS_PIXELGEN_C void pixelgen_ctrl_reg_store(
+ const pixelgen_ID_t ID,
+ const hrt_address reg,
+ const hrt_data value)
+{
+ assert(ID < N_PIXELGEN_ID);
+ assert(PIXELGEN_CTRL_BASE[ID] != (hrt_address)-1);
+
+ ia_css_device_store_uint32(PIXELGEN_CTRL_BASE[ID] + reg * sizeof(hrt_data),
+ value);
+}
+
+/* end of DLI */
+
/*****************************************************
*
* Native command interface (NCI).
}
/* end of NCI */
-/*****************************************************
- *
- * Device level interface (DLI).
- *
- *****************************************************/
-/**
- * @brief Load the register value.
- * Refer to "pixelgen_public.h" for details.
- */
-STORAGE_CLASS_PIXELGEN_C hrt_data pixelgen_ctrl_reg_load(
- const pixelgen_ID_t ID,
- const hrt_address reg)
-{
- assert(ID < N_PIXELGEN_ID);
- assert(PIXELGEN_CTRL_BASE[ID] != (hrt_address) - 1);
- return ia_css_device_load_uint32(PIXELGEN_CTRL_BASE[ID] + reg * sizeof(
- hrt_data));
-}
-
-/**
- * @brief Store a value to the register.
- * Refer to "pixelgen_ctrl_public.h" for details.
- */
-STORAGE_CLASS_PIXELGEN_C void pixelgen_ctrl_reg_store(
- const pixelgen_ID_t ID,
- const hrt_address reg,
- const hrt_data value)
-{
- assert(ID < N_PIXELGEN_ID);
- assert(PIXELGEN_CTRL_BASE[ID] != (hrt_address)-1);
-
- ia_css_device_store_uint32(PIXELGEN_CTRL_BASE[ID] + reg * sizeof(hrt_data),
- value);
-}
-
-/* end of DLI */
#endif /* __PIXELGEN_PRIVATE_H_INCLUDED__ */
#define _IBUF_CNTRL_DMA_SYNC_WAIT_FOR_SYNC 1
#define _IBUF_CNTRL_DMA_SYNC_FSM_WAIT_FOR_ACK (0x3 << 1)
-typedef struct ib_buffer_s ib_buffer_t;
-struct ib_buffer_s {
+struct isp2401_ib_buffer_s {
u32 start_addr; /* start address of the buffer in the
* "input-buffer hardware block"
*/
u32 stride; /* stride per buffer line (in bytes) */
u32 lines; /* lines in the buffer */
};
+typedef struct isp2401_ib_buffer_s isp2401_ib_buffer_t;
typedef struct ibuf_ctrl_cfg_s ibuf_ctrl_cfg_t;
struct ibuf_ctrl_cfg_s {
u32 elems_per_word_in_dest;
} dma_cfg;
- ib_buffer_t ib_buffer;
+ isp2401_ib_buffer_t ib_buffer;
struct {
u32 stride;
#ifndef __ISYS_IRQ_GLOBAL_H__
#define __ISYS_IRQ_GLOBAL_H__
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
/* Register offset/index from base location */
#define ISYS_IRQ_EDGE_REG_IDX (0)
#define ISYS_IRQ_CLEAR_REG_VALUE (0xFFFF)
#define ISYS_IRQ_ENABLE_REG_VALUE (0xFFFF)
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2401) */
+#endif /* defined(ISP2401) */
#endif /* __ISYS_IRQ_GLOBAL_H__ */
/*
* Duplicates "sync_generator_cfg_t" in "input_system_global.h".
*/
-typedef struct sync_generator_cfg_s sync_generator_cfg_t;
-struct sync_generator_cfg_s {
+typedef struct isp2401_sync_generator_cfg_s isp2401_sync_generator_cfg_t;
+struct isp2401_sync_generator_cfg_s {
u32 hblank_cycles;
u32 vblank_cycles;
u32 pixels_per_clock;
s32 v_delta; /* vertical delta? */
} delta_cfg;
- sync_generator_cfg_t sync_gen_cfg;
+ isp2401_sync_generator_cfg_t sync_gen_cfg;
};
/*
s32 seed0;
s32 seed1;
- sync_generator_cfg_t sync_gen_cfg;
+ isp2401_sync_generator_cfg_t sync_gen_cfg;
};
/* end of Pixel-generator: TPG. ("pixelgen_global.h") */
#include "system_global.h"
-#ifdef USE_INPUT_SYSTEM_VERSION_2
+#ifndef ISP2401
#include "input_formatter.h"
#include <type_support.h>
#include "input_formatter_private.h"
#endif /* __INLINE_INPUT_FORMATTER__ */
+static const unsigned int input_formatter_alignment[N_INPUT_FORMATTER_ID] = {
+ ISP_VEC_ALIGN, ISP_VEC_ALIGN, HIVE_ISP_CTRL_DATA_BYTES
+};
+
const hrt_address HIVE_IF_SRST_ADDRESS[N_INPUT_FORMATTER_ID] = {
INPUT_FORMATTER0_SRST_OFFSET,
INPUT_FORMATTER1_SRST_OFFSET,
u32 en_status_update;
};
-static const unsigned int input_formatter_alignment[N_INPUT_FORMATTER_ID] = {
- ISP_VEC_ALIGN, ISP_VEC_ALIGN, HIVE_ISP_CTRL_DATA_BYTES
-};
-
#endif /* __INPUT_FORMATTER_LOCAL_H_INCLUDED__ */
#include "system_global.h"
-#ifdef USE_INPUT_SYSTEM_VERSION_2
+#ifndef ISP2401
#include "input_system.h"
#include <type_support.h>
#define ZERO (0x0)
#define ONE (1U)
-static const ib_buffer_t IB_BUFFER_NULL = {0, 0, 0 };
+static const isp2400_ib_buffer_t IB_BUFFER_NULL = {0, 0, 0 };
-static input_system_error_t input_system_configure_channel(
+static input_system_err_t input_system_configure_channel(
const channel_cfg_t channel);
-static input_system_error_t input_system_configure_channel_sensor(
+static input_system_err_t input_system_configure_channel_sensor(
const channel_cfg_t channel);
-static input_system_error_t input_buffer_configuration(void);
+static input_system_err_t input_buffer_configuration(void);
-static input_system_error_t configuration_to_registers(void);
+static input_system_err_t configuration_to_registers(void);
static void receiver_rst(const rx_ID_t ID);
static void input_system_network_rst(const input_system_ID_t ID);
static void capture_unit_configure(
const input_system_ID_t ID,
const sub_system_ID_t sub_id,
- const ib_buffer_t *const cfg);
+ const isp2400_ib_buffer_t *const cfg);
static void acquisition_unit_configure(
const input_system_ID_t ID,
const sub_system_ID_t sub_id,
- const ib_buffer_t *const cfg);
+ const isp2400_ib_buffer_t *const cfg);
static void ctrl_unit_configure(
const input_system_ID_t ID,
const input_system_network_cfg_t *const cfg);
// MW: CSI is previously named as "rx" short for "receiver"
-static input_system_error_t set_csi_cfg(
+static input_system_err_t set_csi_cfg(
csi_cfg_t *const lhs,
const csi_cfg_t *const rhs,
input_system_config_flags_t *const flags);
-static input_system_error_t set_source_type(
+static input_system_err_t set_source_type(
input_system_source_t *const lhs,
const input_system_source_t rhs,
input_system_config_flags_t *const flags);
-static input_system_error_t input_system_multiplexer_cfg(
+static input_system_err_t input_system_multiplexer_cfg(
input_system_multiplex_t *const lhs,
const input_system_multiplex_t rhs,
input_system_config_flags_t *const flags);
}
// Function that resets current configuration.
-input_system_error_t input_system_configuration_reset(void)
+input_system_err_t input_system_configuration_reset(void)
{
unsigned int i;
// MW: Comments are good, but doxygen is required, place it at the declaration
// Function that appends the channel to current configuration.
-static input_system_error_t input_system_configure_channel(
+static input_system_err_t input_system_configure_channel(
const channel_cfg_t channel)
{
- input_system_error_t error = INPUT_SYSTEM_ERR_NO_ERROR;
+ input_system_err_t error = INPUT_SYSTEM_ERR_NO_ERROR;
// Check if channel is not already configured.
if (config.ch_flags[channel.ch_id] & INPUT_SYSTEM_CFG_FLAG_SET) {
return INPUT_SYSTEM_ERR_CHANNEL_ALREADY_SET;
}
// Function that partitions input buffer space with determining addresses.
-static input_system_error_t input_buffer_configuration(void)
+static input_system_err_t input_buffer_configuration(void)
{
u32 current_address = 0;
u32 unallocated_memory = IB_CAPACITY_IN_WORDS;
- ib_buffer_t candidate_buffer_acq = IB_BUFFER_NULL;
+ isp2400_ib_buffer_t candidate_buffer_acq = IB_BUFFER_NULL;
u32 size_requested;
input_system_config_flags_t acq_already_specified = INPUT_SYSTEM_CFG_FLAG_RESET;
input_system_csi_port_t port;
static void capture_unit_configure(
const input_system_ID_t ID,
const sub_system_ID_t sub_id,
- const ib_buffer_t *const cfg)
+ const isp2400_ib_buffer_t *const cfg)
{
assert(ID < N_INPUT_SYSTEM_ID);
assert(/*(sub_id >= CAPTURE_UNIT0_ID) &&*/ (sub_id <=
static void acquisition_unit_configure(
const input_system_ID_t ID,
const sub_system_ID_t sub_id,
- const ib_buffer_t *const cfg)
+ const isp2400_ib_buffer_t *const cfg)
{
assert(ID < N_INPUT_SYSTEM_ID);
assert(sub_id == ACQUISITION_UNIT0_ID);
return;
}
-static input_system_error_t configuration_to_registers(void)
+static input_system_err_t configuration_to_registers(void)
{
input_system_network_cfg_t input_system_network_cfg;
int i;
}
// Function that applies the whole configuration.
-input_system_error_t input_system_configuration_commit(void)
+input_system_err_t input_system_configuration_commit(void)
{
// The last configuration step is to configure the input buffer.
- input_system_error_t error = input_buffer_configuration();
+ input_system_err_t error = input_buffer_configuration();
if (error != INPUT_SYSTEM_ERR_NO_ERROR) {
return error;
// FIFO
-input_system_error_t input_system_csi_fifo_channel_cfg(
+input_system_err_t input_system_csi_fifo_channel_cfg(
u32 ch_id,
input_system_csi_port_t port,
backend_channel_cfg_t backend_ch,
return input_system_configure_channel(channel);
}
-input_system_error_t input_system_csi_fifo_channel_with_counting_cfg(
+input_system_err_t input_system_csi_fifo_channel_with_counting_cfg(
u32 ch_id,
u32 nof_frames,
input_system_csi_port_t port,
// SRAM
-input_system_error_t input_system_csi_sram_channel_cfg(
+input_system_err_t input_system_csi_sram_channel_cfg(
u32 ch_id,
input_system_csi_port_t port,
backend_channel_cfg_t backend_ch,
//XMEM
// Collects all parameters and puts them in channel_cfg_t.
-input_system_error_t input_system_csi_xmem_channel_cfg(
+input_system_err_t input_system_csi_xmem_channel_cfg(
u32 ch_id,
input_system_csi_port_t port,
backend_channel_cfg_t backend_ch,
return input_system_configure_channel(channel);
}
-input_system_error_t input_system_csi_xmem_acquire_only_channel_cfg(
+input_system_err_t input_system_csi_xmem_acquire_only_channel_cfg(
u32 ch_id,
u32 nof_frames,
input_system_csi_port_t port,
return input_system_configure_channel(channel);
}
-input_system_error_t input_system_csi_xmem_capture_only_channel_cfg(
+input_system_err_t input_system_csi_xmem_capture_only_channel_cfg(
u32 ch_id,
u32 nof_frames,
input_system_csi_port_t port,
// Non - CSI
-input_system_error_t input_system_prbs_channel_cfg(
+input_system_err_t input_system_prbs_channel_cfg(
u32 ch_id,
u32 nof_frames,//not used yet
u32 seed,
return input_system_configure_channel(channel);
}
-input_system_error_t input_system_tpg_channel_cfg(
+input_system_err_t input_system_tpg_channel_cfg(
u32 ch_id,
u32 nof_frames,//not used yet
u32 x_mask,
}
// MW: Don't use system specific names, (even in system specific files) "cfg2400" -> cfg
-input_system_error_t input_system_gpfifo_channel_cfg(
+input_system_err_t input_system_gpfifo_channel_cfg(
u32 ch_id,
u32 nof_frames, //not used yet
///////////////////////////////////////////////////////////////////////////
// Fills the parameters to config.csi_value[port]
-static input_system_error_t input_system_configure_channel_sensor(
+static input_system_err_t input_system_configure_channel_sensor(
const channel_cfg_t channel)
{
const u32 port = channel.source_cfg.csi_cfg.csi_port;
- input_system_error_t status = INPUT_SYSTEM_ERR_NO_ERROR;
+ input_system_err_t status = INPUT_SYSTEM_ERR_NO_ERROR;
input_system_multiplex_t mux;
}
// Test flags and set structure.
-static input_system_error_t set_source_type(
+static input_system_err_t set_source_type(
input_system_source_t *const lhs,
const input_system_source_t rhs,
input_system_config_flags_t *const flags)
}
// Test flags and set structure.
-static input_system_error_t set_csi_cfg(
+static input_system_err_t set_csi_cfg(
csi_cfg_t *const lhs,
const csi_cfg_t *const rhs,
input_system_config_flags_t *const flags)
}
// Test flags and set structure.
-static input_system_error_t input_system_multiplexer_cfg(
+static input_system_err_t input_system_multiplexer_cfg(
input_system_multiplex_t *const lhs,
const input_system_multiplex_t rhs,
input_system_config_flags_t *const flags)
#ifndef __CSI_RX_PUBLIC_H_INCLUDED__
#define __CSI_RX_PUBLIC_H_INCLUDED__
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/*****************************************************
*
* Native command interface (NCI).
const hrt_address reg,
const hrt_data value);
/* end of DLI */
-#endif /* USE_INPUT_SYSTEM_VERSION_2401 */
+#endif /* ISP2401 */
#endif /* __CSI_RX_PUBLIC_H_INCLUDED__ */
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IBUF_CTRL_PUBLIC_H_INCLUDED__
-#define __IBUF_CTRL_PUBLIC_H_INCLUDED__
-
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-/*****************************************************
- *
- * Native command interface (NCI).
- *
- *****************************************************/
-/**
- * @brief Get the ibuf-controller state.
- * Get the state of the ibuf-controller regiester-set.
- *
- * @param[in] id The global unique ID of the input-buffer controller.
- * @param[out] state Point to the register-state.
- */
-STORAGE_CLASS_IBUF_CTRL_H void ibuf_ctrl_get_state(
- const ibuf_ctrl_ID_t ID,
- ibuf_ctrl_state_t *state);
-
-/**
- * @brief Get the state of the ibuf-controller process.
- * Get the state of the register set per buf-controller process.
- *
- * @param[in] id The global unique ID of the input-buffer controller.
- * @param[in] proc_id The process ID.
- * @param[out] state Point to the process state.
- */
-STORAGE_CLASS_IBUF_CTRL_H void ibuf_ctrl_get_proc_state(
- const ibuf_ctrl_ID_t ID,
- const u32 proc_id,
- ibuf_ctrl_proc_state_t *state);
-/**
- * @brief Dump the ibuf-controller state.
- * Dump the state of the ibuf-controller regiester-set.
- *
- * @param[in] id The global unique ID of the input-buffer controller.
- * @param[in] state Pointer to the register-state.
- */
-STORAGE_CLASS_IBUF_CTRL_H void ibuf_ctrl_dump_state(
- const ibuf_ctrl_ID_t ID,
- ibuf_ctrl_state_t *state);
-/* end of NCI */
-
-/*****************************************************
- *
- * Device level interface (DLI).
- *
- *****************************************************/
-/**
- * @brief Load the register value.
- * Load the value of the register of the ibuf-controller.
- *
- * @param[in] ID The global unique ID for the ibuf-controller instance.
- * @param[in] reg The offset address of the register.
- *
- * @return the value of the register.
- */
-STORAGE_CLASS_IBUF_CTRL_H hrt_data ibuf_ctrl_reg_load(
- const ibuf_ctrl_ID_t ID,
- const hrt_address reg);
-
-/**
- * @brief Store a value to the register.
- * Store a value to the registe of the ibuf-controller.
- *
- * @param[in] ID The global unique ID for the ibuf-controller instance.
- * @param[in] reg The offset address of the register.
- * @param[in] value The value to be stored.
- *
- */
-STORAGE_CLASS_IBUF_CTRL_H void ibuf_ctrl_reg_store(
- const ibuf_ctrl_ID_t ID,
- const hrt_address reg,
- const hrt_data value);
-/* end of DLI */
-
-#endif /* USE_INPUT_SYSTEM_VERSION_2401 */
-#endif /* __IBUF_CTRL_PUBLIC_H_INCLUDED__ */
#ifndef __ISYS_DMA_PUBLIC_H_INCLUDED__
#define __ISYS_DMA_PUBLIC_H_INCLUDED__
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
#include "system_local.h"
#include "type_support.h"
-STORAGE_CLASS_ISYS2401_DMA_H void isys2401_dma_reg_store(
+extern void isys2401_dma_reg_store(
const isys2401_dma_ID_t dma_id,
const unsigned int reg,
const hrt_data value);
-STORAGE_CLASS_ISYS2401_DMA_H hrt_data isys2401_dma_reg_load(
+extern hrt_data isys2401_dma_reg_load(
const isys2401_dma_ID_t dma_id,
const unsigned int reg);
const isys2401_dma_ID_t dma_id,
uint32_t max_burst_size);
-#endif /* USE_INPUT_SYSTEM_VERSION_2401 */
+#endif /* ISP2401 */
#endif /* __ISYS_DMA_PUBLIC_H_INCLUDED__ */
#include "isys_irq_global.h"
#include "isys_irq_local.h"
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
-STORAGE_CLASS_ISYS2401_IRQ_H void isys_irqc_state_get(
- const isys_irq_ID_t isys_irqc_id,
- isys_irqc_state_t *state);
+void isys_irqc_state_get(const isys_irq_ID_t isys_irqc_id,
+ isys_irqc_state_t *state);
-STORAGE_CLASS_ISYS2401_IRQ_H void isys_irqc_state_dump(
- const isys_irq_ID_t isys_irqc_id,
- const isys_irqc_state_t *state);
+void isys_irqc_state_dump(const isys_irq_ID_t isys_irqc_id,
+ const isys_irqc_state_t *state);
-STORAGE_CLASS_ISYS2401_IRQ_H void isys_irqc_reg_store(
- const isys_irq_ID_t isys_irqc_id,
- const unsigned int reg_idx,
- const hrt_data value);
+void isys_irqc_reg_store(const isys_irq_ID_t isys_irqc_id,
+ const unsigned int reg_idx,
+ const hrt_data value);
-STORAGE_CLASS_ISYS2401_IRQ_H hrt_data isys_irqc_reg_load(
- const isys_irq_ID_t isys_irqc_id,
- const unsigned int reg_idx);
+hrt_data isys_irqc_reg_load(const isys_irq_ID_t isys_irqc_id,
+ const unsigned int reg_idx);
-STORAGE_CLASS_ISYS2401_IRQ_H void isys_irqc_status_enable(
- const isys_irq_ID_t isys_irqc_id);
+void isys_irqc_status_enable(const isys_irq_ID_t isys_irqc_id);
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2401) */
+#endif /* defined(ISP2401) */
#endif /* __ISYS_IRQ_PUBLIC_H__ */
#ifndef __ISYS_PUBLIC_H_INCLUDED__
#define __ISYS_PUBLIC_H_INCLUDED__
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/*! Read the state of INPUT_SYSTEM[ID]
\param ID[in] INPUT_SYSTEM identifier
\param state[out] pointer to input system state structure
STORAGE_CLASS_INPUT_SYSTEM_H void input_system_dump_state(
const input_system_ID_t ID,
input_system_state_t *state);
-#endif /* USE_INPUT_SYSTEM_VERSION_2401 */
+#endif /* ISP2401 */
#endif /* __ISYS_PUBLIC_H_INCLUDED__ */
#ifndef __PIXELGEN_PUBLIC_H_INCLUDED__
#define __PIXELGEN_PUBLIC_H_INCLUDED__
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/*****************************************************
*
* Native command interface (NCI).
const hrt_data value);
/* end of DLI */
-#endif /* USE_INPUT_SYSTEM_VERSION_2401 */
+#endif /* ISP2401 */
#endif /* __PIXELGEN_PUBLIC_H_INCLUDED__ */
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- */
-
-#ifndef __IBUF_CTRL_H_INCLUDED__
-#define __IBUF_CTRL_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- * - system and cell agnostic interfaces, constants and identifiers
- * - public: system agnostic, cell specific interfaces
- * - private: system dependent, cell specific interfaces &
- * inline implementations
- * - global: system specific constants and identifiers
- * - local: system and cell specific constants and identifiers
- */
-
-#include "system_local.h"
-#include "ibuf_ctrl_local.h"
-
-#ifndef __INLINE_IBUF_CTRL__
-#define STORAGE_CLASS_IBUF_CTRL_H extern
-#define STORAGE_CLASS_IBUF_CTRL_C
-#include "ibuf_ctrl_public.h"
-#else /* __INLINE_IBUF_CTRL__ */
-#define STORAGE_CLASS_IBUF_CTRL_H static inline
-#define STORAGE_CLASS_IBUF_CTRL_C static inline
-#include "ibuf_ctrl_private.h"
-#endif /* __INLINE_IBUF_CTRL__ */
-
-#endif /* __IBUF_CTRL_H_INCLUDED__ */
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- */
-
-#ifndef __ISYS_DMA_H_INCLUDED__
-#define __ISYS_DMA_H_INCLUDED__
-
-/*
- * This file is included on every cell {SP,ISP,host} and on every system
- * that uses the input system device(s). It defines the API to DLI bridge
- *
- * System and cell specific interfaces and inline code are included
- * conditionally through Makefile path settings.
- *
- * - system and cell agnostic interfaces, constants and identifiers
- * - public: system agnostic, cell specific interfaces
- * - private: system dependent, cell specific interfaces &
- * inline implementations
- * - global: system specific constants and identifiers
- * - local: system and cell specific constants and identifiers
- */
-
-#include "system_local.h"
-#include "isys_dma_local.h"
-
-#ifndef __INLINE_ISYS2401_DMA__
-#define STORAGE_CLASS_ISYS2401_DMA_H extern
-#define STORAGE_CLASS_ISYS2401_DMA_C
-#include "isys_dma_public.h"
-#else /* __INLINE_ISYS2401_DMA__ */
-#define STORAGE_CLASS_ISYS2401_DMA_H static inline
-#define STORAGE_CLASS_ISYS2401_DMA_C static inline
-#include "isys_dma_private.h"
-#endif /* __INLINE_ISYS2401_DMA__ */
-
-#endif /* __ISYS_DMA_H_INCLUDED__ */
#include <type_support.h>
#include <system_local.h>
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
-#ifndef __INLINE_ISYS2401_IRQ__
-
-#define STORAGE_CLASS_ISYS2401_IRQ_H extern
-#define STORAGE_CLASS_ISYS2401_IRQ_C extern
#include "isys_irq_public.h"
-#else /* __INLINE_ISYS2401_IRQ__ */
-
-#define STORAGE_CLASS_ISYS2401_IRQ_H static inline
-#define STORAGE_CLASS_ISYS2401_IRQ_C static inline
-#include "isys_irq_private.h"
-
-#endif /* __INLINE_ISYS2401_IRQ__ */
-
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2401) */
+#endif /* defined(ISP2401) */
#endif /* __IA_CSS_ISYS_IRQ_H__ */
extern int (*sh_css_printf)(const char *fmt, va_list args);
/* depends on host supplied print function in ia_css_init() */
-static inline void ia_css_print(const char *fmt, ...)
+static inline __printf(1, 2) void ia_css_print(const char *fmt, ...)
{
va_list ap;
if (attrs & ATOMISP_MAP_FLAG_CLEARED)
hmm_set(bo->start, 0, bytes);
- dev_dbg(atomisp_dev,
- "%s: pages: 0x%08x (%ld bytes), type: %d from highmem %d, user ptr %p, cached %d\n",
- __func__, bo->start, bytes, type, from_highmem, userptr, cached);
+ dev_dbg(atomisp_dev,
+ "%s: pages: 0x%08x (%ld bytes), type: %d from highmem %d, user ptr %p, cached %d\n",
+ __func__, bo->start, bytes, type, from_highmem, userptr, cached);
return bo->start;
/* Environment with function pointers to print error and debug messages.
*/
struct ia_css_print_env {
- int (*debug_print)(const char *fmt, va_list args);
+ int __printf(1, 0) (*debug_print)(const char *fmt, va_list args);
/** Print a debug message. */
- int (*error_print)(const char *fmt, va_list args);
+ int __printf(1, 0) (*error_print)(const char *fmt, va_list args);
/** Print an error message.*/
};
ia_css_mipi_frame_specify(const unsigned int size_mem_words,
const bool contiguous);
-#if !defined(HAS_NO_INPUT_SYSTEM)
/* @brief Register size of a CSS MIPI frame for check during capturing.
*
* @param[in] port CSI-2 port this check is registered.
int
ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port,
const unsigned int size_mem_words);
-#endif
/* @brief Calculate the size of a mipi frame.
*
#include <type_support.h>
#include <system_local.h>
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if !defined(ISP2401)
#include <input_system.h>
#endif
#include "ia_css_types.h"
struct ia_css_stream {
struct ia_css_stream_config config;
struct ia_css_stream_info info;
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if !defined(ISP2401)
rx_cfg_t csi_rx_config;
#endif
bool reconfigure_css_rx;
* (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
*/
-#ifdef ISP2401
-# include "isp2401_input_system_global.h"
-#else
-# include "isp2400_input_system_global.h"
-#endif
+
+#ifndef __INPUT_SYSTEM_GLOBAL_H_INCLUDED__
+#define __INPUT_SYSTEM_GLOBAL_H_INCLUDED__
+typedef enum {
+ INPUT_SYSTEM_ERR_NO_ERROR = 0,
+ /* ISP2401 */
+ INPUT_SYSTEM_ERR_CREATE_CHANNEL_FAIL,
+ INPUT_SYSTEM_ERR_CONFIGURE_CHANNEL_FAIL,
+ INPUT_SYSTEM_ERR_OPEN_CHANNEL_FAIL,
+ INPUT_SYSTEM_ERR_TRANSFER_FAIL,
+ INPUT_SYSTEM_ERR_CREATE_INPUT_PORT_FAIL,
+ INPUT_SYSTEM_ERR_CONFIGURE_INPUT_PORT_FAIL,
+ INPUT_SYSTEM_ERR_OPEN_INPUT_PORT_FAIL,
+ /* ISP2400 */
+ INPUT_SYSTEM_ERR_GENERIC,
+ INPUT_SYSTEM_ERR_CHANNEL_ALREADY_SET,
+ INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE,
+ INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED,
+} input_system_err_t;
+
+#include "isp2401_input_system_global.h"
+#include "isp2400_input_system_global.h"
+
+#endif /* __INPUT_SYSTEM_GLOBAL_H_INCLUDED__ */
#include "ia_css_ctc_types.h"
#ifndef PIPE_GENERATION
-#if defined(HAS_VAMEM_VERSION_2)
#define SH_CSS_ISP_CTC_TABLE_SIZE_LOG2 IA_CSS_VAMEM_2_CTC_TABLE_SIZE_LOG2
#define SH_CSS_ISP_CTC_TABLE_SIZE IA_CSS_VAMEM_2_CTC_TABLE_SIZE
-#elif defined(HAS_VAMEM_VERSION_1)
-#define SH_CSS_ISP_CTC_TABLE_SIZE_LOG2 IA_CSS_VAMEM_1_CTC_TABLE_SIZE_LOG2
-#define SH_CSS_ISP_CTC_TABLE_SIZE IA_CSS_VAMEM_1_CTC_TABLE_SIZE
-#else
-#error "VAMEM should be {VERSION1, VERSION2}"
-#endif
#else
/* For pipe generation, the size is not relevant */
struct ia_css_ctc_table default_ctc_table;
-#if defined(HAS_VAMEM_VERSION_2)
static const uint16_t
default_ctc_table_data[IA_CSS_VAMEM_2_CTC_TABLE_SIZE] = {
0
};
-#elif defined(HAS_VAMEM_VERSION_1)
-
-/* Default Parameters */
-static const uint16_t
-default_ctc_table_data[IA_CSS_VAMEM_1_CTC_TABLE_SIZE] = {
- 0, 0, 256, 384, 384, 497, 765, 806,
- 837, 851, 888, 901, 957, 981, 993, 1001,
- 1011, 1029, 1028, 1039, 1062, 1059, 1073, 1080,
- 1083, 1085, 1085, 1098, 1080, 1084, 1085, 1093,
- 1078, 1073, 1070, 1069, 1077, 1066, 1072, 1063,
- 1053, 1044, 1046, 1053, 1039, 1028, 1025, 1024,
- 1012, 1013, 1016, 996, 992, 990, 990, 980,
- 969, 968, 961, 955, 951, 949, 933, 930,
- 929, 925, 921, 916, 906, 901, 895, 893,
- 886, 877, 872, 869, 866, 861, 857, 849,
- 845, 838, 836, 832, 823, 821, 815, 813,
- 809, 805, 796, 793, 790, 785, 784, 778,
- 772, 768, 766, 763, 758, 752, 749, 745,
- 741, 740, 736, 730, 726, 724, 723, 718,
- 711, 709, 706, 704, 701, 698, 691, 689,
- 688, 683, 683, 678, 675, 673, 671, 669,
- 666, 663, 661, 660, 656, 656, 653, 650,
- 648, 647, 646, 643, 639, 638, 637, 635,
- 633, 632, 629, 627, 626, 625, 622, 621,
- 618, 618, 614, 614, 612, 609, 606, 606,
- 603, 600, 600, 597, 594, 591, 590, 586,
- 582, 581, 578, 575, 572, 569, 563, 560,
- 557, 554, 551, 548, 545, 539, 536, 533,
- 529, 527, 524, 519, 516, 513, 510, 507,
- 504, 501, 498, 493, 491, 488, 485, 484,
- 480, 476, 474, 471, 467, 466, 464, 460,
- 459, 455, 453, 449, 447, 446, 443, 441,
- 438, 435, 432, 432, 429, 427, 426, 422,
- 419, 418, 416, 414, 412, 410, 408, 406,
- 404, 402, 401, 398, 397, 395, 393, 390,
- 389, 388, 387, 384, 382, 380, 378, 377,
- 376, 375, 372, 370, 368, 368, 366, 364,
- 363, 361, 360, 358, 357, 355, 354, 352,
- 351, 350, 349, 346, 345, 344, 344, 342,
- 340, 339, 337, 337, 336, 335, 333, 331,
- 330, 329, 328, 326, 326, 324, 324, 322,
- 321, 320, 318, 318, 318, 317, 315, 313,
- 312, 311, 311, 310, 308, 307, 306, 306,
- 304, 304, 302, 301, 300, 300, 299, 297,
- 297, 296, 296, 294, 294, 292, 291, 291,
- 291, 290, 288, 287, 286, 286, 287, 285,
- 284, 283, 282, 282, 281, 281, 279, 278,
- 278, 278, 276, 276, 275, 274, 274, 273,
- 271, 270, 269, 268, 268, 267, 265, 262,
- 261, 260, 260, 259, 257, 254, 252, 252,
- 251, 251, 249, 246, 245, 244, 243, 242,
- 240, 239, 239, 237, 235, 235, 233, 231,
- 232, 230, 229, 226, 225, 224, 225, 224,
- 223, 220, 219, 219, 218, 217, 217, 214,
- 213, 213, 212, 211, 209, 209, 209, 208,
- 206, 205, 204, 203, 204, 203, 201, 200,
- 199, 197, 198, 198, 197, 195, 194, 194,
- 193, 192, 192, 191, 189, 190, 189, 188,
- 186, 187, 186, 185, 185, 184, 183, 181,
- 183, 182, 181, 180, 179, 178, 178, 178,
- 177, 176, 175, 176, 175, 174, 174, 173,
- 172, 173, 172, 171, 170, 170, 169, 169,
- 169, 168, 167, 166, 167, 167, 166, 165,
- 164, 164, 164, 163, 164, 163, 162, 163,
- 162, 161, 160, 161, 160, 160, 160, 159,
- 158, 157, 158, 158, 157, 157, 156, 156,
- 156, 156, 155, 155, 154, 154, 154, 154,
- 154, 153, 152, 153, 152, 152, 151, 152,
- 151, 152, 151, 150, 150, 149, 149, 150,
- 149, 149, 148, 148, 148, 149, 148, 147,
- 146, 146, 147, 146, 147, 146, 145, 146,
- 146, 145, 144, 145, 144, 145, 144, 144,
- 143, 143, 143, 144, 143, 142, 142, 142,
- 142, 142, 142, 141, 141, 141, 141, 140,
- 140, 141, 140, 140, 141, 140, 139, 139,
- 139, 140, 139, 139, 138, 138, 137, 139,
- 138, 138, 138, 137, 138, 137, 137, 137,
- 137, 136, 137, 136, 136, 136, 136, 135,
- 136, 135, 135, 135, 135, 136, 135, 135,
- 134, 134, 133, 135, 134, 134, 134, 133,
- 134, 133, 134, 133, 133, 132, 133, 133,
- 132, 133, 132, 132, 132, 132, 131, 131,
- 131, 132, 131, 131, 130, 131, 130, 132,
- 131, 130, 130, 129, 130, 129, 130, 129,
- 129, 129, 130, 129, 128, 128, 128, 128,
- 129, 128, 128, 127, 127, 128, 128, 127,
- 127, 126, 126, 127, 127, 126, 126, 126,
- 127, 126, 126, 126, 125, 125, 126, 125,
- 125, 124, 124, 124, 125, 125, 124, 124,
- 123, 124, 124, 123, 123, 122, 122, 122,
- 122, 122, 121, 120, 120, 119, 118, 118,
- 118, 117, 117, 116, 115, 115, 115, 114,
- 114, 113, 113, 112, 111, 111, 111, 110,
- 110, 109, 109, 108, 108, 108, 107, 107,
- 106, 106, 105, 105, 105, 104, 104, 103,
- 103, 102, 102, 102, 102, 101, 101, 100,
- 100, 99, 99, 99, 99, 99, 99, 98,
- 97, 98, 97, 97, 97, 96, 96, 95,
- 96, 95, 96, 95, 95, 94, 94, 95,
- 94, 94, 94, 93, 93, 92, 93, 93,
- 93, 93, 92, 92, 91, 92, 92, 92,
- 91, 91, 90, 90, 91, 91, 91, 90,
- 90, 90, 90, 91, 90, 90, 90, 89,
- 89, 89, 90, 89, 89, 89, 89, 89,
- 88, 89, 89, 88, 88, 88, 88, 87,
- 89, 88, 88, 88, 88, 88, 87, 88,
- 88, 88, 87, 87, 87, 87, 87, 88,
- 87, 87, 87, 87, 87, 87, 88, 87,
- 87, 87, 87, 86, 86, 87, 87, 87,
- 87, 86, 86, 86, 87, 87, 86, 87,
- 86, 86, 86, 87, 87, 86, 86, 86,
- 86, 86, 87, 87, 86, 85, 85, 85,
- 84, 85, 85, 84, 84, 83, 83, 82,
- 82, 82, 81, 81, 80, 79, 79, 79,
- 78, 77, 77, 76, 76, 76, 75, 74,
- 74, 74, 73, 73, 72, 71, 71, 71,
- 70, 70, 69, 69, 68, 68, 67, 67,
- 67, 66, 66, 65, 65, 64, 64, 63,
- 62, 62, 62, 61, 60, 60, 59, 59,
- 58, 58, 57, 57, 56, 56, 56, 55,
- 55, 54, 55, 55, 54, 53, 53, 52,
- 53, 53, 52, 51, 51, 50, 51, 50,
- 49, 49, 50, 49, 49, 48, 48, 47,
- 47, 48, 46, 45, 45, 45, 46, 45,
- 45, 44, 45, 45, 45, 43, 42, 42,
- 41, 43, 41, 40, 40, 39, 40, 41,
- 39, 39, 39, 39, 39, 38, 35, 35,
- 34, 37, 36, 34, 33, 33, 33, 35,
- 34, 32, 32, 31, 32, 30, 29, 26,
- 25, 25, 27, 26, 23, 23, 23, 25,
- 24, 24, 22, 21, 20, 19, 16, 14,
- 13, 13, 13, 10, 9, 7, 7, 7,
- 12, 12, 12, 7, 0, 0, 0, 0
-};
-
-#else
-#error "VAMEM version must be one of {VAMEM_VERSION_1, VAMEM_VERSION_2}"
-#endif
void
ia_css_config_ctc_table(void)
{
-#if defined(HAS_VAMEM_VERSION_2)
memcpy(default_ctc_table.data.vamem_2, default_ctc_table_data,
sizeof(default_ctc_table_data));
default_ctc_table.vamem_type = IA_CSS_VAMEM_TYPE_2;
-#else
- memcpy(default_ctc_table.data.vamem_1, default_ctc_table_data,
- sizeof(default_ctc_table_data));
- default_ctc_table.vamem_type = 1IA_CSS_VAMEM_TYPE_1;
-#endif
}
unsigned int o_width;
unsigned int o_height;
struct ia_css_host_data *me;
- struct gdc_warp_param_mem_s *isp_data_ptr;
assert(binary);
assert(dvs_6axis_config);
assert((dvs_in_frame_info->format == IA_CSS_FRAME_FORMAT_NV12)
|| (dvs_in_frame_info->format == IA_CSS_FRAME_FORMAT_YUV420));
- isp_data_ptr = (struct gdc_warp_param_mem_s *)me->address;
-
i_stride = dvs_in_frame_info->padded_width;
o_width = binary->out_frame_info[0].res.width;
#include "ia_css_types.h"
#include "ia_css_gc_table.host.h"
-#if defined(HAS_VAMEM_VERSION_2)
struct ia_css_gamma_table default_gamma_table;
255
};
-#elif defined(HAS_VAMEM_VERSION_1)
-
-static const uint16_t
-default_gamma_table_data[IA_CSS_VAMEM_1_GAMMA_TABLE_SIZE] = {
- 0, 1, 2, 3, 4, 5, 6, 7,
- 8, 9, 10, 11, 12, 13, 14, 16,
- 17, 18, 19, 20, 21, 23, 24, 25,
- 27, 28, 29, 31, 32, 33, 35, 36,
- 38, 39, 41, 42, 44, 45, 47, 48,
- 49, 51, 52, 54, 55, 57, 58, 60,
- 61, 62, 64, 65, 66, 68, 69, 70,
- 71, 72, 74, 75, 76, 77, 78, 79,
- 80, 81, 82, 83, 84, 85, 86, 87,
- 88, 89, 90, 91, 92, 93, 93, 94,
- 95, 96, 97, 98, 98, 99, 100, 101,
- 102, 102, 103, 104, 105, 105, 106, 107,
- 108, 108, 109, 110, 110, 111, 112, 112,
- 113, 114, 114, 115, 116, 116, 117, 118,
- 118, 119, 120, 120, 121, 121, 122, 123,
- 123, 124, 125, 125, 126, 126, 127, 127, /* 128 */
- 128, 129, 129, 130, 130, 131, 131, 132,
- 132, 133, 134, 134, 135, 135, 136, 136,
- 137, 137, 138, 138, 139, 139, 140, 140,
- 141, 141, 142, 142, 143, 143, 144, 144,
- 145, 145, 145, 146, 146, 147, 147, 148,
- 148, 149, 149, 150, 150, 150, 151, 151,
- 152, 152, 152, 153, 153, 154, 154, 155,
- 155, 155, 156, 156, 156, 157, 157, 158,
- 158, 158, 159, 159, 160, 160, 160, 161,
- 161, 161, 162, 162, 162, 163, 163, 163,
- 164, 164, 164, 165, 165, 165, 166, 166,
- 166, 167, 167, 167, 168, 168, 168, 169,
- 169, 169, 170, 170, 170, 170, 171, 171,
- 171, 172, 172, 172, 172, 173, 173, 173,
- 174, 174, 174, 174, 175, 175, 175, 176,
- 176, 176, 176, 177, 177, 177, 177, 178, /* 256 */
- 178, 178, 178, 179, 179, 179, 179, 180,
- 180, 180, 180, 181, 181, 181, 181, 182,
- 182, 182, 182, 182, 183, 183, 183, 183,
- 184, 184, 184, 184, 184, 185, 185, 185,
- 185, 186, 186, 186, 186, 186, 187, 187,
- 187, 187, 187, 188, 188, 188, 188, 188,
- 189, 189, 189, 189, 189, 190, 190, 190,
- 190, 190, 191, 191, 191, 191, 191, 192,
- 192, 192, 192, 192, 192, 193, 193, 193,
- 193, 193, 194, 194, 194, 194, 194, 194,
- 195, 195, 195, 195, 195, 195, 196, 196,
- 196, 196, 196, 196, 197, 197, 197, 197,
- 197, 197, 198, 198, 198, 198, 198, 198,
- 198, 199, 199, 199, 199, 199, 199, 200,
- 200, 200, 200, 200, 200, 200, 201, 201,
- 201, 201, 201, 201, 201, 202, 202, 202, /* 384 */
- 202, 202, 202, 202, 203, 203, 203, 203,
- 203, 203, 203, 204, 204, 204, 204, 204,
- 204, 204, 204, 205, 205, 205, 205, 205,
- 205, 205, 205, 206, 206, 206, 206, 206,
- 206, 206, 206, 207, 207, 207, 207, 207,
- 207, 207, 207, 208, 208, 208, 208, 208,
- 208, 208, 208, 209, 209, 209, 209, 209,
- 209, 209, 209, 209, 210, 210, 210, 210,
- 210, 210, 210, 210, 210, 211, 211, 211,
- 211, 211, 211, 211, 211, 211, 212, 212,
- 212, 212, 212, 212, 212, 212, 212, 213,
- 213, 213, 213, 213, 213, 213, 213, 213,
- 214, 214, 214, 214, 214, 214, 214, 214,
- 214, 214, 215, 215, 215, 215, 215, 215,
- 215, 215, 215, 216, 216, 216, 216, 216,
- 216, 216, 216, 216, 216, 217, 217, 217, /* 512 */
- 217, 217, 217, 217, 217, 217, 217, 218,
- 218, 218, 218, 218, 218, 218, 218, 218,
- 218, 219, 219, 219, 219, 219, 219, 219,
- 219, 219, 219, 220, 220, 220, 220, 220,
- 220, 220, 220, 220, 220, 221, 221, 221,
- 221, 221, 221, 221, 221, 221, 221, 221,
- 222, 222, 222, 222, 222, 222, 222, 222,
- 222, 222, 223, 223, 223, 223, 223, 223,
- 223, 223, 223, 223, 223, 224, 224, 224,
- 224, 224, 224, 224, 224, 224, 224, 224,
- 225, 225, 225, 225, 225, 225, 225, 225,
- 225, 225, 225, 226, 226, 226, 226, 226,
- 226, 226, 226, 226, 226, 226, 226, 227,
- 227, 227, 227, 227, 227, 227, 227, 227,
- 227, 227, 228, 228, 228, 228, 228, 228,
- 228, 228, 228, 228, 228, 228, 229, 229,
- 229, 229, 229, 229, 229, 229, 229, 229,
- 229, 229, 230, 230, 230, 230, 230, 230,
- 230, 230, 230, 230, 230, 230, 231, 231,
- 231, 231, 231, 231, 231, 231, 231, 231,
- 231, 231, 231, 232, 232, 232, 232, 232,
- 232, 232, 232, 232, 232, 232, 232, 233,
- 233, 233, 233, 233, 233, 233, 233, 233,
- 233, 233, 233, 233, 234, 234, 234, 234,
- 234, 234, 234, 234, 234, 234, 234, 234,
- 234, 235, 235, 235, 235, 235, 235, 235,
- 235, 235, 235, 235, 235, 235, 236, 236,
- 236, 236, 236, 236, 236, 236, 236, 236,
- 236, 236, 236, 236, 237, 237, 237, 237,
- 237, 237, 237, 237, 237, 237, 237, 237,
- 237, 237, 238, 238, 238, 238, 238, 238,
- 238, 238, 238, 238, 238, 238, 238, 238,
- 239, 239, 239, 239, 239, 239, 239, 239,
- 239, 239, 239, 239, 239, 239, 240, 240,
- 240, 240, 240, 240, 240, 240, 240, 240,
- 240, 240, 240, 240, 241, 241, 241, 241,
- 241, 241, 241, 241, 241, 241, 241, 241,
- 241, 241, 241, 242, 242, 242, 242, 242,
- 242, 242, 242, 242, 242, 242, 242, 242,
- 242, 242, 243, 243, 243, 243, 243, 243,
- 243, 243, 243, 243, 243, 243, 243, 243,
- 243, 244, 244, 244, 244, 244, 244, 244,
- 244, 244, 244, 244, 244, 244, 244, 244,
- 245, 245, 245, 245, 245, 245, 245, 245,
- 245, 245, 245, 245, 245, 245, 245, 246,
- 246, 246, 246, 246, 246, 246, 246, 246,
- 246, 246, 246, 246, 246, 246, 246, 247,
- 247, 247, 247, 247, 247, 247, 247, 247,
- 247, 247, 247, 247, 247, 247, 247, 248,
- 248, 248, 248, 248, 248, 248, 248, 248,
- 248, 248, 248, 248, 248, 248, 248, 249,
- 249, 249, 249, 249, 249, 249, 249, 249,
- 249, 249, 249, 249, 249, 249, 249, 250,
- 250, 250, 250, 250, 250, 250, 250, 250,
- 250, 250, 250, 250, 250, 250, 250, 251,
- 251, 251, 251, 251, 251, 251, 251, 251,
- 251, 251, 251, 251, 251, 251, 251, 252,
- 252, 252, 252, 252, 252, 252, 252, 252,
- 252, 252, 252, 252, 252, 252, 252, 253,
- 253, 253, 253, 253, 253, 253, 253, 253,
- 253, 253, 253, 253, 253, 253, 253, 253,
- 254, 254, 254, 254, 254, 254, 254, 254,
- 254, 254, 254, 254, 254, 254, 254, 254,
- 255, 255, 255, 255, 255, 255, 255, 255
-};
-
-#else
-#error "VAMEM version must be one of {VAMEM_VERSION_1, VAMEM_VERSION_2}"
-#endif
void
ia_css_config_gamma_table(void)
{
-#if defined(HAS_VAMEM_VERSION_2)
memcpy(default_gamma_table.data.vamem_2, default_gamma_table_data,
sizeof(default_gamma_table_data));
default_gamma_table.vamem_type = IA_CSS_VAMEM_TYPE_2;
-#else
- memcpy(default_gamma_table.data.vamem_1, default_gamma_table_data,
- sizeof(default_gamma_table_data));
- default_gamma_table.vamem_type = IA_CSS_VAMEM_TYPE_1;
-#endif
}
/* Identical default gamma table for R, G, and B. */
-#if defined(HAS_VAMEM_VERSION_2)
static const uint16_t
default_gamma_table_data[IA_CSS_VAMEM_2_RGB_GAMMA_TABLE_SIZE] = {
4032, 4040, 4048, 4056, 4064, 4072, 4080, 4088,
4095
};
-#elif defined(HAS_VAMEM_VERSION_1)
-
-static const uint16_t
-default_gamma_table_data[IA_CSS_VAMEM_1_RGB_GAMMA_TABLE_SIZE] = {
- 0, 72, 144, 216, 288, 360, 426, 486,
- 541, 592, 641, 687, 730, 772, 812, 850,
- 887, 923, 958, 991, 1024, 1055, 1086, 1117,
- 1146, 1175, 1203, 1230, 1257, 1284, 1310, 1335,
- 1360, 1385, 1409, 1433, 1457, 1480, 1502, 1525,
- 1547, 1569, 1590, 1612, 1632, 1653, 1674, 1694,
- 1714, 1734, 1753, 1772, 1792, 1811, 1829, 1848,
- 1866, 1884, 1902, 1920, 1938, 1955, 1973, 1990,
- 2007, 2024, 2040, 2057, 2074, 2090, 2106, 2122,
- 2138, 2154, 2170, 2185, 2201, 2216, 2231, 2247,
- 2262, 2277, 2291, 2306, 2321, 2335, 2350, 2364,
- 2378, 2393, 2407, 2421, 2435, 2449, 2462, 2476,
- 2490, 2503, 2517, 2530, 2543, 2557, 2570, 2583,
- 2596, 2609, 2622, 2634, 2647, 2660, 2673, 2685,
- 2698, 2710, 2722, 2735, 2747, 2759, 2771, 2783,
- 2795, 2807, 2819, 2831, 2843, 2855, 2867, 2878,
- 2890, 2901, 2913, 2924, 2936, 2947, 2958, 2970,
- 2981, 2992, 3003, 3014, 3025, 3036, 3047, 3058,
- 3069, 3080, 3091, 3102, 3112, 3123, 3134, 3144,
- 3155, 3165, 3176, 3186, 3197, 3207, 3217, 3228,
- 3238, 3248, 3258, 3268, 3279, 3289, 3299, 3309,
- 3319, 3329, 3339, 3349, 3358, 3368, 3378, 3388,
- 3398, 3407, 3417, 3427, 3436, 3446, 3455, 3465,
- 3474, 3484, 3493, 3503, 3512, 3521, 3531, 3540,
- 3549, 3559, 3568, 3577, 3586, 3595, 3605, 3614,
- 3623, 3632, 3641, 3650, 3659, 3668, 3677, 3686,
- 3694, 3703, 3712, 3721, 3730, 3739, 3747, 3756,
- 3765, 3773, 3782, 3791, 3799, 3808, 3816, 3825,
- 3833, 3842, 3850, 3859, 3867, 3876, 3884, 3893,
- 3901, 3909, 3918, 3926, 3934, 3942, 3951, 3959,
- 3967, 3975, 3984, 3992, 4000, 4008, 4016, 4024,
- 4032, 4040, 4048, 4056, 4064, 4072, 4080, 4088
-};
-#else
-#error "VAMEM version must be one of {VAMEM_VERSION_1, VAMEM_VERSION_2}"
-#endif
void
ia_css_config_rgb_gamma_tables(void)
{
-#if defined(HAS_VAMEM_VERSION_2)
default_r_gamma_table.vamem_type = IA_CSS_VAMEM_TYPE_2;
default_g_gamma_table.vamem_type = IA_CSS_VAMEM_TYPE_2;
default_b_gamma_table.vamem_type = IA_CSS_VAMEM_TYPE_2;
sizeof(default_gamma_table_data));
memcpy(default_b_gamma_table.data.vamem_2, default_gamma_table_data,
sizeof(default_gamma_table_data));
-#else
- memcpy(default_r_gamma_table.data.vamem_1, default_gamma_table_data,
- sizeof(default_gamma_table_data));
- memcpy(default_g_gamma_table.data.vamem_1, default_gamma_table_data,
- sizeof(default_gamma_table_data));
- memcpy(default_b_gamma_table.data.vamem_1, default_gamma_table_data,
- sizeof(default_gamma_table_data));
- default_r_gamma_table.vamem_type = IA_CSS_VAMEM_TYPE_1;
- default_g_gamma_table.vamem_type = IA_CSS_VAMEM_TYPE_1;
- default_b_gamma_table.vamem_type = IA_CSS_VAMEM_TYPE_1;
-#endif
}
const struct ia_css_frame_info *internal_info = from->internal_info;
(void)size;
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if !defined(ISP2401)
/* 2401 input system uses input width width */
in_info = internal_info;
#else
to->two_ppc = from->two_ppc;
to->stream_format = css2isp_stream_format(from->stream_format);
to->deinterleaved = from->deinterleaved;
-#if (defined(USE_INPUT_SYSTEM_VERSION_2401) || defined(CONFIG_CSI2_PLUS))
+#if defined(ISP2401)
to->start_column = in_info->crop_info.start_column;
to->start_line = in_info->crop_info.start_line;
to->enable_left_padding = from->enable_left_padding;
{
struct ia_css_isp_parameters *params;
unsigned int hor_num_3a, ver_num_3a;
- unsigned int hor_num_isp, ver_num_isp;
struct ia_css_binary *dvs_binary;
IA_CSS_ENTER("void");
hor_num_3a = dvs_binary->dis.coef.dim.width;
ver_num_3a = dvs_binary->dis.coef.dim.height;
- hor_num_isp = dvs_binary->dis.coef.pad.width;
- ver_num_isp = dvs_binary->dis.coef.pad.height;
memcpy(hor_coefs_odd_real, params->dvs2_coefs.hor_coefs.odd_real,
hor_num_3a * sizeof(short));
* more details.
*/
+#include "atomisp_internal.h"
+
#include "ia_css_vf.host.h"
#include <assert_support.h>
#include <ia_css_err.h>
unsigned int ds_log2 = 0;
unsigned int out_width;
- if ((!out_info) | (!vf_info))
+ if ((!out_info) || (!vf_info))
return -EINVAL;
out_width = out_info->res.width;
const struct ia_css_binary_info *info = &binary->info->sp;
err = configure_kernel(info, out_info, vf_info, downscale_log2, &config);
+ if (err)
+ dev_warn(atomisp_dev, "Couldn't setup downscale\n");
+
configure_dma(&config, vf_info);
if (vf_info)
#include <system_global.h>
#ifndef PIPE_GENERATION
-#if defined(HAS_VAMEM_VERSION_2)
#define SH_CSS_ISP_XNR_TABLE_SIZE_LOG2 IA_CSS_VAMEM_2_XNR_TABLE_SIZE_LOG2
#define SH_CSS_ISP_XNR_TABLE_SIZE IA_CSS_VAMEM_2_XNR_TABLE_SIZE
-#elif defined(HAS_VAMEM_VERSION_1)
-#define SH_CSS_ISP_XNR_TABLE_SIZE_LOG2 IA_CSS_VAMEM_1_XNR_TABLE_SIZE_LOG2
-#define SH_CSS_ISP_XNR_TABLE_SIZE IA_CSS_VAMEM_1_XNR_TABLE_SIZE
-#else
-#error "Unknown vamem type"
-#endif
#else
/* For pipe generation, the size is not relevant */
struct ia_css_xnr_table default_xnr_table;
-#if defined(HAS_VAMEM_VERSION_2)
static const uint16_t
default_xnr_table_data[IA_CSS_VAMEM_2_XNR_TABLE_SIZE] = {
167 >> 1, 163 >> 1, 160 >> 1, 157 >> 1, 154 >> 1, 151 >> 1, 148 >> 1, 146 >> 1, 143 >> 1, 141 >> 1, 138 >> 1, 136 >> 1, 134 >> 1, 132 >> 1, 130 >> 1, 128 >> 1
};
-#elif defined(HAS_VAMEM_VERSION_1)
-
-static const uint16_t
-default_xnr_table_data[IA_CSS_VAMEM_1_XNR_TABLE_SIZE] = {
- /* 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 */
- 8191 >> 1, 4096 >> 1, 2730 >> 1, 2048 >> 1, 1638 >> 1, 1365 >> 1, 1170 >> 1, 1024 >> 1, 910 >> 1, 819 >> 1, 744 >> 1, 682 >> 1, 630 >> 1, 585 >> 1,
- 546 >> 1, 512 >> 1,
-
- /* 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 */
- 481 >> 1, 455 >> 1, 431 >> 1, 409 >> 1, 390 >> 1, 372 >> 1, 356 >> 1, 341 >> 1, 327 >> 1, 315 >> 1, 303 >> 1, 292 >> 1, 282 >> 1, 273 >> 1, 264 >> 1,
- 256 >> 1,
-
- /* 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 */
- 248 >> 1, 240 >> 1, 234 >> 1, 227 >> 1, 221 >> 1, 215 >> 1, 210 >> 1, 204 >> 1, 199 >> 1, 195 >> 1, 190 >> 1, 186 >> 1, 182 >> 1, 178 >> 1, 174 >> 1,
- 170 >> 1,
-
- /* 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 */
- 167 >> 1, 163 >> 1, 160 >> 1, 157 >> 1, 154 >> 1, 151 >> 1, 148 >> 1, 146 >> 1, 143 >> 1, 141 >> 1, 138 >> 1, 136 >> 1, 134 >> 1, 132 >> 1, 130 >> 1, 128 >> 1
-};
-
-#else
-#error "sh_css_params.c: VAMEM version must \
-be one of {VAMEM_VERSION_1, VAMEM_VERSION_2}"
-#endif
void
ia_css_config_xnr_table(void)
{
-#if defined(HAS_VAMEM_VERSION_2)
memcpy(default_xnr_table.data.vamem_2, default_xnr_table_data,
sizeof(default_xnr_table_data));
default_xnr_table.vamem_type = IA_CSS_VAMEM_TYPE_2;
-#else
- memcpy(default_xnr_table.data.vamem_1, default_xnr_table_data,
- sizeof(default_xnr_table_data));
- default_xnr_table.vamem_type = IA_CSS_VAMEM_TYPE_1;
-#endif
}
* more details.
*/
-#ifndef __INPUT_SYSTEM_GLOBAL_H_INCLUDED__
-#define __INPUT_SYSTEM_GLOBAL_H_INCLUDED__
-
-#define IS_INPUT_SYSTEM_VERSION_2
-
#include <type_support.h>
//CSI reveiver has 3 ports.
N_INPUT_SYSTEM_BUFFERING_MODE
} buffering_mode_t;
-typedef struct input_system_cfg_s input_system_cfg_t;
+typedef struct isp2400_input_system_cfg_s input_system_cfg_t;
typedef struct sync_generator_cfg_s sync_generator_cfg_t;
typedef struct tpg_cfg_s tpg_cfg_t;
typedef struct prbs_cfg_s prbs_cfg_t;
/* MW: uint16_t should be sufficient */
-struct input_system_cfg_s {
+struct isp2400_input_system_cfg_s {
u32 no_side_band;
u32 fmt_type;
u32 ch_id;
typedef struct gpfifo_cfg_s gpfifo_cfg_t;
//ALX:Commented out to pass the compilation.
-//typedef struct input_system_cfg_s input_system_cfg_t;
+//typedef struct isp2400_input_system_cfg_s input_system_cfg_t;
struct ib_buffer_s {
u32 mem_reg_size;
u32 mem_reg_addr;
};
-typedef struct ib_buffer_s ib_buffer_t;
+typedef struct ib_buffer_s isp2400_ib_buffer_t;
struct csi_cfg_s {
u32 csi_port;
buffering_mode_t buffering_mode;
- ib_buffer_t csi_buffer;
- ib_buffer_t acquisition_buffer;
+ isp2400_ib_buffer_t csi_buffer;
+ isp2400_ib_buffer_t acquisition_buffer;
u32 nof_xmem_buffers;
};
INPUT_SYSTEM_CFG_FLAG_BLOCKED = 1U << 1,
INPUT_SYSTEM_CFG_FLAG_REQUIRED = 1U << 2,
INPUT_SYSTEM_CFG_FLAG_CONFLICT = 1U << 3 // To mark a conflicting configuration.
-} input_system_cfg_flag_t;
+} isp2400_input_system_cfg_flag_t;
typedef u32 input_system_config_flags_t;
-
-#endif /* __INPUT_SYSTEM_GLOBAL_H_INCLUDED__ */
#include "isp_acquisition_defs.h"
#include "input_system_ctrl_defs.h"
-typedef enum {
- INPUT_SYSTEM_ERR_NO_ERROR = 0,
- INPUT_SYSTEM_ERR_GENERIC,
- INPUT_SYSTEM_ERR_CHANNEL_ALREADY_SET,
- INPUT_SYSTEM_ERR_CONFLICT_ON_RESOURCE,
- INPUT_SYSTEM_ERR_PARAMETER_NOT_SUPPORTED,
- N_INPUT_SYSTEM_ERR
-} input_system_error_t;
-
typedef enum {
INPUT_SYSTEM_PORT_A = 0,
INPUT_SYSTEM_PORT_B,
typedef struct input_switch_cfg_s input_switch_cfg_t;
struct ctrl_unit_cfg_s {
- ib_buffer_t buffer_mipi[N_CAPTURE_UNIT_ID];
- ib_buffer_t buffer_acquire[N_ACQUISITION_UNIT_ID];
+ isp2400_ib_buffer_t buffer_mipi[N_CAPTURE_UNIT_ID];
+ isp2400_ib_buffer_t buffer_acquire[N_ACQUISITION_UNIT_ID];
};
struct input_system_network_cfg_s {
// Possible another struct for ib.
// This buffers set at the end, based on the all configurations.
- ib_buffer_t csi_buffer[N_CSI_PORTS];
+ isp2400_ib_buffer_t csi_buffer[N_CSI_PORTS];
input_system_config_flags_t csi_buffer_flags[N_CSI_PORTS];
- ib_buffer_t acquisition_buffer_unique;
+ isp2400_ib_buffer_t acquisition_buffer_unique;
input_system_config_flags_t acquisition_buffer_unique_flags;
u32 unallocated_ib_mem_words; // Used for check.DEFAULT = IB_CAPACITY_IN_WORDS.
//uint32_t acq_allocated_ib_mem_words;
#define __INPUT_SYSTEM_PUBLIC_H_INCLUDED__
#include <type_support.h>
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
#include "isys_public.h"
#else
// Function that resets current configuration.
// remove the argument since it should be private.
-input_system_error_t input_system_configuration_reset(void);
+input_system_err_t input_system_configuration_reset(void);
// Function that commits current configuration.
// remove the argument since it should be private.
-input_system_error_t input_system_configuration_commit(void);
+input_system_err_t input_system_configuration_commit(void);
///////////////////////////////////////////////////////////////////////////
//
// FIFO channel config function user
-input_system_error_t input_system_csi_fifo_channel_cfg(
+input_system_err_t input_system_csi_fifo_channel_cfg(
u32 ch_id,
input_system_csi_port_t port,
backend_channel_cfg_t backend_ch,
target_cfg2400_t target
);
-input_system_error_t input_system_csi_fifo_channel_with_counting_cfg(
+input_system_err_t input_system_csi_fifo_channel_with_counting_cfg(
u32 ch_id,
u32 nof_frame,
input_system_csi_port_t port,
// SRAM channel config function user
-input_system_error_t input_system_csi_sram_channel_cfg(
+input_system_err_t input_system_csi_sram_channel_cfg(
u32 ch_id,
input_system_csi_port_t port,
backend_channel_cfg_t backend_ch,
//XMEM channel config function user
-input_system_error_t input_system_csi_xmem_channel_cfg(
+input_system_err_t input_system_csi_xmem_channel_cfg(
u32 ch_id,
input_system_csi_port_t port,
backend_channel_cfg_t backend_ch,
uint32_t nof_xmem_buffers
);
-input_system_error_t input_system_csi_xmem_capture_only_channel_cfg(
+input_system_err_t input_system_csi_xmem_capture_only_channel_cfg(
u32 ch_id,
u32 nof_frames,
input_system_csi_port_t port,
target_cfg2400_t target
);
-input_system_error_t input_system_csi_xmem_acquire_only_channel_cfg(
+input_system_err_t input_system_csi_xmem_acquire_only_channel_cfg(
u32 ch_id,
u32 nof_frames,
input_system_csi_port_t port,
// Non - CSI channel config function user
-input_system_error_t input_system_prbs_channel_cfg(
+input_system_err_t input_system_prbs_channel_cfg(
u32 ch_id,
u32 nof_frames,
u32 seed,
target_cfg2400_t target
);
-input_system_error_t input_system_tpg_channel_cfg(
+input_system_err_t input_system_tpg_channel_cfg(
u32 ch_id,
u32 nof_frames,//not used yet
u32 x_mask,
target_cfg2400_t target
);
-input_system_error_t input_system_gpfifo_channel_cfg(
+input_system_err_t input_system_gpfifo_channel_cfg(
u32 ch_id,
u32 nof_frames,
target_cfg2400_t target
);
-#endif /* #ifdef USE_INPUT_SYSTEM_VERSION_2401 */
+#endif /* #ifdef ISP2401 */
#endif /* __INPUT_SYSTEM_PUBLIC_H_INCLUDED__ */
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- */
-
-#define USE_INPUT_SYSTEM_VERSION_2
* more details.
*/
-#ifndef __INPUT_SYSTEM_GLOBAL_H_INCLUDED__
-#define __INPUT_SYSTEM_GLOBAL_H_INCLUDED__
-
-#define IS_INPUT_SYSTEM_VERSION_VERSION_2401
-
/* CSI reveiver has 3 ports. */
#define N_CSI_PORTS (3)
-#include "isys_dma.h" /* isys2401_dma_channel,
+#include "system_local.h"
+#include "isys_dma_global.h" /* isys2401_dma_channel,
* isys2401_dma_cfg_t
*/
-#include "ibuf_ctrl.h" /* ibuf_cfg_t,
+#include "ibuf_ctrl_local.h" /* ibuf_cfg_t,
* ibuf_ctrl_cfg_t
*/
#define INPUT_SYSTEM_N_STREAM_ID 6 /* maximum number of simultaneous
virtual channels supported*/
-typedef enum {
- INPUT_SYSTEM_ERR_NO_ERROR = 0,
- INPUT_SYSTEM_ERR_CREATE_CHANNEL_FAIL,
- INPUT_SYSTEM_ERR_CONFIGURE_CHANNEL_FAIL,
- INPUT_SYSTEM_ERR_OPEN_CHANNEL_FAIL,
- INPUT_SYSTEM_ERR_TRANSFER_FAIL,
- INPUT_SYSTEM_ERR_CREATE_INPUT_PORT_FAIL,
- INPUT_SYSTEM_ERR_CONFIGURE_INPUT_PORT_FAIL,
- INPUT_SYSTEM_ERR_OPEN_INPUT_PORT_FAIL,
- N_INPUT_SYSTEM_ERR
-} input_system_err_t;
-
typedef enum {
INPUT_SYSTEM_SOURCE_TYPE_UNDEFINED = 0,
INPUT_SYSTEM_SOURCE_TYPE_SENSOR,
stream2mmio_sid_ID_t stream2mmio_sid_id;
ibuf_ctrl_ID_t ibuf_ctrl_id;
- ib_buffer_t ib_buffer;
+ isp2401_ib_buffer_t ib_buffer;
isys2401_dma_ID_t dma_id;
isys2401_dma_channel dma_channel;
} pixelgen_cfg;
};
-typedef struct input_system_cfg_s input_system_cfg_t;
-struct input_system_cfg_s {
+typedef struct isp2401_input_system_cfg_s isp2401_input_system_cfg_t;
+struct isp2401_input_system_cfg_s {
input_system_input_port_ID_t input_port_id;
input_system_source_type_t mode;
#define NUM_OF_LINES_PER_BUF 2
#define LINES_OF_ISP_INPUT_BUF (NUM_OF_INPUT_BUF * NUM_OF_LINES_PER_BUF)
#define ISP_INPUT_BUF_STRIDE SH_CSS_MAX_SENSOR_WIDTH
-
-#endif /* __INPUT_SYSTEM_GLOBAL_H_INCLUDED__ */
#include "type_support.h"
#include "input_system_global.h"
-#include "ibuf_ctrl.h"
#include "csi_rx.h"
#include "pixelgen.h"
#include "isys_stream2mmio.h"
#include "isys_irq.h"
-typedef input_system_err_t input_system_error_t;
-
typedef enum {
MIPI_FORMAT_SHORT1 = 0x08,
MIPI_FORMAT_SHORT2,
#include "input_system_public.h"
-STORAGE_CLASS_INPUT_SYSTEM_C input_system_err_t input_system_get_state(
- const input_system_ID_t ID,
- input_system_state_t *state)
+#include "device_access.h" /* ia_css_device_load_uint32 */
+
+#include "assert_support.h" /* assert */
+#include "print_support.h" /* print */
+
+/* Load the register value */
+static inline hrt_data ibuf_ctrl_reg_load(const ibuf_ctrl_ID_t ID,
+ const hrt_address reg)
+{
+ assert(ID < N_IBUF_CTRL_ID);
+ assert(IBUF_CTRL_BASE[ID] != (hrt_address)-1);
+ return ia_css_device_load_uint32(IBUF_CTRL_BASE[ID] + reg * sizeof(hrt_data));
+}
+
+/* Store a value to the register */
+static inline void ibuf_ctrl_reg_store(const ibuf_ctrl_ID_t ID,
+ const hrt_address reg,
+ const hrt_data value)
+{
+ assert(ID < N_IBUF_CTRL_ID);
+ assert(IBUF_CTRL_BASE[ID] != (hrt_address)-1);
+
+ ia_css_device_store_uint32(IBUF_CTRL_BASE[ID] + reg * sizeof(hrt_data), value);
+}
+
+/* Get the state of the ibuf-controller process */
+static inline void ibuf_ctrl_get_proc_state(const ibuf_ctrl_ID_t ID,
+ const u32 proc_id,
+ ibuf_ctrl_proc_state_t *state)
+{
+ hrt_address reg_bank_offset;
+
+ reg_bank_offset =
+ _IBUF_CNTRL_PROC_REG_ALIGN * (1 + proc_id);
+
+ state->num_items =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_NUM_ITEMS_PER_STORE);
+
+ state->num_stores =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_NUM_STORES_PER_FRAME);
+
+ state->dma_channel =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DMA_CHANNEL);
+
+ state->dma_command =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DMA_CMD);
+
+ state->ibuf_st_addr =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_BUFFER_START_ADDRESS);
+
+ state->ibuf_stride =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_BUFFER_STRIDE);
+
+ state->ibuf_end_addr =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_BUFFER_END_ADDRESS);
+
+ state->dest_st_addr =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DEST_START_ADDRESS);
+
+ state->dest_stride =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DEST_STRIDE);
+
+ state->dest_end_addr =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DEST_END_ADDRESS);
+
+ state->sync_frame =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_SYNC_FRAME);
+
+ state->sync_command =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_STR2MMIO_SYNC_CMD);
+
+ state->store_command =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_STR2MMIO_STORE_CMD);
+
+ state->shift_returned_items =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_SHIFT_ITEMS);
+
+ state->elems_ibuf =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_ELEMS_P_WORD_IBUF);
+
+ state->elems_dest =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_ELEMS_P_WORD_DEST);
+
+ state->cur_stores =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_STORES);
+
+ state->cur_acks =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_ACKS);
+
+ state->cur_s2m_ibuf_addr =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_S2M_IBUF_ADDR);
+
+ state->cur_dma_ibuf_addr =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_DMA_IBUF_ADDR);
+
+ state->cur_dma_dest_addr =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_DMA_DEST_ADDR);
+
+ state->cur_isp_dest_addr =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_ISP_DEST_ADDR);
+
+ state->dma_cmds_send =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_CUR_NR_DMA_CMDS_SEND);
+
+ state->main_cntrl_state =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_MAIN_CNTRL_STATE);
+
+ state->dma_sync_state =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_DMA_SYNC_STATE);
+
+ state->isp_sync_state =
+ ibuf_ctrl_reg_load(ID, reg_bank_offset + _IBUF_CNTRL_ISP_SYNC_STATE);
+}
+
+/* Get the ibuf-controller state. */
+static inline void ibuf_ctrl_get_state(const ibuf_ctrl_ID_t ID,
+ ibuf_ctrl_state_t *state)
+{
+ u32 i;
+
+ state->recalc_words =
+ ibuf_ctrl_reg_load(ID, _IBUF_CNTRL_RECALC_WORDS_STATUS);
+ state->arbiters =
+ ibuf_ctrl_reg_load(ID, _IBUF_CNTRL_ARBITERS_STATUS);
+
+ /*
+ * Get the values of the register-set per
+ * ibuf-controller process.
+ */
+ for (i = 0; i < N_IBUF_CTRL_PROCS[ID]; i++) {
+ ibuf_ctrl_get_proc_state(
+ ID,
+ i,
+ &state->proc_state[i]);
+ }
+}
+
+/* Dump the ibuf-controller state */
+static inline void ibuf_ctrl_dump_state(const ibuf_ctrl_ID_t ID,
+ ibuf_ctrl_state_t *state)
+{
+ u32 i;
+
+ ia_css_print("IBUF controller ID %d recalculate words 0x%x\n", ID,
+ state->recalc_words);
+ ia_css_print("IBUF controller ID %d arbiters 0x%x\n", ID, state->arbiters);
+
+ /*
+ * Dump the values of the register-set per
+ * ibuf-controller process.
+ */
+ for (i = 0; i < N_IBUF_CTRL_PROCS[ID]; i++) {
+ ia_css_print("IBUF controller ID %d Process ID %d num_items 0x%x\n", ID, i,
+ state->proc_state[i].num_items);
+ ia_css_print("IBUF controller ID %d Process ID %d num_stores 0x%x\n", ID, i,
+ state->proc_state[i].num_stores);
+ ia_css_print("IBUF controller ID %d Process ID %d dma_channel 0x%x\n", ID, i,
+ state->proc_state[i].dma_channel);
+ ia_css_print("IBUF controller ID %d Process ID %d dma_command 0x%x\n", ID, i,
+ state->proc_state[i].dma_command);
+ ia_css_print("IBUF controller ID %d Process ID %d ibuf_st_addr 0x%x\n", ID, i,
+ state->proc_state[i].ibuf_st_addr);
+ ia_css_print("IBUF controller ID %d Process ID %d ibuf_stride 0x%x\n", ID, i,
+ state->proc_state[i].ibuf_stride);
+ ia_css_print("IBUF controller ID %d Process ID %d ibuf_end_addr 0x%x\n", ID, i,
+ state->proc_state[i].ibuf_end_addr);
+ ia_css_print("IBUF controller ID %d Process ID %d dest_st_addr 0x%x\n", ID, i,
+ state->proc_state[i].dest_st_addr);
+ ia_css_print("IBUF controller ID %d Process ID %d dest_stride 0x%x\n", ID, i,
+ state->proc_state[i].dest_stride);
+ ia_css_print("IBUF controller ID %d Process ID %d dest_end_addr 0x%x\n", ID, i,
+ state->proc_state[i].dest_end_addr);
+ ia_css_print("IBUF controller ID %d Process ID %d sync_frame 0x%x\n", ID, i,
+ state->proc_state[i].sync_frame);
+ ia_css_print("IBUF controller ID %d Process ID %d sync_command 0x%x\n", ID, i,
+ state->proc_state[i].sync_command);
+ ia_css_print("IBUF controller ID %d Process ID %d store_command 0x%x\n", ID, i,
+ state->proc_state[i].store_command);
+ ia_css_print("IBUF controller ID %d Process ID %d shift_returned_items 0x%x\n",
+ ID, i,
+ state->proc_state[i].shift_returned_items);
+ ia_css_print("IBUF controller ID %d Process ID %d elems_ibuf 0x%x\n", ID, i,
+ state->proc_state[i].elems_ibuf);
+ ia_css_print("IBUF controller ID %d Process ID %d elems_dest 0x%x\n", ID, i,
+ state->proc_state[i].elems_dest);
+ ia_css_print("IBUF controller ID %d Process ID %d cur_stores 0x%x\n", ID, i,
+ state->proc_state[i].cur_stores);
+ ia_css_print("IBUF controller ID %d Process ID %d cur_acks 0x%x\n", ID, i,
+ state->proc_state[i].cur_acks);
+ ia_css_print("IBUF controller ID %d Process ID %d cur_s2m_ibuf_addr 0x%x\n", ID,
+ i,
+ state->proc_state[i].cur_s2m_ibuf_addr);
+ ia_css_print("IBUF controller ID %d Process ID %d cur_dma_ibuf_addr 0x%x\n", ID,
+ i,
+ state->proc_state[i].cur_dma_ibuf_addr);
+ ia_css_print("IBUF controller ID %d Process ID %d cur_dma_dest_addr 0x%x\n", ID,
+ i,
+ state->proc_state[i].cur_dma_dest_addr);
+ ia_css_print("IBUF controller ID %d Process ID %d cur_isp_dest_addr 0x%x\n", ID,
+ i,
+ state->proc_state[i].cur_isp_dest_addr);
+ ia_css_print("IBUF controller ID %d Process ID %d dma_cmds_send 0x%x\n", ID, i,
+ state->proc_state[i].dma_cmds_send);
+ ia_css_print("IBUF controller ID %d Process ID %d main_cntrl_state 0x%x\n", ID,
+ i,
+ state->proc_state[i].main_cntrl_state);
+ ia_css_print("IBUF controller ID %d Process ID %d dma_sync_state 0x%x\n", ID, i,
+ state->proc_state[i].dma_sync_state);
+ ia_css_print("IBUF controller ID %d Process ID %d isp_sync_state 0x%x\n", ID, i,
+ state->proc_state[i].isp_sync_state);
+ }
+}
+
+static inline input_system_err_t
+input_system_get_state(const input_system_ID_t ID,
+ input_system_state_t *state)
{
u32 i;
return INPUT_SYSTEM_ERR_NO_ERROR;
}
-STORAGE_CLASS_INPUT_SYSTEM_C void input_system_dump_state(
- const input_system_ID_t ID,
- input_system_state_t *state)
+static inline void input_system_dump_state(const input_system_ID_t ID,
+ input_system_state_t *state)
{
u32 i;
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Support for Intel Camera Imaging ISP subsystem.
- * Copyright (c) 2015, Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- */
-
-#define HAS_NO_INPUT_FORMATTER
-#define USE_INPUT_SYSTEM_VERSION_2401
-#define HAS_INPUT_SYSTEM_VERSION_2401
-#define CSI2P_DISABLE_ISYS2401_ONLINE_MODE
at shading correction. */
};
-/* Get the requirements for the shading correction. */
+/* ISP2400: Get the requirements for the shading correction. */
static int
-#ifndef ISP2401
ia_css_binary_compute_shading_table_bayer_origin(
const struct ia_css_binary *binary, /* [in] */
unsigned int required_bds_factor, /* [in] */
const struct ia_css_stream_config *stream_config, /* [in] */
struct sh_css_shading_table_bayer_origin_compute_results *res) /* [out] */
-#else
-sh_css_binary_get_sc_requirements(
- const struct ia_css_binary *binary, /* [in] */
- unsigned int required_bds_factor, /* [in] */
- const struct ia_css_stream_config *stream_config, /* [in] */
- struct sh_css_binary_sc_requirements *scr) /* [out] */
-#endif
{
int err;
-#ifndef ISP2401
/* Numerator and denominator of the fixed bayer downscaling factor.
(numerator >= denominator) */
-#else
- /* Numerator and denominator of the fixed bayer downscaling factor. (numerator >= denominator) */
-#endif
unsigned int bds_num, bds_den;
-#ifndef ISP2401
/* Horizontal/Vertical ratio of bayer scaling
between input area and output area. */
unsigned int bs_hor_ratio_in;
unsigned int bs_hor_ratio_out;
unsigned int bs_ver_ratio_in;
unsigned int bs_ver_ratio_out;
-#else
- /* Horizontal/Vertical ratio of bayer scaling between input area and output area. */
- unsigned int bs_hor_ratio_in, bs_hor_ratio_out, bs_ver_ratio_in, bs_ver_ratio_out;
-#endif
/* Left padding set by InputFormatter. */
-#ifndef ISP2401
unsigned int left_padding_bqs; /* in bqs */
-#else
- unsigned int left_padding_bqs;
-#endif
-#ifndef ISP2401
/* Flag for the NEED_BDS_FACTOR_2_00 macro defined in isp kernels. */
unsigned int need_bds_factor_2_00;
err = sh_css_bds_factor_get_numerator_denominator
(required_bds_factor, &bds_num, &bds_den);
if (err)
-#else
+ return err;
+
+ /* Set the horizontal/vertical ratio of bayer scaling
+ between input area and output area. */
+ bs_hor_ratio_in = bds_num;
+ bs_hor_ratio_out = bds_den;
+ bs_ver_ratio_in = bds_num;
+ bs_ver_ratio_out = bds_den;
+
+ /* Set the left padding set by InputFormatter. (ifmtr.c) */
+ if (stream_config->left_padding == -1)
+ left_padding_bqs = _ISP_BQS(binary->left_padding);
+ else
+ left_padding_bqs = (unsigned int)((int)ISP_VEC_NELEMS
+ - _ISP_BQS(stream_config->left_padding));
+
+ /* Set the left padding adjusted inside the isp.
+ When bds_factor 2.00 is needed, some padding is added to left_padding
+ inside the isp, before bayer downscaling. (raw.isp.c)
+ (Hopefully, left_crop/left_padding/top_crop should be defined in css
+ appropriately, depending on bds_factor.)
+ */
+ need_bds_factor_2_00 = ((binary->info->sp.bds.supported_bds_factors &
+ (PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_00) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_50) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_3_00) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_4_00) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_4_50) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_5_00) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_6_00) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_8_00))) != 0);
+
+ if (need_bds_factor_2_00 && binary->info->sp.pipeline.left_cropping > 0)
+ left_padding_adjusted_bqs = left_padding_bqs + ISP_VEC_NELEMS;
+ else
+ left_padding_adjusted_bqs = left_padding_bqs;
+
+ /* Currently, the bad pixel caused by filters before bayer scaling
+ is NOT considered, because the bad pixel is subtle.
+ When some large filter is used in the future,
+ we need to consider the bad pixel.
+
+ Currently, when bds_factor isn't 1.00, 3x3 anti-alias filter is applied
+ to each color plane(Gr/R/B/Gb) before bayer downscaling.
+ This filter moves each color plane to right/bottom directions
+ by 1 pixel at the most, depending on downscaling factor.
+ */
+ bad_bqs_on_left_before_bs = 0;
+ bad_bqs_on_top_before_bs = 0;
+
+ /* Currently, the bad pixel caused by filters after bayer scaling
+ is NOT considered, because the bad pixel is subtle.
+ When some large filter is used in the future,
+ we need to consider the bad pixel.
+
+ Currently, when DPC&BNR is processed between bayer scaling and
+ shading correction, DPC&BNR moves each color plane to
+ right/bottom directions by 1 pixel.
+ */
+ bad_bqs_on_left_after_bs = 0;
+ bad_bqs_on_top_after_bs = 0;
+
+ /* Calculate the origin of bayer (real sensor data area)
+ located on the shading table during the shading correction. */
+ res->sc_bayer_origin_x_bqs_on_shading_table =
+ ((left_padding_adjusted_bqs + bad_bqs_on_left_before_bs)
+ * bs_hor_ratio_out + bs_hor_ratio_in / 2) / bs_hor_ratio_in
+ + bad_bqs_on_left_after_bs;
+ /* "+ bs_hor_ratio_in/2": rounding for division by bs_hor_ratio_in */
+ res->sc_bayer_origin_y_bqs_on_shading_table =
+ (bad_bqs_on_top_before_bs * bs_ver_ratio_out + bs_ver_ratio_in / 2) / bs_ver_ratio_in
+ + bad_bqs_on_top_after_bs;
+ /* "+ bs_ver_ratio_in/2": rounding for division by bs_ver_ratio_in */
+
+ res->bayer_scale_hor_ratio_in = (uint32_t)bs_hor_ratio_in;
+ res->bayer_scale_hor_ratio_out = (uint32_t)bs_hor_ratio_out;
+ res->bayer_scale_ver_ratio_in = (uint32_t)bs_ver_ratio_in;
+ res->bayer_scale_ver_ratio_out = (uint32_t)bs_ver_ratio_out;
+
+ return err;
+}
+
+/* ISP2401: Get the requirements for the shading correction. */
+static int
+sh_css_binary_get_sc_requirements(const struct ia_css_binary *binary, /* [in] */
+ unsigned int required_bds_factor, /* [in] */
+ const struct ia_css_stream_config *stream_config, /* [in] */
+ struct sh_css_binary_sc_requirements *scr) /* [out] */
+{
+ int err;
+
+ /* Numerator and denominator of the fixed bayer downscaling factor. (numerator >= denominator) */
+ unsigned int bds_num, bds_den;
+
+ /* Horizontal/Vertical ratio of bayer scaling between input area and output area. */
+ unsigned int bs_hor_ratio_in, bs_hor_ratio_out, bs_ver_ratio_in, bs_ver_ratio_out;
+
+ /* Left padding set by InputFormatter. */
+ unsigned int left_padding_bqs;
+
/* Flags corresponding to NEED_BDS_FACTOR_2_00/NEED_BDS_FACTOR_1_50/NEED_BDS_FACTOR_1_25 macros
* defined in isp kernels. */
unsigned int need_bds_factor_2_00, need_bds_factor_1_50, need_bds_factor_1_25;
unsigned int sensor_data_origin_x_bqs_on_internal;
unsigned int sensor_data_origin_y_bqs_on_internal;
+ unsigned int bs_frac = bds_frac_acc; /* scaling factor 1.0 in fixed point */
+ unsigned int bs_out, bs_in; /* scaling ratio in fixed point */
+
IA_CSS_ENTER_PRIVATE("binary=%p, required_bds_factor=%d, stream_config=%p",
binary, required_bds_factor, stream_config);
/* Get the numerator and denominator of the required bayer downscaling factor. */
- err = sh_css_bds_factor_get_numerator_denominator(required_bds_factor, &bds_num, &bds_den);
- if (err)
- {
+ err = sh_css_bds_factor_get_numerator_denominator(required_bds_factor,
+ &bds_num, &bds_den);
+ if (err) {
IA_CSS_LEAVE_ERR_PRIVATE(err);
-#endif
return err;
-#ifdef ISP2401
-}
-#endif
-
-#ifndef ISP2401
-/* Set the horizontal/vertical ratio of bayer scaling
-between input area and output area. */
-#else
-IA_CSS_LOG("bds_num=%d, bds_den=%d", bds_num, bds_den);
-
-/* Set the horizontal/vertical ratio of bayer scaling between input area and output area. */
-#endif
-bs_hor_ratio_in = bds_num;
-bs_hor_ratio_out = bds_den;
-bs_ver_ratio_in = bds_num;
-bs_ver_ratio_out = bds_den;
+ }
-#ifndef ISP2401
-/* Set the left padding set by InputFormatter. (ifmtr.c) */
-#else
-/* Set the left padding set by InputFormatter. (ia_css_ifmtr_configure() in ifmtr.c) */
-#endif
-if (stream_config->left_padding == -1)
- left_padding_bqs = _ISP_BQS(binary->left_padding);
-else
-#ifndef ISP2401
- left_padding_bqs = (unsigned int)((int)ISP_VEC_NELEMS
- - _ISP_BQS(stream_config->left_padding));
-#else
- left_padding_bqs = (unsigned int)((int)ISP_VEC_NELEMS - _ISP_BQS(stream_config->left_padding));
-#endif
+ IA_CSS_LOG("bds_num=%d, bds_den=%d", bds_num, bds_den);
-#ifndef ISP2401
-/* Set the left padding adjusted inside the isp.
-When bds_factor 2.00 is needed, some padding is added to left_padding
-inside the isp, before bayer downscaling. (raw.isp.c)
-(Hopefully, left_crop/left_padding/top_crop should be defined in css
-appropriately, depending on bds_factor.)
-*/
-#else
-IA_CSS_LOG("stream.left_padding=%d, binary.left_padding=%d, left_padding_bqs=%d",
- stream_config->left_padding, binary->left_padding, left_padding_bqs);
+ /* Set the horizontal/vertical ratio of bayer scaling between input area and output area. */
+ bs_hor_ratio_in = bds_num;
+ bs_hor_ratio_out = bds_den;
+ bs_ver_ratio_in = bds_num;
+ bs_ver_ratio_out = bds_den;
-/* Set the left padding adjusted inside the isp kernels.
- * When the bds_factor isn't 1.00, the left padding size is adjusted inside the isp,
- * before bayer downscaling. (scaled_hor_plane_index(), raw_compute_hphase() in raw.isp.c)
- */
-#endif
-need_bds_factor_2_00 = ((binary->info->sp.bds.supported_bds_factors &
- (PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_00) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_50) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_3_00) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_4_00) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_4_50) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_5_00) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_6_00) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_8_00))) != 0);
+ /* Set the left padding set by InputFormatter. (ia_css_ifmtr_configure() in ifmtr.c) */
+ if (stream_config->left_padding == -1)
+ left_padding_bqs = _ISP_BQS(binary->left_padding);
+ else
+ left_padding_bqs = (unsigned int)((int)ISP_VEC_NELEMS - _ISP_BQS(stream_config->left_padding));
-#ifndef ISP2401
-if (need_bds_factor_2_00 && binary->info->sp.pipeline.left_cropping > 0)
- left_padding_adjusted_bqs = left_padding_bqs + ISP_VEC_NELEMS;
-else
-#else
-need_bds_factor_1_50 = ((binary->info->sp.bds.supported_bds_factors &
- (PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_1_50) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_25) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_3_00) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_4_50) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_6_00))) != 0);
-
-need_bds_factor_1_25 = ((binary->info->sp.bds.supported_bds_factors &
- (PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_1_25) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_50) |
- PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_5_00))) != 0);
-
-if (binary->info->sp.pipeline.left_cropping > 0 &&
- (need_bds_factor_2_00 || need_bds_factor_1_50 || need_bds_factor_1_25))
-{
- /*
- * downscale 2.0 -> first_vec_adjusted_bqs = 128
- * downscale 1.5 -> first_vec_adjusted_bqs = 96
- * downscale 1.25 -> first_vec_adjusted_bqs = 80
- */
- unsigned int first_vec_adjusted_bqs
- = ISP_VEC_NELEMS * bs_hor_ratio_in / bs_hor_ratio_out;
- left_padding_adjusted_bqs = first_vec_adjusted_bqs
- - _ISP_BQS(binary->info->sp.pipeline.left_cropping);
-} else
-#endif
- left_padding_adjusted_bqs = left_padding_bqs;
+ IA_CSS_LOG("stream.left_padding=%d, binary.left_padding=%d, left_padding_bqs=%d",
+ stream_config->left_padding, binary->left_padding,
+ left_padding_bqs);
-#ifndef ISP2401
-/* Currently, the bad pixel caused by filters before bayer scaling
-is NOT considered, because the bad pixel is subtle.
-When some large filter is used in the future,
-we need to consider the bad pixel.
-
-Currently, when bds_factor isn't 1.00, 3x3 anti-alias filter is applied
-to each color plane(Gr/R/B/Gb) before bayer downscaling.
-This filter moves each color plane to right/bottom directions
-by 1 pixel at the most, depending on downscaling factor.
-*/
-bad_bqs_on_left_before_bs = 0;
-bad_bqs_on_top_before_bs = 0;
-#else
-IA_CSS_LOG("supported_bds_factors=%d, need_bds_factor:2_00=%d, 1_50=%d, 1_25=%d",
- binary->info->sp.bds.supported_bds_factors,
- need_bds_factor_2_00, need_bds_factor_1_50, need_bds_factor_1_25);
-IA_CSS_LOG("left_cropping=%d, left_padding_adjusted_bqs=%d",
- binary->info->sp.pipeline.left_cropping, left_padding_adjusted_bqs);
-
-/* Set the top padding padded inside the isp kernel for bayer downscaling binaries.
- * When the bds_factor isn't 1.00, the top padding is padded inside the isp
- * before bayer downscaling, because the top cropping size (input margin) is not enough.
- * (calculate_input_line(), raw_compute_vphase(), dma_read_raw() in raw.isp.c)
- * NOTE: In dma_read_raw(), the factor passed to raw_compute_vphase() is got by get_bds_factor_for_dma_read().
- * This factor is BDS_FPVAL_100/BDS_FPVAL_125/BDS_FPVAL_150/BDS_FPVAL_200.
- */
-top_padding_bqs = 0;
-if (binary->info->sp.pipeline.top_cropping > 0 &&
- (required_bds_factor == SH_CSS_BDS_FACTOR_1_25 ||
- required_bds_factor == SH_CSS_BDS_FACTOR_1_50 ||
- required_bds_factor == SH_CSS_BDS_FACTOR_2_00))
-{
- /* Calculation from calculate_input_line() and raw_compute_vphase() in raw.isp.c. */
- int top_cropping_bqs = _ISP_BQS(binary->info->sp.pipeline.top_cropping);
- /* top cropping (in bqs) */
- int factor = bds_num * bds_frac_acc /
- bds_den; /* downscaling factor by fixed-point */
- int top_padding_bqsxfrac_acc = (top_cropping_bqs * factor - top_cropping_bqs *
- bds_frac_acc)
- + (2 * bds_frac_acc - factor); /* top padding by fixed-point (in bqs) */
-
- top_padding_bqs = (unsigned int)((top_padding_bqsxfrac_acc + bds_frac_acc / 2 -
- 1) / bds_frac_acc);
-}
+ /* Set the left padding adjusted inside the isp kernels.
+ * When the bds_factor isn't 1.00, the left padding size is adjusted inside the isp,
+ * before bayer downscaling. (scaled_hor_plane_index(), raw_compute_hphase() in raw.isp.c)
+ */
+ need_bds_factor_2_00 = ((binary->info->sp.bds.supported_bds_factors &
+ (PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_00) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_50) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_3_00) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_4_00) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_4_50) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_5_00) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_6_00) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_8_00))) != 0);
+
+ need_bds_factor_1_50 = ((binary->info->sp.bds.supported_bds_factors &
+ (PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_1_50) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_25) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_3_00) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_4_50) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_6_00))) != 0);
+
+ need_bds_factor_1_25 = ((binary->info->sp.bds.supported_bds_factors &
+ (PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_1_25) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_2_50) |
+ PACK_BDS_FACTOR(SH_CSS_BDS_FACTOR_5_00))) != 0);
+
+ if (binary->info->sp.pipeline.left_cropping > 0 &&
+ (need_bds_factor_2_00 || need_bds_factor_1_50 || need_bds_factor_1_25)) {
+ /*
+ * downscale 2.0 -> first_vec_adjusted_bqs = 128
+ * downscale 1.5 -> first_vec_adjusted_bqs = 96
+ * downscale 1.25 -> first_vec_adjusted_bqs = 80
+ */
+ unsigned int first_vec_adjusted_bqs = ISP_VEC_NELEMS * bs_hor_ratio_in / bs_hor_ratio_out;
+ left_padding_adjusted_bqs = first_vec_adjusted_bqs
+ - _ISP_BQS(binary->info->sp.pipeline.left_cropping);
+ } else {
+ left_padding_adjusted_bqs = left_padding_bqs;
+ }
-IA_CSS_LOG("top_cropping=%d, top_padding_bqs=%d", binary->info->sp.pipeline.top_cropping, top_padding_bqs);
+ IA_CSS_LOG("supported_bds_factors=%d, need_bds_factor:2_00=%d, 1_50=%d, 1_25=%d",
+ binary->info->sp.bds.supported_bds_factors,
+ need_bds_factor_2_00, need_bds_factor_1_50,
+ need_bds_factor_1_25);
+ IA_CSS_LOG("left_cropping=%d, left_padding_adjusted_bqs=%d",
+ binary->info->sp.pipeline.left_cropping,
+ left_padding_adjusted_bqs);
+
+ /* Set the top padding padded inside the isp kernel for bayer downscaling binaries.
+ * When the bds_factor isn't 1.00, the top padding is padded inside the isp
+ * before bayer downscaling, because the top cropping size (input margin) is not enough.
+ * (calculate_input_line(), raw_compute_vphase(), dma_read_raw() in raw.isp.c)
+ * NOTE: In dma_read_raw(), the factor passed to raw_compute_vphase() is got by get_bds_factor_for_dma_read().
+ * This factor is BDS_FPVAL_100/BDS_FPVAL_125/BDS_FPVAL_150/BDS_FPVAL_200.
+ */
+ top_padding_bqs = 0;
+ if (binary->info->sp.pipeline.top_cropping > 0 &&
+ (required_bds_factor == SH_CSS_BDS_FACTOR_1_25 ||
+ required_bds_factor == SH_CSS_BDS_FACTOR_1_50 ||
+ required_bds_factor == SH_CSS_BDS_FACTOR_2_00)) {
+ /* Calculation from calculate_input_line() and raw_compute_vphase() in raw.isp.c. */
+ int top_cropping_bqs = _ISP_BQS(binary->info->sp.pipeline.top_cropping);
+ /* top cropping (in bqs) */
+ int factor = bds_num * bds_frac_acc /
+ bds_den; /* downscaling factor by fixed-point */
+ int top_padding_bqsxfrac_acc = (top_cropping_bqs * factor - top_cropping_bqs *
+ bds_frac_acc)
+ + (2 * bds_frac_acc - factor); /* top padding by fixed-point (in bqs) */
+
+ top_padding_bqs = (unsigned int)((top_padding_bqsxfrac_acc + bds_frac_acc / 2 -
+ 1) / bds_frac_acc);
+ }
-/* Set the right/down shift amount caused by filters applied BEFORE bayer scaling,
- * which scaling is applied BEFORE shading corrertion.
- *
- * When the bds_factor isn't 1.00, 3x3 anti-alias filter is applied to each color plane(Gr/R/B/Gb)
- * before bayer downscaling.
- * This filter shifts each color plane (Gr/R/B/Gb) to right/down directions by 1 pixel.
- */
-right_shift_bqs_before_bs = 0;
-down_shift_bqs_before_bs = 0;
-#endif
+ IA_CSS_LOG("top_cropping=%d, top_padding_bqs=%d",
+ binary->info->sp.pipeline.top_cropping, top_padding_bqs);
-#ifndef ISP2401
-/* Currently, the bad pixel caused by filters after bayer scaling
-is NOT considered, because the bad pixel is subtle.
-When some large filter is used in the future,
-we need to consider the bad pixel.
-
-Currently, when DPC&BNR is processed between bayer scaling and
-shading correction, DPC&BNR moves each color plane to
-right/bottom directions by 1 pixel.
-*/
-bad_bqs_on_left_after_bs = 0;
-bad_bqs_on_top_after_bs = 0;
-#else
-if (need_bds_factor_2_00 || need_bds_factor_1_50 || need_bds_factor_1_25)
-{
- right_shift_bqs_before_bs = 1;
- down_shift_bqs_before_bs = 1;
-}
+ /* Set the right/down shift amount caused by filters applied BEFORE bayer scaling,
+ * which scaling is applied BEFORE shading corrertion.
+ *
+ * When the bds_factor isn't 1.00, 3x3 anti-alias filter is applied to each color plane(Gr/R/B/Gb)
+ * before bayer downscaling.
+ * This filter shifts each color plane (Gr/R/B/Gb) to right/down directions by 1 pixel.
+ */
+ right_shift_bqs_before_bs = 0;
+ down_shift_bqs_before_bs = 0;
-IA_CSS_LOG("right_shift_bqs_before_bs=%d, down_shift_bqs_before_bs=%d",
- right_shift_bqs_before_bs, down_shift_bqs_before_bs);
+ if (need_bds_factor_2_00 || need_bds_factor_1_50 || need_bds_factor_1_25) {
+ right_shift_bqs_before_bs = 1;
+ down_shift_bqs_before_bs = 1;
+ }
-/* Set the right/down shift amount caused by filters applied AFTER bayer scaling,
- * which scaling is applied BEFORE shading corrertion.
- *
- * When DPC&BNR is processed between bayer scaling and shading correction,
- * DPC&BNR moves each color plane (Gr/R/B/Gb) to right/down directions by 1 pixel.
- */
-right_shift_bqs_after_bs = 0;
-down_shift_bqs_after_bs = 0;
-#endif
+ IA_CSS_LOG("right_shift_bqs_before_bs=%d, down_shift_bqs_before_bs=%d",
+ right_shift_bqs_before_bs, down_shift_bqs_before_bs);
-#ifndef ISP2401
-/* Calculate the origin of bayer (real sensor data area)
-located on the shading table during the shading correction. */
-res->sc_bayer_origin_x_bqs_on_shading_table
-= ((left_padding_adjusted_bqs + bad_bqs_on_left_before_bs)
- * bs_hor_ratio_out + bs_hor_ratio_in / 2) / bs_hor_ratio_in
-+ bad_bqs_on_left_after_bs;
-/* "+ bs_hor_ratio_in/2": rounding for division by bs_hor_ratio_in */
-res->sc_bayer_origin_y_bqs_on_shading_table
-= (bad_bqs_on_top_before_bs
- * bs_ver_ratio_out + bs_ver_ratio_in / 2) / bs_ver_ratio_in
-+ bad_bqs_on_top_after_bs;
-/* "+ bs_ver_ratio_in/2": rounding for division by bs_ver_ratio_in */
-
-res->bayer_scale_hor_ratio_in = (uint32_t)bs_hor_ratio_in;
-res->bayer_scale_hor_ratio_out = (uint32_t)bs_hor_ratio_out;
-res->bayer_scale_ver_ratio_in = (uint32_t)bs_ver_ratio_in;
-res->bayer_scale_ver_ratio_out = (uint32_t)bs_ver_ratio_out;
-#else
-if (binary->info->mem_offsets.offsets.param->dmem.dp.size != 0) /* if DPC&BNR is enabled in the binary */
-{
- right_shift_bqs_after_bs = 1;
- down_shift_bqs_after_bs = 1;
-}
+ /* Set the right/down shift amount caused by filters applied AFTER bayer scaling,
+ * which scaling is applied BEFORE shading corrertion.
+ *
+ * When DPC&BNR is processed between bayer scaling and shading correction,
+ * DPC&BNR moves each color plane (Gr/R/B/Gb) to right/down directions by 1 pixel.
+ */
+ right_shift_bqs_after_bs = 0;
+ down_shift_bqs_after_bs = 0;
-IA_CSS_LOG("right_shift_bqs_after_bs=%d, down_shift_bqs_after_bs=%d",
- right_shift_bqs_after_bs, down_shift_bqs_after_bs);
+ /* if DPC&BNR is enabled in the binary */
+ if (binary->info->mem_offsets.offsets.param->dmem.dp.size != 0) {
+ right_shift_bqs_after_bs = 1;
+ down_shift_bqs_after_bs = 1;
+ }
-/* Set the origin of the sensor data area on the internal frame at shading correction. */
-{
- unsigned int bs_frac = bds_frac_acc; /* scaling factor 1.0 in fixed point */
- unsigned int bs_out, bs_in; /* scaling ratio in fixed point */
+ IA_CSS_LOG("right_shift_bqs_after_bs=%d, down_shift_bqs_after_bs=%d",
+ right_shift_bqs_after_bs, down_shift_bqs_after_bs);
bs_out = bs_hor_ratio_out * bs_frac;
bs_in = bs_hor_ratio_in * bs_frac;
- sensor_data_origin_x_bqs_on_internal
- = ((left_padding_adjusted_bqs + right_shift_bqs_before_bs) * bs_out + bs_in / 2) / bs_in
- + right_shift_bqs_after_bs; /* "+ bs_in/2": rounding */
+ sensor_data_origin_x_bqs_on_internal =
+ ((left_padding_adjusted_bqs + right_shift_bqs_before_bs) * bs_out + bs_in / 2) / bs_in
+ + right_shift_bqs_after_bs; /* "+ bs_in/2": rounding */
bs_out = bs_ver_ratio_out * bs_frac;
bs_in = bs_ver_ratio_in * bs_frac;
- sensor_data_origin_y_bqs_on_internal
- = ((top_padding_bqs + down_shift_bqs_before_bs) * bs_out + bs_in / 2) / bs_in
- + down_shift_bqs_after_bs; /* "+ bs_in/2": rounding */
-}
-
-scr->bayer_scale_hor_ratio_in = (uint32_t)bs_hor_ratio_in;
-scr->bayer_scale_hor_ratio_out = (uint32_t)bs_hor_ratio_out;
-scr->bayer_scale_ver_ratio_in = (uint32_t)bs_ver_ratio_in;
-scr->bayer_scale_ver_ratio_out = (uint32_t)bs_ver_ratio_out;
-scr->sensor_data_origin_x_bqs_on_internal = (uint32_t)sensor_data_origin_x_bqs_on_internal;
-scr->sensor_data_origin_y_bqs_on_internal = (uint32_t)sensor_data_origin_y_bqs_on_internal;
-
-IA_CSS_LOG("sc_requirements: %d, %d, %d, %d, %d, %d",
- scr->bayer_scale_hor_ratio_in, scr->bayer_scale_hor_ratio_out,
- scr->bayer_scale_ver_ratio_in, scr->bayer_scale_ver_ratio_out,
- scr->sensor_data_origin_x_bqs_on_internal, scr->sensor_data_origin_y_bqs_on_internal);
-#endif
+ sensor_data_origin_y_bqs_on_internal =
+ ((top_padding_bqs + down_shift_bqs_before_bs) * bs_out + bs_in / 2) / bs_in
+ + down_shift_bqs_after_bs; /* "+ bs_in/2": rounding */
+
+ scr->bayer_scale_hor_ratio_in = (uint32_t)bs_hor_ratio_in;
+ scr->bayer_scale_hor_ratio_out = (uint32_t)bs_hor_ratio_out;
+ scr->bayer_scale_ver_ratio_in = (uint32_t)bs_ver_ratio_in;
+ scr->bayer_scale_ver_ratio_out = (uint32_t)bs_ver_ratio_out;
+ scr->sensor_data_origin_x_bqs_on_internal = (uint32_t)sensor_data_origin_x_bqs_on_internal;
+ scr->sensor_data_origin_y_bqs_on_internal = (uint32_t)sensor_data_origin_y_bqs_on_internal;
+
+ IA_CSS_LOG("sc_requirements: %d, %d, %d, %d, %d, %d",
+ scr->bayer_scale_hor_ratio_in,
+ scr->bayer_scale_hor_ratio_out,
+ scr->bayer_scale_ver_ratio_in, scr->bayer_scale_ver_ratio_out,
+ scr->sensor_data_origin_x_bqs_on_internal,
+ scr->sensor_data_origin_y_bqs_on_internal);
-#ifdef ISP2401
-IA_CSS_LEAVE_ERR_PRIVATE(err);
-#endif
-return err;
+ IA_CSS_LEAVE_ERR_PRIVATE(err);
+ return err;
}
/* Get the shading information of Shading Correction Type 1. */
static int
-ia_css_binary_get_shading_info_type_1(const struct ia_css_binary
- *binary, /* [in] */
- unsigned int required_bds_factor, /* [in] */
- const struct ia_css_stream_config *stream_config, /* [in] */
-#ifndef ISP2401
- struct ia_css_shading_info *info) /* [out] */
-#else
- struct ia_css_shading_info *shading_info, /* [out] */
- struct ia_css_pipe_config *pipe_config) /* [out] */
-#endif
+isp2400_binary_get_shading_info_type_1(const struct ia_css_binary *binary, /* [in] */
+ unsigned int required_bds_factor, /* [in] */
+ const struct ia_css_stream_config *stream_config, /* [in] */
+ struct ia_css_shading_info *info) /* [out] */
{
int err;
-#ifndef ISP2401
struct sh_css_shading_table_bayer_origin_compute_results res;
-#else
- struct sh_css_binary_sc_requirements scr;
-#endif
-#ifndef ISP2401
assert(binary);
assert(info);
-#else
- u32 in_width_bqs, in_height_bqs, internal_width_bqs, internal_height_bqs;
- u32 num_hor_grids, num_ver_grids, bqs_per_grid_cell, tbl_width_bqs, tbl_height_bqs;
- u32 sensor_org_x_bqs_on_internal, sensor_org_y_bqs_on_internal, sensor_width_bqs, sensor_height_bqs;
- u32 sensor_center_x_bqs_on_internal, sensor_center_y_bqs_on_internal;
- u32 left, right, upper, lower;
- u32 adjust_left, adjust_right, adjust_upper, adjust_lower, adjust_width_bqs, adjust_height_bqs;
- u32 internal_org_x_bqs_on_tbl, internal_org_y_bqs_on_tbl;
- u32 sensor_org_x_bqs_on_tbl, sensor_org_y_bqs_on_tbl;
-#endif
-#ifndef ISP2401
info->type = IA_CSS_SHADING_CORRECTION_TYPE_1;
-#else
- assert(binary);
- assert(stream_config);
- assert(shading_info);
- assert(pipe_config);
-#endif
-#ifndef ISP2401
info->info.type_1.enable = binary->info->sp.enable.sc;
info->info.type_1.num_hor_grids = binary->sctbl_width_per_color;
info->info.type_1.num_ver_grids = binary->sctbl_height;
info->info.type_1.bqs_per_grid_cell = (1 << binary->deci_factor_log2);
-#else
- IA_CSS_ENTER_PRIVATE("binary=%p, required_bds_factor=%d, stream_config=%p",
- binary, required_bds_factor, stream_config);
-#endif
/* Initialize by default values. */
-#ifndef ISP2401
info->info.type_1.bayer_scale_hor_ratio_in = 1;
info->info.type_1.bayer_scale_hor_ratio_out = 1;
info->info.type_1.bayer_scale_ver_ratio_in = 1;
stream_config,
&res);
if (err)
-#else
+ return err;
+
+ info->info.type_1.bayer_scale_hor_ratio_in = res.bayer_scale_hor_ratio_in;
+ info->info.type_1.bayer_scale_hor_ratio_out = res.bayer_scale_hor_ratio_out;
+ info->info.type_1.bayer_scale_ver_ratio_in = res.bayer_scale_ver_ratio_in;
+ info->info.type_1.bayer_scale_ver_ratio_out = res.bayer_scale_ver_ratio_out;
+ info->info.type_1.sc_bayer_origin_x_bqs_on_shading_table = res.sc_bayer_origin_x_bqs_on_shading_table;
+ info->info.type_1.sc_bayer_origin_y_bqs_on_shading_table = res.sc_bayer_origin_y_bqs_on_shading_table;
+
+ return err;
+}
+
+/* Get the shading information of Shading Correction Type 1. */
+static int
+isp2401_binary_get_shading_info_type_1(const struct ia_css_binary *binary, /* [in] */
+ unsigned int required_bds_factor, /* [in] */
+ const struct ia_css_stream_config *stream_config, /* [in] */
+ struct ia_css_shading_info *shading_info, /* [out] */
+ struct ia_css_pipe_config *pipe_config) /* [out] */
+{
+ int err;
+ struct sh_css_binary_sc_requirements scr;
+
+ u32 in_width_bqs, in_height_bqs, internal_width_bqs, internal_height_bqs;
+ u32 num_hor_grids, num_ver_grids, bqs_per_grid_cell, tbl_width_bqs, tbl_height_bqs;
+ u32 sensor_org_x_bqs_on_internal, sensor_org_y_bqs_on_internal, sensor_width_bqs, sensor_height_bqs;
+ u32 sensor_center_x_bqs_on_internal, sensor_center_y_bqs_on_internal;
+ u32 left, right, upper, lower;
+ u32 adjust_left, adjust_right, adjust_upper, adjust_lower, adjust_width_bqs, adjust_height_bqs;
+ u32 internal_org_x_bqs_on_tbl, internal_org_y_bqs_on_tbl;
+ u32 sensor_org_x_bqs_on_tbl, sensor_org_y_bqs_on_tbl;
+
+ assert(binary);
+ assert(stream_config);
+ assert(shading_info);
+ assert(pipe_config);
+
+ IA_CSS_ENTER_PRIVATE("binary=%p, required_bds_factor=%d, stream_config=%p",
+ binary, required_bds_factor, stream_config);
+
+ /* Initialize by default values. */
*shading_info = DEFAULT_SHADING_INFO_TYPE_1;
err = sh_css_binary_get_sc_requirements(binary, required_bds_factor, stream_config, &scr);
- if (err)
- {
+ if (err) {
IA_CSS_LEAVE_ERR_PRIVATE(err);
-#endif
return err;
-#ifdef ISP2401
-}
+ }
-IA_CSS_LOG("binary: id=%d, sctbl=%dx%d, deci=%d",
- binary->info->sp.id, binary->sctbl_width_per_color, binary->sctbl_height, binary->deci_factor_log2);
-IA_CSS_LOG("binary: in=%dx%d, in_padded_w=%d, int=%dx%d, int_padded_w=%d, out=%dx%d, out_padded_w=%d",
- binary->in_frame_info.res.width, binary->in_frame_info.res.height, binary->in_frame_info.padded_width,
- binary->internal_frame_info.res.width, binary->internal_frame_info.res.height,
- binary->internal_frame_info.padded_width,
- binary->out_frame_info[0].res.width, binary->out_frame_info[0].res.height,
- binary->out_frame_info[0].padded_width);
-
-/* Set the input size from sensor, which includes left/top crop size. */
-in_width_bqs = _ISP_BQS(binary->in_frame_info.res.width);
-in_height_bqs = _ISP_BQS(binary->in_frame_info.res.height);
-
-/* Frame size internally used in ISP, including sensor data and padding.
- * This is the frame size, to which the shading correction is applied.
- */
-internal_width_bqs = _ISP_BQS(binary->internal_frame_info.res.width);
-internal_height_bqs = _ISP_BQS(binary->internal_frame_info.res.height);
-
-/* Shading table. */
-num_hor_grids = binary->sctbl_width_per_color;
-num_ver_grids = binary->sctbl_height;
-bqs_per_grid_cell = (1 << binary->deci_factor_log2);
-tbl_width_bqs = (num_hor_grids - 1) * bqs_per_grid_cell;
-tbl_height_bqs = (num_ver_grids - 1) * bqs_per_grid_cell;
-#endif
+ IA_CSS_LOG("binary: id=%d, sctbl=%dx%d, deci=%d",
+ binary->info->sp.id, binary->sctbl_width_per_color, binary->sctbl_height, binary->deci_factor_log2);
+ IA_CSS_LOG("binary: in=%dx%d, in_padded_w=%d, int=%dx%d, int_padded_w=%d, out=%dx%d, out_padded_w=%d",
+ binary->in_frame_info.res.width, binary->in_frame_info.res.height, binary->in_frame_info.padded_width,
+ binary->internal_frame_info.res.width, binary->internal_frame_info.res.height,
+ binary->internal_frame_info.padded_width,
+ binary->out_frame_info[0].res.width, binary->out_frame_info[0].res.height,
+ binary->out_frame_info[0].padded_width);
-#ifndef ISP2401
-info->info.type_1.bayer_scale_hor_ratio_in = res.bayer_scale_hor_ratio_in;
-info->info.type_1.bayer_scale_hor_ratio_out = res.bayer_scale_hor_ratio_out;
-info->info.type_1.bayer_scale_ver_ratio_in = res.bayer_scale_ver_ratio_in;
-info->info.type_1.bayer_scale_ver_ratio_out = res.bayer_scale_ver_ratio_out;
-info->info.type_1.sc_bayer_origin_x_bqs_on_shading_table = res.sc_bayer_origin_x_bqs_on_shading_table;
-info->info.type_1.sc_bayer_origin_y_bqs_on_shading_table = res.sc_bayer_origin_y_bqs_on_shading_table;
-#else
-IA_CSS_LOG("tbl_width_bqs=%d, tbl_height_bqs=%d", tbl_width_bqs, tbl_height_bqs);
-#endif
+ /* Set the input size from sensor, which includes left/top crop size. */
+ in_width_bqs = _ISP_BQS(binary->in_frame_info.res.width);
+ in_height_bqs = _ISP_BQS(binary->in_frame_info.res.height);
-#ifdef ISP2401
-/* Real sensor data area on the internal frame at shading correction.
- * Filters and scaling are applied to the internal frame before shading correction, depending on the binary.
- */
-sensor_org_x_bqs_on_internal = scr.sensor_data_origin_x_bqs_on_internal;
-sensor_org_y_bqs_on_internal = scr.sensor_data_origin_y_bqs_on_internal;
-{
- unsigned int bs_frac = 8; /* scaling factor 1.0 in fixed point (8 == FRAC_ACC macro in ISP) */
- unsigned int bs_out, bs_in; /* scaling ratio in fixed point */
+ /*
+ * Frame size internally used in ISP, including sensor data and padding.
+ * This is the frame size, to which the shading correction is applied.
+ */
+ internal_width_bqs = _ISP_BQS(binary->internal_frame_info.res.width);
+ internal_height_bqs = _ISP_BQS(binary->internal_frame_info.res.height);
- bs_out = scr.bayer_scale_hor_ratio_out * bs_frac;
- bs_in = scr.bayer_scale_hor_ratio_in * bs_frac;
- sensor_width_bqs = (in_width_bqs * bs_out + bs_in / 2) / bs_in; /* "+ bs_in/2": rounding */
+ /* Shading table. */
+ num_hor_grids = binary->sctbl_width_per_color;
+ num_ver_grids = binary->sctbl_height;
+ bqs_per_grid_cell = (1 << binary->deci_factor_log2);
+ tbl_width_bqs = (num_hor_grids - 1) * bqs_per_grid_cell;
+ tbl_height_bqs = (num_ver_grids - 1) * bqs_per_grid_cell;
- bs_out = scr.bayer_scale_ver_ratio_out * bs_frac;
- bs_in = scr.bayer_scale_ver_ratio_in * bs_frac;
- sensor_height_bqs = (in_height_bqs * bs_out + bs_in / 2) / bs_in; /* "+ bs_in/2": rounding */
-}
+ IA_CSS_LOG("tbl_width_bqs=%d, tbl_height_bqs=%d", tbl_width_bqs, tbl_height_bqs);
+
+ /*
+ * Real sensor data area on the internal frame at shading correction.
+ * Filters and scaling are applied to the internal frame before
+ * shading correction, depending on the binary.
+ */
+ sensor_org_x_bqs_on_internal = scr.sensor_data_origin_x_bqs_on_internal;
+ sensor_org_y_bqs_on_internal = scr.sensor_data_origin_y_bqs_on_internal;
+ {
+ unsigned int bs_frac = 8; /* scaling factor 1.0 in fixed point (8 == FRAC_ACC macro in ISP) */
+ unsigned int bs_out, bs_in; /* scaling ratio in fixed point */
-/* Center of the sensor data on the internal frame at shading correction. */
-sensor_center_x_bqs_on_internal = sensor_org_x_bqs_on_internal + sensor_width_bqs / 2;
-sensor_center_y_bqs_on_internal = sensor_org_y_bqs_on_internal + sensor_height_bqs / 2;
+ bs_out = scr.bayer_scale_hor_ratio_out * bs_frac;
+ bs_in = scr.bayer_scale_hor_ratio_in * bs_frac;
+ sensor_width_bqs = (in_width_bqs * bs_out + bs_in / 2) / bs_in; /* "+ bs_in/2": rounding */
-/* Size of left/right/upper/lower sides of the sensor center on the internal frame. */
-left = sensor_center_x_bqs_on_internal;
-right = internal_width_bqs - sensor_center_x_bqs_on_internal;
-upper = sensor_center_y_bqs_on_internal;
-lower = internal_height_bqs - sensor_center_y_bqs_on_internal;
+ bs_out = scr.bayer_scale_ver_ratio_out * bs_frac;
+ bs_in = scr.bayer_scale_ver_ratio_in * bs_frac;
+ sensor_height_bqs = (in_height_bqs * bs_out + bs_in / 2) / bs_in; /* "+ bs_in/2": rounding */
+ }
-/* Align the size of left/right/upper/lower sides to a multiple of the grid cell size. */
-adjust_left = CEIL_MUL(left, bqs_per_grid_cell);
-adjust_right = CEIL_MUL(right, bqs_per_grid_cell);
-adjust_upper = CEIL_MUL(upper, bqs_per_grid_cell);
-adjust_lower = CEIL_MUL(lower, bqs_per_grid_cell);
+ /* Center of the sensor data on the internal frame at shading correction. */
+ sensor_center_x_bqs_on_internal = sensor_org_x_bqs_on_internal + sensor_width_bqs / 2;
+ sensor_center_y_bqs_on_internal = sensor_org_y_bqs_on_internal + sensor_height_bqs / 2;
-/* Shading table should cover the adjusted frame size. */
-adjust_width_bqs = adjust_left + adjust_right;
-adjust_height_bqs = adjust_upper + adjust_lower;
+ /* Size of left/right/upper/lower sides of the sensor center on the internal frame. */
+ left = sensor_center_x_bqs_on_internal;
+ right = internal_width_bqs - sensor_center_x_bqs_on_internal;
+ upper = sensor_center_y_bqs_on_internal;
+ lower = internal_height_bqs - sensor_center_y_bqs_on_internal;
-IA_CSS_LOG("adjust_width_bqs=%d, adjust_height_bqs=%d", adjust_width_bqs, adjust_height_bqs);
+ /* Align the size of left/right/upper/lower sides to a multiple of the grid cell size. */
+ adjust_left = CEIL_MUL(left, bqs_per_grid_cell);
+ adjust_right = CEIL_MUL(right, bqs_per_grid_cell);
+ adjust_upper = CEIL_MUL(upper, bqs_per_grid_cell);
+ adjust_lower = CEIL_MUL(lower, bqs_per_grid_cell);
-if (adjust_width_bqs > tbl_width_bqs || adjust_height_bqs > tbl_height_bqs)
-{
- IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
- return -EINVAL;
-}
+ /* Shading table should cover the adjusted frame size. */
+ adjust_width_bqs = adjust_left + adjust_right;
+ adjust_height_bqs = adjust_upper + adjust_lower;
-/* Origin of the internal frame on the shading table. */
-internal_org_x_bqs_on_tbl = adjust_left - left;
-internal_org_y_bqs_on_tbl = adjust_upper - upper;
-
-/* Origin of the real sensor data area on the shading table. */
-sensor_org_x_bqs_on_tbl = internal_org_x_bqs_on_tbl + sensor_org_x_bqs_on_internal;
-sensor_org_y_bqs_on_tbl = internal_org_y_bqs_on_tbl + sensor_org_y_bqs_on_internal;
-
-/* The shading information necessary as API is stored in the shading_info. */
-shading_info->info.type_1.num_hor_grids = num_hor_grids;
-shading_info->info.type_1.num_ver_grids = num_ver_grids;
-shading_info->info.type_1.bqs_per_grid_cell = bqs_per_grid_cell;
-
-shading_info->info.type_1.bayer_scale_hor_ratio_in = scr.bayer_scale_hor_ratio_in;
-shading_info->info.type_1.bayer_scale_hor_ratio_out = scr.bayer_scale_hor_ratio_out;
-shading_info->info.type_1.bayer_scale_ver_ratio_in = scr.bayer_scale_ver_ratio_in;
-shading_info->info.type_1.bayer_scale_ver_ratio_out = scr.bayer_scale_ver_ratio_out;
-
-shading_info->info.type_1.isp_input_sensor_data_res_bqs.width = in_width_bqs;
-shading_info->info.type_1.isp_input_sensor_data_res_bqs.height = in_height_bqs;
-
-shading_info->info.type_1.sensor_data_res_bqs.width = sensor_width_bqs;
-shading_info->info.type_1.sensor_data_res_bqs.height = sensor_height_bqs;
-
-shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.x = (int32_t)sensor_org_x_bqs_on_tbl;
-shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.y = (int32_t)sensor_org_y_bqs_on_tbl;
-
-/* The shading information related to ISP (but, not necessary as API) is stored in the pipe_config. */
-pipe_config->internal_frame_origin_bqs_on_sctbl.x = (int32_t)internal_org_x_bqs_on_tbl;
-pipe_config->internal_frame_origin_bqs_on_sctbl.y = (int32_t)internal_org_y_bqs_on_tbl;
-
-IA_CSS_LOG("shading_info: grids=%dx%d, cell=%d, scale=%d,%d,%d,%d, input=%dx%d, data=%dx%d, origin=(%d,%d)",
- shading_info->info.type_1.num_hor_grids,
- shading_info->info.type_1.num_ver_grids,
- shading_info->info.type_1.bqs_per_grid_cell,
- shading_info->info.type_1.bayer_scale_hor_ratio_in,
- shading_info->info.type_1.bayer_scale_hor_ratio_out,
- shading_info->info.type_1.bayer_scale_ver_ratio_in,
- shading_info->info.type_1.bayer_scale_ver_ratio_out,
- shading_info->info.type_1.isp_input_sensor_data_res_bqs.width,
- shading_info->info.type_1.isp_input_sensor_data_res_bqs.height,
- shading_info->info.type_1.sensor_data_res_bqs.width,
- shading_info->info.type_1.sensor_data_res_bqs.height,
- shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.x,
- shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.y);
-
-IA_CSS_LOG("pipe_config: origin=(%d,%d)",
- pipe_config->internal_frame_origin_bqs_on_sctbl.x,
- pipe_config->internal_frame_origin_bqs_on_sctbl.y);
-
-IA_CSS_LEAVE_ERR_PRIVATE(err);
-#endif
-return err;
+ IA_CSS_LOG("adjust_width_bqs=%d, adjust_height_bqs=%d", adjust_width_bqs, adjust_height_bqs);
+
+ if (adjust_width_bqs > tbl_width_bqs || adjust_height_bqs > tbl_height_bqs) {
+ IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
+ return -EINVAL;
+ }
+
+ /* Origin of the internal frame on the shading table. */
+ internal_org_x_bqs_on_tbl = adjust_left - left;
+ internal_org_y_bqs_on_tbl = adjust_upper - upper;
+
+ /* Origin of the real sensor data area on the shading table. */
+ sensor_org_x_bqs_on_tbl = internal_org_x_bqs_on_tbl + sensor_org_x_bqs_on_internal;
+ sensor_org_y_bqs_on_tbl = internal_org_y_bqs_on_tbl + sensor_org_y_bqs_on_internal;
+
+ /* The shading information necessary as API is stored in the shading_info. */
+ shading_info->info.type_1.num_hor_grids = num_hor_grids;
+ shading_info->info.type_1.num_ver_grids = num_ver_grids;
+ shading_info->info.type_1.bqs_per_grid_cell = bqs_per_grid_cell;
+
+ shading_info->info.type_1.bayer_scale_hor_ratio_in = scr.bayer_scale_hor_ratio_in;
+ shading_info->info.type_1.bayer_scale_hor_ratio_out = scr.bayer_scale_hor_ratio_out;
+ shading_info->info.type_1.bayer_scale_ver_ratio_in = scr.bayer_scale_ver_ratio_in;
+ shading_info->info.type_1.bayer_scale_ver_ratio_out = scr.bayer_scale_ver_ratio_out;
+
+ shading_info->info.type_1.isp_input_sensor_data_res_bqs.width = in_width_bqs;
+ shading_info->info.type_1.isp_input_sensor_data_res_bqs.height = in_height_bqs;
+
+ shading_info->info.type_1.sensor_data_res_bqs.width = sensor_width_bqs;
+ shading_info->info.type_1.sensor_data_res_bqs.height = sensor_height_bqs;
+
+ shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.x = (int32_t)sensor_org_x_bqs_on_tbl;
+ shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.y = (int32_t)sensor_org_y_bqs_on_tbl;
+
+ /* The shading information related to ISP (but, not necessary as API) is stored in the pipe_config. */
+ pipe_config->internal_frame_origin_bqs_on_sctbl.x = (int32_t)internal_org_x_bqs_on_tbl;
+ pipe_config->internal_frame_origin_bqs_on_sctbl.y = (int32_t)internal_org_y_bqs_on_tbl;
+
+ IA_CSS_LOG("shading_info: grids=%dx%d, cell=%d, scale=%d,%d,%d,%d, input=%dx%d, data=%dx%d, origin=(%d,%d)",
+ shading_info->info.type_1.num_hor_grids,
+ shading_info->info.type_1.num_ver_grids,
+ shading_info->info.type_1.bqs_per_grid_cell,
+ shading_info->info.type_1.bayer_scale_hor_ratio_in,
+ shading_info->info.type_1.bayer_scale_hor_ratio_out,
+ shading_info->info.type_1.bayer_scale_ver_ratio_in,
+ shading_info->info.type_1.bayer_scale_ver_ratio_out,
+ shading_info->info.type_1.isp_input_sensor_data_res_bqs.width,
+ shading_info->info.type_1.isp_input_sensor_data_res_bqs.height,
+ shading_info->info.type_1.sensor_data_res_bqs.width,
+ shading_info->info.type_1.sensor_data_res_bqs.height,
+ shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.x,
+ shading_info->info.type_1.sensor_data_origin_bqs_on_sctbl.y);
+
+ IA_CSS_LOG("pipe_config: origin=(%d,%d)",
+ pipe_config->internal_frame_origin_bqs_on_sctbl.x,
+ pipe_config->internal_frame_origin_bqs_on_sctbl.y);
+
+ IA_CSS_LEAVE_ERR_PRIVATE(err);
+ return err;
}
+
int
ia_css_binary_get_shading_info(const struct ia_css_binary *binary, /* [in] */
enum ia_css_shading_correction_type type, /* [in] */
IA_CSS_ENTER_PRIVATE("binary=%p, type=%d, required_bds_factor=%d, stream_config=%p",
binary, type, required_bds_factor, stream_config);
- if (type == IA_CSS_SHADING_CORRECTION_TYPE_1)
-#ifndef ISP2401
- err = ia_css_binary_get_shading_info_type_1(binary, required_bds_factor, stream_config,
- shading_info);
-#else
- err = ia_css_binary_get_shading_info_type_1(binary, required_bds_factor, stream_config,
- shading_info, pipe_config);
-#endif
+ if (type != IA_CSS_SHADING_CORRECTION_TYPE_1) {
+ err = -ENOTSUPP;
- /* Other function calls can be added here when other shading correction types will be added in the future. */
+ IA_CSS_LEAVE_ERR_PRIVATE(err);
+ return err;
+ }
+ if (!IS_ISP2401)
+ err = isp2400_binary_get_shading_info_type_1(binary,
+ required_bds_factor,
+ stream_config,
+ shading_info);
else
- err = -ENOTSUPP;
+ err = isp2401_binary_get_shading_info_type_1(binary,
+ required_bds_factor,
+ stream_config,
+ shading_info,
+ pipe_config);
IA_CSS_LEAVE_ERR_PRIVATE(err);
return err;
int rval;
int nr_of_left_paddings; /* number of paddings pixels on the left of an image line */
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
/* the output image line of Input System 2401 does not have the left paddings */
nr_of_left_paddings = 0;
#else
SH_CSS_QUEUE_E_ID,
SH_CSS_QUEUE_F_ID,
SH_CSS_QUEUE_G_ID,
-#if defined(HAS_NO_INPUT_SYSTEM)
- /* input frame queue for skycam */
- SH_CSS_QUEUE_H_ID,
-#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
SH_CSS_QUEUE_H_ID, /* for metadata */
-#endif
-#if defined(HAS_NO_INPUT_SYSTEM) || defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
#define SH_CSS_MAX_NUM_QUEUES (SH_CSS_QUEUE_H_ID + 1)
-#else
-#define SH_CSS_MAX_NUM_QUEUES (SH_CSS_QUEUE_G_ID + 1)
-#endif
};
/* SP2Host event queue */
ia_css_queue_t sp2host_psys_event_queue_handle;
-#if !defined(HAS_NO_INPUT_SYSTEM)
/* Host2SP ISYS event queue */
ia_css_queue_t host2sp_isys_event_queue_handle;
/* SP2Host ISYS event queue */
ia_css_queue_t sp2host_isys_event_queue_handle;
-#endif
/* Tagger command queue */
ia_css_queue_t host2sp_tag_cmd_queue_handle;
};
case sh_css_sp2host_psys_event_queue:
q = &css_queues.sp2host_psys_event_queue_handle;
break;
-#if !defined(HAS_NO_INPUT_SYSTEM)
case sh_css_host2sp_isys_event_queue:
q = &css_queues.host2sp_isys_event_queue_handle;
break;
case sh_css_sp2host_isys_event_queue:
q = &css_queues.sp2host_isys_event_queue_handle;
break;
-#endif
case sh_css_host2sp_tag_cmd_queue:
q = &css_queues.host2sp_tag_cmd_queue_handle;
break;
(uint32_t)offsetof(struct host_sp_queues, sp2host_psys_event_queue_elems),
&css_queues.sp2host_psys_event_queue_handle);
-#if !defined(HAS_NO_INPUT_SYSTEM)
/* Host2SP ISYS event queue */
init_bufq((uint32_t)offsetof(struct host_sp_queues,
host2sp_isys_event_queue_desc),
init_bufq((uint32_t)offsetof(struct host_sp_queues, host2sp_tag_cmd_queue_desc),
(uint32_t)offsetof(struct host_sp_queues, host2sp_tag_cmd_queue_elems),
&css_queues.host2sp_tag_cmd_queue_handle);
-#endif
IA_CSS_LEAVE_PRIVATE("");
}
u8 evt_payload_1,
uint8_t evt_payload_2)
{
-
- int error = 0;
+ int error = 0;
ia_css_queue_t *q;
IA_CSS_ENTER_PRIVATE("evt_id=%d", evt_id);
int ia_css_bufq_dequeue_isys_event(
u8 item[BUFQ_EVENT_SIZE])
{
-#if !defined(HAS_NO_INPUT_SYSTEM)
int error = 0;
ia_css_queue_t *q;
}
error = ia_css_eventq_recv(q, item);
return error;
-#else
- (void)item;
- return -EBUSY;
-#endif
}
int ia_css_bufq_enqueue_isys_event(uint8_t evt_id)
{
-#if !defined(HAS_NO_INPUT_SYSTEM)
int error = 0;
ia_css_queue_t *q;
IA_CSS_LEAVE_ERR_PRIVATE(error);
return error;
-#else
- (void)evt_id;
- return -EBUSY;
-#endif
}
int ia_css_bufq_enqueue_tag_cmd(
uint32_t item)
{
-#if !defined(HAS_NO_INPUT_SYSTEM)
int error;
ia_css_queue_t *q;
IA_CSS_LEAVE_ERR_PRIVATE(error);
return error;
-#else
- (void)item;
- return -EBUSY;
-#endif
}
int ia_css_bufq_deinit(void)
bufq_dump_queue_info("sp2host_psys_event",
&css_queues.sp2host_psys_event_queue_handle);
-#if !defined(HAS_NO_INPUT_SYSTEM)
bufq_dump_queue_info("host2sp_isys_event",
&css_queues.host2sp_isys_event_queue_handle);
bufq_dump_queue_info("sp2host_isys_event",
&css_queues.sp2host_isys_event_queue_handle);
bufq_dump_queue_info("host2sp_tag_cmd",
&css_queues.host2sp_tag_cmd_queue_handle);
-#endif
}
* @param[in] fmt printf like format string
* @param[in] args arguments for the format string
*/
-static inline void
-ia_css_debug_vdtrace(unsigned int level, const char *fmt, va_list args)
+static inline void __printf(2, 0) ia_css_debug_vdtrace(unsigned int level,
+ const char *fmt,
+ va_list args)
{
if (dbg_level >= level)
sh_css_vprint(fmt, args);
}
-__printf(2, 3)
-void ia_css_debug_dtrace(unsigned int level, const char *fmt, ...);
+__printf(2, 3) void ia_css_debug_dtrace(unsigned int level,
+ const char *fmt, ...);
/*! @brief Dump sp thread's stack contents
* SP thread's stack contents are set to 0xcafecafe. This function dumps the
*/
unsigned int ia_css_debug_get_dtrace_level(void);
-/*! @brief Dump input formatter state.
- * Dumps the input formatter state to tracing output.
- * @return None
- */
-void ia_css_debug_dump_if_state(void);
-
/*! @brief Dump isp hardware state.
* Dumps the isp hardware state to tracing output.
* @return None
#include "fifo_monitor.h"
-#if !defined(HAS_NO_INPUT_FORMATTER)
#include "input_formatter.h"
-#endif
#include "dma.h"
#include "irq.h"
#include "gp_device.h"
#include "isp.h"
#include "type_support.h"
#include "math_support.h" /* CEIL_DIV */
-#if defined(HAS_INPUT_FORMATTER_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
#include "input_system.h" /* input_formatter_reg_load */
-#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
#include "ia_css_tagger_common.h"
-#endif
#include "sh_css_internal.h"
-#if !defined(HAS_NO_INPUT_SYSTEM)
#include "ia_css_isys.h"
-#endif
#include "sh_css_sp.h" /* sh_css_sp_get_debug_state() */
#include "css_trace.h" /* tracer */
#define ENABLE_LINE_MAX_LENGTH (25)
-#ifdef ISP2401
-#define DBG_EXT_CMD_TRACE_PNTS_DUMP BIT(8)
-#define DBG_EXT_CMD_PUB_CFG_DUMP BIT(9)
-#define DBG_EXT_CMD_GAC_REG_DUMP BIT(10)
-#define DBG_EXT_CMD_GAC_ACB_REG_DUMP BIT(11)
-#define DBG_EXT_CMD_FIFO_DUMP BIT(12)
-#define DBG_EXT_CMD_QUEUE_DUMP BIT(13)
-#define DBG_EXT_CMD_DMA_DUMP BIT(14)
-#define DBG_EXT_CMD_MASK 0xAB0000CD
-
-#endif
/*
* TODO:SH_CSS_MAX_SP_THREADS is not the max number of sp threads
* future rework should fix this and remove the define MAX_THREAD_NUM
debug_print_isp_state(&state, "ISP");
if (state.is_stalling) {
-#if !defined(HAS_NO_INPUT_FORMATTER)
- ia_css_debug_dtrace(2, "\t%-32s: %d\n",
- "[0] if_prim_a_FIFO stalled", stall.fifo0);
- ia_css_debug_dtrace(2, "\t%-32s: %d\n",
- "[1] if_prim_b_FIFO stalled", stall.fifo1);
-#endif
+ if (!IS_ISP2401) {
+ ia_css_debug_dtrace(2, "\t%-32s: %d\n",
+ "[0] if_prim_a_FIFO stalled", stall.fifo0);
+ ia_css_debug_dtrace(2, "\t%-32s: %d\n",
+ "[1] if_prim_b_FIFO stalled", stall.fifo1);
+ }
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[2] dma_FIFO stalled",
stall.fifo2);
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[3] gdc0_FIFO stalled",
stall.fifo3);
-#if !defined(IS_ISP_2500_SYSTEM)
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[4] gdc1_FIFO stalled",
stall.fifo4);
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[5] gpio_FIFO stalled",
stall.fifo5);
-#endif
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[6] sp_FIFO stalled",
stall.fifo6);
ia_css_debug_dtrace(2, "\t%-32s: %d\n",
sp_get_state(SP0_ID, &state, &stall);
debug_print_sp_state(&state, "SP");
if (state.is_stalling) {
-#if !defined(HAS_NO_INPUT_SYSTEM)
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "isys_FIFO stalled",
stall.fifo0);
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "if_sec_FIFO stalled",
stall.fifo1);
-#endif
ia_css_debug_dtrace(2, "\t%-32s: %d\n",
"str_to_mem_FIFO stalled", stall.fifo2);
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "dma_FIFO stalled",
stall.fifo3);
-#if !defined(HAS_NO_INPUT_FORMATTER)
- ia_css_debug_dtrace(2, "\t%-32s: %d\n",
- "if_prim_a_FIFO stalled", stall.fifo4);
-#endif
+ if (!IS_ISP2401)
+ ia_css_debug_dtrace(2, "\t%-32s: %d\n",
+ "if_prim_a_FIFO stalled", stall.fifo4);
+
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "isp_FIFO stalled",
stall.fifo5);
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "gp_FIFO stalled",
stall.fifo6);
-#if !defined(HAS_NO_INPUT_FORMATTER)
- ia_css_debug_dtrace(2, "\t%-32s: %d\n",
- "if_prim_b_FIFO stalled", stall.fifo7);
-#endif
+ if (!IS_ISP2401)
+ ia_css_debug_dtrace(2, "\t%-32s: %d\n",
+ "if_prim_b_FIFO stalled", stall.fifo7);
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "gdc0_FIFO stalled",
stall.fifo8);
-#if !defined(IS_ISP_2500_SYSTEM)
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "gdc1_FIFO stalled",
stall.fifo9);
-#endif
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "irq FIFO stalled",
stall.fifoa);
ia_css_debug_dtrace(2, "\t%-32s: %d\n", "dmem stalled",
return;
}
-#if !defined(HAS_NO_INPUT_FORMATTER) && defined(USE_INPUT_SYSTEM_VERSION_2)
void ia_css_debug_dump_pif_a_isp_fifo_state(void)
{
fifo_channel_state_t pif_to_isp, isp_to_pif;
debug_print_fifo_channel_state(&sp_to_s2m, "SP to stream-to-memory");
}
+#ifndef ISP2401
static void debug_print_if_state(input_formatter_state_t *state, const char *id)
{
unsigned int val;
-#if defined(HAS_INPUT_FORMATTER_VERSION_1)
- const char *st_reset = (state->reset ? "Active" : "Not active");
-#endif
const char *st_vsync_active_low =
(state->vsync_active_low ? "low" : "high");
const char *st_hsync_active_low =
ia_css_debug_dtrace(2, "\tConfiguration:\n");
-#if defined(HAS_INPUT_FORMATTER_VERSION_1)
- ia_css_debug_dtrace(2, "\t\t%-32s: %s\n", "Software reset", st_reset);
-#endif
ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Start line", st_stline);
ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Start column", st_stcol);
ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Cropped height", st_crpht);
ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
"Block when no request", st_block_fifo_when_no_req);
-#if defined(HAS_INPUT_FORMATTER_VERSION_2)
ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
"IF_BLOCKED_FIFO_NO_REQ_ADDRESS",
input_formatter_reg_load(INPUT_FORMATTER0_ID,
"_REG_GP_IFMT_slv_reg_srst",
gp_device_reg_load(GP_DEVICE0_ID,
_REG_GP_IFMT_slv_reg_srst));
-#endif
ia_css_debug_dtrace(2, "\tFSM Status:\n");
state->vector_support);
ia_css_debug_dtrace(2, "\t\t%-32s: %d\n", "Fifo sensor data lost",
state->sensor_data_lost);
- return;
}
static void debug_print_if_bin_state(input_formatter_bin_state_t *state)
state->en_status_update);
}
-void ia_css_debug_dump_if_state(void)
+static void ia_css_debug_dump_if_state(void)
{
input_formatter_state_t if_state;
input_formatter_bin_state_t if_bin_state;
"frame_buffer.sp.c"
};
-#if 1
/* Example SH_CSS_SP_DBG_NR_OF_TRACES==1 */
/* Adjust this to your trace case */
static char const *trace_name[SH_CSS_SP_DBG_NR_OF_TRACES] = {
"default"
};
-#else
- /* Example SH_CSS_SP_DBG_NR_OF_TRACES==4 */
- /* Adjust this to your trace case */
- static char const *trace_name[SH_CSS_SP_DBG_NR_OF_TRACES] = {
- "copy", "preview/video", "capture", "acceleration"
- };
-#endif
/* Remember host_index_last because we only want to print new entries */
static int host_index_last[SH_CSS_SP_DBG_NR_OF_TRACES] = { 0 };
}
#endif
-#if defined(HAS_INPUT_FORMATTER_VERSION_2) && !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
static void debug_print_rx_mipi_port_state(mipi_port_state_t *state)
{
int i;
}
#endif
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
void ia_css_debug_dump_rx_state(void)
{
-#if defined(HAS_INPUT_FORMATTER_VERSION_2) && !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
receiver_state_t state;
receiver_get_state(RX0_ID, &state);
debug_print_rx_state(&state);
#endif
}
-#endif
void ia_css_debug_dump_sp_sw_debug_info(void)
{
return;
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
static void debug_print_isys_capture_unit_state(capture_unit_state_t *state)
{
assert(state);
}
/* end of control unit state */
}
-
-void ia_css_debug_dump_isys_state(void)
-{
- input_system_state_t state;
-
- input_system_get_state(INPUT_SYSTEM0_ID, &state);
- debug_print_isys_state(&state);
-
- return;
-}
#endif
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
+
void ia_css_debug_dump_isys_state(void)
{
- /* Android compilation fails if made a local variable
- stack size on android is limited to 2k and this structure
- is around 3.5K, in place of static malloc can be done but
- if this call is made too often it will lead to fragment memory
- versus a fixed allocation */
static input_system_state_t state;
input_system_get_state(INPUT_SYSTEM0_ID, &state);
+
+#ifndef ISP2401
+ debug_print_isys_state(&state);
+#else
input_system_dump_state(INPUT_SYSTEM0_ID, &state);
-}
#endif
+}
void ia_css_debug_dump_debug_info(const char *context)
{
context = "No Context provided";
ia_css_debug_dtrace(2, "CSS Debug Info dump [Context = %s]\n", context);
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
- ia_css_debug_dump_rx_state();
-#endif
-#if !defined(HAS_NO_INPUT_FORMATTER) && defined(USE_INPUT_SYSTEM_VERSION_2)
+ if (!IS_ISP2401)
+ ia_css_debug_dump_rx_state();
+
+#ifndef ISP2401
ia_css_debug_dump_if_state();
#endif
ia_css_debug_dump_isp_state();
ia_css_debug_dump_dma_isp_fifo_state();
ia_css_debug_dump_dma_sp_fifo_state();
ia_css_debug_dump_dma_state();
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
- ia_css_debug_dump_isys_state();
- {
+ if (!IS_ISP2401) {
struct irq_controller_state state;
+ ia_css_debug_dump_isys_state();
+
irq_controller_get_state(IRQ2_ID, &state);
ia_css_debug_dtrace(2, "\t%-32s:\n",
ia_css_debug_dtrace(2, "\t\t%-32s: %d\n",
"irq_level_not_pulse",
state.irq_level_not_pulse);
+ } else {
+ ia_css_debug_dump_isys_state();
}
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
- ia_css_debug_dump_isys_state();
-#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
+
ia_css_debug_tagger_state();
-#endif
+
return;
}
sp_ctrl_setbit(SP0_ID, SP_SC_REG, SP_START_BIT);
}
-#if !defined(IS_ISP_2500_SYSTEM)
#define FIND_DMEM_PARAMS_TYPE(stream, kernel, type) \
(struct HRTCAT(HRTCAT(sh_css_isp_, type), _params) *) \
findf_dmem_params(stream, offsetof(struct ia_css_memory_offsets, dmem.kernel))
}
return NULL;
}
-#endif
void ia_css_debug_dump_isp_params(struct ia_css_stream *stream,
unsigned int enable)
{
ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "ISP PARAMETERS:\n");
-#if defined(IS_ISP_2500_SYSTEM)
- (void)enable;
- (void)stream;
-#else
assert(stream);
if ((enable & IA_CSS_DEBUG_DUMP_FPN)
|| (enable & IA_CSS_DEBUG_DUMP_ALL)) {
ia_css_ce_dump(FIND_DMEM_PARAMS(stream, ce), IA_CSS_DEBUG_VERBOSE);
}
-#endif
}
void sh_css_dump_sp_raw_copy_linecount(bool reduced)
void ia_css_debug_dump_perf_counters(void)
{
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
const struct ia_css_fw_info *fw;
int i;
unsigned int HIVE_ADDR_ia_css_isys_sp_error_cnt;
- s32 ia_css_sp_input_system_error_cnt[N_MIPI_PORT_ID +
- 1]; /* 3 Capture Units and 1 Acquire Unit. */
+ /* N_MIPI_PORT_ID + 1: 3 Capture Units and 1 Acquire Unit. */
+ s32 ia_css_sp_input_system_error_cnt[N_MIPI_PORT_ID + 1];
+
+ if (IS_ISP2401)
+ return;
ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "Input System Error Counters:\n");
ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE, "\tport[%d] = %d\n",
i, ia_css_sp_input_system_error_cnt[i]);
}
-#endif
}
-/*
-
-void sh_css_init_ddr_debug_queue(void)
-{
- ia_css_ptr ddr_debug_queue_addr =
- hmm_alloc(sizeof(debug_data_ddr_t), HMM_BO_PRIVATE, 0, NULL, 0);
- const struct ia_css_fw_info *fw;
- unsigned int HIVE_ADDR_debug_buffer_ddr_address;
-
- fw = &sh_css_sp_fw;
- HIVE_ADDR_debug_buffer_ddr_address =
- fw->info.sp.debug_buffer_ddr_address;
-
- (void)HIVE_ADDR_debug_buffer_ddr_address;
-
- debug_buffer_ddr_init(ddr_debug_queue_addr);
-
- sp_dmem_store_uint32(SP0_ID,
- (unsigned int)sp_address_of(debug_buffer_ddr_address),
- (uint32_t)(ddr_debug_queue_addr));
-}
-
-void sh_css_load_ddr_debug_queue(void)
-{
- debug_synch_queue_ddr();
-}
-
-void ia_css_debug_dump_ddr_debug_queue(void)
-{
- int i;
- sh_css_load_ddr_debug_queue();
- for (i = 0; i < DEBUG_BUF_SIZE; i++) {
- ia_css_debug_dtrace(IA_CSS_DEBUG_VERBOSE,
- "ddr_debug_queue[%d] = 0x%x\n",
- i, debug_data_ptr->buf[i]);
- }
-}
-*/
-
/*
* @brief Initialize the debug mode.
* Refer to "ia_css_debug.h" for more details.
return rc;
}
-static
-void dtrace_dot(const char *fmt, ...)
+static void __printf(1, 2) dtrace_dot(const char *fmt, ...)
{
va_list ap;
byte 2-3: data
*/
#if TRACE_ENABLE_SP0 || TRACE_ENABLE_SP1 || TRACE_ENABLE_ISP
-#ifndef ISP2401
-static void debug_dump_one_trace(TRACE_CORE_ID proc_id)
-#else
static void debug_dump_one_trace(enum TRACE_CORE_ID proc_id)
-#endif
{
#if defined(HAS_TRACER_V2)
u32 start_addr;
u32 start_addr_data;
u32 item_size;
-#ifndef ISP2401
u32 tmp;
-#else
u8 tid_val;
enum TRACE_DUMP_FORMAT dump_format;
-#endif
+
int i, j, max_trace_points, point_num, limit = -1;
/* using a static buffer here as the driver has issues allocating memory */
static u32 trace_read_buf[TRACE_BUFF_SIZE] = {0};
#endif
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
/* Tagger state dump function. The tagger is only available when the CSS
* contains an input system (2400 or 2401). */
void ia_css_debug_tagger_state(void)
i, tbuf_frames[i].exp_id, tbuf_frames[i].mark, tbuf_frames[i].lock);
}
}
-#endif /* defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401) */
/* ISP2401 */
void ia_css_debug_pc_dump(sp_ID_t id, unsigned int num_of_dumps)
#include "system_global.h"
#include <linux/kernel.h>
-#ifdef USE_INPUT_SYSTEM_VERSION_2
+#ifndef ISP2401
#include "ia_css_ifmtr.h"
#include <math_support.h>
#include "event_fifo.h"
#define __INLINE_SP__
-#if !defined(HAS_NO_INPUT_SYSTEM)
#include "input_system.h" /* MIPI_PREDICTOR_NONE,... */
-#endif
#include "assert_support.h"
/* System independent */
#include "sh_css_internal.h"
-#if !defined(HAS_NO_INPUT_SYSTEM)
#include "ia_css_isys.h"
-#endif
#define HBLANK_CYCLES (187)
#define MARKER_CYCLES (6)
-#if !defined(HAS_NO_INPUT_SYSTEM)
#include <hive_isp_css_streaming_to_mipi_types_hrt.h>
-#endif
/* The data type is used to send special cases:
* yuv420: odd lines (1, 3 etc) are twice as wide as even
inputfifo_mipi_data_type_rgb,
};
-#if !defined(HAS_NO_INPUT_SYSTEM)
static unsigned int inputfifo_curr_ch_id, inputfifo_curr_fmt_type;
-#endif
struct inputfifo_instance {
unsigned int ch_id;
enum atomisp_input_format input_format;
enum inputfifo_mipi_data_type type;
};
-#if !defined(HAS_NO_INPUT_SYSTEM)
/*
* Maintain a basic streaming to Mipi administration with ch_id as index
* ch_id maps on the "Mipi virtual channel ID" and can have value 0..3
s2mi->streaming = false;
return;
}
-#endif /* #if !defined(HAS_NO_INPUT_SYSTEM) */
#include <system_global.h>
#include "ia_css_isys_comm.h"
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/**
* Virtual Input System. (Input System 2401)
*/
-typedef input_system_cfg_t ia_css_isys_descr_t;
+typedef isp2401_input_system_cfg_t ia_css_isys_descr_t;
/* end of Virtual Input System */
#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
-input_system_error_t ia_css_isys_init(void);
+input_system_err_t ia_css_isys_init(void);
void ia_css_isys_uninit(void);
enum mipi_port_id ia_css_isys_port_to_mipi_port(
enum mipi_port_id api_port);
-#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
/**
* @brief Register one (virtual) stream. This is used to track when all
int ia_css_isys_convert_compressed_format(
struct ia_css_csi2_compression *comp,
- struct input_system_cfg_s *cfg);
+ struct isp2401_input_system_cfg_s *cfg);
unsigned int ia_css_csi2_calculate_input_system_alignment(
enum atomisp_input_format fmt_type);
#endif
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if !defined(ISP2401)
/* CSS Receiver */
void ia_css_isys_rx_configure(
const rx_cfg_t *config,
unsigned int irq_infos);
unsigned int ia_css_isys_rx_translate_irq_infos(unsigned int bits);
-#endif /* #if !defined(USE_INPUT_SYSTEM_VERSION_2401) */
+#endif /* #if !defined(ISP2401) */
/* @brief Translate format and compression to format type.
*
mipi_predictor_t compression,
unsigned int *fmt_type);
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/**
* Virtual Input System. (Input System 2401)
*/
#include <type_support.h>
#include <input_system.h>
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
#include <platform_support.h> /* inline */
#include <input_system_global.h>
#include <ia_css_stream_public.h> /* IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH */
return sp_thread_id * IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH + stream_id;
}
-#endif /* USE_INPUT_SYSTEM_VERSION_2401*/
+#endif /* ISP2401*/
#endif /*_IA_CSS_ISYS_COMM_H */
#include "system_global.h"
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
#include "assert_support.h"
#include "platform_support.h"
#include "system_global.h"
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
#include "assert_support.h"
#include "platform_support.h"
#include "input_system.h"
-#ifdef HAS_INPUT_SYSTEM_VERSION_2
#include "ia_css_isys.h"
#include "platform_support.h"
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
-#include "isys_dma.h" /* isys2401_dma_set_max_burst_size() */
+#ifdef ISP2401
+#include "isys_dma_public.h" /* isys2401_dma_set_max_burst_size() */
#include "isys_irq.h"
#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
-input_system_error_t ia_css_isys_init(void)
+#if !defined(ISP2401)
+input_system_err_t ia_css_isys_init(void)
{
backend_channel_cfg_t backend_ch0;
backend_channel_cfg_t backend_ch1;
target_cfg2400_t targetC;
u32 acq_mem_region_size = 24;
u32 acq_nof_mem_regions = 2;
- input_system_error_t error = INPUT_SYSTEM_ERR_NO_ERROR;
+ input_system_err_t error = INPUT_SYSTEM_ERR_NO_ERROR;
memset(&backend_ch0, 0, sizeof(backend_channel_cfg_t));
memset(&backend_ch1, 0, sizeof(backend_channel_cfg_t));
return error;
}
-#elif defined(USE_INPUT_SYSTEM_VERSION_2401)
-input_system_error_t ia_css_isys_init(void)
+#elif defined(ISP2401)
+input_system_err_t ia_css_isys_init(void)
{
ia_css_isys_csi_rx_lut_rmgr_init();
ia_css_isys_ibuf_rmgr_init();
}
#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
void ia_css_isys_uninit(void)
{
}
-#elif defined(USE_INPUT_SYSTEM_VERSION_2401)
+#elif defined(ISP2401)
void ia_css_isys_uninit(void)
{
ia_css_isys_csi_rx_lut_rmgr_uninit();
}
#endif
-#endif
#include "system_global.h"
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
#include "assert_support.h"
#include "platform_support.h"
#include "ia_css_irq.h"
#include "sh_css_internal.h"
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if !defined(ISP2401)
void ia_css_isys_rx_enable_all_interrupts(enum mipi_port_id port)
{
hrt_data bits = receiver_port_reg_load(RX0_ID,
_HRT_CSS_RECEIVER_IRQ_ENABLE_REG_IDX);
bits |= (1U << _HRT_CSS_RECEIVER_IRQ_OVERRUN_BIT) |
-#if defined(HAS_RX_VERSION_2)
(1U << _HRT_CSS_RECEIVER_IRQ_INIT_TIMEOUT_BIT) |
-#endif
(1U << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_ENTRY_BIT) |
(1U << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_EXIT_BIT) |
(1U << _HRT_CSS_RECEIVER_IRQ_ERR_SOT_HS_BIT) |
if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_OVERRUN_BIT))
infos |= IA_CSS_RX_IRQ_INFO_BUFFER_OVERRUN;
-#if defined(HAS_RX_VERSION_2)
if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_INIT_TIMEOUT_BIT))
infos |= IA_CSS_RX_IRQ_INFO_INIT_TIMEOUT;
-#endif
if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_ENTRY_BIT))
infos |= IA_CSS_RX_IRQ_INFO_ENTER_SLEEP_MODE;
if (bits & (1U << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_EXIT_BIT))
/* MW: Why do we remap the receiver bitmap */
if (irq_infos & IA_CSS_RX_IRQ_INFO_BUFFER_OVERRUN)
bits |= 1U << _HRT_CSS_RECEIVER_IRQ_OVERRUN_BIT;
-#if defined(HAS_RX_VERSION_2)
if (irq_infos & IA_CSS_RX_IRQ_INFO_INIT_TIMEOUT)
bits |= 1U << _HRT_CSS_RECEIVER_IRQ_INIT_TIMEOUT_BIT;
-#endif
if (irq_infos & IA_CSS_RX_IRQ_INFO_ENTER_SLEEP_MODE)
bits |= 1U << _HRT_CSS_RECEIVER_IRQ_SLEEP_MODE_ENTRY_BIT;
if (irq_infos & IA_CSS_RX_IRQ_INFO_EXIT_SLEEP_MODE)
return;
}
-#endif /* #if !defined(USE_INPUT_SYSTEM_VERSION_2401) */
+#endif /* #if !defined(ISP2401) */
int ia_css_isys_convert_stream_format_to_mipi_format(
enum atomisp_input_format input_format,
case ATOMISP_INPUT_FORMAT_EMBEDDED:
*fmt_type = MIPI_FORMAT_EMBEDDED;
break;
-#ifndef USE_INPUT_SYSTEM_VERSION_2401
+#ifndef ISP2401
case ATOMISP_INPUT_FORMAT_RAW_16:
/* This is not specified by Arasan, so we use
* 17 for now.
return 0;
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
static mipi_predictor_t sh_css_csi2_compression_type_2_mipi_predictor(
enum ia_css_csi2_compression_type type)
{
int ia_css_isys_convert_compressed_format(
struct ia_css_csi2_compression *comp,
- struct input_system_cfg_s *cfg)
+ struct isp2401_input_system_cfg_s *cfg)
{
int err = 0;
#endif
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if !defined(ISP2401)
void ia_css_isys_rx_configure(const rx_cfg_t *config,
const enum ia_css_input_mode input_mode)
{
-#if defined(HAS_RX_VERSION_2)
bool port_enabled[N_MIPI_PORT_ID];
bool any_port_enabled = false;
enum mipi_port_id port;
* INPUT_SYSTEM_CSI_RECEIVER_SELECT_BACKENG, 1);
*/
input_system_reg_store(INPUT_SYSTEM0_ID, 0x207, 1);
-#else
-#error "rx.c: RX version must be one of {RX_VERSION_2}"
-#endif
return;
}
}
return;
}
-#endif /* if !defined(USE_INPUT_SYSTEM_VERSION_2401) */
+#endif /* if !defined(ISP2401) */
#include "system_global.h"
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
#include "ia_css_isys.h"
#include "ia_css_debug.h"
*************************************************/
static bool create_input_system_channel(
- input_system_cfg_t *cfg,
+ isp2401_input_system_cfg_t *cfg,
bool metadata,
input_system_channel_t *channel);
input_system_channel_t *channel);
static bool create_input_system_input_port(
- input_system_cfg_t *cfg,
+ isp2401_input_system_cfg_t *cfg,
input_system_input_port_t *input_port);
static void destroy_input_system_input_port(
static bool calculate_input_system_channel_cfg(
input_system_channel_t *channel,
input_system_input_port_t *input_port,
- input_system_cfg_t *isys_cfg,
+ isp2401_input_system_cfg_t *isys_cfg,
input_system_channel_cfg_t *channel_cfg,
bool metadata);
static bool calculate_input_system_input_port_cfg(
input_system_channel_t *channel,
input_system_input_port_t *input_port,
- input_system_cfg_t *isys_cfg,
+ isp2401_input_system_cfg_t *isys_cfg,
input_system_input_port_cfg_t *input_port_cfg);
static bool acquire_sid(
s32 lines_per_frame,
s32 align_in_bytes,
bool online,
- ib_buffer_t *buf);
+ isp2401_ib_buffer_t *buf);
static void release_ib_buffer(
- ib_buffer_t *buf);
+ isp2401_ib_buffer_t *buf);
static bool acquire_dma_channel(
isys2401_dma_ID_t dma_id,
static bool calculate_tpg_cfg(
input_system_channel_t *channel,
input_system_input_port_t *input_port,
- input_system_cfg_t *isys_cfg,
+ isp2401_input_system_cfg_t *isys_cfg,
pixelgen_tpg_cfg_t *cfg);
static bool calculate_prbs_cfg(
input_system_channel_t *channel,
input_system_input_port_t *input_port,
- input_system_cfg_t *isys_cfg,
+ isp2401_input_system_cfg_t *isys_cfg,
pixelgen_prbs_cfg_t *cfg);
static bool calculate_fe_cfg(
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
csi_rx_frontend_cfg_t *cfg);
static bool calculate_be_cfg(
const input_system_input_port_t *input_port,
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
bool metadata,
csi_rx_backend_cfg_t *cfg);
static bool calculate_stream2mmio_cfg(
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
bool metadata,
stream2mmio_cfg_t *cfg);
static bool calculate_ibuf_ctrl_cfg(
const input_system_channel_t *channel,
const input_system_input_port_t *input_port,
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
ibuf_ctrl_cfg_t *cfg);
static bool calculate_isys2401_dma_cfg(
const input_system_channel_t *channel,
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
isys2401_dma_cfg_t *cfg);
static bool calculate_isys2401_dma_port_cfg(
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
bool raw_packed,
bool metadata,
isys2401_dma_port_cfg_t *cfg);
*
**************************************************/
static bool create_input_system_channel(
- input_system_cfg_t *cfg,
+ isp2401_input_system_cfg_t *cfg,
bool metadata,
input_system_channel_t *me)
{
}
static bool create_input_system_input_port(
- input_system_cfg_t *cfg,
+ isp2401_input_system_cfg_t *cfg,
input_system_input_port_t *me)
{
csi_mipi_packet_type_t packet_type;
static bool calculate_input_system_channel_cfg(
input_system_channel_t *channel,
input_system_input_port_t *input_port,
- input_system_cfg_t *isys_cfg,
+ isp2401_input_system_cfg_t *isys_cfg,
input_system_channel_cfg_t *channel_cfg,
bool metadata)
{
static bool calculate_input_system_input_port_cfg(
input_system_channel_t *channel,
input_system_input_port_t *input_port,
- input_system_cfg_t *isys_cfg,
+ isp2401_input_system_cfg_t *isys_cfg,
input_system_input_port_cfg_t *input_port_cfg)
{
bool rc;
s32 lines_per_frame,
s32 align_in_bytes,
bool online,
- ib_buffer_t *buf)
+ isp2401_ib_buffer_t *buf)
{
buf->stride = calculate_stride(bits_per_pixel, pixels_per_line, false,
align_in_bytes);
}
static void release_ib_buffer(
- ib_buffer_t *buf)
+ isp2401_ib_buffer_t *buf)
{
ia_css_isys_ibuf_rmgr_release(&buf->start_addr);
}
static bool calculate_tpg_cfg(
input_system_channel_t *channel,
input_system_input_port_t *input_port,
- input_system_cfg_t *isys_cfg,
+ isp2401_input_system_cfg_t *isys_cfg,
pixelgen_tpg_cfg_t *cfg)
{
memcpy(cfg, &isys_cfg->tpg_port_attr, sizeof(pixelgen_tpg_cfg_t));
static bool calculate_prbs_cfg(
input_system_channel_t *channel,
input_system_input_port_t *input_port,
- input_system_cfg_t *isys_cfg,
+ isp2401_input_system_cfg_t *isys_cfg,
pixelgen_prbs_cfg_t *cfg)
{
memcpy(cfg, &isys_cfg->prbs_port_attr, sizeof(pixelgen_prbs_cfg_t));
}
static bool calculate_fe_cfg(
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
csi_rx_frontend_cfg_t *cfg)
{
cfg->active_lanes = isys_cfg->csi_port_attr.active_lanes;
static bool calculate_be_cfg(
const input_system_input_port_t *input_port,
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
bool metadata,
csi_rx_backend_cfg_t *cfg)
{
}
static bool calculate_stream2mmio_cfg(
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
bool metadata,
stream2mmio_cfg_t *cfg
)
static bool calculate_ibuf_ctrl_cfg(
const input_system_channel_t *channel,
const input_system_input_port_t *input_port,
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
ibuf_ctrl_cfg_t *cfg)
{
const s32 bits_per_byte = 8;
static bool calculate_isys2401_dma_cfg(
const input_system_channel_t *channel,
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
isys2401_dma_cfg_t *cfg)
{
cfg->channel = channel->dma_channel;
/* See also: ia_css_dma_configure_from_info() */
static bool calculate_isys2401_dma_port_cfg(
- const input_system_cfg_t *isys_cfg,
+ const isp2401_input_system_cfg_t *isys_cfg,
bool raw_packed,
bool metadata,
isys2401_dma_port_cfg_t *cfg)
*/
bool ia_css_pipeline_get_sp_thread_id(unsigned int key, unsigned int *val);
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
/**
* @brief Get the pipeline io status
*
false, false, false, true, SH_CSS_BDS_FACTOR_1_00,
SH_CSS_PIPE_CONFIG_OVRD_NO_OVRD,
IA_CSS_INPUT_MODE_MEMORY, NULL, NULL,
-#if !defined(HAS_NO_INPUT_SYSTEM)
(enum mipi_port_id)0,
-#endif
NULL, NULL);
ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
return sp_group.pipe[thread_id].num_stages == 0;
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
struct sh_css_sp_pipeline_io_status *ia_css_pipeline_get_pipe_io_status(void)
{
return(&sh_css_sp_group.pipe_io_status);
the value as zero. This causes division by 0
exception as the size is used in a modular
division operation. */
- return EDOM;
+ return -EDOM;
}
}
struct ia_css_rmgr_vbuf_handle **handle)
{
u32 i;
- bool succes = false;
assert(pool);
assert(pool->recycle);
pool->handles[i] = NULL;
/* dont release, we are returning it...
ia_css_rmgr_refcount_release_vbuf(handle); */
- succes = true;
- break;
+ return;
}
}
}
#include "sh_css_internal.h"
#include "sh_css_mipi.h"
#include "sh_css_sp.h" /* sh_css_sp_group */
-#if !defined(HAS_NO_INPUT_SYSTEM)
#include "ia_css_isys.h"
-#endif
#include "ia_css_frame.h"
#include "sh_css_defs.h"
#include "sh_css_firmware.h"
#include "ia_css_pipe_util.h"
#include "ia_css_pipe_binarydesc.h"
#include "ia_css_pipe_stagedesc.h"
-#ifdef USE_INPUT_SYSTEM_VERSION_2
+#ifndef ISP2401
#include "ia_css_isys.h"
#endif
#include "assert_support.h"
#include "math_support.h"
#include "sw_event_global.h" /* Event IDs.*/
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
#include "ia_css_ifmtr.h"
#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
#include "input_system.h"
-#endif
#include "mmu_device.h" /* mmu_set_page_table_base_index(), ... */
#include "ia_css_mmu_private.h" /* sh_css_mmu_set_page_table_base_index() */
#include "gdc_device.h" /* HRT_GDC_N */
struct sh_css my_css;
-int (*sh_css_printf)(const char *fmt, va_list args) = NULL;
+int __printf(1, 0) (*sh_css_printf)(const char *fmt, va_list args) = NULL;
/* modes of work: stream_create and stream_destroy will update the save/restore data
only when in working mode, not suspend/resume
*info, struct frame_data_wrapper *frame);
#endif
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
static unsigned int get_crop_lines_for_bayer_order(const struct
ia_css_stream_config *config);
static unsigned int get_crop_columns_for_bayer_order(const struct
#define GP_ISEL_TPG_MODE 0x90058
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
static int
sh_css_config_input_network(struct ia_css_stream *stream) {
unsigned int fmt_type;
"sh_css_config_input_network() leave:\n");
return 0;
}
-#elif !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
+#elif defined(ISP2401)
static unsigned int csi2_protocol_calculate_max_subpixels_per_line(
enum atomisp_input_format format,
unsigned int pixels_per_line)
stream_cfg->source.port.num_lanes;
isys_stream_descr->csi_port_attr.fmt_type = fmt_type;
isys_stream_descr->csi_port_attr.ch_id = stream_cfg->channel_id;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
isys_stream_descr->online = stream_cfg->online;
#endif
err |= ia_css_isys_convert_compressed_format(
stream_cfg->metadata_config.resolution.width;
isys_stream_descr->metadata.lines_per_frame =
stream_cfg->metadata_config.resolution.height;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/* For new input system, number of str2mmio requests must be even.
* So we round up number of metadata lines to be even. */
if (isys_stream_descr->metadata.lines_per_frame > 0)
start_binary(struct ia_css_pipe *pipe,
struct ia_css_binary *binary)
{
- struct ia_css_stream *stream;
-
assert(pipe);
/* Acceleration uses firmware, the binary thus can be NULL */
- /* assert(binary != NULL); */
-
- (void)binary;
-
-#if !defined(HAS_NO_INPUT_SYSTEM)
- stream = pipe->stream;
-#else
- (void)pipe;
- (void)stream;
-#endif
if (binary)
sh_css_metrics_start_binary(&binary->metrics);
sh_binary_running = true;
#endif
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
- if (stream->reconfigure_css_rx) {
+#if !defined(ISP2401)
+ if (pipe->stream->reconfigure_css_rx) {
ia_css_isys_rx_configure(&pipe->stream->csi_rx_config,
pipe->stream->config.mode);
- stream->reconfigure_css_rx = false;
+ pipe->stream->reconfigure_css_rx = false;
}
#endif
}
if ((!pipe) || (!pipe->stream))
return -EINVAL;
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if !defined(ISP2401)
if (pipe->stream->reconfigure_css_rx)
ia_css_isys_rx_disable();
#endif
return -EINVAL;
sh_css_sp_start_binary_copy(ia_css_pipe_get_pipe_num(pipe), out_frame, pipe->stream->config.pixels_per_clock == 2);
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if !defined(ISP2401)
if (pipe->stream->reconfigure_css_rx)
{
ia_css_isys_rx_configure(&pipe->stream->csi_rx_config,
const struct ia_css_coordinate *coord = NULL;
const struct ia_css_isp_parameters *params = NULL;
-#if defined(HAS_NO_INPUT_SYSTEM)
- (void)input_mode;
-#endif
IA_CSS_ENTER_PRIVATE("me = %p, copy_ovrd = %d, input_mode = %d",
me, copy_ovrd, input_mode);
input_mode,
&me->stream->config.metadata_config,
&me->stream->info.metadata_info
-#if !defined(HAS_NO_INPUT_SYSTEM)
, (input_mode == IA_CSS_INPUT_MODE_MEMORY) ?
(enum mipi_port_id)0 :
me->stream->config.source.port.port,
-#endif
coord,
params);
static void
enable_interrupts(enum ia_css_irq_type irq_type)
{
-#ifdef USE_INPUT_SYSTEM_VERSION_2
+#ifndef ISP2401
enum mipi_port_id port;
#endif
bool enable_pulse = irq_type != IA_CSS_IRQ_TYPE_EDGE;
cnd_virq_enable_channel(
(enum virq_id)(IRQ_SW_CHANNEL1_ID + IRQ_SW_CHANNEL_OFFSET),
true);
-#if !defined(HAS_IRQ_MAP_VERSION_2)
- /* IRQ_SW_CHANNEL2_ID does not exist on 240x systems */
- cnd_virq_enable_channel(
- (enum virq_id)(IRQ_SW_CHANNEL2_ID + IRQ_SW_CHANNEL_OFFSET),
- true);
- virq_clear_all();
-#endif
-#ifdef USE_INPUT_SYSTEM_VERSION_2
+#ifndef ISP2401
for (port = 0; port < N_MIPI_PORT_ID; port++)
ia_css_isys_rx_enable_all_interrupts(port);
#endif
sh_css_init_buffer_queues();
*/
-#if defined(HAS_INPUT_SYSTEM_VERSION_2) && defined(HAS_INPUT_SYSTEM_VERSION_2401)
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
- gp_device_reg_store(GP_DEVICE0_ID, _REG_GP_SWITCH_ISYS2401_ADDR, 0);
-#elif defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
gp_device_reg_store(GP_DEVICE0_ID, _REG_GP_SWITCH_ISYS2401_ADDR, 1);
#endif
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
if (!IS_ISP2401)
dma_set_max_burst_size(DMA0_ID, HIVE_DMA_BUS_DDR_CONN,
if (ia_css_isys_init() != INPUT_SYSTEM_ERR_NO_ERROR)
err = -EINVAL;
-#endif
sh_css_params_map_and_store_default_gdc_lut();
}
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
/* old isys: need to allocate_mipi_frames() even in IA_CSS_PIPE_MODE_COPY */
if (pipe_id != IA_CSS_PIPE_ID_ACC)
{
if (err)
goto ERR;
}
-#elif defined(USE_INPUT_SYSTEM_VERSION_2401)
+#elif defined(ISP2401)
if ((pipe_id != IA_CSS_PIPE_ID_ACC) &&
(main_pipe->config.mode != IA_CSS_PIPE_MODE_COPY))
{
ia_css_rmgr_uninit();
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
/* needed for reprogramming the inputformatter after power cycle of css */
ifmtr_set_if_blocking_mode_reset = true;
#endif
}
ia_css_spctrl_unload_fw(SP0_ID);
sh_css_sp_set_sp_running(false);
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
/* check and free any remaining mipi frames */
free_mipi_frames(NULL);
-#endif
sh_css_sp_reset_global_vars();
-#if !defined(HAS_NO_INPUT_SYSTEM)
ia_css_isys_uninit();
-#endif
ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_uninit() leave: return_void\n");
}
-#if defined(HAS_IRQ_MAP_VERSION_2)
int ia_css_irq_translate(
unsigned int *irq_infos)
{
break;
case virq_isp:
break;
-#if !defined(HAS_NO_INPUT_SYSTEM)
case virq_isys_sof:
infos |= IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF;
break;
case virq_isys_csi:
infos |= IA_CSS_IRQ_INFO_INPUT_SYSTEM_ERROR;
break;
-#endif
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
case virq_ifmt0_id:
infos |= IA_CSS_IRQ_INFO_IF_ERROR;
break;
IA_CSS_ENTER("info=%d, enable=%d", info, enable);
switch (info) {
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
case IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF:
irq = virq_isys_sof;
break;
return 0;
}
-#else
-#error "sh_css.c: IRQ MAP must be one of { IRQ_MAP_VERSION_2 }"
-#endif
static unsigned int
sh_css_get_sw_interrupt_value(unsigned int irq)
bool continuous;
unsigned int i, idx;
unsigned int num_frames;
- struct ia_css_pipe *capture_pipe = NULL;
IA_CSS_ENTER_PRIVATE("pipe = %p, init_time = %d", pipe, init_time);
return -EINVAL;
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
/* For CSI2+, the continuous frame will hold the full input frame */
ref_info.res.width = pipe->stream->config.input_config.input_res.width;
ref_info.res.height = pipe->stream->config.input_config.input_res.height;
}
/* Write format back to binary */
- if (pipe_id == IA_CSS_PIPE_ID_PREVIEW)
- {
+ if (pipe_id == IA_CSS_PIPE_ID_PREVIEW) {
pipe->pipe_settings.preview.preview_binary.in_frame_info.format =
ref_info.format;
- capture_pipe = pipe->pipe_settings.preview.capture_pipe;
- } else if (pipe_id == IA_CSS_PIPE_ID_VIDEO)
- {
+ } else if (pipe_id == IA_CSS_PIPE_ID_VIDEO) {
pipe->pipe_settings.video.video_binary.in_frame_info.format = ref_info.format;
- capture_pipe = pipe->pipe_settings.video.capture_pipe;
- } else
- {
+ } else {
/* should not happen */
IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
return -EINVAL;
struct ia_css_binary_descr preview_descr;
bool online;
int err = 0;
- bool continuous, need_vf_pp = false;
+ bool need_vf_pp = false;
bool need_isp_copy_binary = false;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
bool sensor = false;
+#else
+ bool continuous;
#endif
/* preview only have 1 output pin now */
struct ia_css_frame_info *pipe_out_info = &pipe->output_info[0];
assert(pipe->mode == IA_CSS_PIPE_ID_PREVIEW);
online = pipe->stream->config.online;
- continuous = pipe->stream->config.continuous;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
sensor = pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR;
+#else
+ continuous = pipe->stream->config.continuous;
#endif
if (mycs->preview_binary.info)
return err;
}
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/* When the input system is 2401, only the Direct Sensor Mode
* Offline Preview uses the ISP copy binary.
*/
return err;
}
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
static unsigned int
get_crop_lines_for_bayer_order(
const struct ia_css_stream_config *config)
in_frame->info.format = format;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
if (format == IA_CSS_FRAME_FORMAT_RAW)
in_frame->info.format = (pipe->stream->config.pack_raw_pixels) ?
IA_CSS_FRAME_FORMAT_RAW_PACKED : IA_CSS_FRAME_FORMAT_RAW;
ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_INPUT_FRAME, thread_id, &queue_id);
in_frame->dynamic_queue_id = queue_id;
in_frame->buf_type = IA_CSS_BUFFER_TYPE_INPUT_FRAME;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
ia_css_get_crop_offsets(pipe, &in_frame->info);
#endif
err = ia_css_frame_init_planes(in_frame);
bool need_copy = false;
bool need_vf_pp = false;
bool need_yuv_pp = false;
- unsigned int num_output_pins;
bool need_in_frameinfo_memory = false;
unsigned int i, num_yuv_scaler;
me->dvs_frame_delay = pipe->dvs_frame_delay;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/* When the input system is 2401, always enable 'in_frameinfo_memory'
* except for the following: online or continuous
*/
copy_binary = &pipe->pipe_settings.video.copy_binary;
video_binary = &pipe->pipe_settings.video.video_binary;
vf_pp_binary = &pipe->pipe_settings.video.vf_pp_binary;
- num_output_pins = video_binary->info->num_output_pins;
yuv_scaler_binary = pipe->pipe_settings.video.yuv_scaler_binary;
num_yuv_scaler = pipe->pipe_settings.video.num_yuv_scaler;
goto ERR;
in_frame = me->stages->args.out_frame[0];
} else if (pipe->stream->config.continuous) {
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/* When continuous is enabled, configure in_frame with the
* last pipe, which is the copy pipe.
*/
struct ia_css_frame *tmp_out_frame = NULL;
for (i = 0; i < num_yuv_scaler; i++) {
- if (is_output_stage[i] == true) {
+ if (is_output_stage[i]) {
tmp_out_frame = out_frame;
} else {
tmp_out_frame = NULL;
struct ia_css_frame *out_frame;
struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS];
bool need_in_frameinfo_memory = false;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
bool sensor = false;
bool buffered_sensor = false;
bool online = false;
me = &pipe->pipeline;
ia_css_pipeline_clean(me);
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/* When the input system is 2401, always enable 'in_frameinfo_memory'
* except for the following:
* - Direct Sensor Mode Online Preview
if (err)
goto ERR;
in_frame = me->stages->args.out_frame[0];
-#ifndef ISP2401
- } else
- {
-#else
- } else if (pipe->stream->config.continuous)
- {
-#endif
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+ } else if (pipe->stream->config.continuous) {
+#ifdef ISP2401
/* When continuous is enabled, configure in_frame with the
* last pipe, which is the copy pipe.
*/
static int
preview_start(struct ia_css_pipe *pipe) {
- struct ia_css_pipeline *me;
- struct ia_css_binary *copy_binary, *preview_binary, *vf_pp_binary = NULL;
int err = 0;
struct ia_css_pipe *copy_pipe, *capture_pipe;
struct ia_css_pipe *acc_pipe;
return -EINVAL;
}
- me = &pipe->pipeline;
-
preview_pipe_input_mode = pipe->stream->config.mode;
copy_pipe = pipe->pipe_settings.preview.copy_pipe;
capture_pipe = pipe->pipe_settings.preview.capture_pipe;
acc_pipe = pipe->pipe_settings.preview.acc_pipe;
- copy_binary = &pipe->pipe_settings.preview.copy_binary;
- preview_binary = &pipe->pipe_settings.preview.preview_binary;
- if (pipe->pipe_settings.preview.vf_pp_binary.info)
- vf_pp_binary = &pipe->pipe_settings.preview.vf_pp_binary;
-
sh_css_metrics_start_frame();
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
/* multi stream video needs mipi buffers */
err = send_mipi_frames(pipe);
if (err) {
IA_CSS_LEAVE_ERR_PRIVATE(err);
return err;
}
-#endif
send_raw_frames(pipe);
{
pipe->stream->config.mode,
&pipe->stream->config.metadata_config,
&pipe->stream->info.metadata_info,
-#if !defined(HAS_NO_INPUT_SYSTEM)
pipe->stream->config.source.port.port,
-#endif
coord,
params);
IA_CSS_INPUT_MODE_MEMORY,
&pipe->stream->config.metadata_config,
&pipe->stream->info.metadata_info,
-#if !defined(HAS_NO_INPUT_SYSTEM)
(enum mipi_port_id)0,
-#endif
coord,
params);
}
IA_CSS_INPUT_MODE_MEMORY,
NULL,
NULL,
-#if !defined(HAS_NO_INPUT_SYSTEM)
(enum mipi_port_id)0,
-#endif
coord,
params);
}
case IA_CSS_BUFFER_TYPE_INPUT_FRAME:
case IA_CSS_BUFFER_TYPE_OUTPUT_FRAME:
case IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME:
- if ((pipe) && (pipe->stop_requested == true)) {
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
+ if (pipe && pipe->stop_requested) {
+#if !defined(ISP2401)
/* free mipi frames only for old input system
* for 2401 it is done in ia_css_stream_destroy call
*/
pipe->num_invalid_frames--;
if (frame->info.format == IA_CSS_FRAME_FORMAT_BINARY_8) {
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
frame->planes.binary.size = frame->data_bytes;
#else
frame->planes.binary.size =
pipe_id = pipe->mode;
- if (stream->started == true)
+ if (stream->started)
{
IA_CSS_WARNING("Cannot start stream that is already started");
IA_CSS_LEAVE_ERR(err);
stream->pipes[i]->pipeline.pipe_id);
err = ia_css_pipeline_request_stop(&stream->pipes[i]->pipeline);
- /*
- * Exit this loop if "ia_css_pipeline_request_stop()"
- * returns the error code.
- *
- * The error code would be generated in the following
- * two cases:
- * (1) The Scalar Processor has already been stopped.
- * (2) The "Host->SP" event queue is full.
- *
- * As the convention of using CSS API 2.0/2.1, such CSS
- * error code would be propogated from the CSS-internal
- * API returned value to the CSS API returned value. Then
- * the CSS driver should capture these error code and
- * handle it in the driver exception handling mechanism.
- */
- if (err) {
- goto ERR;
- }
+ /*
+ * Exit this loop if "ia_css_pipeline_request_stop()"
+ * returns the error code.
+ *
+ * The error code would be generated in the following
+ * two cases:
+ * (1) The Scalar Processor has already been stopped.
+ * (2) The "Host->SP" event queue is full.
+ *
+ * As the convention of using CSS API 2.0/2.1, such CSS
+ * error code would be propogated from the CSS-internal
+ * API returned value to the CSS API returned value. Then
+ * the CSS driver should capture these error code and
+ * handle it in the driver exception handling mechanism.
+ */
+ if (err)
+ goto ERR;
}
/*
return rval;
}
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
unsigned int
sh_css_get_mipi_sizes_for_check(const unsigned int port, const unsigned int idx)
{
info->isp_in_height = binary->internal_frame_info.res.height;
}
-#if defined(HAS_VAMEM_VERSION_2)
info->vamem_type = IA_CSS_VAMEM_TYPE_2;
-#elif defined(HAS_VAMEM_VERSION_1)
- info->vamem_type = IA_CSS_VAMEM_TYPE_1;
-#else
-#error "Unknown VAMEM version"
-#endif
ERR :
IA_CSS_LEAVE_ERR_PRIVATE(err);
pipe->num_invalid_frames, pipe->dvs_frame_delay);
/* pqiao TODO: temp hack for PO, should be removed after offline YUVPP is enabled */
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if !defined(ISP2401)
/* Copy */
if (!online && !continuous) {
/* TODO: what exactly needs doing, prepend the copy binary to
static int video_start(struct ia_css_pipe *pipe)
{
- struct ia_css_binary *copy_binary;
int err = 0;
struct ia_css_pipe *copy_pipe, *capture_pipe;
enum sh_css_pipe_config_override copy_ovrd;
copy_pipe = pipe->pipe_settings.video.copy_pipe;
capture_pipe = pipe->pipe_settings.video.capture_pipe;
- copy_binary = &pipe->pipe_settings.video.copy_binary;
-
sh_css_metrics_start_frame();
/* multi stream video needs mipi buffers */
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
err = send_mipi_frames(pipe);
if (err)
return err;
-#endif
send_raw_frames(pipe);
{
pipe->stream->config.mode,
&pipe->stream->config.metadata_config,
&pipe->stream->info.metadata_info,
-#if !defined(HAS_NO_INPUT_SYSTEM)
pipe->stream->config.source.port.port,
-#endif
coord,
params);
IA_CSS_INPUT_MODE_MEMORY,
&pipe->stream->config.metadata_config,
&pipe->stream->info.metadata_info,
-#if !defined(HAS_NO_INPUT_SYSTEM)
(enum mipi_port_id)0,
-#endif
coord,
params);
}
if (IS_ISP2401) {
/* ldc and capture_pp are not supported in the same pipeline */
- if (need_capt_ldc(pipe) == true)
+ if (need_capt_ldc(pipe))
return false;
}
struct ia_css_pipe *pipe)
{
bool online = false;
- bool memory = false;
- bool continuous = false;
bool need_pp = false;
bool need_isp_copy_binary = false;
bool need_ldc = false;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
bool sensor = false;
+#else
+ bool memory, continuous;
#endif
struct ia_css_frame_info prim_in_info,
prim_out_info,
pipe->mode == IA_CSS_PIPE_ID_COPY);
online = pipe->stream->config.online;
+#ifdef ISP2401
+ sensor = (pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR);
+#else
memory = pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY;
continuous = pipe->stream->config.continuous;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
- sensor = (pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR);
#endif
mycs = &pipe->pipe_settings.capture;
IA_CSS_LEAVE_ERR_PRIVATE(err);
return err;
}
- need_pp = 0;
- need_ldc = 0;
+ need_pp = false;
+ need_ldc = false;
}
/* we build up the pipeline starting at the end */
if (err)
return err;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/* When the input system is 2401, only the Direct Sensor Mode
* Offline Capture uses the ISP copy binary.
*/
}
/* Copy */
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/* For CSI2+, only the direct sensor mode/online requires ISP copy */
need_isp_copy = pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR;
#endif
}
/* Copy */
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/* For CSI2+, only the direct sensor mode/online requires ISP copy */
need_isp_copy = pipe->stream->config.mode == IA_CSS_INPUT_MODE_SENSOR;
#endif
switch (pipe->config.default_capture_config.mode) {
case IA_CSS_CAPTURE_MODE_RAW:
err = load_copy_binaries(pipe);
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
if (!err)
pipe->pipe_settings.capture.copy_binary.online = pipe->stream->config.online;
#endif
next_binary = NULL;
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
/*
* NOTES
* - Why does the "yuvpp" pipe needs "isp_copy_binary" (i.e. ISP Copy) when
*/
need_isp_copy_binary =
(pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_YUV422_8);
-#else /* !USE_INPUT_SYSTEM_VERSION_2401 */
+#else /* !ISP2401 */
need_isp_copy_binary = true;
-#endif /* USE_INPUT_SYSTEM_VERSION_2401 */
+#endif /* ISP2401 */
if (need_isp_copy_binary)
{
static int yuvpp_start(struct ia_css_pipe *pipe)
{
- struct ia_css_binary *copy_binary;
int err = 0;
enum sh_css_pipe_config_override copy_ovrd;
enum ia_css_input_mode yuvpp_pipe_input_mode;
yuvpp_pipe_input_mode = pipe->stream->config.mode;
- copy_binary = &pipe->pipe_settings.yuvpp.copy_binary;
-
sh_css_metrics_start_frame();
/* multi stream video needs mipi buffers */
-#if !defined(HAS_NO_INPUT_SYSTEM) && (defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401))
err = send_mipi_frames(pipe);
if (err) {
IA_CSS_LEAVE_ERR_PRIVATE(err);
return err;
}
-#endif
{
unsigned int thread_id;
*vf_pp_binary,
*yuv_scaler_binary;
bool need_scaler = false;
- unsigned int num_stage, num_vf_pp_stage, num_output_stage;
+ unsigned int num_stage, num_output_stage;
unsigned int i, j;
struct ia_css_frame *in_frame = NULL;
struct ia_css_frame *vf_frame[IA_CSS_PIPE_MAX_OUTPUT_STAGE];
struct ia_css_pipeline_stage_desc stage_desc;
bool need_in_frameinfo_memory = false;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
bool sensor = false;
bool buffered_sensor = false;
bool online = false;
}
ia_css_pipe_util_create_output_frames(bin_out_frame);
num_stage = pipe->pipe_settings.yuvpp.num_yuv_scaler;
- num_vf_pp_stage = pipe->pipe_settings.yuvpp.num_vf_pp;
num_output_stage = pipe->pipe_settings.yuvpp.num_output;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/* When the input system is 2401, always enable 'in_frameinfo_memory'
* except for the following:
* - Direct Sensor Mode Online Capture
{
struct ia_css_frame *in_frame_local = NULL;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/* After isp copy is enabled in_frame needs to be passed. */
if (!online)
in_frame_local = in_frame;
struct ia_css_frame *vf_frame;
struct ia_css_pipeline_stage_desc stage_desc;
bool need_in_frameinfo_memory = false;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
bool sensor = false;
bool buffered_sensor = false;
bool online = false;
ia_css_pipeline_clean(me);
ia_css_pipe_util_create_output_frames(out_frames);
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
/* When the input system is 2401, always enable 'in_frameinfo_memory'
* except for the following:
* - Direct Sensor Mode Online Capture
{
if (raw) {
ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame);
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
if (!continuous) {
ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary,
out_frames, in_frame, NULL);
}
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
/* old isys: need to send_mipi_frames() in all pipe modes */
err = send_mipi_frames(pipe);
if (err) {
IA_CSS_LEAVE_ERR_PRIVATE(err);
return err;
}
-#elif defined(USE_INPUT_SYSTEM_VERSION_2401)
+#elif defined(ISP2401)
if (pipe->config.mode != IA_CSS_PIPE_MODE_COPY) {
err = send_mipi_frames(pipe);
if (err) {
}
start_pipe(pipe, copy_ovrd, pipe->stream->config.mode);
-#if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if !defined(ISP2401)
/*
* old isys: for IA_CSS_PIPE_MODE_COPY pipe, isys rx has to be configured,
* which is currently done in start_binary(); but COPY pipe contains no binary,
return 0;
}
-#if !defined(HAS_NO_INPUT_SYSTEM)
void
ia_css_stream_send_input_frame(const struct ia_css_stream *stream,
const unsigned short *data,
ia_css_inputfifo_end_frame(stream->config.channel_id);
}
-#endif
static void
append_firmware(struct ia_css_fw_info **l, struct ia_css_fw_info *firmware) {
unsigned int o = offsetof(struct host_sp_communication, host2sp_command)
/ sizeof(int);
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
unsigned int i;
-#endif
ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
"sh_css_init_host_sp_control_vars() enter: void\n");
#endif
store_sp_array_uint(host_sp_com, o, host2sp_cmd_ready);
-#if !defined(HAS_NO_INPUT_SYSTEM)
for (i = 0; i < N_CSI_PORTS; i++) {
sh_css_update_host2sp_num_mipi_frames
(my_css.num_mipi_frames[i]);
}
-#endif
ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
"sh_css_init_host_sp_control_vars() leave: return_void\n");
return err;
}
-int
-ia_css_pipe_create(const struct ia_css_pipe_config *config,
- struct ia_css_pipe **pipe) {
-#ifndef ISP2401
- if (!config)
-#else
+int ia_css_pipe_create(const struct ia_css_pipe_config *config,
+ struct ia_css_pipe **pipe)
+{
int err = 0;
IA_CSS_ENTER_PRIVATE("config = %p, pipe = %p", config, pipe);
- if (!config)
- {
+ if (!config || !pipe) {
IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
-#endif
return -EINVAL;
-#ifndef ISP2401
- if (!pipe)
-#else
-}
-
-if (!pipe)
-{
- IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
-#endif
- return -EINVAL;
-#ifndef ISP2401
- return ia_css_pipe_create_extra(config, NULL, pipe);
-#else
-}
+ }
-err = ia_css_pipe_create_extra(config, NULL, pipe);
+ err = ia_css_pipe_create_extra(config, NULL, pipe);
-if (err == 0)
-{
- IA_CSS_LOG("pipe created successfully = %p", *pipe);
-}
+ if (err == 0) {
+ IA_CSS_LOG("pipe created successfully = %p", *pipe);
+ }
-IA_CSS_LEAVE_ERR_PRIVATE(err);
+ IA_CSS_LEAVE_ERR_PRIVATE(err);
-return err;
-#endif
+ return err;
}
int
return err;
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
/* Configuration of INPUT_SYSTEM_VERSION_2401 is done on SP */
static int
ia_css_stream_configure_rx(struct ia_css_stream *stream) {
int err = -EINVAL;
struct ia_css_metadata_info md_info;
struct ia_css_resolution effective_res;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
bool aspect_ratio_crop_enabled = false;
#endif
return err;
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
/* We don't support metadata for JPEG stream, since they both use str2mem */
if (stream_config->input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8 &&
stream_config->metadata_config.resolution.height > 0)
}
#endif
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
if (stream_config->online && stream_config->pack_raw_pixels)
{
IA_CSS_LOG("online and pack raw is invalid on input system 2401");
}
#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
ia_css_debug_pipe_graph_dump_stream_config(stream_config);
/* check if mipi size specified */
if (stream_config->mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
if (!stream_config->online)
#endif
{
return err;
}
}
-#endif
/* Currently we only supported metadata up to a certain size. */
err = metadata_info_init(&stream_config->metadata_config, &md_info);
/* take over stream config */
curr_stream->config = *stream_config;
-#if defined(USE_INPUT_SYSTEM_VERSION_2401) && defined(CSI2P_DISABLE_ISYS2401_ONLINE_MODE)
+#if defined(ISP2401)
if (stream_config->mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR &&
stream_config->online)
curr_stream->config.online = false;
#endif
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
if (curr_stream->config.online)
{
curr_stream->config.source.port.num_lanes =
{
case IA_CSS_INPUT_MODE_SENSOR:
case IA_CSS_INPUT_MODE_BUFFERED_SENSOR:
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
ia_css_stream_configure_rx(curr_stream);
#endif
break;
case IA_CSS_INPUT_MODE_TPG:
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
IA_CSS_LOG("tpg_configuration: x_mask=%d, y_mask=%d, x_delta=%d, y_delta=%d, xy_mask=%d",
curr_stream->config.source.tpg.x_mask,
curr_stream->config.source.tpg.y_mask,
#endif
break;
case IA_CSS_INPUT_MODE_PRBS:
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
IA_CSS_LOG("mode prbs");
sh_css_sp_configure_prbs(curr_stream->config.source.prbs.seed);
#endif
IA_CSS_LOG("mode sensor/default");
}
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
err = aspect_ratio_crop_init(curr_stream,
pipes,
&aspect_ratio_crop_enabled);
if (err)
{
IA_CSS_LEAVE_ERR(err);
- return err;
+ goto ERR;
}
#endif
for (i = 0; i < num_pipes; i++)
if (effective_res.height == 0 || effective_res.width == 0) {
effective_res = curr_pipe->stream->config.input_config.effective_res;
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
/* The aspect ratio cropping is currently only
* supported on the new input system. */
if (aspect_ratio_crop_check(aspect_ratio_crop_enabled, curr_pipe)) {
IA_CSS_PIPE_MODE_VIDEO, false);
acc_pipe = find_pipe(pipes, num_pipes,
IA_CSS_PIPE_MODE_ACC, false);
- if (acc_pipe && num_pipes == 2 && curr_stream->cont_capt == true)
+ if (acc_pipe && num_pipes == 2 && curr_stream->cont_capt)
curr_stream->cont_capt =
false; /* preview + QoS case will not need cont_capt switch */
- if (curr_stream->cont_capt == true) {
+ if (curr_stream->cont_capt) {
capture_pipe = find_pipe(pipes, num_pipes,
IA_CSS_PIPE_MODE_CAPTURE, false);
if (!capture_pipe) {
preview_pipe->pipe_settings.preview.copy_pipe = copy_pipe;
copy_pipe->stream = curr_stream;
}
- if (preview_pipe && (curr_stream->cont_capt == true)) {
+ if (preview_pipe && curr_stream->cont_capt) {
preview_pipe->pipe_settings.preview.capture_pipe = capture_pipe;
}
if (video_pipe && !video_pipe->pipe_settings.video.copy_pipe) {
video_pipe->pipe_settings.video.copy_pipe = copy_pipe;
copy_pipe->stream = curr_stream;
}
- if (video_pipe && (curr_stream->cont_capt == true)) {
+ if (video_pipe && curr_stream->cont_capt) {
video_pipe->pipe_settings.video.capture_pipe = capture_pipe;
}
if (preview_pipe && acc_pipe) {
if ((stream->last_pipe) &&
ia_css_pipeline_is_mapped(stream->last_pipe->pipe_num))
{
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
bool free_mpi;
for (i = 0; i < stream->num_pipes; i++) {
return err;
}
-#if !defined(HAS_NO_INPUT_SYSTEM)
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
if ((stream->config.mode == IA_CSS_INPUT_MODE_SENSOR) ||
(stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR))
stream_register_with_csi_rx(stream);
#endif
-#endif
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
/* Initialize mipi size checks */
if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
{
}
#endif
-#if !defined(HAS_NO_INPUT_SYSTEM)
if (stream->config.mode != IA_CSS_INPUT_MODE_MEMORY)
{
err = sh_css_config_input_network(stream);
if (err)
return err;
}
-#endif /* !HAS_NO_INPUT_SYSTEM */
err = sh_css_pipe_start(stream);
IA_CSS_LEAVE_ERR(err);
ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_stop: stopping %d\n",
stream->last_pipe->mode);
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
/* De-initialize mipi size checks */
if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
{
void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map)
{
unsigned int thread_id;
- enum ia_css_pipe_id pipe_id;
unsigned int pipe_num;
bool need_input_queue;
IA_CSS_ENTER("");
assert(pipe);
- pipe_id = pipe->mode;
pipe_num = pipe->pipe_num;
ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
-#if defined(HAS_NO_INPUT_SYSTEM) || defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
need_input_queue = true;
#else
need_input_queue = pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY;
return err;
}
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
static int
aspect_ratio_crop_init(struct ia_css_stream *curr_stream,
struct ia_css_pipe *pipes[],
#define SH_CSS_MAX_STAGES 8 /* primary_stage[1-6], capture_pp, vf_pp */
/* For CSI2+ input system, it requires extra paddinga from vmem */
-#ifdef CONFIG_CSI2_PLUS
-#define _ISP_EXTRA_PADDING_VECS 2
-#else
#define _ISP_EXTRA_PADDING_VECS 0
-#endif /* CONFIG_CSI2_PLUS */
#endif /* _SH_CSS_DEFS_H_ */
static struct firmware_header *firmware_header;
-/* The string STR is a place holder
+/*
+ * The string STR is a place holder
* which will be replaced with the actual RELEASE_VERSION
- * during package generation. Please do not modify */
+ * during package generation. Please do not modify
+ */
static const char *isp2400_release_version = STR(irci_stable_candrpv_0415_20150521_0458);
static const char *isp2401_release_version = STR(irci_ecr - master_20150911_0724);
/* Setup sp/sp1 binary */
static int
setup_binary(struct ia_css_fw_info *fw, const char *fw_data,
- struct ia_css_fw_info *sh_css_fw, unsigned int binary_id) {
+ struct ia_css_fw_info *sh_css_fw, unsigned int binary_id)
+{
const char *blob_data;
if ((!fw) || (!fw_data))
int
sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi,
struct ia_css_blob_descr *bd,
- unsigned int index) {
+ unsigned int index)
+{
const char *name;
const unsigned char *blob;
return -EINVAL;
/* Special case: only one binary in fw */
- if (!bi) bi = (const struct ia_css_fw_info *)fw;
+ if (!bi)
+ bi = (const struct ia_css_fw_info *)fw;
name = fw + bi->blob.prog_name_offset;
blob = (const unsigned char *)fw + bi->blob.offset;
/* sanity check */
- if (bi->blob.size != bi->blob.text_size + bi->blob.icache_size + bi->blob.data_size + bi->blob.padding_size)
- {
+ if (bi->blob.size !=
+ bi->blob.text_size + bi->blob.icache_size +
+ bi->blob.data_size + bi->blob.padding_size) {
/* sanity check, note the padding bytes added for section to DDR alignment */
return -EINVAL;
}
bd->blob = blob;
bd->header = *bi;
- if (bi->type == ia_css_isp_firmware || bi->type == ia_css_sp_firmware)
- {
+ if (bi->type == ia_css_isp_firmware || bi->type == ia_css_sp_firmware) {
char *namebuffer;
namebuffer = kstrdup(name, GFP_KERNEL);
if (!namebuffer)
return -ENOMEM;
bd->name = fw_minibuffer[index].name = namebuffer;
- } else
- {
+ } else {
bd->name = name;
}
- if (bi->type == ia_css_isp_firmware)
- {
+ if (bi->type == ia_css_isp_firmware) {
size_t paramstruct_size = sizeof(struct ia_css_memory_offsets);
size_t configstruct_size = sizeof(struct ia_css_config_memory_offsets);
size_t statestruct_size = sizeof(struct ia_css_state_memory_offsets);
}
/* For now, let's just accept a wrong version, even if wrong */
- return 0;
+ return false;
}
static const char * const fw_type_name[] = {
int
sh_css_load_firmware(struct device *dev, const char *fw_data,
- unsigned int fw_size) {
+ unsigned int fw_size)
+{
unsigned int i;
struct ia_css_fw_info *binaries;
struct sh_css_fw_bi_file_h *file_header;
firmware_header = (struct firmware_header *)fw_data;
file_header = &firmware_header->file_header;
binaries = &firmware_header->binary_header;
- strscpy(FW_rel_ver_name, file_header->version, min(sizeof(FW_rel_ver_name), sizeof(file_header->version)));
+ strscpy(FW_rel_ver_name, file_header->version,
+ min(sizeof(FW_rel_ver_name), sizeof(file_header->version)));
ret = sh_css_check_firmware_version(dev, fw_data);
if (ret) {
IA_CSS_ERROR("CSS code version (%s) and firmware version (%s) mismatch!",
sh_css_num_binaries = file_header->binary_nr;
/* Only allocate memory for ISP blob info */
- if (sh_css_num_binaries > NUM_OF_SPS)
- {
+ if (sh_css_num_binaries > NUM_OF_SPS) {
sh_css_blob_info = kmalloc(
(sh_css_num_binaries - NUM_OF_SPS) *
sizeof(*sh_css_blob_info), GFP_KERNEL);
if (!fw_minibuffer)
return -ENOMEM;
- for (i = 0; i < sh_css_num_binaries; i++)
- {
+ for (i = 0; i < sh_css_num_binaries; i++) {
struct ia_css_fw_info *bi = &binaries[i];
- /* note: the var below is made static as it is quite large;
- if it is not static it ends up on the stack which could
- cause issues for drivers
- */
+ /*
+ * note: the var below is made static as it is quite large;
+ * if it is not static it ends up on the stack which could
+ * cause issues for drivers
+ */
static struct ia_css_blob_descr bd;
int err;
return err;
} else {
- /* All subsequent binaries (including bootloaders) (i>NUM_OF_SPS) are ISP firmware */
+ /*
+ * All subsequent binaries
+ * (including bootloaders) (i>NUM_OF_SPS)
+ * are ISP firmware
+ */
if (i < NUM_OF_SPS)
return -EINVAL;
sh_css_load_blob(const unsigned char *blob, unsigned int size)
{
ia_css_ptr target_addr = hmm_alloc(size, HMM_BO_PRIVATE, 0, NULL, 0);
- /* this will allocate memory aligned to a DDR word boundary which
- is required for the CSS DMA to read the instructions. */
+ /*
+ * this will allocate memory aligned to a DDR word boundary which
+ * is required for the CSS DMA to read the instructions.
+ */
assert(blob);
if (target_addr)
int sh_css_hrt_sp_wait(void)
{
-#if defined(HAS_IRQ_MAP_VERSION_2)
irq_sw_channel_id_t irq_id = IRQ_SW_CHANNEL0_ID;
-#else
- irq_sw_channel_id_t irq_id = IRQ_SW_CHANNEL2_ID;
-#endif
/*
* Wait till SP is idle or till there is a SW2 interrupt
* The SW2 interrupt will be used when frameloop runs on SP
#include <platform_support.h>
#include <stdarg.h>
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
#include "input_formatter.h"
#endif
#include "input_system.h"
#define SH_CSS_MAX_IF_CONFIGS 3 /* Must match with IA_CSS_NR_OF_CONFIGS (not defined yet).*/
#define SH_CSS_IF_CONFIG_NOT_NEEDED 0xFF
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
#define SH_CSS_ENABLE_METADATA
-#endif
-#if defined(SH_CSS_ENABLE_METADATA) && !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(SH_CSS_ENABLE_METADATA) && !defined(ISP2401)
#define SH_CSS_ENABLE_METADATA_THREAD
#endif
#elif SP_DEBUG == SP_DEBUG_TRACE
-#if 1
/* Example of just one global trace */
#define SH_CSS_SP_DBG_NR_OF_TRACES (1)
#define SH_CSS_SP_DBG_TRACE_DEPTH (40)
-#else
-/* E.g. if you like separate traces for 4 threads */
-#define SH_CSS_SP_DBG_NR_OF_TRACES (4)
-#define SH_CSS_SP_DBG_TRACE_DEPTH (10)
-#endif
#define SH_CSS_SP_DBG_TRACE_FILE_ID_BIT_POS (13)
u32 dma_sw_reg;
};
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
/* SP input formatter configuration.*/
struct sh_css_sp_input_formatter_set {
u32 stream_format;
frames are locked when their EOF event is successfully sent to the
host (true) or when they are passed to the preview/video pipe
(false). */
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
struct {
u8 a_changed;
u8 b_changed;
set[SH_CSS_MAX_IF_CONFIGS]; /* CSI-2 port is used as index. */
} input_formatter;
#endif
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
sync_generator_cfg_t sync_gen;
tpg_cfg_t tpg;
prbs_cfg_t prbs;
#define SH_CSS_PIPE_CONFIG_SAMPLE_PARAMS_MASK \
((SH_CSS_PIPE_CONFIG_SAMPLE_PARAMS << SH_CSS_MAX_SP_THREADS) - 1)
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
struct sh_css_sp_pipeline_terminal {
union {
/* Input System 2401 */
struct sh_css_sp_group {
struct sh_css_sp_config config;
struct sh_css_sp_pipeline pipe[SH_CSS_MAX_SP_THREADS];
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
struct sh_css_sp_pipeline_io pipe_io[SH_CSS_MAX_SP_THREADS];
struct sh_css_sp_pipeline_io_status pipe_io_status;
#endif
ia_css_ptr host2sp_offline_frames[NUM_CONTINUOUS_FRAMES];
ia_css_ptr host2sp_offline_metadata[NUM_CONTINUOUS_FRAMES];
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
ia_css_ptr host2sp_mipi_frames[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM];
ia_css_ptr host2sp_mipi_metadata[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM];
u32 host2sp_num_mipi_frames[N_CSI_PORTS];
-#endif
u32 host2sp_cont_avail_num_raw_frames;
u32 host2sp_cont_extra_num_raw_frames;
u32 host2sp_cont_target_num_raw_frames;
};
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
#define SIZE_OF_HOST_SP_COMMUNICATION_STRUCT \
(sizeof(uint32_t) + \
(NUM_CONTINUOUS_FRAMES * SIZE_OF_HRT_VADDRESS * 2) + \
(N_CSI_PORTS * NUM_MIPI_FRAMES_PER_STREAM * SIZE_OF_HRT_VADDRESS * 2) + \
((3 + N_CSI_PORTS) * sizeof(uint32_t)) + \
(NR_OF_PIPELINES * SIZE_OF_SH_CSS_EVENT_IRQ_MASK_STRUCT))
-#else
-#define SIZE_OF_HOST_SP_COMMUNICATION_STRUCT \
- (sizeof(uint32_t) + \
- (NUM_CONTINUOUS_FRAMES * SIZE_OF_HRT_VADDRESS * 2) + \
- (3 * sizeof(uint32_t)) + \
- (NR_OF_PIPELINES * SIZE_OF_SH_CSS_EVENT_IRQ_MASK_STRUCT))
-#endif
struct host_sp_queues {
/*
#define SIZE_OF_HOST_SP_QUEUES_STRUCT \
(SIZE_OF_QUEUES_ELEMS + SIZE_OF_QUEUES_DESC)
-extern int (*sh_css_printf)(const char *fmt, va_list args);
+extern int __printf(1, 0) (*sh_css_printf)(const char *fmt, va_list args);
-static inline void
-sh_css_print(const char *fmt, ...)
+static inline void __printf(1, 2) sh_css_print(const char *fmt, ...)
{
va_list ap;
}
}
-static inline void
-sh_css_vprint(const char *fmt, va_list args)
+static inline void __printf(1, 0) sh_css_vprint(const char *fmt, va_list args)
{
if (sh_css_printf)
sh_css_printf(fmt, args);
unsigned int width,
unsigned int aligned);
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
unsigned int
sh_css_get_mipi_sizes_for_check(const unsigned int port,
struct ia_css_pipe *
find_pipe_by_num(uint32_t pipe_num);
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
void
ia_css_get_crop_offsets(
struct ia_css_pipe *pipe,
#include "sh_css_sp.h" /* sh_css_update_host2sp_mipi_frame sh_css_update_host2sp_num_mipi_frames ... */
#include "sw_event_global.h" /* IA_CSS_PSYS_SW_EVENT_MIPI_BUFFERS_READY */
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
static u32
ref_count_mipi_allocation[N_CSI_PORTS]; /* Initialized in mipi_init */
-#endif
int
ia_css_mipi_frame_specify(const unsigned int size_mem_words,
unsigned int mem_words = 0;
unsigned int width_padded = width;
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
/* The changes will be reverted as soon as RAW
* Buffers are deployed by the 2401 Input System
* in the non-continuous use scenario.
return err;
}
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
int
ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port,
const unsigned int size_mem_words) {
void
mipi_init(void)
{
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
unsigned int i;
for (i = 0; i < N_CSI_PORTS; i++)
ref_count_mipi_allocation[i] = 0;
-#endif
}
int
calculate_mipi_buff_size(
struct ia_css_stream_config *stream_cfg,
unsigned int *size_mem_words) {
-#if !defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if !defined(ISP2401)
int err = -EINVAL;
(void)stream_cfg;
(void)size_mem_words;
int
allocate_mipi_frames(struct ia_css_pipe *pipe,
struct ia_css_stream_info *info) {
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
int err = -EINVAL;
unsigned int port;
- struct ia_css_frame_info mipi_intermediate_info;
ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
"allocate_mipi_frames(%p) enter:\n", pipe);
return -EINVAL;
}
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
if (pipe->stream->config.online)
{
ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
return -EINVAL;
}
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
err = calculate_mipi_buff_size(
&pipe->stream->config,
&my_css.mipi_frame_size[port]);
#endif
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
if (ref_count_mipi_allocation[port] != 0)
{
ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
ref_count_mipi_allocation[port]++;
- /* TODO: Cleaning needed. */
- /* This code needs to modified to allocate the MIPI frames in the correct normal way
- with an allocate from info, by justin */
- mipi_intermediate_info = pipe->pipe_settings.video.video_binary.internal_frame_info;
- mipi_intermediate_info.res.width = 0;
- mipi_intermediate_info.res.height = 0;
- /* To indicate it is not (yet) valid format. */
- mipi_intermediate_info.format = IA_CSS_FRAME_FORMAT_NUM;
- mipi_intermediate_info.padded_width = 0;
- mipi_intermediate_info.raw_bit_depth = 0;
-
/* AM TODO: mipi frames number should come from stream struct. */
my_css.num_mipi_frames[port] = NUM_MIPI_FRAMES_PER_STREAM;
"allocate_mipi_frames(%p) exit:\n", pipe);
return err;
-#else
- (void)pipe;
- (void)info;
- return 0;
-#endif
}
int
free_mipi_frames(struct ia_css_pipe *pipe) {
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
int err = -EINVAL;
unsigned int port;
}
if (ref_count_mipi_allocation[port] > 0) {
-#if defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
assert(ref_count_mipi_allocation[port] == 1);
if (ref_count_mipi_allocation[port] != 1) {
ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
"free_mipi_frames(%p) exit (deallocated).\n", pipe);
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
else {
/* 2401 system allows multiple streams to use same physical port. This is not
* true for 2400 system. Currently 2401 uses MIPI buffers as a temporary solution.
ref_count_mipi_allocation[port] = 0;
}
}
-#else
- (void)pipe;
-#endif
return 0;
}
int
send_mipi_frames(struct ia_css_pipe *pipe) {
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
int err = -EINVAL;
unsigned int i;
#ifndef ISP2401
(uint8_t)my_css.num_mipi_frames[port],
0 /* not used */);
IA_CSS_LEAVE_ERR_PRIVATE(0);
-#else
- (void)pipe;
-#endif
return 0;
}
const struct ia_css_binary *binary,
unsigned int bds_factor)
{
- unsigned int input_width,
- input_height,
- table_width,
- table_height,
- left_padding,
- top_padding,
- padded_width,
- left_cropping,
- i;
+ unsigned int input_width, input_height, table_width, table_height, i;
+ unsigned int left_padding, top_padding, left_cropping;
unsigned int bds_numerator, bds_denominator;
int right_padding;
return;
}
- padded_width = binary->in_frame_info.padded_width;
- /* We use the ISP input resolution for the shading table because
- shading correction is performed in the bayer domain (before bayer
- down scaling). */
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
- padded_width = CEIL_MUL(binary->effective_in_frame_res.width + 2 *
- ISP_VEC_NELEMS,
- 2 * ISP_VEC_NELEMS);
-#endif
+ /*
+ * We use the ISP input resolution for the shading table because
+ * shading correction is performed in the bayer domain (before bayer
+ * down scaling).
+ */
input_height = binary->in_frame_info.res.height;
input_width = binary->in_frame_info.res.width;
left_padding = binary->left_padding;
#include "ia_css_host_data.h"
#include "ia_css_pipe.h"
#include "ia_css_pipe_binarydesc.h"
-#if 0
-#include "ia_css_system_ctrl.h"
-#endif
/* Include all kernel host interfaces for ISP1 */
/* Digital Zoom lookup table. See documentation for more details about the
* contents of this table.
*/
-#if defined(HAS_GDC_VERSION_2)
-#if defined(CONFIG_CSI2_PLUS)
-/*
- * Coefficients from
- * Css_Mizuchi/regressions/20140424_0930/all/applications/common/gdc_v2_common/lut.h
- */
-
-static const int zoom_table[4][HRT_GDC_N] = {
- {
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, -1,
- -1, -1, -1, -1, -1, -1, -1, -1,
- -1, -2, -2, -2, -2, -2, -2, -2,
- -3, -3, -3, -3, -3, -3, -3, -4,
- -4, -4, -4, -4, -5, -5, -5, -5,
- -5, -5, -6, -6, -6, -6, -7, -7,
- -7, -7, -7, -8, -8, -8, -8, -9,
- -9, -9, -9, -10, -10, -10, -10, -11,
- -11, -11, -12, -12, -12, -12, -13, -13,
- -13, -14, -14, -14, -15, -15, -15, -15,
- -16, -16, -16, -17, -17, -17, -18, -18,
- -18, -19, -19, -20, -20, -20, -21, -21,
- -21, -22, -22, -22, -23, -23, -24, -24,
- -24, -25, -25, -25, -26, -26, -27, -27,
- -28, -28, -28, -29, -29, -30, -30, -30,
- -31, -31, -32, -32, -33, -33, -33, -34,
- -34, -35, -35, -36, -36, -37, -37, -37,
- -38, -38, -39, -39, -40, -40, -41, -41,
- -42, -42, -43, -43, -44, -44, -45, -45,
- -46, -46, -47, -47, -48, -48, -49, -49,
- -50, -50, -51, -51, -52, -52, -53, -53,
- -54, -54, -55, -55, -56, -56, -57, -57,
- -58, -59, -59, -60, -60, -61, -61, -62,
- -62, -63, -63, -64, -65, -65, -66, -66,
- -67, -67, -68, -69, -69, -70, -70, -71,
- -71, -72, -73, -73, -74, -74, -75, -75,
- -76, -77, -77, -78, -78, -79, -80, -80,
- -81, -81, -82, -83, -83, -84, -84, -85,
- -86, -86, -87, -87, -88, -89, -89, -90,
- -91, -91, -92, -92, -93, -94, -94, -95,
- -96, -96, -97, -97, -98, -99, -99, -100,
- -101, -101, -102, -102, -103, -104, -104, -105,
- -106, -106, -107, -108, -108, -109, -109, -110,
- -111, -111, -112, -113, -113, -114, -115, -115,
- -116, -117, -117, -118, -119, -119, -120, -121,
- -121, -122, -122, -123, -124, -124, -125, -126,
- -126, -127, -128, -128, -129, -130, -130, -131,
- -132, -132, -133, -134, -134, -135, -136, -136,
- -137, -138, -138, -139, -140, -140, -141, -142,
- -142, -143, -144, -144, -145, -146, -146, -147,
- -148, -148, -149, -150, -150, -151, -152, -152,
- -153, -154, -154, -155, -156, -156, -157, -158,
- -158, -159, -160, -160, -161, -162, -162, -163,
- -164, -164, -165, -166, -166, -167, -168, -168,
- -169, -170, -170, -171, -172, -172, -173, -174,
- -174, -175, -176, -176, -177, -178, -178, -179,
- -180, -180, -181, -181, -182, -183, -183, -184,
- -185, -185, -186, -187, -187, -188, -189, -189,
- -190, -191, -191, -192, -193, -193, -194, -194,
- -195, -196, -196, -197, -198, -198, -199, -200,
- -200, -201, -201, -202, -203, -203, -204, -205,
- -205, -206, -206, -207, -208, -208, -209, -210,
- -210, -211, -211, -212, -213, -213, -214, -215,
- -215, -216, -216, -217, -218, -218, -219, -219,
- -220, -221, -221, -222, -222, -223, -224, -224,
- -225, -225, -226, -227, -227, -228, -228, -229,
- -229, -230, -231, -231, -232, -232, -233, -233,
- -234, -235, -235, -236, -236, -237, -237, -238,
- -239, -239, -240, -240, -241, -241, -242, -242,
- -243, -244, -244, -245, -245, -246, -246, -247,
- -247, -248, -248, -249, -249, -250, -250, -251,
- -251, -252, -252, -253, -253, -254, -254, -255,
- -256, -256, -256, -257, -257, -258, -258, -259,
- -259, -260, -260, -261, -261, -262, -262, -263,
- -263, -264, -264, -265, -265, -266, -266, -266,
- -267, -267, -268, -268, -269, -269, -270, -270,
- -270, -271, -271, -272, -272, -273, -273, -273,
- -274, -274, -275, -275, -275, -276, -276, -277,
- -277, -277, -278, -278, -279, -279, -279, -280,
- -280, -280, -281, -281, -282, -282, -282, -283,
- -283, -283, -284, -284, -284, -285, -285, -285,
- -286, -286, -286, -287, -287, -287, -288, -288,
- -288, -289, -289, -289, -289, -290, -290, -290,
- -291, -291, -291, -291, -292, -292, -292, -293,
- -293, -293, -293, -294, -294, -294, -294, -295,
- -295, -295, -295, -295, -296, -296, -296, -296,
- -297, -297, -297, -297, -297, -298, -298, -298,
- -298, -298, -299, -299, -299, -299, -299, -299,
- -300, -300, -300, -300, -300, -300, -300, -301,
- -301, -301, -301, -301, -301, -301, -301, -301,
- -302, -302, -302, -302, -302, -302, -302, -302,
- -302, -302, -302, -302, -302, -303, -303, -303,
- -303, -303, -303, -303, -303, -303, -303, -303,
- -303, -303, -303, -303, -303, -303, -303, -303,
- -303, -303, -303, -303, -303, -303, -303, -303,
- -303, -303, -302, -302, -302, -302, -302, -302,
- -302, -302, -302, -302, -302, -302, -301, -301,
- -301, -301, -301, -301, -301, -301, -300, -300,
- -300, -300, -300, -300, -299, -299, -299, -299,
- -299, -299, -298, -298, -298, -298, -298, -297,
- -297, -297, -297, -296, -296, -296, -296, -295,
- -295, -295, -295, -294, -294, -294, -293, -293,
- -293, -293, -292, -292, -292, -291, -291, -291,
- -290, -290, -290, -289, -289, -289, -288, -288,
- -288, -287, -287, -286, -286, -286, -285, -285,
- -284, -284, -284, -283, -283, -282, -282, -281,
- -281, -280, -280, -279, -279, -279, -278, -278,
- -277, -277, -276, -276, -275, -275, -274, -273,
- -273, -272, -272, -271, -271, -270, -270, -269,
- -268, -268, -267, -267, -266, -266, -265, -264,
- -264, -263, -262, -262, -261, -260, -260, -259,
- -259, -258, -257, -256, -256, -255, -254, -254,
- -253, -252, -252, -251, -250, -249, -249, -248,
- -247, -246, -246, -245, -244, -243, -242, -242,
- -241, -240, -239, -238, -238, -237, -236, -235,
- -234, -233, -233, -232, -231, -230, -229, -228,
- -227, -226, -226, -225, -224, -223, -222, -221,
- -220, -219, -218, -217, -216, -215, -214, -213,
- -212, -211, -210, -209, -208, -207, -206, -205,
- -204, -203, -202, -201, -200, -199, -198, -197,
- -196, -194, -193, -192, -191, -190, -189, -188,
- -187, -185, -184, -183, -182, -181, -180, -178,
- -177, -176, -175, -174, -172, -171, -170, -169,
- -167, -166, -165, -164, -162, -161, -160, -158,
- -157, -156, -155, -153, -152, -151, -149, -148,
- -147, -145, -144, -142, -141, -140, -138, -137,
- -135, -134, -133, -131, -130, -128, -127, -125,
- -124, -122, -121, -120, -118, -117, -115, -114,
- -112, -110, -109, -107, -106, -104, -103, -101,
- -100, -98, -96, -95, -93, -92, -90, -88,
- -87, -85, -83, -82, -80, -78, -77, -75,
- -73, -72, -70, -68, -67, -65, -63, -61,
- -60, -58, -56, -54, -52, -51, -49, -47,
- -45, -43, -42, -40, -38, -36, -34, -32,
- -31, -29, -27, -25, -23, -21, -19, -17,
- -15, -13, -11, -9, -7, -5, -3, -1
- },
- {
- 0, 2, 4, 6, 8, 10, 12, 14,
- 16, 18, 20, 22, 25, 27, 29, 31,
- 33, 36, 38, 40, 43, 45, 47, 50,
- 52, 54, 57, 59, 61, 64, 66, 69,
- 71, 74, 76, 79, 81, 84, 86, 89,
- 92, 94, 97, 99, 102, 105, 107, 110,
- 113, 116, 118, 121, 124, 127, 129, 132,
- 135, 138, 141, 144, 146, 149, 152, 155,
- 158, 161, 164, 167, 170, 173, 176, 179,
- 182, 185, 188, 191, 194, 197, 200, 203,
- 207, 210, 213, 216, 219, 222, 226, 229,
- 232, 235, 239, 242, 245, 248, 252, 255,
- 258, 262, 265, 269, 272, 275, 279, 282,
- 286, 289, 292, 296, 299, 303, 306, 310,
- 313, 317, 321, 324, 328, 331, 335, 338,
- 342, 346, 349, 353, 357, 360, 364, 368,
- 372, 375, 379, 383, 386, 390, 394, 398,
- 402, 405, 409, 413, 417, 421, 425, 429,
- 432, 436, 440, 444, 448, 452, 456, 460,
- 464, 468, 472, 476, 480, 484, 488, 492,
- 496, 500, 504, 508, 512, 516, 521, 525,
- 529, 533, 537, 541, 546, 550, 554, 558,
- 562, 567, 571, 575, 579, 584, 588, 592,
- 596, 601, 605, 609, 614, 618, 622, 627,
- 631, 635, 640, 644, 649, 653, 657, 662,
- 666, 671, 675, 680, 684, 689, 693, 698,
- 702, 707, 711, 716, 720, 725, 729, 734,
- 738, 743, 747, 752, 757, 761, 766, 771,
- 775, 780, 784, 789, 794, 798, 803, 808,
- 813, 817, 822, 827, 831, 836, 841, 846,
- 850, 855, 860, 865, 870, 874, 879, 884,
- 889, 894, 898, 903, 908, 913, 918, 923,
- 928, 932, 937, 942, 947, 952, 957, 962,
- 967, 972, 977, 982, 986, 991, 996, 1001,
- 1006, 1011, 1016, 1021, 1026, 1031, 1036, 1041,
- 1046, 1051, 1056, 1062, 1067, 1072, 1077, 1082,
- 1087, 1092, 1097, 1102, 1107, 1112, 1117, 1122,
- 1128, 1133, 1138, 1143, 1148, 1153, 1158, 1164,
- 1169, 1174, 1179, 1184, 1189, 1195, 1200, 1205,
- 1210, 1215, 1221, 1226, 1231, 1236, 1242, 1247,
- 1252, 1257, 1262, 1268, 1273, 1278, 1284, 1289,
- 1294, 1299, 1305, 1310, 1315, 1321, 1326, 1331,
- 1336, 1342, 1347, 1352, 1358, 1363, 1368, 1374,
- 1379, 1384, 1390, 1395, 1400, 1406, 1411, 1417,
- 1422, 1427, 1433, 1438, 1443, 1449, 1454, 1460,
- 1465, 1470, 1476, 1481, 1487, 1492, 1497, 1503,
- 1508, 1514, 1519, 1525, 1530, 1535, 1541, 1546,
- 1552, 1557, 1563, 1568, 1574, 1579, 1585, 1590,
- 1596, 1601, 1606, 1612, 1617, 1623, 1628, 1634,
- 1639, 1645, 1650, 1656, 1661, 1667, 1672, 1678,
- 1683, 1689, 1694, 1700, 1705, 1711, 1716, 1722,
- 1727, 1733, 1738, 1744, 1749, 1755, 1761, 1766,
- 1772, 1777, 1783, 1788, 1794, 1799, 1805, 1810,
- 1816, 1821, 1827, 1832, 1838, 1844, 1849, 1855,
- 1860, 1866, 1871, 1877, 1882, 1888, 1893, 1899,
- 1905, 1910, 1916, 1921, 1927, 1932, 1938, 1943,
- 1949, 1955, 1960, 1966, 1971, 1977, 1982, 1988,
- 1993, 1999, 2005, 2010, 2016, 2021, 2027, 2032,
- 2038, 2043, 2049, 2055, 2060, 2066, 2071, 2077,
- 2082, 2088, 2093, 2099, 2105, 2110, 2116, 2121,
- 2127, 2132, 2138, 2143, 2149, 2154, 2160, 2165,
- 2171, 2177, 2182, 2188, 2193, 2199, 2204, 2210,
- 2215, 2221, 2226, 2232, 2237, 2243, 2248, 2254,
- 2259, 2265, 2270, 2276, 2281, 2287, 2292, 2298,
- 2304, 2309, 2314, 2320, 2325, 2331, 2336, 2342,
- 2347, 2353, 2358, 2364, 2369, 2375, 2380, 2386,
- 2391, 2397, 2402, 2408, 2413, 2419, 2424, 2429,
- 2435, 2440, 2446, 2451, 2457, 2462, 2467, 2473,
- 2478, 2484, 2489, 2495, 2500, 2505, 2511, 2516,
- 2522, 2527, 2532, 2538, 2543, 2549, 2554, 2559,
- 2565, 2570, 2575, 2581, 2586, 2591, 2597, 2602,
- 2607, 2613, 2618, 2623, 2629, 2634, 2639, 2645,
- 2650, 2655, 2661, 2666, 2671, 2676, 2682, 2687,
- 2692, 2698, 2703, 2708, 2713, 2719, 2724, 2729,
- 2734, 2740, 2745, 2750, 2755, 2760, 2766, 2771,
- 2776, 2781, 2786, 2792, 2797, 2802, 2807, 2812,
- 2817, 2823, 2828, 2833, 2838, 2843, 2848, 2853,
- 2859, 2864, 2869, 2874, 2879, 2884, 2889, 2894,
- 2899, 2904, 2909, 2914, 2919, 2924, 2930, 2935,
- 2940, 2945, 2950, 2955, 2960, 2965, 2970, 2975,
- 2980, 2984, 2989, 2994, 2999, 3004, 3009, 3014,
- 3019, 3024, 3029, 3034, 3039, 3044, 3048, 3053,
- 3058, 3063, 3068, 3073, 3078, 3082, 3087, 3092,
- 3097, 3102, 3106, 3111, 3116, 3121, 3126, 3130,
- 3135, 3140, 3145, 3149, 3154, 3159, 3163, 3168,
- 3173, 3177, 3182, 3187, 3191, 3196, 3201, 3205,
- 3210, 3215, 3219, 3224, 3228, 3233, 3238, 3242,
- 3247, 3251, 3256, 3260, 3265, 3269, 3274, 3279,
- 3283, 3287, 3292, 3296, 3301, 3305, 3310, 3314,
- 3319, 3323, 3327, 3332, 3336, 3341, 3345, 3349,
- 3354, 3358, 3362, 3367, 3371, 3375, 3380, 3384,
- 3388, 3393, 3397, 3401, 3405, 3410, 3414, 3418,
- 3422, 3426, 3431, 3435, 3439, 3443, 3447, 3451,
- 3455, 3460, 3464, 3468, 3472, 3476, 3480, 3484,
- 3488, 3492, 3496, 3500, 3504, 3508, 3512, 3516,
- 3520, 3524, 3528, 3532, 3536, 3540, 3544, 3548,
- 3552, 3555, 3559, 3563, 3567, 3571, 3575, 3578,
- 3582, 3586, 3590, 3593, 3597, 3601, 3605, 3608,
- 3612, 3616, 3619, 3623, 3627, 3630, 3634, 3638,
- 3641, 3645, 3649, 3652, 3656, 3659, 3663, 3666,
- 3670, 3673, 3677, 3680, 3684, 3687, 3691, 3694,
- 3698, 3701, 3704, 3708, 3711, 3714, 3718, 3721,
- 3724, 3728, 3731, 3734, 3738, 3741, 3744, 3747,
- 3751, 3754, 3757, 3760, 3763, 3767, 3770, 3773,
- 3776, 3779, 3782, 3785, 3788, 3791, 3794, 3798,
- 3801, 3804, 3807, 3809, 3812, 3815, 3818, 3821,
- 3824, 3827, 3830, 3833, 3836, 3839, 3841, 3844,
- 3847, 3850, 3853, 3855, 3858, 3861, 3864, 3866,
- 3869, 3872, 3874, 3877, 3880, 3882, 3885, 3887,
- 3890, 3893, 3895, 3898, 3900, 3903, 3905, 3908,
- 3910, 3913, 3915, 3917, 3920, 3922, 3925, 3927,
- 3929, 3932, 3934, 3936, 3939, 3941, 3943, 3945,
- 3948, 3950, 3952, 3954, 3956, 3958, 3961, 3963,
- 3965, 3967, 3969, 3971, 3973, 3975, 3977, 3979,
- 3981, 3983, 3985, 3987, 3989, 3991, 3993, 3994,
- 3996, 3998, 4000, 4002, 4004, 4005, 4007, 4009,
- 4011, 4012, 4014, 4016, 4017, 4019, 4021, 4022,
- 4024, 4025, 4027, 4028, 4030, 4031, 4033, 4034,
- 4036, 4037, 4039, 4040, 4042, 4043, 4044, 4046,
- 4047, 4048, 4050, 4051, 4052, 4053, 4055, 4056,
- 4057, 4058, 4059, 4060, 4062, 4063, 4064, 4065,
- 4066, 4067, 4068, 4069, 4070, 4071, 4072, 4073,
- 4074, 4075, 4075, 4076, 4077, 4078, 4079, 4079,
- 4080, 4081, 4082, 4082, 4083, 4084, 4084, 4085,
- 4086, 4086, 4087, 4087, 4088, 4088, 4089, 4089,
- 4090, 4090, 4091, 4091, 4092, 4092, 4092, 4093,
- 4093, 4093, 4094, 4094, 4094, 4094, 4095, 4095,
- 4095, 4095, 4095, 4095, 4095, 4095, 4095, 4095
- },
- {
- 4096, 4095, 4095, 4095, 4095, 4095, 4095, 4095,
- 4095, 4095, 4095, 4094, 4094, 4094, 4094, 4093,
- 4093, 4093, 4092, 4092, 4092, 4091, 4091, 4090,
- 4090, 4089, 4089, 4088, 4088, 4087, 4087, 4086,
- 4086, 4085, 4084, 4084, 4083, 4082, 4082, 4081,
- 4080, 4079, 4079, 4078, 4077, 4076, 4075, 4075,
- 4074, 4073, 4072, 4071, 4070, 4069, 4068, 4067,
- 4066, 4065, 4064, 4063, 4062, 4060, 4059, 4058,
- 4057, 4056, 4055, 4053, 4052, 4051, 4050, 4048,
- 4047, 4046, 4044, 4043, 4042, 4040, 4039, 4037,
- 4036, 4034, 4033, 4031, 4030, 4028, 4027, 4025,
- 4024, 4022, 4021, 4019, 4017, 4016, 4014, 4012,
- 4011, 4009, 4007, 4005, 4004, 4002, 4000, 3998,
- 3996, 3994, 3993, 3991, 3989, 3987, 3985, 3983,
- 3981, 3979, 3977, 3975, 3973, 3971, 3969, 3967,
- 3965, 3963, 3961, 3958, 3956, 3954, 3952, 3950,
- 3948, 3945, 3943, 3941, 3939, 3936, 3934, 3932,
- 3929, 3927, 3925, 3922, 3920, 3917, 3915, 3913,
- 3910, 3908, 3905, 3903, 3900, 3898, 3895, 3893,
- 3890, 3887, 3885, 3882, 3880, 3877, 3874, 3872,
- 3869, 3866, 3864, 3861, 3858, 3855, 3853, 3850,
- 3847, 3844, 3841, 3839, 3836, 3833, 3830, 3827,
- 3824, 3821, 3818, 3815, 3812, 3809, 3807, 3804,
- 3801, 3798, 3794, 3791, 3788, 3785, 3782, 3779,
- 3776, 3773, 3770, 3767, 3763, 3760, 3757, 3754,
- 3751, 3747, 3744, 3741, 3738, 3734, 3731, 3728,
- 3724, 3721, 3718, 3714, 3711, 3708, 3704, 3701,
- 3698, 3694, 3691, 3687, 3684, 3680, 3677, 3673,
- 3670, 3666, 3663, 3659, 3656, 3652, 3649, 3645,
- 3641, 3638, 3634, 3630, 3627, 3623, 3619, 3616,
- 3612, 3608, 3605, 3601, 3597, 3593, 3590, 3586,
- 3582, 3578, 3575, 3571, 3567, 3563, 3559, 3555,
- 3552, 3548, 3544, 3540, 3536, 3532, 3528, 3524,
- 3520, 3516, 3512, 3508, 3504, 3500, 3496, 3492,
- 3488, 3484, 3480, 3476, 3472, 3468, 3464, 3460,
- 3455, 3451, 3447, 3443, 3439, 3435, 3431, 3426,
- 3422, 3418, 3414, 3410, 3405, 3401, 3397, 3393,
- 3388, 3384, 3380, 3375, 3371, 3367, 3362, 3358,
- 3354, 3349, 3345, 3341, 3336, 3332, 3327, 3323,
- 3319, 3314, 3310, 3305, 3301, 3296, 3292, 3287,
- 3283, 3279, 3274, 3269, 3265, 3260, 3256, 3251,
- 3247, 3242, 3238, 3233, 3228, 3224, 3219, 3215,
- 3210, 3205, 3201, 3196, 3191, 3187, 3182, 3177,
- 3173, 3168, 3163, 3159, 3154, 3149, 3145, 3140,
- 3135, 3130, 3126, 3121, 3116, 3111, 3106, 3102,
- 3097, 3092, 3087, 3082, 3078, 3073, 3068, 3063,
- 3058, 3053, 3048, 3044, 3039, 3034, 3029, 3024,
- 3019, 3014, 3009, 3004, 2999, 2994, 2989, 2984,
- 2980, 2975, 2970, 2965, 2960, 2955, 2950, 2945,
- 2940, 2935, 2930, 2924, 2919, 2914, 2909, 2904,
- 2899, 2894, 2889, 2884, 2879, 2874, 2869, 2864,
- 2859, 2853, 2848, 2843, 2838, 2833, 2828, 2823,
- 2817, 2812, 2807, 2802, 2797, 2792, 2786, 2781,
- 2776, 2771, 2766, 2760, 2755, 2750, 2745, 2740,
- 2734, 2729, 2724, 2719, 2713, 2708, 2703, 2698,
- 2692, 2687, 2682, 2676, 2671, 2666, 2661, 2655,
- 2650, 2645, 2639, 2634, 2629, 2623, 2618, 2613,
- 2607, 2602, 2597, 2591, 2586, 2581, 2575, 2570,
- 2565, 2559, 2554, 2549, 2543, 2538, 2532, 2527,
- 2522, 2516, 2511, 2505, 2500, 2495, 2489, 2484,
- 2478, 2473, 2467, 2462, 2457, 2451, 2446, 2440,
- 2435, 2429, 2424, 2419, 2413, 2408, 2402, 2397,
- 2391, 2386, 2380, 2375, 2369, 2364, 2358, 2353,
- 2347, 2342, 2336, 2331, 2325, 2320, 2314, 2309,
- 2304, 2298, 2292, 2287, 2281, 2276, 2270, 2265,
- 2259, 2254, 2248, 2243, 2237, 2232, 2226, 2221,
- 2215, 2210, 2204, 2199, 2193, 2188, 2182, 2177,
- 2171, 2165, 2160, 2154, 2149, 2143, 2138, 2132,
- 2127, 2121, 2116, 2110, 2105, 2099, 2093, 2088,
- 2082, 2077, 2071, 2066, 2060, 2055, 2049, 2043,
- 2038, 2032, 2027, 2021, 2016, 2010, 2005, 1999,
- 1993, 1988, 1982, 1977, 1971, 1966, 1960, 1955,
- 1949, 1943, 1938, 1932, 1927, 1921, 1916, 1910,
- 1905, 1899, 1893, 1888, 1882, 1877, 1871, 1866,
- 1860, 1855, 1849, 1844, 1838, 1832, 1827, 1821,
- 1816, 1810, 1805, 1799, 1794, 1788, 1783, 1777,
- 1772, 1766, 1761, 1755, 1749, 1744, 1738, 1733,
- 1727, 1722, 1716, 1711, 1705, 1700, 1694, 1689,
- 1683, 1678, 1672, 1667, 1661, 1656, 1650, 1645,
- 1639, 1634, 1628, 1623, 1617, 1612, 1606, 1601,
- 1596, 1590, 1585, 1579, 1574, 1568, 1563, 1557,
- 1552, 1546, 1541, 1535, 1530, 1525, 1519, 1514,
- 1508, 1503, 1497, 1492, 1487, 1481, 1476, 1470,
- 1465, 1460, 1454, 1449, 1443, 1438, 1433, 1427,
- 1422, 1417, 1411, 1406, 1400, 1395, 1390, 1384,
- 1379, 1374, 1368, 1363, 1358, 1352, 1347, 1342,
- 1336, 1331, 1326, 1321, 1315, 1310, 1305, 1299,
- 1294, 1289, 1284, 1278, 1273, 1268, 1262, 1257,
- 1252, 1247, 1242, 1236, 1231, 1226, 1221, 1215,
- 1210, 1205, 1200, 1195, 1189, 1184, 1179, 1174,
- 1169, 1164, 1158, 1153, 1148, 1143, 1138, 1133,
- 1128, 1122, 1117, 1112, 1107, 1102, 1097, 1092,
- 1087, 1082, 1077, 1072, 1067, 1062, 1056, 1051,
- 1046, 1041, 1036, 1031, 1026, 1021, 1016, 1011,
- 1006, 1001, 996, 991, 986, 982, 977, 972,
- 967, 962, 957, 952, 947, 942, 937, 932,
- 928, 923, 918, 913, 908, 903, 898, 894,
- 889, 884, 879, 874, 870, 865, 860, 855,
- 850, 846, 841, 836, 831, 827, 822, 817,
- 813, 808, 803, 798, 794, 789, 784, 780,
- 775, 771, 766, 761, 757, 752, 747, 743,
- 738, 734, 729, 725, 720, 716, 711, 707,
- 702, 698, 693, 689, 684, 680, 675, 671,
- 666, 662, 657, 653, 649, 644, 640, 635,
- 631, 627, 622, 618, 614, 609, 605, 601,
- 596, 592, 588, 584, 579, 575, 571, 567,
- 562, 558, 554, 550, 546, 541, 537, 533,
- 529, 525, 521, 516, 512, 508, 504, 500,
- 496, 492, 488, 484, 480, 476, 472, 468,
- 464, 460, 456, 452, 448, 444, 440, 436,
- 432, 429, 425, 421, 417, 413, 409, 405,
- 402, 398, 394, 390, 386, 383, 379, 375,
- 372, 368, 364, 360, 357, 353, 349, 346,
- 342, 338, 335, 331, 328, 324, 321, 317,
- 313, 310, 306, 303, 299, 296, 292, 289,
- 286, 282, 279, 275, 272, 269, 265, 262,
- 258, 255, 252, 248, 245, 242, 239, 235,
- 232, 229, 226, 222, 219, 216, 213, 210,
- 207, 203, 200, 197, 194, 191, 188, 185,
- 182, 179, 176, 173, 170, 167, 164, 161,
- 158, 155, 152, 149, 146, 144, 141, 138,
- 135, 132, 129, 127, 124, 121, 118, 116,
- 113, 110, 107, 105, 102, 99, 97, 94,
- 92, 89, 86, 84, 81, 79, 76, 74,
- 71, 69, 66, 64, 61, 59, 57, 54,
- 52, 50, 47, 45, 43, 40, 38, 36,
- 33, 31, 29, 27, 25, 22, 20, 18,
- 16, 14, 12, 10, 8, 6, 4, 2
- },
- {
- 0, -1, -3, -5, -7, -9, -11, -13,
- -15, -17, -19, -20, -23, -25, -27, -28,
- -30, -33, -34, -36, -39, -40, -42, -43,
- -45, -46, -49, -50, -52, -54, -56, -58,
- -60, -61, -62, -65, -66, -68, -70, -72,
- -73, -74, -77, -78, -80, -82, -83, -85,
- -87, -89, -90, -92, -93, -95, -96, -98,
- -100, -102, -103, -105, -106, -107, -108, -110,
- -112, -114, -116, -116, -118, -120, -122, -122,
- -124, -126, -127, -128, -130, -131, -133, -133,
- -136, -137, -138, -139, -141, -142, -144, -145,
- -147, -147, -150, -151, -151, -153, -155, -156,
- -157, -159, -160, -161, -163, -164, -165, -166,
- -168, -168, -170, -171, -172, -174, -174, -176,
- -177, -178, -180, -181, -182, -183, -184, -185,
- -187, -188, -189, -190, -191, -192, -193, -195,
- -196, -196, -198, -199, -200, -200, -202, -204,
- -204, -205, -206, -207, -208, -209, -211, -212,
- -212, -213, -214, -215, -216, -217, -218, -220,
- -220, -221, -222, -223, -224, -225, -225, -227,
- -227, -228, -229, -230, -230, -231, -233, -234,
- -234, -235, -235, -237, -238, -239, -239, -240,
- -240, -242, -242, -243, -243, -245, -246, -247,
- -247, -249, -248, -249, -250, -251, -251, -253,
- -253, -253, -255, -255, -256, -256, -257, -258,
- -259, -259, -260, -261, -261, -262, -262, -264,
- -263, -265, -265, -265, -266, -267, -267, -268,
- -269, -269, -269, -270, -271, -271, -272, -273,
- -273, -273, -274, -274, -276, -275, -276, -277,
- -277, -278, -278, -278, -279, -279, -280, -281,
- -280, -281, -282, -283, -283, -282, -284, -284,
- -284, -285, -285, -286, -286, -286, -287, -287,
- -288, -288, -288, -289, -289, -289, -290, -290,
- -290, -291, -291, -292, -291, -291, -292, -292,
- -292, -293, -293, -293, -294, -294, -295, -295,
- -294, -295, -295, -296, -297, -297, -297, -297,
- -297, -297, -298, -298, -297, -298, -298, -298,
- -299, -299, -300, -299, -299, -300, -299, -300,
- -301, -300, -300, -301, -300, -301, -301, -301,
- -301, -301, -302, -301, -302, -301, -302, -302,
- -302, -302, -302, -302, -302, -302, -303, -302,
- -303, -302, -303, -303, -302, -303, -303, -303,
- -302, -303, -303, -302, -303, -303, -302, -303,
- -303, -302, -303, -303, -302, -303, -303, -303,
- -303, -302, -303, -303, -302, -302, -302, -303,
- -302, -302, -302, -301, -303, -302, -301, -302,
- -301, -301, -301, -302, -301, -301, -301, -300,
- -301, -300, -300, -300, -300, -299, -300, -299,
- -300, -300, -299, -300, -299, -299, -299, -299,
- -298, -299, -298, -297, -297, -297, -296, -297,
- -296, -296, -296, -296, -295, -296, -295, -296,
- -295, -294, -294, -294, -293, -294, -294, -293,
- -293, -292, -293, -292, -292, -292, -291, -290,
- -291, -290, -291, -289, -289, -290, -289, -289,
- -288, -288, -288, -288, -286, -287, -286, -286,
- -286, -285, -286, -284, -284, -284, -284, -283,
- -283, -283, -282, -282, -282, -281, -280, -281,
- -279, -280, -280, -278, -279, -278, -278, -277,
- -278, -276, -276, -277, -275, -276, -274, -275,
- -274, -273, -273, -272, -273, -272, -272, -271,
- -270, -270, -269, -269, -269, -268, -268, -267,
- -267, -266, -266, -266, -265, -265, -264, -264,
- -263, -263, -262, -262, -261, -261, -260, -260,
- -259, -259, -258, -258, -257, -257, -256, -256,
- -256, -255, -254, -254, -253, -253, -252, -252,
- -251, -251, -250, -250, -249, -249, -248, -248,
- -247, -247, -246, -246, -245, -245, -244, -244,
- -243, -242, -242, -241, -241, -240, -239, -239,
- -239, -238, -238, -237, -237, -235, -235, -235,
- -234, -234, -232, -233, -232, -232, -231, -229,
- -230, -229, -228, -228, -227, -226, -227, -225,
- -224, -225, -223, -223, -222, -222, -221, -221,
- -220, -219, -219, -218, -218, -216, -217, -216,
- -215, -215, -214, -213, -212, -213, -211, -211,
- -210, -210, -209, -209, -208, -206, -207, -206,
- -205, -204, -204, -204, -203, -202, -202, -200,
- -200, -200, -200, -198, -197, -197, -196, -195,
- -195, -195, -194, -194, -192, -192, -191, -191,
- -189, -189, -188, -188, -187, -186, -186, -186,
- -185, -185, -183, -183, -182, -182, -181, -181,
- -180, -178, -178, -177, -177, -176, -176, -174,
- -174, -173, -173, -172, -172, -172, -170, -170,
- -168, -168, -167, -167, -167, -165, -165, -164,
- -164, -164, -162, -162, -161, -160, -160, -158,
- -158, -158, -157, -156, -155, -155, -154, -153,
- -153, -152, -151, -151, -150, -149, -149, -148,
- -147, -147, -146, -146, -144, -144, -144, -142,
- -142, -141, -142, -140, -140, -139, -138, -138,
- -137, -136, -136, -134, -134, -133, -134, -132,
- -132, -131, -130, -130, -128, -128, -128, -127,
- -127, -126, -124, -124, -124, -123, -123, -122,
- -121, -120, -120, -119, -118, -118, -117, -117,
- -116, -115, -115, -115, -114, -113, -111, -111,
- -110, -110, -109, -109, -108, -107, -107, -106,
- -105, -104, -104, -103, -102, -103, -102, -101,
- -101, -100, -99, -99, -98, -97, -97, -96,
- -96, -95, -94, -94, -93, -92, -92, -91,
- -91, -90, -89, -88, -88, -88, -87, -86,
- -85, -86, -84, -84, -83, -82, -82, -81,
- -81, -80, -80, -78, -79, -77, -77, -77,
- -76, -76, -75, -74, -74, -73, -72, -72,
- -72, -71, -70, -70, -69, -68, -68, -68,
- -66, -67, -66, -65, -65, -65, -63, -63,
- -62, -62, -61, -61, -60, -60, -60, -58,
- -58, -58, -56, -56, -56, -55, -54, -55,
- -54, -54, -53, -52, -51, -51, -51, -50,
- -49, -49, -49, -49, -48, -47, -46, -46,
- -46, -46, -45, -43, -43, -43, -43, -42,
- -42, -42, -40, -40, -40, -39, -39, -38,
- -38, -38, -37, -37, -36, -36, -35, -35,
- -34, -35, -34, -33, -33, -32, -32, -31,
- -31, -31, -30, -29, -29, -29, -28, -27,
- -28, -28, -27, -26, -26, -25, -25, -25,
- -24, -24, -24, -23, -23, -22, -22, -22,
- -21, -21, -20, -20, -20, -20, -19, -18,
- -19, -18, -18, -17, -18, -17, -16, -17,
- -16, -15, -15, -15, -14, -14, -15, -13,
- -13, -13, -13, -12, -12, -11, -12, -11,
- -12, -10, -10, -10, -10, -10, -9, -10,
- -9, -9, -9, -8, -8, -7, -8, -7,
- -7, -7, -6, -6, -6, -7, -6, -6,
- -5, -5, -5, -5, -5, -4, -4, -5,
- -4, -4, -3, -3, -3, -3, -3, -2,
- -3, -2, -2, -2, -1, -2, -1, -2,
- -1, -1, -1, -1, -1, 0, -1, 0,
- -1, -1, 0, 0, -1, 0, 0, -1,
- 1, 1, 0, 0, 0, 1, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0
- }
-};
-#else /* defined(CONFIG_CSI2_PLUS) */
static const int zoom_table[4][HRT_GDC_N] = {
{
0 << 4, 0 << 4, 0 << 4, 0 << 4, 0 << 4, 0 << 4, 0 << 4, 0 << 4,
0 << 4, 0 << 4, 0 << 4, 0 << 4, 0 << 4, 0 << 4, 0 << 4, 0 << 4
}
};
-#endif
-#else
-#error "sh_css_params.c: GDC version must be \
-one of {GDC_VERSION_2}"
-#endif
static const struct ia_css_dz_config default_dz_config = {
HRT_GDC_N,
{
const struct ia_css_binary *binary = stage->binary;
struct ia_css_host_data *sctbl;
- unsigned int i, j, aligned_width, row_padding;
+ unsigned int i, j, aligned_width;
unsigned int sctbl_size;
short int *ptr;
}
aligned_width = binary->sctbl_aligned_width_per_color;
- row_padding = aligned_width - shading_table->width;
sctbl_size = shading_table->height * IA_CSS_SC_NUM_COLORS * aligned_width *
sizeof(short);
if (me) {
kvfree(me->rgby_data);
kvfree(me->data);
- memset(me, 0, sizeof(struct ia_css_3a_statistics));
kvfree(me);
}
}
if (me) {
kvfree(me->hor_proj);
kvfree(me->ver_proj);
- memset(me, 0, sizeof(struct ia_css_dvs_statistics));
kvfree(me);
}
}
if (me) {
kvfree(me->hor_coefs);
kvfree(me->ver_coefs);
- memset(me, 0, sizeof(struct ia_css_dvs_coefficients));
kvfree(me);
}
}
kvfree(me->ver_prod.odd_imag);
kvfree(me->ver_prod.even_real);
kvfree(me->ver_prod.even_imag);
- memset(me, 0, sizeof(struct ia_css_dvs2_statistics));
kvfree(me);
}
}
kvfree(me->ver_coefs.odd_imag);
kvfree(me->ver_coefs.even_real);
kvfree(me->ver_coefs.even_imag);
- memset(me, 0, sizeof(struct ia_css_dvs2_coefficients));
kvfree(me);
}
}
kvfree(dvs_6axis_config->ycoords_y);
kvfree(dvs_6axis_config->xcoords_uv);
kvfree(dvs_6axis_config->ycoords_uv);
- memset(dvs_6axis_config, 0, sizeof(struct ia_css_dvs_6axis_config));
kvfree(dvs_6axis_config);
}
}
ia_css_get_properties(struct ia_css_properties *properties)
{
assert(properties);
-#if defined(HAS_GDC_VERSION_2) || defined(HAS_GDC_VERSION_3)
/*
* MW: We don't want to store the coordinates
* full range in memory: Truncate
*/
properties->gdc_coord_one = gdc_get_unity(GDC0_ID) / HRT_GDC_COORD_SCALE;
-#else
-#error "Unknown GDC version"
-#endif
properties->l1_base_is_index = true;
-#if defined(HAS_VAMEM_VERSION_1)
- properties->vamem_type = IA_CSS_VAMEM_TYPE_1;
-#elif defined(HAS_VAMEM_VERSION_2)
properties->vamem_type = IA_CSS_VAMEM_TYPE_2;
-#else
-#error "Unknown VAMEM version"
-#endif
}
#include "sh_css_sp.h"
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
#include "input_formatter.h"
#endif
#include "sh_css_params.h"
#include "sh_css_legacy.h"
#include "ia_css_frame_comm.h"
-#if !defined(HAS_NO_INPUT_SYSTEM)
#include "ia_css_isys.h"
-#endif
#include "gdc_device.h" /* HRT_GDC_N */
IA_CSS_LOG("pipe_id %d port_config %08x",
pipe->pipe_id, pipe->inout_port_config);
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
sh_css_sp_group.config.input_formatter.isp_2ppc = (uint8_t)two_ppc;
#else
(void)two_ppc;
IA_CSS_LOG("pipe_id %d port_config %08x",
pipe->pipe_id, pipe->inout_port_config);
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
sh_css_sp_group.config.input_formatter.isp_2ppc = (uint8_t)two_ppc;
#else
(void)two_ppc;
return 0;
}
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
void sh_css_sp_set_if_configs(
const input_formatter_cfg_t *config_a,
const input_formatter_cfg_t *config_b,
}
#endif
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
void
sh_css_sp_program_input_circuit(int fmt_type,
int ch_id,
}
#endif
-#if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2)
+#if !defined(ISP2401)
void
sh_css_sp_configure_sync_gen(int width, int height,
int hblank_cycles,
void
sh_css_sp_enable_isys_event_queue(bool enable)
{
-#if !defined(HAS_NO_INPUT_SYSTEM)
sh_css_sp_group.config.enable_isys_event_queue = enable;
-#else
- (void)enable;
-#endif
}
void
bool no_isp_sync,
uint8_t if_config_index)
{
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
sh_css_sp_group.config.input_formatter.isp_2ppc = two_ppc;
#else
(void)two_ppc;
sh_css_sp_group.config.no_isp_sync = (uint8_t)no_isp_sync;
/* decide whether the frame is processed online or offline */
if (if_config_index == SH_CSS_IF_CONFIG_NOT_NEEDED) return;
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
assert(if_config_index < SH_CSS_MAX_IF_CONFIGS);
sh_css_sp_group.config.input_formatter.set[if_config_index].stream_format =
input_format;
return 0;
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2401)
+#if defined(ISP2401)
(void)continuous;
sh_css_sp_stage.deinterleaved = 0;
#else
if (err)
return err;
-#ifdef USE_INPUT_SYSTEM_VERSION_2401
+#ifdef ISP2401
if (stage == 0) {
pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num);
if (!pipe)
enum ia_css_input_mode input_mode,
const struct ia_css_metadata_config *md_config,
const struct ia_css_metadata_info *md_info,
-#if !defined(HAS_NO_INPUT_SYSTEM)
const enum mipi_port_id port_id,
-#endif
const struct ia_css_coordinate
*internal_frame_origin_bqs_on_sctbl, /* Origin of internal frame
positioned on shading table at shading correction in ISP. */
assert(me);
-#if !defined(HAS_NO_INPUT_SYSTEM)
assert(me->stages);
first_binary = me->stages->binary;
{
if_config_index = 0x0;
}
-#else
- (void)input_mode;
- if_config_index = SH_CSS_IF_CONFIG_NOT_NEEDED;
-#endif
ia_css_pipeline_get_sp_thread_id(pipe_num, &thread_id);
memset(&sh_css_sp_group.pipe[thread_id], 0, sizeof(struct sh_css_sp_pipeline));
offline, if_config_index);
} /* if (first_binary != NULL) */
-#if defined(USE_INPUT_SYSTEM_VERSION_2401) || defined(USE_INPUT_SYSTEM_VERSION_2)
/* Signal the host immediately after start for SP_ISYS_COPY only */
if ((me->num_stages == 1) && me->stages &&
(me->stages->sp_func == IA_CSS_PIPELINE_ISYS_COPY))
sh_css_sp_group.config.no_isp_sync = true;
-#endif
/* Init stage data */
sh_css_init_host2sp_frame_data();
sh_css_sp_group.pipe[thread_id].num_execs = me->num_execs;
sh_css_sp_group.pipe[thread_id].pipe_qos_config = me->pipe_qos_config;
sh_css_sp_group.pipe[thread_id].required_bds_factor = required_bds_factor;
-#if !defined(HAS_NO_INPUT_SYSTEM)
sh_css_sp_group.pipe[thread_id].input_system_mode
= (uint32_t)input_mode;
sh_css_sp_group.pipe[thread_id].port_id = port_id;
-#endif
sh_css_sp_group.pipe[thread_id].dvs_frame_delay = (uint32_t)me->dvs_frame_delay;
/* TODO: next indicates from which queues parameters need to be
store_sp_array_uint(host_sp_com, offset, metadata ? metadata->address : 0);
}
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
/*
* @brief Update the mipi frame information in host_sp_communication.
* Refer to "sh_css_sp.h" for more details.
store_sp_array_uint(host_sp_com, offset, num_frames);
}
-#endif
void
sh_css_update_host2sp_cont_num_raw_frames(unsigned int num_frames,
#include <system_global.h>
#include <type_support.h>
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
#include "input_formatter.h"
#endif
enum ia_css_input_mode input_mode,
const struct ia_css_metadata_config *md_config,
const struct ia_css_metadata_info *md_info,
-#if !defined(HAS_NO_INPUT_SYSTEM)
const enum mipi_port_id port_id,
-#endif
const struct ia_css_coordinate
*internal_frame_origin_bqs_on_sctbl, /* Origin of internal frame
positioned on shading table at shading correction in ISP. */
struct ia_css_frame *frame,
struct ia_css_metadata *metadata);
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
/**
* @brief Update the mipi frame information in host_sp_communication.
*
*/
void
sh_css_update_host2sp_num_mipi_frames(unsigned int num_frames);
-#endif
/**
* @brief Update the nr of offline frames to use in host_sp_communication.
#endif
-#if !defined(HAS_NO_INPUT_FORMATTER)
+#if !defined(ISP2401)
void
sh_css_sp_set_if_configs(
const input_formatter_cfg_t *config_a,
bool check_system_idle;
unsigned int num_cont_raw_frames;
-#if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)
unsigned int num_mipi_frames[N_CSI_PORTS];
struct ia_css_frame
*mipi_frames[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM];
unsigned int
mipi_sizes_for_check[N_CSI_PORTS][IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT];
unsigned int mipi_frame_size[N_CSI_PORTS];
-#endif
ia_css_ptr sp_bin_addr;
hrt_data page_table_base_index;
* N.B. the 3 input formatters are of 2 different classess
*/
-#define HAS_MMU_VERSION_2
-#define HAS_DMA_VERSION_2
-#define HAS_GDC_VERSION_2
-#define HAS_VAMEM_VERSION_2
-#define HAS_HMEM_VERSION_1
-#define HAS_BAMEM_VERSION_2
-#define HAS_IRQ_VERSION_2
-#define HAS_IRQ_MAP_VERSION_2
-#define HAS_INPUT_FORMATTER_VERSION_2
-#define HAS_INPUT_SYSTEM_VERSION_2
-#define HAS_BUFFERED_SENSOR
-#define HAS_FIFO_MONITORS_VERSION_2
-#define HAS_GP_DEVICE_VERSION_2
-#define HAS_GPIO_VERSION_1
-#define HAS_TIMED_CTRL_VERSION_1
-#define HAS_RX_VERSION_2
-
/* per-frame parameter handling support */
#define SH_CSS_ENABLE_PER_FRAME_PARAMS
#define ISP2400_DMA_MAX_BURST_LENGTH 128
#define ISP2401_DMA_MAX_BURST_LENGTH 2
-#ifdef ISP2401
-# include "isp2401_system_global.h"
-#else
-# include "isp2400_system_global.h"
-#endif
-
#include <hive_isp_css_defs.h>
#include <type_support.h>
.cfg = {
.id = V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS,
},
- }, {
- .codec = HANTRO_H264_DECODER,
- .cfg = {
- .id = V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS,
- },
}, {
.codec = HANTRO_H264_DECODER,
.cfg = {
{
const struct hantro_h264_dec_ctrls *ctrls = &ctx->h264_dec.ctrls;
const struct v4l2_ctrl_h264_decode_params *dec_param = ctrls->decode;
- const struct v4l2_ctrl_h264_slice_params *slices = ctrls->slices;
const struct v4l2_ctrl_h264_sps *sps = ctrls->sps;
const struct v4l2_ctrl_h264_pps *pps = ctrls->pps;
struct vb2_v4l2_buffer *src_buf = hantro_get_src_buf(ctx);
if (!(sps->flags & V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY) &&
(sps->flags & V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD ||
- slices[0].flags & V4L2_H264_SLICE_FLAG_FIELD_PIC))
+ dec_param->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC))
reg |= G1_REG_DEC_CTRL0_PIC_INTERLACE_E;
- if (slices[0].flags & V4L2_H264_SLICE_FLAG_FIELD_PIC)
+ if (dec_param->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC)
reg |= G1_REG_DEC_CTRL0_PIC_FIELDMODE_E;
- if (!(slices[0].flags & V4L2_H264_SLICE_FLAG_BOTTOM_FIELD))
+ if (!(dec_param->flags & V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD))
reg |= G1_REG_DEC_CTRL0_PIC_TOPFIELD_E;
vdpu_write_relaxed(vpu, reg, G1_REG_DEC_CTRL0);
reg = G1_REG_DEC_CTRL2_CH_QP_OFFSET(pps->chroma_qp_index_offset) |
G1_REG_DEC_CTRL2_CH_QP_OFFSET2(pps->second_chroma_qp_index_offset);
- /* always use the matrix sent from userspace */
- reg |= G1_REG_DEC_CTRL2_TYPE1_QUANT_E;
-
+ if (pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT)
+ reg |= G1_REG_DEC_CTRL2_TYPE1_QUANT_E;
if (!(sps->flags & V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY))
reg |= G1_REG_DEC_CTRL2_FIELDPIC_FLAG_E;
vdpu_write_relaxed(vpu, reg, G1_REG_DEC_CTRL2);
/* Decoder control register 4. */
reg = G1_REG_DEC_CTRL4_FRAMENUM_LEN(sps->log2_max_frame_num_minus4 + 4) |
- G1_REG_DEC_CTRL4_FRAMENUM(slices[0].frame_num) |
+ G1_REG_DEC_CTRL4_FRAMENUM(dec_param->frame_num) |
G1_REG_DEC_CTRL4_WEIGHT_BIPR_IDC(pps->weighted_bipred_idc);
if (pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE)
reg |= G1_REG_DEC_CTRL4_CABAC_E;
vdpu_write_relaxed(vpu, reg, G1_REG_DEC_CTRL4);
/* Decoder control register 5. */
- reg = G1_REG_DEC_CTRL5_REFPIC_MK_LEN(slices[0].dec_ref_pic_marking_bit_size) |
- G1_REG_DEC_CTRL5_IDR_PIC_ID(slices[0].idr_pic_id);
+ reg = G1_REG_DEC_CTRL5_REFPIC_MK_LEN(dec_param->dec_ref_pic_marking_bit_size) |
+ G1_REG_DEC_CTRL5_IDR_PIC_ID(dec_param->idr_pic_id);
if (pps->flags & V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED)
reg |= G1_REG_DEC_CTRL5_CONST_INTRA_E;
if (pps->flags & V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT)
vdpu_write_relaxed(vpu, reg, G1_REG_DEC_CTRL5);
/* Decoder control register 6. */
- reg = G1_REG_DEC_CTRL6_PPS_ID(slices[0].pic_parameter_set_id) |
+ reg = G1_REG_DEC_CTRL6_PPS_ID(pps->pic_parameter_set_id) |
G1_REG_DEC_CTRL6_REFIDX0_ACTIVE(pps->num_ref_idx_l0_default_active_minus1 + 1) |
G1_REG_DEC_CTRL6_REFIDX1_ACTIVE(pps->num_ref_idx_l1_default_active_minus1 + 1) |
- G1_REG_DEC_CTRL6_POC_LENGTH(slices[0].pic_order_cnt_bit_size);
+ G1_REG_DEC_CTRL6_POC_LENGTH(dec_param->pic_order_cnt_bit_size);
vdpu_write_relaxed(vpu, reg, G1_REG_DEC_CTRL6);
/* Error concealment register. */
/* Destination (decoded frame) buffer. */
dst_dma = hantro_get_dec_buf_addr(ctx, &dst_buf->vb2_buf);
/* Adjust dma addr to start at second line for bottom field */
- if (ctrls->slices[0].flags & V4L2_H264_SLICE_FLAG_BOTTOM_FIELD)
+ if (ctrls->decode->flags & V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD)
offset = ALIGN(ctx->src_fmt.width, MB_DIM);
vdpu_write_relaxed(vpu, dst_dma + offset, G1_REG_ADDR_DST);
* DMV buffer is split in two for field encoded frames,
* adjust offset for bottom field
*/
- if (ctrls->slices[0].flags & V4L2_H264_SLICE_FLAG_BOTTOM_FIELD)
+ if (ctrls->decode->flags & V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD)
offset += 32 * MB_WIDTH(ctx->src_fmt.width) *
MB_HEIGHT(ctx->src_fmt.height);
vdpu_write_relaxed(vpu, dst_dma + offset, G1_REG_ADDR_DIR_MV);
{
const struct hantro_h264_dec_ctrls *ctrls = &ctx->h264_dec.ctrls;
const struct v4l2_ctrl_h264_scaling_matrix *scaling = ctrls->scaling;
+ const struct v4l2_ctrl_h264_pps *pps = ctrls->pps;
const size_t num_list_4x4 = ARRAY_SIZE(scaling->scaling_list_4x4);
const size_t list_len_4x4 = ARRAY_SIZE(scaling->scaling_list_4x4[0]);
const size_t list_len_8x8 = ARRAY_SIZE(scaling->scaling_list_8x8[0]);
const u32 *src;
int i, j;
+ if (!(pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT))
+ return;
+
for (i = 0; i < num_list_4x4; i++) {
src = (u32 *)&scaling->scaling_list_4x4[i];
for (j = 0; j < list_len_4x4 / 4; j++)
*/
dst_buf = hantro_get_dst_buf(ctx);
buf = &dst_buf->vb2_buf;
- dma_addr = vb2_dma_contig_plane_dma_addr(buf, 0);
+ dma_addr = hantro_get_dec_buf_addr(ctx, buf);
}
return dma_addr;
if (WARN_ON(!ctrls->decode))
return -EINVAL;
- ctrls->slices =
- hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS);
- if (WARN_ON(!ctrls->slices))
- return -EINVAL;
-
ctrls->sps =
hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_H264_SPS);
if (WARN_ON(!ctrls->sps))
/* Build the P/B{0,1} ref lists. */
v4l2_h264_init_reflist_builder(&reflist_builder, ctrls->decode,
- &ctrls->slices[0], ctrls->sps,
- ctx->h264_dec.dpb);
+ ctrls->sps, ctx->h264_dec.dpb);
v4l2_h264_build_p_ref_list(&reflist_builder, h264_ctx->reflists.p);
v4l2_h264_build_b_ref_lists(&reflist_builder, h264_ctx->reflists.b0,
h264_ctx->reflists.b1);
* struct hantro_h264_dec_ctrls
* @decode: Decode params
* @scaling: Scaling info
- * @slice: Slice params
* @sps: SPS info
* @pps: PPS info
*/
struct hantro_h264_dec_ctrls {
const struct v4l2_ctrl_h264_decode_params *decode;
const struct v4l2_ctrl_h264_scaling_matrix *scaling;
- const struct v4l2_ctrl_h264_slice_params *slices;
const struct v4l2_ctrl_h264_sps *sps;
const struct v4l2_ctrl_h264_pps *pps;
};
unsigned int num_buffers = cap_queue->num_buffers;
unsigned int i, buf_size;
- buf_size = ctx->dst_fmt.plane_fmt[0].sizeimage;
+ buf_size = ctx->dst_fmt.plane_fmt[0].sizeimage +
+ hantro_h264_mv_size(ctx->dst_fmt.width,
+ ctx->dst_fmt.height);
for (i = 0; i < num_buffers; ++i) {
struct hantro_aux_buf *priv = &ctx->postproc.dec_q[i];
driver uses the parsed DT bus config method until this issue is
resolved.
+ 2020-06: g_mbus has been removed in favour of the get_mbus_config pad
+ operation which should be used to avoid parsing the remote endpoint
+ configuration.
+
- This media driver supports inheriting V4L2 controls to the
video capture devices, from the subdevices in the capture device's
pipeline. The controls for each capture device are updated in the
#define IPU3_UAPI_AE_WEIGHTS 96
/**
- + * struct ipu3_uapi_ae_raw_buffer - AE global weighted histogram
- + *
- + * @vals: Sum of IPU3_UAPI_AE_COLORS in cell
- + *
- + * Each histogram contains IPU3_UAPI_AE_BINS bins. Each bin has 24 bit unsigned
- + * for counting the number of the pixel.
- + */
+ * struct ipu3_uapi_ae_raw_buffer - AE global weighted histogram
+ *
+ * @vals: Sum of IPU3_UAPI_AE_COLORS in cell
+ *
+ * Each histogram contains IPU3_UAPI_AE_BINS bins. Each bin has 24 bit unsigned
+ * for counting the number of the pixel.
+ */
struct ipu3_uapi_ae_raw_buffer {
__u32 vals[IPU3_UAPI_AE_BINS * IPU3_UAPI_AE_COLORS];
} __packed;
memset(&cfg->scaler_coeffs_chroma, 0,
sizeof(cfg->scaler_coeffs_chroma));
- memset(&cfg->scaler_coeffs_luma, 0, sizeof(*cfg->scaler_coeffs_luma));
+ memset(&cfg->scaler_coeffs_luma, 0, sizeof(cfg->scaler_coeffs_luma));
do {
phase_step_correction++;
static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe)
{
struct imgu_device *imgu = dev_get_drvdata(css->dev);
+ struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
unsigned int i;
- imgu_css_pool_cleanup(imgu,
- &css->pipes[pipe].pool.parameter_set_info);
- imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.acc);
- imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.gdc);
- imgu_css_pool_cleanup(imgu, &css->pipes[pipe].pool.obgrid);
+ imgu_css_pool_cleanup(imgu, &css_pipe->pool.parameter_set_info);
+ imgu_css_pool_cleanup(imgu, &css_pipe->pool.acc);
+ imgu_css_pool_cleanup(imgu, &css_pipe->pool.gdc);
+ imgu_css_pool_cleanup(imgu, &css_pipe->pool.obgrid);
for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
- imgu_css_pool_cleanup(imgu,
- &css->pipes[pipe].pool.binary_params_p[i]);
+ imgu_css_pool_cleanup(imgu, &css_pipe->pool.binary_params_p[i]);
}
/*
unsigned int i, j;
struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
+ struct imgu_css_queue *css_queue_in =
+ &css_pipe->queue[IPU3_CSS_QUEUE_IN];
+ struct imgu_css_queue *css_queue_out =
+ &css_pipe->queue[IPU3_CSS_QUEUE_OUT];
+ struct imgu_css_queue *css_queue_vf =
+ &css_pipe->queue[IPU3_CSS_QUEUE_VF];
const struct imgu_fw_info *bi =
&css->fwp->binary_header[css_pipe->bindex];
const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes;
struct imgu_abi_isp_stage *isp_stage;
struct imgu_abi_sp_stage *sp_stage;
struct imgu_abi_sp_group *sp_group;
+ struct imgu_abi_frames_sp *frames_sp;
+ struct imgu_abi_frame_sp *frame_sp;
+ struct imgu_abi_frame_sp_info *frame_sp_info;
const unsigned int bds_width_pad =
ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
if (!cfg_iter)
goto bad_firmware;
- cfg_iter->input_info.res.width =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width;
- cfg_iter->input_info.res.height =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height;
- cfg_iter->input_info.padded_width =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].width_pad;
- cfg_iter->input_info.format =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format;
- cfg_iter->input_info.raw_bit_depth =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth;
- cfg_iter->input_info.raw_bayer_order =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order;
- cfg_iter->input_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
-
- cfg_iter->internal_info.res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
- cfg_iter->internal_info.res.height =
- css_pipe->rect[IPU3_CSS_RECT_BDS].height;
- cfg_iter->internal_info.padded_width = bds_width_pad;
- cfg_iter->internal_info.format =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
- cfg_iter->internal_info.raw_bit_depth =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
- cfg_iter->internal_info.raw_bayer_order =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
- cfg_iter->internal_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
-
- cfg_iter->output_info.res.width =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width;
- cfg_iter->output_info.res.height =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height;
- cfg_iter->output_info.padded_width =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
- cfg_iter->output_info.format =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
- cfg_iter->output_info.raw_bit_depth =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
- cfg_iter->output_info.raw_bayer_order =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
- cfg_iter->output_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
-
- cfg_iter->vf_info.res.width =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width;
- cfg_iter->vf_info.res.height =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height;
- cfg_iter->vf_info.padded_width =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
- cfg_iter->vf_info.format =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
- cfg_iter->vf_info.raw_bit_depth =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth;
- cfg_iter->vf_info.raw_bayer_order =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order;
- cfg_iter->vf_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
-
- cfg_iter->dvs_envelope.width = css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
+ frame_sp_info = &cfg_iter->input_info;
+ frame_sp_info->res.width = css_queue_in->fmt.mpix.width;
+ frame_sp_info->res.height = css_queue_in->fmt.mpix.height;
+ frame_sp_info->padded_width = css_queue_in->width_pad;
+ frame_sp_info->format = css_queue_in->css_fmt->frame_format;
+ frame_sp_info->raw_bit_depth = css_queue_in->css_fmt->bit_depth;
+ frame_sp_info->raw_bayer_order = css_queue_in->css_fmt->bayer_order;
+ frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
+
+ frame_sp_info = &cfg_iter->internal_info;
+ frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
+ frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
+ frame_sp_info->padded_width = bds_width_pad;
+ frame_sp_info->format = css_queue_out->css_fmt->frame_format;
+ frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
+ frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
+ frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
+
+ frame_sp_info = &cfg_iter->output_info;
+ frame_sp_info->res.width = css_queue_out->fmt.mpix.width;
+ frame_sp_info->res.height = css_queue_out->fmt.mpix.height;
+ frame_sp_info->padded_width = css_queue_out->width_pad;
+ frame_sp_info->format = css_queue_out->css_fmt->frame_format;
+ frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
+ frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
+ frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
+
+ frame_sp_info = &cfg_iter->vf_info;
+ frame_sp_info->res.width = css_queue_vf->fmt.mpix.width;
+ frame_sp_info->res.height = css_queue_vf->fmt.mpix.height;
+ frame_sp_info->padded_width = css_queue_vf->width_pad;
+ frame_sp_info->format = css_queue_vf->css_fmt->frame_format;
+ frame_sp_info->raw_bit_depth = css_queue_vf->css_fmt->bit_depth;
+ frame_sp_info->raw_bayer_order = css_queue_vf->css_fmt->bayer_order;
+ frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
+
+ cfg_iter->dvs_envelope.width =
+ css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
cfg_iter->dvs_envelope.height =
css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr;
memset(sp_stage, 0, sizeof(*sp_stage));
- sp_stage->frames.in.buf_attr = buffer_sp_init;
+ frames_sp = &sp_stage->frames;
+ frames_sp->in.buf_attr = buffer_sp_init;
for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++)
- sp_stage->frames.out[i].buf_attr = buffer_sp_init;
- sp_stage->frames.out_vf.buf_attr = buffer_sp_init;
- sp_stage->frames.s3a_buf = buffer_sp_init;
- sp_stage->frames.dvs_buf = buffer_sp_init;
+ frames_sp->out[i].buf_attr = buffer_sp_init;
+ frames_sp->out_vf.buf_attr = buffer_sp_init;
+ frames_sp->s3a_buf = buffer_sp_init;
+ frames_sp->dvs_buf = buffer_sp_init;
sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP;
sp_stage->num = stage;
sp_stage->enable.vf_output = css_pipe->vf_output_en;
- sp_stage->frames.effective_in_res.width =
+ frames_sp->effective_in_res.width =
css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width;
- sp_stage->frames.effective_in_res.height =
+ frames_sp->effective_in_res.height =
css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height;
- sp_stage->frames.in.info.res.width =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.width;
- sp_stage->frames.in.info.res.height =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].fmt.mpix.height;
- sp_stage->frames.in.info.padded_width =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].width_pad;
- sp_stage->frames.in.info.format =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->frame_format;
- sp_stage->frames.in.info.raw_bit_depth =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bit_depth;
- sp_stage->frames.in.info.raw_bayer_order =
- css_pipe->queue[IPU3_CSS_QUEUE_IN].css_fmt->bayer_order;
- sp_stage->frames.in.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
- sp_stage->frames.in.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID;
- sp_stage->frames.in.buf_attr.buf_type =
- IMGU_ABI_BUFFER_TYPE_INPUT_FRAME;
-
- sp_stage->frames.out[0].info.res.width =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.width;
- sp_stage->frames.out[0].info.res.height =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height;
- sp_stage->frames.out[0].info.padded_width =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
- sp_stage->frames.out[0].info.format =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
- sp_stage->frames.out[0].info.raw_bit_depth =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
- sp_stage->frames.out[0].info.raw_bayer_order =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
- sp_stage->frames.out[0].info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
- sp_stage->frames.out[0].planes.nv.uv.offset =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad *
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].fmt.mpix.height;
- sp_stage->frames.out[0].buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID;
- sp_stage->frames.out[0].buf_attr.buf_type =
- IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME;
-
- sp_stage->frames.out[1].buf_attr.buf_src.queue_id =
- IMGU_ABI_QUEUE_EVENT_ID;
-
- sp_stage->frames.internal_frame_info.res.width =
- css_pipe->rect[IPU3_CSS_RECT_BDS].width;
- sp_stage->frames.internal_frame_info.res.height =
- css_pipe->rect[IPU3_CSS_RECT_BDS].height;
- sp_stage->frames.internal_frame_info.padded_width = bds_width_pad;
-
- sp_stage->frames.internal_frame_info.format =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
- sp_stage->frames.internal_frame_info.raw_bit_depth =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bit_depth;
- sp_stage->frames.internal_frame_info.raw_bayer_order =
- css_pipe->queue[IPU3_CSS_QUEUE_OUT].css_fmt->bayer_order;
- sp_stage->frames.internal_frame_info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
-
- sp_stage->frames.out_vf.info.res.width =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.width;
- sp_stage->frames.out_vf.info.res.height =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height;
- sp_stage->frames.out_vf.info.padded_width =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
- sp_stage->frames.out_vf.info.format =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
- sp_stage->frames.out_vf.info.raw_bit_depth =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bit_depth;
- sp_stage->frames.out_vf.info.raw_bayer_order =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].css_fmt->bayer_order;
- sp_stage->frames.out_vf.info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
- sp_stage->frames.out_vf.planes.yuv.u.offset =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad *
- css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height;
- sp_stage->frames.out_vf.planes.yuv.v.offset =
- css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad *
- css_pipe->queue[IPU3_CSS_QUEUE_VF].fmt.mpix.height * 5 / 4;
- sp_stage->frames.out_vf.buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID;
- sp_stage->frames.out_vf.buf_attr.buf_type =
- IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME;
-
- sp_stage->frames.s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID;
- sp_stage->frames.s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS;
-
- sp_stage->frames.dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID;
- sp_stage->frames.dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS;
-
- sp_stage->dvs_envelope.width = css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
+
+ frame_sp = &frames_sp->in;
+ frame_sp->info.res.width = css_queue_in->fmt.mpix.width;
+ frame_sp->info.res.height = css_queue_in->fmt.mpix.height;
+ frame_sp->info.padded_width = css_queue_in->width_pad;
+ frame_sp->info.format = css_queue_in->css_fmt->frame_format;
+ frame_sp->info.raw_bit_depth = css_queue_in->css_fmt->bit_depth;
+ frame_sp->info.raw_bayer_order = css_queue_in->css_fmt->bayer_order;
+ frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
+ frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID;
+ frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_INPUT_FRAME;
+
+ frame_sp = &frames_sp->out[0];
+ frame_sp->info.res.width = css_queue_out->fmt.mpix.width;
+ frame_sp->info.res.height = css_queue_out->fmt.mpix.height;
+ frame_sp->info.padded_width = css_queue_out->width_pad;
+ frame_sp->info.format = css_queue_out->css_fmt->frame_format;
+ frame_sp->info.raw_bit_depth = css_queue_out->css_fmt->bit_depth;
+ frame_sp->info.raw_bayer_order = css_queue_out->css_fmt->bayer_order;
+ frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
+ frame_sp->planes.nv.uv.offset = css_queue_out->width_pad *
+ css_queue_out->fmt.mpix.height;
+ frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID;
+ frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME;
+
+ frame_sp = &frames_sp->out[1];
+ frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_EVENT_ID;
+
+ frame_sp_info = &frames_sp->internal_frame_info;
+ frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
+ frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
+ frame_sp_info->padded_width = bds_width_pad;
+ frame_sp_info->format = css_queue_out->css_fmt->frame_format;
+ frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
+ frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
+ frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
+
+ frame_sp = &frames_sp->out_vf;
+ frame_sp->info.res.width = css_queue_vf->fmt.mpix.width;
+ frame_sp->info.res.height = css_queue_vf->fmt.mpix.height;
+ frame_sp->info.padded_width = css_queue_vf->width_pad;
+ frame_sp->info.format = css_queue_vf->css_fmt->frame_format;
+ frame_sp->info.raw_bit_depth = css_queue_vf->css_fmt->bit_depth;
+ frame_sp->info.raw_bayer_order = css_queue_vf->css_fmt->bayer_order;
+ frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
+ frame_sp->planes.yuv.u.offset = css_queue_vf->width_pad *
+ css_queue_vf->fmt.mpix.height;
+ frame_sp->planes.yuv.v.offset = css_queue_vf->width_pad *
+ css_queue_vf->fmt.mpix.height * 5 / 4;
+ frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID;
+ frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME;
+
+ frames_sp->s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID;
+ frames_sp->s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS;
+
+ frames_sp->dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID;
+ frames_sp->dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS;
+
+ sp_stage->dvs_envelope.width =
+ css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
sp_stage->dvs_envelope.height =
css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
dst_vq->min_buffers_needed = 1;
dst_vq->dev = sess->core->dev;
dst_vq->lock = &sess->lock;
- ret = vb2_queue_init(dst_vq);
- if (ret) {
- vb2_queue_release(src_vq);
- return ret;
- }
-
- return 0;
+ return vb2_queue_init(dst_vq);
}
static int vdec_init_ctrls(struct amvdec_session *sess)
* readback the same register, in this case the revision register.
*
* See this link for reference:
- * http://www.mail-archive.com/linux-omap@vger.kernel.org/msg08149.html
+ * https://www.mail-archive.com/linux-omap@vger.kernel.org/msg08149.html
*/
static void omap4iss_flush(struct iss_device *iss)
{
+++ /dev/null
-.. SPDX-License-Identifier: (GPL-2.0+ OR MIT)
-
-.. _v4l2-meta-fmt-rkisp1-params:
-
-============================
-V4L2_META_FMT_RK_ISP1_PARAMS
-============================
-
-Rockchip ISP1 Parameters Data
-
-Description
-===========
-
-This format describes input parameters for the Rockchip ISP1.
-
-It uses c-struct :c:type:`rkisp1_params_cfg`, which is defined in
-the ``linux/rkisp1-config.h`` header file.
-
-The parameters consist of multiple modules.
-The module won't be updated if the corresponding bit was not set in module_*_update.
-
-.. kernel-doc:: include/uapi/linux/rkisp1-config.h
- :functions: rkisp1_params_cfg
+++ /dev/null
-.. SPDX-License-Identifier: (GPL-2.0+ OR MIT)
-
-.. _v4l2-meta-fmt-rkisp1-stat:
-
-=============================
-V4L2_META_FMT_RK_ISP1_STAT_3A
-=============================
-
-
-Rockchip ISP1 Statistics Data
-
-Description
-===========
-
-This format describes image color statistics information generated by the Rockchip
-ISP1.
-
-It uses c-struct :c:type:`rkisp1_stat_buffer`, which is defined in
-the ``linux/rkisp1-config.h`` header file.
-
-.. kernel-doc:: include/uapi/linux/rkisp1-config.h
- :functions: rkisp1_stat_buffer
* Fix pad format size for statistics and parameters entities.
* Fix checkpatch errors.
-* Review and comment every lock
-* Handle quantization
-* Document rkisp1-common.h
+* Add uapi docs. Remember to add documentation of how quantization is handled.
* streaming paths (mainpath and selfpath) check if the other path is streaming
in several places of the code, review this, specially that it doesn't seem it
supports streaming from both paths at the same time.
* @uv_swap: if cb cr swaped, for yuv
* @write_format: defines how YCbCr self picture data is written to memory
* @output_format: defines sp output format
+ * @mbus: the mbus code on the src resizer pad that matches the pixel format
*/
struct rkisp1_capture_fmt_cfg {
u32 fourcc;
u8 uv_swap;
u32 write_format;
u32 output_format;
+ u32 mbus;
};
struct rkisp1_capture_ops {
} mi;
};
+/*
+ * The supported pixel formats for mainpath. NOTE, pixel formats with identical 'mbus'
+ * are grouped together. This is assumed and used by the function rkisp1_cap_enum_mbus_codes
+ */
static const struct rkisp1_capture_fmt_cfg rkisp1_mp_fmts[] = {
/* yuv422 */
{
.fourcc = V4L2_PIX_FMT_YUYV,
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUVINT,
- }, {
- .fourcc = V4L2_PIX_FMT_YVYU,
- .uv_swap = 1,
- .write_format = RKISP1_MI_CTRL_MP_WRITE_YUVINT,
- }, {
- .fourcc = V4L2_PIX_FMT_VYUY,
- .write_format = RKISP1_MI_CTRL_MP_WRITE_YUVINT,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
}, {
.fourcc = V4L2_PIX_FMT_YUV422P,
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
}, {
.fourcc = V4L2_PIX_FMT_NV16,
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
}, {
.fourcc = V4L2_PIX_FMT_NV61,
.uv_swap = 1,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
}, {
.fourcc = V4L2_PIX_FMT_YVU422M,
.uv_swap = 1,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
+ },
+ /* yuv400 */
+ {
+ .fourcc = V4L2_PIX_FMT_GREY,
+ .uv_swap = 0,
+ .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
},
/* yuv420 */
{
.fourcc = V4L2_PIX_FMT_NV21,
.uv_swap = 1,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
}, {
.fourcc = V4L2_PIX_FMT_NV12,
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
}, {
.fourcc = V4L2_PIX_FMT_NV21M,
.uv_swap = 1,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
}, {
.fourcc = V4L2_PIX_FMT_NV12M,
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_SPLA,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
}, {
.fourcc = V4L2_PIX_FMT_YUV420,
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
}, {
.fourcc = V4L2_PIX_FMT_YVU420,
.uv_swap = 1,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8,
- },
- /* yuv444 */
- {
- .fourcc = V4L2_PIX_FMT_YUV444M,
- .uv_swap = 0,
- .write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8,
- },
- /* yuv400 */
- {
- .fourcc = V4L2_PIX_FMT_GREY,
- .uv_swap = 0,
- .write_format = RKISP1_MI_CTRL_MP_WRITE_YUVINT,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
},
/* raw */
{
.fourcc = V4L2_PIX_FMT_SRGGB8,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8,
+ .mbus = MEDIA_BUS_FMT_SRGGB8_1X8,
}, {
.fourcc = V4L2_PIX_FMT_SGRBG8,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8,
+ .mbus = MEDIA_BUS_FMT_SGRBG8_1X8,
}, {
.fourcc = V4L2_PIX_FMT_SGBRG8,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8,
+ .mbus = MEDIA_BUS_FMT_SGBRG8_1X8,
}, {
.fourcc = V4L2_PIX_FMT_SBGGR8,
.write_format = RKISP1_MI_CTRL_MP_WRITE_YUV_PLA_OR_RAW8,
+ .mbus = MEDIA_BUS_FMT_SBGGR8_1X8,
}, {
.fourcc = V4L2_PIX_FMT_SRGGB10,
.write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12,
+ .mbus = MEDIA_BUS_FMT_SRGGB10_1X10,
}, {
.fourcc = V4L2_PIX_FMT_SGRBG10,
.write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12,
+ .mbus = MEDIA_BUS_FMT_SGRBG10_1X10,
}, {
.fourcc = V4L2_PIX_FMT_SGBRG10,
.write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12,
+ .mbus = MEDIA_BUS_FMT_SGBRG10_1X10,
}, {
.fourcc = V4L2_PIX_FMT_SBGGR10,
.write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12,
+ .mbus = MEDIA_BUS_FMT_SBGGR10_1X10,
}, {
.fourcc = V4L2_PIX_FMT_SRGGB12,
.write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12,
+ .mbus = MEDIA_BUS_FMT_SRGGB12_1X12,
}, {
.fourcc = V4L2_PIX_FMT_SGRBG12,
.write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12,
+ .mbus = MEDIA_BUS_FMT_SGRBG12_1X12,
}, {
.fourcc = V4L2_PIX_FMT_SGBRG12,
.write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12,
+ .mbus = MEDIA_BUS_FMT_SGBRG12_1X12,
}, {
.fourcc = V4L2_PIX_FMT_SBGGR12,
.write_format = RKISP1_MI_CTRL_MP_WRITE_RAW12,
+ .mbus = MEDIA_BUS_FMT_SBGGR12_1X12,
},
};
+/*
+ * The supported pixel formats for selfpath. NOTE, pixel formats with identical 'mbus'
+ * are grouped together. This is assumed and used by the function rkisp1_cap_enum_mbus_codes
+ */
static const struct rkisp1_capture_fmt_cfg rkisp1_sp_fmts[] = {
/* yuv422 */
{
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_SP_WRITE_INT,
.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422,
- }, {
- .fourcc = V4L2_PIX_FMT_YVYU,
- .uv_swap = 1,
- .write_format = RKISP1_MI_CTRL_SP_WRITE_INT,
- .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422,
- }, {
- .fourcc = V4L2_PIX_FMT_VYUY,
- .uv_swap = 1,
- .write_format = RKISP1_MI_CTRL_SP_WRITE_INT,
- .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
}, {
.fourcc = V4L2_PIX_FMT_YUV422P,
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_SP_WRITE_PLA,
.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
}, {
.fourcc = V4L2_PIX_FMT_NV16,
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA,
.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
}, {
.fourcc = V4L2_PIX_FMT_NV61,
.uv_swap = 1,
.write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA,
.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
}, {
.fourcc = V4L2_PIX_FMT_YVU422M,
.uv_swap = 1,
.write_format = RKISP1_MI_CTRL_SP_WRITE_PLA,
.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV422,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
+ },
+ /* yuv400 */
+ {
+ .fourcc = V4L2_PIX_FMT_GREY,
+ .uv_swap = 0,
+ .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA,
+ .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV400,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
+ },
+ /* rgb */
+ {
+ .fourcc = V4L2_PIX_FMT_XBGR32,
+ .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA,
+ .output_format = RKISP1_MI_CTRL_SP_OUTPUT_RGB888,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
+ }, {
+ .fourcc = V4L2_PIX_FMT_RGB565,
+ .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA,
+ .output_format = RKISP1_MI_CTRL_SP_OUTPUT_RGB565,
+ .mbus = MEDIA_BUS_FMT_YUYV8_2X8,
},
/* yuv420 */
{
.uv_swap = 1,
.write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA,
.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
}, {
.fourcc = V4L2_PIX_FMT_NV12,
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA,
.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
}, {
.fourcc = V4L2_PIX_FMT_NV21M,
.uv_swap = 1,
.write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA,
.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
}, {
.fourcc = V4L2_PIX_FMT_NV12M,
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_SP_WRITE_SPLA,
.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
}, {
.fourcc = V4L2_PIX_FMT_YUV420,
.uv_swap = 0,
.write_format = RKISP1_MI_CTRL_SP_WRITE_PLA,
.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
}, {
.fourcc = V4L2_PIX_FMT_YVU420,
.uv_swap = 1,
.write_format = RKISP1_MI_CTRL_SP_WRITE_PLA,
.output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV420,
- },
- /* yuv444 */
- {
- .fourcc = V4L2_PIX_FMT_YUV444M,
- .uv_swap = 0,
- .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA,
- .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV444,
- },
- /* yuv400 */
- {
- .fourcc = V4L2_PIX_FMT_GREY,
- .uv_swap = 0,
- .write_format = RKISP1_MI_CTRL_SP_WRITE_INT,
- .output_format = RKISP1_MI_CTRL_SP_OUTPUT_YUV400,
- },
- /* rgb */
- {
- .fourcc = V4L2_PIX_FMT_RGB24,
- .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA,
- .output_format = RKISP1_MI_CTRL_SP_OUTPUT_RGB888,
- }, {
- .fourcc = V4L2_PIX_FMT_RGB565,
- .write_format = RKISP1_MI_CTRL_SP_WRITE_PLA,
- .output_format = RKISP1_MI_CTRL_SP_OUTPUT_RGB565,
+ .mbus = MEDIA_BUS_FMT_YUYV8_1_5X8,
},
};
return container_of(vdev, struct rkisp1_vdev_node, vdev);
}
+int rkisp1_cap_enum_mbus_codes(struct rkisp1_capture *cap,
+ struct v4l2_subdev_mbus_code_enum *code)
+{
+ const struct rkisp1_capture_fmt_cfg *fmts = cap->config->fmts;
+ /*
+ * initialize curr_mbus to non existing mbus code 0 to ensure it is
+ * different from fmts[0].mbus
+ */
+ u32 curr_mbus = 0;
+ int i, n = 0;
+
+ for (i = 0; i < cap->config->fmt_size; i++) {
+ if (fmts[i].mbus == curr_mbus)
+ continue;
+
+ curr_mbus = fmts[i].mbus;
+ if (n++ == code->index) {
+ code->code = curr_mbus;
+ return 0;
+ }
+ }
+ return -EINVAL;
+}
+
/* ----------------------------------------------------------------------------
* Stream operations for self-picture path (sp) and main-picture path (mp)
*/
{
struct rkisp1_isp *isp = &cap->rkisp1->isp;
struct rkisp1_buffer *curr_buf;
- unsigned long flags;
- spin_lock_irqsave(&cap->buf.lock, flags);
+ spin_lock(&cap->buf.lock);
curr_buf = cap->buf.curr;
if (curr_buf) {
- curr_buf->vb.sequence = atomic_read(&isp->frame_sequence);
+ curr_buf->vb.sequence = isp->frame_sequence;
curr_buf->vb.vb2_buf.timestamp = ktime_get_boottime_ns();
curr_buf->vb.field = V4L2_FIELD_NONE;
vb2_buffer_done(&curr_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
}
rkisp1_set_next_buf(cap);
- spin_unlock_irqrestore(&cap->buf.lock, flags);
+ spin_unlock(&cap->buf.lock);
}
void rkisp1_capture_isr(struct rkisp1_device *rkisp1)
container_of(vbuf, struct rkisp1_buffer, vb);
struct rkisp1_capture *cap = vb->vb2_queue->drv_priv;
const struct v4l2_pix_format_mplane *pixm = &cap->pix.fmt;
- unsigned long flags;
unsigned int i;
memset(ispbuf->buff_addr, 0, sizeof(ispbuf->buff_addr));
swap(ispbuf->buff_addr[RKISP1_PLANE_CR],
ispbuf->buff_addr[RKISP1_PLANE_CB]);
- spin_lock_irqsave(&cap->buf.lock, flags);
+ spin_lock_irq(&cap->buf.lock);
list_add_tail(&ispbuf->queue, &cap->buf.queue);
- spin_unlock_irqrestore(&cap->buf.lock, flags);
+ spin_unlock_irq(&cap->buf.lock);
}
static int rkisp1_vb2_buf_prepare(struct vb2_buffer *vb)
static void rkisp1_return_all_buffers(struct rkisp1_capture *cap,
enum vb2_buffer_state state)
{
- unsigned long flags;
struct rkisp1_buffer *buf;
- spin_lock_irqsave(&cap->buf.lock, flags);
+ spin_lock_irq(&cap->buf.lock);
if (cap->buf.curr) {
vb2_buffer_done(&cap->buf.curr->vb.vb2_buf, state);
cap->buf.curr = NULL;
list_del(&buf->queue);
vb2_buffer_done(&buf->vb.vb2_buf, state);
}
- spin_unlock_irqrestore(&cap->buf.lock, flags);
+ spin_unlock_irq(&cap->buf.lock);
}
/*
cap->ops->config(cap);
/* Setup a buffer for the next frame */
+ spin_lock_irq(&cap->buf.lock);
rkisp1_set_next_buf(cap);
cap->ops->enable(cap);
/* It's safe to config ACTIVE and SHADOW regs for the
RKISP1_CIF_MI_INIT_SOFT_UPD, RKISP1_CIF_MI_INIT);
rkisp1_set_next_buf(cap);
}
+ spin_unlock_irq(&cap->buf.lock);
cap->is_streaming = true;
}
unsigned int i;
u32 stride;
+ memset(pixm->plane_fmt, 0, sizeof(pixm->plane_fmt));
info = v4l2_format_info(pixm->pixelformat);
pixm->num_planes = info->mem_planes;
stride = info->bpp[0] * pixm->width;
const struct v4l2_format_info **fmt_info)
{
const struct rkisp1_capture_config *config = cap->config;
- struct rkisp1_capture *other_cap =
- &cap->rkisp1->capture_devs[cap->id ^ 1];
const struct rkisp1_capture_fmt_cfg *fmt;
const struct v4l2_format_info *info;
const unsigned int max_widths[] = { RKISP1_RSZ_MP_SRC_MAX_WIDTH,
info = rkisp1_fill_pixfmt(pixm, cap->id);
- /* can not change quantization when stream-on */
- if (other_cap->is_streaming)
- pixm->quantization = other_cap->pix.fmt.quantization;
- /* output full range by default, take effect in params */
- else if (!pixm->quantization ||
- pixm->quantization > V4L2_QUANTIZATION_LIM_RANGE)
- pixm->quantization = V4L2_QUANTIZATION_FULL_RANGE;
-
if (fmt_cfg)
*fmt_cfg = fmt;
if (fmt_info)
{
struct rkisp1_capture *cap = video_drvdata(file);
const struct rkisp1_capture_fmt_cfg *fmt = NULL;
+ unsigned int i, n = 0;
- if (f->index >= cap->config->fmt_size)
- return -EINVAL;
+ if (!f->mbus_code) {
+ if (f->index >= cap->config->fmt_size)
+ return -EINVAL;
- fmt = &cap->config->fmts[f->index];
- f->pixelformat = fmt->fourcc;
+ fmt = &cap->config->fmts[f->index];
+ f->pixelformat = fmt->fourcc;
+ return 0;
+ }
- return 0;
+ for (i = 0; i < cap->config->fmt_size; i++) {
+ if (cap->config->fmts[i].mbus != f->mbus_code)
+ continue;
+
+ if (n++ == f->index) {
+ f->pixelformat = cap->config->fmts[i].fourcc;
+ return 0;
+ }
+ }
+ return -EINVAL;
}
static int rkisp1_s_fmt_vid_cap_mplane(struct file *file,
struct v4l2_subdev *sd =
media_entity_to_v4l2_subdev(link->source->entity);
struct rkisp1_capture *cap = video_get_drvdata(vdev);
- struct rkisp1_isp *isp = &cap->rkisp1->isp;
- u8 isp_pix_enc = isp->src_fmt->pixel_enc;
- u8 cap_pix_enc = cap->pix.info->pixel_enc;
+ const struct rkisp1_capture_fmt_cfg *fmt =
+ rkisp1_find_fmt_cfg(cap, cap->pix.fmt.pixelformat);
struct v4l2_subdev_format sd_fmt;
int ret;
- if (cap->id == RKISP1_SELFPATH &&
- isp->src_fmt->mbus_code != MEDIA_BUS_FMT_YUYV8_2X8) {
- dev_err(cap->rkisp1->dev,
- "selfpath only supports MEDIA_BUS_FMT_YUYV8_2X8\n");
- return -EPIPE;
- }
-
- if (cap_pix_enc != isp_pix_enc &&
- !(isp_pix_enc == V4L2_PIXEL_ENC_YUV &&
- cap_pix_enc == V4L2_PIXEL_ENC_RGB)) {
- dev_err(cap->rkisp1->dev,
- "format type mismatch in link '%s:%d->%s:%d'\n",
- link->source->entity->name, link->source->index,
- link->sink->entity->name, link->sink->index);
- return -EPIPE;
- }
-
sd_fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
sd_fmt.pad = link->source->index;
ret = v4l2_subdev_call(sd, pad, get_fmt, NULL, &sd_fmt);
return ret;
if (sd_fmt.format.height != cap->pix.fmt.height ||
- sd_fmt.format.width != cap->pix.fmt.width)
+ sd_fmt.format.width != cap->pix.fmt.width ||
+ sd_fmt.format.code != fmt->mbus)
return -EPIPE;
return 0;
static void rkisp1_unregister_capture(struct rkisp1_capture *cap)
{
media_entity_cleanup(&cap->vnode.vdev.entity);
- video_unregister_device(&cap->vnode.vdev);
+ vb2_video_unregister_device(&cap->vnode.vdev);
}
void rkisp1_capture_devs_unregister(struct rkisp1_device *rkisp1)
vdev->v4l2_dev = v4l2_dev;
vdev->lock = &node->vlock;
vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE |
- V4L2_CAP_STREAMING;
+ V4L2_CAP_STREAMING | V4L2_CAP_IO_MC;
vdev->entity.ops = &rkisp1_media_ops;
video_set_drvdata(vdev, cap);
vdev->vfl_dir = VFL_DIR_RX;
#include "rkisp1-regs.h"
#include "uapi/rkisp1-config.h"
+/*
+ * flags on the 'direction' field in struct 'rkisp1_isp_mbus_info' that indicate
+ * on which pad the media bus format is supported
+ */
#define RKISP1_ISP_SD_SRC BIT(0)
#define RKISP1_ISP_SD_SINK BIT(1)
+/* min and max values for the widths and heights of the entities */
#define RKISP1_ISP_MAX_WIDTH 4032
#define RKISP1_ISP_MAX_HEIGHT 3024
#define RKISP1_ISP_MIN_WIDTH 32
#define RKISP1_RSZ_SRC_MIN_WIDTH 32
#define RKISP1_RSZ_SRC_MIN_HEIGHT 16
+/* the default width and height of all the entities */
#define RKISP1_DEFAULT_WIDTH 800
#define RKISP1_DEFAULT_HEIGHT 600
#define RKISP1_DRIVER_NAME "rkisp1"
#define RKISP1_BUS_INFO "platform:" RKISP1_DRIVER_NAME
+/* maximum number of clocks */
#define RKISP1_MAX_BUS_CLK 8
+/* a bitmask of the ready stats */
#define RKISP1_STATS_MEAS_MASK (RKISP1_CIF_ISP_AWB_DONE | \
RKISP1_CIF_ISP_AFM_FIN | \
RKISP1_CIF_ISP_EXP_END | \
RKISP1_CIF_ISP_HIST_MEASURE_RDY)
+
+/* enum for the resizer pads */
enum rkisp1_rsz_pad {
RKISP1_RSZ_PAD_SINK,
RKISP1_RSZ_PAD_SRC,
RKISP1_RSZ_PAD_MAX
};
+/* enum for the capture id */
enum rkisp1_stream_id {
RKISP1_MAINPATH,
RKISP1_SELFPATH,
};
+/* bayer patterns */
enum rkisp1_fmt_raw_pat_type {
RKISP1_RAW_RGGB = 0,
RKISP1_RAW_GRBG,
RKISP1_RAW_BGGR,
};
+/* enum for the isp pads */
enum rkisp1_isp_pad {
RKISP1_ISP_PAD_SINK_VIDEO,
RKISP1_ISP_PAD_SINK_PARAMS,
};
/*
- * struct rkisp1_sensor_async - Sensor information
- * @mbus: media bus configuration
+ * struct rkisp1_sensor_async - A container for the v4l2_async_subdev to add to the notifier
+ * of the v4l2-async API
+ *
+ * @asd: async_subdev variable for the sensor
+ * @lanes: number of lanes
+ * @mbus_type: type of bus (currently only CSI2 is supported)
+ * @mbus_flags: media bus (V4L2_MBUS_*) flags
+ * @sd: a pointer to v4l2_subdev struct of the sensor
+ * @pixel_rate_ctrl: pixel rate of the sensor, used to initialize the phy
+ * @dphy: a pointer to the phy
*/
struct rkisp1_sensor_async {
struct v4l2_async_subdev asd;
};
/*
- * struct rkisp1_isp - ISP sub-device
+ * struct rkisp1_isp - ISP subdev entity
*
- * See Cropping regions of ISP in rkisp1.c for details
- * @sink_frm: input size, don't have to be equal to sensor size
- * @sink_fmt: input format
- * @sink_crop: crop for sink pad
- * @src_fmt: output format
- * @src_crop: output size
- * @ops_lock: ops serialization
- *
- * @is_dphy_errctrl_disabled : if dphy errctrl is disabled (avoid endless interrupt)
- * @frame_sequence: used to synchronize frame_id between video devices.
- * @quantization: output quantization
+ * @sd: v4l2_subdev variable
+ * @rkisp1: pointer to rkisp1_device
+ * @pads: media pads
+ * @pad_cfg: pads configurations
+ * @sink_fmt: input format
+ * @src_fmt: output format
+ * @ops_lock: ops serialization
+ * @is_dphy_errctrl_disabled: if dphy errctrl is disabled (avoid endless interrupt)
+ * @frame_sequence: used to synchronize frame_id between video devices.
*/
struct rkisp1_isp {
struct v4l2_subdev sd;
struct v4l2_subdev_pad_config pad_cfg[RKISP1_ISP_PAD_MAX];
const struct rkisp1_isp_mbus_info *sink_fmt;
const struct rkisp1_isp_mbus_info *src_fmt;
- struct mutex ops_lock;
+ struct mutex ops_lock; /* serialize the subdevice ops */
bool is_dphy_errctrl_disabled;
- atomic_t frame_sequence;
+ __u32 frame_sequence;
};
+/*
+ * struct rkisp1_vdev_node - Container for the video nodes: params, stats, mainpath, selfpath
+ *
+ * @buf_queue: queue of buffers
+ * @vlock: lock of the video node
+ * @vdev: video node
+ * @pad: media pad
+ */
struct rkisp1_vdev_node {
struct vb2_queue buf_queue;
struct mutex vlock; /* ioctl serialization mutex */
struct media_pad pad;
};
+/*
+ * struct rkisp1_buffer - A container for the vb2 buffers used by the video devices:
+ * params, stats, mainpath, selfpath
+ *
+ * @vb: vb2 buffer
+ * @queue: entry of the buffer in the queue
+ * @buff_addr: dma addresses of each plane, used only by the capture devices: selfpath, mainpath
+ * @vaddr: virtual address for buffers used by params and stats devices
+ */
struct rkisp1_buffer {
struct vb2_v4l2_buffer vb;
struct list_head queue;
union {
u32 buff_addr[VIDEO_MAX_PLANES];
- void *vaddr[VIDEO_MAX_PLANES];
+ void *vaddr;
};
};
+/*
+ * struct rkisp1_dummy_buffer - A buffer to write the next frame to in case
+ * there are no vb2 buffers available.
+ *
+ * @vaddr: return value of call to dma_alloc_attrs.
+ * @dma_addr: dma address of the buffer.
+ * @size: size of the buffer.
+ */
struct rkisp1_dummy_buffer {
void *vaddr;
dma_addr_t dma_addr;
/*
* struct rkisp1_capture - ISP capture video device
*
- * @pix.fmt: buffer format
- * @pix.info: pixel information
- * @pix.cfg: pixel configuration
+ * @vnode: video node
+ * @rkisp1: pointer to rkisp1_device
+ * @id: id of the capture, one of RKISP1_SELFPATH, RKISP1_MAINPATH
+ * @ops: list of callbacks to configure the capture device.
+ * @config: a pointer to the list of registers to configure the capture format.
+ * @is_streaming: device is streaming
+ * @is_stopping: stop_streaming callback was called and the device is in the process of
+ * stopping the streaming.
+ * @done: when stop_streaming callback is called, the device waits for the next irq
+ * handler to stop the streaming by waiting on the 'done' wait queue.
+ * If the irq handler is not called, the stream is stopped by the callback
+ * after timeout.
+ * @sp_y_stride: the selfpath allows to configure a y stride that is longer than the image width.
+ * @buf.lock: lock to protect buf.queue
+ * @buf.queue: queued buffer list
+ * @buf.dummy: dummy space to store dropped data
*
- * @buf.lock: lock to protect buf_queue
- * @buf.queue: queued buffer list
- * @buf.dummy: dummy space to store dropped data
- *
- * rkisp1 use shadowsock registers, so it need two buffer at a time
- * @buf.curr: the buffer used for current frame
- * @buf.next: the buffer used for next frame
+ * rkisp1 uses shadow registers, so it needs two buffers at a time
+ * @buf.curr: the buffer used for current frame
+ * @buf.next: the buffer used for next frame
+ * @pix.cfg: pixel configuration
+ * @pix.info: a pointer to the v4l2_format_info of the pixel format
+ * @pix.fmt: buffer format
*/
struct rkisp1_capture {
struct rkisp1_vdev_node vnode;
/*
* struct rkisp1_stats - ISP Statistics device
*
- * @lock: locks the buffer list 'stat' and 'is_streaming'
- * @stat: stats buffer list
+ * @vnode: video node
+ * @rkisp1: pointer to the rkisp1 device
+ * @lock: locks the buffer list 'stat' and 'is_streaming'
+ * @stat: queue of rkisp1_buffer
+ * @vdev_fmt: v4l2_format of the metadata format
+ * @is_streaming: device is streaming
*/
struct rkisp1_stats {
struct rkisp1_vdev_node vnode;
struct rkisp1_device *rkisp1;
- spinlock_t lock; /* locks 'is_streaming', and 'stats' */
+ spinlock_t lock; /* locks the buffers list 'stats' and 'is_streaming' */
struct list_head stat;
struct v4l2_format vdev_fmt;
bool is_streaming;
/*
* struct rkisp1_params - ISP input parameters device
*
- * @cur_params: Current ISP parameters
- * @is_first_params: the first params should take effect immediately
+ * @vnode: video node
+ * @rkisp1: pointer to the rkisp1 device
+ * @config_lock: locks the buffer list 'params' and 'is_streaming'
+ * @params: queue of rkisp1_buffer
+ * @vdev_fmt: v4l2_format of the metadata format
+ * @is_streaming: device is streaming
+ * @quantization: the quantization configured on the isp's src pad
+ * @raw_type: the bayer pattern on the isp video sink pad
*/
struct rkisp1_params {
struct rkisp1_vdev_node vnode;
struct rkisp1_device *rkisp1;
- spinlock_t config_lock;
+ spinlock_t config_lock; /* locks the buffers list 'params' and 'is_streaming' */
struct list_head params;
- struct rkisp1_params_cfg cur_params;
struct v4l2_format vdev_fmt;
bool is_streaming;
- bool is_first_params;
enum v4l2_quantization quantization;
enum rkisp1_fmt_raw_pat_type raw_type;
};
+/*
+ * struct rkisp1_resizer - Resizer subdev
+ *
+ * @sd: v4l2_subdev variable
+ * @id: id of the resizer, one of RKISP1_SELFPATH, RKISP1_MAINPATH
+ * @rkisp1: pointer to the rkisp1 device
+ * @pads: media pads
+ * @pad_cfg: configurations for the pads
+ * @config: the set of registers to configure the resizer
+ * @pixel_enc: pixel encoding of the resizer
+ * @ops_lock: a lock for the subdev ops
+ */
struct rkisp1_resizer {
struct v4l2_subdev sd;
enum rkisp1_stream_id id;
struct v4l2_subdev_pad_config pad_cfg[RKISP1_RSZ_PAD_MAX];
const struct rkisp1_rsz_config *config;
enum v4l2_pixel_encoding pixel_enc;
- struct mutex ops_lock;
+ struct mutex ops_lock; /* serialize the subdevice ops */
};
+/*
+ * struct rkisp1_debug - Values to be exposed on debugfs.
+ * The parameters are counters of the number of times the
+ * event occurred since the driver was loaded.
+ *
+ * @data_loss: loss of data occurred within a line, processing failure
+ * @outform_size_error: size error is generated in outmux submodule
+ * @img_stabilization_size_error: size error is generated in image stabilization submodule
+ * @inform_size_err: size error is generated in inform submodule
+ * @mipi_error: mipi error occurred
+ * @stats_error: writing to the 'Interrupt clear register' did not clear
+ * it in the register 'Masked interrupt status'
+ * @stop_timeout: upon stream stop, the capture waits 1 second for the isr to stop
+ * the stream. This param is incremented in case of timeout.
+ * @frame_drop: a frame was ready but the buffer queue was empty so the frame
+ * was not sent to userspace
+ */
struct rkisp1_debug {
struct dentry *debugfs_dir;
unsigned long data_loss;
unsigned long outform_size_error;
unsigned long img_stabilization_size_error;
unsigned long inform_size_error;
+ unsigned long irq_delay;
unsigned long mipi_error;
unsigned long stats_error;
unsigned long stop_timeout[2];
/*
* struct rkisp1_device - ISP platform device
- * @base_addr: base register address
+ *
+ * @base_addr: base register address
+ * @irq: the irq number
+ * @dev: a pointer to the struct device
+ * @clk_size: number of clocks
+ * @clks: array of clocks
+ * @v4l2_dev: v4l2_device variable
+ * @media_dev: media_device variable
+ * @notifier: a notifier to register on the v4l2-async API to be notified on the sensor
* @active_sensor: sensor in-use, set when streaming on
- * @isp: ISP sub-device
- * @rkisp1_capture: capture video device
- * @stats: ISP statistics output device
- * @params: ISP input parameters device
- * @stream_lock: lock to serialize start/stop streaming in capture devices.
+ * @isp: ISP sub-device
+ * @resizer_devs: resizer sub-devices
+ * @capture_devs: capture devices
+ * @stats: ISP statistics metadata capture device
+ * @params: ISP parameters metadata output device
+ * @pipe: media pipeline
+ * @stream_lock: serializes {start/stop}_streaming callbacks between the capture devices.
+ * @debug: debug params to be exposed on debugfs
*/
struct rkisp1_device {
void __iomem *base_addr;
struct rkisp1_stats stats;
struct rkisp1_params params;
struct media_pipeline pipe;
- struct mutex stream_lock;
+ struct mutex stream_lock; /* serialize {start/stop}_streaming cb between capture devices */
struct rkisp1_debug debug;
};
/*
- * struct rkisp1_isp_mbus_info - ISP pad format info
- *
- * Translate mbus_code to hardware format values
+ * struct rkisp1_isp_mbus_info - ISP media bus info, Translates media bus code to hardware
+ * format values
*
- * @bus_width: used for parallel
+ * @mbus_code: media bus code
+ * @pixel_enc: pixel encoding
+ * @mipi_dt: mipi data type
+ * @yuv_seq: the order of the Y, Cb, Cr values
+ * @bus_width: bus width
+ * @bayer_pat: bayer pattern
+ * @direction: a bitmask of the flags indicating on which pad the format is supported on
*/
struct rkisp1_isp_mbus_info {
u32 mbus_code;
return readl(rkisp1->base_addr + addr);
}
+/*
+ * rkisp1_cap_enum_mbus_codes - A helper function that return the i'th supported mbus code
+ * of the capture entity. This is used to enumerate the supported
+ * mbus codes on the source pad of the resizer.
+ *
+ * @cap: the capture entity
+ * @code: the mbus code, the function reads the code->index and fills the code->code
+ */
+int rkisp1_cap_enum_mbus_codes(struct rkisp1_capture *cap,
+ struct v4l2_subdev_mbus_code_enum *code);
+
+/*
+ * rkisp1_sd_adjust_crop_rect - adjust a rectangle to fit into another rectangle.
+ *
+ * @crop: rectangle to adjust.
+ * @bounds: rectangle used as bounds.
+ */
void rkisp1_sd_adjust_crop_rect(struct v4l2_rect *crop,
const struct v4l2_rect *bounds);
+/*
+ * rkisp1_sd_adjust_crop - adjust a rectangle to fit into media bus format
+ *
+ * @crop: rectangle to adjust.
+ * @bounds: media bus format used as bounds.
+ */
void rkisp1_sd_adjust_crop(struct v4l2_rect *crop,
const struct v4l2_mbus_framefmt *bounds);
-int rkisp1_isp_register(struct rkisp1_device *rkisp1,
- struct v4l2_device *v4l2_dev);
-void rkisp1_isp_unregister(struct rkisp1_device *rkisp1);
-
+/*
+ * rkisp1_isp_mbus_info - get the isp info of the media bus code
+ *
+ * @mbus_code: the media bus code
+ */
const struct rkisp1_isp_mbus_info *rkisp1_isp_mbus_info_get(u32 mbus_code);
+/* rkisp1_params_configure - configure the params when stream starts.
+ * This function is called by the isp entity upon stream starts.
+ * The function applies the initial configuration of the parameters.
+ *
+ * @params: pointer to rkisp1_params.
+ * @bayer_pat: the bayer pattern on the isp video sink pad
+ * @quantization: the quantization configured on the isp's src pad
+ */
+void rkisp1_params_configure(struct rkisp1_params *params,
+ enum rkisp1_fmt_raw_pat_type bayer_pat,
+ enum v4l2_quantization quantization);
+
+/* rkisp1_params_disable - disable all parameters.
+ * This function is called by the isp entity upon stream start
+ * when capturing bayer format.
+ *
+ * @params: pointer to rkisp1_params.
+ */
+void rkisp1_params_disable(struct rkisp1_params *params);
+
+/* irq handlers */
void rkisp1_isp_isr(struct rkisp1_device *rkisp1);
void rkisp1_mipi_isr(struct rkisp1_device *rkisp1);
void rkisp1_capture_isr(struct rkisp1_device *rkisp1);
void rkisp1_stats_isr(struct rkisp1_stats *stats, u32 isp_ris);
-void rkisp1_params_isr(struct rkisp1_device *rkisp1, u32 isp_mis);
+void rkisp1_params_isr(struct rkisp1_device *rkisp1);
+/* register/unregisters functions of the entities */
int rkisp1_capture_devs_register(struct rkisp1_device *rkisp1);
void rkisp1_capture_devs_unregister(struct rkisp1_device *rkisp1);
+int rkisp1_isp_register(struct rkisp1_device *rkisp1);
+void rkisp1_isp_unregister(struct rkisp1_device *rkisp1);
+
int rkisp1_resizer_devs_register(struct rkisp1_device *rkisp1);
void rkisp1_resizer_devs_unregister(struct rkisp1_device *rkisp1);
-int rkisp1_stats_register(struct rkisp1_stats *stats,
- struct v4l2_device *v4l2_dev,
- struct rkisp1_device *rkisp1);
-void rkisp1_stats_unregister(struct rkisp1_stats *stats);
-
-void rkisp1_params_configure(struct rkisp1_params *params,
- enum rkisp1_fmt_raw_pat_type bayer_pat,
- enum v4l2_quantization quantization);
-void rkisp1_params_disable(struct rkisp1_params *params);
-int rkisp1_params_register(struct rkisp1_params *params,
- struct v4l2_device *v4l2_dev,
- struct rkisp1_device *rkisp1);
-void rkisp1_params_unregister(struct rkisp1_params *params);
+int rkisp1_stats_register(struct rkisp1_device *rkisp1);
+void rkisp1_stats_unregister(struct rkisp1_device *rkisp1);
-void rkisp1_params_isr_handler(struct rkisp1_device *rkisp1, u32 isp_mis);
+int rkisp1_params_register(struct rkisp1_device *rkisp1);
+void rkisp1_params_unregister(struct rkisp1_device *rkisp1);
#endif /* _RKISP1_COMMON_H */
{
int ret;
- ret = rkisp1_isp_register(rkisp1, &rkisp1->v4l2_dev);
+ ret = rkisp1_isp_register(rkisp1);
if (ret)
return ret;
if (ret)
goto err_unreg_resizer_devs;
- ret = rkisp1_stats_register(&rkisp1->stats, &rkisp1->v4l2_dev, rkisp1);
+ ret = rkisp1_stats_register(rkisp1);
if (ret)
goto err_unreg_capture_devs;
- ret = rkisp1_params_register(&rkisp1->params,
- &rkisp1->v4l2_dev, rkisp1);
+ ret = rkisp1_params_register(rkisp1);
if (ret)
goto err_unreg_stats;
return 0;
err_unreg_params:
- rkisp1_params_unregister(&rkisp1->params);
+ rkisp1_params_unregister(rkisp1);
err_unreg_stats:
- rkisp1_stats_unregister(&rkisp1->stats);
+ rkisp1_stats_unregister(rkisp1);
err_unreg_capture_devs:
rkisp1_capture_devs_unregister(rkisp1);
err_unreg_resizer_devs:
&debug->img_stabilization_size_error);
debugfs_create_ulong("inform_size_error", 0444, debug->debugfs_dir,
&debug->inform_size_error);
+ debugfs_create_ulong("irq_delay", 0444, debug->debugfs_dir,
+ &debug->irq_delay);
debugfs_create_ulong("mipi_error", 0444, debug->debugfs_dir,
&debug->mipi_error);
debugfs_create_ulong("stats_error", 0444, debug->debugfs_dir,
v4l2_async_notifier_unregister(&rkisp1->notifier);
v4l2_async_notifier_cleanup(&rkisp1->notifier);
- rkisp1_params_unregister(&rkisp1->params);
- rkisp1_stats_unregister(&rkisp1->stats);
+ rkisp1_params_unregister(rkisp1);
+ rkisp1_stats_unregister(rkisp1);
rkisp1_capture_devs_unregister(rkisp1);
rkisp1_resizer_devs_unregister(rkisp1);
rkisp1_isp_unregister(rkisp1);
rkisp1_write(rkisp1, sink_crop->height, RKISP1_CIF_ISP_OUT_V_SIZE);
irq_mask |= RKISP1_CIF_ISP_FRAME | RKISP1_CIF_ISP_V_START |
- RKISP1_CIF_ISP_PIC_SIZE_ERROR | RKISP1_CIF_ISP_FRAME_IN;
+ RKISP1_CIF_ISP_PIC_SIZE_ERROR;
rkisp1_write(rkisp1, irq_mask, RKISP1_CIF_ISP_IMSC);
if (src_fmt->pixel_enc == V4L2_PIXEL_ENC_BAYER) {
if (code->index == pos - 1) {
code->code = fmt->mbus_code;
+ if (fmt->pixel_enc == V4L2_PIXEL_ENC_YUV &&
+ dir == RKISP1_ISP_SD_SRC)
+ code->flags =
+ V4L2_SUBDEV_MBUS_CODE_CSC_QUANTIZATION;
return 0;
}
}
RKISP1_ISP_PAD_SOURCE_VIDEO);
*src_fmt = *sink_fmt;
src_fmt->code = RKISP1_DEF_SRC_PAD_FMT;
- src_fmt->quantization = V4L2_QUANTIZATION_FULL_RANGE;
src_crop = v4l2_subdev_get_try_crop(sd, cfg,
RKISP1_ISP_PAD_SOURCE_VIDEO);
isp->src_fmt = mbus_info;
src_fmt->width = src_crop->width;
src_fmt->height = src_crop->height;
- src_fmt->quantization = format->quantization;
- /* full range by default */
- if (!src_fmt->quantization)
+
+ /*
+ * The CSC API is used to allow userspace to force full
+ * quantization on YUV formats.
+ */
+ if (format->flags & V4L2_MBUS_FRAMEFMT_SET_CSC &&
+ format->quantization == V4L2_QUANTIZATION_FULL_RANGE &&
+ mbus_info->pixel_enc == V4L2_PIXEL_ENC_YUV)
+ src_fmt->quantization = V4L2_QUANTIZATION_FULL_RANGE;
+ else if (mbus_info->pixel_enc == V4L2_PIXEL_ENC_YUV)
+ src_fmt->quantization = V4L2_QUANTIZATION_LIM_RANGE;
+ else
src_fmt->quantization = V4L2_QUANTIZATION_FULL_RANGE;
*format = *src_fmt;
if (rkisp1->active_sensor->mbus_type != V4L2_MBUS_CSI2_DPHY)
return -EINVAL;
- atomic_set(&rkisp1->isp.frame_sequence, -1);
+ rkisp1->isp.frame_sequence = -1;
mutex_lock(&isp->ops_lock);
ret = rkisp1_config_cif(rkisp1);
if (ret)
.pad = &rkisp1_isp_pad_ops,
};
-int rkisp1_isp_register(struct rkisp1_device *rkisp1,
- struct v4l2_device *v4l2_dev)
+int rkisp1_isp_register(struct rkisp1_device *rkisp1)
{
struct rkisp1_isp *isp = &rkisp1->isp;
struct media_pad *pads = isp->pads;
if (ret)
return ret;
- ret = v4l2_device_register_subdev(v4l2_dev, sd);
+ ret = v4l2_device_register_subdev(&rkisp1->v4l2_dev, sd);
if (ret) {
dev_err(rkisp1->dev, "Failed to register isp subdev\n");
goto err_cleanup_media_entity;
struct v4l2_event event = {
.type = V4L2_EVENT_FRAME_SYNC,
};
+ event.u.frame_sync.frame_sequence = isp->frame_sequence;
- /*
- * Increment the frame sequence on the vsync signal.
- * This will allow applications to detect dropped.
- * Note that there is a debugfs counter for dropped
- * frames, but using this event is more accurate.
- */
- event.u.frame_sync.frame_sequence =
- atomic_inc_return(&isp->frame_sequence);
v4l2_event_queue(isp->sd.devnode, &event);
}
rkisp1_write(rkisp1, status, RKISP1_CIF_ISP_ICR);
/* Vertical sync signal, starting generating new frame */
- if (status & RKISP1_CIF_ISP_V_START)
+ if (status & RKISP1_CIF_ISP_V_START) {
+ rkisp1->isp.frame_sequence++;
rkisp1_isp_queue_event_sof(&rkisp1->isp);
-
+ if (status & RKISP1_CIF_ISP_FRAME) {
+ WARN_ONCE(1, "irq delay is too long, buffers might not be in sync\n");
+ rkisp1->debug.irq_delay++;
+ }
+ }
if (status & RKISP1_CIF_ISP_PIC_SIZE_ERROR) {
/* Clear pic_size_error */
isp_err = rkisp1_read(rkisp1, RKISP1_CIF_ISP_ERR);
isp_ris = rkisp1_read(rkisp1, RKISP1_CIF_ISP_RIS);
if (isp_ris & RKISP1_STATS_MEAS_MASK)
rkisp1_stats_isr(&rkisp1->stats, isp_ris);
+ /*
+ * Then update changed configs. Some of them involve
+ * lot of register writes. Do those only one per frame.
+ * Do the updates in the order of the processing flow.
+ */
+ rkisp1_params_isr(rkisp1);
}
- /*
- * Then update changed configs. Some of them involve
- * lot of register writes. Do those only one per frame.
- * Do the updates in the order of the processing flow.
- */
- rkisp1_params_isr(rkisp1, status);
}
RKISP1_CIF_ISP_LSC_B_TABLE_ADDR);
/* program data tables (table size is 9 * 17 = 153) */
- for (i = 0;
- i < RKISP1_CIF_ISP_LSC_SECTORS_MAX * RKISP1_CIF_ISP_LSC_SECTORS_MAX;
- i += RKISP1_CIF_ISP_LSC_SECTORS_MAX) {
+ for (i = 0; i < RKISP1_CIF_ISP_LSC_SAMPLES_MAX; i++) {
/*
* 17 sectors with 2 values in one DWORD = 9
* DWORDs (2nd value of last DWORD unused)
*/
- for (j = 0; j < RKISP1_CIF_ISP_LSC_SECTORS_MAX - 1; j += 2) {
- data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->r_data_tbl[i + j],
- pconfig->r_data_tbl[i + j + 1]);
+ for (j = 0; j < RKISP1_CIF_ISP_LSC_SAMPLES_MAX - 1; j += 2) {
+ data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->r_data_tbl[i][j],
+ pconfig->r_data_tbl[i][j + 1]);
rkisp1_write(params->rkisp1, data,
RKISP1_CIF_ISP_LSC_R_TABLE_DATA);
- data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gr_data_tbl[i + j],
- pconfig->gr_data_tbl[i + j + 1]);
+ data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gr_data_tbl[i][j],
+ pconfig->gr_data_tbl[i][j + 1]);
rkisp1_write(params->rkisp1, data,
RKISP1_CIF_ISP_LSC_GR_TABLE_DATA);
- data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gb_data_tbl[i + j],
- pconfig->gb_data_tbl[i + j + 1]);
+ data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gb_data_tbl[i][j],
+ pconfig->gb_data_tbl[i][j + 1]);
rkisp1_write(params->rkisp1, data,
RKISP1_CIF_ISP_LSC_GB_TABLE_DATA);
- data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->b_data_tbl[i + j],
- pconfig->b_data_tbl[i + j + 1]);
+ data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->b_data_tbl[i][j],
+ pconfig->b_data_tbl[i][j + 1]);
rkisp1_write(params->rkisp1, data,
RKISP1_CIF_ISP_LSC_B_TABLE_DATA);
}
- data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->r_data_tbl[i + j], 0);
+ data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->r_data_tbl[i][j], 0);
rkisp1_write(params->rkisp1, data,
RKISP1_CIF_ISP_LSC_R_TABLE_DATA);
- data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gr_data_tbl[i + j], 0);
+ data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gr_data_tbl[i][j], 0);
rkisp1_write(params->rkisp1, data,
RKISP1_CIF_ISP_LSC_GR_TABLE_DATA);
- data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gb_data_tbl[i + j], 0);
+ data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->gb_data_tbl[i][j], 0);
rkisp1_write(params->rkisp1, data,
RKISP1_CIF_ISP_LSC_GB_TABLE_DATA);
- data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->b_data_tbl[i + j], 0);
+ data = RKISP1_CIF_ISP_LSC_TABLE_DATA(pconfig->b_data_tbl[i][j], 0);
rkisp1_write(params->rkisp1, data,
RKISP1_CIF_ISP_LSC_B_TABLE_DATA);
}
RKISP1_CIF_ISP_LSC_CTRL_ENA);
rkisp1_lsc_correct_matrix_config(params, arg);
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < RKISP1_CIF_ISP_LSC_SECTORS_TBL_SIZE / 2; i++) {
/* program x size tables */
data = RKISP1_CIF_ISP_LSC_SECT_SIZE(arg->x_size_tbl[i * 2],
arg->x_size_tbl[i * 2 + 1]);
static void rkisp1_ctk_config(struct rkisp1_params *params,
const struct rkisp1_cif_isp_ctk_config *arg)
{
- rkisp1_write(params->rkisp1, arg->coeff0, RKISP1_CIF_ISP_CT_COEFF_0);
- rkisp1_write(params->rkisp1, arg->coeff1, RKISP1_CIF_ISP_CT_COEFF_1);
- rkisp1_write(params->rkisp1, arg->coeff2, RKISP1_CIF_ISP_CT_COEFF_2);
- rkisp1_write(params->rkisp1, arg->coeff3, RKISP1_CIF_ISP_CT_COEFF_3);
- rkisp1_write(params->rkisp1, arg->coeff4, RKISP1_CIF_ISP_CT_COEFF_4);
- rkisp1_write(params->rkisp1, arg->coeff5, RKISP1_CIF_ISP_CT_COEFF_5);
- rkisp1_write(params->rkisp1, arg->coeff6, RKISP1_CIF_ISP_CT_COEFF_6);
- rkisp1_write(params->rkisp1, arg->coeff7, RKISP1_CIF_ISP_CT_COEFF_7);
- rkisp1_write(params->rkisp1, arg->coeff8, RKISP1_CIF_ISP_CT_COEFF_8);
- rkisp1_write(params->rkisp1, arg->ct_offset_r,
- RKISP1_CIF_ISP_CT_OFFSET_R);
- rkisp1_write(params->rkisp1, arg->ct_offset_g,
- RKISP1_CIF_ISP_CT_OFFSET_G);
- rkisp1_write(params->rkisp1, arg->ct_offset_b,
- RKISP1_CIF_ISP_CT_OFFSET_B);
+ unsigned int i, j, k = 0;
+
+ for (i = 0; i < 3; i++)
+ for (j = 0; j < 3; j++)
+ rkisp1_write(params->rkisp1, arg->coeff[i][j],
+ RKISP1_CIF_ISP_CT_COEFF_0 + 4 * k++);
+ for (i = 0; i < 3; i++)
+ rkisp1_write(params->rkisp1, arg->ct_offset[i],
+ RKISP1_CIF_ISP_CT_OFFSET_R + i * 4);
}
static void rkisp1_ctk_enable(struct rkisp1_params *params, bool en)
const struct rkisp1_cif_isp_cproc_config *arg)
{
struct rkisp1_cif_isp_isp_other_cfg *cur_other_cfg =
- ¶ms->cur_params.others;
+ container_of(arg, struct rkisp1_cif_isp_isp_other_cfg, cproc_config);
struct rkisp1_cif_isp_ie_config *cur_ie_config =
&cur_other_cfg->ie_config;
u32 effect = cur_ie_config->effect;
}
}
-void rkisp1_params_isr(struct rkisp1_device *rkisp1, u32 isp_mis)
+static void rkisp1_params_apply_params_cfg(struct rkisp1_params *params,
+ unsigned int frame_sequence)
{
- unsigned int frame_sequence = atomic_read(&rkisp1->isp.frame_sequence);
- struct rkisp1_params *params = &rkisp1->params;
struct rkisp1_params_cfg *new_params;
struct rkisp1_buffer *cur_buf = NULL;
- spin_lock(¶ms->config_lock);
- if (!params->is_streaming) {
- spin_unlock(¶ms->config_lock);
+ if (list_empty(¶ms->params))
return;
- }
- /* get one empty buffer */
- if (!list_empty(¶ms->params))
- cur_buf = list_first_entry(¶ms->params,
- struct rkisp1_buffer, queue);
- spin_unlock(¶ms->config_lock);
+ cur_buf = list_first_entry(¶ms->params,
+ struct rkisp1_buffer, queue);
- if (!cur_buf)
- return;
+ new_params = (struct rkisp1_params_cfg *)(cur_buf->vaddr);
- new_params = (struct rkisp1_params_cfg *)(cur_buf->vaddr[0]);
+ rkisp1_isp_isr_other_config(params, new_params);
+ rkisp1_isp_isr_meas_config(params, new_params);
- if (isp_mis & RKISP1_CIF_ISP_FRAME) {
- u32 isp_ctrl;
+ /* update shadow register immediately */
+ rkisp1_param_set_bits(params, RKISP1_CIF_ISP_CTRL, RKISP1_CIF_ISP_CTRL_ISP_CFG_UPD);
- rkisp1_isp_isr_other_config(params, new_params);
- rkisp1_isp_isr_meas_config(params, new_params);
+ list_del(&cur_buf->queue);
- /* update shadow register immediately */
- isp_ctrl = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_CTRL);
- isp_ctrl |= RKISP1_CIF_ISP_CTRL_ISP_CFG_UPD;
- rkisp1_write(params->rkisp1, isp_ctrl, RKISP1_CIF_ISP_CTRL);
+ cur_buf->vb.sequence = frame_sequence;
+ vb2_buffer_done(&cur_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
+}
- spin_lock(¶ms->config_lock);
- list_del(&cur_buf->queue);
- spin_unlock(¶ms->config_lock);
+void rkisp1_params_isr(struct rkisp1_device *rkisp1)
+{
+ /*
+ * This isr is called when the ISR finishes processing a frame (RKISP1_CIF_ISP_FRAME).
+ * Configurations performed here will be applied on the next frame.
+ * Since frame_sequence is updated on the vertical sync signal, we should use
+ * frame_sequence + 1 here to indicate to userspace on which frame these parameters
+ * are being applied.
+ */
+ unsigned int frame_sequence = rkisp1->isp.frame_sequence + 1;
+ struct rkisp1_params *params = &rkisp1->params;
- cur_buf->vb.sequence = frame_sequence;
- vb2_buffer_done(&cur_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
+ spin_lock(¶ms->config_lock);
+ if (!params->is_streaming) {
+ spin_unlock(¶ms->config_lock);
+ return;
}
+ rkisp1_params_apply_params_cfg(params, frame_sequence);
+
+ spin_unlock(¶ms->config_lock);
}
static const struct rkisp1_cif_isp_awb_meas_config rkisp1_awb_params_default_config = {
{
struct rkisp1_cif_isp_hst_config hst = rkisp1_hst_params_default_config;
- spin_lock(¶ms->config_lock);
-
rkisp1_awb_meas_config(params, &rkisp1_awb_params_default_config);
rkisp1_awb_meas_enable(params, &rkisp1_awb_params_default_config,
true);
else
rkisp1_csm_config(params, false);
- /* override the default things */
- rkisp1_isp_isr_other_config(params, ¶ms->cur_params);
- rkisp1_isp_isr_meas_config(params, ¶ms->cur_params);
+ spin_lock_irq(¶ms->config_lock);
- spin_unlock(¶ms->config_lock);
+ /* apply the first buffer if there is one already */
+ if (params->is_streaming)
+ rkisp1_params_apply_params_cfg(params, 0);
+
+ spin_unlock_irq(¶ms->config_lock);
}
-/* Not called when the camera active, thus not isr protection. */
void rkisp1_params_configure(struct rkisp1_params *params,
enum rkisp1_fmt_raw_pat_type bayer_pat,
enum v4l2_quantization quantization)
sizes[0] = sizeof(struct rkisp1_params_cfg);
INIT_LIST_HEAD(¶ms->params);
- params->is_first_params = true;
-
return 0;
}
container_of(vbuf, struct rkisp1_buffer, vb);
struct vb2_queue *vq = vb->vb2_queue;
struct rkisp1_params *params = vq->drv_priv;
- struct rkisp1_params_cfg *new_params;
- unsigned long flags;
- unsigned int frame_sequence =
- atomic_read(¶ms->rkisp1->isp.frame_sequence);
-
- if (params->is_first_params) {
- new_params = (struct rkisp1_params_cfg *)
- (vb2_plane_vaddr(vb, 0));
- vbuf->sequence = frame_sequence;
- vb2_buffer_done(¶ms_buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
- params->is_first_params = false;
- params->cur_params = *new_params;
- return;
- }
- params_buf->vaddr[0] = vb2_plane_vaddr(vb, 0);
- spin_lock_irqsave(¶ms->config_lock, flags);
+ params_buf->vaddr = vb2_plane_vaddr(vb, 0);
+ spin_lock_irq(¶ms->config_lock);
list_add_tail(¶ms_buf->queue, ¶ms->params);
- spin_unlock_irqrestore(¶ms->config_lock, flags);
+ spin_unlock_irq(¶ms->config_lock);
}
static int rkisp1_params_vb2_buf_prepare(struct vb2_buffer *vb)
{
struct rkisp1_params *params = vq->drv_priv;
struct rkisp1_buffer *buf;
- unsigned long flags;
- unsigned int i;
+ struct list_head tmp_list;
+
+ INIT_LIST_HEAD(&tmp_list);
- /* stop params input firstly */
- spin_lock_irqsave(¶ms->config_lock, flags);
+ /*
+ * we first move the buffers into a local list 'tmp_list'
+ * and then we can iterate it and call vb2_buffer_done
+ * without holding the lock
+ */
+ spin_lock_irq(¶ms->config_lock);
params->is_streaming = false;
- spin_unlock_irqrestore(¶ms->config_lock, flags);
-
- for (i = 0; i < RKISP1_ISP_PARAMS_REQ_BUFS_MAX; i++) {
- spin_lock_irqsave(¶ms->config_lock, flags);
- if (!list_empty(¶ms->params)) {
- buf = list_first_entry(¶ms->params,
- struct rkisp1_buffer, queue);
- list_del(&buf->queue);
- spin_unlock_irqrestore(¶ms->config_lock,
- flags);
- } else {
- spin_unlock_irqrestore(¶ms->config_lock,
- flags);
- break;
- }
+ list_cut_position(&tmp_list, ¶ms->params, params->params.prev);
+ spin_unlock_irq(¶ms->config_lock);
- if (buf)
- vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
- buf = NULL;
- }
+ list_for_each_entry(buf, &tmp_list, queue)
+ vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
}
static int
rkisp1_params_vb2_start_streaming(struct vb2_queue *queue, unsigned int count)
{
struct rkisp1_params *params = queue->drv_priv;
- unsigned long flags;
- spin_lock_irqsave(¶ms->config_lock, flags);
+ spin_lock_irq(¶ms->config_lock);
params->is_streaming = true;
- spin_unlock_irqrestore(¶ms->config_lock, flags);
+ spin_unlock_irq(¶ms->config_lock);
return 0;
}
sizeof(struct rkisp1_params_cfg);
}
-int rkisp1_params_register(struct rkisp1_params *params,
- struct v4l2_device *v4l2_dev,
- struct rkisp1_device *rkisp1)
+int rkisp1_params_register(struct rkisp1_device *rkisp1)
{
+ struct rkisp1_params *params = &rkisp1->params;
struct rkisp1_vdev_node *node = ¶ms->vnode;
struct video_device *vdev = &node->vdev;
int ret;
* to protect all fops and v4l2 ioctls.
*/
vdev->lock = &node->vlock;
- vdev->v4l2_dev = v4l2_dev;
+ vdev->v4l2_dev = &rkisp1->v4l2_dev;
vdev->queue = &node->buf_queue;
vdev->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_META_OUTPUT;
vdev->vfl_dir = VFL_DIR_TX;
node->pad.flags = MEDIA_PAD_FL_SOURCE;
ret = media_entity_pads_init(&vdev->entity, 1, &node->pad);
if (ret)
- goto err_release_queue;
+ return ret;
ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
if (ret) {
dev_err(rkisp1->dev,
return 0;
err_cleanup_media_entity:
media_entity_cleanup(&vdev->entity);
-err_release_queue:
- vb2_queue_release(vdev->queue);
return ret;
}
-void rkisp1_params_unregister(struct rkisp1_params *params)
+void rkisp1_params_unregister(struct rkisp1_device *rkisp1)
{
+ struct rkisp1_params *params = &rkisp1->params;
struct rkisp1_vdev_node *node = ¶ms->vnode;
struct video_device *vdev = &node->vdev;
- video_unregister_device(vdev);
+ vb2_video_unregister_device(vdev);
media_entity_cleanup(&vdev->entity);
- vb2_queue_release(vdev->queue);
}
#define RKISP1_CIF_ISP_LSC_SECT_SIZE_RESERVED 0xFC00FC00
#define RKISP1_CIF_ISP_LSC_GRAD_RESERVED 0xF000F000
#define RKISP1_CIF_ISP_LSC_SAMPLE_RESERVED 0xF000F000
-#define RKISP1_CIF_ISP_LSC_SECTORS_MAX 17
#define RKISP1_CIF_ISP_LSC_TABLE_DATA(v0, v1) \
(((v0) & 0xFFF) | (((v1) & 0xFFF) << 12))
#define RKISP1_CIF_ISP_LSC_SECT_SIZE(v0, v1) \
#define RKISP1_DEF_FMT MEDIA_BUS_FMT_YUYV8_2X8
#define RKISP1_DEF_PIXEL_ENC V4L2_PIXEL_ENC_YUV
-#define RKISP1_MBUS_FMT_HDIV 2
-#define RKISP1_MBUS_FMT_VDIV 1
+struct rkisp1_rsz_yuv_mbus_info {
+ u32 mbus_code;
+ u32 hdiv;
+ u32 vdiv;
+};
+
+static const struct rkisp1_rsz_yuv_mbus_info rkisp1_rsz_yuv_src_formats[] = {
+ {
+ .mbus_code = MEDIA_BUS_FMT_YUYV8_2X8, /* YUV422 */
+ .hdiv = 2,
+ .vdiv = 1,
+ },
+ {
+ .mbus_code = MEDIA_BUS_FMT_YUYV8_1_5X8, /* YUV420 */
+ .hdiv = 2,
+ .vdiv = 2,
+ },
+};
+
+static const struct rkisp1_rsz_yuv_mbus_info *rkisp1_rsz_get_yuv_mbus_info(u32 mbus_code)
+{
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(rkisp1_rsz_yuv_src_formats); i++) {
+ if (rkisp1_rsz_yuv_src_formats[i].mbus_code == mbus_code)
+ return &rkisp1_rsz_yuv_src_formats[i];
+ }
+
+ return NULL;
+}
enum rkisp1_shadow_regs_when {
RKISP1_SHADOW_REGS_SYNC,
static void rkisp1_rsz_config(struct rkisp1_resizer *rsz,
enum rkisp1_shadow_regs_when when)
{
- u8 hdiv = RKISP1_MBUS_FMT_HDIV, vdiv = RKISP1_MBUS_FMT_VDIV;
+ const struct rkisp1_rsz_yuv_mbus_info *sink_yuv_info, *src_yuv_info;
struct v4l2_rect sink_y, sink_c, src_y, src_c;
- struct v4l2_mbus_framefmt *src_fmt;
+ struct v4l2_mbus_framefmt *src_fmt, *sink_fmt;
struct v4l2_rect *sink_crop;
- struct rkisp1_capture *cap = &rsz->rkisp1->capture_devs[rsz->id];
sink_crop = rkisp1_rsz_get_pad_crop(rsz, NULL, RKISP1_RSZ_PAD_SINK,
V4L2_SUBDEV_FORMAT_ACTIVE);
src_fmt = rkisp1_rsz_get_pad_fmt(rsz, NULL, RKISP1_RSZ_PAD_SRC,
V4L2_SUBDEV_FORMAT_ACTIVE);
+ src_yuv_info = rkisp1_rsz_get_yuv_mbus_info(src_fmt->code);
+ sink_fmt = rkisp1_rsz_get_pad_fmt(rsz, NULL, RKISP1_RSZ_PAD_SINK,
+ V4L2_SUBDEV_FORMAT_ACTIVE);
+ sink_yuv_info = rkisp1_rsz_get_yuv_mbus_info(sink_fmt->code);
/*
* The resizer only works on yuv formats,
src_y.width = src_fmt->width;
src_y.height = src_fmt->height;
- sink_c.width = sink_y.width / RKISP1_MBUS_FMT_HDIV;
- sink_c.height = sink_y.height / RKISP1_MBUS_FMT_VDIV;
+ sink_c.width = sink_y.width / sink_yuv_info->hdiv;
+ sink_c.height = sink_y.height / sink_yuv_info->vdiv;
/*
* The resizer is used not only to change the dimensions of the frame
* but also to change the scale for YUV formats,
* (4:2:2 -> 4:2:0 for example). So the width/height of the CbCr
- * streams should be set according to the pixel format in the capture.
- * The resizer always gets the input as YUV422. If the capture format
- * is RGB then the memory input should be YUV422 so we don't change the
- * default hdiv, vdiv in that case.
+ * streams should be set according to the media bus format in the src pad.
*/
- if (v4l2_is_format_yuv(cap->pix.info)) {
- hdiv = cap->pix.info->hdiv;
- vdiv = cap->pix.info->vdiv;
- }
-
- src_c.width = src_y.width / hdiv;
- src_c.height = src_y.height / vdiv;
+ src_c.width = src_y.width / src_yuv_info->hdiv;
+ src_c.height = src_y.height / src_yuv_info->vdiv;
if (sink_c.width == src_c.width && sink_c.height == src_c.height) {
rkisp1_rsz_disable(rsz, when);
u32 pad = code->pad;
int ret;
- /* supported mbus codes are the same in isp video src pad */
+ if (code->pad == RKISP1_RSZ_PAD_SRC) {
+ /* supported mbus codes on the src are the same as in the capture */
+ struct rkisp1_capture *cap = &rsz->rkisp1->capture_devs[rsz->id];
+
+ return rkisp1_cap_enum_mbus_codes(cap, code);
+ }
+
+ /*
+ * The selfpath capture doesn't support bayer formats. Therefore the selfpath resizer
+ * should support only YUV422 on the sink pad
+ */
+ if (rsz->id == RKISP1_SELFPATH) {
+ if (code->index > 0)
+ return -EINVAL;
+ code->code = MEDIA_BUS_FMT_YUYV8_2X8;
+ return 0;
+ }
+
+ /* supported mbus codes on the sink pad are the same as isp src pad */
code->pad = RKISP1_ISP_PAD_SOURCE_VIDEO;
ret = v4l2_subdev_call(&rsz->rkisp1->isp.sd, pad, enum_mbus_code,
&dummy_cfg, code);
/* restore pad */
code->pad = pad;
+ code->flags = 0;
return ret;
}
struct v4l2_mbus_framefmt *format,
unsigned int which)
{
+ const struct rkisp1_isp_mbus_info *mbus_info;
struct v4l2_mbus_framefmt *src_fmt;
src_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SRC, which);
+ mbus_info = rkisp1_isp_mbus_info_get(src_fmt->code);
+
+ /* for YUV formats, userspace can change the mbus code on the src pad if it is supported */
+ if (mbus_info->pixel_enc == V4L2_PIXEL_ENC_YUV &&
+ rkisp1_rsz_get_yuv_mbus_info(format->code))
+ src_fmt->code = format->code;
+
src_fmt->width = clamp_t(u32, format->width,
rsz->config->min_rsz_width,
rsz->config->max_rsz_width);
src_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SRC, which);
sink_crop = rkisp1_rsz_get_pad_crop(rsz, cfg, RKISP1_RSZ_PAD_SINK,
which);
- sink_fmt->code = format->code;
+ if (rsz->id == RKISP1_SELFPATH)
+ sink_fmt->code = MEDIA_BUS_FMT_YUYV8_2X8;
+ else
+ sink_fmt->code = format->code;
+
mbus_info = rkisp1_isp_mbus_info_get(sink_fmt->code);
if (!mbus_info || !(mbus_info->direction & RKISP1_ISP_SD_SRC)) {
sink_fmt->code = RKISP1_DEF_FMT;
struct vb2_queue *vq = vb->vb2_queue;
struct rkisp1_stats *stats_dev = vq->drv_priv;
- stats_buf->vaddr[0] = vb2_plane_vaddr(vb, 0);
+ stats_buf->vaddr = vb2_plane_vaddr(vb, 0);
spin_lock_irq(&stats_dev->lock);
list_add_tail(&stats_buf->queue, &stats_dev->stat);
{
struct rkisp1_stats *stats = queue->drv_priv;
+ spin_lock_irq(&stats->lock);
stats->is_streaming = true;
+ spin_unlock_irq(&stats->lock);
return 0;
}
struct rkisp1_device *rkisp1 = stats->rkisp1;
struct rkisp1_cif_isp_af_stat *af;
- pbuf->meas_type |= RKISP1_CIF_ISP_STAT_AFM_FIN;
+ pbuf->meas_type |= RKISP1_CIF_ISP_STAT_AFM;
af = &pbuf->params.af;
af->window[0].sum = rkisp1_read(rkisp1, RKISP1_CIF_ISP_AFM_SUM_A);
{
struct rkisp1_stat_buffer *cur_stat_buf;
struct rkisp1_buffer *cur_buf = NULL;
- unsigned int frame_sequence =
- atomic_read(&stats->rkisp1->isp.frame_sequence);
+ unsigned int frame_sequence = stats->rkisp1->isp.frame_sequence;
u64 timestamp = ktime_get_ns();
/* get one empty buffer */
return;
cur_stat_buf =
- (struct rkisp1_stat_buffer *)(cur_buf->vaddr[0]);
+ (struct rkisp1_stat_buffer *)(cur_buf->vaddr);
if (isp_ris & RKISP1_CIF_ISP_AWB_DONE)
rkisp1_stats_get_awb_meas(stats, cur_stat_buf);
sizeof(struct rkisp1_stat_buffer);
}
-int rkisp1_stats_register(struct rkisp1_stats *stats,
- struct v4l2_device *v4l2_dev,
- struct rkisp1_device *rkisp1)
+int rkisp1_stats_register(struct rkisp1_device *rkisp1)
{
+ struct rkisp1_stats *stats = &rkisp1->stats;
struct rkisp1_vdev_node *node = &stats->vnode;
struct video_device *vdev = &node->vdev;
int ret;
vdev->fops = &rkisp1_stats_fops;
vdev->release = video_device_release_empty;
vdev->lock = &node->vlock;
- vdev->v4l2_dev = v4l2_dev;
+ vdev->v4l2_dev = &rkisp1->v4l2_dev;
vdev->queue = &node->buf_queue;
vdev->device_caps = V4L2_CAP_META_CAPTURE | V4L2_CAP_STREAMING;
vdev->vfl_dir = VFL_DIR_RX;
node->pad.flags = MEDIA_PAD_FL_SINK;
ret = media_entity_pads_init(&vdev->entity, 1, &node->pad);
if (ret)
- goto err_release_queue;
+ goto err_mutex_destroy;
ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
if (ret) {
err_cleanup_media_entity:
media_entity_cleanup(&vdev->entity);
-err_release_queue:
- vb2_queue_release(vdev->queue);
+err_mutex_destroy:
mutex_destroy(&node->vlock);
return ret;
}
-void rkisp1_stats_unregister(struct rkisp1_stats *stats)
+void rkisp1_stats_unregister(struct rkisp1_device *rkisp1)
{
+ struct rkisp1_stats *stats = &rkisp1->stats;
struct rkisp1_vdev_node *node = &stats->vnode;
struct video_device *vdev = &node->vdev;
- video_unregister_device(vdev);
+ vb2_video_unregister_device(vdev);
media_entity_cleanup(&vdev->entity);
- vb2_queue_release(vdev->queue);
mutex_destroy(&node->vlock);
}
* Copyright (C) 2017 Rockchip Electronics Co., Ltd.
*/
-/*
- * TODO: Improve documentation, mostly regarding abbreviation and hardware
- * specificities. Reference: "REF_01 - ISP_user_manual, Rev 2.57" (not public)
- */
-
#ifndef _UAPI_RKISP1_CONFIG_H
#define _UAPI_RKISP1_CONFIG_H
#define V4L2_META_FMT_RK_ISP1_PARAMS v4l2_fourcc('R', 'K', '1', 'P') /* Rockchip ISP1 params */
#define V4L2_META_FMT_RK_ISP1_STAT_3A v4l2_fourcc('R', 'K', '1', 'S') /* Rockchip ISP1 3A statistics */
-#define RKISP1_CIF_ISP_MODULE_DPCC BIT(0)
-#define RKISP1_CIF_ISP_MODULE_BLS BIT(1)
-#define RKISP1_CIF_ISP_MODULE_SDG BIT(2)
-#define RKISP1_CIF_ISP_MODULE_HST BIT(3)
-#define RKISP1_CIF_ISP_MODULE_LSC BIT(4)
-#define RKISP1_CIF_ISP_MODULE_AWB_GAIN BIT(5)
-#define RKISP1_CIF_ISP_MODULE_FLT BIT(6)
-#define RKISP1_CIF_ISP_MODULE_BDM BIT(7)
-#define RKISP1_CIF_ISP_MODULE_CTK BIT(8)
-#define RKISP1_CIF_ISP_MODULE_GOC BIT(9)
-#define RKISP1_CIF_ISP_MODULE_CPROC BIT(10)
-#define RKISP1_CIF_ISP_MODULE_AFC BIT(11)
-#define RKISP1_CIF_ISP_MODULE_AWB BIT(12)
-#define RKISP1_CIF_ISP_MODULE_IE BIT(13)
-#define RKISP1_CIF_ISP_MODULE_AEC BIT(14)
-#define RKISP1_CIF_ISP_MODULE_WDR BIT(15)
-#define RKISP1_CIF_ISP_MODULE_DPF BIT(16)
-#define RKISP1_CIF_ISP_MODULE_DPF_STRENGTH BIT(17)
+/* Defect Pixel Cluster Detection */
+#define RKISP1_CIF_ISP_MODULE_DPCC (1U << 0)
+/* Black Level Subtraction */
+#define RKISP1_CIF_ISP_MODULE_BLS (1U << 1)
+/* Sensor De-gamma */
+#define RKISP1_CIF_ISP_MODULE_SDG (1U << 2)
+/* Histogram */
+#define RKISP1_CIF_ISP_MODULE_HST (1U << 3)
+/* Lens Shade Control */
+#define RKISP1_CIF_ISP_MODULE_LSC (1U << 4)
+/* Auto White Balance Gain */
+#define RKISP1_CIF_ISP_MODULE_AWB_GAIN (1U << 5)
+/* Filter */
+#define RKISP1_CIF_ISP_MODULE_FLT (1U << 6)
+/* Bayer Demosaic */
+#define RKISP1_CIF_ISP_MODULE_BDM (1U << 7)
+/* Cross Talk */
+#define RKISP1_CIF_ISP_MODULE_CTK (1U << 8)
+/* Gamma Out Curve */
+#define RKISP1_CIF_ISP_MODULE_GOC (1U << 9)
+/* Color Processing */
+#define RKISP1_CIF_ISP_MODULE_CPROC (1U << 10)
+/* Auto Focus Control */
+#define RKISP1_CIF_ISP_MODULE_AFC (1U << 11)
+/* Auto White Balancing */
+#define RKISP1_CIF_ISP_MODULE_AWB (1U << 12)
+/* Image Effect */
+#define RKISP1_CIF_ISP_MODULE_IE (1U << 13)
+/* Auto Exposure Control */
+#define RKISP1_CIF_ISP_MODULE_AEC (1U << 14)
+/* Wide Dynamic Range */
+#define RKISP1_CIF_ISP_MODULE_WDR (1U << 15)
+/* Denoise Pre-Filter */
+#define RKISP1_CIF_ISP_MODULE_DPF (1U << 16)
+/* Denoise Pre-Filter Strength */
+#define RKISP1_CIF_ISP_MODULE_DPF_STRENGTH (1U << 17)
#define RKISP1_CIF_ISP_CTK_COEFF_MAX 0x100
#define RKISP1_CIF_ISP_CTK_OFFSET_MAX 0x800
/*
* Lens shade correction
*/
-#define RKISP1_CIF_ISP_LSC_GRAD_TBL_SIZE 8
-#define RKISP1_CIF_ISP_LSC_SIZE_TBL_SIZE 8
+#define RKISP1_CIF_ISP_LSC_SECTORS_TBL_SIZE 8
+
/*
* The following matches the tuning process,
* not the max capabilities of the chip.
- * Last value unused.
*/
-#define RKISP1_CIF_ISP_LSC_DATA_TBL_SIZE 290
+#define RKISP1_CIF_ISP_LSC_SAMPLES_MAX 17
/*
* Histogram calculation
/*
* Measurement types
*/
-#define RKISP1_CIF_ISP_STAT_AWB BIT(0)
-#define RKISP1_CIF_ISP_STAT_AUTOEXP BIT(1)
-#define RKISP1_CIF_ISP_STAT_AFM_FIN BIT(2)
-#define RKISP1_CIF_ISP_STAT_HIST BIT(3)
+#define RKISP1_CIF_ISP_STAT_AWB (1U << 0)
+#define RKISP1_CIF_ISP_STAT_AUTOEXP (1U << 1)
+#define RKISP1_CIF_ISP_STAT_AFM (1U << 2)
+#define RKISP1_CIF_ISP_STAT_HIST (1U << 3)
enum rkisp1_cif_isp_histogram_mode {
RKISP1_CIF_ISP_HISTOGRAM_MODE_DISABLE,
/*---------- PART1: Input Parameters ------------*/
+/**
+ * struct rkisp1_cif_isp_window - measurement window.
+ *
+ * Measurements are calculated per window inside the frame.
+ * This struct represents a window for a measurement.
+ *
+ * @h_offs: the horizontal offset of the window from the left of the frame in pixels.
+ * @v_offs: the vertical offset of the window from the top of the frame in pixels.
+ * @h_size: the horizontal size of the window in pixels
+ * @v_size: the vertical size of the window in pixels.
+ */
struct rkisp1_cif_isp_window {
__u16 h_offs;
__u16 v_offs;
__u16 h_size;
__u16 v_size;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_bls_fixed_val - BLS fixed subtraction values
__s16 gr;
__s16 gb;
__s16 b;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_bls_config - Configuration used by black level subtraction
struct rkisp1_cif_isp_window bls_window2;
__u8 bls_samples;
struct rkisp1_cif_isp_bls_fixed_val fixed_val;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_dpcc_methods_config - Methods Configuration used by DPCC
__u32 pg_fac;
__u32 rnd_thresh;
__u32 rg_fac;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_dpcc_config - Configuration used by DPCC
struct rkisp1_cif_isp_dpcc_methods_config methods[RKISP1_CIF_ISP_DPCC_METHODS_MAX];
__u32 ro_limits;
__u32 rnd_offs;
-} __packed;
+};
+/**
+ * struct rkisp1_cif_isp_gamma_corr_curve - gamma curve point definition y-axis (output).
+ *
+ * The reset values define a linear curve which has the same effect as bypass. Reset values are:
+ * gamma_y[0] = 0x0000, gamma_y[1] = 0x0100, ... gamma_y[15] = 0x0f00, gamma_y[16] = 0xfff
+ *
+ * @gamma_y: the values for the y-axis of gamma curve points. Each value is 12 bit.
+ */
struct rkisp1_cif_isp_gamma_corr_curve {
__u16 gamma_y[RKISP1_CIF_ISP_DEGAMMA_CURVE_SIZE];
-} __packed;
+};
+/**
+ * struct rkisp1_cif_isp_gamma_curve_x_axis_pnts - De-Gamma Curve definition x increments
+ * (sampling points). gamma_dx0 is for the lower samples (1-8), gamma_dx1 is for the
+ * higher samples (9-16). The reset values for both fields is 0x44444444. This means
+ * that each sample is 4 units away from the previous one on the x-axis.
+ *
+ * @gamma_dx0: gamma curve sample points definitions. Bits 0:2 for sample 1. Bit 3 unused.
+ * Bits 4:6 for sample 2. bit 7 unused ... Bits 28:30 for sample 8. Bit 31 unused
+ * @gamma_dx1: gamma curve sample points definitions. Bits 0:2 for sample 9. Bit 3 unused.
+ * Bits 4:6 for sample 10. bit 7 unused ... Bits 28:30 for sample 16. Bit 31 unused
+ */
struct rkisp1_cif_isp_gamma_curve_x_axis_pnts {
__u32 gamma_dx0;
__u32 gamma_dx1;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_sdg_config - Configuration used by sensor degamma
*
- * @curve_x: gamma curve point definition axis for x
- * @xa_pnts: x increments
+ * @curve_r: gamma curve point definition axis for red
+ * @curve_g: gamma curve point definition axis for green
+ * @curve_b: gamma curve point definition axis for blue
+ * @xa_pnts: x axis increments
*/
struct rkisp1_cif_isp_sdg_config {
struct rkisp1_cif_isp_gamma_corr_curve curve_r;
struct rkisp1_cif_isp_gamma_corr_curve curve_g;
struct rkisp1_cif_isp_gamma_corr_curve curve_b;
struct rkisp1_cif_isp_gamma_curve_x_axis_pnts xa_pnts;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_lsc_config - Configuration used by Lens shading correction
*
- * refer to REF_01 for details
+ * @r_data_tbl: sample table red
+ * @gr_data_tbl: sample table green (red)
+ * @gb_data_tbl: sample table green (blue)
+ * @b_data_tbl: sample table blue
+ * @x_grad_tbl: gradient table x
+ * @y_grad_tbl: gradient table y
+ * @x_size_tbl: size table x
+ * @y_size_tbl: size table y
+ * @config_width: not used at the moment
+ * @config_height: not used at the moment
*/
struct rkisp1_cif_isp_lsc_config {
- __u32 r_data_tbl[RKISP1_CIF_ISP_LSC_DATA_TBL_SIZE];
- __u32 gr_data_tbl[RKISP1_CIF_ISP_LSC_DATA_TBL_SIZE];
- __u32 gb_data_tbl[RKISP1_CIF_ISP_LSC_DATA_TBL_SIZE];
- __u32 b_data_tbl[RKISP1_CIF_ISP_LSC_DATA_TBL_SIZE];
+ __u16 r_data_tbl[RKISP1_CIF_ISP_LSC_SAMPLES_MAX][RKISP1_CIF_ISP_LSC_SAMPLES_MAX];
+ __u16 gr_data_tbl[RKISP1_CIF_ISP_LSC_SAMPLES_MAX][RKISP1_CIF_ISP_LSC_SAMPLES_MAX];
+ __u16 gb_data_tbl[RKISP1_CIF_ISP_LSC_SAMPLES_MAX][RKISP1_CIF_ISP_LSC_SAMPLES_MAX];
+ __u16 b_data_tbl[RKISP1_CIF_ISP_LSC_SAMPLES_MAX][RKISP1_CIF_ISP_LSC_SAMPLES_MAX];
- __u32 x_grad_tbl[RKISP1_CIF_ISP_LSC_GRAD_TBL_SIZE];
- __u32 y_grad_tbl[RKISP1_CIF_ISP_LSC_GRAD_TBL_SIZE];
+ __u16 x_grad_tbl[RKISP1_CIF_ISP_LSC_SECTORS_TBL_SIZE];
+ __u16 y_grad_tbl[RKISP1_CIF_ISP_LSC_SECTORS_TBL_SIZE];
- __u32 x_size_tbl[RKISP1_CIF_ISP_LSC_SIZE_TBL_SIZE];
- __u32 y_size_tbl[RKISP1_CIF_ISP_LSC_SIZE_TBL_SIZE];
+ __u16 x_size_tbl[RKISP1_CIF_ISP_LSC_SECTORS_TBL_SIZE];
+ __u16 y_size_tbl[RKISP1_CIF_ISP_LSC_SECTORS_TBL_SIZE];
__u16 config_width;
__u16 config_height;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_ie_config - Configuration used by image effects
*
+ * @effect: values from 'enum v4l2_colorfx'. Possible values are: V4L2_COLORFX_SEPIA,
+ * V4L2_COLORFX_SET_CBCR, V4L2_COLORFX_AQUA, V4L2_COLORFX_EMBOSS,
+ * V4L2_COLORFX_SKETCH, V4L2_COLORFX_BW, V4L2_COLORFX_NEGATIVE
+ * @color_sel: bits 0:2 - colors bitmask (001 - blue, 010 - green, 100 - red).
+ * bits 8:15 - Threshold value of the RGB colors for the color selection effect.
* @eff_mat_1: 3x3 Matrix Coefficients for Emboss Effect 1
* @eff_mat_2: 3x3 Matrix Coefficients for Emboss Effect 2
* @eff_mat_3: 3x3 Matrix Coefficients for Emboss 3/Sketch 1
__u16 eff_mat_4;
__u16 eff_mat_5;
__u16 eff_tint;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_cproc_config - Configuration used by Color Processing
__u8 brightness;
__u8 sat;
__u8 hue;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_awb_meas_config - Configuration used by auto white balance
*
+ * @awb_mode: the awb meas mode. From enum rkisp1_cif_isp_awb_mode_type.
* @awb_wnd: white balance measurement window (in pixels)
- * (from enum rkisp1_cif_isp_awb_mode_type)
* @max_y: only pixels values < max_y contribute to awb measurement, set to 0
* to disable this feature
* @min_y: only pixels values > min_y contribute to awb measurement
* (ucFrames=0 means 1 Frame)
* @awb_ref_cr: reference Cr value for AWB regulation, target for AWB
* @awb_ref_cb: reference Cb value for AWB regulation, target for AWB
+ * @enable_ymax_cmp: enable Y_MAX compare (Not valid in RGB measurement mode.)
*/
struct rkisp1_cif_isp_awb_meas_config {
/*
__u8 awb_ref_cr;
__u8 awb_ref_cb;
__u8 enable_ymax_cmp;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_awb_gain_config - Configuration used by auto white balance gain
*
- * out_data_x = ( AWB_GEAIN_X * in_data + 128) >> 8
+ * All fields in this struct are 10 bit, where:
+ * 0x100h = 1, unsigned integer value, range 0 to 4 with 8 bit fractional part.
+ *
+ * out_data_x = ( AWB_GAIN_X * in_data + 128) >> 8
+ *
+ * @gain_red: gain value for red component.
+ * @gain_green_r: gain value for green component in red line.
+ * @gain_blue: gain value for blue component.
+ * @gain_green_b: gain value for green component in blue line.
*/
struct rkisp1_cif_isp_awb_gain_config {
__u16 gain_red;
__u16 gain_green_r;
__u16 gain_blue;
__u16 gain_green_b;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_flt_config - Configuration used by ISP filtering
*
- * @mode: ISP_FILT_MODE register fields (from enum rkisp1_cif_isp_flt_mode)
- * @grn_stage1: ISP_FILT_MODE register fields
- * @chr_h_mode: ISP_FILT_MODE register fields
- * @chr_v_mode: ISP_FILT_MODE register fields
+ * All 4 threshold fields (thresh_*) are 10 bits.
+ * All 6 factor fields (fac_*) are 6 bits.
*
- * refer to REF_01 for details.
+ * @mode: ISP_FILT_MODE register fields (from enum rkisp1_cif_isp_flt_mode)
+ * @grn_stage1: Green filter stage 1 select (range 0x0...0x8)
+ * @chr_h_mode: Chroma filter horizontal mode
+ * @chr_v_mode: Chroma filter vertical mode
+ * @thresh_bl0: If thresh_bl1 < sum_grad < thresh_bl0 then fac_bl0 is selected (blurring th)
+ * @thresh_bl1: If sum_grad < thresh_bl1 then fac_bl1 is selected (blurring th)
+ * @thresh_sh0: If thresh_sh0 < sum_grad < thresh_sh1 then thresh_sh0 is selected (sharpening th)
+ * @thresh_sh1: If thresh_sh1 < sum_grad then thresh_sh1 is selected (sharpening th)
+ * @lum_weight: Parameters for luminance weight function.
+ * @fac_sh1: filter factor for sharp1 level
+ * @fac_sh0: filter factor for sharp0 level
+ * @fac_mid: filter factor for mid level and for static filter mode
+ * @fac_bl0: filter factor for blur 0 level
+ * @fac_bl1: filter factor for blur 1 level (max blur)
*/
-
struct rkisp1_cif_isp_flt_config {
__u32 mode;
__u8 grn_stage1;
__u32 fac_mid;
__u32 fac_bl0;
__u32 fac_bl1;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_bdm_config - Configuration used by Bayer DeMosaic
*/
struct rkisp1_cif_isp_bdm_config {
__u8 demosaic_th;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_ctk_config - Configuration used by Cross Talk correction
*
- * @coeff: color correction matrix
- * @ct_offset_b: offset for the crosstalk correction matrix
+ * @coeff: color correction matrix. Values are 11-bit signed fixed-point numbers with 4 bit integer
+ * and 7 bit fractional part, ranging from -8 (0x400) to +7.992 (0x3FF). 0 is
+ * represented by 0x000 and a coefficient value of 1 as 0x080.
+ * @ct_offset: Red, Green, Blue offsets for the crosstalk correction matrix
*/
struct rkisp1_cif_isp_ctk_config {
- __u16 coeff0;
- __u16 coeff1;
- __u16 coeff2;
- __u16 coeff3;
- __u16 coeff4;
- __u16 coeff5;
- __u16 coeff6;
- __u16 coeff7;
- __u16 coeff8;
- __u16 ct_offset_r;
- __u16 ct_offset_g;
- __u16 ct_offset_b;
-} __packed;
+ __u16 coeff[3][3];
+ __u16 ct_offset[3];
+};
enum rkisp1_cif_isp_goc_mode {
RKISP1_CIF_ISP_GOC_MODE_LOGARITHMIC,
struct rkisp1_cif_isp_goc_config {
__u32 mode;
__u16 gamma_y[RKISP1_CIF_ISP_GAMMA_OUT_MAX_SAMPLES];
-} __packed;
+};
/**
* struct rkisp1_cif_isp_hst_config - Configuration used by Histogram
__u8 histogram_predivider;
struct rkisp1_cif_isp_window meas_window;
__u8 hist_weight[RKISP1_CIF_ISP_HISTOGRAM_WEIGHT_GRIDS_SIZE];
-} __packed;
+};
/**
* struct rkisp1_cif_isp_aec_config - Configuration used by Auto Exposure Control
__u32 mode;
__u32 autostop;
struct rkisp1_cif_isp_window meas_window;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_afc_config - Configuration used by Auto Focus Control
struct rkisp1_cif_isp_window afm_win[RKISP1_CIF_ISP_AFM_MAX_WINDOWS];
__u32 thres;
__u32 var_shift;
-} __packed;
+};
/**
* enum rkisp1_cif_isp_dpf_gain_usage - dpf gain usage
struct rkisp1_cif_isp_dpf_nll {
__u16 coeff[RKISP1_CIF_ISP_DPF_MAX_NLF_COEFFS];
__u32 scale_mode;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_dpf_rb_flt - Red blue filter config
__u8 spatial_coeff[RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS];
__u8 r_enable;
__u8 b_enable;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_dpf_g_flt - Green filter Configuration
__u8 spatial_coeff[RKISP1_CIF_ISP_DPF_MAX_SPATIAL_COEFFS];
__u8 gr_enable;
__u8 gb_enable;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_dpf_gain - Noise function Configuration
__u16 nf_b_gain;
__u16 nf_gr_gain;
__u16 nf_gb_gain;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_dpf_config - Configuration used by De-noising pre-filter
struct rkisp1_cif_isp_dpf_g_flt g_flt;
struct rkisp1_cif_isp_dpf_rb_flt rb_flt;
struct rkisp1_cif_isp_dpf_nll nll;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_dpf_strength_config - strength of the filter
__u8 r;
__u8 g;
__u8 b;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_isp_other_cfg - Parameters for some blocks in rockchip isp1
struct rkisp1_cif_isp_dpf_strength_config dpf_strength_config;
struct rkisp1_cif_isp_cproc_config cproc_config;
struct rkisp1_cif_isp_ie_config ie_config;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_isp_meas_cfg - Rockchip ISP1 Measure Parameters
struct rkisp1_cif_isp_hst_config hst_config;
struct rkisp1_cif_isp_aec_config aec_config;
struct rkisp1_cif_isp_afc_config afc_config;
-} __packed;
+};
/**
* struct rkisp1_params_cfg - Rockchip ISP1 Input Parameters Meta Data
struct rkisp1_cif_isp_isp_meas_cfg meas;
struct rkisp1_cif_isp_isp_other_cfg others;
-} __packed;
+};
/*---------- PART2: Measurement Statistics ------------*/
__u8 mean_y_or_g;
__u8 mean_cb_or_b;
__u8 mean_cr_or_r;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_awb_stat - statistics automatic white balance data
*/
struct rkisp1_cif_isp_awb_stat {
struct rkisp1_cif_isp_awb_meas awb_mean[RKISP1_CIF_ISP_AWB_MAX_GRID];
-} __packed;
+};
/**
* struct rkisp1_cif_isp_bls_meas_val - BLS measured values
__u16 meas_gr;
__u16 meas_gb;
__u16 meas_b;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_ae_stat - statistics auto exposure data
struct rkisp1_cif_isp_ae_stat {
__u8 exp_mean[RKISP1_CIF_ISP_AE_MEAN_MAX];
struct rkisp1_cif_isp_bls_meas_val bls_val;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_af_meas_val - AF measured values
*
- * @sum: sharpness, refer to REF_01 for definition
- * @lum: luminance, refer to REF_01 for definition
+ * @sum: sharpness value
+ * @lum: luminance value
*/
struct rkisp1_cif_isp_af_meas_val {
__u32 sum;
__u32 lum;
-} __packed;
+};
/**
* struct rkisp1_cif_isp_af_stat - statistics auto focus data
*/
struct rkisp1_cif_isp_af_stat {
struct rkisp1_cif_isp_af_meas_val window[RKISP1_CIF_ISP_AFM_MAX_WINDOWS];
-} __packed;
+};
/**
* struct rkisp1_cif_isp_hist_stat - statistics histogram data
*/
struct rkisp1_cif_isp_hist_stat {
__u16 hist_bins[RKISP1_CIF_ISP_HIST_BIN_N_MAX];
-} __packed;
+};
/**
- * struct rkisp1_stat_buffer - Rockchip ISP1 Statistics Data
+ * struct rkisp1_cif_isp_stat - Rockchip ISP1 Statistics Data
*
- * @rkisp1_cif_isp_awb_stat: statistics data for automatic white balance
- * @rkisp1_cif_isp_ae_stat: statistics data for auto exposure
- * @rkisp1_cif_isp_af_stat: statistics data for auto focus
- * @rkisp1_cif_isp_hist_stat: statistics histogram data
+ * @awb: statistics data for automatic white balance
+ * @ae: statistics data for auto exposure
+ * @af: statistics data for auto focus
+ * @hist: statistics histogram data
*/
struct rkisp1_cif_isp_stat {
struct rkisp1_cif_isp_awb_stat awb;
struct rkisp1_cif_isp_ae_stat ae;
struct rkisp1_cif_isp_af_stat af;
struct rkisp1_cif_isp_hist_stat hist;
-} __packed;
+};
/**
* struct rkisp1_stat_buffer - Rockchip ISP1 Statistics Meta Data
*
- * @meas_type: measurement types (RKISP1_CIF_ISP_STAT_ definitions)
+ * @meas_type: measurement types (RKISP1_CIF_ISP_STAT_* definitions)
* @frame_id: frame ID for sync
* @params: statistics data
*/
__u32 meas_type;
__u32 frame_id;
struct rkisp1_cif_isp_stat params;
-} __packed;
+};
#endif /* _UAPI_RKISP1_CONFIG_H */
struct rkvdec_h264_run {
struct rkvdec_run base;
const struct v4l2_ctrl_h264_decode_params *decode_params;
- const struct v4l2_ctrl_h264_slice_params *slices_params;
const struct v4l2_ctrl_h264_sps *sps;
const struct v4l2_ctrl_h264_pps *pps;
const struct v4l2_ctrl_h264_scaling_matrix *scaling_matrix;
WRITE_PPS(pps->second_chroma_qp_index_offset,
SECOND_CHROMA_QP_INDEX_OFFSET);
- /* always use the matrix sent from userspace */
- WRITE_PPS(1, SCALING_LIST_ENABLE_FLAG);
-
+ WRITE_PPS(!!(pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT),
+ SCALING_LIST_ENABLE_FLAG);
+ /* To be on the safe side, program the scaling matrix address */
scaling_distance = offsetof(struct rkvdec_h264_priv_tbl, scaling_list);
scaling_list_address = h264_ctx->priv_tbl.dma + scaling_distance;
WRITE_PPS(scaling_list_address, SCALING_LIST_ADDRESS);
struct rkvdec_h264_run *run)
{
const struct v4l2_ctrl_h264_decode_params *dec_params = run->decode_params;
- const struct v4l2_ctrl_h264_slice_params *sl_params = &run->slices_params[0];
const struct v4l2_h264_dpb_entry *dpb = dec_params->dpb;
struct rkvdec_h264_ctx *h264_ctx = ctx->priv;
const struct v4l2_ctrl_h264_sps *sps = run->sps;
continue;
if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM ||
- dpb[i].frame_num < sl_params->frame_num) {
+ dpb[i].frame_num < dec_params->frame_num) {
p[i] = dpb[i].frame_num;
continue;
}
struct rkvdec_h264_run *run)
{
const struct v4l2_ctrl_h264_scaling_matrix *scaling = run->scaling_matrix;
+ const struct v4l2_ctrl_h264_pps *pps = run->pps;
struct rkvdec_h264_ctx *h264_ctx = ctx->priv;
struct rkvdec_h264_priv_tbl *tbl = h264_ctx->priv_tbl.cpu;
+ if (!(pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT))
+ return;
+
BUILD_BUG_ON(sizeof(tbl->scaling_list.scaling_list_4x4) !=
sizeof(scaling->scaling_list_4x4));
BUILD_BUG_ON(sizeof(tbl->scaling_list.scaling_list_8x8) !=
for (i = 0; i < ARRAY_SIZE(dec_params->dpb); i++) {
struct vb2_buffer *vb_buf = get_ref_buf(ctx, run, i);
- refer_addr = vb2_dma_contig_plane_dma_addr(vb_buf, 0) |
- RKVDEC_COLMV_USED_FLAG_REF;
+ refer_addr = vb2_dma_contig_plane_dma_addr(vb_buf, 0);
- if (!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD))
- refer_addr |= RKVDEC_TOPFIELD_USED_REF |
- RKVDEC_BOTFIELD_USED_REF;
- else if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_BOTTOM_FIELD)
- refer_addr |= RKVDEC_BOTFIELD_USED_REF;
- else
+ if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE)
+ refer_addr |= RKVDEC_COLMV_USED_FLAG_REF;
+ if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD)
+ refer_addr |= RKVDEC_FIELD_REF;
+
+ if (dpb[i].fields & V4L2_H264_TOP_FIELD_REF)
refer_addr |= RKVDEC_TOPFIELD_USED_REF;
+ if (dpb[i].fields & V4L2_H264_BOTTOM_FIELD_REF)
+ refer_addr |= RKVDEC_BOTFIELD_USED_REF;
writel_relaxed(dpb[i].top_field_order_cnt,
rkvdec->regs + poc_reg_tbl_top_field[i]);
ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl,
V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS);
run->decode_params = ctrl ? ctrl->p_cur.p : NULL;
- ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl,
- V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS);
- run->slices_params = ctrl ? ctrl->p_cur.p : NULL;
ctrl = v4l2_ctrl_find(&ctx->ctrl_hdl,
V4L2_CID_MPEG_VIDEO_H264_SPS);
run->sps = ctrl ? ctrl->p_cur.p : NULL;
/* Build the P/B{0,1} ref lists. */
v4l2_h264_init_reflist_builder(&reflist_builder, run.decode_params,
- &run.slices_params[0], run.sps,
- run.decode_params->dpb);
+ run.sps, run.decode_params->dpb);
h264_ctx->reflists.num_valid = reflist_builder.num_valid;
v4l2_h264_build_p_ref_list(&reflist_builder, h264_ctx->reflists.p);
v4l2_h264_build_b_ref_lists(&reflist_builder, h264_ctx->reflists.b0,
static const struct rkvdec_ctrl_desc rkvdec_h264_ctrl_descs[] = {
{
- .per_request = true,
.mandatory = true,
.cfg.id = V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS,
},
{
- .per_request = true,
- .mandatory = true,
- .cfg.id = V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS,
- },
- {
- .per_request = true,
.mandatory = true,
.cfg.id = V4L2_CID_MPEG_VIDEO_H264_SPS,
.cfg.ops = &rkvdec_ctrl_ops,
},
{
- .per_request = true,
.mandatory = true,
.cfg.id = V4L2_CID_MPEG_VIDEO_H264_PPS,
},
{
- .per_request = true,
- .mandatory = true,
.cfg.id = V4L2_CID_MPEG_VIDEO_H264_SCALING_MATRIX,
},
{
- .mandatory = true,
.cfg.id = V4L2_CID_MPEG_VIDEO_H264_DECODE_MODE,
.cfg.min = V4L2_MPEG_VIDEO_H264_DECODE_MODE_FRAME_BASED,
.cfg.max = V4L2_MPEG_VIDEO_H264_DECODE_MODE_FRAME_BASED,
.cfg.def = V4L2_MPEG_VIDEO_H264_DECODE_MODE_FRAME_BASED,
},
{
- .mandatory = true,
.cfg.id = V4L2_CID_MPEG_VIDEO_H264_START_CODE,
.cfg.min = V4L2_MPEG_VIDEO_H264_START_CODE_ANNEX_B,
.cfg.def = V4L2_MPEG_VIDEO_H264_START_CODE_ANNEX_B,
u32 id = ctrls->ctrls[i].cfg.id;
struct v4l2_ctrl *ctrl;
- if (!ctrls->ctrls[i].per_request || !ctrls->ctrls[i].mandatory)
+ if (!ctrls->ctrls[i].mandatory)
continue;
ctrl = v4l2_ctrl_request_hdl_ctrl_find(hdl, id);
struct rkvdec_ctx;
struct rkvdec_ctrl_desc {
- u32 per_request : 1;
u32 mandatory : 1;
struct v4l2_ctrl_config cfg;
};
.id = V4L2_CID_MPEG_VIDEO_H264_SCALING_MATRIX,
},
.codec = CEDRUS_CODEC_H264,
- .required = true,
+ .required = false,
+ },
+ {
+ .cfg = {
+ .id = V4L2_CID_MPEG_VIDEO_H264_PRED_WEIGHTS,
+ },
+ .codec = CEDRUS_CODEC_H264,
+ .required = false,
},
{
.cfg = {
const struct v4l2_ctrl_h264_scaling_matrix *scaling_matrix;
const struct v4l2_ctrl_h264_slice_params *slice_params;
const struct v4l2_ctrl_h264_sps *sps;
+ const struct v4l2_ctrl_h264_pred_weights *pred_weights;
};
struct cedrus_mpeg2_run {
V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS);
run.h264.sps = cedrus_find_control_data(ctx,
V4L2_CID_MPEG_VIDEO_H264_SPS);
+ run.h264.pred_weights = cedrus_find_control_data(ctx,
+ V4L2_CID_MPEG_VIDEO_H264_PRED_WEIGHTS);
break;
case V4L2_PIX_FMT_HEVC_SLICE:
{
struct cedrus_h264_sram_ref_pic pic_list[CEDRUS_H264_FRAME_NUM];
const struct v4l2_ctrl_h264_decode_params *decode = run->h264.decode_params;
- const struct v4l2_ctrl_h264_slice_params *slice = run->h264.slice_params;
const struct v4l2_ctrl_h264_sps *sps = run->h264.sps;
struct vb2_queue *cap_q;
struct cedrus_buffer *output_buf;
struct cedrus_dev *dev = ctx->dev;
unsigned long used_dpbs = 0;
unsigned int position;
- unsigned int output = 0;
+ int output = -1;
unsigned int i;
cap_q = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
position = cedrus_buf->codec.h264.position;
used_dpbs |= BIT(position);
+ if (run->dst->vb2_buf.timestamp == dpb->reference_ts) {
+ output = position;
+ continue;
+ }
+
if (!(dpb->flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE))
continue;
dpb->top_field_order_cnt,
dpb->bottom_field_order_cnt,
&pic_list[position]);
-
- output = max(position, output);
}
- position = find_next_zero_bit(&used_dpbs, CEDRUS_H264_FRAME_NUM,
- output);
- if (position >= CEDRUS_H264_FRAME_NUM)
+ if (output >= 0)
+ position = output;
+ else
position = find_first_zero_bit(&used_dpbs, CEDRUS_H264_FRAME_NUM);
output_buf = vb2_to_cedrus_buffer(&run->dst->vb2_buf);
output_buf->codec.h264.position = position;
- if (slice->flags & V4L2_H264_SLICE_FLAG_FIELD_PIC)
+ if (decode->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC)
output_buf->codec.h264.pic_type = CEDRUS_H264_PIC_TYPE_FIELD;
else if (sps->flags & V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD)
output_buf->codec.h264.pic_type = CEDRUS_H264_PIC_TYPE_MBAFF;
static void _cedrus_write_ref_list(struct cedrus_ctx *ctx,
struct cedrus_run *run,
- const u8 *ref_list, u8 num_ref,
- enum cedrus_h264_sram_off sram)
+ const struct v4l2_h264_reference *ref_list,
+ u8 num_ref, enum cedrus_h264_sram_off sram)
{
const struct v4l2_ctrl_h264_decode_params *decode = run->h264.decode_params;
struct vb2_queue *cap_q;
for (i = 0; i < num_ref; i++) {
const struct v4l2_h264_dpb_entry *dpb;
const struct cedrus_buffer *cedrus_buf;
- const struct vb2_v4l2_buffer *ref_buf;
unsigned int position;
int buf_idx;
u8 dpb_idx;
- dpb_idx = ref_list[i];
+ dpb_idx = ref_list[i].index;
dpb = &decode->dpb[dpb_idx];
if (!(dpb->flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE))
if (buf_idx < 0)
continue;
- ref_buf = to_vb2_v4l2_buffer(cap_q->bufs[buf_idx]);
- cedrus_buf = vb2_v4l2_to_cedrus_buffer(ref_buf);
+ cedrus_buf = vb2_to_cedrus_buffer(cap_q->bufs[buf_idx]);
position = cedrus_buf->codec.h264.position;
sram_array[i] |= position << 1;
- if (ref_buf->field == V4L2_FIELD_BOTTOM)
+ if (ref_list[i].fields & V4L2_H264_BOTTOM_FIELD_REF)
sram_array[i] |= BIT(0);
}
{
const struct v4l2_ctrl_h264_scaling_matrix *scaling =
run->h264.scaling_matrix;
+ const struct v4l2_ctrl_h264_pps *pps = run->h264.pps;
struct cedrus_dev *dev = ctx->dev;
+ if (!(pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT))
+ return;
+
cedrus_h264_write_sram(dev, CEDRUS_SRAM_H264_SCALING_LIST_8x8_0,
scaling->scaling_list_8x8[0],
sizeof(scaling->scaling_list_8x8[0]));
static void cedrus_write_pred_weight_table(struct cedrus_ctx *ctx,
struct cedrus_run *run)
{
- const struct v4l2_ctrl_h264_slice_params *slice =
- run->h264.slice_params;
- const struct v4l2_h264_pred_weight_table *pred_weight =
- &slice->pred_weight_table;
+ const struct v4l2_ctrl_h264_pred_weights *pred_weight =
+ run->h264.pred_weights;
struct cedrus_dev *dev = ctx->dev;
int i, j, k;
struct vb2_buffer *src_buf = &run->src->vb2_buf;
struct cedrus_dev *dev = ctx->dev;
dma_addr_t src_buf_addr;
- u32 len = slice->size * 8;
+ size_t slice_bytes = vb2_get_plane_payload(src_buf, 0);
unsigned int pic_width_in_mbs;
bool mbaff_pic;
u32 reg;
- cedrus_write(dev, VE_H264_VLD_LEN, len);
+ cedrus_write(dev, VE_H264_VLD_LEN, slice_bytes * 8);
cedrus_write(dev, VE_H264_VLD_OFFSET, 0);
src_buf_addr = vb2_dma_contig_plane_dma_addr(src_buf, 0);
- cedrus_write(dev, VE_H264_VLD_END,
- src_buf_addr + vb2_get_plane_payload(src_buf, 0));
+ cedrus_write(dev, VE_H264_VLD_END, src_buf_addr + slice_bytes);
cedrus_write(dev, VE_H264_VLD_ADDR,
VE_H264_VLD_ADDR_VAL(src_buf_addr) |
VE_H264_VLD_ADDR_FIRST | VE_H264_VLD_ADDR_VALID |
cedrus_skip_bits(dev, slice->header_bit_size);
- if (((pps->flags & V4L2_H264_PPS_FLAG_WEIGHTED_PRED) &&
- (slice->slice_type == V4L2_H264_SLICE_TYPE_P ||
- slice->slice_type == V4L2_H264_SLICE_TYPE_SP)) ||
- (pps->weighted_bipred_idc == 1 &&
- slice->slice_type == V4L2_H264_SLICE_TYPE_B))
+ if (V4L2_H264_CTRL_PRED_WEIGHTS_REQUIRED(pps, slice))
cedrus_write_pred_weight_table(ctx, run);
if ((slice->slice_type == V4L2_H264_SLICE_TYPE_P) ||
reg |= VE_H264_SPS_DIRECT_8X8_INFERENCE;
cedrus_write(dev, VE_H264_SPS, reg);
- mbaff_pic = !(slice->flags & V4L2_H264_SLICE_FLAG_FIELD_PIC) &&
+ mbaff_pic = !(decode->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC) &&
(sps->flags & V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD);
pic_width_in_mbs = sps->pic_width_in_mbs_minus1 + 1;
reg |= slice->cabac_init_idc & 0x3;
if (ctx->fh.m2m_ctx->new_frame)
reg |= VE_H264_SHS_FIRST_SLICE_IN_PIC;
- if (slice->flags & V4L2_H264_SLICE_FLAG_FIELD_PIC)
+ if (decode->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC)
reg |= VE_H264_SHS_FIELD_PIC;
- if (slice->flags & V4L2_H264_SLICE_FLAG_BOTTOM_FIELD)
+ if (decode->flags & V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD)
reg |= VE_H264_SHS_BOTTOM_FIELD;
if (slice->flags & V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED)
reg |= VE_H264_SHS_DIRECT_SPATIAL_MV_PRED;
reg |= (pps->second_chroma_qp_index_offset & 0x3f) << 16;
reg |= (pps->chroma_qp_index_offset & 0x3f) << 8;
reg |= (pps->pic_init_qp_minus26 + 26 + slice->slice_qp_delta) & 0x3f;
+ if (pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT)
+ reg |= VE_H264_SHS_QP_SCALING_MATRIX_DEFAULT;
cedrus_write(dev, VE_H264_SHS_QP, reg);
// clear status flags
return -EINVAL;
pix_fmt->pixelformat = fmt->pixelformat;
+ pix_fmt->width = ctx->src_fmt.width;
+ pix_fmt->height = ctx->src_fmt.height;
cedrus_prepare_format(pix_fmt);
return 0;
{
struct cedrus_ctx *ctx = cedrus_file2ctx(file);
struct vb2_queue *vq;
+ struct vb2_queue *peer_vq;
int ret;
+ ret = cedrus_try_fmt_vid_out(file, priv, f);
+ if (ret)
+ return ret;
+
vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type);
- if (vb2_is_busy(vq))
+ /*
+ * In order to support dynamic resolution change,
+ * the decoder admits a resolution change, as long
+ * as the pixelformat remains. Can't be done if streaming.
+ */
+ if (vb2_is_streaming(vq) || (vb2_is_busy(vq) &&
+ f->fmt.pix.pixelformat != ctx->src_fmt.pixelformat))
+ return -EBUSY;
+ /*
+ * Since format change on the OUTPUT queue will reset
+ * the CAPTURE queue, we can't allow doing so
+ * when the CAPTURE queue has buffers allocated.
+ */
+ peer_vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx,
+ V4L2_BUF_TYPE_VIDEO_CAPTURE);
+ if (vb2_is_busy(peer_vq))
return -EBUSY;
ret = cedrus_try_fmt_vid_out(file, priv, f);
break;
}
- /* Propagate colorspace information to capture. */
+ /* Propagate format information to capture. */
ctx->dst_fmt.colorspace = f->fmt.pix.colorspace;
ctx->dst_fmt.xfer_func = f->fmt.pix.xfer_func;
ctx->dst_fmt.ycbcr_enc = f->fmt.pix.ycbcr_enc;
ctx->dst_fmt.quantization = f->fmt.pix.quantization;
+ ctx->dst_fmt.width = ctx->src_fmt.width;
+ ctx->dst_fmt.height = ctx->src_fmt.height;
+ cedrus_prepare_format(&ctx->dst_fmt);
return 0;
}
return IRQ_HANDLED;
}
-static int tegra_vde_runtime_suspend(struct device *dev)
+static __maybe_unused int tegra_vde_runtime_suspend(struct device *dev)
{
struct tegra_vde *vde = dev_get_drvdata(dev);
int err;
return 0;
}
-static int tegra_vde_runtime_resume(struct device *dev)
+static __maybe_unused int tegra_vde_runtime_resume(struct device *dev)
{
struct tegra_vde *vde = dev_get_drvdata(dev);
int err;
depends on VIDEO_V4L2
select MEDIA_CONTROLLER
select VIDEOBUF2_DMA_CONTIG
+ select V4L2_FWNODE
help
Choose this option if you have an NVIDIA Tegra SoC.
To compile this driver as a module, choose M here: the module
will be called tegra-video.
+
+config VIDEO_TEGRA_TPG
+ bool "NVIDIA Tegra VI driver TPG mode"
+ depends on VIDEO_TEGRA
+ help
+ Say yes here to enable Tegra internal TPG mode
TODO list
-* Currently driver supports Tegra build-in TPG only with direct media links
- from CSI to VI. Add kernel config CONFIG_VIDEO_TEGRA_TPG and update the
- driver to do TPG Vs Sensor media links based on CONFIG_VIDEO_TEGRA_TPG.
-* Add real camera sensor capture support.
-* Add Tegra CSI MIPI pads calibration.
-* Add MIPI clock Settle time computation based on the data rate.
* Add support for Ganged mode.
* Add RAW10 packed video format support to Tegra210 video formats.
* Add support for suspend and resume.
#include <linux/host1x.h>
#include <linux/module.h>
#include <linux/of.h>
+#include <linux/of_graph.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
+#include <media/v4l2-fwnode.h>
+
#include "csi.h"
#include "video.h"
+#define MHZ 1000000
+
static inline struct tegra_csi *
host1x_client_to_csi(struct host1x_client *client)
{
struct v4l2_subdev_pad_config *cfg,
struct v4l2_subdev_mbus_code_enum *code)
{
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ return -ENOIOCTLCMD;
+
if (code->index >= ARRAY_SIZE(tegra_csi_tpg_fmts))
return -EINVAL;
{
struct tegra_csi_channel *csi_chan = to_csi_chan(subdev);
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ return -ENOIOCTLCMD;
+
fmt->format = csi_chan->format;
return 0;
{
unsigned int i;
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ return -ENOIOCTLCMD;
+
if (fse->index >= ARRAY_SIZE(tegra_csi_tpg_sizes))
return -EINVAL;
const struct tpg_framerate *frmrate = csi->soc->tpg_frmrate_table;
int index;
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ return -ENOIOCTLCMD;
+
/* one framerate per format and resolution */
if (fie->index > 0)
return -EINVAL;
const struct v4l2_frmsize_discrete *sizes;
unsigned int i;
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ return -ENOIOCTLCMD;
+
sizes = v4l2_find_nearest_size(tegra_csi_tpg_sizes,
ARRAY_SIZE(tegra_csi_tpg_sizes),
width, height,
{
struct tegra_csi_channel *csi_chan = to_csi_chan(subdev);
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ return -ENOIOCTLCMD;
+
vfi->interval.numerator = 1;
vfi->interval.denominator = csi_chan->framerate;
return 0;
}
-static int tegra_csi_s_stream(struct v4l2_subdev *subdev, int enable)
+static unsigned int csi_get_pixel_rate(struct tegra_csi_channel *csi_chan)
+{
+ struct tegra_vi_channel *chan;
+ struct v4l2_subdev *src_subdev;
+ struct v4l2_ctrl *ctrl;
+
+ chan = v4l2_get_subdev_hostdata(&csi_chan->subdev);
+ src_subdev = tegra_channel_get_remote_source_subdev(chan);
+ ctrl = v4l2_ctrl_find(src_subdev->ctrl_handler, V4L2_CID_PIXEL_RATE);
+ if (ctrl)
+ return v4l2_ctrl_g_ctrl_int64(ctrl);
+
+ return 0;
+}
+
+void tegra_csi_calc_settle_time(struct tegra_csi_channel *csi_chan,
+ u8 *clk_settle_time,
+ u8 *ths_settle_time)
+{
+ struct tegra_csi *csi = csi_chan->csi;
+ unsigned int cil_clk_mhz;
+ unsigned int pix_clk_mhz;
+ int clk_idx = (csi_chan->csi_port_num >> 1) + 1;
+
+ cil_clk_mhz = clk_get_rate(csi->clks[clk_idx].clk) / MHZ;
+ pix_clk_mhz = csi_get_pixel_rate(csi_chan) / MHZ;
+
+ /*
+ * CLK Settle time is the interval during which HS receiver should
+ * ignore any clock lane HS transitions, starting from the beginning
+ * of T-CLK-PREPARE.
+ * Per DPHY specification, T-CLK-SETTLE should be between 95ns ~ 300ns
+ *
+ * 95ns < (clk-settle-programmed + 7) * lp clk period < 300ns
+ * midpoint = 197.5 ns
+ */
+ *clk_settle_time = ((95 + 300) * cil_clk_mhz - 14000) / 2000;
+
+ /*
+ * THS Settle time is the interval during which HS receiver should
+ * ignore any data lane HS transitions, starting from the beginning
+ * of THS-PREPARE.
+ *
+ * Per DPHY specification, T-HS-SETTLE should be between 85ns + 6UI
+ * and 145ns+10UI.
+ * 85ns + 6UI < (Ths-settle-prog + 5) * lp_clk_period < 145ns + 10UI
+ * midpoint = 115ns + 8UI
+ */
+ if (pix_clk_mhz)
+ *ths_settle_time = (115 * cil_clk_mhz + 8000 * cil_clk_mhz
+ / (2 * pix_clk_mhz) - 5000) / 1000;
+}
+
+static int tegra_csi_enable_stream(struct v4l2_subdev *subdev)
{
struct tegra_vi_channel *chan = v4l2_get_subdev_hostdata(subdev);
struct tegra_csi_channel *csi_chan = to_csi_chan(subdev);
struct tegra_csi *csi = csi_chan->csi;
- int ret = 0;
+ int ret, err;
- csi_chan->pg_mode = chan->pg_mode;
- if (enable) {
- ret = pm_runtime_get_sync(csi->dev);
+ ret = pm_runtime_get_sync(csi->dev);
+ if (ret < 0) {
+ dev_err(csi->dev, "failed to get runtime PM: %d\n", ret);
+ pm_runtime_put_noidle(csi->dev);
+ return ret;
+ }
+
+ if (csi_chan->mipi) {
+ ret = tegra_mipi_enable(csi_chan->mipi);
if (ret < 0) {
dev_err(csi->dev,
- "failed to get runtime PM: %d\n", ret);
- pm_runtime_put_noidle(csi->dev);
- return ret;
+ "failed to enable MIPI pads: %d\n", ret);
+ goto rpm_put;
}
- ret = csi->ops->csi_start_streaming(csi_chan);
- if (ret < 0)
- goto rpm_put;
+ /*
+ * CSI MIPI pads PULLUP, PULLDN and TERM impedances need to
+ * be calibrated after power on.
+ * So, trigger the calibration start here and results will
+ * be latched and applied to the pads when link is in LP11
+ * state during start of sensor streaming.
+ */
+ ret = tegra_mipi_start_calibration(csi_chan->mipi);
+ if (ret < 0) {
+ dev_err(csi->dev,
+ "failed to start MIPI calibration: %d\n", ret);
+ goto disable_mipi;
+ }
+ }
- return 0;
+ csi_chan->pg_mode = chan->pg_mode;
+ ret = csi->ops->csi_start_streaming(csi_chan);
+ if (ret < 0)
+ goto finish_calibration;
+
+ return 0;
+
+finish_calibration:
+ if (csi_chan->mipi)
+ tegra_mipi_finish_calibration(csi_chan->mipi);
+disable_mipi:
+ if (csi_chan->mipi) {
+ err = tegra_mipi_disable(csi_chan->mipi);
+ if (err < 0)
+ dev_err(csi->dev,
+ "failed to disable MIPI pads: %d\n", err);
}
+rpm_put:
+ pm_runtime_put(csi->dev);
+ return ret;
+}
+
+static int tegra_csi_disable_stream(struct v4l2_subdev *subdev)
+{
+ struct tegra_csi_channel *csi_chan = to_csi_chan(subdev);
+ struct tegra_csi *csi = csi_chan->csi;
+ int err;
+
csi->ops->csi_stop_streaming(csi_chan);
-rpm_put:
+ if (csi_chan->mipi) {
+ err = tegra_mipi_disable(csi_chan->mipi);
+ if (err < 0)
+ dev_err(csi->dev,
+ "failed to disable MIPI pads: %d\n", err);
+ }
+
pm_runtime_put(csi->dev);
+
+ return 0;
+}
+
+static int tegra_csi_s_stream(struct v4l2_subdev *subdev, int enable)
+{
+ int ret;
+
+ if (enable)
+ ret = tegra_csi_enable_stream(subdev);
+ else
+ ret = tegra_csi_disable_stream(subdev);
+
return ret;
}
.pad = &tegra_csi_pad_ops,
};
+static int tegra_csi_channel_alloc(struct tegra_csi *csi,
+ struct device_node *node,
+ unsigned int port_num, unsigned int lanes,
+ unsigned int num_pads)
+{
+ struct tegra_csi_channel *chan;
+ int ret = 0;
+
+ chan = kzalloc(sizeof(*chan), GFP_KERNEL);
+ if (!chan)
+ return -ENOMEM;
+
+ list_add_tail(&chan->list, &csi->csi_chans);
+ chan->csi = csi;
+ chan->csi_port_num = port_num;
+ chan->numlanes = lanes;
+ chan->of_node = node;
+ chan->numpads = num_pads;
+ if (num_pads & 0x2) {
+ chan->pads[0].flags = MEDIA_PAD_FL_SINK;
+ chan->pads[1].flags = MEDIA_PAD_FL_SOURCE;
+ } else {
+ chan->pads[0].flags = MEDIA_PAD_FL_SOURCE;
+ }
+
+ if (IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ return 0;
+
+ chan->mipi = tegra_mipi_request(csi->dev, node);
+ if (IS_ERR(chan->mipi)) {
+ ret = PTR_ERR(chan->mipi);
+ dev_err(csi->dev, "failed to get mipi device: %d\n", ret);
+ }
+
+ return ret;
+}
+
static int tegra_csi_tpg_channels_alloc(struct tegra_csi *csi)
{
struct device_node *node = csi->dev->of_node;
unsigned int port_num;
- struct tegra_csi_channel *chan;
unsigned int tpg_channels = csi->soc->csi_max_channels;
+ int ret;
/* allocate CSI channel for each CSI x2 ports */
for (port_num = 0; port_num < tpg_channels; port_num++) {
- chan = kzalloc(sizeof(*chan), GFP_KERNEL);
- if (!chan)
- return -ENOMEM;
-
- list_add_tail(&chan->list, &csi->csi_chans);
- chan->csi = csi;
- chan->csi_port_num = port_num;
- chan->numlanes = 2;
- chan->of_node = node;
- chan->numpads = 1;
- chan->pads[0].flags = MEDIA_PAD_FL_SOURCE;
+ ret = tegra_csi_channel_alloc(csi, node, port_num, 2, 1);
+ if (ret < 0)
+ return ret;
+ }
+
+ return 0;
+}
+
+static int tegra_csi_channels_alloc(struct tegra_csi *csi)
+{
+ struct device_node *node = csi->dev->of_node;
+ struct v4l2_fwnode_endpoint v4l2_ep = {
+ .bus_type = V4L2_MBUS_CSI2_DPHY
+ };
+ struct fwnode_handle *fwh;
+ struct device_node *channel;
+ struct device_node *ep;
+ unsigned int lanes, portno, num_pads;
+ int ret;
+
+ for_each_child_of_node(node, channel) {
+ if (!of_node_name_eq(channel, "channel"))
+ continue;
+
+ ret = of_property_read_u32(channel, "reg", &portno);
+ if (ret < 0)
+ continue;
+
+ if (portno >= csi->soc->csi_max_channels) {
+ dev_err(csi->dev, "invalid port num %d for %pOF\n",
+ portno, channel);
+ ret = -EINVAL;
+ goto err_node_put;
+ }
+
+ ep = of_graph_get_endpoint_by_regs(channel, 0, 0);
+ if (!ep)
+ continue;
+
+ fwh = of_fwnode_handle(ep);
+ ret = v4l2_fwnode_endpoint_parse(fwh, &v4l2_ep);
+ of_node_put(ep);
+ if (ret) {
+ dev_err(csi->dev,
+ "failed to parse v4l2 endpoint for %pOF: %d\n",
+ channel, ret);
+ goto err_node_put;
+ }
+
+ lanes = v4l2_ep.bus.mipi_csi2.num_data_lanes;
+ if (!lanes || ((lanes & (lanes - 1)) != 0)) {
+ dev_err(csi->dev, "invalid data-lanes %d for %pOF\n",
+ lanes, channel);
+ ret = -EINVAL;
+ goto err_node_put;
+ }
+
+ num_pads = of_graph_get_endpoint_count(channel);
+ if (num_pads == TEGRA_CSI_PADS_NUM) {
+ ret = tegra_csi_channel_alloc(csi, channel, portno,
+ lanes, num_pads);
+ if (ret < 0)
+ goto err_node_put;
+ }
}
return 0;
+
+err_node_put:
+ of_node_put(channel);
+ return ret;
}
static int tegra_csi_channel_init(struct tegra_csi_channel *chan)
subdev = &chan->subdev;
v4l2_subdev_init(subdev, &tegra_csi_ops);
subdev->dev = csi->dev;
- snprintf(subdev->name, V4L2_SUBDEV_NAME_SIZE, "%s-%d", "tpg",
- chan->csi_port_num);
+ if (IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ snprintf(subdev->name, V4L2_SUBDEV_NAME_SIZE, "%s-%d", "tpg",
+ chan->csi_port_num);
+ else
+ snprintf(subdev->name, V4L2_SUBDEV_NAME_SIZE, "%s",
+ kbasename(chan->of_node->full_name));
v4l2_set_subdevdata(subdev, chan);
subdev->fwnode = of_fwnode_handle(chan->of_node);
return ret;
}
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)) {
+ ret = v4l2_async_register_subdev(subdev);
+ if (ret < 0) {
+ dev_err(csi->dev,
+ "failed to register subdev: %d\n", ret);
+ return ret;
+ }
+ }
+
return 0;
}
struct tegra_csi_channel *chan, *tmp;
list_for_each_entry_safe(chan, tmp, &csi->csi_chans, list) {
+ if (chan->mipi)
+ tegra_mipi_free(chan->mipi);
+
subdev = &chan->subdev;
- if (subdev->dev)
+ if (subdev->dev) {
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ v4l2_async_unregister_subdev(subdev);
media_entity_cleanup(&subdev->entity);
+ }
+
list_del(&chan->list);
kfree(chan);
}
INIT_LIST_HEAD(&csi->csi_chans);
- ret = tegra_csi_tpg_channels_alloc(csi);
+ if (IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ ret = tegra_csi_tpg_channels_alloc(csi);
+ else
+ ret = tegra_csi_channels_alloc(csi);
if (ret < 0) {
dev_err(csi->dev,
- "failed to allocate tpg channels: %d\n", ret);
+ "failed to allocate channels: %d\n", ret);
goto cleanup;
}
#define __TEGRA_CSI_H__
#include <media/media-entity.h>
+#include <media/v4l2-async.h>
#include <media/v4l2-subdev.h>
/*
* @framerate: active framerate for TPG
* @h_blank: horizontal blanking for TPG active format
* @v_blank: vertical blanking for TPG active format
+ * @mipi: mipi device for corresponding csi channel pads
+ * @pixel_rate: active pixel rate from the sensor on this channel
*/
struct tegra_csi_channel {
struct list_head list;
unsigned int framerate;
unsigned int h_blank;
unsigned int v_blank;
+ struct tegra_mipi_device *mipi;
+ unsigned int pixel_rate;
};
/**
#endif
void tegra_csi_error_recover(struct v4l2_subdev *subdev);
+void tegra_csi_calc_settle_time(struct tegra_csi_channel *csi_chan,
+ u8 *clk_settle_time,
+ u8 *ths_settle_time);
#endif
* This source file contains Tegra210 supported video formats,
* VI and CSI SoC specific data, operations and registers accessors.
*/
+#include <linux/bitfield.h>
#include <linux/clk.h>
#include <linux/clk/tegra.h>
#include <linux/delay.h>
#define BRICK_CLOCK_B_4X (0x2 << 16)
#define TEGRA_CSI_CIL_PAD_CONFIG1 0x004
#define TEGRA_CSI_CIL_PHY_CONTROL 0x008
+#define CLK_SETTLE_MASK GENMASK(13, 8)
+#define THS_SETTLE_MASK GENMASK(5, 0)
#define TEGRA_CSI_CIL_INTERRUPT_MASK 0x00c
#define TEGRA_CSI_CIL_STATUS 0x010
#define TEGRA_CSI_CILX_STATUS 0x014
tegra_channel_capture_setup(chan);
/* recover CSI block */
- subdev = tegra_channel_get_remote_subdev(chan);
+ subdev = tegra_channel_get_remote_csi_subdev(chan);
tegra_csi_error_recover(subdev);
}
.ops = &tegra210_vi_ops,
.hw_revision = 3,
.vi_max_channels = 6,
+#if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
.vi_max_clk_hz = 499200000,
+#else
+ .vi_max_clk_hz = 998400000,
+#endif
};
/* Tegra210 CSI PHY registers accessors */
{
struct tegra_csi *csi = csi_chan->csi;
unsigned int portno = csi_chan->csi_port_num;
+ u8 clk_settle_time = 0;
+ u8 ths_settle_time = 10;
u32 val;
+ if (!csi_chan->pg_mode)
+ tegra_csi_calc_settle_time(csi_chan, &clk_settle_time,
+ &ths_settle_time);
+
csi_write(csi, portno, TEGRA_CSI_CLKEN_OVERRIDE, 0);
/* clean up status */
/* CIL PHY registers setup */
cil_write(csi, portno, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
- cil_write(csi, portno, TEGRA_CSI_CIL_PHY_CONTROL, 0xa);
+ cil_write(csi, portno, TEGRA_CSI_CIL_PHY_CONTROL,
+ FIELD_PREP(CLK_SETTLE_MASK, clk_settle_time) |
+ FIELD_PREP(THS_SETTLE_MASK, ths_settle_time));
/*
* The CSI unit provides for connection of up to six cameras in
BRICK_CLOCK_A_4X);
cil_write(csi, portno + 1, TEGRA_CSI_CIL_PAD_CONFIG0, 0x0);
cil_write(csi, portno + 1, TEGRA_CSI_CIL_INTERRUPT_MASK, 0x0);
- cil_write(csi, portno + 1, TEGRA_CSI_CIL_PHY_CONTROL, 0xa);
+ cil_write(csi, portno + 1, TEGRA_CSI_CIL_PHY_CONTROL,
+ FIELD_PREP(CLK_SETTLE_MASK, clk_settle_time) |
+ FIELD_PREP(THS_SETTLE_MASK, ths_settle_time));
csi_write(csi, portno, TEGRA_CSI_PHY_CIL_COMMAND,
CSI_A_PHY_CIL_ENABLE | CSI_B_PHY_CIL_ENABLE);
} else {
"cilab",
"cilcd",
"cile",
+#if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
"csi_tpg",
+#endif
};
/* Tegra210 CSI operations */
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_device.h>
+#include <linux/of_graph.h>
#include <linux/platform_device.h>
#include <linux/regulator/consumer.h>
#include <linux/pm_runtime.h>
return container_of(vb, struct tegra_channel_buffer, buf);
}
+static inline struct tegra_vi_graph_entity *
+to_tegra_vi_graph_entity(struct v4l2_async_subdev *asd)
+{
+ return container_of(asd, struct tegra_vi_graph_entity, asd);
+}
+
static int tegra_get_format_idx_by_code(struct tegra_vi *vi,
- unsigned int code)
+ unsigned int code,
+ unsigned int offset)
{
unsigned int i;
- for (i = 0; i < vi->soc->nformats; ++i) {
+ for (i = offset; i < vi->soc->nformats; ++i) {
if (vi->soc->video_formats[i].code == code)
return i;
}
}
struct v4l2_subdev *
-tegra_channel_get_remote_subdev(struct tegra_vi_channel *chan)
+tegra_channel_get_remote_csi_subdev(struct tegra_vi_channel *chan)
+{
+ struct media_pad *pad;
+
+ pad = media_entity_remote_pad(&chan->pad);
+ if (!pad)
+ return NULL;
+
+ return media_entity_to_v4l2_subdev(pad->entity);
+}
+
+struct v4l2_subdev *
+tegra_channel_get_remote_source_subdev(struct tegra_vi_channel *chan)
{
struct media_pad *pad;
struct v4l2_subdev *subdev;
struct media_entity *entity;
- pad = media_entity_remote_pad(&chan->pad);
- entity = pad->entity;
- subdev = media_entity_to_v4l2_subdev(entity);
+ subdev = tegra_channel_get_remote_csi_subdev(chan);
+ if (!subdev)
+ return NULL;
+
+ pad = &subdev->entity.pads[0];
+ while (!(pad->flags & MEDIA_PAD_FL_SOURCE)) {
+ pad = media_entity_remote_pad(pad);
+ if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
+ break;
+ entity = pad->entity;
+ pad = &entity->pads[0];
+ subdev = media_entity_to_v4l2_subdev(entity);
+ }
return subdev;
}
-int tegra_channel_set_stream(struct tegra_vi_channel *chan, bool on)
+static int tegra_channel_enable_stream(struct tegra_vi_channel *chan)
+{
+ struct v4l2_subdev *csi_subdev, *src_subdev;
+ struct tegra_csi_channel *csi_chan;
+ int ret, err;
+
+ /*
+ * Tegra CSI receiver can detect the first LP to HS transition.
+ * So, start the CSI stream-on prior to sensor stream-on and
+ * vice-versa for stream-off.
+ */
+ csi_subdev = tegra_channel_get_remote_csi_subdev(chan);
+ ret = v4l2_subdev_call(csi_subdev, video, s_stream, true);
+ if (ret < 0 && ret != -ENOIOCTLCMD)
+ return ret;
+
+ if (IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ return 0;
+
+ csi_chan = v4l2_get_subdevdata(csi_subdev);
+ /*
+ * TRM has incorrectly documented to wait for done status from
+ * calibration logic after CSI interface power on.
+ * As per the design, calibration results are latched and applied
+ * to the pads only when the link is in LP11 state which will happen
+ * during the sensor stream-on.
+ * CSI subdev stream-on triggers start of MIPI pads calibration.
+ * Wait for calibration to finish here after sensor subdev stream-on.
+ */
+ src_subdev = tegra_channel_get_remote_source_subdev(chan);
+ ret = v4l2_subdev_call(src_subdev, video, s_stream, true);
+ err = tegra_mipi_finish_calibration(csi_chan->mipi);
+
+ if (ret < 0 && ret != -ENOIOCTLCMD)
+ goto err_disable_csi_stream;
+
+ if (err < 0)
+ dev_warn(csi_chan->csi->dev,
+ "MIPI calibration failed: %d\n", err);
+
+ return 0;
+
+err_disable_csi_stream:
+ v4l2_subdev_call(csi_subdev, video, s_stream, false);
+ return ret;
+}
+
+static int tegra_channel_disable_stream(struct tegra_vi_channel *chan)
{
struct v4l2_subdev *subdev;
int ret;
- /* stream CSI */
- subdev = tegra_channel_get_remote_subdev(chan);
- ret = v4l2_subdev_call(subdev, video, s_stream, on);
- if (on && ret < 0 && ret != -ENOIOCTLCMD)
+ /*
+ * Stream-off subdevices in reverse order to stream-on.
+ * Remote source subdev in TPG mode is same as CSI subdev.
+ */
+ subdev = tegra_channel_get_remote_source_subdev(chan);
+ ret = v4l2_subdev_call(subdev, video, s_stream, false);
+ if (ret < 0 && ret != -ENOIOCTLCMD)
+ return ret;
+
+ if (IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ return 0;
+
+ subdev = tegra_channel_get_remote_csi_subdev(chan);
+ ret = v4l2_subdev_call(subdev, video, s_stream, false);
+ if (ret < 0 && ret != -ENOIOCTLCMD)
return ret;
return 0;
}
+int tegra_channel_set_stream(struct tegra_vi_channel *chan, bool on)
+{
+ int ret;
+
+ if (on)
+ ret = tegra_channel_enable_stream(chan);
+ else
+ ret = tegra_channel_disable_stream(chan);
+
+ return ret;
+}
+
void tegra_channel_release_buffers(struct tegra_vi_channel *chan,
enum vb2_buffer_state state)
{
struct tegra_vi_channel *chan = video_drvdata(file);
struct v4l2_subdev *subdev;
- subdev = tegra_channel_get_remote_subdev(chan);
+ subdev = tegra_channel_get_remote_source_subdev(chan);
return v4l2_g_parm_cap(&chan->video, subdev, a);
}
struct tegra_vi_channel *chan = video_drvdata(file);
struct v4l2_subdev *subdev;
- subdev = tegra_channel_get_remote_subdev(chan);
+ subdev = tegra_channel_get_remote_source_subdev(chan);
return v4l2_s_parm_cap(&chan->video, subdev, a);
}
fse.code = fmtinfo->code;
- subdev = tegra_channel_get_remote_subdev(chan);
+ subdev = tegra_channel_get_remote_source_subdev(chan);
ret = v4l2_subdev_call(subdev, pad, enum_frame_size, NULL, &fse);
if (ret)
return ret;
fie.code = fmtinfo->code;
- subdev = tegra_channel_get_remote_subdev(chan);
+ subdev = tegra_channel_get_remote_source_subdev(chan);
ret = v4l2_subdev_call(subdev, pad, enum_frame_interval, NULL, &fie);
if (ret)
return ret;
unsigned int index = 0, i;
unsigned long *fmts_bitmap = chan->tpg_fmts_bitmap;
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ fmts_bitmap = chan->fmts_bitmap;
+
if (f->index >= bitmap_weight(fmts_bitmap, MAX_FORMAT_NUM))
return -EINVAL;
struct v4l2_pix_format *pix,
unsigned int bpp)
{
- unsigned int align;
- unsigned int min_width;
- unsigned int max_width;
- unsigned int width;
unsigned int min_bpl;
unsigned int max_bpl;
unsigned int bpl;
/*
- * The transfer alignment requirements are expressed in bytes. Compute
- * minimum and maximum values, clamp the requested width and convert
- * it back to pixels. Use bytesperline to adjust the width.
+ * The transfer alignment requirements are expressed in bytes.
+ * Clamp the requested width and height to the limits.
*/
- align = lcm(SURFACE_ALIGN_BYTES, bpp);
- min_width = roundup(TEGRA_MIN_WIDTH, align);
- max_width = rounddown(TEGRA_MAX_WIDTH, align);
- width = roundup(pix->width * bpp, align);
-
- pix->width = clamp(width, min_width, max_width) / bpp;
+ pix->width = clamp(pix->width, TEGRA_MIN_WIDTH, TEGRA_MAX_WIDTH);
pix->height = clamp(pix->height, TEGRA_MIN_HEIGHT, TEGRA_MAX_HEIGHT);
/* Clamp the requested bytes per line value. If the maximum bytes per
struct v4l2_subdev *subdev;
struct v4l2_subdev_format fmt;
struct v4l2_subdev_pad_config *pad_cfg;
+ struct v4l2_subdev_frame_size_enum fse = {
+ .which = V4L2_SUBDEV_FORMAT_TRY,
+ };
+ struct v4l2_subdev_selection sdsel = {
+ .which = V4L2_SUBDEV_FORMAT_ACTIVE,
+ .target = V4L2_SEL_TGT_CROP_BOUNDS,
+ };
+ int ret;
+
+ subdev = tegra_channel_get_remote_source_subdev(chan);
+ if (!subdev)
+ return -ENODEV;
- subdev = tegra_channel_get_remote_subdev(chan);
pad_cfg = v4l2_subdev_alloc_pad_config(subdev);
if (!pad_cfg)
return -ENOMEM;
fmt.which = V4L2_SUBDEV_FORMAT_TRY;
fmt.pad = 0;
v4l2_fill_mbus_format(&fmt.format, pix, fmtinfo->code);
- v4l2_subdev_call(subdev, pad, set_fmt, pad_cfg, &fmt);
+
+ /*
+ * Attempt to obtain the format size from subdev.
+ * If not available, try to get crop boundary from subdev.
+ */
+ fse.code = fmtinfo->code;
+ ret = v4l2_subdev_call(subdev, pad, enum_frame_size, pad_cfg, &fse);
+ if (ret) {
+ ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel);
+ if (ret)
+ return -EINVAL;
+ pad_cfg->try_crop.width = sdsel.r.width;
+ pad_cfg->try_crop.height = sdsel.r.height;
+ } else {
+ pad_cfg->try_crop.width = fse.max_width;
+ pad_cfg->try_crop.height = fse.max_height;
+ }
+
+ ret = v4l2_subdev_call(subdev, pad, set_fmt, pad_cfg, &fmt);
+ if (ret < 0)
+ return ret;
+
v4l2_fill_pix_format(pix, &fmt.format);
tegra_channel_fmt_align(chan, pix, fmtinfo->bpp);
fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
fmt.pad = 0;
v4l2_fill_mbus_format(&fmt.format, pix, fmtinfo->code);
- subdev = tegra_channel_get_remote_subdev(chan);
- v4l2_subdev_call(subdev, pad, set_fmt, NULL, &fmt);
+ subdev = tegra_channel_get_remote_source_subdev(chan);
+ ret = v4l2_subdev_call(subdev, pad, set_fmt, NULL, &fmt);
+ if (ret < 0)
+ return ret;
+
v4l2_fill_pix_format(pix, &fmt.format);
tegra_channel_fmt_align(chan, pix, fmtinfo->bpp);
return 0;
}
+static int tegra_channel_set_subdev_active_fmt(struct tegra_vi_channel *chan)
+{
+ int ret, index;
+ struct v4l2_subdev *subdev;
+ struct v4l2_subdev_format fmt = {
+ .which = V4L2_SUBDEV_FORMAT_ACTIVE,
+ };
+
+ /*
+ * Initialize channel format to the sub-device active format if there
+ * is corresponding match in the Tegra supported video formats.
+ */
+ subdev = tegra_channel_get_remote_source_subdev(chan);
+ ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &fmt);
+ if (ret)
+ return ret;
+
+ index = tegra_get_format_idx_by_code(chan->vi, fmt.format.code, 0);
+ if (index < 0)
+ return -EINVAL;
+
+ chan->fmtinfo = &chan->vi->soc->video_formats[index];
+ v4l2_fill_pix_format(&chan->format, &fmt.format);
+ chan->format.pixelformat = chan->fmtinfo->fourcc;
+ chan->format.bytesperline = chan->format.width * chan->fmtinfo->bpp;
+ chan->format.sizeimage = chan->format.bytesperline *
+ chan->format.height;
+ tegra_channel_fmt_align(chan, &chan->format, chan->fmtinfo->bpp);
+
+ return 0;
+}
+
+static int tegra_channel_g_selection(struct file *file, void *priv,
+ struct v4l2_selection *sel)
+{
+ struct tegra_vi_channel *chan = video_drvdata(file);
+ struct v4l2_subdev *subdev;
+ struct v4l2_subdev_format fmt = {
+ .which = V4L2_SUBDEV_FORMAT_ACTIVE,
+ };
+ struct v4l2_subdev_selection sdsel = {
+ .which = V4L2_SUBDEV_FORMAT_ACTIVE,
+ .target = sel->target,
+ };
+ int ret;
+
+ subdev = tegra_channel_get_remote_source_subdev(chan);
+ if (!v4l2_subdev_has_op(subdev, pad, get_selection))
+ return -ENOTTY;
+
+ if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+ return -EINVAL;
+ /*
+ * Try the get selection operation and fallback to get format if not
+ * implemented.
+ */
+ ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel);
+ if (!ret)
+ sel->r = sdsel.r;
+ if (ret != -ENOIOCTLCMD)
+ return ret;
+
+ ret = v4l2_subdev_call(subdev, pad, get_fmt, NULL, &fmt);
+ if (ret < 0)
+ return ret;
+
+ sel->r.left = 0;
+ sel->r.top = 0;
+ sel->r.width = fmt.format.width;
+ sel->r.height = fmt.format.height;
+
+ return 0;
+}
+
+static int tegra_channel_s_selection(struct file *file, void *fh,
+ struct v4l2_selection *sel)
+{
+ struct tegra_vi_channel *chan = video_drvdata(file);
+ struct v4l2_subdev *subdev;
+ int ret;
+ struct v4l2_subdev_selection sdsel = {
+ .which = V4L2_SUBDEV_FORMAT_ACTIVE,
+ .target = sel->target,
+ .flags = sel->flags,
+ .r = sel->r,
+ };
+
+ subdev = tegra_channel_get_remote_source_subdev(chan);
+ if (!v4l2_subdev_has_op(subdev, pad, set_selection))
+ return -ENOTTY;
+
+ if (sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+ return -EINVAL;
+
+ if (vb2_is_busy(&chan->queue))
+ return -EBUSY;
+
+ ret = v4l2_subdev_call(subdev, pad, set_selection, NULL, &sdsel);
+ if (!ret) {
+ sel->r = sdsel.r;
+ /*
+ * Subdev active format resolution may have changed during
+ * set selection operation. So, update channel format to
+ * the sub-device active format.
+ */
+ return tegra_channel_set_subdev_active_fmt(chan);
+ }
+
+ return ret;
+}
+
static int tegra_channel_enum_input(struct file *file, void *fh,
struct v4l2_input *inp)
{
- /* currently driver supports internal TPG only */
+ struct tegra_vi_channel *chan = video_drvdata(file);
+ struct v4l2_subdev *subdev;
+
if (inp->index)
return -EINVAL;
inp->type = V4L2_INPUT_TYPE_CAMERA;
- strscpy(inp->name, "Tegra TPG", sizeof(inp->name));
+ subdev = tegra_channel_get_remote_source_subdev(chan);
+ strscpy(inp->name, subdev->name, sizeof(inp->name));
return 0;
}
.vidioc_streamoff = vb2_ioctl_streamoff,
.vidioc_subscribe_event = v4l2_ctrl_subscribe_event,
.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
+ .vidioc_g_selection = tegra_channel_g_selection,
+ .vidioc_s_selection = tegra_channel_s_selection,
};
/*
/*
* V4L2 control operations
*/
+#if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
static int vi_s_ctrl(struct v4l2_ctrl *ctrl)
{
struct tegra_vi_channel *chan = container_of(ctrl->handler,
"Black/White Direct Mode",
"Color Patch Mode",
};
+#endif
static int tegra_channel_setup_ctrl_handler(struct tegra_vi_channel *chan)
{
int ret;
+#if IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)
/* add test pattern control handler to v4l2 device */
v4l2_ctrl_new_std_menu_items(&chan->ctrl_handler, &vi_ctrl_ops,
V4L2_CID_TEST_PATTERN,
v4l2_ctrl_handler_free(&chan->ctrl_handler);
return chan->ctrl_handler.error;
}
+#else
+ struct v4l2_subdev *subdev;
+
+ subdev = tegra_channel_get_remote_source_subdev(chan);
+ if (!subdev)
+ return -ENODEV;
+
+ ret = v4l2_ctrl_add_handler(&chan->ctrl_handler, subdev->ctrl_handler,
+ NULL, true);
+ if (ret < 0) {
+ dev_err(chan->vi->dev,
+ "failed to add subdev %s ctrl handler: %d\n",
+ subdev->name, ret);
+ v4l2_ctrl_handler_free(&chan->ctrl_handler);
+ return ret;
+ }
+#endif
/* setup the controls */
ret = v4l2_ctrl_handler_setup(&chan->ctrl_handler);
bitmap_zero(chan->tpg_fmts_bitmap, MAX_FORMAT_NUM);
index = tegra_get_format_idx_by_code(chan->vi,
- MEDIA_BUS_FMT_SRGGB10_1X10);
+ MEDIA_BUS_FMT_SRGGB10_1X10, 0);
bitmap_set(chan->tpg_fmts_bitmap, index, 1);
index = tegra_get_format_idx_by_code(chan->vi,
- MEDIA_BUS_FMT_RGB888_1X32_PADHI);
+ MEDIA_BUS_FMT_RGB888_1X32_PADHI,
+ 0);
bitmap_set(chan->tpg_fmts_bitmap, index, 1);
}
+static int vi_fmts_bitmap_init(struct tegra_vi_channel *chan)
+{
+ int index, ret, match_code = 0;
+ struct v4l2_subdev *subdev;
+ struct v4l2_subdev_mbus_code_enum code = {
+ .which = V4L2_SUBDEV_FORMAT_ACTIVE,
+ };
+
+ bitmap_zero(chan->fmts_bitmap, MAX_FORMAT_NUM);
+
+ /*
+ * Set the bitmap bits based on all the matched formats between the
+ * available media bus formats of sub-device and the pre-defined Tegra
+ * supported video formats.
+ */
+ subdev = tegra_channel_get_remote_source_subdev(chan);
+ while (1) {
+ ret = v4l2_subdev_call(subdev, pad, enum_mbus_code,
+ NULL, &code);
+ if (ret < 0)
+ break;
+
+ index = tegra_get_format_idx_by_code(chan->vi, code.code, 0);
+ while (index >= 0) {
+ bitmap_set(chan->fmts_bitmap, index, 1);
+ if (!match_code)
+ match_code = code.code;
+ /* look for other formats with same mbus code */
+ index = tegra_get_format_idx_by_code(chan->vi,
+ code.code,
+ index + 1);
+ }
+
+ code.index++;
+ }
+
+ /*
+ * Set the bitmap bit corresponding to default tegra video format if
+ * there are no matched formats.
+ */
+ if (!match_code) {
+ match_code = tegra_default_format.code;
+ index = tegra_get_format_idx_by_code(chan->vi, match_code, 0);
+ if (WARN_ON(index < 0))
+ return -EINVAL;
+
+ bitmap_set(chan->fmts_bitmap, index, 1);
+ }
+
+ /* initialize channel format to the sub-device active format */
+ tegra_channel_set_subdev_active_fmt(chan);
+
+ return 0;
+}
+
static void tegra_channel_cleanup(struct tegra_vi_channel *chan)
{
v4l2_ctrl_handler_free(&chan->ctrl_handler);
goto free_v4l2_ctrl_hdl;
}
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ v4l2_async_notifier_init(&chan->notifier);
+
return 0;
free_v4l2_ctrl_hdl:
return ret;
}
-static int tegra_vi_tpg_channels_alloc(struct tegra_vi *vi)
+static int tegra_vi_channel_alloc(struct tegra_vi *vi, unsigned int port_num,
+ struct device_node *node)
{
struct tegra_vi_channel *chan;
+
+ /*
+ * Do not use devm_kzalloc as memory is freed immediately
+ * when device instance is unbound but application might still
+ * be holding the device node open. Channel memory allocated
+ * with kzalloc is freed during video device release callback.
+ */
+ chan = kzalloc(sizeof(*chan), GFP_KERNEL);
+ if (!chan)
+ return -ENOMEM;
+
+ chan->vi = vi;
+ chan->portno = port_num;
+ chan->of_node = node;
+ list_add_tail(&chan->list, &vi->vi_chans);
+
+ return 0;
+}
+
+static int tegra_vi_tpg_channels_alloc(struct tegra_vi *vi)
+{
unsigned int port_num;
unsigned int nchannels = vi->soc->vi_max_channels;
+ int ret;
for (port_num = 0; port_num < nchannels; port_num++) {
- /*
- * Do not use devm_kzalloc as memory is freed immediately
- * when device instance is unbound but application might still
- * be holding the device node open. Channel memory allocated
- * with kzalloc is freed during video device release callback.
- */
- chan = kzalloc(sizeof(*chan), GFP_KERNEL);
- if (!chan)
- return -ENOMEM;
-
- chan->vi = vi;
- chan->portno = port_num;
- list_add_tail(&chan->list, &vi->vi_chans);
+ ret = tegra_vi_channel_alloc(vi, port_num, vi->dev->of_node);
+ if (ret < 0)
+ return ret;
}
return 0;
}
+static int tegra_vi_channels_alloc(struct tegra_vi *vi)
+{
+ struct device_node *node = vi->dev->of_node;
+ struct device_node *ep = NULL;
+ struct device_node *ports;
+ struct device_node *port;
+ unsigned int port_num;
+ int ret = 0;
+
+ ports = of_get_child_by_name(node, "ports");
+ if (!ports)
+ return -ENODEV;
+
+ for_each_child_of_node(ports, port) {
+ if (!of_node_name_eq(port, "port"))
+ continue;
+
+ ret = of_property_read_u32(port, "reg", &port_num);
+ if (ret < 0)
+ continue;
+
+ if (port_num > vi->soc->vi_max_channels) {
+ dev_err(vi->dev, "invalid port num %d for %pOF\n",
+ port_num, port);
+ ret = -EINVAL;
+ of_node_put(port);
+ goto cleanup;
+ }
+
+ ep = of_get_child_by_name(port, "endpoint");
+ if (!ep)
+ continue;
+
+ of_node_put(ep);
+ ret = tegra_vi_channel_alloc(vi, port_num, port);
+ if (ret < 0) {
+ of_node_put(port);
+ goto cleanup;
+ }
+ }
+
+cleanup:
+ of_node_put(ports);
+ return ret;
+}
+
static int tegra_vi_channels_init(struct tegra_vi *vi)
{
struct tegra_vi_channel *chan;
struct tegra_csi_channel *csi_chan;
struct tegra_vi_channel *chan;
- list_for_each_entry(chan, &vi->vi_chans, list) {
- video_unregister_device(&chan->video);
- mutex_lock(&chan->video_lock);
- vb2_queue_release(&chan->queue);
- mutex_unlock(&chan->video_lock);
- }
+ list_for_each_entry(chan, &vi->vi_chans, list)
+ vb2_video_unregister_device(&chan->video);
list_for_each_entry(csi_chan, &csi->csi_chans, list)
v4l2_device_unregister_subdev(&csi_chan->subdev);
return 0;
}
+/*
+ * Graph Management
+ */
+static struct tegra_vi_graph_entity *
+tegra_vi_graph_find_entity(struct tegra_vi_channel *chan,
+ const struct fwnode_handle *fwnode)
+{
+ struct tegra_vi_graph_entity *entity;
+ struct v4l2_async_subdev *asd;
+
+ list_for_each_entry(asd, &chan->notifier.asd_list, asd_list) {
+ entity = to_tegra_vi_graph_entity(asd);
+ if (entity->asd.match.fwnode == fwnode)
+ return entity;
+ }
+
+ return NULL;
+}
+
+static int tegra_vi_graph_build(struct tegra_vi_channel *chan,
+ struct tegra_vi_graph_entity *entity)
+{
+ struct tegra_vi *vi = chan->vi;
+ struct tegra_vi_graph_entity *ent;
+ struct fwnode_handle *ep = NULL;
+ struct v4l2_fwnode_link link;
+ struct media_entity *local = entity->entity;
+ struct media_entity *remote;
+ struct media_pad *local_pad;
+ struct media_pad *remote_pad;
+ u32 link_flags = MEDIA_LNK_FL_ENABLED;
+ int ret = 0;
+
+ dev_dbg(vi->dev, "creating links for entity %s\n", local->name);
+
+ while (1) {
+ ep = fwnode_graph_get_next_endpoint(entity->asd.match.fwnode,
+ ep);
+ if (!ep)
+ break;
+
+ ret = v4l2_fwnode_parse_link(ep, &link);
+ if (ret < 0) {
+ dev_err(vi->dev, "failed to parse link for %pOF: %d\n",
+ to_of_node(ep), ret);
+ continue;
+ }
+
+ if (link.local_port >= local->num_pads) {
+ dev_err(vi->dev, "invalid port number %u on %pOF\n",
+ link.local_port, to_of_node(link.local_node));
+ v4l2_fwnode_put_link(&link);
+ ret = -EINVAL;
+ break;
+ }
+
+ local_pad = &local->pads[link.local_port];
+ /* Remote node is vi node. So use channel video entity and pad
+ * as remote/sink.
+ */
+ if (link.remote_node == of_fwnode_handle(vi->dev->of_node)) {
+ remote = &chan->video.entity;
+ remote_pad = &chan->pad;
+ goto create_link;
+ }
+
+ /*
+ * Skip sink ports, they will be processed from the other end
+ * of the link.
+ */
+ if (local_pad->flags & MEDIA_PAD_FL_SINK) {
+ dev_dbg(vi->dev, "skipping sink port %pOF:%u\n",
+ to_of_node(link.local_node), link.local_port);
+ v4l2_fwnode_put_link(&link);
+ continue;
+ }
+
+ /* find the remote entity from notifier list */
+ ent = tegra_vi_graph_find_entity(chan, link.remote_node);
+ if (!ent) {
+ dev_err(vi->dev, "no entity found for %pOF\n",
+ to_of_node(link.remote_node));
+ v4l2_fwnode_put_link(&link);
+ ret = -ENODEV;
+ break;
+ }
+
+ remote = ent->entity;
+ if (link.remote_port >= remote->num_pads) {
+ dev_err(vi->dev, "invalid port number %u on %pOF\n",
+ link.remote_port,
+ to_of_node(link.remote_node));
+ v4l2_fwnode_put_link(&link);
+ ret = -EINVAL;
+ break;
+ }
+
+ remote_pad = &remote->pads[link.remote_port];
+
+create_link:
+ dev_dbg(vi->dev, "creating %s:%u -> %s:%u link\n",
+ local->name, local_pad->index,
+ remote->name, remote_pad->index);
+
+ ret = media_create_pad_link(local, local_pad->index,
+ remote, remote_pad->index,
+ link_flags);
+ v4l2_fwnode_put_link(&link);
+ if (ret < 0) {
+ dev_err(vi->dev,
+ "failed to create %s:%u -> %s:%u link: %d\n",
+ local->name, local_pad->index,
+ remote->name, remote_pad->index, ret);
+ break;
+ }
+ }
+
+ fwnode_handle_put(ep);
+ return ret;
+}
+
+static int tegra_vi_graph_notify_complete(struct v4l2_async_notifier *notifier)
+{
+ struct tegra_vi_graph_entity *entity;
+ struct v4l2_async_subdev *asd;
+ struct v4l2_subdev *subdev;
+ struct tegra_vi_channel *chan;
+ struct tegra_vi *vi;
+ int ret;
+
+ chan = container_of(notifier, struct tegra_vi_channel, notifier);
+ vi = chan->vi;
+
+ dev_dbg(vi->dev, "notify complete, all subdevs registered\n");
+
+ /*
+ * Video device node should be created at the end of all the device
+ * related initialization/setup.
+ * Current video_register_device() does both initialize and register
+ * video device in same API.
+ *
+ * TODO: Update v4l2-dev driver to split initialize and register into
+ * separate APIs and then update Tegra video driver to do video device
+ * initialize followed by all video device related setup and then
+ * register the video device.
+ */
+ ret = video_register_device(&chan->video, VFL_TYPE_VIDEO, -1);
+ if (ret < 0) {
+ dev_err(vi->dev,
+ "failed to register video device: %d\n", ret);
+ goto unregister_video;
+ }
+
+ /* create links between the entities */
+ list_for_each_entry(asd, &chan->notifier.asd_list, asd_list) {
+ entity = to_tegra_vi_graph_entity(asd);
+ ret = tegra_vi_graph_build(chan, entity);
+ if (ret < 0)
+ goto unregister_video;
+ }
+
+ ret = tegra_channel_setup_ctrl_handler(chan);
+ if (ret < 0) {
+ dev_err(vi->dev,
+ "failed to setup channel controls: %d\n", ret);
+ goto unregister_video;
+ }
+
+ ret = vi_fmts_bitmap_init(chan);
+ if (ret < 0) {
+ dev_err(vi->dev,
+ "failed to initialize formats bitmap: %d\n", ret);
+ goto unregister_video;
+ }
+
+ subdev = tegra_channel_get_remote_csi_subdev(chan);
+ if (!subdev) {
+ ret = -ENODEV;
+ dev_err(vi->dev,
+ "failed to get remote csi subdev: %d\n", ret);
+ goto unregister_video;
+ }
+
+ v4l2_set_subdev_hostdata(subdev, chan);
+
+ return 0;
+
+unregister_video:
+ vb2_video_unregister_device(&chan->video);
+ return ret;
+}
+
+static int tegra_vi_graph_notify_bound(struct v4l2_async_notifier *notifier,
+ struct v4l2_subdev *subdev,
+ struct v4l2_async_subdev *asd)
+{
+ struct tegra_vi_graph_entity *entity;
+ struct tegra_vi *vi;
+ struct tegra_vi_channel *chan;
+
+ chan = container_of(notifier, struct tegra_vi_channel, notifier);
+ vi = chan->vi;
+
+ /*
+ * Locate the entity corresponding to the bound subdev and store the
+ * subdev pointer.
+ */
+ entity = tegra_vi_graph_find_entity(chan, subdev->fwnode);
+ if (!entity) {
+ dev_err(vi->dev, "no entity for subdev %s\n", subdev->name);
+ return -EINVAL;
+ }
+
+ if (entity->subdev) {
+ dev_err(vi->dev, "duplicate subdev for node %pOF\n",
+ to_of_node(entity->asd.match.fwnode));
+ return -EINVAL;
+ }
+
+ dev_dbg(vi->dev, "subdev %s bound\n", subdev->name);
+ entity->entity = &subdev->entity;
+ entity->subdev = subdev;
+
+ return 0;
+}
+
+static const struct v4l2_async_notifier_operations tegra_vi_async_ops = {
+ .bound = tegra_vi_graph_notify_bound,
+ .complete = tegra_vi_graph_notify_complete,
+};
+
+static int tegra_vi_graph_parse_one(struct tegra_vi_channel *chan,
+ struct fwnode_handle *fwnode)
+{
+ struct tegra_vi *vi = chan->vi;
+ struct fwnode_handle *ep = NULL;
+ struct fwnode_handle *remote = NULL;
+ struct v4l2_async_subdev *asd;
+ struct device_node *node = NULL;
+ int ret;
+
+ dev_dbg(vi->dev, "parsing node %pOF\n", to_of_node(fwnode));
+
+ /* parse all the remote entities and put them into the list */
+ for_each_endpoint_of_node(to_of_node(fwnode), node) {
+ ep = of_fwnode_handle(node);
+ remote = fwnode_graph_get_remote_port_parent(ep);
+ if (!remote) {
+ dev_err(vi->dev,
+ "remote device at %pOF not found\n", node);
+ ret = -EINVAL;
+ goto cleanup;
+ }
+
+ /* skip entities that are already processed */
+ if (remote == dev_fwnode(vi->dev) ||
+ tegra_vi_graph_find_entity(chan, remote)) {
+ fwnode_handle_put(remote);
+ continue;
+ }
+
+ asd = v4l2_async_notifier_add_fwnode_subdev(&chan->notifier,
+ remote, sizeof(struct tegra_vi_graph_entity));
+ if (IS_ERR(asd)) {
+ ret = PTR_ERR(asd);
+ dev_err(vi->dev,
+ "failed to add subdev to notifier: %d\n", ret);
+ fwnode_handle_put(remote);
+ goto cleanup;
+ }
+
+ ret = tegra_vi_graph_parse_one(chan, remote);
+ if (ret < 0) {
+ fwnode_handle_put(remote);
+ goto cleanup;
+ }
+
+ fwnode_handle_put(remote);
+ }
+
+ return 0;
+
+cleanup:
+ dev_err(vi->dev, "failed parsing the graph: %d\n", ret);
+ v4l2_async_notifier_cleanup(&chan->notifier);
+ of_node_put(node);
+ return ret;
+}
+
+static int tegra_vi_graph_init(struct tegra_vi *vi)
+{
+ struct tegra_video_device *vid = dev_get_drvdata(vi->client.host);
+ struct tegra_vi_channel *chan;
+ struct fwnode_handle *fwnode = dev_fwnode(vi->dev);
+ int ret;
+ struct fwnode_handle *remote = NULL;
+
+ /*
+ * Walk the links to parse the full graph. Each channel will have
+ * one endpoint of the composite node. Start by parsing the
+ * composite node and parse the remote entities in turn.
+ * Each channel will register v4l2 async notifier to make the graph
+ * independent between the channels so we can the current channel
+ * in case of something wrong during graph parsing and continue with
+ * next channels.
+ */
+ list_for_each_entry(chan, &vi->vi_chans, list) {
+ remote = fwnode_graph_get_remote_node(fwnode, chan->portno, 0);
+ if (!remote)
+ continue;
+
+ ret = tegra_vi_graph_parse_one(chan, remote);
+ fwnode_handle_put(remote);
+ if (ret < 0 || list_empty(&chan->notifier.asd_list))
+ continue;
+
+ chan->notifier.ops = &tegra_vi_async_ops;
+ ret = v4l2_async_notifier_register(&vid->v4l2_dev,
+ &chan->notifier);
+ if (ret < 0) {
+ dev_err(vi->dev,
+ "failed to register channel %d notifier: %d\n",
+ chan->portno, ret);
+ v4l2_async_notifier_cleanup(&chan->notifier);
+ }
+ }
+
+ return 0;
+}
+
+static void tegra_vi_graph_cleanup(struct tegra_vi *vi)
+{
+ struct tegra_vi_channel *chan;
+
+ list_for_each_entry(chan, &vi->vi_chans, list) {
+ vb2_video_unregister_device(&chan->video);
+ v4l2_async_notifier_unregister(&chan->notifier);
+ v4l2_async_notifier_cleanup(&chan->notifier);
+ }
+}
+
static int tegra_vi_init(struct host1x_client *client)
{
struct tegra_video_device *vid = dev_get_drvdata(client->host);
INIT_LIST_HEAD(&vi->vi_chans);
- ret = tegra_vi_tpg_channels_alloc(vi);
+ if (IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ ret = tegra_vi_tpg_channels_alloc(vi);
+ else
+ ret = tegra_vi_channels_alloc(vi);
if (ret < 0) {
- dev_err(vi->dev, "failed to allocate tpg channels: %d\n", ret);
+ dev_err(vi->dev,
+ "failed to allocate vi channels: %d\n", ret);
goto free_chans;
}
vid->vi = vi;
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)) {
+ ret = tegra_vi_graph_init(vi);
+ if (ret < 0)
+ goto free_chans;
+ }
+
return 0;
free_chans:
static int tegra_vi_exit(struct host1x_client *client)
{
+ struct tegra_vi *vi = host1x_client_to_vi(client);
+
/*
* Do not cleanup the channels here as application might still be
* holding video device nodes. Channels cleanup will happen during
* device nodes are released.
*/
+ if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ tegra_vi_graph_cleanup(vi);
+
return 0;
}
#include <linux/wait.h>
#include <media/media-entity.h>
+#include <media/v4l2-async.h>
#include <media/v4l2-ctrls.h>
#include <media/v4l2-device.h>
#include <media/v4l2-dev.h>
struct list_head vi_chans;
};
+/**
+ * struct tegra_vi_graph_entity - Entity in the video graph
+ *
+ * @asd: subdev asynchronous registration information
+ * @entity: media entity from the corresponding V4L2 subdev
+ * @subdev: V4L2 subdev
+ */
+struct tegra_vi_graph_entity {
+ struct v4l2_async_subdev asd;
+ struct media_entity *entity;
+ struct v4l2_subdev *subdev;
+};
+
/**
* struct tegra_vi_channel - Tegra video channel
*
* @done_lock: protects the capture done queue list
*
* @portno: VI channel port number
+ * @of_node: device node of VI channel
*
* @ctrl_handler: V4L2 control handler of this video channel
+ * @fmts_bitmap: a bitmap for supported formats matching v4l2 subdev formats
* @tpg_fmts_bitmap: a bitmap for supported TPG formats
* @pg_mode: test pattern generator mode (disabled/direct/patch)
+ * @notifier: V4L2 asynchronous subdevs notifier
*/
struct tegra_vi_channel {
struct list_head list;
spinlock_t done_lock;
unsigned char portno;
+ struct device_node *of_node;
struct v4l2_ctrl_handler ctrl_handler;
+ DECLARE_BITMAP(fmts_bitmap, MAX_FORMAT_NUM);
DECLARE_BITMAP(tpg_fmts_bitmap, MAX_FORMAT_NUM);
enum tegra_vi_pg_mode pg_mode;
+
+ struct v4l2_async_notifier notifier;
};
/**
#endif
struct v4l2_subdev *
-tegra_channel_get_remote_subdev(struct tegra_vi_channel *chan);
+tegra_channel_get_remote_csi_subdev(struct tegra_vi_channel *chan);
+struct v4l2_subdev *
+tegra_channel_get_remote_source_subdev(struct tegra_vi_channel *chan);
int tegra_channel_set_stream(struct tegra_vi_channel *chan, bool on);
void tegra_channel_release_buffers(struct tegra_vi_channel *chan,
enum vb2_buffer_state state);
if (ret < 0)
goto unregister_v4l2;
- /*
- * Both vi and csi channels are available now.
- * Register v4l2 nodes and create media links for TPG.
- */
- ret = tegra_v4l2_nodes_setup_tpg(vid);
- if (ret < 0) {
- dev_err(&dev->dev,
- "failed to setup tpg graph: %d\n", ret);
- goto device_exit;
+ if (IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)) {
+ /*
+ * Both vi and csi channels are available now.
+ * Register v4l2 nodes and create media links for TPG.
+ */
+ ret = tegra_v4l2_nodes_setup_tpg(vid);
+ if (ret < 0) {
+ dev_err(&dev->dev,
+ "failed to setup tpg graph: %d\n", ret);
+ goto device_exit;
+ }
}
return 0;
{
struct tegra_video_device *vid = dev_get_drvdata(&dev->dev);
- tegra_v4l2_nodes_cleanup_tpg(vid);
+ if (IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG))
+ tegra_v4l2_nodes_cleanup_tpg(vid);
host1x_device_exit(dev);
+++ /dev/null
-# SPDX-License-Identifier: GPL-2.0-only
-config VIDEO_USBVISION
- tristate "USB video devices based on Nogatech NT1003/1004/1005 (Deprecated)"
- depends on MEDIA_USB_SUPPORT && I2C && VIDEO_V4L2 && USB
- select VIDEO_TUNER
- select VIDEO_SAA711X if MEDIA_SUBDRV_AUTOSELECT
- help
- There are more than 50 different USB video devices based on
- NT1003/1004/1005 USB Bridges. This driver enables using those
- devices.
-
- This driver is deprecated and scheduled for removal by the
- end of 2020. See the TODO file in drivers/staging/media/usbvision
- for a list of actions that have to be done in order to prevent
- removal of this driver.
-
- To compile this driver as a module, choose M here: the
- module will be called usbvision.
+++ /dev/null
-# SPDX-License-Identifier: GPL-2.0-only
-usbvision-objs := usbvision-core.o usbvision-video.o usbvision-i2c.o usbvision-cards.o
-
-obj-$(CONFIG_VIDEO_USBVISION) += usbvision.o
+++ /dev/null
-The driver is deprecated and scheduled for removal by the end
-of 2020.
-
-In order to prevent removal the following actions would have to
-be taken:
-
-- clean up the code
-- convert to the vb2 framework
-- fix the disconnect and free-on-last-user handling (i.e., add
- a release callback for struct v4l2_device and rework the code
- to use that correctly).
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * usbvision-cards.c
- * usbvision cards definition file
- *
- * Copyright (c) 1999-2005 Joerg Heckenbach <joerg@heckenbach-aw.de>
- *
- * This module is part of usbvision driver project.
- * Updates to driver completed by Dwaine P. Garden
- */
-
-
-#include <linux/list.h>
-#include <linux/module.h>
-#include <media/v4l2-dev.h>
-#include <media/tuner.h>
-#include "usbvision.h"
-#include "usbvision-cards.h"
-
-/* Supported Devices: A table for usbvision.c*/
-struct usbvision_device_data_st usbvision_device_data[] = {
- [XANBOO] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 4,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Xanboo",
- },
- [BELKIN_VIDEOBUS_II] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 2,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Belkin USB VideoBus II Adapter",
- },
- [BELKIN_VIDEOBUS] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 2,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Belkin Components USB VideoBus",
- },
- [BELKIN_USB_VIDEOBUS_II] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 2,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Belkin USB VideoBus II",
- },
- [ECHOFX_INTERVIEW_LITE] = {
- .interface = 0,
- .codec = CODEC_SAA7111,
- .video_channels = 2,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 0,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = -1,
- .y_offset = -1,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "echoFX InterView Lite",
- },
- [USBGEAR_USBG_V1] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 2,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "USBGear USBG-V1 resp. HAMA USB",
- },
- [D_LINK_V100] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 4,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 0,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "D-Link V100",
- },
- [X10_USB_CAMERA] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 2,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "X10 USB Camera",
- },
- [HPG_WINTV_LIVE_PAL_BG] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 2,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = -1,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Live (PAL B/G)",
- },
- [HPG_WINTV_LIVE_PRO_NTSC_MN] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 2,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 0,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Live Pro (NTSC M/N)",
- },
- [ZORAN_PMD_NOGATECH] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 2,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 2,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Zoran Co. PMD (Nogatech) AV-grabber Manhattan",
- },
- [NOGATECH_USB_TV_NTSC_FM] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_NTSC_M,
- .x_offset = -1,
- .y_offset = 20,
- .model_string = "Nogatech USB-TV (NTSC) FM",
- },
- [PNY_USB_TV_NTSC_FM] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_NTSC_M,
- .x_offset = -1,
- .y_offset = 20,
- .model_string = "PNY USB-TV (NTSC) FM",
- },
- [PV_PLAYTV_USB_PRO_PAL_FM] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "PixelView PlayTv-USB PRO (PAL) FM",
- },
- [ZT_721] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "ZTV ZT-721 2.4GHz USB A/V Receiver",
- },
- [HPG_WINTV_NTSC_MN] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_NTSC_M,
- .x_offset = -1,
- .y_offset = 20,
- .model_string = "Hauppauge WinTV USB (NTSC M/N)",
- },
- [HPG_WINTV_PAL_BG] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Hauppauge WinTV USB (PAL B/G)",
- },
- [HPG_WINTV_PAL_I] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Hauppauge WinTV USB (PAL I)",
- },
- [HPG_WINTV_PAL_SECAM_L] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_SECAM,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_SECAM,
- .x_offset = 0x80,
- .y_offset = 0x16,
- .model_string = "Hauppauge WinTV USB (PAL/SECAM L)",
- },
- [HPG_WINTV_PAL_D_K] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Hauppauge WinTV USB (PAL D/K)",
- },
- [HPG_WINTV_NTSC_FM] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_NTSC_M,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Hauppauge WinTV USB (NTSC FM)",
- },
- [HPG_WINTV_PAL_BG_FM] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Hauppauge WinTV USB (PAL B/G FM)",
- },
- [HPG_WINTV_PAL_I_FM] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Hauppauge WinTV USB (PAL I FM)",
- },
- [HPG_WINTV_PAL_D_K_FM] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Hauppauge WinTV USB (PAL D/K FM)",
- },
- [HPG_WINTV_PRO_NTSC_MN] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_MICROTUNE_4049FM5,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (NTSC M/N)",
- },
- [HPG_WINTV_PRO_NTSC_MN_V2] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_MICROTUNE_4049FM5,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (NTSC M/N) V2",
- },
- [HPG_WINTV_PRO_PAL] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_FM1216ME_MK3,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL/SECAM B/G/I/D/K/L)",
- },
- [HPG_WINTV_PRO_NTSC_MN_V3] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_NTSC_M,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (NTSC M/N) V3",
- },
- [HPG_WINTV_PRO_PAL_BG] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL B/G)",
- },
- [HPG_WINTV_PRO_PAL_I] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL I)",
- },
- [HPG_WINTV_PRO_PAL_SECAM_L] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_SECAM,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_SECAM,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL/SECAM L)",
- },
- [HPG_WINTV_PRO_PAL_D_K] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL D/K)",
- },
- [HPG_WINTV_PRO_PAL_SECAM] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_SECAM,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_SECAM,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL/SECAM BGDK/I/L)",
- },
- [HPG_WINTV_PRO_PAL_SECAM_V2] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_SECAM,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_SECAM,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL/SECAM BGDK/I/L) V2",
- },
- [HPG_WINTV_PRO_PAL_BG_V2] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_ALPS_TSBE1_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL B/G) V2",
- },
- [HPG_WINTV_PRO_PAL_BG_D_K] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_ALPS_TSBE1_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL B/G,D/K)",
- },
- [HPG_WINTV_PRO_PAL_I_D_K] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_LG_PAL_NEW_TAPC,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL I,D/K)",
- },
- [HPG_WINTV_PRO_NTSC_MN_FM] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_NTSC_M,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (NTSC M/N FM)",
- },
- [HPG_WINTV_PRO_PAL_BG_FM] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL B/G FM)",
- },
- [HPG_WINTV_PRO_PAL_I_FM] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL I FM)",
- },
- [HPG_WINTV_PRO_PAL_D_K_FM] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL D/K FM)",
- },
- [HPG_WINTV_PRO_TEMIC_PAL_FM] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_MICROTUNE_4049FM5,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (Temic PAL/SECAM B/G/I/D/K/L FM)",
- },
- [HPG_WINTV_PRO_TEMIC_PAL_BG_FM] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_MICROTUNE_4049FM5,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (Temic PAL B/G FM)",
- },
- [HPG_WINTV_PRO_PAL_FM] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_FM1216ME_MK3,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (PAL/SECAM B/G/I/D/K/L FM)",
- },
- [HPG_WINTV_PRO_NTSC_MN_FM_V2] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_NTSC_M,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Hauppauge WinTV USB Pro (NTSC M/N FM) V2",
- },
- [CAMTEL_TVB330] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_NTSC_M,
- .x_offset = 5,
- .y_offset = 5,
- .model_string = "Camtel Technology USB TV Genie Pro FM Model TVB330",
- },
- [DIGITAL_VIDEO_CREATOR_I] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 2,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 0,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Digital Video Creator I",
- },
- [GLOBAL_VILLAGE_GV_007_NTSC] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 2,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 0,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 82,
- .y_offset = 20,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Global Village GV-007 (NTSC)",
- },
- [DAZZLE_DVC_50_REV_1_NTSC] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 2,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 0,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Dazzle Fusion Model DVC-50 Rev 1 (NTSC)",
- },
- [DAZZLE_DVC_80_REV_1_PAL] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 2,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 0,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Dazzle Fusion Model DVC-80 Rev 1 (PAL)",
- },
- [DAZZLE_DVC_90_REV_1_SECAM] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 2,
- .video_norm = V4L2_STD_SECAM,
- .audio_channels = 0,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Dazzle Fusion Model DVC-90 Rev 1 (SECAM)",
- },
- [ESKAPE_LABS_MYTV2GO] = {
- .interface = 0,
- .codec = CODEC_SAA7113,
- .video_channels = 2,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_FM1216ME_MK3,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Eskape Labs MyTV2Go",
- },
- [PINNA_PCTV_USB_PAL] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 0,
- .tuner = 1,
- .tuner_type = TUNER_TEMIC_4066FY5_PAL_I,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Pinnacle Studio PCTV USB (PAL)",
- },
- [PINNA_PCTV_USB_SECAM] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_SECAM,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_SECAM,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Pinnacle Studio PCTV USB (SECAM)",
- },
- [PINNA_PCTV_USB_PAL_FM] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = 128,
- .y_offset = 23,
- .model_string = "Pinnacle Studio PCTV USB (PAL) FM",
- },
- [MIRO_PCTV_USB] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_PAL,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Miro PCTV USB",
- },
- [PINNA_PCTV_USB_NTSC_FM] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_NTSC_M,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Pinnacle Studio PCTV USB (NTSC) FM",
- },
- [PINNA_PCTV_USB_NTSC_FM_V3] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_NTSC_M,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Pinnacle Studio PCTV USB (NTSC) FM V3",
- },
- [PINNA_PCTV_USB_PAL_FM_V2] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_TEMIC_4009FR5_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Pinnacle Studio PCTV USB (PAL) FM V2",
- },
- [PINNA_PCTV_USB_NTSC_FM_V2] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_TEMIC_4039FR5_NTSC,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Pinnacle Studio PCTV USB (NTSC) FM V2",
- },
- [PINNA_PCTV_USB_PAL_FM_V3] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_TEMIC_4009FR5_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Pinnacle Studio PCTV USB (PAL) FM V3",
- },
- [PINNA_LINX_VD_IN_CAB_NTSC] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 2,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Pinnacle Studio Linx Video input cable (NTSC)",
- },
- [PINNA_LINX_VD_IN_CAB_PAL] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 2,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Pinnacle Studio Linx Video input cable (PAL)",
- },
- [PINNA_PCTV_BUNGEE_PAL_FM] = {
- .interface = -1,
- .codec = CODEC_SAA7113,
- .video_channels = 3,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 1,
- .radio = 1,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_TEMIC_4009FR5_PAL,
- .x_offset = 0,
- .y_offset = 3,
- .dvi_yuv_override = 1,
- .dvi_yuv = 7,
- .model_string = "Pinnacle PCTV Bungee USB (PAL) FM",
- },
- [HPG_WINTV] = {
- .interface = -1,
- .codec = CODEC_SAA7111,
- .video_channels = 3,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 1,
- .radio = 0,
- .vbi = 1,
- .tuner = 1,
- .tuner_type = TUNER_PHILIPS_NTSC_M,
- .x_offset = -1,
- .y_offset = -1,
- .model_string = "Hauppauge WinTv-USB",
- },
- [MICROCAM_NTSC] = {
- .interface = -1,
- .codec = CODEC_WEBCAM,
- .video_channels = 1,
- .video_norm = V4L2_STD_NTSC,
- .audio_channels = 0,
- .radio = 0,
- .vbi = 0,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 71,
- .y_offset = 15,
- .model_string = "Nogatech USB MicroCam NTSC (NV3000N)",
- },
- [MICROCAM_PAL] = {
- .interface = -1,
- .codec = CODEC_WEBCAM,
- .video_channels = 1,
- .video_norm = V4L2_STD_PAL,
- .audio_channels = 0,
- .radio = 0,
- .vbi = 0,
- .tuner = 0,
- .tuner_type = 0,
- .x_offset = 71,
- .y_offset = 18,
- .model_string = "Nogatech USB MicroCam PAL (NV3001P)",
- },
-};
-const int usbvision_device_data_size = ARRAY_SIZE(usbvision_device_data);
-
-/* Supported Devices */
-
-struct usb_device_id usbvision_table[] = {
- { USB_DEVICE(0x0a6f, 0x0400), .driver_info = XANBOO },
- { USB_DEVICE(0x050d, 0x0106), .driver_info = BELKIN_VIDEOBUS_II },
- { USB_DEVICE(0x050d, 0x0207), .driver_info = BELKIN_VIDEOBUS },
- { USB_DEVICE(0x050d, 0x0208), .driver_info = BELKIN_USB_VIDEOBUS_II },
- { USB_DEVICE(0x0571, 0x0002), .driver_info = ECHOFX_INTERVIEW_LITE },
- { USB_DEVICE(0x0573, 0x0003), .driver_info = USBGEAR_USBG_V1 },
- { USB_DEVICE(0x0573, 0x0400), .driver_info = D_LINK_V100 },
- { USB_DEVICE(0x0573, 0x2000), .driver_info = X10_USB_CAMERA },
- { USB_DEVICE(0x0573, 0x2d00), .driver_info = HPG_WINTV_LIVE_PAL_BG },
- { USB_DEVICE(0x0573, 0x2d01), .driver_info = HPG_WINTV_LIVE_PRO_NTSC_MN },
- { USB_DEVICE(0x0573, 0x2101), .driver_info = ZORAN_PMD_NOGATECH },
- { USB_DEVICE(0x0573, 0x3000), .driver_info = MICROCAM_NTSC },
- { USB_DEVICE(0x0573, 0x3001), .driver_info = MICROCAM_PAL },
- { USB_DEVICE(0x0573, 0x4100), .driver_info = NOGATECH_USB_TV_NTSC_FM },
- { USB_DEVICE(0x0573, 0x4110), .driver_info = PNY_USB_TV_NTSC_FM },
- { USB_DEVICE(0x0573, 0x4450), .driver_info = PV_PLAYTV_USB_PRO_PAL_FM },
- { USB_DEVICE(0x0573, 0x4550), .driver_info = ZT_721 },
- { USB_DEVICE(0x0573, 0x4d00), .driver_info = HPG_WINTV_NTSC_MN },
- { USB_DEVICE(0x0573, 0x4d01), .driver_info = HPG_WINTV_PAL_BG },
- { USB_DEVICE(0x0573, 0x4d02), .driver_info = HPG_WINTV_PAL_I },
- { USB_DEVICE(0x0573, 0x4d03), .driver_info = HPG_WINTV_PAL_SECAM_L },
- { USB_DEVICE(0x0573, 0x4d04), .driver_info = HPG_WINTV_PAL_D_K },
- { USB_DEVICE(0x0573, 0x4d10), .driver_info = HPG_WINTV_NTSC_FM },
- { USB_DEVICE(0x0573, 0x4d11), .driver_info = HPG_WINTV_PAL_BG_FM },
- { USB_DEVICE(0x0573, 0x4d12), .driver_info = HPG_WINTV_PAL_I_FM },
- { USB_DEVICE(0x0573, 0x4d14), .driver_info = HPG_WINTV_PAL_D_K_FM },
- { USB_DEVICE(0x0573, 0x4d2a), .driver_info = HPG_WINTV_PRO_NTSC_MN },
- { USB_DEVICE(0x0573, 0x4d2b), .driver_info = HPG_WINTV_PRO_NTSC_MN_V2 },
- { USB_DEVICE(0x0573, 0x4d2c), .driver_info = HPG_WINTV_PRO_PAL },
- { USB_DEVICE(0x0573, 0x4d20), .driver_info = HPG_WINTV_PRO_NTSC_MN_V3 },
- { USB_DEVICE(0x0573, 0x4d21), .driver_info = HPG_WINTV_PRO_PAL_BG },
- { USB_DEVICE(0x0573, 0x4d22), .driver_info = HPG_WINTV_PRO_PAL_I },
- { USB_DEVICE(0x0573, 0x4d23), .driver_info = HPG_WINTV_PRO_PAL_SECAM_L },
- { USB_DEVICE(0x0573, 0x4d24), .driver_info = HPG_WINTV_PRO_PAL_D_K },
- { USB_DEVICE(0x0573, 0x4d25), .driver_info = HPG_WINTV_PRO_PAL_SECAM },
- { USB_DEVICE(0x0573, 0x4d26), .driver_info = HPG_WINTV_PRO_PAL_SECAM_V2 },
- { USB_DEVICE(0x0573, 0x4d27), .driver_info = HPG_WINTV_PRO_PAL_BG_V2 },
- { USB_DEVICE(0x0573, 0x4d28), .driver_info = HPG_WINTV_PRO_PAL_BG_D_K },
- { USB_DEVICE(0x0573, 0x4d29), .driver_info = HPG_WINTV_PRO_PAL_I_D_K },
- { USB_DEVICE(0x0573, 0x4d30), .driver_info = HPG_WINTV_PRO_NTSC_MN_FM },
- { USB_DEVICE(0x0573, 0x4d31), .driver_info = HPG_WINTV_PRO_PAL_BG_FM },
- { USB_DEVICE(0x0573, 0x4d32), .driver_info = HPG_WINTV_PRO_PAL_I_FM },
- { USB_DEVICE(0x0573, 0x4d34), .driver_info = HPG_WINTV_PRO_PAL_D_K_FM },
- { USB_DEVICE(0x0573, 0x4d35), .driver_info = HPG_WINTV_PRO_TEMIC_PAL_FM },
- { USB_DEVICE(0x0573, 0x4d36), .driver_info = HPG_WINTV_PRO_TEMIC_PAL_BG_FM },
- { USB_DEVICE(0x0573, 0x4d37), .driver_info = HPG_WINTV_PRO_PAL_FM },
- { USB_DEVICE(0x0573, 0x4d38), .driver_info = HPG_WINTV_PRO_NTSC_MN_FM_V2 },
- { USB_DEVICE(0x0768, 0x0006), .driver_info = CAMTEL_TVB330 },
- { USB_DEVICE(0x07d0, 0x0001), .driver_info = DIGITAL_VIDEO_CREATOR_I },
- { USB_DEVICE(0x07d0, 0x0002), .driver_info = GLOBAL_VILLAGE_GV_007_NTSC },
- { USB_DEVICE(0x07d0, 0x0003), .driver_info = DAZZLE_DVC_50_REV_1_NTSC },
- { USB_DEVICE(0x07d0, 0x0004), .driver_info = DAZZLE_DVC_80_REV_1_PAL },
- { USB_DEVICE(0x07d0, 0x0005), .driver_info = DAZZLE_DVC_90_REV_1_SECAM },
- { USB_DEVICE(0x07f8, 0x9104), .driver_info = ESKAPE_LABS_MYTV2GO },
- { USB_DEVICE(0x2304, 0x010d), .driver_info = PINNA_PCTV_USB_PAL },
- { USB_DEVICE(0x2304, 0x0109), .driver_info = PINNA_PCTV_USB_SECAM },
- { USB_DEVICE(0x2304, 0x0110), .driver_info = PINNA_PCTV_USB_PAL_FM },
- { USB_DEVICE(0x2304, 0x0111), .driver_info = MIRO_PCTV_USB },
- { USB_DEVICE(0x2304, 0x0112), .driver_info = PINNA_PCTV_USB_NTSC_FM },
- { USB_DEVICE(0x2304, 0x0113), .driver_info = PINNA_PCTV_USB_NTSC_FM_V3 },
- { USB_DEVICE(0x2304, 0x0210), .driver_info = PINNA_PCTV_USB_PAL_FM_V2 },
- { USB_DEVICE(0x2304, 0x0212), .driver_info = PINNA_PCTV_USB_NTSC_FM_V2 },
- { USB_DEVICE(0x2304, 0x0214), .driver_info = PINNA_PCTV_USB_PAL_FM_V3 },
- { USB_DEVICE(0x2304, 0x0300), .driver_info = PINNA_LINX_VD_IN_CAB_NTSC },
- { USB_DEVICE(0x2304, 0x0301), .driver_info = PINNA_LINX_VD_IN_CAB_PAL },
- { USB_DEVICE(0x2304, 0x0419), .driver_info = PINNA_PCTV_BUNGEE_PAL_FM },
- { USB_DEVICE(0x2400, 0x4200), .driver_info = HPG_WINTV },
- { }, /* terminate list */
-};
-
-MODULE_DEVICE_TABLE(usb, usbvision_table);
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-#define XANBOO 0
-#define BELKIN_VIDEOBUS_II 1
-#define BELKIN_VIDEOBUS 2
-#define BELKIN_USB_VIDEOBUS_II 3
-#define ECHOFX_INTERVIEW_LITE 4
-#define USBGEAR_USBG_V1 5
-#define D_LINK_V100 6
-#define X10_USB_CAMERA 7
-#define HPG_WINTV_LIVE_PAL_BG 8
-#define HPG_WINTV_LIVE_PRO_NTSC_MN 9
-#define ZORAN_PMD_NOGATECH 10
-#define NOGATECH_USB_TV_NTSC_FM 11
-#define PNY_USB_TV_NTSC_FM 12
-#define PV_PLAYTV_USB_PRO_PAL_FM 13
-#define ZT_721 14
-#define HPG_WINTV_NTSC_MN 15
-#define HPG_WINTV_PAL_BG 16
-#define HPG_WINTV_PAL_I 17
-#define HPG_WINTV_PAL_SECAM_L 18
-#define HPG_WINTV_PAL_D_K 19
-#define HPG_WINTV_NTSC_FM 20
-#define HPG_WINTV_PAL_BG_FM 21
-#define HPG_WINTV_PAL_I_FM 22
-#define HPG_WINTV_PAL_D_K_FM 23
-#define HPG_WINTV_PRO_NTSC_MN 24
-#define HPG_WINTV_PRO_NTSC_MN_V2 25
-#define HPG_WINTV_PRO_PAL 26
-#define HPG_WINTV_PRO_NTSC_MN_V3 27
-#define HPG_WINTV_PRO_PAL_BG 28
-#define HPG_WINTV_PRO_PAL_I 29
-#define HPG_WINTV_PRO_PAL_SECAM_L 30
-#define HPG_WINTV_PRO_PAL_D_K 31
-#define HPG_WINTV_PRO_PAL_SECAM 32
-#define HPG_WINTV_PRO_PAL_SECAM_V2 33
-#define HPG_WINTV_PRO_PAL_BG_V2 34
-#define HPG_WINTV_PRO_PAL_BG_D_K 35
-#define HPG_WINTV_PRO_PAL_I_D_K 36
-#define HPG_WINTV_PRO_NTSC_MN_FM 37
-#define HPG_WINTV_PRO_PAL_BG_FM 38
-#define HPG_WINTV_PRO_PAL_I_FM 39
-#define HPG_WINTV_PRO_PAL_D_K_FM 40
-#define HPG_WINTV_PRO_TEMIC_PAL_FM 41
-#define HPG_WINTV_PRO_TEMIC_PAL_BG_FM 42
-#define HPG_WINTV_PRO_PAL_FM 43
-#define HPG_WINTV_PRO_NTSC_MN_FM_V2 44
-#define CAMTEL_TVB330 45
-#define DIGITAL_VIDEO_CREATOR_I 46
-#define GLOBAL_VILLAGE_GV_007_NTSC 47
-#define DAZZLE_DVC_50_REV_1_NTSC 48
-#define DAZZLE_DVC_80_REV_1_PAL 49
-#define DAZZLE_DVC_90_REV_1_SECAM 50
-#define ESKAPE_LABS_MYTV2GO 51
-#define PINNA_PCTV_USB_PAL 52
-#define PINNA_PCTV_USB_SECAM 53
-#define PINNA_PCTV_USB_PAL_FM 54
-#define MIRO_PCTV_USB 55
-#define PINNA_PCTV_USB_NTSC_FM 56
-#define PINNA_PCTV_USB_PAL_FM_V2 57
-#define PINNA_PCTV_USB_NTSC_FM_V2 58
-#define PINNA_PCTV_USB_PAL_FM_V3 59
-#define PINNA_LINX_VD_IN_CAB_NTSC 60
-#define PINNA_LINX_VD_IN_CAB_PAL 61
-#define PINNA_PCTV_BUNGEE_PAL_FM 62
-#define HPG_WINTV 63
-#define PINNA_PCTV_USB_NTSC_FM_V3 64
-#define MICROCAM_NTSC 65
-#define MICROCAM_PAL 66
-
-extern const int usbvision_device_data_size;
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * usbvision-core.c - driver for NT100x USB video capture devices
- *
- * Copyright (c) 1999-2005 Joerg Heckenbach <joerg@heckenbach-aw.de>
- * Dwaine Garden <dwainegarden@rogers.com>
- *
- * This module is part of usbvision driver project.
- * Updates to driver completed by Dwaine P. Garden
- */
-
-#include <linux/kernel.h>
-#include <linux/list.h>
-#include <linux/timer.h>
-#include <linux/gfp.h>
-#include <linux/mm.h>
-#include <linux/highmem.h>
-#include <linux/vmalloc.h>
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/spinlock.h>
-#include <linux/io.h>
-#include <linux/videodev2.h>
-#include <linux/i2c.h>
-
-#include <media/i2c/saa7115.h>
-#include <media/v4l2-common.h>
-#include <media/tuner.h>
-
-#include <linux/workqueue.h>
-
-#include "usbvision.h"
-
-static unsigned int core_debug;
-module_param(core_debug, int, 0644);
-MODULE_PARM_DESC(core_debug, "enable debug messages [core]");
-
-static int adjust_compression = 1; /* Set the compression to be adaptive */
-module_param(adjust_compression, int, 0444);
-MODULE_PARM_DESC(adjust_compression, " Set the ADPCM compression for the device. Default: 1 (On)");
-
-/* To help people with Black and White output with using s-video input.
- * Some cables and input device are wired differently. */
-static int switch_svideo_input;
-module_param(switch_svideo_input, int, 0444);
-MODULE_PARM_DESC(switch_svideo_input, " Set the S-Video input. Some cables and input device are wired differently. Default: 0 (Off)");
-
-static unsigned int adjust_x_offset = -1;
-module_param(adjust_x_offset, int, 0644);
-MODULE_PARM_DESC(adjust_x_offset, "adjust X offset display [core]");
-
-static unsigned int adjust_y_offset = -1;
-module_param(adjust_y_offset, int, 0644);
-MODULE_PARM_DESC(adjust_y_offset, "adjust Y offset display [core]");
-
-
-#define ENABLE_HEXDUMP 0 /* Enable if you need it */
-
-
-#ifdef USBVISION_DEBUG
- #define PDEBUG(level, fmt, args...) { \
- if (core_debug & (level)) \
- printk(KERN_INFO KBUILD_MODNAME ":[%s:%d] " fmt, \
- __func__, __LINE__ , ## args); \
- }
-#else
- #define PDEBUG(level, fmt, args...) do {} while (0)
-#endif
-
-#define DBG_HEADER (1 << 0)
-#define DBG_IRQ (1 << 1)
-#define DBG_ISOC (1 << 2)
-#define DBG_PARSE (1 << 3)
-#define DBG_SCRATCH (1 << 4)
-#define DBG_FUNC (1 << 5)
-
-/* The value of 'scratch_buf_size' affects quality of the picture
- * in many ways. Shorter buffers may cause loss of data when client
- * is too slow. Larger buffers are memory-consuming and take longer
- * to work with. This setting can be adjusted, but the default value
- * should be OK for most desktop users.
- */
-#define DEFAULT_SCRATCH_BUF_SIZE (0x20000) /* 128kB memory scratch buffer */
-static const int scratch_buf_size = DEFAULT_SCRATCH_BUF_SIZE;
-
-/* Function prototypes */
-static int usbvision_request_intra(struct usb_usbvision *usbvision);
-static int usbvision_unrequest_intra(struct usb_usbvision *usbvision);
-static int usbvision_adjust_compression(struct usb_usbvision *usbvision);
-static int usbvision_measure_bandwidth(struct usb_usbvision *usbvision);
-
-/*******************************/
-/* Memory management functions */
-/*******************************/
-
-/*
- * Here we want the physical address of the memory.
- * This is used when initializing the contents of the area.
- */
-
-static void *usbvision_rvmalloc(unsigned long size)
-{
- void *mem;
- unsigned long adr;
-
- size = PAGE_ALIGN(size);
- mem = vmalloc_32(size);
- if (!mem)
- return NULL;
-
- memset(mem, 0, size); /* Clear the ram out, no junk to the user */
- adr = (unsigned long) mem;
- while (size > 0) {
- SetPageReserved(vmalloc_to_page((void *)adr));
- adr += PAGE_SIZE;
- size -= PAGE_SIZE;
- }
-
- return mem;
-}
-
-static void usbvision_rvfree(void *mem, unsigned long size)
-{
- unsigned long adr;
-
- if (!mem)
- return;
-
- size = PAGE_ALIGN(size);
-
- adr = (unsigned long) mem;
- while ((long) size > 0) {
- ClearPageReserved(vmalloc_to_page((void *)adr));
- adr += PAGE_SIZE;
- size -= PAGE_SIZE;
- }
-
- vfree(mem);
-}
-
-
-#if ENABLE_HEXDUMP
-static void usbvision_hexdump(const unsigned char *data, int len)
-{
- char tmp[80];
- int i, k;
-
- for (i = k = 0; len > 0; i++, len--) {
- if (i > 0 && (i % 16 == 0)) {
- printk("%s\n", tmp);
- k = 0;
- }
- k += sprintf(&tmp[k], "%02x ", data[i]);
- }
- if (k > 0)
- printk(KERN_CONT "%s\n", tmp);
-}
-#endif
-
-/********************************
- * scratch ring buffer handling
- ********************************/
-static int scratch_len(struct usb_usbvision *usbvision) /* This returns the amount of data actually in the buffer */
-{
- int len = usbvision->scratch_write_ptr - usbvision->scratch_read_ptr;
-
- if (len < 0)
- len += scratch_buf_size;
- PDEBUG(DBG_SCRATCH, "scratch_len() = %d\n", len);
-
- return len;
-}
-
-
-/* This returns the free space left in the buffer */
-static int scratch_free(struct usb_usbvision *usbvision)
-{
- int free = usbvision->scratch_read_ptr - usbvision->scratch_write_ptr;
- if (free <= 0)
- free += scratch_buf_size;
- if (free) {
- free -= 1; /* at least one byte in the buffer must */
- /* left blank, otherwise there is no chance to differ between full and empty */
- }
- PDEBUG(DBG_SCRATCH, "return %d\n", free);
-
- return free;
-}
-
-
-/* This puts data into the buffer */
-static int scratch_put(struct usb_usbvision *usbvision, unsigned char *data,
- int len)
-{
- int len_part;
-
- if (usbvision->scratch_write_ptr + len < scratch_buf_size) {
- memcpy(usbvision->scratch + usbvision->scratch_write_ptr, data, len);
- usbvision->scratch_write_ptr += len;
- } else {
- len_part = scratch_buf_size - usbvision->scratch_write_ptr;
- memcpy(usbvision->scratch + usbvision->scratch_write_ptr, data, len_part);
- if (len == len_part) {
- usbvision->scratch_write_ptr = 0; /* just set write_ptr to zero */
- } else {
- memcpy(usbvision->scratch, data + len_part, len - len_part);
- usbvision->scratch_write_ptr = len - len_part;
- }
- }
-
- PDEBUG(DBG_SCRATCH, "len=%d, new write_ptr=%d\n", len, usbvision->scratch_write_ptr);
-
- return len;
-}
-
-/* This marks the write_ptr as position of new frame header */
-static void scratch_mark_header(struct usb_usbvision *usbvision)
-{
- PDEBUG(DBG_SCRATCH, "header at write_ptr=%d\n", usbvision->scratch_headermarker_write_ptr);
-
- usbvision->scratch_headermarker[usbvision->scratch_headermarker_write_ptr] =
- usbvision->scratch_write_ptr;
- usbvision->scratch_headermarker_write_ptr += 1;
- usbvision->scratch_headermarker_write_ptr %= USBVISION_NUM_HEADERMARKER;
-}
-
-/* This gets data from the buffer at the given "ptr" position */
-static int scratch_get_extra(struct usb_usbvision *usbvision,
- unsigned char *data, int *ptr, int len)
-{
- int len_part;
-
- if (*ptr + len < scratch_buf_size) {
- memcpy(data, usbvision->scratch + *ptr, len);
- *ptr += len;
- } else {
- len_part = scratch_buf_size - *ptr;
- memcpy(data, usbvision->scratch + *ptr, len_part);
- if (len == len_part) {
- *ptr = 0; /* just set the y_ptr to zero */
- } else {
- memcpy(data + len_part, usbvision->scratch, len - len_part);
- *ptr = len - len_part;
- }
- }
-
- PDEBUG(DBG_SCRATCH, "len=%d, new ptr=%d\n", len, *ptr);
-
- return len;
-}
-
-
-/* This sets the scratch extra read pointer */
-static void scratch_set_extra_ptr(struct usb_usbvision *usbvision, int *ptr,
- int len)
-{
- *ptr = (usbvision->scratch_read_ptr + len) % scratch_buf_size;
-
- PDEBUG(DBG_SCRATCH, "ptr=%d\n", *ptr);
-}
-
-
-/* This increments the scratch extra read pointer */
-static void scratch_inc_extra_ptr(int *ptr, int len)
-{
- *ptr = (*ptr + len) % scratch_buf_size;
-
- PDEBUG(DBG_SCRATCH, "ptr=%d\n", *ptr);
-}
-
-
-/* This gets data from the buffer */
-static int scratch_get(struct usb_usbvision *usbvision, unsigned char *data,
- int len)
-{
- int len_part;
-
- if (usbvision->scratch_read_ptr + len < scratch_buf_size) {
- memcpy(data, usbvision->scratch + usbvision->scratch_read_ptr, len);
- usbvision->scratch_read_ptr += len;
- } else {
- len_part = scratch_buf_size - usbvision->scratch_read_ptr;
- memcpy(data, usbvision->scratch + usbvision->scratch_read_ptr, len_part);
- if (len == len_part) {
- usbvision->scratch_read_ptr = 0; /* just set the read_ptr to zero */
- } else {
- memcpy(data + len_part, usbvision->scratch, len - len_part);
- usbvision->scratch_read_ptr = len - len_part;
- }
- }
-
- PDEBUG(DBG_SCRATCH, "len=%d, new read_ptr=%d\n", len, usbvision->scratch_read_ptr);
-
- return len;
-}
-
-
-/* This sets read pointer to next header and returns it */
-static int scratch_get_header(struct usb_usbvision *usbvision,
- struct usbvision_frame_header *header)
-{
- int err_code = 0;
-
- PDEBUG(DBG_SCRATCH, "from read_ptr=%d", usbvision->scratch_headermarker_read_ptr);
-
- while (usbvision->scratch_headermarker_write_ptr -
- usbvision->scratch_headermarker_read_ptr != 0) {
- usbvision->scratch_read_ptr =
- usbvision->scratch_headermarker[usbvision->scratch_headermarker_read_ptr];
- usbvision->scratch_headermarker_read_ptr += 1;
- usbvision->scratch_headermarker_read_ptr %= USBVISION_NUM_HEADERMARKER;
- scratch_get(usbvision, (unsigned char *)header, USBVISION_HEADER_LENGTH);
- if ((header->magic_1 == USBVISION_MAGIC_1)
- && (header->magic_2 == USBVISION_MAGIC_2)
- && (header->header_length == USBVISION_HEADER_LENGTH)) {
- err_code = USBVISION_HEADER_LENGTH;
- header->frame_width = header->frame_width_lo + (header->frame_width_hi << 8);
- header->frame_height = header->frame_height_lo + (header->frame_height_hi << 8);
- break;
- }
- }
-
- return err_code;
-}
-
-
-/* This removes len bytes of old data from the buffer */
-static void scratch_rm_old(struct usb_usbvision *usbvision, int len)
-{
- usbvision->scratch_read_ptr += len;
- usbvision->scratch_read_ptr %= scratch_buf_size;
- PDEBUG(DBG_SCRATCH, "read_ptr is now %d\n", usbvision->scratch_read_ptr);
-}
-
-
-/* This resets the buffer - kills all data in it too */
-static void scratch_reset(struct usb_usbvision *usbvision)
-{
- PDEBUG(DBG_SCRATCH, "\n");
-
- usbvision->scratch_read_ptr = 0;
- usbvision->scratch_write_ptr = 0;
- usbvision->scratch_headermarker_read_ptr = 0;
- usbvision->scratch_headermarker_write_ptr = 0;
- usbvision->isocstate = isoc_state_no_frame;
-}
-
-int usbvision_scratch_alloc(struct usb_usbvision *usbvision)
-{
- usbvision->scratch = vmalloc_32(scratch_buf_size);
- scratch_reset(usbvision);
- if (usbvision->scratch == NULL) {
- dev_err(&usbvision->dev->dev,
- "%s: unable to allocate %d bytes for scratch\n",
- __func__, scratch_buf_size);
- return -ENOMEM;
- }
- return 0;
-}
-
-void usbvision_scratch_free(struct usb_usbvision *usbvision)
-{
- vfree(usbvision->scratch);
- usbvision->scratch = NULL;
-}
-
-/*
- * usbvision_decompress_alloc()
- *
- * allocates intermediate buffer for decompression
- */
-int usbvision_decompress_alloc(struct usb_usbvision *usbvision)
-{
- int IFB_size = MAX_FRAME_WIDTH * MAX_FRAME_HEIGHT * 3 / 2;
-
- usbvision->intra_frame_buffer = vmalloc_32(IFB_size);
- if (usbvision->intra_frame_buffer == NULL) {
- dev_err(&usbvision->dev->dev,
- "%s: unable to allocate %d for compr. frame buffer\n",
- __func__, IFB_size);
- return -ENOMEM;
- }
- return 0;
-}
-
-/*
- * usbvision_decompress_free()
- *
- * frees intermediate buffer for decompression
- */
-void usbvision_decompress_free(struct usb_usbvision *usbvision)
-{
- vfree(usbvision->intra_frame_buffer);
- usbvision->intra_frame_buffer = NULL;
-
-}
-
-/************************************************************
- * Here comes the data parsing stuff that is run as interrupt
- ************************************************************/
-/*
- * usbvision_find_header()
- *
- * Locate one of supported header markers in the scratch buffer.
- */
-static enum parse_state usbvision_find_header(struct usb_usbvision *usbvision)
-{
- struct usbvision_frame *frame;
- int found_header = 0;
-
- frame = usbvision->cur_frame;
-
- while (scratch_get_header(usbvision, &frame->isoc_header) == USBVISION_HEADER_LENGTH) {
- /* found header in scratch */
- PDEBUG(DBG_HEADER, "found header: 0x%02x%02x %d %d %d %d %#x 0x%02x %u %u",
- frame->isoc_header.magic_2,
- frame->isoc_header.magic_1,
- frame->isoc_header.header_length,
- frame->isoc_header.frame_num,
- frame->isoc_header.frame_phase,
- frame->isoc_header.frame_latency,
- frame->isoc_header.data_format,
- frame->isoc_header.format_param,
- frame->isoc_header.frame_width,
- frame->isoc_header.frame_height);
-
- if (usbvision->request_intra) {
- if (frame->isoc_header.format_param & 0x80) {
- found_header = 1;
- usbvision->last_isoc_frame_num = -1; /* do not check for lost frames this time */
- usbvision_unrequest_intra(usbvision);
- break;
- }
- } else {
- found_header = 1;
- break;
- }
- }
-
- if (found_header) {
- frame->frmwidth = frame->isoc_header.frame_width * usbvision->stretch_width;
- frame->frmheight = frame->isoc_header.frame_height * usbvision->stretch_height;
- frame->v4l2_linesize = (frame->frmwidth * frame->v4l2_format.depth) >> 3;
- } else { /* no header found */
- PDEBUG(DBG_HEADER, "skipping scratch data, no header");
- scratch_reset(usbvision);
- return parse_state_end_parse;
- }
-
- /* found header */
- if (frame->isoc_header.data_format == ISOC_MODE_COMPRESS) {
- /* check isoc_header.frame_num for lost frames */
- if (usbvision->last_isoc_frame_num >= 0) {
- if (((usbvision->last_isoc_frame_num + 1) % 32) != frame->isoc_header.frame_num) {
- /* unexpected frame drop: need to request new intra frame */
- PDEBUG(DBG_HEADER, "Lost frame before %d on USB", frame->isoc_header.frame_num);
- usbvision_request_intra(usbvision);
- return parse_state_next_frame;
- }
- }
- usbvision->last_isoc_frame_num = frame->isoc_header.frame_num;
- }
- usbvision->header_count++;
- frame->scanstate = scan_state_lines;
- frame->curline = 0;
-
- return parse_state_continue;
-}
-
-static enum parse_state usbvision_parse_lines_422(struct usb_usbvision *usbvision,
- long *pcopylen)
-{
- volatile struct usbvision_frame *frame;
- unsigned char *f;
- int len;
- int i;
- unsigned char yuyv[4] = { 180, 128, 10, 128 }; /* YUV components */
- unsigned char rv, gv, bv; /* RGB components */
- int clipmask_index, bytes_per_pixel;
- int stretch_bytes, clipmask_add;
-
- frame = usbvision->cur_frame;
- f = frame->data + (frame->v4l2_linesize * frame->curline);
-
- /* Make sure there's enough data for the entire line */
- len = (frame->isoc_header.frame_width * 2) + 5;
- if (scratch_len(usbvision) < len) {
- PDEBUG(DBG_PARSE, "out of data in line %d, need %u.\n", frame->curline, len);
- return parse_state_out;
- }
-
- if ((frame->curline + 1) >= frame->frmheight)
- return parse_state_next_frame;
-
- bytes_per_pixel = frame->v4l2_format.bytes_per_pixel;
- stretch_bytes = (usbvision->stretch_width - 1) * bytes_per_pixel;
- clipmask_index = frame->curline * MAX_FRAME_WIDTH;
- clipmask_add = usbvision->stretch_width;
-
- for (i = 0; i < frame->frmwidth; i += (2 * usbvision->stretch_width)) {
- scratch_get(usbvision, &yuyv[0], 4);
-
- if (frame->v4l2_format.format == V4L2_PIX_FMT_YUYV) {
- *f++ = yuyv[0]; /* Y */
- *f++ = yuyv[3]; /* U */
- } else {
- YUV_TO_RGB_BY_THE_BOOK(yuyv[0], yuyv[1], yuyv[3], rv, gv, bv);
- switch (frame->v4l2_format.format) {
- case V4L2_PIX_FMT_RGB565:
- *f++ = (0x1F & rv) |
- (0xE0 & (gv << 5));
- *f++ = (0x07 & (gv >> 3)) |
- (0xF8 & bv);
- break;
- case V4L2_PIX_FMT_RGB24:
- *f++ = rv;
- *f++ = gv;
- *f++ = bv;
- break;
- case V4L2_PIX_FMT_RGB32:
- *f++ = rv;
- *f++ = gv;
- *f++ = bv;
- f++;
- break;
- case V4L2_PIX_FMT_RGB555:
- *f++ = (0x1F & rv) |
- (0xE0 & (gv << 5));
- *f++ = (0x03 & (gv >> 3)) |
- (0x7C & (bv << 2));
- break;
- }
- }
- clipmask_index += clipmask_add;
- f += stretch_bytes;
-
- if (frame->v4l2_format.format == V4L2_PIX_FMT_YUYV) {
- *f++ = yuyv[2]; /* Y */
- *f++ = yuyv[1]; /* V */
- } else {
- YUV_TO_RGB_BY_THE_BOOK(yuyv[2], yuyv[1], yuyv[3], rv, gv, bv);
- switch (frame->v4l2_format.format) {
- case V4L2_PIX_FMT_RGB565:
- *f++ = (0x1F & rv) |
- (0xE0 & (gv << 5));
- *f++ = (0x07 & (gv >> 3)) |
- (0xF8 & bv);
- break;
- case V4L2_PIX_FMT_RGB24:
- *f++ = rv;
- *f++ = gv;
- *f++ = bv;
- break;
- case V4L2_PIX_FMT_RGB32:
- *f++ = rv;
- *f++ = gv;
- *f++ = bv;
- f++;
- break;
- case V4L2_PIX_FMT_RGB555:
- *f++ = (0x1F & rv) |
- (0xE0 & (gv << 5));
- *f++ = (0x03 & (gv >> 3)) |
- (0x7C & (bv << 2));
- break;
- }
- }
- clipmask_index += clipmask_add;
- f += stretch_bytes;
- }
-
- frame->curline += usbvision->stretch_height;
- *pcopylen += frame->v4l2_linesize * usbvision->stretch_height;
-
- if (frame->curline >= frame->frmheight)
- return parse_state_next_frame;
- return parse_state_continue;
-}
-
-/* The decompression routine */
-static int usbvision_decompress(struct usb_usbvision *usbvision, unsigned char *compressed,
- unsigned char *decompressed, int *start_pos,
- int *block_typestart_pos, int len)
-{
- int rest_pixel, idx, pos, extra_pos, block_len, block_type_pos, block_type_len;
- unsigned char block_byte, block_code, block_type, block_type_byte, integrator;
-
- integrator = 0;
- pos = *start_pos;
- block_type_pos = *block_typestart_pos;
- extra_pos = pos;
- block_len = 0;
- block_byte = 0;
- block_code = 0;
- block_type = 0;
- block_type_byte = 0;
- block_type_len = 0;
- rest_pixel = len;
-
- for (idx = 0; idx < len; idx++) {
- if (block_len == 0) {
- if (block_type_len == 0) {
- block_type_byte = compressed[block_type_pos];
- block_type_pos++;
- block_type_len = 4;
- }
- block_type = (block_type_byte & 0xC0) >> 6;
-
- /* statistic: */
- usbvision->compr_block_types[block_type]++;
-
- pos = extra_pos;
- if (block_type == 0) {
- if (rest_pixel >= 24) {
- idx += 23;
- rest_pixel -= 24;
- integrator = decompressed[idx];
- } else {
- idx += rest_pixel - 1;
- rest_pixel = 0;
- }
- } else {
- block_code = compressed[pos];
- pos++;
- if (rest_pixel >= 24)
- block_len = 24;
- else
- block_len = rest_pixel;
- rest_pixel -= block_len;
- extra_pos = pos + (block_len / 4);
- }
- block_type_byte <<= 2;
- block_type_len -= 1;
- }
- if (block_len > 0) {
- if ((block_len % 4) == 0) {
- block_byte = compressed[pos];
- pos++;
- }
- if (block_type == 1) /* inter Block */
- integrator = decompressed[idx];
- switch (block_byte & 0xC0) {
- case 0x03 << 6:
- integrator += compressed[extra_pos];
- extra_pos++;
- break;
- case 0x02 << 6:
- integrator += block_code;
- break;
- case 0x00:
- integrator -= block_code;
- break;
- }
- decompressed[idx] = integrator;
- block_byte <<= 2;
- block_len -= 1;
- }
- }
- *start_pos = extra_pos;
- *block_typestart_pos = block_type_pos;
- return idx;
-}
-
-
-/*
- * usbvision_parse_compress()
- *
- * Parse compressed frame from the scratch buffer, put
- * decoded RGB value into the current frame buffer and add the written
- * number of bytes (RGB) to the *pcopylen.
- *
- */
-static enum parse_state usbvision_parse_compress(struct usb_usbvision *usbvision,
- long *pcopylen)
-{
-#define USBVISION_STRIP_MAGIC 0x5A
-#define USBVISION_STRIP_LEN_MAX 400
-#define USBVISION_STRIP_HEADER_LEN 3
-
- struct usbvision_frame *frame;
- unsigned char *f, *u = NULL, *v = NULL;
- unsigned char strip_data[USBVISION_STRIP_LEN_MAX];
- unsigned char strip_header[USBVISION_STRIP_HEADER_LEN];
- int idx, idx_end, strip_len, strip_ptr, startblock_pos, block_pos, block_type_pos;
- int clipmask_index;
- int image_size;
- unsigned char rv, gv, bv;
- static unsigned char *Y, *U, *V;
-
- frame = usbvision->cur_frame;
- image_size = frame->frmwidth * frame->frmheight;
- if ((frame->v4l2_format.format == V4L2_PIX_FMT_YUV422P) ||
- (frame->v4l2_format.format == V4L2_PIX_FMT_YVU420)) { /* this is a planar format */
- /* ... v4l2_linesize not used here. */
- f = frame->data + (frame->width * frame->curline);
- } else
- f = frame->data + (frame->v4l2_linesize * frame->curline);
-
- if (frame->v4l2_format.format == V4L2_PIX_FMT_YUYV) { /* initialise u and v pointers */
- /* get base of u and b planes add halfoffset */
- u = frame->data
- + image_size
- + (frame->frmwidth >> 1) * frame->curline;
- v = u + (image_size >> 1);
- } else if (frame->v4l2_format.format == V4L2_PIX_FMT_YVU420) {
- v = frame->data + image_size + ((frame->curline * (frame->width)) >> 2);
- u = v + (image_size >> 2);
- }
-
- if (frame->curline == 0)
- usbvision_adjust_compression(usbvision);
-
- if (scratch_len(usbvision) < USBVISION_STRIP_HEADER_LEN)
- return parse_state_out;
-
- /* get strip header without changing the scratch_read_ptr */
- scratch_set_extra_ptr(usbvision, &strip_ptr, 0);
- scratch_get_extra(usbvision, &strip_header[0], &strip_ptr,
- USBVISION_STRIP_HEADER_LEN);
-
- if (strip_header[0] != USBVISION_STRIP_MAGIC) {
- /* wrong strip magic */
- usbvision->strip_magic_errors++;
- return parse_state_next_frame;
- }
-
- if (frame->curline != (int)strip_header[2]) {
- /* line number mismatch error */
- usbvision->strip_line_number_errors++;
- }
-
- strip_len = 2 * (unsigned int)strip_header[1];
- if (strip_len > USBVISION_STRIP_LEN_MAX) {
- /* strip overrun */
- /* I think this never happens */
- usbvision_request_intra(usbvision);
- }
-
- if (scratch_len(usbvision) < strip_len) {
- /* there is not enough data for the strip */
- return parse_state_out;
- }
-
- if (usbvision->intra_frame_buffer) {
- Y = usbvision->intra_frame_buffer + frame->frmwidth * frame->curline;
- U = usbvision->intra_frame_buffer + image_size + (frame->frmwidth / 2) * (frame->curline / 2);
- V = usbvision->intra_frame_buffer + image_size / 4 * 5 + (frame->frmwidth / 2) * (frame->curline / 2);
- } else {
- return parse_state_next_frame;
- }
-
- clipmask_index = frame->curline * MAX_FRAME_WIDTH;
-
- scratch_get(usbvision, strip_data, strip_len);
-
- idx_end = frame->frmwidth;
- block_type_pos = USBVISION_STRIP_HEADER_LEN;
- startblock_pos = block_type_pos + (idx_end - 1) / 96 + (idx_end / 2 - 1) / 96 + 2;
- block_pos = startblock_pos;
-
- usbvision->block_pos = block_pos;
-
- usbvision_decompress(usbvision, strip_data, Y, &block_pos, &block_type_pos, idx_end);
- if (strip_len > usbvision->max_strip_len)
- usbvision->max_strip_len = strip_len;
-
- if (frame->curline % 2)
- usbvision_decompress(usbvision, strip_data, V, &block_pos, &block_type_pos, idx_end / 2);
- else
- usbvision_decompress(usbvision, strip_data, U, &block_pos, &block_type_pos, idx_end / 2);
-
- if (block_pos > usbvision->comprblock_pos)
- usbvision->comprblock_pos = block_pos;
- if (block_pos > strip_len)
- usbvision->strip_len_errors++;
-
- for (idx = 0; idx < idx_end; idx++) {
- if (frame->v4l2_format.format == V4L2_PIX_FMT_YUYV) {
- *f++ = Y[idx];
- *f++ = idx & 0x01 ? U[idx / 2] : V[idx / 2];
- } else if (frame->v4l2_format.format == V4L2_PIX_FMT_YUV422P) {
- *f++ = Y[idx];
- if (idx & 0x01)
- *u++ = U[idx >> 1];
- else
- *v++ = V[idx >> 1];
- } else if (frame->v4l2_format.format == V4L2_PIX_FMT_YVU420) {
- *f++ = Y[idx];
- if (!((idx & 0x01) | (frame->curline & 0x01))) {
- /* only need do this for 1 in 4 pixels */
- /* intraframe buffer is YUV420 format */
- *u++ = U[idx >> 1];
- *v++ = V[idx >> 1];
- }
- } else {
- YUV_TO_RGB_BY_THE_BOOK(Y[idx], U[idx / 2], V[idx / 2], rv, gv, bv);
- switch (frame->v4l2_format.format) {
- case V4L2_PIX_FMT_GREY:
- *f++ = Y[idx];
- break;
- case V4L2_PIX_FMT_RGB555:
- *f++ = (0x1F & rv) |
- (0xE0 & (gv << 5));
- *f++ = (0x03 & (gv >> 3)) |
- (0x7C & (bv << 2));
- break;
- case V4L2_PIX_FMT_RGB565:
- *f++ = (0x1F & rv) |
- (0xE0 & (gv << 5));
- *f++ = (0x07 & (gv >> 3)) |
- (0xF8 & bv);
- break;
- case V4L2_PIX_FMT_RGB24:
- *f++ = rv;
- *f++ = gv;
- *f++ = bv;
- break;
- case V4L2_PIX_FMT_RGB32:
- *f++ = rv;
- *f++ = gv;
- *f++ = bv;
- f++;
- break;
- }
- }
- clipmask_index++;
- }
- /* Deal with non-integer no. of bytes for YUV420P */
- if (frame->v4l2_format.format != V4L2_PIX_FMT_YVU420)
- *pcopylen += frame->v4l2_linesize;
- else
- *pcopylen += frame->curline & 0x01 ? frame->v4l2_linesize : frame->v4l2_linesize << 1;
-
- frame->curline += 1;
-
- if (frame->curline >= frame->frmheight)
- return parse_state_next_frame;
- return parse_state_continue;
-
-}
-
-
-/*
- * usbvision_parse_lines_420()
- *
- * Parse two lines from the scratch buffer, put
- * decoded RGB value into the current frame buffer and add the written
- * number of bytes (RGB) to the *pcopylen.
- *
- */
-static enum parse_state usbvision_parse_lines_420(struct usb_usbvision *usbvision,
- long *pcopylen)
-{
- struct usbvision_frame *frame;
- unsigned char *f_even = NULL, *f_odd = NULL;
- unsigned int pixel_per_line, block;
- int pixel, block_split;
- int y_ptr, u_ptr, v_ptr, y_odd_offset;
- const int y_block_size = 128;
- const int uv_block_size = 64;
- const int sub_block_size = 32;
- const int y_step[] = { 0, 0, 0, 2 }, y_step_size = 4;
- const int uv_step[] = { 0, 0, 0, 4 }, uv_step_size = 4;
- unsigned char y[2], u, v; /* YUV components */
- int y_, u_, v_, vb, uvg, ur;
- int r_, g_, b_; /* RGB components */
- unsigned char g;
- int clipmask_even_index, clipmask_odd_index, bytes_per_pixel;
- int clipmask_add, stretch_bytes;
-
- frame = usbvision->cur_frame;
- f_even = frame->data + (frame->v4l2_linesize * frame->curline);
- f_odd = f_even + frame->v4l2_linesize * usbvision->stretch_height;
-
- /* Make sure there's enough data for the entire line */
- /* In this mode usbvision transfer 3 bytes for every 2 pixels */
- /* I need two lines to decode the color */
- bytes_per_pixel = frame->v4l2_format.bytes_per_pixel;
- stretch_bytes = (usbvision->stretch_width - 1) * bytes_per_pixel;
- clipmask_even_index = frame->curline * MAX_FRAME_WIDTH;
- clipmask_odd_index = clipmask_even_index + MAX_FRAME_WIDTH;
- clipmask_add = usbvision->stretch_width;
- pixel_per_line = frame->isoc_header.frame_width;
-
- if (scratch_len(usbvision) < (int)pixel_per_line * 3) {
- /* printk(KERN_DEBUG "out of data, need %d\n", len); */
- return parse_state_out;
- }
-
- if ((frame->curline + 1) >= frame->frmheight)
- return parse_state_next_frame;
-
- block_split = (pixel_per_line%y_block_size) ? 1 : 0; /* are some blocks split into different lines? */
-
- y_odd_offset = (pixel_per_line / y_block_size) * (y_block_size + uv_block_size)
- + block_split * uv_block_size;
-
- scratch_set_extra_ptr(usbvision, &y_ptr, y_odd_offset);
- scratch_set_extra_ptr(usbvision, &u_ptr, y_block_size);
- scratch_set_extra_ptr(usbvision, &v_ptr, y_odd_offset
- + (4 - block_split) * sub_block_size);
-
- for (block = 0; block < (pixel_per_line / sub_block_size); block++) {
- for (pixel = 0; pixel < sub_block_size; pixel += 2) {
- scratch_get(usbvision, &y[0], 2);
- scratch_get_extra(usbvision, &u, &u_ptr, 1);
- scratch_get_extra(usbvision, &v, &v_ptr, 1);
-
- /* I don't use the YUV_TO_RGB macro for better performance */
- v_ = v - 128;
- u_ = u - 128;
- vb = 132252 * v_;
- uvg = -53281 * u_ - 25625 * v_;
- ur = 104595 * u_;
-
- if (frame->v4l2_format.format == V4L2_PIX_FMT_YUYV) {
- *f_even++ = y[0];
- *f_even++ = v;
- } else {
- y_ = 76284 * (y[0] - 16);
-
- b_ = (y_ + vb) >> 16;
- g_ = (y_ + uvg) >> 16;
- r_ = (y_ + ur) >> 16;
-
- switch (frame->v4l2_format.format) {
- case V4L2_PIX_FMT_RGB565:
- g = LIMIT_RGB(g_);
- *f_even++ =
- (0x1F & LIMIT_RGB(r_)) |
- (0xE0 & (g << 5));
- *f_even++ =
- (0x07 & (g >> 3)) |
- (0xF8 & LIMIT_RGB(b_));
- break;
- case V4L2_PIX_FMT_RGB24:
- *f_even++ = LIMIT_RGB(r_);
- *f_even++ = LIMIT_RGB(g_);
- *f_even++ = LIMIT_RGB(b_);
- break;
- case V4L2_PIX_FMT_RGB32:
- *f_even++ = LIMIT_RGB(r_);
- *f_even++ = LIMIT_RGB(g_);
- *f_even++ = LIMIT_RGB(b_);
- f_even++;
- break;
- case V4L2_PIX_FMT_RGB555:
- g = LIMIT_RGB(g_);
- *f_even++ = (0x1F & LIMIT_RGB(r_)) |
- (0xE0 & (g << 5));
- *f_even++ = (0x03 & (g >> 3)) |
- (0x7C & (LIMIT_RGB(b_) << 2));
- break;
- }
- }
- clipmask_even_index += clipmask_add;
- f_even += stretch_bytes;
-
- if (frame->v4l2_format.format == V4L2_PIX_FMT_YUYV) {
- *f_even++ = y[1];
- *f_even++ = u;
- } else {
- y_ = 76284 * (y[1] - 16);
-
- b_ = (y_ + vb) >> 16;
- g_ = (y_ + uvg) >> 16;
- r_ = (y_ + ur) >> 16;
-
- switch (frame->v4l2_format.format) {
- case V4L2_PIX_FMT_RGB565:
- g = LIMIT_RGB(g_);
- *f_even++ =
- (0x1F & LIMIT_RGB(r_)) |
- (0xE0 & (g << 5));
- *f_even++ =
- (0x07 & (g >> 3)) |
- (0xF8 & LIMIT_RGB(b_));
- break;
- case V4L2_PIX_FMT_RGB24:
- *f_even++ = LIMIT_RGB(r_);
- *f_even++ = LIMIT_RGB(g_);
- *f_even++ = LIMIT_RGB(b_);
- break;
- case V4L2_PIX_FMT_RGB32:
- *f_even++ = LIMIT_RGB(r_);
- *f_even++ = LIMIT_RGB(g_);
- *f_even++ = LIMIT_RGB(b_);
- f_even++;
- break;
- case V4L2_PIX_FMT_RGB555:
- g = LIMIT_RGB(g_);
- *f_even++ = (0x1F & LIMIT_RGB(r_)) |
- (0xE0 & (g << 5));
- *f_even++ = (0x03 & (g >> 3)) |
- (0x7C & (LIMIT_RGB(b_) << 2));
- break;
- }
- }
- clipmask_even_index += clipmask_add;
- f_even += stretch_bytes;
-
- scratch_get_extra(usbvision, &y[0], &y_ptr, 2);
-
- if (frame->v4l2_format.format == V4L2_PIX_FMT_YUYV) {
- *f_odd++ = y[0];
- *f_odd++ = v;
- } else {
- y_ = 76284 * (y[0] - 16);
-
- b_ = (y_ + vb) >> 16;
- g_ = (y_ + uvg) >> 16;
- r_ = (y_ + ur) >> 16;
-
- switch (frame->v4l2_format.format) {
- case V4L2_PIX_FMT_RGB565:
- g = LIMIT_RGB(g_);
- *f_odd++ =
- (0x1F & LIMIT_RGB(r_)) |
- (0xE0 & (g << 5));
- *f_odd++ =
- (0x07 & (g >> 3)) |
- (0xF8 & LIMIT_RGB(b_));
- break;
- case V4L2_PIX_FMT_RGB24:
- *f_odd++ = LIMIT_RGB(r_);
- *f_odd++ = LIMIT_RGB(g_);
- *f_odd++ = LIMIT_RGB(b_);
- break;
- case V4L2_PIX_FMT_RGB32:
- *f_odd++ = LIMIT_RGB(r_);
- *f_odd++ = LIMIT_RGB(g_);
- *f_odd++ = LIMIT_RGB(b_);
- f_odd++;
- break;
- case V4L2_PIX_FMT_RGB555:
- g = LIMIT_RGB(g_);
- *f_odd++ = (0x1F & LIMIT_RGB(r_)) |
- (0xE0 & (g << 5));
- *f_odd++ = (0x03 & (g >> 3)) |
- (0x7C & (LIMIT_RGB(b_) << 2));
- break;
- }
- }
- clipmask_odd_index += clipmask_add;
- f_odd += stretch_bytes;
-
- if (frame->v4l2_format.format == V4L2_PIX_FMT_YUYV) {
- *f_odd++ = y[1];
- *f_odd++ = u;
- } else {
- y_ = 76284 * (y[1] - 16);
-
- b_ = (y_ + vb) >> 16;
- g_ = (y_ + uvg) >> 16;
- r_ = (y_ + ur) >> 16;
-
- switch (frame->v4l2_format.format) {
- case V4L2_PIX_FMT_RGB565:
- g = LIMIT_RGB(g_);
- *f_odd++ =
- (0x1F & LIMIT_RGB(r_)) |
- (0xE0 & (g << 5));
- *f_odd++ =
- (0x07 & (g >> 3)) |
- (0xF8 & LIMIT_RGB(b_));
- break;
- case V4L2_PIX_FMT_RGB24:
- *f_odd++ = LIMIT_RGB(r_);
- *f_odd++ = LIMIT_RGB(g_);
- *f_odd++ = LIMIT_RGB(b_);
- break;
- case V4L2_PIX_FMT_RGB32:
- *f_odd++ = LIMIT_RGB(r_);
- *f_odd++ = LIMIT_RGB(g_);
- *f_odd++ = LIMIT_RGB(b_);
- f_odd++;
- break;
- case V4L2_PIX_FMT_RGB555:
- g = LIMIT_RGB(g_);
- *f_odd++ = (0x1F & LIMIT_RGB(r_)) |
- (0xE0 & (g << 5));
- *f_odd++ = (0x03 & (g >> 3)) |
- (0x7C & (LIMIT_RGB(b_) << 2));
- break;
- }
- }
- clipmask_odd_index += clipmask_add;
- f_odd += stretch_bytes;
- }
-
- scratch_rm_old(usbvision, y_step[block % y_step_size] * sub_block_size);
- scratch_inc_extra_ptr(&y_ptr, y_step[(block + 2 * block_split) % y_step_size]
- * sub_block_size);
- scratch_inc_extra_ptr(&u_ptr, uv_step[block % uv_step_size]
- * sub_block_size);
- scratch_inc_extra_ptr(&v_ptr, uv_step[(block + 2 * block_split) % uv_step_size]
- * sub_block_size);
- }
-
- scratch_rm_old(usbvision, pixel_per_line * 3 / 2
- + block_split * sub_block_size);
-
- frame->curline += 2 * usbvision->stretch_height;
- *pcopylen += frame->v4l2_linesize * 2 * usbvision->stretch_height;
-
- if (frame->curline >= frame->frmheight)
- return parse_state_next_frame;
- return parse_state_continue;
-}
-
-/*
- * usbvision_parse_data()
- *
- * Generic routine to parse the scratch buffer. It employs either
- * usbvision_find_header() or usbvision_parse_lines() to do most
- * of work.
- *
- */
-static void usbvision_parse_data(struct usb_usbvision *usbvision)
-{
- struct usbvision_frame *frame;
- enum parse_state newstate;
- long copylen = 0;
- unsigned long lock_flags;
-
- frame = usbvision->cur_frame;
-
- PDEBUG(DBG_PARSE, "parsing len=%d\n", scratch_len(usbvision));
-
- while (1) {
- newstate = parse_state_out;
- if (scratch_len(usbvision)) {
- if (frame->scanstate == scan_state_scanning) {
- newstate = usbvision_find_header(usbvision);
- } else if (frame->scanstate == scan_state_lines) {
- if (usbvision->isoc_mode == ISOC_MODE_YUV420)
- newstate = usbvision_parse_lines_420(usbvision, ©len);
- else if (usbvision->isoc_mode == ISOC_MODE_YUV422)
- newstate = usbvision_parse_lines_422(usbvision, ©len);
- else if (usbvision->isoc_mode == ISOC_MODE_COMPRESS)
- newstate = usbvision_parse_compress(usbvision, ©len);
- }
- }
- if (newstate == parse_state_continue)
- continue;
- if ((newstate == parse_state_next_frame) || (newstate == parse_state_out))
- break;
- return; /* parse_state_end_parse */
- }
-
- if (newstate == parse_state_next_frame) {
- frame->grabstate = frame_state_done;
- frame->ts = ktime_get_ns();
- frame->sequence = usbvision->frame_num;
-
- spin_lock_irqsave(&usbvision->queue_lock, lock_flags);
- list_move_tail(&(frame->frame), &usbvision->outqueue);
- usbvision->cur_frame = NULL;
- spin_unlock_irqrestore(&usbvision->queue_lock, lock_flags);
-
- usbvision->frame_num++;
-
- /* This will cause the process to request another frame. */
- if (waitqueue_active(&usbvision->wait_frame)) {
- PDEBUG(DBG_PARSE, "Wake up !");
- wake_up_interruptible(&usbvision->wait_frame);
- }
- } else {
- frame->grabstate = frame_state_grabbing;
- }
-
- /* Update the frame's uncompressed length. */
- frame->scanlength += copylen;
-}
-
-
-/*
- * Make all of the blocks of data contiguous
- */
-static int usbvision_compress_isochronous(struct usb_usbvision *usbvision,
- struct urb *urb)
-{
- unsigned char *packet_data;
- int i, totlen = 0;
-
- for (i = 0; i < urb->number_of_packets; i++) {
- int packet_len = urb->iso_frame_desc[i].actual_length;
- int packet_stat = urb->iso_frame_desc[i].status;
-
- packet_data = urb->transfer_buffer + urb->iso_frame_desc[i].offset;
-
- /* Detect and ignore errored packets */
- if (packet_stat) { /* packet_stat != 0 ????????????? */
- PDEBUG(DBG_ISOC, "data error: [%d] len=%d, status=%X", i, packet_len, packet_stat);
- usbvision->isoc_err_count++;
- continue;
- }
-
- /* Detect and ignore empty packets */
- if (packet_len < 0) {
- PDEBUG(DBG_ISOC, "error packet [%d]", i);
- usbvision->isoc_skip_count++;
- continue;
- } else if (packet_len == 0) { /* Frame end ????? */
- PDEBUG(DBG_ISOC, "null packet [%d]", i);
- usbvision->isocstate = isoc_state_no_frame;
- usbvision->isoc_skip_count++;
- continue;
- } else if (packet_len > usbvision->isoc_packet_size) {
- PDEBUG(DBG_ISOC, "packet[%d] > isoc_packet_size", i);
- usbvision->isoc_skip_count++;
- continue;
- }
-
- PDEBUG(DBG_ISOC, "packet ok [%d] len=%d", i, packet_len);
-
- if (usbvision->isocstate == isoc_state_no_frame) { /* new frame begins */
- usbvision->isocstate = isoc_state_in_frame;
- scratch_mark_header(usbvision);
- usbvision_measure_bandwidth(usbvision);
- PDEBUG(DBG_ISOC, "packet with header");
- }
-
- /*
- * If usbvision continues to feed us with data but there is no
- * consumption (if, for example, V4L client fell asleep) we
- * may overflow the buffer. We have to move old data over to
- * free room for new data. This is bad for old data. If we
- * just drop new data then it's bad for new data... choose
- * your favorite evil here.
- */
- if (scratch_free(usbvision) < packet_len) {
- usbvision->scratch_ovf_count++;
- PDEBUG(DBG_ISOC, "scratch buf overflow! scr_len: %d, n: %d",
- scratch_len(usbvision), packet_len);
- scratch_rm_old(usbvision, packet_len - scratch_free(usbvision));
- }
-
- /* Now we know that there is enough room in scratch buffer */
- scratch_put(usbvision, packet_data, packet_len);
- totlen += packet_len;
- usbvision->isoc_data_count += packet_len;
- usbvision->isoc_packet_count++;
- }
-#if ENABLE_HEXDUMP
- if (totlen > 0) {
- static int foo;
-
- if (foo < 1) {
- printk(KERN_DEBUG "+%d.\n", usbvision->scratchlen);
- usbvision_hexdump(data0, (totlen > 64) ? 64 : totlen);
- ++foo;
- }
- }
-#endif
- return totlen;
-}
-
-static void usbvision_isoc_irq(struct urb *urb)
-{
- int err_code = 0;
- int len;
- struct usb_usbvision *usbvision = urb->context;
- int i;
- struct usbvision_frame **f;
-
- /* We don't want to do anything if we are about to be removed! */
- if (!USBVISION_IS_OPERATIONAL(usbvision))
- return;
-
- /* any urb with wrong status is ignored without acknowledgment */
- if (urb->status == -ENOENT)
- return;
-
- f = &usbvision->cur_frame;
-
- /* Manage streaming interruption */
- if (usbvision->streaming == stream_interrupt) {
- usbvision->streaming = stream_idle;
- if ((*f)) {
- (*f)->grabstate = frame_state_ready;
- (*f)->scanstate = scan_state_scanning;
- }
- PDEBUG(DBG_IRQ, "stream interrupted");
- wake_up_interruptible(&usbvision->wait_stream);
- }
-
- /* Copy the data received into our scratch buffer */
- len = usbvision_compress_isochronous(usbvision, urb);
-
- usbvision->isoc_urb_count++;
- usbvision->urb_length = len;
-
- if (usbvision->streaming == stream_on) {
- /* If we collected enough data let's parse! */
- if (scratch_len(usbvision) > USBVISION_HEADER_LENGTH &&
- !list_empty(&(usbvision->inqueue))) {
- if (!(*f)) {
- (*f) = list_entry(usbvision->inqueue.next,
- struct usbvision_frame,
- frame);
- }
- usbvision_parse_data(usbvision);
- } else {
- /* If we don't have a frame
- we're current working on, complain */
- PDEBUG(DBG_IRQ,
- "received data, but no one needs it");
- scratch_reset(usbvision);
- }
- } else {
- PDEBUG(DBG_IRQ, "received data, but no one needs it");
- scratch_reset(usbvision);
- }
-
- for (i = 0; i < USBVISION_URB_FRAMES; i++) {
- urb->iso_frame_desc[i].status = 0;
- urb->iso_frame_desc[i].actual_length = 0;
- }
-
- urb->status = 0;
- urb->dev = usbvision->dev;
- err_code = usb_submit_urb(urb, GFP_ATOMIC);
-
- if (err_code) {
- dev_err(&usbvision->dev->dev,
- "%s: usb_submit_urb failed: error %d\n",
- __func__, err_code);
- }
-
- return;
-}
-
-/*************************************/
-/* Low level usbvision access functions */
-/*************************************/
-
-/*
- * usbvision_read_reg()
- *
- * return < 0 -> Error
- * >= 0 -> Data
- */
-
-int usbvision_read_reg(struct usb_usbvision *usbvision, unsigned char reg)
-{
- int err_code = 0;
- unsigned char *buffer = usbvision->ctrl_urb_buffer;
-
- if (!USBVISION_IS_OPERATIONAL(usbvision))
- return -1;
-
- err_code = usb_control_msg(usbvision->dev, usb_rcvctrlpipe(usbvision->dev, 1),
- USBVISION_OP_CODE,
- USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_ENDPOINT,
- 0, (__u16) reg, buffer, 1, HZ);
-
- if (err_code < 0) {
- dev_err(&usbvision->dev->dev,
- "%s: failed: error %d\n", __func__, err_code);
- return err_code;
- }
- return buffer[0];
-}
-
-/*
- * usbvision_write_reg()
- *
- * return 1 -> Reg written
- * 0 -> usbvision is not yet ready
- * -1 -> Something went wrong
- */
-
-int usbvision_write_reg(struct usb_usbvision *usbvision, unsigned char reg,
- unsigned char value)
-{
- int err_code = 0;
-
- if (!USBVISION_IS_OPERATIONAL(usbvision))
- return 0;
-
- usbvision->ctrl_urb_buffer[0] = value;
- err_code = usb_control_msg(usbvision->dev, usb_sndctrlpipe(usbvision->dev, 1),
- USBVISION_OP_CODE,
- USB_DIR_OUT | USB_TYPE_VENDOR |
- USB_RECIP_ENDPOINT, 0, (__u16) reg,
- usbvision->ctrl_urb_buffer, 1, HZ);
-
- if (err_code < 0) {
- dev_err(&usbvision->dev->dev,
- "%s: failed: error %d\n", __func__, err_code);
- }
- return err_code;
-}
-
-
-static void usbvision_ctrl_urb_complete(struct urb *urb)
-{
- struct usb_usbvision *usbvision = (struct usb_usbvision *)urb->context;
-
- PDEBUG(DBG_IRQ, "");
- usbvision->ctrl_urb_busy = 0;
-}
-
-
-static int usbvision_write_reg_irq(struct usb_usbvision *usbvision, int address,
- unsigned char *data, int len)
-{
- int err_code = 0;
-
- PDEBUG(DBG_IRQ, "");
- if (len > 8)
- return -EFAULT;
- if (usbvision->ctrl_urb_busy)
- return -EBUSY;
- usbvision->ctrl_urb_busy = 1;
-
- usbvision->ctrl_urb_setup.bRequestType = USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_ENDPOINT;
- usbvision->ctrl_urb_setup.bRequest = USBVISION_OP_CODE;
- usbvision->ctrl_urb_setup.wValue = 0;
- usbvision->ctrl_urb_setup.wIndex = cpu_to_le16(address);
- usbvision->ctrl_urb_setup.wLength = cpu_to_le16(len);
- usb_fill_control_urb(usbvision->ctrl_urb, usbvision->dev,
- usb_sndctrlpipe(usbvision->dev, 1),
- (unsigned char *)&usbvision->ctrl_urb_setup,
- (void *)usbvision->ctrl_urb_buffer, len,
- usbvision_ctrl_urb_complete,
- (void *)usbvision);
-
- memcpy(usbvision->ctrl_urb_buffer, data, len);
-
- err_code = usb_submit_urb(usbvision->ctrl_urb, GFP_ATOMIC);
- if (err_code < 0) {
- /* error in usb_submit_urb() */
- usbvision->ctrl_urb_busy = 0;
- }
- PDEBUG(DBG_IRQ, "submit %d byte: error %d", len, err_code);
- return err_code;
-}
-
-
-static int usbvision_init_compression(struct usb_usbvision *usbvision)
-{
- usbvision->last_isoc_frame_num = -1;
- usbvision->isoc_data_count = 0;
- usbvision->isoc_packet_count = 0;
- usbvision->isoc_skip_count = 0;
- usbvision->compr_level = 50;
- usbvision->last_compr_level = -1;
- usbvision->isoc_urb_count = 0;
- usbvision->request_intra = 1;
- usbvision->isoc_measure_bandwidth_count = 0;
-
- return 0;
-}
-
-/* this function measures the used bandwidth since last call
- * return: 0 : no error
- * sets used_bandwidth to 1-100 : 1-100% of full bandwidth resp. to isoc_packet_size
- */
-static int usbvision_measure_bandwidth(struct usb_usbvision *usbvision)
-{
- if (usbvision->isoc_measure_bandwidth_count < 2) { /* this gives an average bandwidth of 3 frames */
- usbvision->isoc_measure_bandwidth_count++;
- return 0;
- }
- if ((usbvision->isoc_packet_size > 0) && (usbvision->isoc_packet_count > 0)) {
- usbvision->used_bandwidth = usbvision->isoc_data_count /
- (usbvision->isoc_packet_count + usbvision->isoc_skip_count) *
- 100 / usbvision->isoc_packet_size;
- }
- usbvision->isoc_measure_bandwidth_count = 0;
- usbvision->isoc_data_count = 0;
- usbvision->isoc_packet_count = 0;
- usbvision->isoc_skip_count = 0;
- return 0;
-}
-
-static int usbvision_adjust_compression(struct usb_usbvision *usbvision)
-{
- int err_code = 0;
- unsigned char buffer[6];
-
- PDEBUG(DBG_IRQ, "");
- if ((adjust_compression) && (usbvision->used_bandwidth > 0)) {
- usbvision->compr_level += (usbvision->used_bandwidth - 90) / 2;
- RESTRICT_TO_RANGE(usbvision->compr_level, 0, 100);
- if (usbvision->compr_level != usbvision->last_compr_level) {
- int distortion;
-
- if (usbvision->bridge_type == BRIDGE_NT1004 || usbvision->bridge_type == BRIDGE_NT1005) {
- buffer[0] = (unsigned char)(4 + 16 * usbvision->compr_level / 100); /* PCM Threshold 1 */
- buffer[1] = (unsigned char)(4 + 8 * usbvision->compr_level / 100); /* PCM Threshold 2 */
- distortion = 7 + 248 * usbvision->compr_level / 100;
- buffer[2] = (unsigned char)(distortion & 0xFF); /* Average distortion Threshold (inter) */
- buffer[3] = (unsigned char)(distortion & 0xFF); /* Average distortion Threshold (intra) */
- distortion = 1 + 42 * usbvision->compr_level / 100;
- buffer[4] = (unsigned char)(distortion & 0xFF); /* Maximum distortion Threshold (inter) */
- buffer[5] = (unsigned char)(distortion & 0xFF); /* Maximum distortion Threshold (intra) */
- } else { /* BRIDGE_NT1003 */
- buffer[0] = (unsigned char)(4 + 16 * usbvision->compr_level / 100); /* PCM threshold 1 */
- buffer[1] = (unsigned char)(4 + 8 * usbvision->compr_level / 100); /* PCM threshold 2 */
- distortion = 2 + 253 * usbvision->compr_level / 100;
- buffer[2] = (unsigned char)(distortion & 0xFF); /* distortion threshold bit0-7 */
- buffer[3] = 0; /* (unsigned char)((distortion >> 8) & 0x0F); distortion threshold bit 8-11 */
- distortion = 0 + 43 * usbvision->compr_level / 100;
- buffer[4] = (unsigned char)(distortion & 0xFF); /* maximum distortion bit0-7 */
- buffer[5] = 0; /* (unsigned char)((distortion >> 8) & 0x01); maximum distortion bit 8 */
- }
- err_code = usbvision_write_reg_irq(usbvision, USBVISION_PCM_THR1, buffer, 6);
- if (err_code == 0) {
- PDEBUG(DBG_IRQ, "new compr params %#02x %#02x %#02x %#02x %#02x %#02x", buffer[0],
- buffer[1], buffer[2], buffer[3], buffer[4], buffer[5]);
- usbvision->last_compr_level = usbvision->compr_level;
- }
- }
- }
- return err_code;
-}
-
-static int usbvision_request_intra(struct usb_usbvision *usbvision)
-{
- unsigned char buffer[1];
-
- PDEBUG(DBG_IRQ, "");
- usbvision->request_intra = 1;
- buffer[0] = 1;
- usbvision_write_reg_irq(usbvision, USBVISION_FORCE_INTRA, buffer, 1);
- return 0;
-}
-
-static int usbvision_unrequest_intra(struct usb_usbvision *usbvision)
-{
- unsigned char buffer[1];
-
- PDEBUG(DBG_IRQ, "");
- usbvision->request_intra = 0;
- buffer[0] = 0;
- usbvision_write_reg_irq(usbvision, USBVISION_FORCE_INTRA, buffer, 1);
- return 0;
-}
-
-/*******************************
- * usbvision utility functions
- *******************************/
-
-int usbvision_power_off(struct usb_usbvision *usbvision)
-{
- int err_code = 0;
-
- PDEBUG(DBG_FUNC, "");
-
- err_code = usbvision_write_reg(usbvision, USBVISION_PWR_REG, USBVISION_SSPND_EN);
- if (err_code == 1)
- usbvision->power = 0;
- PDEBUG(DBG_FUNC, "%s: err_code %d", (err_code != 1) ? "ERROR" : "power is off", err_code);
- return err_code;
-}
-
-/* configure webcam image sensor using the serial port */
-static int usbvision_init_webcam(struct usb_usbvision *usbvision)
-{
- int rc;
- int i;
- static char init_values[38][3] = {
- { 0x04, 0x12, 0x08 }, { 0x05, 0xff, 0xc8 }, { 0x06, 0x18, 0x07 }, { 0x07, 0x90, 0x00 },
- { 0x09, 0x00, 0x00 }, { 0x0a, 0x00, 0x00 }, { 0x0b, 0x08, 0x00 }, { 0x0d, 0xcc, 0xcc },
- { 0x0e, 0x13, 0x14 }, { 0x10, 0x9b, 0x83 }, { 0x11, 0x5a, 0x3f }, { 0x12, 0xe4, 0x73 },
- { 0x13, 0x88, 0x84 }, { 0x14, 0x89, 0x80 }, { 0x15, 0x00, 0x20 }, { 0x16, 0x00, 0x00 },
- { 0x17, 0xff, 0xa0 }, { 0x18, 0x6b, 0x20 }, { 0x19, 0x22, 0x40 }, { 0x1a, 0x10, 0x07 },
- { 0x1b, 0x00, 0x47 }, { 0x1c, 0x03, 0xe0 }, { 0x1d, 0x00, 0x00 }, { 0x1e, 0x00, 0x00 },
- { 0x1f, 0x00, 0x00 }, { 0x20, 0x00, 0x00 }, { 0x21, 0x00, 0x00 }, { 0x22, 0x00, 0x00 },
- { 0x23, 0x00, 0x00 }, { 0x24, 0x00, 0x00 }, { 0x25, 0x00, 0x00 }, { 0x26, 0x00, 0x00 },
- { 0x27, 0x00, 0x00 }, { 0x28, 0x00, 0x00 }, { 0x29, 0x00, 0x00 }, { 0x08, 0x80, 0x60 },
- { 0x0f, 0x2d, 0x24 }, { 0x0c, 0x80, 0x80 }
- };
- unsigned char *value = usbvision->ctrl_urb_buffer;
-
- /* the only difference between PAL and NTSC init_values */
- if (usbvision_device_data[usbvision->dev_model].video_norm == V4L2_STD_NTSC)
- init_values[4][1] = 0x34;
-
- for (i = 0; i < sizeof(init_values) / 3; i++) {
- usbvision_write_reg(usbvision, USBVISION_SER_MODE, USBVISION_SER_MODE_SOFT);
- memcpy(value, init_values[i], 3);
- rc = usb_control_msg(usbvision->dev,
- usb_sndctrlpipe(usbvision->dev, 1),
- USBVISION_OP_CODE,
- USB_DIR_OUT | USB_TYPE_VENDOR |
- USB_RECIP_ENDPOINT, 0,
- (__u16) USBVISION_SER_DAT1, value,
- 3, HZ);
- if (rc < 0)
- return rc;
- usbvision_write_reg(usbvision, USBVISION_SER_MODE, USBVISION_SER_MODE_SIO);
- /* write 3 bytes to the serial port using SIO mode */
- usbvision_write_reg(usbvision, USBVISION_SER_CONT, 3 | 0x10);
- usbvision_write_reg(usbvision, USBVISION_IOPIN_REG, 0);
- usbvision_write_reg(usbvision, USBVISION_SER_MODE, USBVISION_SER_MODE_SOFT);
- usbvision_write_reg(usbvision, USBVISION_IOPIN_REG, USBVISION_IO_2);
- usbvision_write_reg(usbvision, USBVISION_SER_MODE, USBVISION_SER_MODE_SOFT | USBVISION_CLK_OUT);
- usbvision_write_reg(usbvision, USBVISION_SER_MODE, USBVISION_SER_MODE_SOFT | USBVISION_DAT_IO);
- usbvision_write_reg(usbvision, USBVISION_SER_MODE, USBVISION_SER_MODE_SOFT | USBVISION_CLK_OUT | USBVISION_DAT_IO);
- }
-
- return 0;
-}
-
-/*
- * usbvision_set_video_format()
- *
- */
-static int usbvision_set_video_format(struct usb_usbvision *usbvision, int format)
-{
- static const char proc[] = "usbvision_set_video_format";
- unsigned char *value = usbvision->ctrl_urb_buffer;
- int rc;
-
- if (!USBVISION_IS_OPERATIONAL(usbvision))
- return 0;
-
- PDEBUG(DBG_FUNC, "isoc_mode %#02x", format);
-
- if ((format != ISOC_MODE_YUV422)
- && (format != ISOC_MODE_YUV420)
- && (format != ISOC_MODE_COMPRESS)) {
- printk(KERN_ERR "usbvision: unknown video format %02x, using default YUV420",
- format);
- format = ISOC_MODE_YUV420;
- }
- value[0] = 0x0A; /* TODO: See the effect of the filter */
- value[1] = format; /* Sets the VO_MODE register which follows FILT_CONT */
- rc = usb_control_msg(usbvision->dev, usb_sndctrlpipe(usbvision->dev, 1),
- USBVISION_OP_CODE,
- USB_DIR_OUT | USB_TYPE_VENDOR |
- USB_RECIP_ENDPOINT, 0,
- (__u16) USBVISION_FILT_CONT, value, 2, HZ);
-
- if (rc < 0) {
- printk(KERN_ERR "%s: ERROR=%d. USBVISION stopped - reconnect or reload driver.\n",
- proc, rc);
- }
- usbvision->isoc_mode = format;
- return rc;
-}
-
-/*
- * usbvision_set_output()
- *
- */
-
-int usbvision_set_output(struct usb_usbvision *usbvision, int width,
- int height)
-{
- int err_code = 0;
- int usb_width, usb_height;
- unsigned int frame_rate = 0, frame_drop = 0;
- unsigned char *value = usbvision->ctrl_urb_buffer;
-
- if (!USBVISION_IS_OPERATIONAL(usbvision))
- return 0;
-
- if (width > MAX_USB_WIDTH) {
- usb_width = width / 2;
- usbvision->stretch_width = 2;
- } else {
- usb_width = width;
- usbvision->stretch_width = 1;
- }
-
- if (height > MAX_USB_HEIGHT) {
- usb_height = height / 2;
- usbvision->stretch_height = 2;
- } else {
- usb_height = height;
- usbvision->stretch_height = 1;
- }
-
- RESTRICT_TO_RANGE(usb_width, MIN_FRAME_WIDTH, MAX_USB_WIDTH);
- usb_width &= ~(MIN_FRAME_WIDTH-1);
- RESTRICT_TO_RANGE(usb_height, MIN_FRAME_HEIGHT, MAX_USB_HEIGHT);
- usb_height &= ~(1);
-
- PDEBUG(DBG_FUNC, "usb %dx%d; screen %dx%d; stretch %dx%d",
- usb_width, usb_height, width, height,
- usbvision->stretch_width, usbvision->stretch_height);
-
- /* I'll not rewrite the same values */
- if ((usb_width != usbvision->curwidth) || (usb_height != usbvision->curheight)) {
- value[0] = usb_width & 0xff; /* LSB */
- value[1] = (usb_width >> 8) & 0x03; /* MSB */
- value[2] = usb_height & 0xff; /* LSB */
- value[3] = (usb_height >> 8) & 0x03; /* MSB */
-
- err_code = usb_control_msg(usbvision->dev, usb_sndctrlpipe(usbvision->dev, 1),
- USBVISION_OP_CODE,
- USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_ENDPOINT,
- 0, (__u16) USBVISION_LXSIZE_O, value, 4, HZ);
-
- if (err_code < 0) {
- dev_err(&usbvision->dev->dev,
- "%s failed: error %d\n", __func__, err_code);
- return err_code;
- }
- usbvision->curwidth = usbvision->stretch_width * usb_width;
- usbvision->curheight = usbvision->stretch_height * usb_height;
- }
-
- if (usbvision->isoc_mode == ISOC_MODE_YUV422)
- frame_rate = (usbvision->isoc_packet_size * 1000) / (usb_width * usb_height * 2);
- else if (usbvision->isoc_mode == ISOC_MODE_YUV420)
- frame_rate = (usbvision->isoc_packet_size * 1000) / ((usb_width * usb_height * 12) / 8);
- else
- frame_rate = FRAMERATE_MAX;
-
- if (usbvision->tvnorm_id & V4L2_STD_625_50)
- frame_drop = frame_rate * 32 / 25 - 1;
- else if (usbvision->tvnorm_id & V4L2_STD_525_60)
- frame_drop = frame_rate * 32 / 30 - 1;
-
- RESTRICT_TO_RANGE(frame_drop, FRAMERATE_MIN, FRAMERATE_MAX);
-
- PDEBUG(DBG_FUNC, "frame_rate %d fps, frame_drop %d", frame_rate, frame_drop);
-
- frame_drop = FRAMERATE_MAX; /* We can allow the maximum here, because dropping is controlled */
-
- if (usbvision_device_data[usbvision->dev_model].codec == CODEC_WEBCAM) {
- if (usbvision_device_data[usbvision->dev_model].video_norm == V4L2_STD_PAL)
- frame_drop = 25;
- else
- frame_drop = 30;
- }
-
- /* frame_drop = 7; => frame_phase = 1, 5, 9, 13, 17, 21, 25, 0, 4, 8, ...
- => frame_skip = 4;
- => frame_rate = (7 + 1) * 25 / 32 = 200 / 32 = 6.25;
-
- frame_drop = 9; => frame_phase = 1, 5, 8, 11, 14, 17, 21, 24, 27, 1, 4, 8, ...
- => frame_skip = 4, 3, 3, 3, 3, 4, 3, 3, 3, 3, 4, ...
- => frame_rate = (9 + 1) * 25 / 32 = 250 / 32 = 7.8125;
- */
- err_code = usbvision_write_reg(usbvision, USBVISION_FRM_RATE, frame_drop);
- return err_code;
-}
-
-
-/*
- * usbvision_frames_alloc
- * allocate the required frames
- */
-int usbvision_frames_alloc(struct usb_usbvision *usbvision, int number_of_frames)
-{
- int i;
-
- /* needs to be page aligned cause the buffers can be mapped individually! */
- usbvision->max_frame_size = PAGE_ALIGN(usbvision->curwidth *
- usbvision->curheight *
- usbvision->palette.bytes_per_pixel);
-
- /* Try to do my best to allocate the frames the user want in the remaining memory */
- usbvision->num_frames = number_of_frames;
- while (usbvision->num_frames > 0) {
- usbvision->fbuf_size = usbvision->num_frames * usbvision->max_frame_size;
- usbvision->fbuf = usbvision_rvmalloc(usbvision->fbuf_size);
- if (usbvision->fbuf)
- break;
- usbvision->num_frames--;
- }
-
- /* Allocate all buffers */
- for (i = 0; i < usbvision->num_frames; i++) {
- usbvision->frame[i].index = i;
- usbvision->frame[i].grabstate = frame_state_unused;
- usbvision->frame[i].data = usbvision->fbuf +
- i * usbvision->max_frame_size;
- /*
- * Set default sizes for read operation.
- */
- usbvision->stretch_width = 1;
- usbvision->stretch_height = 1;
- usbvision->frame[i].width = usbvision->curwidth;
- usbvision->frame[i].height = usbvision->curheight;
- usbvision->frame[i].bytes_read = 0;
- }
- PDEBUG(DBG_FUNC, "allocated %d frames (%d bytes per frame)",
- usbvision->num_frames, usbvision->max_frame_size);
- return usbvision->num_frames;
-}
-
-/*
- * usbvision_frames_free
- * frees memory allocated for the frames
- */
-void usbvision_frames_free(struct usb_usbvision *usbvision)
-{
- /* Have to free all that memory */
- PDEBUG(DBG_FUNC, "free %d frames", usbvision->num_frames);
-
- if (usbvision->fbuf != NULL) {
- usbvision_rvfree(usbvision->fbuf, usbvision->fbuf_size);
- usbvision->fbuf = NULL;
-
- usbvision->num_frames = 0;
- }
-}
-/*
- * usbvision_empty_framequeues()
- * prepare queues for incoming and outgoing frames
- */
-void usbvision_empty_framequeues(struct usb_usbvision *usbvision)
-{
- u32 i;
-
- INIT_LIST_HEAD(&(usbvision->inqueue));
- INIT_LIST_HEAD(&(usbvision->outqueue));
-
- for (i = 0; i < USBVISION_NUMFRAMES; i++) {
- usbvision->frame[i].grabstate = frame_state_unused;
- usbvision->frame[i].bytes_read = 0;
- }
-}
-
-/*
- * usbvision_stream_interrupt()
- * stops streaming
- */
-int usbvision_stream_interrupt(struct usb_usbvision *usbvision)
-{
- int ret = 0;
-
- /* stop reading from the device */
-
- usbvision->streaming = stream_interrupt;
- ret = wait_event_timeout(usbvision->wait_stream,
- (usbvision->streaming == stream_idle),
- msecs_to_jiffies(USBVISION_NUMSBUF*USBVISION_URB_FRAMES));
- return ret;
-}
-
-/*
- * usbvision_set_compress_params()
- *
- */
-
-static int usbvision_set_compress_params(struct usb_usbvision *usbvision)
-{
- static const char proc[] = "usbvision_set_compression_params: ";
- int rc;
- unsigned char *value = usbvision->ctrl_urb_buffer;
-
- value[0] = 0x0F; /* Intra-Compression cycle */
- value[1] = 0x01; /* Reg.45 one line per strip */
- value[2] = 0x00; /* Reg.46 Force intra mode on all new frames */
- value[3] = 0x00; /* Reg.47 FORCE_UP <- 0 normal operation (not force) */
- value[4] = 0xA2; /* Reg.48 BUF_THR I'm not sure if this does something in not compressed mode. */
- value[5] = 0x00; /* Reg.49 DVI_YUV This has nothing to do with compression */
-
- /* caught values for NT1004 */
- /* value[0] = 0xFF; Never apply intra mode automatically */
- /* value[1] = 0xF1; Use full frame height for virtual strip width; One line per strip */
- /* value[2] = 0x01; Force intra mode on all new frames */
- /* value[3] = 0x00; Strip size 400 Bytes; do not force up */
- /* value[4] = 0xA2; */
- if (!USBVISION_IS_OPERATIONAL(usbvision))
- return 0;
-
- rc = usb_control_msg(usbvision->dev, usb_sndctrlpipe(usbvision->dev, 1),
- USBVISION_OP_CODE,
- USB_DIR_OUT | USB_TYPE_VENDOR |
- USB_RECIP_ENDPOINT, 0,
- (__u16) USBVISION_INTRA_CYC, value, 5, HZ);
-
- if (rc < 0) {
- printk(KERN_ERR "%sERROR=%d. USBVISION stopped - reconnect or reload driver.\n",
- proc, rc);
- return rc;
- }
-
- if (usbvision->bridge_type == BRIDGE_NT1004) {
- value[0] = 20; /* PCM Threshold 1 */
- value[1] = 12; /* PCM Threshold 2 */
- value[2] = 255; /* Distortion Threshold inter */
- value[3] = 255; /* Distortion Threshold intra */
- value[4] = 43; /* Max Distortion inter */
- value[5] = 43; /* Max Distortion intra */
- } else {
- value[0] = 20; /* PCM Threshold 1 */
- value[1] = 12; /* PCM Threshold 2 */
- value[2] = 255; /* Distortion Threshold d7-d0 */
- value[3] = 0; /* Distortion Threshold d11-d8 */
- value[4] = 43; /* Max Distortion d7-d0 */
- value[5] = 0; /* Max Distortion d8 */
- }
-
- if (!USBVISION_IS_OPERATIONAL(usbvision))
- return 0;
-
- rc = usb_control_msg(usbvision->dev, usb_sndctrlpipe(usbvision->dev, 1),
- USBVISION_OP_CODE,
- USB_DIR_OUT | USB_TYPE_VENDOR |
- USB_RECIP_ENDPOINT, 0,
- (__u16) USBVISION_PCM_THR1, value, 6, HZ);
-
- if (rc < 0) {
- printk(KERN_ERR "%sERROR=%d. USBVISION stopped - reconnect or reload driver.\n",
- proc, rc);
- }
- return rc;
-}
-
-
-/*
- * usbvision_set_input()
- *
- * Set the input (saa711x, ...) size x y and other misc input params
- * I've no idea if this parameters are right
- *
- */
-int usbvision_set_input(struct usb_usbvision *usbvision)
-{
- static const char proc[] = "usbvision_set_input: ";
- int rc;
- unsigned char *value = usbvision->ctrl_urb_buffer;
- unsigned char dvi_yuv_value;
-
- if (!USBVISION_IS_OPERATIONAL(usbvision))
- return 0;
-
- /* Set input format expected from decoder*/
- if (usbvision_device_data[usbvision->dev_model].vin_reg1_override) {
- value[0] = usbvision_device_data[usbvision->dev_model].vin_reg1;
- } else if (usbvision_device_data[usbvision->dev_model].codec == CODEC_SAA7113) {
- /* SAA7113 uses 8 bit output */
- value[0] = USBVISION_8_422_SYNC;
- } else {
- /* I'm sure only about d2-d0 [010] 16 bit 4:2:2 using sync pulses
- * as that is how saa7111 is configured */
- value[0] = USBVISION_16_422_SYNC;
- /* | USBVISION_VSNC_POL | USBVISION_VCLK_POL);*/
- }
-
- rc = usbvision_write_reg(usbvision, USBVISION_VIN_REG1, value[0]);
- if (rc < 0) {
- printk(KERN_ERR "%sERROR=%d. USBVISION stopped - reconnect or reload driver.\n",
- proc, rc);
- return rc;
- }
-
-
- if (usbvision->tvnorm_id & V4L2_STD_PAL) {
- value[0] = 0xC0;
- value[1] = 0x02; /* 0x02C0 -> 704 Input video line length */
- value[2] = 0x20;
- value[3] = 0x01; /* 0x0120 -> 288 Input video n. of lines */
- value[4] = 0x60;
- value[5] = 0x00; /* 0x0060 -> 96 Input video h offset */
- value[6] = 0x16;
- value[7] = 0x00; /* 0x0016 -> 22 Input video v offset */
- } else if (usbvision->tvnorm_id & V4L2_STD_SECAM) {
- value[0] = 0xC0;
- value[1] = 0x02; /* 0x02C0 -> 704 Input video line length */
- value[2] = 0x20;
- value[3] = 0x01; /* 0x0120 -> 288 Input video n. of lines */
- value[4] = 0x01;
- value[5] = 0x00; /* 0x0001 -> 01 Input video h offset */
- value[6] = 0x01;
- value[7] = 0x00; /* 0x0001 -> 01 Input video v offset */
- } else { /* V4L2_STD_NTSC */
- value[0] = 0xD0;
- value[1] = 0x02; /* 0x02D0 -> 720 Input video line length */
- value[2] = 0xF0;
- value[3] = 0x00; /* 0x00F0 -> 240 Input video number of lines */
- value[4] = 0x50;
- value[5] = 0x00; /* 0x0050 -> 80 Input video h offset */
- value[6] = 0x10;
- value[7] = 0x00; /* 0x0010 -> 16 Input video v offset */
- }
-
- /* webcam is only 480 pixels wide, both PAL and NTSC version */
- if (usbvision_device_data[usbvision->dev_model].codec == CODEC_WEBCAM) {
- value[0] = 0xe0;
- value[1] = 0x01; /* 0x01E0 -> 480 Input video line length */
- }
-
- if (usbvision_device_data[usbvision->dev_model].x_offset >= 0) {
- value[4] = usbvision_device_data[usbvision->dev_model].x_offset & 0xff;
- value[5] = (usbvision_device_data[usbvision->dev_model].x_offset & 0x0300) >> 8;
- }
-
- if (adjust_x_offset != -1) {
- value[4] = adjust_x_offset & 0xff;
- value[5] = (adjust_x_offset & 0x0300) >> 8;
- }
-
- if (usbvision_device_data[usbvision->dev_model].y_offset >= 0) {
- value[6] = usbvision_device_data[usbvision->dev_model].y_offset & 0xff;
- value[7] = (usbvision_device_data[usbvision->dev_model].y_offset & 0x0300) >> 8;
- }
-
- if (adjust_y_offset != -1) {
- value[6] = adjust_y_offset & 0xff;
- value[7] = (adjust_y_offset & 0x0300) >> 8;
- }
-
- rc = usb_control_msg(usbvision->dev, usb_sndctrlpipe(usbvision->dev, 1),
- USBVISION_OP_CODE, /* USBVISION specific code */
- USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_ENDPOINT, 0,
- (__u16) USBVISION_LXSIZE_I, value, 8, HZ);
- if (rc < 0) {
- printk(KERN_ERR "%sERROR=%d. USBVISION stopped - reconnect or reload driver.\n",
- proc, rc);
- return rc;
- }
-
-
- dvi_yuv_value = 0x00; /* U comes after V, Ya comes after U/V, Yb comes after Yb */
-
- if (usbvision_device_data[usbvision->dev_model].dvi_yuv_override) {
- dvi_yuv_value = usbvision_device_data[usbvision->dev_model].dvi_yuv;
- } else if (usbvision_device_data[usbvision->dev_model].codec == CODEC_SAA7113) {
- /* This changes as the fine sync control changes. Further investigation necessary */
- dvi_yuv_value = 0x06;
- }
-
- return usbvision_write_reg(usbvision, USBVISION_DVI_YUV, dvi_yuv_value);
-}
-
-
-/*
- * usbvision_set_dram_settings()
- *
- * Set the buffer address needed by the usbvision dram to operate
- * This values has been taken with usbsnoop.
- *
- */
-
-static int usbvision_set_dram_settings(struct usb_usbvision *usbvision)
-{
- unsigned char *value = usbvision->ctrl_urb_buffer;
- int rc;
-
- if (usbvision->isoc_mode == ISOC_MODE_COMPRESS) {
- value[0] = 0x42;
- value[1] = 0x71;
- value[2] = 0xff;
- value[3] = 0x00;
- value[4] = 0x98;
- value[5] = 0xe0;
- value[6] = 0x71;
- value[7] = 0xff;
- /* UR: 0x0E200-0x3FFFF = 204288 Words (1 Word = 2 Byte) */
- /* FDL: 0x00000-0x0E099 = 57498 Words */
- /* VDW: 0x0E3FF-0x3FFFF */
- } else {
- value[0] = 0x42;
- value[1] = 0x00;
- value[2] = 0xff;
- value[3] = 0x00;
- value[4] = 0x00;
- value[5] = 0x00;
- value[6] = 0x00;
- value[7] = 0xff;
- }
- /* These are the values of the address of the video buffer,
- * they have to be loaded into the USBVISION_DRM_PRM1-8
- *
- * Start address of video output buffer for read: drm_prm1-2 -> 0x00000
- * End address of video output buffer for read: drm_prm1-3 -> 0x1ffff
- * Start address of video frame delay buffer: drm_prm1-4 -> 0x20000
- * Only used in compressed mode
- * End address of video frame delay buffer: drm_prm1-5-6 -> 0x3ffff
- * Only used in compressed mode
- * Start address of video output buffer for write: drm_prm1-7 -> 0x00000
- * End address of video output buffer for write: drm_prm1-8 -> 0x1ffff
- */
-
- if (!USBVISION_IS_OPERATIONAL(usbvision))
- return 0;
-
- rc = usb_control_msg(usbvision->dev, usb_sndctrlpipe(usbvision->dev, 1),
- USBVISION_OP_CODE, /* USBVISION specific code */
- USB_DIR_OUT | USB_TYPE_VENDOR |
- USB_RECIP_ENDPOINT, 0,
- (__u16) USBVISION_DRM_PRM1, value, 8, HZ);
-
- if (rc < 0) {
- dev_err(&usbvision->dev->dev, "%s: ERROR=%d\n", __func__, rc);
- return rc;
- }
-
- /* Restart the video buffer logic */
- rc = usbvision_write_reg(usbvision, USBVISION_DRM_CONT, USBVISION_RES_UR |
- USBVISION_RES_FDL | USBVISION_RES_VDW);
- if (rc < 0)
- return rc;
- rc = usbvision_write_reg(usbvision, USBVISION_DRM_CONT, 0x00);
-
- return rc;
-}
-
-/*
- * ()
- *
- * Power on the device, enables suspend-resume logic
- * & reset the isoc End-Point
- *
- */
-
-int usbvision_power_on(struct usb_usbvision *usbvision)
-{
- int err_code = 0;
-
- PDEBUG(DBG_FUNC, "");
-
- usbvision_write_reg(usbvision, USBVISION_PWR_REG, USBVISION_SSPND_EN);
- usbvision_write_reg(usbvision, USBVISION_PWR_REG,
- USBVISION_SSPND_EN | USBVISION_RES2);
-
- if (usbvision_device_data[usbvision->dev_model].codec == CODEC_WEBCAM) {
- usbvision_write_reg(usbvision, USBVISION_VIN_REG1,
- USBVISION_16_422_SYNC | USBVISION_HVALID_PO);
- usbvision_write_reg(usbvision, USBVISION_VIN_REG2,
- USBVISION_NOHVALID | USBVISION_KEEP_BLANK);
- }
- usbvision_write_reg(usbvision, USBVISION_PWR_REG,
- USBVISION_SSPND_EN | USBVISION_PWR_VID);
- mdelay(10);
- err_code = usbvision_write_reg(usbvision, USBVISION_PWR_REG,
- USBVISION_SSPND_EN | USBVISION_PWR_VID | USBVISION_RES2);
- if (err_code == 1)
- usbvision->power = 1;
- PDEBUG(DBG_FUNC, "%s: err_code %d", (err_code < 0) ? "ERROR" : "power is on", err_code);
- return err_code;
-}
-
-
-/*
- * usbvision_begin_streaming()
- * Sure you have to put bit 7 to 0, if not incoming frames are dropped, but no
- * idea about the rest
- */
-int usbvision_begin_streaming(struct usb_usbvision *usbvision)
-{
- if (usbvision->isoc_mode == ISOC_MODE_COMPRESS)
- usbvision_init_compression(usbvision);
- return usbvision_write_reg(usbvision, USBVISION_VIN_REG2,
- USBVISION_NOHVALID | usbvision->vin_reg2_preset);
-}
-
-/*
- * usbvision_restart_isoc()
- * Not sure yet if touching here PWR_REG make loose the config
- */
-
-int usbvision_restart_isoc(struct usb_usbvision *usbvision)
-{
- int ret;
-
- ret = usbvision_write_reg(usbvision, USBVISION_PWR_REG,
- USBVISION_SSPND_EN | USBVISION_PWR_VID);
- if (ret < 0)
- return ret;
- ret = usbvision_write_reg(usbvision, USBVISION_PWR_REG,
- USBVISION_SSPND_EN | USBVISION_PWR_VID |
- USBVISION_RES2);
- if (ret < 0)
- return ret;
- ret = usbvision_write_reg(usbvision, USBVISION_VIN_REG2,
- USBVISION_KEEP_BLANK | USBVISION_NOHVALID |
- usbvision->vin_reg2_preset);
- if (ret < 0)
- return ret;
-
- /* TODO: schedule timeout */
- while ((usbvision_read_reg(usbvision, USBVISION_STATUS_REG) & 0x01) != 1)
- ;
-
- return 0;
-}
-
-int usbvision_audio_off(struct usb_usbvision *usbvision)
-{
- if (usbvision_write_reg(usbvision, USBVISION_IOPIN_REG, USBVISION_AUDIO_MUTE) < 0) {
- printk(KERN_ERR "usbvision_audio_off: can't write reg\n");
- return -1;
- }
- usbvision->audio_mute = 0;
- usbvision->audio_channel = USBVISION_AUDIO_MUTE;
- return 0;
-}
-
-int usbvision_set_audio(struct usb_usbvision *usbvision, int audio_channel)
-{
- if (!usbvision->audio_mute) {
- if (usbvision_write_reg(usbvision, USBVISION_IOPIN_REG, audio_channel) < 0) {
- printk(KERN_ERR "usbvision_set_audio: can't write iopin register for audio switching\n");
- return -1;
- }
- }
- usbvision->audio_channel = audio_channel;
- return 0;
-}
-
-int usbvision_setup(struct usb_usbvision *usbvision, int format)
-{
- if (usbvision_device_data[usbvision->dev_model].codec == CODEC_WEBCAM)
- usbvision_init_webcam(usbvision);
- usbvision_set_video_format(usbvision, format);
- usbvision_set_dram_settings(usbvision);
- usbvision_set_compress_params(usbvision);
- usbvision_set_input(usbvision);
- usbvision_set_output(usbvision, MAX_USB_WIDTH, MAX_USB_HEIGHT);
- usbvision_restart_isoc(usbvision);
-
- /* cosas del PCM */
- return USBVISION_IS_OPERATIONAL(usbvision);
-}
-
-int usbvision_set_alternate(struct usb_usbvision *dev)
-{
- int err_code, prev_alt = dev->iface_alt;
- int i;
-
- dev->iface_alt = 0;
- for (i = 0; i < dev->num_alt; i++)
- if (dev->alt_max_pkt_size[i] > dev->alt_max_pkt_size[dev->iface_alt])
- dev->iface_alt = i;
-
- if (dev->iface_alt != prev_alt) {
- dev->isoc_packet_size = dev->alt_max_pkt_size[dev->iface_alt];
- PDEBUG(DBG_FUNC, "setting alternate %d with max_packet_size=%u",
- dev->iface_alt, dev->isoc_packet_size);
- err_code = usb_set_interface(dev->dev, dev->iface, dev->iface_alt);
- if (err_code < 0) {
- dev_err(&dev->dev->dev,
- "cannot change alternate number to %d (error=%i)\n",
- dev->iface_alt, err_code);
- return err_code;
- }
- }
-
- PDEBUG(DBG_ISOC, "ISO Packet Length:%d", dev->isoc_packet_size);
-
- return 0;
-}
-
-/*
- * usbvision_init_isoc()
- *
- */
-int usbvision_init_isoc(struct usb_usbvision *usbvision)
-{
- struct usb_device *dev = usbvision->dev;
- int buf_idx, err_code, reg_value;
- int sb_size;
-
- if (!USBVISION_IS_OPERATIONAL(usbvision))
- return -EFAULT;
-
- usbvision->cur_frame = NULL;
- scratch_reset(usbvision);
-
- /* Alternate interface 1 is is the biggest frame size */
- err_code = usbvision_set_alternate(usbvision);
- if (err_code < 0) {
- usbvision->last_error = err_code;
- return -EBUSY;
- }
- sb_size = USBVISION_URB_FRAMES * usbvision->isoc_packet_size;
-
- reg_value = (16 - usbvision_read_reg(usbvision,
- USBVISION_ALTER_REG)) & 0x0F;
-
- usbvision->usb_bandwidth = reg_value >> 1;
- PDEBUG(DBG_ISOC, "USB Bandwidth Usage: %dMbit/Sec",
- usbvision->usb_bandwidth);
-
-
-
- /* We double buffer the Iso lists */
-
- for (buf_idx = 0; buf_idx < USBVISION_NUMSBUF; buf_idx++) {
- int j, k;
- struct urb *urb;
-
- urb = usb_alloc_urb(USBVISION_URB_FRAMES, GFP_KERNEL);
- if (urb == NULL)
- return -ENOMEM;
- usbvision->sbuf[buf_idx].urb = urb;
- usbvision->sbuf[buf_idx].data =
- usb_alloc_coherent(usbvision->dev,
- sb_size,
- GFP_KERNEL,
- &urb->transfer_dma);
- if (!usbvision->sbuf[buf_idx].data)
- return -ENOMEM;
-
- urb->dev = dev;
- urb->context = usbvision;
- urb->pipe = usb_rcvisocpipe(dev, usbvision->video_endp);
- urb->transfer_flags = URB_ISO_ASAP | URB_NO_TRANSFER_DMA_MAP;
- urb->interval = 1;
- urb->transfer_buffer = usbvision->sbuf[buf_idx].data;
- urb->complete = usbvision_isoc_irq;
- urb->number_of_packets = USBVISION_URB_FRAMES;
- urb->transfer_buffer_length =
- usbvision->isoc_packet_size * USBVISION_URB_FRAMES;
- for (j = k = 0; j < USBVISION_URB_FRAMES; j++,
- k += usbvision->isoc_packet_size) {
- urb->iso_frame_desc[j].offset = k;
- urb->iso_frame_desc[j].length =
- usbvision->isoc_packet_size;
- }
- }
-
- /* Submit all URBs */
- for (buf_idx = 0; buf_idx < USBVISION_NUMSBUF; buf_idx++) {
- err_code = usb_submit_urb(usbvision->sbuf[buf_idx].urb,
- GFP_KERNEL);
- if (err_code) {
- dev_err(&usbvision->dev->dev,
- "%s: usb_submit_urb(%d) failed: error %d\n",
- __func__, buf_idx, err_code);
- }
- }
-
- usbvision->streaming = stream_idle;
- PDEBUG(DBG_ISOC, "%s: streaming=1 usbvision->video_endp=$%02x",
- __func__,
- usbvision->video_endp);
- return 0;
-}
-
-/*
- * usbvision_stop_isoc()
- *
- * This procedure stops streaming and deallocates URBs. Then it
- * activates zero-bandwidth alt. setting of the video interface.
- *
- */
-void usbvision_stop_isoc(struct usb_usbvision *usbvision)
-{
- int buf_idx, err_code, reg_value;
- int sb_size = USBVISION_URB_FRAMES * usbvision->isoc_packet_size;
-
- if ((usbvision->streaming == stream_off) || (usbvision->dev == NULL))
- return;
-
- /* Unschedule all of the iso td's */
- for (buf_idx = 0; buf_idx < USBVISION_NUMSBUF; buf_idx++) {
- usb_kill_urb(usbvision->sbuf[buf_idx].urb);
- if (usbvision->sbuf[buf_idx].data) {
- usb_free_coherent(usbvision->dev,
- sb_size,
- usbvision->sbuf[buf_idx].data,
- usbvision->sbuf[buf_idx].urb->transfer_dma);
- }
- usb_free_urb(usbvision->sbuf[buf_idx].urb);
- usbvision->sbuf[buf_idx].urb = NULL;
- }
-
- PDEBUG(DBG_ISOC, "%s: streaming=stream_off\n", __func__);
- usbvision->streaming = stream_off;
-
- if (!usbvision->remove_pending) {
- /* Set packet size to 0 */
- usbvision->iface_alt = 0;
- err_code = usb_set_interface(usbvision->dev, usbvision->iface,
- usbvision->iface_alt);
- if (err_code < 0) {
- dev_err(&usbvision->dev->dev,
- "%s: usb_set_interface() failed: error %d\n",
- __func__, err_code);
- usbvision->last_error = err_code;
- }
- reg_value = (16-usbvision_read_reg(usbvision, USBVISION_ALTER_REG)) & 0x0F;
- usbvision->isoc_packet_size =
- (reg_value == 0) ? 0 : (reg_value * 64) - 1;
- PDEBUG(DBG_ISOC, "ISO Packet Length:%d",
- usbvision->isoc_packet_size);
-
- usbvision->usb_bandwidth = reg_value >> 1;
- PDEBUG(DBG_ISOC, "USB Bandwidth Usage: %dMbit/Sec",
- usbvision->usb_bandwidth);
- }
-}
-
-int usbvision_muxsel(struct usb_usbvision *usbvision, int channel)
-{
- /* inputs #0 and #3 are constant for every SAA711x. */
- /* inputs #1 and #2 are variable for SAA7111 and SAA7113 */
- int mode[4] = { SAA7115_COMPOSITE0, 0, 0, SAA7115_COMPOSITE3 };
- int audio[] = { 1, 0, 0, 0 };
- /* channel 0 is TV with audiochannel 1 (tuner mono) */
- /* channel 1 is Composite with audio channel 0 (line in) */
- /* channel 2 is S-Video with audio channel 0 (line in) */
- /* channel 3 is additional video inputs to the device with audio channel 0 (line in) */
-
- RESTRICT_TO_RANGE(channel, 0, usbvision->video_inputs);
- usbvision->ctl_input = channel;
-
- /* set the new channel */
- /* Regular USB TV Tuners -> channel: 0 = Television, 1 = Composite, 2 = S-Video */
- /* Four video input devices -> channel: 0 = Chan White, 1 = Chan Green, 2 = Chan Yellow, 3 = Chan Red */
-
- switch (usbvision_device_data[usbvision->dev_model].codec) {
- case CODEC_SAA7113:
- mode[1] = SAA7115_COMPOSITE2;
- if (switch_svideo_input) {
- /* To handle problems with S-Video Input for
- * some devices. Use switch_svideo_input
- * parameter when loading the module.*/
- mode[2] = SAA7115_COMPOSITE1;
- } else {
- mode[2] = SAA7115_SVIDEO1;
- }
- break;
- case CODEC_SAA7111:
- default:
- /* modes for saa7111 */
- mode[1] = SAA7115_COMPOSITE1;
- mode[2] = SAA7115_SVIDEO1;
- break;
- }
- call_all(usbvision, video, s_routing, mode[channel], 0, 0);
- usbvision_set_audio(usbvision, audio[channel]);
- return 0;
-}
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * usbvision_i2c.c
- * i2c algorithm for USB-I2C Bridges
- *
- * Copyright (c) 1999-2007 Joerg Heckenbach <joerg@heckenbach-aw.de>
- * Dwaine Garden <dwainegarden@rogers.com>
- *
- * This module is part of usbvision driver project.
- * Updates to driver completed by Dwaine P. Garden
- */
-
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/delay.h>
-#include <linux/init.h>
-#include <linux/uaccess.h>
-#include <linux/ioport.h>
-#include <linux/errno.h>
-#include <linux/usb.h>
-#include <linux/i2c.h>
-#include "usbvision.h"
-
-#define DBG_I2C (1 << 0)
-
-static int i2c_debug;
-
-module_param(i2c_debug, int, 0644); /* debug_i2c_usb mode of the device driver */
-MODULE_PARM_DESC(i2c_debug, "enable debug messages [i2c]");
-
-#define PDEBUG(level, fmt, args...) { \
- if (i2c_debug & (level)) \
- printk(KERN_INFO KBUILD_MODNAME ":[%s:%d] " fmt, \
- __func__, __LINE__ , ## args); \
- }
-
-static int usbvision_i2c_write(struct usb_usbvision *usbvision, unsigned char addr, char *buf,
- short len);
-static int usbvision_i2c_read(struct usb_usbvision *usbvision, unsigned char addr, char *buf,
- short len);
-
-static inline int try_write_address(struct i2c_adapter *i2c_adap,
- unsigned char addr, int retries)
-{
- struct usb_usbvision *usbvision;
- int i, ret = -1;
- char buf[4];
-
- usbvision = (struct usb_usbvision *)i2c_get_adapdata(i2c_adap);
- buf[0] = 0x00;
- for (i = 0; i <= retries; i++) {
- ret = (usbvision_i2c_write(usbvision, addr, buf, 1));
- if (ret == 1)
- break; /* success! */
- udelay(5);
- if (i == retries) /* no success */
- break;
- udelay(10);
- }
- if (i) {
- PDEBUG(DBG_I2C, "Needed %d retries for address %#2x", i, addr);
- PDEBUG(DBG_I2C, "Maybe there's no device at this address");
- }
- return ret;
-}
-
-static inline int try_read_address(struct i2c_adapter *i2c_adap,
- unsigned char addr, int retries)
-{
- struct usb_usbvision *usbvision;
- int i, ret = -1;
- char buf[4];
-
- usbvision = (struct usb_usbvision *)i2c_get_adapdata(i2c_adap);
- for (i = 0; i <= retries; i++) {
- ret = (usbvision_i2c_read(usbvision, addr, buf, 1));
- if (ret == 1)
- break; /* success! */
- udelay(5);
- if (i == retries) /* no success */
- break;
- udelay(10);
- }
- if (i) {
- PDEBUG(DBG_I2C, "Needed %d retries for address %#2x", i, addr);
- PDEBUG(DBG_I2C, "Maybe there's no device at this address");
- }
- return ret;
-}
-
-static inline int usb_find_address(struct i2c_adapter *i2c_adap,
- struct i2c_msg *msg, int retries,
- unsigned char *add)
-{
- unsigned short flags = msg->flags;
-
- unsigned char addr;
- int ret;
-
- addr = (msg->addr << 1);
- if (flags & I2C_M_RD)
- addr |= 1;
-
- add[0] = addr;
- if (flags & I2C_M_RD)
- ret = try_read_address(i2c_adap, addr, retries);
- else
- ret = try_write_address(i2c_adap, addr, retries);
-
- if (ret != 1)
- return -EREMOTEIO;
-
- return 0;
-}
-
-static int
-usbvision_i2c_xfer(struct i2c_adapter *i2c_adap, struct i2c_msg msgs[], int num)
-{
- struct i2c_msg *pmsg;
- struct usb_usbvision *usbvision;
- int i, ret;
- unsigned char addr = 0;
-
- usbvision = (struct usb_usbvision *)i2c_get_adapdata(i2c_adap);
-
- for (i = 0; i < num; i++) {
- pmsg = &msgs[i];
- ret = usb_find_address(i2c_adap, pmsg, i2c_adap->retries, &addr);
- if (ret != 0) {
- PDEBUG(DBG_I2C, "got NAK from device, message #%d", i);
- return (ret < 0) ? ret : -EREMOTEIO;
- }
-
- if (pmsg->flags & I2C_M_RD) {
- /* read bytes into buffer */
- ret = (usbvision_i2c_read(usbvision, addr, pmsg->buf, pmsg->len));
- if (ret < pmsg->len)
- return (ret < 0) ? ret : -EREMOTEIO;
- } else {
- /* write bytes from buffer */
- ret = (usbvision_i2c_write(usbvision, addr, pmsg->buf, pmsg->len));
- if (ret < pmsg->len)
- return (ret < 0) ? ret : -EREMOTEIO;
- }
- }
- return num;
-}
-
-static u32 functionality(struct i2c_adapter *adap)
-{
- return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
-}
-
-/* -----exported algorithm data: ------------------------------------- */
-
-static const struct i2c_algorithm usbvision_algo = {
- .master_xfer = usbvision_i2c_xfer,
- .smbus_xfer = NULL,
- .functionality = functionality,
-};
-
-
-/* ----------------------------------------------------------------------- */
-/* usbvision specific I2C functions */
-/* ----------------------------------------------------------------------- */
-static const struct i2c_adapter i2c_adap_template;
-
-int usbvision_i2c_register(struct usb_usbvision *usbvision)
-{
- static unsigned short saa711x_addrs[] = {
- 0x4a >> 1, 0x48 >> 1, /* SAA7111, SAA7111A and SAA7113 */
- 0x42 >> 1, 0x40 >> 1, /* SAA7114, SAA7115 and SAA7118 */
- I2C_CLIENT_END };
-
- if (usbvision->registered_i2c)
- return 0;
-
- usbvision->i2c_adap = i2c_adap_template;
-
- snprintf(usbvision->i2c_adap.name, sizeof(usbvision->i2c_adap.name),
- "usbvision-%d-%s",
- usbvision->dev->bus->busnum, usbvision->dev->devpath);
- PDEBUG(DBG_I2C, "Adaptername: %s", usbvision->i2c_adap.name);
- usbvision->i2c_adap.dev.parent = &usbvision->dev->dev;
-
- i2c_set_adapdata(&usbvision->i2c_adap, &usbvision->v4l2_dev);
-
- if (usbvision_write_reg(usbvision, USBVISION_SER_MODE, USBVISION_IIC_LRNACK) < 0) {
- printk(KERN_ERR "usbvision_i2c_register: can't write reg\n");
- return -EBUSY;
- }
-
- PDEBUG(DBG_I2C, "I2C debugging is enabled [i2c]");
- PDEBUG(DBG_I2C, "ALGO debugging is enabled [i2c]");
-
- /* register new adapter to i2c module... */
-
- usbvision->i2c_adap.algo = &usbvision_algo;
-
- usbvision->i2c_adap.timeout = 100; /* default values, should */
- usbvision->i2c_adap.retries = 3; /* be replaced by defines */
-
- i2c_add_adapter(&usbvision->i2c_adap);
-
- PDEBUG(DBG_I2C, "i2c bus for %s registered", usbvision->i2c_adap.name);
-
- /* Request the load of the i2c modules we need */
- switch (usbvision_device_data[usbvision->dev_model].codec) {
- case CODEC_SAA7113:
- case CODEC_SAA7111:
- /* Without this delay the detection of the saa711x is
- hit-and-miss. */
- mdelay(10);
- v4l2_i2c_new_subdev(&usbvision->v4l2_dev,
- &usbvision->i2c_adap,
- "saa7115_auto", 0, saa711x_addrs);
- break;
- }
- if (usbvision_device_data[usbvision->dev_model].tuner == 1) {
- struct v4l2_subdev *sd;
- enum v4l2_i2c_tuner_type type;
- struct tuner_setup tun_setup;
-
- sd = v4l2_i2c_new_subdev(&usbvision->v4l2_dev,
- &usbvision->i2c_adap,
- "tuner", 0, v4l2_i2c_tuner_addrs(ADDRS_DEMOD));
- /* depending on whether we found a demod or not, select
- the tuner type. */
- type = sd ? ADDRS_TV_WITH_DEMOD : ADDRS_TV;
-
- sd = v4l2_i2c_new_subdev(&usbvision->v4l2_dev,
- &usbvision->i2c_adap,
- "tuner", 0, v4l2_i2c_tuner_addrs(type));
-
- if (sd == NULL)
- return -ENODEV;
- if (usbvision->tuner_type != -1) {
- tun_setup.mode_mask = T_ANALOG_TV | T_RADIO;
- tun_setup.type = usbvision->tuner_type;
- tun_setup.addr = v4l2_i2c_subdev_addr(sd);
- call_all(usbvision, tuner, s_type_addr, &tun_setup);
- }
- }
- usbvision->registered_i2c = 1;
-
- return 0;
-}
-
-int usbvision_i2c_unregister(struct usb_usbvision *usbvision)
-{
- if (!usbvision->registered_i2c)
- return 0;
-
- i2c_del_adapter(&(usbvision->i2c_adap));
- usbvision->registered_i2c = 0;
-
- PDEBUG(DBG_I2C, "i2c bus for %s unregistered", usbvision->i2c_adap.name);
-
- return 0;
-}
-
-static int
-usbvision_i2c_read_max4(struct usb_usbvision *usbvision, unsigned char addr,
- char *buf, short len)
-{
- int rc, retries;
-
- for (retries = 5;;) {
- rc = usbvision_write_reg(usbvision, USBVISION_SER_ADRS, addr);
- if (rc < 0)
- return rc;
-
- /* Initiate byte read cycle */
- /* USBVISION_SER_CONT <- d0-d2 n. of bytes to r/w */
- /* d3 0=Wr 1=Rd */
- rc = usbvision_write_reg(usbvision, USBVISION_SER_CONT,
- (len & 0x07) | 0x18);
- if (rc < 0)
- return rc;
-
- /* Test for Busy and ACK */
- do {
- /* USBVISION_SER_CONT -> d4 == 0 busy */
- rc = usbvision_read_reg(usbvision, USBVISION_SER_CONT);
- } while (rc > 0 && ((rc & 0x10) != 0)); /* Retry while busy */
- if (rc < 0)
- return rc;
-
- /* USBVISION_SER_CONT -> d5 == 1 Not ack */
- if ((rc & 0x20) == 0) /* Ack? */
- break;
-
- /* I2C abort */
- rc = usbvision_write_reg(usbvision, USBVISION_SER_CONT, 0x00);
- if (rc < 0)
- return rc;
-
- if (--retries < 0)
- return -1;
- }
-
- switch (len) {
- case 4:
- buf[3] = usbvision_read_reg(usbvision, USBVISION_SER_DAT4);
- fallthrough;
- case 3:
- buf[2] = usbvision_read_reg(usbvision, USBVISION_SER_DAT3);
- fallthrough;
- case 2:
- buf[1] = usbvision_read_reg(usbvision, USBVISION_SER_DAT2);
- fallthrough;
- case 1:
- buf[0] = usbvision_read_reg(usbvision, USBVISION_SER_DAT1);
- break;
- default:
- printk(KERN_ERR
- "usbvision_i2c_read_max4: buffer length > 4\n");
- }
-
- if (i2c_debug & DBG_I2C) {
- int idx;
-
- for (idx = 0; idx < len; idx++)
- PDEBUG(DBG_I2C, "read %x from address %x", (unsigned char)buf[idx], addr);
- }
- return len;
-}
-
-
-static int usbvision_i2c_write_max4(struct usb_usbvision *usbvision,
- unsigned char addr, const char *buf,
- short len)
-{
- int rc, retries;
- int i;
- unsigned char *value = usbvision->ctrl_urb_buffer;
- unsigned char ser_cont;
-
- ser_cont = (len & 0x07) | 0x10;
-
- value[0] = addr;
- value[1] = ser_cont;
- for (i = 0; i < len; i++)
- value[i + 2] = buf[i];
-
- for (retries = 5;;) {
- rc = usb_control_msg(usbvision->dev,
- usb_sndctrlpipe(usbvision->dev, 1),
- USBVISION_OP_CODE,
- USB_DIR_OUT | USB_TYPE_VENDOR |
- USB_RECIP_ENDPOINT, 0,
- (__u16) USBVISION_SER_ADRS, value,
- len + 2, HZ);
-
- if (rc < 0)
- return rc;
-
- rc = usbvision_write_reg(usbvision, USBVISION_SER_CONT,
- (len & 0x07) | 0x10);
- if (rc < 0)
- return rc;
-
- /* Test for Busy and ACK */
- do {
- rc = usbvision_read_reg(usbvision, USBVISION_SER_CONT);
- } while (rc > 0 && ((rc & 0x10) != 0)); /* Retry while busy */
- if (rc < 0)
- return rc;
-
- if ((rc & 0x20) == 0) /* Ack? */
- break;
-
- /* I2C abort */
- usbvision_write_reg(usbvision, USBVISION_SER_CONT, 0x00);
-
- if (--retries < 0)
- return -1;
-
- }
-
- if (i2c_debug & DBG_I2C) {
- int idx;
-
- for (idx = 0; idx < len; idx++)
- PDEBUG(DBG_I2C, "wrote %x at address %x", (unsigned char)buf[idx], addr);
- }
- return len;
-}
-
-static int usbvision_i2c_write(struct usb_usbvision *usbvision, unsigned char addr, char *buf,
- short len)
-{
- char *buf_ptr = buf;
- int retval;
- int wrcount = 0;
- int count;
- int max_len = 4;
-
- while (len > 0) {
- count = (len > max_len) ? max_len : len;
- retval = usbvision_i2c_write_max4(usbvision, addr, buf_ptr, count);
- if (retval > 0) {
- len -= count;
- buf_ptr += count;
- wrcount += count;
- } else
- return (retval < 0) ? retval : -EFAULT;
- }
- return wrcount;
-}
-
-static int usbvision_i2c_read(struct usb_usbvision *usbvision, unsigned char addr, char *buf,
- short len)
-{
- char temp[4];
- int retval, i;
- int rdcount = 0;
- int count;
-
- while (len > 0) {
- count = (len > 3) ? 4 : len;
- retval = usbvision_i2c_read_max4(usbvision, addr, temp, count);
- if (retval > 0) {
- for (i = 0; i < len; i++)
- buf[rdcount + i] = temp[i];
- len -= count;
- rdcount += count;
- } else
- return (retval < 0) ? retval : -EFAULT;
- }
- return rdcount;
-}
-
-static const struct i2c_adapter i2c_adap_template = {
- .owner = THIS_MODULE,
- .name = "usbvision",
-};
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * USB USBVISION Video device driver 0.9.10
- *
- * Copyright (c) 1999-2005 Joerg Heckenbach <joerg@heckenbach-aw.de>
- *
- * This module is part of usbvision driver project.
- *
- * Let's call the version 0.... until compression decoding is completely
- * implemented.
- *
- * This driver is written by Jose Ignacio Gijon and Joerg Heckenbach.
- * It was based on USB CPiA driver written by Peter Pregler,
- * Scott J. Bertin and Johannes Erdfelt
- * Ideas are taken from bttv driver by Ralph Metzler, Marcus Metzler &
- * Gerd Knorr and zoran 36120/36125 driver by Pauline Middelink
- * Updates to driver completed by Dwaine P. Garden
- *
- * TODO:
- * - use submit_urb for all setup packets
- * - Fix memory settings for nt1004. It is 4 times as big as the
- * nt1003 memory.
- * - Add audio on endpoint 3 for nt1004 chip.
- * Seems impossible, needs a codec interface. Which one?
- * - Clean up the driver.
- * - optimization for performance.
- * - Add Videotext capability (VBI). Working on it.....
- * - Check audio for other devices
- */
-
-#include <linux/kernel.h>
-#include <linux/list.h>
-#include <linux/timer.h>
-#include <linux/slab.h>
-#include <linux/mm.h>
-#include <linux/highmem.h>
-#include <linux/vmalloc.h>
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/spinlock.h>
-#include <linux/io.h>
-#include <linux/videodev2.h>
-#include <linux/i2c.h>
-
-#include <media/i2c/saa7115.h>
-#include <media/v4l2-common.h>
-#include <media/v4l2-ioctl.h>
-#include <media/v4l2-event.h>
-#include <media/tuner.h>
-
-#include <linux/workqueue.h>
-
-#include "usbvision.h"
-#include "usbvision-cards.h"
-
-#define DRIVER_AUTHOR \
- "Joerg Heckenbach <joerg@heckenbach-aw.de>, " \
- "Dwaine Garden <DwaineGarden@rogers.com>"
-#define DRIVER_NAME "usbvision"
-#define DRIVER_ALIAS "USBVision"
-#define DRIVER_DESC "USBVision USB Video Device Driver for Linux"
-#define USBVISION_VERSION_STRING "0.9.11"
-
-#define ENABLE_HEXDUMP 0 /* Enable if you need it */
-
-
-#ifdef USBVISION_DEBUG
- #define PDEBUG(level, fmt, args...) { \
- if (video_debug & (level)) \
- printk(KERN_INFO KBUILD_MODNAME ":[%s:%d] " fmt, \
- __func__, __LINE__ , ## args); \
- }
-#else
- #define PDEBUG(level, fmt, args...) do {} while (0)
-#endif
-
-#define DBG_IO (1 << 1)
-#define DBG_PROBE (1 << 2)
-#define DBG_MMAP (1 << 3)
-
-/* String operations */
-#define rmspace(str) while (*str == ' ') str++;
-#define goto2next(str) while (*str != ' ') str++; while (*str == ' ') str++;
-
-
-/* sequential number of usbvision device */
-static int usbvision_nr;
-
-static struct usbvision_v4l2_format_st usbvision_v4l2_format[] = {
- { 1, 1, 8, V4L2_PIX_FMT_GREY },
- { 1, 2, 16, V4L2_PIX_FMT_RGB565 },
- { 1, 3, 24, V4L2_PIX_FMT_RGB24 },
- { 1, 4, 32, V4L2_PIX_FMT_RGB32 },
- { 1, 2, 16, V4L2_PIX_FMT_RGB555 },
- { 1, 2, 16, V4L2_PIX_FMT_YUYV },
- { 1, 2, 12, V4L2_PIX_FMT_YVU420 }, /* 1.5 ! */
- { 1, 2, 16, V4L2_PIX_FMT_YUV422P }
-};
-
-/* Function prototypes */
-static void usbvision_release(struct usb_usbvision *usbvision);
-
-/* Default initialization of device driver parameters */
-/* Set the default format for ISOC endpoint */
-static int isoc_mode = ISOC_MODE_COMPRESS;
-/* Set the default Debug Mode of the device driver */
-static int video_debug;
-/* Sequential Number of Video Device */
-static int video_nr = -1;
-/* Sequential Number of Radio Device */
-static int radio_nr = -1;
-
-/* Grab parameters for the device driver */
-
-/* Showing parameters under SYSFS */
-module_param(isoc_mode, int, 0444);
-module_param(video_debug, int, 0444);
-module_param(video_nr, int, 0444);
-module_param(radio_nr, int, 0444);
-
-MODULE_PARM_DESC(isoc_mode, " Set the default format for ISOC endpoint. Default: 0x60 (Compression On)");
-MODULE_PARM_DESC(video_debug, " Set the default Debug Mode of the device driver. Default: 0 (Off)");
-MODULE_PARM_DESC(video_nr, "Set video device number (/dev/videoX). Default: -1 (autodetect)");
-MODULE_PARM_DESC(radio_nr, "Set radio device number (/dev/radioX). Default: -1 (autodetect)");
-
-
-/* Misc stuff */
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL");
-MODULE_VERSION(USBVISION_VERSION_STRING);
-MODULE_ALIAS(DRIVER_ALIAS);
-
-
-/*****************************************************************************/
-/* SYSFS Code - Copied from the stv680.c usb module. */
-/* Device information is located at /sys/class/video4linux/video0 */
-/* Device parameters information is located at /sys/module/usbvision */
-/* Device USB Information is located at */
-/* /sys/bus/usb/drivers/USBVision Video Grabber */
-/*****************************************************************************/
-
-#define YES_NO(x) ((x) ? "Yes" : "No")
-
-static inline struct usb_usbvision *cd_to_usbvision(struct device *cd)
-{
- struct video_device *vdev = to_video_device(cd);
- return video_get_drvdata(vdev);
-}
-
-static ssize_t show_version(struct device *cd,
- struct device_attribute *attr, char *buf)
-{
- return sprintf(buf, "%s\n", USBVISION_VERSION_STRING);
-}
-static DEVICE_ATTR(version, S_IRUGO, show_version, NULL);
-
-static ssize_t show_model(struct device *cd,
- struct device_attribute *attr, char *buf)
-{
- struct video_device *vdev = to_video_device(cd);
- struct usb_usbvision *usbvision = video_get_drvdata(vdev);
- return sprintf(buf, "%s\n",
- usbvision_device_data[usbvision->dev_model].model_string);
-}
-static DEVICE_ATTR(model, S_IRUGO, show_model, NULL);
-
-static ssize_t show_hue(struct device *cd,
- struct device_attribute *attr, char *buf)
-{
- struct video_device *vdev = to_video_device(cd);
- struct usb_usbvision *usbvision = video_get_drvdata(vdev);
- s32 val = v4l2_ctrl_g_ctrl(v4l2_ctrl_find(&usbvision->hdl,
- V4L2_CID_HUE));
-
- return sprintf(buf, "%d\n", val);
-}
-static DEVICE_ATTR(hue, S_IRUGO, show_hue, NULL);
-
-static ssize_t show_contrast(struct device *cd,
- struct device_attribute *attr, char *buf)
-{
- struct video_device *vdev = to_video_device(cd);
- struct usb_usbvision *usbvision = video_get_drvdata(vdev);
- s32 val = v4l2_ctrl_g_ctrl(v4l2_ctrl_find(&usbvision->hdl,
- V4L2_CID_CONTRAST));
-
- return sprintf(buf, "%d\n", val);
-}
-static DEVICE_ATTR(contrast, S_IRUGO, show_contrast, NULL);
-
-static ssize_t show_brightness(struct device *cd,
- struct device_attribute *attr, char *buf)
-{
- struct video_device *vdev = to_video_device(cd);
- struct usb_usbvision *usbvision = video_get_drvdata(vdev);
- s32 val = v4l2_ctrl_g_ctrl(v4l2_ctrl_find(&usbvision->hdl,
- V4L2_CID_BRIGHTNESS));
-
- return sprintf(buf, "%d\n", val);
-}
-static DEVICE_ATTR(brightness, S_IRUGO, show_brightness, NULL);
-
-static ssize_t show_saturation(struct device *cd,
- struct device_attribute *attr, char *buf)
-{
- struct video_device *vdev = to_video_device(cd);
- struct usb_usbvision *usbvision = video_get_drvdata(vdev);
- s32 val = v4l2_ctrl_g_ctrl(v4l2_ctrl_find(&usbvision->hdl,
- V4L2_CID_SATURATION));
-
- return sprintf(buf, "%d\n", val);
-}
-static DEVICE_ATTR(saturation, S_IRUGO, show_saturation, NULL);
-
-static ssize_t show_streaming(struct device *cd,
- struct device_attribute *attr, char *buf)
-{
- struct video_device *vdev = to_video_device(cd);
- struct usb_usbvision *usbvision = video_get_drvdata(vdev);
- return sprintf(buf, "%s\n",
- YES_NO(usbvision->streaming == stream_on ? 1 : 0));
-}
-static DEVICE_ATTR(streaming, S_IRUGO, show_streaming, NULL);
-
-static ssize_t show_compression(struct device *cd,
- struct device_attribute *attr, char *buf)
-{
- struct video_device *vdev = to_video_device(cd);
- struct usb_usbvision *usbvision = video_get_drvdata(vdev);
- return sprintf(buf, "%s\n",
- YES_NO(usbvision->isoc_mode == ISOC_MODE_COMPRESS));
-}
-static DEVICE_ATTR(compression, S_IRUGO, show_compression, NULL);
-
-static ssize_t show_device_bridge(struct device *cd,
- struct device_attribute *attr, char *buf)
-{
- struct video_device *vdev = to_video_device(cd);
- struct usb_usbvision *usbvision = video_get_drvdata(vdev);
- return sprintf(buf, "%d\n", usbvision->bridge_type);
-}
-static DEVICE_ATTR(bridge, S_IRUGO, show_device_bridge, NULL);
-
-static void usbvision_create_sysfs(struct video_device *vdev)
-{
- int res;
-
- if (!vdev)
- return;
- do {
- res = device_create_file(&vdev->dev, &dev_attr_version);
- if (res < 0)
- break;
- res = device_create_file(&vdev->dev, &dev_attr_model);
- if (res < 0)
- break;
- res = device_create_file(&vdev->dev, &dev_attr_hue);
- if (res < 0)
- break;
- res = device_create_file(&vdev->dev, &dev_attr_contrast);
- if (res < 0)
- break;
- res = device_create_file(&vdev->dev, &dev_attr_brightness);
- if (res < 0)
- break;
- res = device_create_file(&vdev->dev, &dev_attr_saturation);
- if (res < 0)
- break;
- res = device_create_file(&vdev->dev, &dev_attr_streaming);
- if (res < 0)
- break;
- res = device_create_file(&vdev->dev, &dev_attr_compression);
- if (res < 0)
- break;
- res = device_create_file(&vdev->dev, &dev_attr_bridge);
- if (res >= 0)
- return;
- } while (0);
-
- dev_err(&vdev->dev, "%s error: %d\n", __func__, res);
-}
-
-static void usbvision_remove_sysfs(struct video_device *vdev)
-{
- if (vdev) {
- device_remove_file(&vdev->dev, &dev_attr_version);
- device_remove_file(&vdev->dev, &dev_attr_model);
- device_remove_file(&vdev->dev, &dev_attr_hue);
- device_remove_file(&vdev->dev, &dev_attr_contrast);
- device_remove_file(&vdev->dev, &dev_attr_brightness);
- device_remove_file(&vdev->dev, &dev_attr_saturation);
- device_remove_file(&vdev->dev, &dev_attr_streaming);
- device_remove_file(&vdev->dev, &dev_attr_compression);
- device_remove_file(&vdev->dev, &dev_attr_bridge);
- }
-}
-
-/*
- * usbvision_open()
- *
- * This is part of Video 4 Linux API. The driver can be opened by one
- * client only (checks internal counter 'usbvision->user'). The procedure
- * then allocates buffers needed for video processing.
- *
- */
-static int usbvision_v4l2_open(struct file *file)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int err_code = 0;
-
- PDEBUG(DBG_IO, "open");
-
- if (mutex_lock_interruptible(&usbvision->v4l2_lock))
- return -ERESTARTSYS;
-
- if (usbvision->remove_pending) {
- err_code = -ENODEV;
- goto unlock;
- }
- if (usbvision->user) {
- err_code = -EBUSY;
- } else {
- err_code = v4l2_fh_open(file);
- if (err_code)
- goto unlock;
-
- /* Allocate memory for the scratch ring buffer */
- err_code = usbvision_scratch_alloc(usbvision);
- if (isoc_mode == ISOC_MODE_COMPRESS) {
- /* Allocate intermediate decompression buffers
- only if needed */
- err_code = usbvision_decompress_alloc(usbvision);
- }
- if (err_code) {
- /* Deallocate all buffers if trouble */
- usbvision_scratch_free(usbvision);
- usbvision_decompress_free(usbvision);
- }
- }
-
- /* If so far no errors then we shall start the camera */
- if (!err_code) {
- /* Send init sequence only once, it's large! */
- if (!usbvision->initialized) {
- int setup_ok = 0;
- setup_ok = usbvision_setup(usbvision, isoc_mode);
- if (setup_ok)
- usbvision->initialized = 1;
- else
- err_code = -EBUSY;
- }
-
- if (!err_code) {
- usbvision_begin_streaming(usbvision);
- err_code = usbvision_init_isoc(usbvision);
- /* device must be initialized before isoc transfer */
- usbvision_muxsel(usbvision, 0);
-
- /* prepare queues */
- usbvision_empty_framequeues(usbvision);
- usbvision->user++;
- }
- }
-
-unlock:
- mutex_unlock(&usbvision->v4l2_lock);
-
- PDEBUG(DBG_IO, "success");
- return err_code;
-}
-
-/*
- * usbvision_v4l2_close()
- *
- * This is part of Video 4 Linux API. The procedure
- * stops streaming and deallocates all buffers that were earlier
- * allocated in usbvision_v4l2_open().
- *
- */
-static int usbvision_v4l2_close(struct file *file)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int r;
-
- PDEBUG(DBG_IO, "close");
-
- mutex_lock(&usbvision->v4l2_lock);
- usbvision_audio_off(usbvision);
- usbvision_restart_isoc(usbvision);
- usbvision_stop_isoc(usbvision);
-
- usbvision_decompress_free(usbvision);
- usbvision_frames_free(usbvision);
- usbvision_empty_framequeues(usbvision);
- usbvision_scratch_free(usbvision);
-
- usbvision->user--;
- r = usbvision->remove_pending;
- mutex_unlock(&usbvision->v4l2_lock);
-
- if (r) {
- printk(KERN_INFO "%s: Final disconnect\n", __func__);
- usbvision_release(usbvision);
- return 0;
- }
-
- PDEBUG(DBG_IO, "success");
- return v4l2_fh_release(file);
-}
-
-
-/*
- * usbvision_ioctl()
- *
- * This is part of Video 4 Linux API. The procedure handles ioctl() calls.
- *
- */
-#ifdef CONFIG_VIDEO_ADV_DEBUG
-static int vidioc_g_register(struct file *file, void *priv,
- struct v4l2_dbg_register *reg)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int err_code;
-
- /* NT100x has a 8-bit register space */
- err_code = usbvision_read_reg(usbvision, reg->reg&0xff);
- if (err_code < 0) {
- dev_err(&usbvision->vdev.dev,
- "%s: VIDIOC_DBG_G_REGISTER failed: error %d\n",
- __func__, err_code);
- return err_code;
- }
- reg->val = err_code;
- reg->size = 1;
- return 0;
-}
-
-static int vidioc_s_register(struct file *file, void *priv,
- const struct v4l2_dbg_register *reg)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int err_code;
-
- /* NT100x has a 8-bit register space */
- err_code = usbvision_write_reg(usbvision, reg->reg & 0xff, reg->val);
- if (err_code < 0) {
- dev_err(&usbvision->vdev.dev,
- "%s: VIDIOC_DBG_S_REGISTER failed: error %d\n",
- __func__, err_code);
- return err_code;
- }
- return 0;
-}
-#endif
-
-static int vidioc_querycap(struct file *file, void *priv,
- struct v4l2_capability *vc)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
-
- if (!usbvision->dev)
- return -ENODEV;
-
- strscpy(vc->driver, "USBVision", sizeof(vc->driver));
- strscpy(vc->card,
- usbvision_device_data[usbvision->dev_model].model_string,
- sizeof(vc->card));
- usb_make_path(usbvision->dev, vc->bus_info, sizeof(vc->bus_info));
- vc->capabilities = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_READWRITE |
- V4L2_CAP_STREAMING | V4L2_CAP_DEVICE_CAPS;
- if (usbvision_device_data[usbvision->dev_model].radio)
- vc->capabilities |= V4L2_CAP_RADIO;
- if (usbvision->have_tuner)
- vc->capabilities |= V4L2_CAP_TUNER;
- return 0;
-}
-
-static int vidioc_enum_input(struct file *file, void *priv,
- struct v4l2_input *vi)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int chan;
-
- if (vi->index >= usbvision->video_inputs)
- return -EINVAL;
- if (usbvision->have_tuner)
- chan = vi->index;
- else
- chan = vi->index + 1; /* skip Television string*/
-
- /* Determine the requested input characteristics
- specific for each usbvision card model */
- switch (chan) {
- case 0:
- if (usbvision_device_data[usbvision->dev_model].video_channels == 4) {
- strscpy(vi->name, "White Video Input", sizeof(vi->name));
- } else {
- strscpy(vi->name, "Television", sizeof(vi->name));
- vi->type = V4L2_INPUT_TYPE_TUNER;
- vi->tuner = chan;
- vi->std = USBVISION_NORMS;
- }
- break;
- case 1:
- vi->type = V4L2_INPUT_TYPE_CAMERA;
- if (usbvision_device_data[usbvision->dev_model].video_channels == 4)
- strscpy(vi->name, "Green Video Input", sizeof(vi->name));
- else
- strscpy(vi->name, "Composite Video Input",
- sizeof(vi->name));
- vi->std = USBVISION_NORMS;
- break;
- case 2:
- vi->type = V4L2_INPUT_TYPE_CAMERA;
- if (usbvision_device_data[usbvision->dev_model].video_channels == 4)
- strscpy(vi->name, "Yellow Video Input", sizeof(vi->name));
- else
- strscpy(vi->name, "S-Video Input", sizeof(vi->name));
- vi->std = USBVISION_NORMS;
- break;
- case 3:
- vi->type = V4L2_INPUT_TYPE_CAMERA;
- strscpy(vi->name, "Red Video Input", sizeof(vi->name));
- vi->std = USBVISION_NORMS;
- break;
- }
- return 0;
-}
-
-static int vidioc_g_input(struct file *file, void *priv, unsigned int *input)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
-
- *input = usbvision->ctl_input;
- return 0;
-}
-
-static int vidioc_s_input(struct file *file, void *priv, unsigned int input)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
-
- if (input >= usbvision->video_inputs)
- return -EINVAL;
-
- usbvision_muxsel(usbvision, input);
- usbvision_set_input(usbvision);
- usbvision_set_output(usbvision,
- usbvision->curwidth,
- usbvision->curheight);
- return 0;
-}
-
-static int vidioc_s_std(struct file *file, void *priv, v4l2_std_id id)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
-
- usbvision->tvnorm_id = id;
-
- call_all(usbvision, video, s_std, usbvision->tvnorm_id);
- /* propagate the change to the decoder */
- usbvision_muxsel(usbvision, usbvision->ctl_input);
-
- return 0;
-}
-
-static int vidioc_g_std(struct file *file, void *priv, v4l2_std_id *id)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
-
- *id = usbvision->tvnorm_id;
- return 0;
-}
-
-static int vidioc_g_tuner(struct file *file, void *priv,
- struct v4l2_tuner *vt)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
-
- if (vt->index) /* Only tuner 0 */
- return -EINVAL;
- if (vt->type == V4L2_TUNER_RADIO)
- strscpy(vt->name, "Radio", sizeof(vt->name));
- else
- strscpy(vt->name, "Television", sizeof(vt->name));
-
- /* Let clients fill in the remainder of this struct */
- call_all(usbvision, tuner, g_tuner, vt);
-
- return 0;
-}
-
-static int vidioc_s_tuner(struct file *file, void *priv,
- const struct v4l2_tuner *vt)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
-
- /* Only one tuner for now */
- if (vt->index)
- return -EINVAL;
- /* let clients handle this */
- call_all(usbvision, tuner, s_tuner, vt);
-
- return 0;
-}
-
-static int vidioc_g_frequency(struct file *file, void *priv,
- struct v4l2_frequency *freq)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
-
- /* Only one tuner */
- if (freq->tuner)
- return -EINVAL;
- if (freq->type == V4L2_TUNER_RADIO)
- freq->frequency = usbvision->radio_freq;
- else
- freq->frequency = usbvision->tv_freq;
-
- return 0;
-}
-
-static int vidioc_s_frequency(struct file *file, void *priv,
- const struct v4l2_frequency *freq)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- struct v4l2_frequency new_freq = *freq;
-
- /* Only one tuner for now */
- if (freq->tuner)
- return -EINVAL;
-
- call_all(usbvision, tuner, s_frequency, freq);
- call_all(usbvision, tuner, g_frequency, &new_freq);
- if (freq->type == V4L2_TUNER_RADIO)
- usbvision->radio_freq = new_freq.frequency;
- else
- usbvision->tv_freq = new_freq.frequency;
-
- return 0;
-}
-
-static int vidioc_reqbufs(struct file *file,
- void *priv, struct v4l2_requestbuffers *vr)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int ret;
-
- RESTRICT_TO_RANGE(vr->count, 1, USBVISION_NUMFRAMES);
-
- /* Check input validity:
- the user must do a VIDEO CAPTURE and MMAP method. */
- if (vr->memory != V4L2_MEMORY_MMAP)
- return -EINVAL;
-
- if (usbvision->streaming == stream_on) {
- ret = usbvision_stream_interrupt(usbvision);
- if (ret)
- return ret;
- }
-
- usbvision_frames_free(usbvision);
- usbvision_empty_framequeues(usbvision);
- vr->count = usbvision_frames_alloc(usbvision, vr->count);
-
- usbvision->cur_frame = NULL;
-
- return 0;
-}
-
-static int vidioc_querybuf(struct file *file,
- void *priv, struct v4l2_buffer *vb)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- struct usbvision_frame *frame;
-
- /* FIXME : must control
- that buffers are mapped (VIDIOC_REQBUFS has been called) */
- if (vb->index >= usbvision->num_frames)
- return -EINVAL;
- /* Updating the corresponding frame state */
- vb->flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
- frame = &usbvision->frame[vb->index];
- if (frame->grabstate >= frame_state_ready)
- vb->flags |= V4L2_BUF_FLAG_QUEUED;
- if (frame->grabstate >= frame_state_done)
- vb->flags |= V4L2_BUF_FLAG_DONE;
- if (frame->grabstate == frame_state_unused)
- vb->flags |= V4L2_BUF_FLAG_MAPPED;
- vb->memory = V4L2_MEMORY_MMAP;
-
- vb->m.offset = vb->index * PAGE_ALIGN(usbvision->max_frame_size);
-
- vb->memory = V4L2_MEMORY_MMAP;
- vb->field = V4L2_FIELD_NONE;
- vb->length = usbvision->curwidth *
- usbvision->curheight *
- usbvision->palette.bytes_per_pixel;
- v4l2_buffer_set_timestamp(vb, usbvision->frame[vb->index].ts);
- vb->sequence = usbvision->frame[vb->index].sequence;
- return 0;
-}
-
-static int vidioc_qbuf(struct file *file, void *priv, struct v4l2_buffer *vb)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- struct usbvision_frame *frame;
- unsigned long lock_flags;
-
- /* FIXME : works only on VIDEO_CAPTURE MODE, MMAP. */
- if (vb->index >= usbvision->num_frames)
- return -EINVAL;
-
- frame = &usbvision->frame[vb->index];
-
- if (frame->grabstate != frame_state_unused)
- return -EAGAIN;
-
- /* Mark it as ready and enqueue frame */
- frame->grabstate = frame_state_ready;
- frame->scanstate = scan_state_scanning;
- frame->scanlength = 0; /* Accumulated in usbvision_parse_data() */
-
- vb->flags &= ~V4L2_BUF_FLAG_DONE;
-
- /* set v4l2_format index */
- frame->v4l2_format = usbvision->palette;
-
- spin_lock_irqsave(&usbvision->queue_lock, lock_flags);
- list_add_tail(&usbvision->frame[vb->index].frame, &usbvision->inqueue);
- spin_unlock_irqrestore(&usbvision->queue_lock, lock_flags);
-
- return 0;
-}
-
-static int vidioc_dqbuf(struct file *file, void *priv, struct v4l2_buffer *vb)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int ret;
- struct usbvision_frame *f;
- unsigned long lock_flags;
-
- if (list_empty(&(usbvision->outqueue))) {
- if (usbvision->streaming == stream_idle)
- return -EINVAL;
- ret = wait_event_interruptible
- (usbvision->wait_frame,
- !list_empty(&(usbvision->outqueue)));
- if (ret)
- return ret;
- }
-
- spin_lock_irqsave(&usbvision->queue_lock, lock_flags);
- f = list_entry(usbvision->outqueue.next,
- struct usbvision_frame, frame);
- list_del(usbvision->outqueue.next);
- spin_unlock_irqrestore(&usbvision->queue_lock, lock_flags);
-
- f->grabstate = frame_state_unused;
-
- vb->memory = V4L2_MEMORY_MMAP;
- vb->flags = V4L2_BUF_FLAG_MAPPED |
- V4L2_BUF_FLAG_QUEUED |
- V4L2_BUF_FLAG_DONE |
- V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
- vb->index = f->index;
- vb->sequence = f->sequence;
- v4l2_buffer_set_timestamp(vb, f->ts);
- vb->field = V4L2_FIELD_NONE;
- vb->bytesused = f->scanlength;
-
- return 0;
-}
-
-static int vidioc_streamon(struct file *file, void *priv, enum v4l2_buf_type i)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
-
- usbvision->streaming = stream_on;
- call_all(usbvision, video, s_stream, 1);
-
- return 0;
-}
-
-static int vidioc_streamoff(struct file *file,
- void *priv, enum v4l2_buf_type type)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
-
- if (type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
- return -EINVAL;
-
- if (usbvision->streaming == stream_on) {
- usbvision_stream_interrupt(usbvision);
- /* Stop all video streamings */
- call_all(usbvision, video, s_stream, 0);
- }
- usbvision_empty_framequeues(usbvision);
-
- return 0;
-}
-
-static int vidioc_enum_fmt_vid_cap(struct file *file, void *priv,
- struct v4l2_fmtdesc *vfd)
-{
- if (vfd->index >= USBVISION_SUPPORTED_PALETTES - 1)
- return -EINVAL;
- vfd->pixelformat = usbvision_v4l2_format[vfd->index].format;
- return 0;
-}
-
-static int vidioc_g_fmt_vid_cap(struct file *file, void *priv,
- struct v4l2_format *vf)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- vf->fmt.pix.width = usbvision->curwidth;
- vf->fmt.pix.height = usbvision->curheight;
- vf->fmt.pix.pixelformat = usbvision->palette.format;
- vf->fmt.pix.bytesperline =
- usbvision->curwidth * usbvision->palette.bytes_per_pixel;
- vf->fmt.pix.sizeimage = vf->fmt.pix.bytesperline * usbvision->curheight;
- vf->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
- vf->fmt.pix.field = V4L2_FIELD_NONE; /* Always progressive image */
-
- return 0;
-}
-
-static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
- struct v4l2_format *vf)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int format_idx;
-
- /* Find requested format in available ones */
- for (format_idx = 0; format_idx < USBVISION_SUPPORTED_PALETTES; format_idx++) {
- if (vf->fmt.pix.pixelformat ==
- usbvision_v4l2_format[format_idx].format) {
- usbvision->palette = usbvision_v4l2_format[format_idx];
- break;
- }
- }
- /* robustness */
- if (format_idx == USBVISION_SUPPORTED_PALETTES)
- return -EINVAL;
- RESTRICT_TO_RANGE(vf->fmt.pix.width, MIN_FRAME_WIDTH, MAX_FRAME_WIDTH);
- RESTRICT_TO_RANGE(vf->fmt.pix.height, MIN_FRAME_HEIGHT, MAX_FRAME_HEIGHT);
-
- vf->fmt.pix.bytesperline = vf->fmt.pix.width*
- usbvision->palette.bytes_per_pixel;
- vf->fmt.pix.sizeimage = vf->fmt.pix.bytesperline*vf->fmt.pix.height;
- vf->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
- vf->fmt.pix.field = V4L2_FIELD_NONE; /* Always progressive image */
-
- return 0;
-}
-
-static int vidioc_s_fmt_vid_cap(struct file *file, void *priv,
- struct v4l2_format *vf)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int ret;
-
- ret = vidioc_try_fmt_vid_cap(file, priv, vf);
- if (ret)
- return ret;
-
- /* stop io in case it is already in progress */
- if (usbvision->streaming == stream_on) {
- ret = usbvision_stream_interrupt(usbvision);
- if (ret)
- return ret;
- }
- usbvision_frames_free(usbvision);
- usbvision_empty_framequeues(usbvision);
-
- usbvision->cur_frame = NULL;
-
- /* by now we are committed to the new data... */
- usbvision_set_output(usbvision, vf->fmt.pix.width, vf->fmt.pix.height);
-
- return 0;
-}
-
-static ssize_t usbvision_read(struct file *file, char __user *buf,
- size_t count, loff_t *ppos)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int noblock = file->f_flags & O_NONBLOCK;
- unsigned long lock_flags;
- int ret, i;
- struct usbvision_frame *frame;
-
- PDEBUG(DBG_IO, "%s: %ld bytes, noblock=%d", __func__,
- (unsigned long)count, noblock);
-
- if (!USBVISION_IS_OPERATIONAL(usbvision) || !buf)
- return -EFAULT;
-
- /* This entry point is compatible with the mmap routines
- so that a user can do either VIDIOC_QBUF/VIDIOC_DQBUF
- to get frames or call read on the device. */
- if (!usbvision->num_frames) {
- /* First, allocate some frames to work with
- if this has not been done with VIDIOC_REQBUF */
- usbvision_frames_free(usbvision);
- usbvision_empty_framequeues(usbvision);
- usbvision_frames_alloc(usbvision, USBVISION_NUMFRAMES);
- }
-
- if (usbvision->streaming != stream_on) {
- /* no stream is running, make it running ! */
- usbvision->streaming = stream_on;
- call_all(usbvision, video, s_stream, 1);
- }
-
- /* Then, enqueue as many frames as possible
- (like a user of VIDIOC_QBUF would do) */
- for (i = 0; i < usbvision->num_frames; i++) {
- frame = &usbvision->frame[i];
- if (frame->grabstate == frame_state_unused) {
- /* Mark it as ready and enqueue frame */
- frame->grabstate = frame_state_ready;
- frame->scanstate = scan_state_scanning;
- /* Accumulated in usbvision_parse_data() */
- frame->scanlength = 0;
-
- /* set v4l2_format index */
- frame->v4l2_format = usbvision->palette;
-
- spin_lock_irqsave(&usbvision->queue_lock, lock_flags);
- list_add_tail(&frame->frame, &usbvision->inqueue);
- spin_unlock_irqrestore(&usbvision->queue_lock,
- lock_flags);
- }
- }
-
- /* Then try to steal a frame (like a VIDIOC_DQBUF would do) */
- if (list_empty(&(usbvision->outqueue))) {
- if (noblock)
- return -EAGAIN;
-
- ret = wait_event_interruptible
- (usbvision->wait_frame,
- !list_empty(&(usbvision->outqueue)));
- if (ret)
- return ret;
- }
-
- spin_lock_irqsave(&usbvision->queue_lock, lock_flags);
- frame = list_entry(usbvision->outqueue.next,
- struct usbvision_frame, frame);
- list_del(usbvision->outqueue.next);
- spin_unlock_irqrestore(&usbvision->queue_lock, lock_flags);
-
- /* An error returns an empty frame */
- if (frame->grabstate == frame_state_error) {
- frame->bytes_read = 0;
- return 0;
- }
-
- PDEBUG(DBG_IO, "%s: frmx=%d, bytes_read=%ld, scanlength=%ld",
- __func__,
- frame->index, frame->bytes_read, frame->scanlength);
-
- /* copy bytes to user space; we allow for partials reads */
- if ((count + frame->bytes_read) > (unsigned long)frame->scanlength)
- count = frame->scanlength - frame->bytes_read;
-
- if (copy_to_user(buf, frame->data + frame->bytes_read, count))
- return -EFAULT;
-
- frame->bytes_read += count;
- PDEBUG(DBG_IO, "%s: {copy} count used=%ld, new bytes_read=%ld",
- __func__,
- (unsigned long)count, frame->bytes_read);
-
- /*
- * FIXME:
- * For now, forget the frame if it has not been read in one shot.
- */
- frame->bytes_read = 0;
-
- /* Mark it as available to be used again. */
- frame->grabstate = frame_state_unused;
-
- return count;
-}
-
-static ssize_t usbvision_v4l2_read(struct file *file, char __user *buf,
- size_t count, loff_t *ppos)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int res;
-
- if (mutex_lock_interruptible(&usbvision->v4l2_lock))
- return -ERESTARTSYS;
- res = usbvision_read(file, buf, count, ppos);
- mutex_unlock(&usbvision->v4l2_lock);
- return res;
-}
-
-static int usbvision_mmap(struct file *file, struct vm_area_struct *vma)
-{
- unsigned long size = vma->vm_end - vma->vm_start,
- start = vma->vm_start;
- void *pos;
- u32 i;
- struct usb_usbvision *usbvision = video_drvdata(file);
-
- PDEBUG(DBG_MMAP, "mmap");
-
- if (!USBVISION_IS_OPERATIONAL(usbvision))
- return -EFAULT;
-
- if (!(vma->vm_flags & VM_WRITE) ||
- size != PAGE_ALIGN(usbvision->max_frame_size)) {
- return -EINVAL;
- }
-
- for (i = 0; i < usbvision->num_frames; i++) {
- if (((PAGE_ALIGN(usbvision->max_frame_size)*i) >> PAGE_SHIFT) ==
- vma->vm_pgoff)
- break;
- }
- if (i == usbvision->num_frames) {
- PDEBUG(DBG_MMAP,
- "mmap: user supplied mapping address is out of range");
- return -EINVAL;
- }
-
- /* VM_IO is eventually going to replace PageReserved altogether */
- vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP;
-
- pos = usbvision->frame[i].data;
- while (size > 0) {
- if (vm_insert_page(vma, start, vmalloc_to_page(pos))) {
- PDEBUG(DBG_MMAP, "mmap: vm_insert_page failed");
- return -EAGAIN;
- }
- start += PAGE_SIZE;
- pos += PAGE_SIZE;
- size -= PAGE_SIZE;
- }
-
- return 0;
-}
-
-static int usbvision_v4l2_mmap(struct file *file, struct vm_area_struct *vma)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int res;
-
- if (mutex_lock_interruptible(&usbvision->v4l2_lock))
- return -ERESTARTSYS;
- res = usbvision_mmap(file, vma);
- mutex_unlock(&usbvision->v4l2_lock);
- return res;
-}
-
-/*
- * Here comes the stuff for radio on usbvision based devices
- *
- */
-static int usbvision_radio_open(struct file *file)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int err_code = 0;
-
- PDEBUG(DBG_IO, "%s:", __func__);
-
- if (mutex_lock_interruptible(&usbvision->v4l2_lock))
- return -ERESTARTSYS;
-
- if (usbvision->remove_pending) {
- err_code = -ENODEV;
- goto out;
- }
- err_code = v4l2_fh_open(file);
- if (err_code)
- goto out;
- if (usbvision->user) {
- dev_err(&usbvision->rdev.dev,
- "%s: Someone tried to open an already opened USBVision Radio!\n",
- __func__);
- err_code = -EBUSY;
- } else {
- /* Alternate interface 1 is is the biggest frame size */
- err_code = usbvision_set_alternate(usbvision);
- if (err_code < 0) {
- usbvision->last_error = err_code;
- err_code = -EBUSY;
- goto out;
- }
-
- /* If so far no errors then we shall start the radio */
- usbvision->radio = 1;
- call_all(usbvision, tuner, s_radio);
- usbvision_set_audio(usbvision, USBVISION_AUDIO_RADIO);
- usbvision->user++;
- }
-out:
- mutex_unlock(&usbvision->v4l2_lock);
- return err_code;
-}
-
-
-static int usbvision_radio_close(struct file *file)
-{
- struct usb_usbvision *usbvision = video_drvdata(file);
- int r;
-
- PDEBUG(DBG_IO, "");
-
- mutex_lock(&usbvision->v4l2_lock);
- /* Set packet size to 0 */
- usbvision->iface_alt = 0;
- if (usbvision->dev)
- usb_set_interface(usbvision->dev, usbvision->iface,
- usbvision->iface_alt);
-
- usbvision_audio_off(usbvision);
- usbvision->radio = 0;
- usbvision->user--;
- r = usbvision->remove_pending;
- mutex_unlock(&usbvision->v4l2_lock);
-
- if (r) {
- printk(KERN_INFO "%s: Final disconnect\n", __func__);
- v4l2_fh_release(file);
- usbvision_release(usbvision);
- return 0;
- }
-
- PDEBUG(DBG_IO, "success");
- return v4l2_fh_release(file);
-}
-
-/* Video registration stuff */
-
-/* Video template */
-static const struct v4l2_file_operations usbvision_fops = {
- .owner = THIS_MODULE,
- .open = usbvision_v4l2_open,
- .release = usbvision_v4l2_close,
- .read = usbvision_v4l2_read,
- .mmap = usbvision_v4l2_mmap,
- .unlocked_ioctl = video_ioctl2,
-};
-
-static const struct v4l2_ioctl_ops usbvision_ioctl_ops = {
- .vidioc_querycap = vidioc_querycap,
- .vidioc_enum_fmt_vid_cap = vidioc_enum_fmt_vid_cap,
- .vidioc_g_fmt_vid_cap = vidioc_g_fmt_vid_cap,
- .vidioc_try_fmt_vid_cap = vidioc_try_fmt_vid_cap,
- .vidioc_s_fmt_vid_cap = vidioc_s_fmt_vid_cap,
- .vidioc_reqbufs = vidioc_reqbufs,
- .vidioc_querybuf = vidioc_querybuf,
- .vidioc_qbuf = vidioc_qbuf,
- .vidioc_dqbuf = vidioc_dqbuf,
- .vidioc_s_std = vidioc_s_std,
- .vidioc_g_std = vidioc_g_std,
- .vidioc_enum_input = vidioc_enum_input,
- .vidioc_g_input = vidioc_g_input,
- .vidioc_s_input = vidioc_s_input,
- .vidioc_streamon = vidioc_streamon,
- .vidioc_streamoff = vidioc_streamoff,
- .vidioc_g_tuner = vidioc_g_tuner,
- .vidioc_s_tuner = vidioc_s_tuner,
- .vidioc_g_frequency = vidioc_g_frequency,
- .vidioc_s_frequency = vidioc_s_frequency,
- .vidioc_log_status = v4l2_ctrl_log_status,
- .vidioc_subscribe_event = v4l2_ctrl_subscribe_event,
- .vidioc_unsubscribe_event = v4l2_event_unsubscribe,
-#ifdef CONFIG_VIDEO_ADV_DEBUG
- .vidioc_g_register = vidioc_g_register,
- .vidioc_s_register = vidioc_s_register,
-#endif
-};
-
-static struct video_device usbvision_video_template = {
- .fops = &usbvision_fops,
- .ioctl_ops = &usbvision_ioctl_ops,
- .name = "usbvision-video",
- .release = video_device_release_empty,
- .tvnorms = USBVISION_NORMS,
-};
-
-
-/* Radio template */
-static const struct v4l2_file_operations usbvision_radio_fops = {
- .owner = THIS_MODULE,
- .open = usbvision_radio_open,
- .release = usbvision_radio_close,
- .poll = v4l2_ctrl_poll,
- .unlocked_ioctl = video_ioctl2,
-};
-
-static const struct v4l2_ioctl_ops usbvision_radio_ioctl_ops = {
- .vidioc_querycap = vidioc_querycap,
- .vidioc_g_tuner = vidioc_g_tuner,
- .vidioc_s_tuner = vidioc_s_tuner,
- .vidioc_g_frequency = vidioc_g_frequency,
- .vidioc_s_frequency = vidioc_s_frequency,
- .vidioc_log_status = v4l2_ctrl_log_status,
- .vidioc_subscribe_event = v4l2_ctrl_subscribe_event,
- .vidioc_unsubscribe_event = v4l2_event_unsubscribe,
-};
-
-static struct video_device usbvision_radio_template = {
- .fops = &usbvision_radio_fops,
- .name = "usbvision-radio",
- .release = video_device_release_empty,
- .ioctl_ops = &usbvision_radio_ioctl_ops,
-};
-
-
-static void usbvision_vdev_init(struct usb_usbvision *usbvision,
- struct video_device *vdev,
- const struct video_device *vdev_template,
- const char *name)
-{
- struct usb_device *usb_dev = usbvision->dev;
-
- if (!usb_dev) {
- dev_err(&usbvision->dev->dev,
- "%s: usbvision->dev is not set\n", __func__);
- return;
- }
-
- *vdev = *vdev_template;
- vdev->lock = &usbvision->v4l2_lock;
- vdev->v4l2_dev = &usbvision->v4l2_dev;
- snprintf(vdev->name, sizeof(vdev->name), "%s", name);
- video_set_drvdata(vdev, usbvision);
-}
-
-/* unregister video4linux devices */
-static void usbvision_unregister_video(struct usb_usbvision *usbvision)
-{
- /* Radio Device: */
- if (video_is_registered(&usbvision->rdev)) {
- PDEBUG(DBG_PROBE, "unregister %s [v4l2]",
- video_device_node_name(&usbvision->rdev));
- video_unregister_device(&usbvision->rdev);
- }
-
- /* Video Device: */
- if (video_is_registered(&usbvision->vdev)) {
- PDEBUG(DBG_PROBE, "unregister %s [v4l2]",
- video_device_node_name(&usbvision->vdev));
- video_unregister_device(&usbvision->vdev);
- }
-}
-
-/* register video4linux devices */
-static int usbvision_register_video(struct usb_usbvision *usbvision)
-{
- int res = -ENOMEM;
-
- /* Video Device: */
- usbvision_vdev_init(usbvision, &usbvision->vdev,
- &usbvision_video_template, "USBVision Video");
- if (!usbvision->have_tuner) {
- v4l2_disable_ioctl(&usbvision->vdev, VIDIOC_G_FREQUENCY);
- v4l2_disable_ioctl(&usbvision->vdev, VIDIOC_S_TUNER);
- v4l2_disable_ioctl(&usbvision->vdev, VIDIOC_G_FREQUENCY);
- v4l2_disable_ioctl(&usbvision->vdev, VIDIOC_S_TUNER);
- }
- usbvision->vdev.device_caps = V4L2_CAP_VIDEO_CAPTURE |
- V4L2_CAP_READWRITE | V4L2_CAP_STREAMING;
- if (usbvision->have_tuner)
- usbvision->vdev.device_caps |= V4L2_CAP_TUNER;
-
- if (video_register_device(&usbvision->vdev, VFL_TYPE_VIDEO, video_nr) < 0)
- goto err_exit;
- printk(KERN_INFO "USBVision[%d]: registered USBVision Video device %s [v4l2]\n",
- usbvision->nr, video_device_node_name(&usbvision->vdev));
-
- /* Radio Device: */
- if (usbvision_device_data[usbvision->dev_model].radio) {
- /* usbvision has radio */
- usbvision_vdev_init(usbvision, &usbvision->rdev,
- &usbvision_radio_template, "USBVision Radio");
- usbvision->rdev.device_caps = V4L2_CAP_RADIO | V4L2_CAP_TUNER;
- if (video_register_device(&usbvision->rdev, VFL_TYPE_RADIO, radio_nr) < 0)
- goto err_exit;
- printk(KERN_INFO "USBVision[%d]: registered USBVision Radio device %s [v4l2]\n",
- usbvision->nr, video_device_node_name(&usbvision->rdev));
- }
- /* all done */
- return 0;
-
- err_exit:
- dev_err(&usbvision->dev->dev,
- "USBVision[%d]: video_register_device() failed\n",
- usbvision->nr);
- usbvision_unregister_video(usbvision);
- return res;
-}
-
-/*
- * usbvision_alloc()
- *
- * This code allocates the struct usb_usbvision.
- * It is filled with default values.
- *
- * Returns NULL on error, a pointer to usb_usbvision else.
- *
- */
-static struct usb_usbvision *usbvision_alloc(struct usb_device *dev,
- struct usb_interface *intf)
-{
- struct usb_usbvision *usbvision;
-
- usbvision = kzalloc(sizeof(*usbvision), GFP_KERNEL);
- if (!usbvision)
- return NULL;
-
- usbvision->dev = dev;
- if (v4l2_device_register(&intf->dev, &usbvision->v4l2_dev))
- goto err_free;
-
- if (v4l2_ctrl_handler_init(&usbvision->hdl, 4))
- goto err_unreg;
- usbvision->v4l2_dev.ctrl_handler = &usbvision->hdl;
- mutex_init(&usbvision->v4l2_lock);
-
- /* prepare control urb for control messages during interrupts */
- usbvision->ctrl_urb = usb_alloc_urb(USBVISION_URB_FRAMES, GFP_KERNEL);
- if (!usbvision->ctrl_urb)
- goto err_unreg;
-
- return usbvision;
-
-err_unreg:
- v4l2_ctrl_handler_free(&usbvision->hdl);
- v4l2_device_unregister(&usbvision->v4l2_dev);
-err_free:
- kfree(usbvision);
- return NULL;
-}
-
-/*
- * usbvision_release()
- *
- * This code does final release of struct usb_usbvision. This happens
- * after the device is disconnected -and- all clients closed their files.
- *
- */
-static void usbvision_release(struct usb_usbvision *usbvision)
-{
- PDEBUG(DBG_PROBE, "");
-
- usbvision->initialized = 0;
-
- usbvision_remove_sysfs(&usbvision->vdev);
- usbvision_unregister_video(usbvision);
- kfree(usbvision->alt_max_pkt_size);
-
- usb_free_urb(usbvision->ctrl_urb);
-
- v4l2_ctrl_handler_free(&usbvision->hdl);
- v4l2_device_unregister(&usbvision->v4l2_dev);
- kfree(usbvision);
-
- PDEBUG(DBG_PROBE, "success");
-}
-
-
-/*********************** usb interface **********************************/
-
-static void usbvision_configure_video(struct usb_usbvision *usbvision)
-{
- int model;
-
- if (!usbvision)
- return;
-
- model = usbvision->dev_model;
- usbvision->palette = usbvision_v4l2_format[2]; /* V4L2_PIX_FMT_RGB24; */
-
- if (usbvision_device_data[usbvision->dev_model].vin_reg2_override) {
- usbvision->vin_reg2_preset =
- usbvision_device_data[usbvision->dev_model].vin_reg2;
- } else {
- usbvision->vin_reg2_preset = 0;
- }
-
- usbvision->tvnorm_id = usbvision_device_data[model].video_norm;
- usbvision->video_inputs = usbvision_device_data[model].video_channels;
- usbvision->ctl_input = 0;
- usbvision->radio_freq = 87.5 * 16000;
- usbvision->tv_freq = 400 * 16;
-
- /* This should be here to make i2c clients to be able to register */
- /* first switch off audio */
- if (usbvision_device_data[model].audio_channels > 0)
- usbvision_audio_off(usbvision);
- /* and then power up the tuner */
- usbvision_power_on(usbvision);
- usbvision_i2c_register(usbvision);
-}
-
-/*
- * usbvision_probe()
- *
- * This procedure queries device descriptor and accepts the interface
- * if it looks like USBVISION video device
- *
- */
-static int usbvision_probe(struct usb_interface *intf,
- const struct usb_device_id *devid)
-{
- struct usb_device *dev = usb_get_dev(interface_to_usbdev(intf));
- struct usb_interface *uif;
- __u8 ifnum = intf->altsetting->desc.bInterfaceNumber;
- const struct usb_host_interface *interface;
- struct usb_usbvision *usbvision = NULL;
- const struct usb_endpoint_descriptor *endpoint;
- int model, i, ret;
-
- PDEBUG(DBG_PROBE, "VID=%#04x, PID=%#04x, ifnum=%u",
- le16_to_cpu(dev->descriptor.idVendor),
- le16_to_cpu(dev->descriptor.idProduct), ifnum);
-
- model = devid->driver_info;
- if (model < 0 || model >= usbvision_device_data_size) {
- PDEBUG(DBG_PROBE, "model out of bounds %d", model);
- ret = -ENODEV;
- goto err_usb;
- }
- printk(KERN_INFO "%s: %s found\n", __func__,
- usbvision_device_data[model].model_string);
-
- if (usbvision_device_data[model].interface >= 0)
- interface = &dev->actconfig->interface[usbvision_device_data[model].interface]->altsetting[0];
- else if (ifnum < dev->actconfig->desc.bNumInterfaces)
- interface = &dev->actconfig->interface[ifnum]->altsetting[0];
- else {
- dev_err(&intf->dev, "interface %d is invalid, max is %d\n",
- ifnum, dev->actconfig->desc.bNumInterfaces - 1);
- ret = -ENODEV;
- goto err_usb;
- }
-
- if (interface->desc.bNumEndpoints < 2) {
- dev_err(&intf->dev, "interface %d has %d endpoints, but must have minimum 2\n",
- ifnum, interface->desc.bNumEndpoints);
- ret = -ENODEV;
- goto err_usb;
- }
- endpoint = &interface->endpoint[1].desc;
-
- if (!usb_endpoint_xfer_isoc(endpoint)) {
- dev_err(&intf->dev, "%s: interface %d. has non-ISO endpoint!\n",
- __func__, ifnum);
- dev_err(&intf->dev, "%s: Endpoint attributes %d",
- __func__, endpoint->bmAttributes);
- ret = -ENODEV;
- goto err_usb;
- }
- if (usb_endpoint_dir_out(endpoint)) {
- dev_err(&intf->dev, "%s: interface %d. has ISO OUT endpoint!\n",
- __func__, ifnum);
- ret = -ENODEV;
- goto err_usb;
- }
-
- usbvision = usbvision_alloc(dev, intf);
- if (!usbvision) {
- dev_err(&intf->dev, "%s: couldn't allocate USBVision struct\n", __func__);
- ret = -ENOMEM;
- goto err_usb;
- }
-
- if (dev->descriptor.bNumConfigurations > 1)
- usbvision->bridge_type = BRIDGE_NT1004;
- else if (model == DAZZLE_DVC_90_REV_1_SECAM)
- usbvision->bridge_type = BRIDGE_NT1005;
- else
- usbvision->bridge_type = BRIDGE_NT1003;
- PDEBUG(DBG_PROBE, "bridge_type %d", usbvision->bridge_type);
-
- /* compute alternate max packet sizes */
- uif = dev->actconfig->interface[0];
-
- usbvision->num_alt = uif->num_altsetting;
- PDEBUG(DBG_PROBE, "Alternate settings: %i", usbvision->num_alt);
- usbvision->alt_max_pkt_size = kmalloc_array(32, usbvision->num_alt,
- GFP_KERNEL);
- if (!usbvision->alt_max_pkt_size) {
- ret = -ENOMEM;
- goto err_pkt;
- }
-
- for (i = 0; i < usbvision->num_alt; i++) {
- u16 tmp;
-
- if (uif->altsetting[i].desc.bNumEndpoints < 2) {
- ret = -ENODEV;
- goto err_pkt;
- }
-
- tmp = le16_to_cpu(uif->altsetting[i].endpoint[1].desc.
- wMaxPacketSize);
- usbvision->alt_max_pkt_size[i] =
- (tmp & 0x07ff) * (((tmp & 0x1800) >> 11) + 1);
- PDEBUG(DBG_PROBE, "Alternate setting %i, max size= %i", i,
- usbvision->alt_max_pkt_size[i]);
- }
-
-
- usbvision->nr = usbvision_nr++;
-
- spin_lock_init(&usbvision->queue_lock);
- init_waitqueue_head(&usbvision->wait_frame);
- init_waitqueue_head(&usbvision->wait_stream);
-
- usbvision->have_tuner = usbvision_device_data[model].tuner;
- if (usbvision->have_tuner)
- usbvision->tuner_type = usbvision_device_data[model].tuner_type;
-
- usbvision->dev_model = model;
- usbvision->remove_pending = 0;
- usbvision->iface = ifnum;
- usbvision->iface_alt = 0;
- usbvision->video_endp = endpoint->bEndpointAddress;
- usbvision->isoc_packet_size = 0;
- usbvision->usb_bandwidth = 0;
- usbvision->user = 0;
- usbvision->streaming = stream_off;
- usbvision_configure_video(usbvision);
- usbvision_register_video(usbvision);
-
- usbvision_create_sysfs(&usbvision->vdev);
-
- PDEBUG(DBG_PROBE, "success");
- return 0;
-
-err_pkt:
- usbvision_release(usbvision);
-err_usb:
- usb_put_dev(dev);
- return ret;
-}
-
-
-/*
- * usbvision_disconnect()
- *
- * This procedure stops all driver activity, deallocates interface-private
- * structure (pointed by 'ptr') and after that driver should be removable
- * with no ill consequences.
- *
- */
-static void usbvision_disconnect(struct usb_interface *intf)
-{
- struct usb_usbvision *usbvision = to_usbvision(usb_get_intfdata(intf));
- int u;
-
- PDEBUG(DBG_PROBE, "");
-
- if (!usbvision) {
- pr_err("%s: usb_get_intfdata() failed\n", __func__);
- return;
- }
-
- mutex_lock(&usbvision->v4l2_lock);
-
- /* At this time we ask to cancel outstanding URBs */
- usbvision_stop_isoc(usbvision);
-
- v4l2_device_disconnect(&usbvision->v4l2_dev);
- usbvision_i2c_unregister(usbvision);
- usbvision->remove_pending = 1; /* Now all ISO data will be ignored */
- u = usbvision->user;
-
- usb_put_dev(usbvision->dev);
- usbvision->dev = NULL; /* USB device is no more */
-
- mutex_unlock(&usbvision->v4l2_lock);
-
- if (u) {
- printk(KERN_INFO "%s: In use, disconnect pending\n",
- __func__);
- wake_up_interruptible(&usbvision->wait_frame);
- wake_up_interruptible(&usbvision->wait_stream);
- } else {
- usbvision_release(usbvision);
- }
-
- PDEBUG(DBG_PROBE, "success");
-}
-
-static struct usb_driver usbvision_driver = {
- .name = "usbvision",
- .id_table = usbvision_table,
- .probe = usbvision_probe,
- .disconnect = usbvision_disconnect,
-};
-
-/*
- * usbvision_init()
- *
- * This code is run to initialize the driver.
- *
- */
-static int __init usbvision_init(void)
-{
- int err_code;
-
- PDEBUG(DBG_PROBE, "");
-
- PDEBUG(DBG_IO, "IO debugging is enabled [video]");
- PDEBUG(DBG_PROBE, "PROBE debugging is enabled [video]");
- PDEBUG(DBG_MMAP, "MMAP debugging is enabled [video]");
-
- /* disable planar mode support unless compression enabled */
- if (isoc_mode != ISOC_MODE_COMPRESS) {
- /* FIXME : not the right way to set supported flag */
- usbvision_v4l2_format[6].supported = 0; /* V4L2_PIX_FMT_YVU420 */
- usbvision_v4l2_format[7].supported = 0; /* V4L2_PIX_FMT_YUV422P */
- }
-
- err_code = usb_register(&usbvision_driver);
-
- if (err_code == 0) {
- printk(KERN_INFO DRIVER_DESC " : " USBVISION_VERSION_STRING "\n");
- PDEBUG(DBG_PROBE, "success");
- }
- return err_code;
-}
-
-static void __exit usbvision_exit(void)
-{
- PDEBUG(DBG_PROBE, "");
-
- usb_deregister(&usbvision_driver);
- PDEBUG(DBG_PROBE, "success");
-}
-
-module_init(usbvision_init);
-module_exit(usbvision_exit);
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * USBVISION.H
- * usbvision header file
- *
- * Copyright (c) 1999-2005 Joerg Heckenbach <joerg@heckenbach-aw.de>
- * Dwaine Garden <dwainegarden@rogers.com>
- *
- * Report problems to v4l MailingList: linux-media@vger.kernel.org
- *
- * This module is part of usbvision driver project.
- * Updates to driver completed by Dwaine P. Garden
- * v4l2 conversion by Thierry Merle <thierry.merle@free.fr>
- */
-
-
-#ifndef __LINUX_USBVISION_H
-#define __LINUX_USBVISION_H
-
-#include <linux/list.h>
-#include <linux/usb.h>
-#include <linux/i2c.h>
-#include <linux/mutex.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-ctrls.h>
-#include <media/tuner.h>
-#include <linux/videodev2.h>
-
-#define USBVISION_DEBUG /* Turn on debug messages */
-
-#define USBVISION_PWR_REG 0x00
- #define USBVISION_SSPND_EN (1 << 1)
- #define USBVISION_RES2 (1 << 2)
- #define USBVISION_PWR_VID (1 << 5)
- #define USBVISION_E2_EN (1 << 7)
-#define USBVISION_CONFIG_REG 0x01
-#define USBVISION_ADRS_REG 0x02
-#define USBVISION_ALTER_REG 0x03
-#define USBVISION_FORCE_ALTER_REG 0x04
-#define USBVISION_STATUS_REG 0x05
-#define USBVISION_IOPIN_REG 0x06
- #define USBVISION_IO_1 (1 << 0)
- #define USBVISION_IO_2 (1 << 1)
- #define USBVISION_AUDIO_IN 0
- #define USBVISION_AUDIO_TV 1
- #define USBVISION_AUDIO_RADIO 2
- #define USBVISION_AUDIO_MUTE 3
-#define USBVISION_SER_MODE 0x07
- #define USBVISION_CLK_OUT (1 << 0)
- #define USBVISION_DAT_IO (1 << 1)
- #define USBVISION_SENS_OUT (1 << 2)
- #define USBVISION_SER_MODE_SOFT (0 << 4)
- #define USBVISION_SER_MODE_SIO (1 << 4)
-#define USBVISION_SER_ADRS 0x08
-#define USBVISION_SER_CONT 0x09
-#define USBVISION_SER_DAT1 0x0A
-#define USBVISION_SER_DAT2 0x0B
-#define USBVISION_SER_DAT3 0x0C
-#define USBVISION_SER_DAT4 0x0D
-#define USBVISION_EE_DATA 0x0E
-#define USBVISION_EE_LSBAD 0x0F
-#define USBVISION_EE_CONT 0x10
-#define USBVISION_DRM_CONT 0x12
- #define USBVISION_REF (1 << 0)
- #define USBVISION_RES_UR (1 << 2)
- #define USBVISION_RES_FDL (1 << 3)
- #define USBVISION_RES_VDW (1 << 4)
-#define USBVISION_DRM_PRM1 0x13
-#define USBVISION_DRM_PRM2 0x14
-#define USBVISION_DRM_PRM3 0x15
-#define USBVISION_DRM_PRM4 0x16
-#define USBVISION_DRM_PRM5 0x17
-#define USBVISION_DRM_PRM6 0x18
-#define USBVISION_DRM_PRM7 0x19
-#define USBVISION_DRM_PRM8 0x1A
-#define USBVISION_VIN_REG1 0x1B
- #define USBVISION_8_422_SYNC 0x01
- #define USBVISION_16_422_SYNC 0x02
- #define USBVISION_VSNC_POL (1 << 3)
- #define USBVISION_HSNC_POL (1 << 4)
- #define USBVISION_FID_POL (1 << 5)
- #define USBVISION_HVALID_PO (1 << 6)
- #define USBVISION_VCLK_POL (1 << 7)
-#define USBVISION_VIN_REG2 0x1C
- #define USBVISION_AUTO_FID (1 << 0)
- #define USBVISION_NONE_INTER (1 << 1)
- #define USBVISION_NOHVALID (1 << 2)
- #define USBVISION_UV_ID (1 << 3)
- #define USBVISION_FIX_2C (1 << 4)
- #define USBVISION_SEND_FID (1 << 5)
- #define USBVISION_KEEP_BLANK (1 << 7)
-#define USBVISION_LXSIZE_I 0x1D
-#define USBVISION_MXSIZE_I 0x1E
-#define USBVISION_LYSIZE_I 0x1F
-#define USBVISION_MYSIZE_I 0x20
-#define USBVISION_LX_OFFST 0x21
-#define USBVISION_MX_OFFST 0x22
-#define USBVISION_LY_OFFST 0x23
-#define USBVISION_MY_OFFST 0x24
-#define USBVISION_FRM_RATE 0x25
-#define USBVISION_LXSIZE_O 0x26
-#define USBVISION_MXSIZE_O 0x27
-#define USBVISION_LYSIZE_O 0x28
-#define USBVISION_MYSIZE_O 0x29
-#define USBVISION_FILT_CONT 0x2A
-#define USBVISION_VO_MODE 0x2B
-#define USBVISION_INTRA_CYC 0x2C
-#define USBVISION_STRIP_SZ 0x2D
-#define USBVISION_FORCE_INTRA 0x2E
-#define USBVISION_FORCE_UP 0x2F
-#define USBVISION_BUF_THR 0x30
-#define USBVISION_DVI_YUV 0x31
-#define USBVISION_AUDIO_CONT 0x32
-#define USBVISION_AUD_PK_LEN 0x33
-#define USBVISION_BLK_PK_LEN 0x34
-#define USBVISION_PCM_THR1 0x38
-#define USBVISION_PCM_THR2 0x39
-#define USBVISION_DIST_THR_L 0x3A
-#define USBVISION_DIST_THR_H 0x3B
-#define USBVISION_MAX_DIST_L 0x3C
-#define USBVISION_MAX_DIST_H 0x3D
-#define USBVISION_OP_CODE 0x33
-
-#define MAX_BYTES_PER_PIXEL 4
-
-#define MIN_FRAME_WIDTH 64
-#define MAX_USB_WIDTH 320 /* 384 */
-#define MAX_FRAME_WIDTH 320 /* 384 */ /* stretching sometimes causes crashes*/
-
-#define MIN_FRAME_HEIGHT 48
-#define MAX_USB_HEIGHT 240 /* 288 */
-#define MAX_FRAME_HEIGHT 240 /* 288 */ /* Stretching sometimes causes crashes*/
-
-#define MAX_FRAME_SIZE (MAX_FRAME_WIDTH * MAX_FRAME_HEIGHT * MAX_BYTES_PER_PIXEL)
-#define USBVISION_CLIPMASK_SIZE (MAX_FRAME_WIDTH * MAX_FRAME_HEIGHT / 8) /* bytesize of clipmask */
-
-#define USBVISION_URB_FRAMES 32
-
-#define USBVISION_NUM_HEADERMARKER 20
-#define USBVISION_NUMFRAMES 3 /* Maximum number of frames an application can get */
-#define USBVISION_NUMSBUF 2 /* Dimensioning the USB S buffering */
-
-#define USBVISION_POWEROFF_TIME (3 * HZ) /* 3 seconds */
-
-
-#define FRAMERATE_MIN 0
-#define FRAMERATE_MAX 31
-
-enum {
- ISOC_MODE_YUV422 = 0x03,
- ISOC_MODE_YUV420 = 0x14,
- ISOC_MODE_COMPRESS = 0x60,
-};
-
-/* This macro restricts an int variable to an inclusive range */
-#define RESTRICT_TO_RANGE(v, mi, ma) \
- { if (((int)v) < (mi)) (v) = (mi); else if ((v) > (ma)) (v) = (ma); }
-
-/*
- * We use macros to do YUV -> RGB conversion because this is
- * very important for speed and totally unimportant for size.
- *
- * YUV -> RGB Conversion
- * ---------------------
- *
- * B = 1.164*(Y-16) + 2.018*(V-128)
- * G = 1.164*(Y-16) - 0.813*(U-128) - 0.391*(V-128)
- * R = 1.164*(Y-16) + 1.596*(U-128)
- *
- * If you fancy integer arithmetic (as you should), hear this:
- *
- * 65536*B = 76284*(Y-16) + 132252*(V-128)
- * 65536*G = 76284*(Y-16) - 53281*(U-128) - 25625*(V-128)
- * 65536*R = 76284*(Y-16) + 104595*(U-128)
- *
- * Make sure the output values are within [0..255] range.
- */
-#define LIMIT_RGB(x) (((x) < 0) ? 0 : (((x) > 255) ? 255 : (x)))
-#define YUV_TO_RGB_BY_THE_BOOK(my, mu, mv, mr, mg, mb) { \
- int mm_y, mm_yc, mm_u, mm_v, mm_r, mm_g, mm_b; \
- mm_y = (my) - 16; \
- mm_u = (mu) - 128; \
- mm_v = (mv) - 128; \
- mm_yc = mm_y * 76284; \
- mm_b = (mm_yc + 132252 * mm_v) >> 16; \
- mm_g = (mm_yc - 53281 * mm_u - 25625 * mm_v) >> 16; \
- mm_r = (mm_yc + 104595 * mm_u) >> 16; \
- mb = LIMIT_RGB(mm_b); \
- mg = LIMIT_RGB(mm_g); \
- mr = LIMIT_RGB(mm_r); \
-}
-
-/*
- * This macro checks if usbvision is still operational. The 'usbvision'
- * pointer must be valid, usbvision->dev must be valid, we are not
- * removing the device and the device has not erred on us.
- */
-#define USBVISION_IS_OPERATIONAL(udevice) (\
- (udevice != NULL) && \
- ((udevice)->dev != NULL) && \
- ((udevice)->last_error == 0) && \
- (!(udevice)->remove_pending))
-
-#define I2C_USB_ADAP_MAX 16
-
-#define USBVISION_NORMS (V4L2_STD_PAL | V4L2_STD_NTSC | V4L2_STD_SECAM | V4L2_STD_PAL_M)
-
-/* ----------------------------------------------------------------- */
-/* usbvision video structures */
-/* ----------------------------------------------------------------- */
-enum scan_state {
- scan_state_scanning, /* Scanning for header */
- scan_state_lines /* Parsing lines */
-};
-
-/* Completion states of the data parser */
-enum parse_state {
- parse_state_continue, /* Just parse next item */
- parse_state_next_frame, /* Frame done, send it to V4L */
- parse_state_out, /* Not enough data for frame */
- parse_state_end_parse /* End parsing */
-};
-
-enum frame_state {
- frame_state_unused, /* Unused (no MCAPTURE) */
- frame_state_ready, /* Ready to start grabbing */
- frame_state_grabbing, /* In the process of being grabbed into */
- frame_state_done, /* Finished grabbing, but not been synced yet */
- frame_state_done_hold, /* Are syncing or reading */
- frame_state_error, /* Something bad happened while processing */
-};
-
-/* stream states */
-enum stream_state {
- stream_off, /* Driver streaming is completely OFF */
- stream_idle, /* Driver streaming is ready to be put ON by the application */
- stream_interrupt, /* Driver streaming must be interrupted */
- stream_on, /* Driver streaming is put ON by the application */
-};
-
-enum isoc_state {
- isoc_state_in_frame, /* Isoc packet is member of frame */
- isoc_state_no_frame, /* Isoc packet is not member of any frame */
-};
-
-struct usb_device;
-
-struct usbvision_sbuf {
- char *data;
- struct urb *urb;
-};
-
-#define USBVISION_MAGIC_1 0x55
-#define USBVISION_MAGIC_2 0xAA
-#define USBVISION_HEADER_LENGTH 0x0c
-#define USBVISION_SAA7111_ADDR 0x48
-#define USBVISION_SAA7113_ADDR 0x4a
-#define USBVISION_IIC_LRACK 0x20
-#define USBVISION_IIC_LRNACK 0x30
-#define USBVISION_FRAME_FORMAT_PARAM_INTRA (1<<7)
-
-struct usbvision_v4l2_format_st {
- int supported;
- int bytes_per_pixel;
- int depth;
- int format;
-};
-#define USBVISION_SUPPORTED_PALETTES ARRAY_SIZE(usbvision_v4l2_format)
-
-struct usbvision_frame_header {
- unsigned char magic_1; /* 0 magic */
- unsigned char magic_2; /* 1 magic */
- unsigned char header_length; /* 2 */
- unsigned char frame_num; /* 3 */
- unsigned char frame_phase; /* 4 */
- unsigned char frame_latency; /* 5 */
- unsigned char data_format; /* 6 */
- unsigned char format_param; /* 7 */
- unsigned char frame_width_lo; /* 8 */
- unsigned char frame_width_hi; /* 9 */
- unsigned char frame_height_lo; /* 10 */
- unsigned char frame_height_hi; /* 11 */
- __u16 frame_width; /* 8 - 9 after endian correction*/
- __u16 frame_height; /* 10 - 11 after endian correction*/
-};
-
-struct usbvision_frame {
- char *data; /* Frame buffer */
- struct usbvision_frame_header isoc_header; /* Header from stream */
-
- int width; /* Width application is expecting */
- int height; /* Height */
- int index; /* Frame index */
- int frmwidth; /* Width the frame actually is */
- int frmheight; /* Height */
-
- volatile int grabstate; /* State of grabbing */
- int scanstate; /* State of scanning */
-
- struct list_head frame;
-
- int curline; /* Line of frame we're working on */
-
- long scanlength; /* uncompressed, raw data length of frame */
- long bytes_read; /* amount of scanlength that has been read from data */
- struct usbvision_v4l2_format_st v4l2_format; /* format the user needs*/
- int v4l2_linesize; /* bytes for one videoline*/
- u64 ts;
- int sequence; /* How many video frames we send to user */
-};
-
-#define CODEC_SAA7113 7113
-#define CODEC_SAA7111 7111
-#define CODEC_WEBCAM 3000
-#define BRIDGE_NT1003 1003
-#define BRIDGE_NT1004 1004
-#define BRIDGE_NT1005 1005
-
-struct usbvision_device_data_st {
- __u64 video_norm;
- const char *model_string;
- int interface; /* to handle special interface number like BELKIN and Hauppauge WinTV-USB II */
- __u16 codec;
- unsigned video_channels:3;
- unsigned audio_channels:2;
- unsigned radio:1;
- unsigned vbi:1;
- unsigned tuner:1;
- unsigned vin_reg1_override:1; /* Override default value with */
- unsigned vin_reg2_override:1; /* vin_reg1, vin_reg2, etc. */
- unsigned dvi_yuv_override:1;
- __u8 vin_reg1;
- __u8 vin_reg2;
- __u8 dvi_yuv;
- __u8 tuner_type;
- __s16 x_offset;
- __s16 y_offset;
-};
-
-/* Declared on usbvision-cards.c */
-extern struct usbvision_device_data_st usbvision_device_data[];
-extern struct usb_device_id usbvision_table[];
-
-struct usb_usbvision {
- struct v4l2_device v4l2_dev;
- struct v4l2_ctrl_handler hdl;
- struct video_device vdev; /* Video Device */
- struct video_device rdev; /* Radio Device */
-
- /* i2c Declaration Section*/
- struct i2c_adapter i2c_adap;
- int registered_i2c;
-
- struct urb *ctrl_urb;
- unsigned char ctrl_urb_buffer[8];
- int ctrl_urb_busy;
- struct usb_ctrlrequest ctrl_urb_setup;
-
- /* configuration part */
- int have_tuner;
- int tuner_type;
- int bridge_type; /* NT1003, NT1004, NT1005 */
- int radio;
- int video_inputs; /* # of inputs */
- unsigned long radio_freq;
- unsigned long tv_freq;
- int audio_mute;
- int audio_channel;
- int isoc_mode; /* format of video data for the usb isoc-transfer */
- unsigned int nr; /* Number of the device */
-
- /* Device structure */
- struct usb_device *dev;
- /* usb transfer */
- int num_alt; /* Number of alternative settings */
- unsigned int *alt_max_pkt_size; /* array of max_packet_size */
- unsigned char iface; /* Video interface number */
- unsigned char iface_alt; /* Alt settings */
- unsigned char vin_reg2_preset;
- struct mutex v4l2_lock;
- int power; /* is the device powered on? */
- int user; /* user count for exclusive use */
- int initialized; /* Had we already sent init sequence? */
- int dev_model; /* What type of USBVISION device we got? */
- enum stream_state streaming; /* Are we streaming Isochronous? */
- int last_error; /* What calamity struck us? */
- int curwidth; /* width of the frame the device is currently set to*/
- int curheight; /* height of the frame the device is currently set to*/
- int stretch_width; /* stretch-factor for frame width (from usb to screen)*/
- int stretch_height; /* stretch-factor for frame height (from usb to screen)*/
- char *fbuf; /* Videodev buffer area for mmap*/
- int max_frame_size; /* Bytes in one video frame */
- int fbuf_size; /* Videodev buffer size */
- spinlock_t queue_lock; /* spinlock for protecting mods on inqueue and outqueue */
- struct list_head inqueue, outqueue; /* queued frame list and ready to dequeue frame list */
- wait_queue_head_t wait_frame; /* Processes waiting */
- wait_queue_head_t wait_stream; /* Processes waiting */
- struct usbvision_frame *cur_frame; /* pointer to current frame, set by usbvision_find_header */
- struct usbvision_frame frame[USBVISION_NUMFRAMES]; /* frame buffer */
- int num_frames; /* number of frames allocated */
- struct usbvision_sbuf sbuf[USBVISION_NUMSBUF]; /* S buffering */
- volatile int remove_pending; /* If set then about to exit */
-
- /* Scratch space from the Isochronous Pipe.*/
- unsigned char *scratch;
- int scratch_read_ptr;
- int scratch_write_ptr;
- int scratch_headermarker[USBVISION_NUM_HEADERMARKER];
- int scratch_headermarker_read_ptr;
- int scratch_headermarker_write_ptr;
- enum isoc_state isocstate;
- struct usbvision_v4l2_format_st palette;
-
- struct v4l2_capability vcap; /* Video capabilities */
- unsigned int ctl_input; /* selected input */
- v4l2_std_id tvnorm_id; /* selected tv norm */
- unsigned char video_endp; /* 0x82 for USBVISION devices based */
-
- /* Decompression stuff: */
- unsigned char *intra_frame_buffer; /* Buffer for reference frame */
- int block_pos; /* for test only */
- int request_intra; /* 0 = normal; 1 = intra frame is requested; */
- int last_isoc_frame_num; /* check for lost isoc frames */
- int isoc_packet_size; /* need to calculate used_bandwidth */
- int used_bandwidth; /* used bandwidth 0-100%, need to set compr_level */
- int compr_level; /* How strong (100) or weak (0) is compression */
- int last_compr_level; /* How strong (100) or weak (0) was compression */
- int usb_bandwidth; /* Mbit/s */
-
- /* Statistics that can be overlaid on the screen */
- unsigned long isoc_urb_count; /* How many URBs we received so far */
- unsigned long urb_length; /* Length of last URB */
- unsigned long isoc_data_count; /* How many bytes we received */
- unsigned long header_count; /* How many frame headers we found */
- unsigned long scratch_ovf_count; /* How many times we overflowed scratch */
- unsigned long isoc_skip_count; /* How many empty ISO packets received */
- unsigned long isoc_err_count; /* How many bad ISO packets received */
- unsigned long isoc_packet_count; /* How many packets we totally got */
- int isoc_measure_bandwidth_count;
- int frame_num; /* How many video frames we send to user */
- int max_strip_len; /* How big is the biggest strip */
- int comprblock_pos;
- int strip_len_errors; /* How many times was block_pos greater than strip_len */
- int strip_magic_errors;
- int strip_line_number_errors;
- int compr_block_types[4];
-};
-
-static inline struct usb_usbvision *to_usbvision(struct v4l2_device *v4l2_dev)
-{
- return container_of(v4l2_dev, struct usb_usbvision, v4l2_dev);
-}
-
-#define call_all(usbvision, o, f, args...) \
- v4l2_device_call_all(&usbvision->v4l2_dev, 0, o, f, ##args)
-
-/* --------------------------------------------------------------- */
-/* defined in usbvision-i2c.c */
-/* i2c-algo-usb declaration */
-/* --------------------------------------------------------------- */
-
-/* ----------------------------------------------------------------------- */
-/* usbvision specific I2C functions */
-/* ----------------------------------------------------------------------- */
-int usbvision_i2c_register(struct usb_usbvision *usbvision);
-int usbvision_i2c_unregister(struct usb_usbvision *usbvision);
-
-/* defined in usbvision-core.c */
-int usbvision_read_reg(struct usb_usbvision *usbvision, unsigned char reg);
-int usbvision_write_reg(struct usb_usbvision *usbvision, unsigned char reg,
- unsigned char value);
-
-int usbvision_frames_alloc(struct usb_usbvision *usbvision, int number_of_frames);
-void usbvision_frames_free(struct usb_usbvision *usbvision);
-int usbvision_scratch_alloc(struct usb_usbvision *usbvision);
-void usbvision_scratch_free(struct usb_usbvision *usbvision);
-int usbvision_decompress_alloc(struct usb_usbvision *usbvision);
-void usbvision_decompress_free(struct usb_usbvision *usbvision);
-
-int usbvision_setup(struct usb_usbvision *usbvision, int format);
-int usbvision_init_isoc(struct usb_usbvision *usbvision);
-int usbvision_restart_isoc(struct usb_usbvision *usbvision);
-void usbvision_stop_isoc(struct usb_usbvision *usbvision);
-int usbvision_set_alternate(struct usb_usbvision *dev);
-
-int usbvision_set_audio(struct usb_usbvision *usbvision, int audio_channel);
-int usbvision_audio_off(struct usb_usbvision *usbvision);
-
-int usbvision_begin_streaming(struct usb_usbvision *usbvision);
-void usbvision_empty_framequeues(struct usb_usbvision *dev);
-int usbvision_stream_interrupt(struct usb_usbvision *dev);
-
-int usbvision_muxsel(struct usb_usbvision *usbvision, int channel);
-int usbvision_set_input(struct usb_usbvision *usbvision);
-int usbvision_set_output(struct usb_usbvision *usbvision, int width, int height);
-
-int usbvision_power_off(struct usb_usbvision *usbvision);
-int usbvision_power_on(struct usb_usbvision *usbvision);
-
-#endif /* __LINUX_USBVISION_H */
--- /dev/null
+config VIDEO_ZORAN
+ tristate "Zoran ZR36057/36067 Video For Linux (Deprecated)"
+ depends on PCI && I2C_ALGOBIT && VIDEO_V4L2
+ depends on !ALPHA
+ select VIDEOBUF2_DMA_CONTIG
+ help
+ Say Y for support for MJPEG capture cards based on the Zoran
+ 36057/36067 PCI controller chipset. This includes the Iomega
+ Buz, Pinnacle DC10+ and the Linux Media Labs LML33. There is
+ a driver homepage at <http://mjpeg.sf.net/driver-zoran/>. For
+ more information, check <file:Documentation/driver-api/media/drivers/zoran.rst>.
+
+ To compile this driver as a module, choose M here: the
+ module will be called zr36067.
+
+config VIDEO_ZORAN_DC30
+ tristate "Pinnacle/Miro DC30(+) support"
+ depends on VIDEO_ZORAN
+ select VIDEO_ADV7175 if MEDIA_SUBDRV_AUTOSELECT
+ select VIDEO_VPX3220 if MEDIA_SUBDRV_AUTOSELECT
+ help
+ Support for the Pinnacle/Miro DC30(+) MJPEG capture/playback
+ card. This also supports really old DC10 cards based on the
+ zr36050 MJPEG codec and zr36016 VFE.
+
+config VIDEO_ZORAN_ZR36060
+ tristate "Zoran ZR36060"
+ depends on VIDEO_ZORAN
+ help
+ Say Y to support Zoran boards based on 36060 chips.
+ This includes Iomega Buz, Pinnacle DC10, Linux media Labs 33
+ and 33 R10 and AverMedia 6 boards.
+
+config VIDEO_ZORAN_BUZ
+ tristate "Iomega Buz support"
+ depends on VIDEO_ZORAN_ZR36060
+ select VIDEO_SAA711X if MEDIA_SUBDRV_AUTOSELECT
+ select VIDEO_SAA7185 if MEDIA_SUBDRV_AUTOSELECT
+ help
+ Support for the Iomega Buz MJPEG capture/playback card.
+
+config VIDEO_ZORAN_DC10
+ tristate "Pinnacle/Miro DC10(+) support"
+ depends on VIDEO_ZORAN_ZR36060
+ select VIDEO_SAA7110 if MEDIA_SUBDRV_AUTOSELECT
+ select VIDEO_ADV7175 if MEDIA_SUBDRV_AUTOSELECT
+ help
+ Support for the Pinnacle/Miro DC10(+) MJPEG capture/playback
+ card.
+
+config VIDEO_ZORAN_LML33
+ tristate "Linux Media Labs LML33 support"
+ depends on VIDEO_ZORAN_ZR36060
+ select VIDEO_BT819 if MEDIA_SUBDRV_AUTOSELECT
+ select VIDEO_BT856 if MEDIA_SUBDRV_AUTOSELECT
+ help
+ Support for the Linux Media Labs LML33 MJPEG capture/playback
+ card.
+
+config VIDEO_ZORAN_LML33R10
+ tristate "Linux Media Labs LML33R10 support"
+ depends on VIDEO_ZORAN_ZR36060
+ select VIDEO_SAA711X if MEDIA_SUBDRV_AUTOSELECT
+ select VIDEO_ADV7170 if MEDIA_SUBDRV_AUTOSELECT
+ help
+ support for the Linux Media Labs LML33R10 MJPEG capture/playback
+ card.
+
+config VIDEO_ZORAN_AVS6EYES
+ tristate "AverMedia 6 Eyes support"
+ depends on VIDEO_ZORAN_ZR36060
+ select VIDEO_BT856 if MEDIA_SUBDRV_AUTOSELECT
+ select VIDEO_BT866 if MEDIA_SUBDRV_AUTOSELECT
+ select VIDEO_KS0127 if MEDIA_SUBDRV_AUTOSELECT
+ help
+ Support for the AverMedia 6 Eyes video surveillance card.
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+zr36067-objs := zoran_device.o \
+ zoran_driver.o zoran_card.o
+
+obj-$(CONFIG_VIDEO_ZORAN) += zr36067.o videocodec.o
+obj-$(CONFIG_VIDEO_ZORAN_DC30) += zr36050.o zr36016.o
+obj-$(CONFIG_VIDEO_ZORAN_ZR36060) += zr36060.o
--- /dev/null
+
+How to test the zoran driver:
+- RAW capture
+ mplayer tv:///dev/video0 -tv driver=v4l2
+
+- MJPEG capture (compression)
+ mplayer tv:///dev/video0 -tv driver=v4l2:outfmt=mjpeg
+ TODO: need two test for both Dcim path
+
+- MJPEG play (decompression)
+ ffmpeg -i test.avi -vcodec mjpeg -an -f v4l2 /dev/video0
+ Note: only recent ffmpeg has the ability of sending non-raw video via v4l2
+
+ The original way of sending video was via mplayer vo_zr/vo_zr2, but it does not compile
+ anymore and is a dead end (usage of some old private ffmpeg structures).
+
+TODO
+- fix the v4l compliance "TRY_FMT cannot handle an invalid pixelformat"
+- Filter JPEG data to made output work
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * VIDEO MOTION CODECs internal API for video devices
+ *
+ * Interface for MJPEG (and maybe later MPEG/WAVELETS) codec's
+ * bound to a master device.
+ *
+ * (c) 2002 Wolfgang Scherr <scherr@net4you.at>
+ */
+
+#define VIDEOCODEC_VERSION "v0.2"
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+
+// kernel config is here (procfs flag)
+
+#ifdef CONFIG_PROC_FS
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
+#endif
+
+#include "videocodec.h"
+
+static int debug;
+module_param(debug, int, 0);
+MODULE_PARM_DESC(debug, "Debug level (0-4)");
+
+#define dprintk(num, format, args...) \
+ do { \
+ if (debug >= num) \
+ printk(format, ##args); \
+ } while (0)
+
+struct attached_list {
+ struct videocodec *codec;
+ struct attached_list *next;
+};
+
+struct codec_list {
+ const struct videocodec *codec;
+ int attached;
+ struct attached_list *list;
+ struct codec_list *next;
+};
+
+static struct codec_list *codeclist_top;
+
+/* ================================================= */
+/* function prototypes of the master/slave interface */
+/* ================================================= */
+
+struct videocodec *videocodec_attach(struct videocodec_master *master)
+{
+ struct codec_list *h = codeclist_top;
+ struct attached_list *a, *ptr;
+ struct videocodec *codec;
+ int res;
+
+ if (!master) {
+ pr_err("%s: no data\n", __func__);
+ return NULL;
+ }
+
+ dprintk(2, "%s: '%s', flags %lx, magic %lx\n", __func__,
+ master->name, master->flags, master->magic);
+
+ if (!h) {
+ pr_err("%s: no device available\n", __func__);
+ return NULL;
+ }
+
+ while (h) {
+ // attach only if the slave has at least the flags
+ // expected by the master
+ if ((master->flags & h->codec->flags) == master->flags) {
+ dprintk(4, "%s: try '%s'\n", __func__, h->codec->name);
+
+ if (!try_module_get(h->codec->owner))
+ return NULL;
+
+ codec = kmemdup(h->codec, sizeof(struct videocodec), GFP_KERNEL);
+ if (!codec)
+ goto out_module_put;
+
+ res = strlen(codec->name);
+ snprintf(codec->name + res, sizeof(codec->name) - res, "[%d]", h->attached);
+ codec->master_data = master;
+ res = codec->setup(codec);
+ if (res == 0) {
+ dprintk(3, "%s: '%s'\n", __func__, codec->name);
+ ptr = kzalloc(sizeof(*ptr), GFP_KERNEL);
+ if (!ptr)
+ goto out_kfree;
+ ptr->codec = codec;
+
+ a = h->list;
+ if (!a) {
+ h->list = ptr;
+ dprintk(4, "videocodec: first element\n");
+ } else {
+ while (a->next)
+ a = a->next; // find end
+ a->next = ptr;
+ dprintk(4, "videocodec: in after '%s'\n", h->codec->name);
+ }
+
+ h->attached += 1;
+ return codec;
+ } else {
+ kfree(codec);
+ }
+ }
+ h = h->next;
+ }
+
+ pr_err("%s: no codec found!\n", __func__);
+ return NULL;
+
+ out_module_put:
+ module_put(h->codec->owner);
+ out_kfree:
+ kfree(codec);
+ return NULL;
+}
+EXPORT_SYMBOL(videocodec_attach);
+
+int videocodec_detach(struct videocodec *codec)
+{
+ struct codec_list *h = codeclist_top;
+ struct attached_list *a, *prev;
+ int res;
+
+ if (!codec) {
+ pr_err("%s: no data\n", __func__);
+ return -EINVAL;
+ }
+
+ dprintk(2, "%s: '%s', type: %x, flags %lx, magic %lx\n", __func__,
+ codec->name, codec->type, codec->flags, codec->magic);
+
+ if (!h) {
+ pr_err("%s: no device left...\n", __func__);
+ return -ENXIO;
+ }
+
+ while (h) {
+ a = h->list;
+ prev = NULL;
+ while (a) {
+ if (codec == a->codec) {
+ res = a->codec->unset(a->codec);
+ if (res >= 0) {
+ dprintk(3, "%s: '%s'\n", __func__, a->codec->name);
+ a->codec->master_data = NULL;
+ } else {
+ pr_err("%s: '%s'\n", __func__, a->codec->name);
+ a->codec->master_data = NULL;
+ }
+ if (!prev) {
+ h->list = a->next;
+ dprintk(4, "videocodec: delete first\n");
+ } else {
+ prev->next = a->next;
+ dprintk(4, "videocodec: delete middle\n");
+ }
+ module_put(a->codec->owner);
+ kfree(a->codec);
+ kfree(a);
+ h->attached -= 1;
+ return 0;
+ }
+ prev = a;
+ a = a->next;
+ }
+ h = h->next;
+ }
+
+ pr_err("%s: given codec not found!\n", __func__);
+ return -EINVAL;
+}
+EXPORT_SYMBOL(videocodec_detach);
+
+int videocodec_register(const struct videocodec *codec)
+{
+ struct codec_list *ptr, *h = codeclist_top;
+
+ if (!codec) {
+ pr_err("%s: no data!\n", __func__);
+ return -EINVAL;
+ }
+
+ dprintk(2,
+ "videocodec: register '%s', type: %x, flags %lx, magic %lx\n",
+ codec->name, codec->type, codec->flags, codec->magic);
+
+ ptr = kzalloc(sizeof(*ptr), GFP_KERNEL);
+ if (!ptr)
+ return -ENOMEM;
+ ptr->codec = codec;
+
+ if (!h) {
+ codeclist_top = ptr;
+ dprintk(4, "videocodec: hooked in as first element\n");
+ } else {
+ while (h->next)
+ h = h->next; // find the end
+ h->next = ptr;
+ dprintk(4, "videocodec: hooked in after '%s'\n",
+ h->codec->name);
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(videocodec_register);
+
+int videocodec_unregister(const struct videocodec *codec)
+{
+ struct codec_list *prev = NULL, *h = codeclist_top;
+
+ if (!codec) {
+ pr_err("%s: no data!\n", __func__);
+ return -EINVAL;
+ }
+
+ dprintk(2,
+ "videocodec: unregister '%s', type: %x, flags %lx, magic %lx\n",
+ codec->name, codec->type, codec->flags, codec->magic);
+
+ if (!h) {
+ pr_err("%s: no device left...\n", __func__);
+ return -ENXIO;
+ }
+
+ while (h) {
+ if (codec == h->codec) {
+ if (h->attached) {
+ pr_err("videocodec: '%s' is used\n", h->codec->name);
+ return -EBUSY;
+ }
+ dprintk(3, "videocodec: unregister '%s' is ok.\n",
+ h->codec->name);
+ if (!prev) {
+ codeclist_top = h->next;
+ dprintk(4,
+ "videocodec: delete first element\n");
+ } else {
+ prev->next = h->next;
+ dprintk(4,
+ "videocodec: delete middle element\n");
+ }
+ kfree(h);
+ return 0;
+ }
+ prev = h;
+ h = h->next;
+ }
+
+ pr_err("%s: given codec not found!\n", __func__);
+ return -EINVAL;
+}
+EXPORT_SYMBOL(videocodec_unregister);
+
+#ifdef CONFIG_PROC_FS
+static int proc_videocodecs_show(struct seq_file *m, void *v)
+{
+ struct codec_list *h = codeclist_top;
+ struct attached_list *a;
+
+ seq_printf(m, "<S>lave or attached <M>aster name type flags magic ");
+ seq_printf(m, "(connected as)\n");
+
+ while (h) {
+ seq_printf(m, "S %32s %04x %08lx %08lx (TEMPLATE)\n",
+ h->codec->name, h->codec->type,
+ h->codec->flags, h->codec->magic);
+ a = h->list;
+ while (a) {
+ seq_printf(m, "M %32s %04x %08lx %08lx (%s)\n",
+ a->codec->master_data->name,
+ a->codec->master_data->type,
+ a->codec->master_data->flags,
+ a->codec->master_data->magic,
+ a->codec->name);
+ a = a->next;
+ }
+ h = h->next;
+ }
+
+ return 0;
+}
+#endif
+
+/* ===================== */
+/* hook in driver module */
+/* ===================== */
+static int __init videocodec_init(void)
+{
+#ifdef CONFIG_PROC_FS
+ static struct proc_dir_entry *videocodec_proc_entry;
+#endif
+
+ pr_info("Linux video codec intermediate layer: %s\n", VIDEOCODEC_VERSION);
+
+#ifdef CONFIG_PROC_FS
+ videocodec_proc_entry = proc_create_single("videocodecs", 0, NULL, proc_videocodecs_show);
+ if (!videocodec_proc_entry)
+ pr_err("videocodec: can't init procfs.\n");
+#endif
+ return 0;
+}
+
+static void __exit videocodec_exit(void)
+{
+#ifdef CONFIG_PROC_FS
+ remove_proc_entry("videocodecs", NULL);
+#endif
+}
+
+module_init(videocodec_init);
+module_exit(videocodec_exit);
+
+MODULE_AUTHOR("Wolfgang Scherr <scherr@net4you.at>");
+MODULE_DESCRIPTION("Intermediate API module for video codecs "
+ VIDEOCODEC_VERSION);
+MODULE_LICENSE("GPL");
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * VIDEO MOTION CODECs internal API for video devices
+ *
+ * Interface for MJPEG (and maybe later MPEG/WAVELETS) codec's
+ * bound to a master device.
+ *
+ * (c) 2002 Wolfgang Scherr <scherr@net4you.at>
+ */
+
+/* =================== */
+/* general description */
+/* =================== */
+
+/* Should ease the (re-)usage of drivers supporting cards with (different)
+ video codecs. The codecs register to this module their functionality,
+ and the processors (masters) can attach to them if they fit.
+
+ The codecs are typically have a "strong" binding to their master - so I
+ don't think it makes sense to have a full blown interfacing as with e.g.
+ i2c. If you have an other opinion, let's discuss & implement it :-)))
+
+ Usage:
+
+ The slave has just to setup the videocodec structure and use two functions:
+ videocodec_register(codecdata);
+ videocodec_unregister(codecdata);
+ The best is just calling them at module (de-)initialisation.
+
+ The master sets up the structure videocodec_master and calls:
+ codecdata=videocodec_attach(master_codecdata);
+ videocodec_detach(codecdata);
+
+ The slave is called during attach/detach via functions setup previously
+ during register. At that time, the master_data pointer is set up
+ and the slave can access any io registers of the master device (in the case
+ the slave is bound to it). Otherwise it doesn't need this functions and
+ therfor they may not be initialized.
+
+ The other functions are just for convenience, as they are for sure used by
+ most/all of the codecs. The last ones may be omitted, too.
+
+ See the structure declaration below for more information and which data has
+ to be set up for the master and the slave.
+
+ ----------------------------------------------------------------------------
+ The master should have "knowledge" of the slave and vice versa. So the data
+ structures sent to/from slave via set_data/get_data set_image/get_image are
+ device dependent and vary between MJPEG/MPEG/WAVELET/... devices. (!!!!)
+ ----------------------------------------------------------------------------
+*/
+
+/* ========================================== */
+/* description of the videocodec_io structure */
+/* ========================================== */
+
+/*
+ ==== master setup ====
+ name -> name of the device structure for reference and debugging
+ master_data -> data ref. for the master (e.g. the zr36055,57,67)
+ readreg -> ref. to read-fn from register (setup by master, used by slave)
+ writereg -> ref. to write-fn to register (setup by master, used by slave)
+ this two functions do the lowlevel I/O job
+
+ ==== slave functionality setup ====
+ slave_data -> data ref. for the slave (e.g. the zr36050,60)
+ check -> fn-ref. checks availability of an device, returns -EIO on failure or
+ the type on success
+ this makes espcecially sense if a driver module supports more than
+ one codec which may be quite similar to access, nevertheless it
+ is good for a first functionality check
+
+ -- main functions you always need for compression/decompression --
+
+ set_mode -> this fn-ref. resets the entire codec, and sets up the mode
+ with the last defined norm/size (or device default if not
+ available) - it returns 0 if the mode is possible
+ set_size -> this fn-ref. sets the norm and image size for
+ compression/decompression (returns 0 on success)
+ the norm param is defined in videodev2.h (V4L2_STD_*)
+
+ additional setup may be available, too - but the codec should work with
+ some default values even without this
+
+ set_data -> sets device-specific data (tables, quality etc.)
+ get_data -> query device-specific data (tables, quality etc.)
+
+ if the device delivers interrupts, they may be setup/handled here
+ setup_interrupt -> codec irq setup (not needed for 36050/60)
+ handle_interrupt -> codec irq handling (not needed for 36050/60)
+
+ if the device delivers pictures, they may be handled here
+ put_image -> puts image data to the codec (not needed for 36050/60)
+ get_image -> gets image data from the codec (not needed for 36050/60)
+ the calls include frame numbers and flags (even/odd/...)
+ if needed and a flag which allows blocking until its ready
+*/
+
+/* ============== */
+/* user interface */
+/* ============== */
+
+/*
+ Currently there is only a information display planned, as the layer
+ is not visible for the user space at all.
+
+ Information is available via procfs. The current entry is "/proc/videocodecs"
+ but it makes sense to "hide" it in the /proc/video tree of v4l(2) --TODO--.
+
+A example for such an output is:
+
+<S>lave or attached <M>aster name type flags magic (connected as)
+S zr36050 0002 0000d001 00000000 (TEMPLATE)
+M zr36055[0] 0001 0000c001 00000000 (zr36050[0])
+M zr36055[1] 0001 0000c001 00000000 (zr36050[1])
+
+*/
+
+/* =============================================== */
+/* special defines for the videocodec_io structure */
+/* =============================================== */
+
+#ifndef __LINUX_VIDEOCODEC_H
+#define __LINUX_VIDEOCODEC_H
+
+#include <linux/videodev2.h>
+
+#define CODEC_DO_COMPRESSION 0
+#define CODEC_DO_EXPANSION 1
+
+/* this are the current codec flags I think they are needed */
+/* -> type value in structure */
+#define CODEC_FLAG_JPEG 0x00000001L // JPEG codec
+#define CODEC_FLAG_MPEG 0x00000002L // MPEG1/2/4 codec
+#define CODEC_FLAG_DIVX 0x00000004L // DIVX codec
+#define CODEC_FLAG_WAVELET 0x00000008L // WAVELET codec
+ // room for other types
+
+#define CODEC_FLAG_MAGIC 0x00000800L // magic key must match
+#define CODEC_FLAG_HARDWARE 0x00001000L // is a hardware codec
+#define CODEC_FLAG_VFE 0x00002000L // has direct video frontend
+#define CODEC_FLAG_ENCODER 0x00004000L // compression capability
+#define CODEC_FLAG_DECODER 0x00008000L // decompression capability
+#define CODEC_FLAG_NEEDIRQ 0x00010000L // needs irq handling
+#define CODEC_FLAG_RDWRPIC 0x00020000L // handles picture I/O
+
+/* a list of modes, some are just examples (is there any HW?) */
+#define CODEC_MODE_BJPG 0x0001 // Baseline JPEG
+#define CODEC_MODE_LJPG 0x0002 // Lossless JPEG
+#define CODEC_MODE_MPEG1 0x0003 // MPEG 1
+#define CODEC_MODE_MPEG2 0x0004 // MPEG 2
+#define CODEC_MODE_MPEG4 0x0005 // MPEG 4
+#define CODEC_MODE_MSDIVX 0x0006 // MS DivX
+#define CODEC_MODE_ODIVX 0x0007 // Open DivX
+#define CODEC_MODE_WAVELET 0x0008 // Wavelet
+
+/* this are the current codec types I want to implement */
+/* -> type value in structure */
+#define CODEC_TYPE_NONE 0
+#define CODEC_TYPE_L64702 1
+#define CODEC_TYPE_ZR36050 2
+#define CODEC_TYPE_ZR36016 3
+#define CODEC_TYPE_ZR36060 4
+
+/* the type of data may be enhanced by future implementations (data-fn.'s) */
+/* -> used in command */
+#define CODEC_G_STATUS 0x0000 /* codec status (query only) */
+#define CODEC_S_CODEC_MODE 0x0001 /* codec mode (baseline JPEG, MPEG1,... */
+#define CODEC_G_CODEC_MODE 0x8001
+#define CODEC_S_VFE 0x0002 /* additional video frontend setup */
+#define CODEC_G_VFE 0x8002
+#define CODEC_S_MMAP 0x0003 /* MMAP setup (if available) */
+
+#define CODEC_S_JPEG_TDS_BYTE 0x0010 /* target data size in bytes */
+#define CODEC_G_JPEG_TDS_BYTE 0x8010
+#define CODEC_S_JPEG_SCALE 0x0011 /* scaling factor for quant. tables */
+#define CODEC_G_JPEG_SCALE 0x8011
+#define CODEC_S_JPEG_HDT_DATA 0x0018 /* huffman-tables */
+#define CODEC_G_JPEG_HDT_DATA 0x8018
+#define CODEC_S_JPEG_QDT_DATA 0x0019 /* quantizing-tables */
+#define CODEC_G_JPEG_QDT_DATA 0x8019
+#define CODEC_S_JPEG_APP_DATA 0x001A /* APP marker */
+#define CODEC_G_JPEG_APP_DATA 0x801A
+#define CODEC_S_JPEG_COM_DATA 0x001B /* COM marker */
+#define CODEC_G_JPEG_COM_DATA 0x801B
+
+#define CODEC_S_PRIVATE 0x1000 /* "private" commands start here */
+#define CODEC_G_PRIVATE 0x9000
+
+#define CODEC_G_FLAG 0x8000 /* this is how 'get' is detected */
+
+/* types of transfer, directly user space or a kernel buffer (image-fn.'s) */
+/* -> used in get_image, put_image */
+#define CODEC_TRANSFER_KERNEL 0 /* use "memcopy" */
+#define CODEC_TRANSFER_USER 1 /* use "to/from_user" */
+
+/* ========================= */
+/* the structures itself ... */
+/* ========================= */
+
+struct vfe_polarity {
+ unsigned int vsync_pol:1;
+ unsigned int hsync_pol:1;
+ unsigned int field_pol:1;
+ unsigned int blank_pol:1;
+ unsigned int subimg_pol:1;
+ unsigned int poe_pol:1;
+ unsigned int pvalid_pol:1;
+ unsigned int vclk_pol:1;
+};
+
+struct vfe_settings {
+ __u32 x, y; /* Offsets into image */
+ __u32 width, height; /* Area to capture */
+ __u16 decimation; /* Decimation divider */
+ __u16 flags; /* Flags for capture */
+ __u16 quality; /* quality of the video */
+};
+
+struct tvnorm {
+ u16 wt, wa, h_start, h_sync_start, ht, ha, v_start;
+};
+
+struct jpeg_com_marker {
+ int len; /* number of usable bytes in data */
+ char data[60];
+};
+
+struct jpeg_app_marker {
+ int appn; /* number app segment */
+ int len; /* number of usable bytes in data */
+ char data[60];
+};
+
+struct videocodec {
+ struct module *owner;
+ /* -- filled in by slave device during register -- */
+ char name[32];
+ unsigned long magic; /* may be used for client<->master attaching */
+ unsigned long flags; /* functionality flags */
+ unsigned int type; /* codec type */
+
+ /* -- these is filled in later during master device attach -- */
+
+ struct videocodec_master *master_data;
+
+ /* -- these are filled in by the slave device during register -- */
+
+ void *data; /* private slave data */
+
+ /* attach/detach client functions (indirect call) */
+ int (*setup)(struct videocodec *codec);
+ int (*unset)(struct videocodec *codec);
+
+ /* main functions, every client needs them for sure! */
+ // set compression or decompression (or freeze, stop, standby, etc)
+ int (*set_mode)(struct videocodec *codec, int mode);
+ // setup picture size and norm (for the codec's video frontend)
+ int (*set_video)(struct videocodec *codec, const struct tvnorm *norm,
+ struct vfe_settings *cap, struct vfe_polarity *pol);
+ // other control commands, also mmap setup etc.
+ int (*control)(struct videocodec *codec, int type, int size, void *data);
+
+ /* additional setup/query/processing (may be NULL pointer) */
+ // interrupt setup / handling (for irq's delivered by master)
+ int (*setup_interrupt)(struct videocodec *codec, long mode);
+ int (*handle_interrupt)(struct videocodec *codec, int source, long flag);
+ // picture interface (if any)
+ long (*put_image)(struct videocodec *codec, int tr_type, int block,
+ long *fr_num, long *flag, long size, void *buf);
+ long (*get_image)(struct videocodec *codec, int tr_type, int block,
+ long *fr_num, long *flag, long size, void *buf);
+};
+
+struct videocodec_master {
+ /* -- filled in by master device for registration -- */
+ char name[32];
+ unsigned long magic; /* may be used for client<->master attaching */
+ unsigned long flags; /* functionality flags */
+ unsigned int type; /* master type */
+
+ void *data; /* private master data */
+
+ __u32 (*readreg)(struct videocodec *codec, __u16 reg);
+ void (*writereg)(struct videocodec *codec, __u16 reg, __u32 value);
+};
+
+/* ================================================= */
+/* function prototypes of the master/slave interface */
+/* ================================================= */
+
+/* attach and detach commands for the master */
+// * master structure needs to be kmalloc'ed before calling attach
+// and free'd after calling detach
+// * returns pointer on success, NULL on failure
+extern struct videocodec *videocodec_attach(struct videocodec_master *);
+// * 0 on success, <0 (errno) on failure
+extern int videocodec_detach(struct videocodec *);
+
+/* register and unregister commands for the slaves */
+// * 0 on success, <0 (errno) on failure
+extern int videocodec_register(const struct videocodec *);
+// * 0 on success, <0 (errno) on failure
+extern int videocodec_unregister(const struct videocodec *);
+
+/* the other calls are directly done via the videocodec structure! */
+
+#endif /*ifndef __LINUX_VIDEOCODEC_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * zoran - Iomega Buz driver
+ *
+ * Copyright (C) 1999 Rainer Johanni <Rainer@Johanni.de>
+ *
+ * based on
+ *
+ * zoran.0.0.3 Copyright (C) 1998 Dave Perks <dperks@ibm.net>
+ *
+ * and
+ *
+ * bttv - Bt848 frame grabber driver
+ * Copyright (C) 1996,97,98 Ralph Metzler (rjkm@thp.uni-koeln.de)
+ * & Marcus Metzler (mocm@thp.uni-koeln.de)
+ */
+
+#ifndef _BUZ_H_
+#define _BUZ_H_
+
+#include <media/v4l2-device.h>
+#include <media/v4l2-ctrls.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/videobuf2-dma-contig.h>
+
+#define ZR_NORM_PAL 0
+#define ZR_NORM_NTSC 1
+#define ZR_NORM_SECAM 2
+
+struct zr_buffer {
+ /* common v4l buffer stuff -- must be first */
+ struct vb2_v4l2_buffer vbuf;
+ struct list_head queue;
+};
+
+static inline struct zr_buffer *vb2_to_zr_buffer(struct vb2_buffer *vb)
+{
+ struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+
+ return container_of(vbuf, struct zr_buffer, vbuf);
+}
+
+#define ZORAN_NAME "ZORAN" /* name of the device */
+
+#define ZR_DEVNAME(zr) ((zr)->name)
+
+#define BUZ_MAX_WIDTH (zr->timing->wa)
+#define BUZ_MAX_HEIGHT (zr->timing->ha)
+#define BUZ_MIN_WIDTH 32 /* never display less than 32 pixels */
+#define BUZ_MIN_HEIGHT 24 /* never display less than 24 rows */
+
+#define BUZ_NUM_STAT_COM 4
+#define BUZ_MASK_STAT_COM 3
+
+#define BUZ_MAX_FRAME 256 /* Must be a power of 2 */
+#define BUZ_MASK_FRAME 255 /* Must be BUZ_MAX_FRAME-1 */
+
+#define BUZ_MAX_INPUT 16
+
+#if VIDEO_MAX_FRAME <= 32
+# define V4L_MAX_FRAME 32
+#elif VIDEO_MAX_FRAME <= 64
+# define V4L_MAX_FRAME 64
+#else
+# error "Too many video frame buffers to handle"
+#endif
+#define V4L_MASK_FRAME (V4L_MAX_FRAME - 1)
+
+#define MAX_FRAME (BUZ_MAX_FRAME > VIDEO_MAX_FRAME ? BUZ_MAX_FRAME : VIDEO_MAX_FRAME)
+
+#include "zr36057.h"
+
+enum card_type {
+ UNKNOWN = -1,
+
+ /* Pinnacle/Miro */
+ DC10_OLD, /* DC30 like */
+ DC10_NEW, /* DC10_PLUS like */
+ DC10_PLUS,
+ DC30,
+ DC30_PLUS,
+
+ /* Linux Media Labs */
+ LML33,
+ LML33R10,
+
+ /* Iomega */
+ BUZ,
+
+ /* AverMedia */
+ AVS6EYES,
+
+ /* total number of cards */
+ NUM_CARDS
+};
+
+enum zoran_codec_mode {
+ BUZ_MODE_IDLE, /* nothing going on */
+ BUZ_MODE_MOTION_COMPRESS, /* grabbing frames */
+ BUZ_MODE_MOTION_DECOMPRESS, /* playing frames */
+ BUZ_MODE_STILL_COMPRESS, /* still frame conversion */
+ BUZ_MODE_STILL_DECOMPRESS /* still frame conversion */
+};
+
+enum zoran_map_mode {
+ ZORAN_MAP_MODE_NONE,
+ ZORAN_MAP_MODE_RAW,
+ ZORAN_MAP_MODE_JPG_REC,
+ ZORAN_MAP_MODE_JPG_PLAY,
+};
+
+enum gpio_type {
+ ZR_GPIO_JPEG_SLEEP = 0,
+ ZR_GPIO_JPEG_RESET,
+ ZR_GPIO_JPEG_FRAME,
+ ZR_GPIO_VID_DIR,
+ ZR_GPIO_VID_EN,
+ ZR_GPIO_VID_RESET,
+ ZR_GPIO_CLK_SEL1,
+ ZR_GPIO_CLK_SEL2,
+ ZR_GPIO_MAX,
+};
+
+enum gpcs_type {
+ GPCS_JPEG_RESET = 0,
+ GPCS_JPEG_START,
+ GPCS_MAX,
+};
+
+struct zoran_format {
+ char *name;
+ __u32 fourcc;
+ int colorspace;
+ int depth;
+ __u32 flags;
+ __u32 vfespfr;
+};
+
+/* flags */
+#define ZORAN_FORMAT_COMPRESSED BIT(0)
+#define ZORAN_FORMAT_OVERLAY BIT(1)
+#define ZORAN_FORMAT_CAPTURE BIT(2)
+#define ZORAN_FORMAT_PLAYBACK BIT(3)
+
+/* v4l-capture settings */
+struct zoran_v4l_settings {
+ int width, height, bytesperline; /* capture size */
+ const struct zoran_format *format; /* capture format */
+};
+
+/* jpg-capture/-playback settings */
+struct zoran_jpg_settings {
+ int decimation; /* this bit is used to set everything to default */
+ int hor_dcm, ver_dcm, tmp_dcm; /* capture decimation settings (tmp_dcm=1 means both fields) */
+ int field_per_buff, odd_even; /* field-settings (odd_even=1 (+tmp_dcm=1) means top-field-first) */
+ int img_x, img_y, img_width, img_height; /* crop settings (subframe capture) */
+ struct v4l2_jpegcompression jpg_comp; /* JPEG-specific capture settings */
+};
+
+
+struct zoran;
+
+/* zoran_fh contains per-open() settings */
+struct zoran_fh {
+ struct v4l2_fh fh;
+ struct zoran *zr;
+};
+
+struct card_info {
+ enum card_type type;
+ char name[32];
+ const char *i2c_decoder; /* i2c decoder device */
+ const unsigned short *addrs_decoder;
+ const char *i2c_encoder; /* i2c encoder device */
+ const unsigned short *addrs_encoder;
+ u16 video_vfe, video_codec; /* videocodec types */
+ u16 audio_chip; /* audio type */
+
+ int inputs; /* number of video inputs */
+ struct input {
+ int muxsel;
+ char name[32];
+ } input[BUZ_MAX_INPUT];
+
+ v4l2_std_id norms;
+ const struct tvnorm *tvn[3]; /* supported TV norms */
+
+ u32 jpeg_int; /* JPEG interrupt */
+ u32 vsync_int; /* VSYNC interrupt */
+ s8 gpio[ZR_GPIO_MAX];
+ u8 gpcs[GPCS_MAX];
+
+ struct vfe_polarity vfe_pol;
+ u8 gpio_pol[ZR_GPIO_MAX];
+
+ /* is the /GWS line connected? */
+ u8 gws_not_connected;
+
+ /* avs6eyes mux setting */
+ u8 input_mux;
+
+ void (*init)(struct zoran *zr);
+};
+
+struct zoran {
+ struct v4l2_device v4l2_dev;
+ struct v4l2_ctrl_handler hdl;
+ struct video_device *video_dev;
+ struct vb2_queue vq;
+
+ struct i2c_adapter i2c_adapter; /* */
+ struct i2c_algo_bit_data i2c_algo; /* */
+ u32 i2cbr;
+
+ struct v4l2_subdev *decoder; /* video decoder sub-device */
+ struct v4l2_subdev *encoder; /* video encoder sub-device */
+
+ struct videocodec *codec; /* video codec */
+ struct videocodec *vfe; /* video front end */
+
+ struct mutex lock; /* file ops serialize lock */
+
+ u8 initialized; /* flag if zoran has been correctly initialized */
+ struct card_info card;
+ const struct tvnorm *timing;
+
+ unsigned short id; /* number of this device */
+ char name[32]; /* name of this device */
+ struct pci_dev *pci_dev; /* PCI device */
+ unsigned char revision; /* revision of zr36057 */
+ unsigned char __iomem *zr36057_mem;/* pointer to mapped IO memory */
+
+ spinlock_t spinlock; /* Spinlock */
+
+ /* Video for Linux parameters */
+ int input; /* card's norm and input */
+ v4l2_std_id norm;
+
+ /* Current buffer params */
+ unsigned int buffer_size;
+
+ struct zoran_v4l_settings v4l_settings; /* structure with a lot of things to play with */
+
+ /* Buz MJPEG parameters */
+ enum zoran_codec_mode codec_mode; /* status of codec */
+ struct zoran_jpg_settings jpg_settings; /* structure with a lot of things to play with */
+
+ /* grab queue counts/indices, mask with BUZ_MASK_STAT_COM before using as index */
+ /* (dma_head - dma_tail) is number active in DMA, must be <= BUZ_NUM_STAT_COM */
+ /* (value & BUZ_MASK_STAT_COM) corresponds to index in stat_com table */
+ unsigned long jpg_que_head; /* Index where to put next buffer which is queued */
+ unsigned long jpg_dma_head; /* Index of next buffer which goes into stat_com */
+ unsigned long jpg_dma_tail; /* Index of last buffer in stat_com */
+ unsigned long jpg_que_tail; /* Index of last buffer in queue */
+ unsigned long jpg_seq_num; /* count of frames since grab/play started */
+ unsigned long jpg_err_seq; /* last seq_num before error */
+ unsigned long jpg_err_shift;
+ unsigned long jpg_queued_num; /* count of frames queued since grab/play started */
+ unsigned long vbseq;
+
+ /* zr36057's code buffer table */
+ __le32 *stat_com; /* stat_com[i] is indexed by dma_head/tail & BUZ_MASK_STAT_COM */
+
+ /* Additional stuff for testing */
+ unsigned int ghost_int;
+ int intr_counter_GIRQ1;
+ int intr_counter_GIRQ0;
+ int intr_counter_cod_rep_irq;
+ int intr_counter_jpeg_rep_irq;
+ int field_counter;
+ int irq1_in;
+ int irq1_out;
+ int jpeg_in;
+ int jpeg_out;
+ int JPEG_0;
+ int JPEG_1;
+ int end_event_missed;
+ int jpeg_missed;
+ int jpeg_error;
+ int num_errors;
+ int jpeg_max_missed;
+ int jpeg_min_missed;
+ unsigned int prepared;
+ unsigned int queued;
+
+ u32 last_isr;
+ unsigned long frame_num;
+ int running;
+ int buf_in_reserve;
+
+ dma_addr_t p_sc;
+ __le32 *stat_comb;
+ dma_addr_t p_scb;
+ enum zoran_map_mode map_mode;
+ struct list_head queued_bufs;
+ spinlock_t queued_bufs_lock; /* Protects queued_bufs */
+ struct zr_buffer *inuse[BUZ_NUM_STAT_COM * 2];
+};
+
+static inline struct zoran *to_zoran(struct v4l2_device *v4l2_dev)
+{
+ return container_of(v4l2_dev, struct zoran, v4l2_dev);
+}
+
+/* There was something called _ALPHA_BUZ that used the PCI address instead of
+ * the kernel iomapped address for btread/btwrite. */
+#define btwrite(dat, adr) writel((dat), zr->zr36057_mem + (adr))
+#define btread(adr) readl(zr->zr36057_mem + (adr))
+
+#define btand(dat, adr) btwrite((dat) & btread(adr), adr)
+#define btor(dat, adr) btwrite((dat) | btread(adr), adr)
+#define btaor(dat, mask, adr) btwrite((dat) | ((mask) & btread(adr)), adr)
+
+#endif
+
+int zoran_queue_init(struct zoran *zr, struct vb2_queue *vq);
+void zoran_queue_exit(struct zoran *zr);
+int zr_set_buf(struct zoran *zr);
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Zoran zr36057/zr36067 PCI controller driver, for the
+ * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
+ * Media Labs LML33/LML33R10.
+ *
+ * This part handles card-specific data and detection
+ *
+ * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
+ */
+
+#include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+
+#include <linux/i2c.h>
+#include <linux/i2c-algo-bit.h>
+#include <linux/videodev2.h>
+#include <linux/spinlock.h>
+
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <media/v4l2-common.h>
+#include <media/i2c/bt819.h>
+
+#include "videocodec.h"
+#include "zoran.h"
+#include "zoran_card.h"
+#include "zoran_device.h"
+
+extern const struct zoran_format zoran_formats[];
+
+static int card[BUZ_MAX] = { [0 ... (BUZ_MAX - 1)] = -1 };
+module_param_array(card, int, NULL, 0444);
+MODULE_PARM_DESC(card, "Card type");
+
+/*
+ * The video mem address of the video card. The driver has a little database for some videocards
+ * to determine it from there. If your video card is not in there you have either to give it to
+ * the driver as a parameter or set in in a VIDIOCSFBUF ioctl
+ */
+
+static unsigned long vidmem; /* default = 0 - Video memory base address */
+module_param_hw(vidmem, ulong, iomem, 0444);
+MODULE_PARM_DESC(vidmem, "Default video memory base address");
+
+/* Default input and video norm at startup of the driver. */
+
+static unsigned int default_input; /* default 0 = Composite, 1 = S-Video */
+module_param(default_input, uint, 0444);
+MODULE_PARM_DESC(default_input,
+ "Default input (0=Composite, 1=S-Video, 2=Internal)");
+
+static int default_mux = 1; /* 6 Eyes input selection */
+module_param(default_mux, int, 0644);
+MODULE_PARM_DESC(default_mux,
+ "Default 6 Eyes mux setting (Input selection)");
+
+static int default_norm; /* default 0 = PAL, 1 = NTSC 2 = SECAM */
+module_param(default_norm, int, 0444);
+MODULE_PARM_DESC(default_norm, "Default norm (0=PAL, 1=NTSC, 2=SECAM)");
+
+/* /dev/videoN, -1 for autodetect */
+static int video_nr[BUZ_MAX] = { [0 ... (BUZ_MAX - 1)] = -1 };
+module_param_array(video_nr, int, NULL, 0444);
+MODULE_PARM_DESC(video_nr, "Video device number (-1=Auto)");
+
+int v4l_nbufs = 4;
+int v4l_bufsize = 864; /* Everybody should be able to work with this setting */
+module_param(v4l_nbufs, int, 0644);
+MODULE_PARM_DESC(v4l_nbufs, "Maximum number of V4L buffers to use");
+module_param(v4l_bufsize, int, 0644);
+MODULE_PARM_DESC(v4l_bufsize, "Maximum size per V4L buffer (in kB)");
+
+int jpg_nbufs = 32;
+int jpg_bufsize = 512; /* max size for 100% quality full-PAL frame */
+module_param(jpg_nbufs, int, 0644);
+MODULE_PARM_DESC(jpg_nbufs, "Maximum number of JPG buffers to use");
+module_param(jpg_bufsize, int, 0644);
+MODULE_PARM_DESC(jpg_bufsize, "Maximum size per JPG buffer (in kB)");
+
+/* 1=Pass through TV signal when device is not used */
+/* 0=Show color bar when device is not used (LML33: only if lml33dpath=1) */
+int pass_through;
+module_param(pass_through, int, 0644);
+MODULE_PARM_DESC(pass_through,
+ "Pass TV signal through to TV-out when idling");
+
+int zr36067_debug = 1;
+module_param_named(debug, zr36067_debug, int, 0644);
+MODULE_PARM_DESC(debug, "Debug level (0-5)");
+
+#define ZORAN_VERSION "0.10.1"
+
+MODULE_DESCRIPTION("Zoran-36057/36067 JPEG codec driver");
+MODULE_AUTHOR("Serguei Miridonov");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(ZORAN_VERSION);
+
+#define ZR_DEVICE(subven, subdev, data) { \
+ .vendor = PCI_VENDOR_ID_ZORAN, .device = PCI_DEVICE_ID_ZORAN_36057, \
+ .subvendor = (subven), .subdevice = (subdev), .driver_data = (data) }
+
+static const struct pci_device_id zr36067_pci_tbl[] = {
+ ZR_DEVICE(PCI_VENDOR_ID_MIRO, PCI_DEVICE_ID_MIRO_DC10PLUS, DC10_PLUS),
+ ZR_DEVICE(PCI_VENDOR_ID_MIRO, PCI_DEVICE_ID_MIRO_DC30PLUS, DC30_PLUS),
+ ZR_DEVICE(PCI_VENDOR_ID_ELECTRONICDESIGNGMBH, PCI_DEVICE_ID_LML_33R10, LML33R10),
+ ZR_DEVICE(PCI_VENDOR_ID_IOMEGA, PCI_DEVICE_ID_IOMEGA_BUZ, BUZ),
+ ZR_DEVICE(PCI_ANY_ID, PCI_ANY_ID, NUM_CARDS),
+ {0}
+};
+MODULE_DEVICE_TABLE(pci, zr36067_pci_tbl);
+
+static unsigned int zoran_num; /* number of cards found */
+
+/* videocodec bus functions ZR36060 */
+static u32 zr36060_read(struct videocodec *codec, u16 reg)
+{
+ struct zoran *zr = (struct zoran *)codec->master_data->data;
+ __u32 data;
+
+ if (post_office_wait(zr) || post_office_write(zr, 0, 1, reg >> 8) ||
+ post_office_write(zr, 0, 2, reg & 0xff))
+ return -1;
+
+ data = post_office_read(zr, 0, 3) & 0xff;
+ return data;
+}
+
+static void zr36060_write(struct videocodec *codec, u16 reg, u32 val)
+{
+ struct zoran *zr = (struct zoran *)codec->master_data->data;
+
+ if (post_office_wait(zr) || post_office_write(zr, 0, 1, reg >> 8) ||
+ post_office_write(zr, 0, 2, reg & 0xff))
+ return;
+
+ post_office_write(zr, 0, 3, val & 0xff);
+}
+
+/* videocodec bus functions ZR36050 */
+static u32 zr36050_read(struct videocodec *codec, u16 reg)
+{
+ struct zoran *zr = (struct zoran *)codec->master_data->data;
+ __u32 data;
+
+ if (post_office_wait(zr) || post_office_write(zr, 1, 0, reg >> 2)) // reg. HIGHBYTES
+ return -1;
+
+ data = post_office_read(zr, 0, reg & 0x03) & 0xff; // reg. LOWBYTES + read
+ return data;
+}
+
+static void zr36050_write(struct videocodec *codec, u16 reg, u32 val)
+{
+ struct zoran *zr = (struct zoran *)codec->master_data->data;
+
+ if (post_office_wait(zr) || post_office_write(zr, 1, 0, reg >> 2)) // reg. HIGHBYTES
+ return;
+
+ post_office_write(zr, 0, reg & 0x03, val & 0xff); // reg. LOWBYTES + wr. data
+}
+
+/* videocodec bus functions ZR36016 */
+static u32 zr36016_read(struct videocodec *codec, u16 reg)
+{
+ struct zoran *zr = (struct zoran *)codec->master_data->data;
+ __u32 data;
+
+ if (post_office_wait(zr))
+ return -1;
+
+ data = post_office_read(zr, 2, reg & 0x03) & 0xff; // read
+ return data;
+}
+
+/* hack for in zoran_device.c */
+void zr36016_write(struct videocodec *codec, u16 reg, u32 val)
+{
+ struct zoran *zr = (struct zoran *)codec->master_data->data;
+
+ if (post_office_wait(zr))
+ return;
+
+ post_office_write(zr, 2, reg & 0x03, val & 0x0ff); // wr. data
+}
+
+/*
+ * Board specific information
+ */
+
+static void dc10_init(struct zoran *zr)
+{
+ pci_dbg(zr->pci_dev, "%s\n", __func__);
+
+ /* Pixel clock selection */
+ GPIO(zr, 4, 0);
+ GPIO(zr, 5, 1);
+ /* Enable the video bus sync signals */
+ GPIO(zr, 7, 0);
+}
+
+static void dc10plus_init(struct zoran *zr)
+{
+ pci_dbg(zr->pci_dev, "%s\n", __func__);
+}
+
+static void buz_init(struct zoran *zr)
+{
+ pci_dbg(zr->pci_dev, "%s\n", __func__);
+
+ /* some stuff from Iomega */
+ pci_write_config_dword(zr->pci_dev, 0xfc, 0x90680f15);
+ pci_write_config_dword(zr->pci_dev, 0x0c, 0x00012020);
+ pci_write_config_dword(zr->pci_dev, 0xe8, 0xc0200000);
+}
+
+static void lml33_init(struct zoran *zr)
+{
+ pci_dbg(zr->pci_dev, "%s\n", __func__);
+
+ GPIO(zr, 2, 1); // Set Composite input/output
+}
+
+static void avs6eyes_init(struct zoran *zr)
+{
+ // AverMedia 6-Eyes original driver by Christer Weinigel
+
+ // Lifted straight from Christer's old driver and
+ // modified slightly by Martin Samuelsson.
+
+ int mux = default_mux; /* 1 = BT866, 7 = VID1 */
+
+ GPIO(zr, 4, 1); /* Bt866 SLEEP on */
+ udelay(2);
+
+ GPIO(zr, 0, 1); /* ZR36060 /RESET on */
+ GPIO(zr, 1, 0); /* ZR36060 /SLEEP on */
+ GPIO(zr, 2, mux & 1); /* MUX S0 */
+ GPIO(zr, 3, 0); /* /FRAME on */
+ GPIO(zr, 4, 0); /* Bt866 SLEEP off */
+ GPIO(zr, 5, mux & 2); /* MUX S1 */
+ GPIO(zr, 6, 0); /* ? */
+ GPIO(zr, 7, mux & 4); /* MUX S2 */
+}
+
+static const char *codecid_to_modulename(u16 codecid)
+{
+ const char *name = NULL;
+
+ switch (codecid) {
+ case CODEC_TYPE_ZR36060:
+ name = "zr36060";
+ break;
+ case CODEC_TYPE_ZR36050:
+ name = "zr36050";
+ break;
+ case CODEC_TYPE_ZR36016:
+ name = "zr36016";
+ break;
+ }
+
+ return name;
+}
+
+// struct tvnorm {
+// u16 wt, wa, h_start, h_sync_start, ht, ha, v_start;
+// };
+
+static const struct tvnorm f50sqpixel = { 944, 768, 83, 880, 625, 576, 16 };
+static const struct tvnorm f60sqpixel = { 780, 640, 51, 716, 525, 480, 12 };
+static const struct tvnorm f50ccir601 = { 864, 720, 75, 804, 625, 576, 18 };
+static const struct tvnorm f60ccir601 = { 858, 720, 57, 788, 525, 480, 16 };
+
+static const struct tvnorm f50ccir601_lml33 = { 864, 720, 75 + 34, 804, 625, 576, 18 };
+static const struct tvnorm f60ccir601_lml33 = { 858, 720, 57 + 34, 788, 525, 480, 16 };
+
+/* The DC10 (57/16/50) uses VActive as HSync, so h_start must be 0 */
+static const struct tvnorm f50sqpixel_dc10 = { 944, 768, 0, 880, 625, 576, 0 };
+static const struct tvnorm f60sqpixel_dc10 = { 780, 640, 0, 716, 525, 480, 12 };
+
+/*
+ * FIXME: I cannot swap U and V in saa7114, so i do one pixel left shift in zoran (75 -> 74)
+ * (Maxim Yevtyushkin <max@linuxmedialabs.com>)
+ */
+static const struct tvnorm f50ccir601_lm33r10 = { 864, 720, 74 + 54, 804, 625, 576, 18 };
+static const struct tvnorm f60ccir601_lm33r10 = { 858, 720, 56 + 54, 788, 525, 480, 16 };
+
+/*
+ * FIXME: The ks0127 seem incapable of swapping U and V, too, which is why I copy Maxim's left
+ * shift hack for the 6 Eyes.
+ *
+ * Christer's driver used the unshifted norms, though...
+ * /Sam
+ */
+static const struct tvnorm f50ccir601_avs6eyes = { 864, 720, 74, 804, 625, 576, 18 };
+static const struct tvnorm f60ccir601_avs6eyes = { 858, 720, 56, 788, 525, 480, 16 };
+
+static const unsigned short vpx3220_addrs[] = { 0x43, 0x47, I2C_CLIENT_END };
+static const unsigned short saa7110_addrs[] = { 0x4e, 0x4f, I2C_CLIENT_END };
+static const unsigned short saa7111_addrs[] = { 0x25, 0x24, I2C_CLIENT_END };
+static const unsigned short saa7114_addrs[] = { 0x21, 0x20, I2C_CLIENT_END };
+static const unsigned short adv717x_addrs[] = { 0x6a, 0x6b, 0x2a, 0x2b, I2C_CLIENT_END };
+static const unsigned short ks0127_addrs[] = { 0x6c, 0x6d, I2C_CLIENT_END };
+static const unsigned short saa7185_addrs[] = { 0x44, I2C_CLIENT_END };
+static const unsigned short bt819_addrs[] = { 0x45, I2C_CLIENT_END };
+static const unsigned short bt856_addrs[] = { 0x44, I2C_CLIENT_END };
+static const unsigned short bt866_addrs[] = { 0x44, I2C_CLIENT_END };
+
+static struct card_info zoran_cards[NUM_CARDS] = {
+ {
+ .type = DC10_OLD,
+ .name = "DC10(old)",
+ .i2c_decoder = "vpx3220a",
+ .addrs_decoder = vpx3220_addrs,
+ .video_codec = CODEC_TYPE_ZR36050,
+ .video_vfe = CODEC_TYPE_ZR36016,
+
+ .inputs = 3,
+ .input = {
+ { 1, "Composite" },
+ { 2, "S-Video" },
+ { 0, "Internal/comp" }
+ },
+ .norms = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM,
+ .tvn = {
+ &f50sqpixel_dc10,
+ &f60sqpixel_dc10,
+ &f50sqpixel_dc10
+ },
+ .jpeg_int = 0,
+ .vsync_int = ZR36057_ISR_GIRQ1,
+ .gpio = { 2, 1, -1, 3, 7, 0, 4, 5 },
+ .gpio_pol = { 0, 0, 0, 1, 0, 0, 0, 0 },
+ .gpcs = { -1, 0 },
+ .vfe_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
+ .gws_not_connected = 0,
+ .input_mux = 0,
+ .init = &dc10_init,
+ }, {
+ .type = DC10_NEW,
+ .name = "DC10(new)",
+ .i2c_decoder = "saa7110",
+ .addrs_decoder = saa7110_addrs,
+ .i2c_encoder = "adv7175",
+ .addrs_encoder = adv717x_addrs,
+ .video_codec = CODEC_TYPE_ZR36060,
+
+ .inputs = 3,
+ .input = {
+ { 0, "Composite" },
+ { 7, "S-Video" },
+ { 5, "Internal/comp" }
+ },
+ .norms = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM,
+ .tvn = {
+ &f50sqpixel,
+ &f60sqpixel,
+ &f50sqpixel},
+ .jpeg_int = ZR36057_ISR_GIRQ0,
+ .vsync_int = ZR36057_ISR_GIRQ1,
+ .gpio = { 3, 0, 6, 1, 2, -1, 4, 5 },
+ .gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
+ .gpcs = { -1, 1},
+ .vfe_pol = { 1, 1, 1, 1, 0, 0, 0, 0 },
+ .gws_not_connected = 0,
+ .input_mux = 0,
+ .init = &dc10plus_init,
+ }, {
+ .type = DC10_PLUS,
+ .name = "DC10_PLUS",
+ .i2c_decoder = "saa7110",
+ .addrs_decoder = saa7110_addrs,
+ .i2c_encoder = "adv7175",
+ .addrs_encoder = adv717x_addrs,
+ .video_codec = CODEC_TYPE_ZR36060,
+
+ .inputs = 3,
+ .input = {
+ { 0, "Composite" },
+ { 7, "S-Video" },
+ { 5, "Internal/comp" }
+ },
+ .norms = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM,
+ .tvn = {
+ &f50sqpixel,
+ &f60sqpixel,
+ &f50sqpixel
+ },
+ .jpeg_int = ZR36057_ISR_GIRQ0,
+ .vsync_int = ZR36057_ISR_GIRQ1,
+ .gpio = { 3, 0, 6, 1, 2, -1, 4, 5 },
+ .gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
+ .gpcs = { -1, 1 },
+ .vfe_pol = { 1, 1, 1, 1, 0, 0, 0, 0 },
+ .gws_not_connected = 0,
+ .input_mux = 0,
+ .init = &dc10plus_init,
+ }, {
+ .type = DC30,
+ .name = "DC30",
+ .i2c_decoder = "vpx3220a",
+ .addrs_decoder = vpx3220_addrs,
+ .i2c_encoder = "adv7175",
+ .addrs_encoder = adv717x_addrs,
+ .video_codec = CODEC_TYPE_ZR36050,
+ .video_vfe = CODEC_TYPE_ZR36016,
+
+ .inputs = 3,
+ .input = {
+ { 1, "Composite" },
+ { 2, "S-Video" },
+ { 0, "Internal/comp" }
+ },
+ .norms = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM,
+ .tvn = {
+ &f50sqpixel_dc10,
+ &f60sqpixel_dc10,
+ &f50sqpixel_dc10
+ },
+ .jpeg_int = 0,
+ .vsync_int = ZR36057_ISR_GIRQ1,
+ .gpio = { 2, 1, -1, 3, 7, 0, 4, 5 },
+ .gpio_pol = { 0, 0, 0, 1, 0, 0, 0, 0 },
+ .gpcs = { -1, 0 },
+ .vfe_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
+ .gws_not_connected = 0,
+ .input_mux = 0,
+ .init = &dc10_init,
+ }, {
+ .type = DC30_PLUS,
+ .name = "DC30_PLUS",
+ .i2c_decoder = "vpx3220a",
+ .addrs_decoder = vpx3220_addrs,
+ .i2c_encoder = "adv7175",
+ .addrs_encoder = adv717x_addrs,
+ .video_codec = CODEC_TYPE_ZR36050,
+ .video_vfe = CODEC_TYPE_ZR36016,
+
+ .inputs = 3,
+ .input = {
+ { 1, "Composite" },
+ { 2, "S-Video" },
+ { 0, "Internal/comp" }
+ },
+ .norms = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM,
+ .tvn = {
+ &f50sqpixel_dc10,
+ &f60sqpixel_dc10,
+ &f50sqpixel_dc10
+ },
+ .jpeg_int = 0,
+ .vsync_int = ZR36057_ISR_GIRQ1,
+ .gpio = { 2, 1, -1, 3, 7, 0, 4, 5 },
+ .gpio_pol = { 0, 0, 0, 1, 0, 0, 0, 0 },
+ .gpcs = { -1, 0 },
+ .vfe_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
+ .gws_not_connected = 0,
+ .input_mux = 0,
+ .init = &dc10_init,
+ }, {
+ .type = LML33,
+ .name = "LML33",
+ .i2c_decoder = "bt819a",
+ .addrs_decoder = bt819_addrs,
+ .i2c_encoder = "bt856",
+ .addrs_encoder = bt856_addrs,
+ .video_codec = CODEC_TYPE_ZR36060,
+
+ .inputs = 2,
+ .input = {
+ { 0, "Composite" },
+ { 7, "S-Video" }
+ },
+ .norms = V4L2_STD_NTSC | V4L2_STD_PAL,
+ .tvn = {
+ &f50ccir601_lml33,
+ &f60ccir601_lml33,
+ NULL
+ },
+ .jpeg_int = ZR36057_ISR_GIRQ1,
+ .vsync_int = ZR36057_ISR_GIRQ0,
+ .gpio = { 1, -1, 3, 5, 7, -1, -1, -1 },
+ .gpio_pol = { 0, 0, 0, 0, 1, 0, 0, 0 },
+ .gpcs = { 3, 1 },
+ .vfe_pol = { 1, 1, 0, 0, 0, 1, 0, 0 },
+ .gws_not_connected = 1,
+ .input_mux = 0,
+ .init = &lml33_init,
+ }, {
+ .type = LML33R10,
+ .name = "LML33R10",
+ .i2c_decoder = "saa7114",
+ .addrs_decoder = saa7114_addrs,
+ .i2c_encoder = "adv7170",
+ .addrs_encoder = adv717x_addrs,
+ .video_codec = CODEC_TYPE_ZR36060,
+
+ .inputs = 2,
+ .input = {
+ { 0, "Composite" },
+ { 7, "S-Video" }
+ },
+ .norms = V4L2_STD_NTSC | V4L2_STD_PAL,
+ .tvn = {
+ &f50ccir601_lm33r10,
+ &f60ccir601_lm33r10,
+ NULL
+ },
+ .jpeg_int = ZR36057_ISR_GIRQ1,
+ .vsync_int = ZR36057_ISR_GIRQ0,
+ .gpio = { 1, -1, 3, 5, 7, -1, -1, -1 },
+ .gpio_pol = { 0, 0, 0, 0, 1, 0, 0, 0 },
+ .gpcs = { 3, 1 },
+ .vfe_pol = { 1, 1, 0, 0, 0, 1, 0, 0 },
+ .gws_not_connected = 1,
+ .input_mux = 0,
+ .init = &lml33_init,
+ }, {
+ .type = BUZ,
+ .name = "Buz",
+ .i2c_decoder = "saa7111",
+ .addrs_decoder = saa7111_addrs,
+ .i2c_encoder = "saa7185",
+ .addrs_encoder = saa7185_addrs,
+ .video_codec = CODEC_TYPE_ZR36060,
+
+ .inputs = 2,
+ .input = {
+ { 3, "Composite" },
+ { 7, "S-Video" }
+ },
+ .norms = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM,
+ .tvn = {
+ &f50ccir601,
+ &f60ccir601,
+ &f50ccir601
+ },
+ .jpeg_int = ZR36057_ISR_GIRQ1,
+ .vsync_int = ZR36057_ISR_GIRQ0,
+ .gpio = { 1, -1, 3, -1, -1, -1, -1, -1 },
+ .gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
+ .gpcs = { 3, 1 },
+ .vfe_pol = { 1, 1, 0, 0, 0, 1, 0, 0 },
+ .gws_not_connected = 1,
+ .input_mux = 0,
+ .init = &buz_init,
+ }, {
+ .type = AVS6EYES,
+ .name = "6-Eyes",
+/* AverMedia chose not to brand the 6-Eyes. Thus it can't be autodetected, and requires card=x. */
+ .i2c_decoder = "ks0127",
+ .addrs_decoder = ks0127_addrs,
+ .i2c_encoder = "bt866",
+ .addrs_encoder = bt866_addrs,
+ .video_codec = CODEC_TYPE_ZR36060,
+
+ .inputs = 10,
+ .input = {
+ { 0, "Composite 1" },
+ { 1, "Composite 2" },
+ { 2, "Composite 3" },
+ { 4, "Composite 4" },
+ { 5, "Composite 5" },
+ { 6, "Composite 6" },
+ { 8, "S-Video 1" },
+ { 9, "S-Video 2" },
+ {10, "S-Video 3" },
+ {15, "YCbCr" }
+ },
+ .norms = V4L2_STD_NTSC | V4L2_STD_PAL,
+ .tvn = {
+ &f50ccir601_avs6eyes,
+ &f60ccir601_avs6eyes,
+ NULL
+ },
+ .jpeg_int = ZR36057_ISR_GIRQ1,
+ .vsync_int = ZR36057_ISR_GIRQ0,
+ .gpio = { 1, 0, 3, -1, -1, -1, -1, -1 },// Validity unknown /Sam
+ .gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 }, // Validity unknown /Sam
+ .gpcs = { 3, 1 }, // Validity unknown /Sam
+ .vfe_pol = { 1, 0, 0, 0, 0, 1, 0, 0 }, // Validity unknown /Sam
+ .gws_not_connected = 1,
+ .input_mux = 1,
+ .init = &avs6eyes_init,
+ }
+
+};
+
+/*
+ * I2C functions
+ */
+/* software I2C functions */
+static int zoran_i2c_getsda(void *data)
+{
+ struct zoran *zr = (struct zoran *)data;
+
+ return (btread(ZR36057_I2CBR) >> 1) & 1;
+}
+
+static int zoran_i2c_getscl(void *data)
+{
+ struct zoran *zr = (struct zoran *)data;
+
+ return btread(ZR36057_I2CBR) & 1;
+}
+
+static void zoran_i2c_setsda(void *data, int state)
+{
+ struct zoran *zr = (struct zoran *)data;
+
+ if (state)
+ zr->i2cbr |= 2;
+ else
+ zr->i2cbr &= ~2;
+ btwrite(zr->i2cbr, ZR36057_I2CBR);
+}
+
+static void zoran_i2c_setscl(void *data, int state)
+{
+ struct zoran *zr = (struct zoran *)data;
+
+ if (state)
+ zr->i2cbr |= 1;
+ else
+ zr->i2cbr &= ~1;
+ btwrite(zr->i2cbr, ZR36057_I2CBR);
+}
+
+static const struct i2c_algo_bit_data zoran_i2c_bit_data_template = {
+ .setsda = zoran_i2c_setsda,
+ .setscl = zoran_i2c_setscl,
+ .getsda = zoran_i2c_getsda,
+ .getscl = zoran_i2c_getscl,
+ .udelay = 10,
+ .timeout = 100,
+};
+
+static int zoran_register_i2c(struct zoran *zr)
+{
+ zr->i2c_algo = zoran_i2c_bit_data_template;
+ zr->i2c_algo.data = zr;
+ strscpy(zr->i2c_adapter.name, ZR_DEVNAME(zr),
+ sizeof(zr->i2c_adapter.name));
+ i2c_set_adapdata(&zr->i2c_adapter, &zr->v4l2_dev);
+ zr->i2c_adapter.algo_data = &zr->i2c_algo;
+ zr->i2c_adapter.dev.parent = &zr->pci_dev->dev;
+ return i2c_bit_add_bus(&zr->i2c_adapter);
+}
+
+static void zoran_unregister_i2c(struct zoran *zr)
+{
+ i2c_del_adapter(&zr->i2c_adapter);
+}
+
+/* Check a zoran_params struct for correctness, insert default params */
+int zoran_check_jpg_settings(struct zoran *zr,
+ struct zoran_jpg_settings *settings, int try)
+{
+ int err = 0, err0 = 0;
+
+ pci_dbg(zr->pci_dev, "%s - dec: %d, Hdcm: %d, Vdcm: %d, Tdcm: %d\n",
+ __func__, settings->decimation, settings->hor_dcm,
+ settings->ver_dcm, settings->tmp_dcm);
+ pci_dbg(zr->pci_dev, "%s - x: %d, y: %d, w: %d, y: %d\n", __func__,
+ settings->img_x, settings->img_y,
+ settings->img_width, settings->img_height);
+ /* Check decimation, set default values for decimation = 1, 2, 4 */
+ switch (settings->decimation) {
+ case 1:
+
+ settings->hor_dcm = 1;
+ settings->ver_dcm = 1;
+ settings->tmp_dcm = 1;
+ settings->field_per_buff = 2;
+ settings->img_x = 0;
+ settings->img_y = 0;
+ settings->img_width = BUZ_MAX_WIDTH;
+ settings->img_height = BUZ_MAX_HEIGHT / 2;
+ break;
+ case 2:
+
+ settings->hor_dcm = 2;
+ settings->ver_dcm = 1;
+ settings->tmp_dcm = 2;
+ settings->field_per_buff = 1;
+ settings->img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
+ settings->img_y = 0;
+ settings->img_width =
+ (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
+ settings->img_height = BUZ_MAX_HEIGHT / 2;
+ break;
+ case 4:
+
+ if (zr->card.type == DC10_NEW) {
+ pci_dbg(zr->pci_dev, "%s - HDec by 4 is not supported on the DC10\n", __func__);
+ err0++;
+ break;
+ }
+
+ settings->hor_dcm = 4;
+ settings->ver_dcm = 2;
+ settings->tmp_dcm = 2;
+ settings->field_per_buff = 1;
+ settings->img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
+ settings->img_y = 0;
+ settings->img_width =
+ (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
+ settings->img_height = BUZ_MAX_HEIGHT / 2;
+ break;
+ case 0:
+
+ /* We have to check the data the user has set */
+
+ if (settings->hor_dcm != 1 && settings->hor_dcm != 2 &&
+ (zr->card.type == DC10_NEW || settings->hor_dcm != 4)) {
+ settings->hor_dcm = clamp(settings->hor_dcm, 1, 2);
+ err0++;
+ }
+ if (settings->ver_dcm != 1 && settings->ver_dcm != 2) {
+ settings->ver_dcm = clamp(settings->ver_dcm, 1, 2);
+ err0++;
+ }
+ if (settings->tmp_dcm != 1 && settings->tmp_dcm != 2) {
+ settings->tmp_dcm = clamp(settings->tmp_dcm, 1, 2);
+ err0++;
+ }
+ if (settings->field_per_buff != 1 &&
+ settings->field_per_buff != 2) {
+ settings->field_per_buff = clamp(settings->field_per_buff, 1, 2);
+ err0++;
+ }
+ if (settings->img_x < 0) {
+ settings->img_x = 0;
+ err0++;
+ }
+ if (settings->img_y < 0) {
+ settings->img_y = 0;
+ err0++;
+ }
+ if (settings->img_width < 0 || settings->img_width > BUZ_MAX_WIDTH) {
+ settings->img_width = clamp(settings->img_width, 0, (int)BUZ_MAX_WIDTH);
+ err0++;
+ }
+ if (settings->img_height < 0 || settings->img_height > BUZ_MAX_HEIGHT / 2) {
+ settings->img_height = clamp(settings->img_height, 0, BUZ_MAX_HEIGHT / 2);
+ err0++;
+ }
+ if (settings->img_x + settings->img_width > BUZ_MAX_WIDTH) {
+ settings->img_x = BUZ_MAX_WIDTH - settings->img_width;
+ err0++;
+ }
+ if (settings->img_y + settings->img_height > BUZ_MAX_HEIGHT / 2) {
+ settings->img_y = BUZ_MAX_HEIGHT / 2 - settings->img_height;
+ err0++;
+ }
+ if (settings->img_width % (16 * settings->hor_dcm) != 0) {
+ settings->img_width -= settings->img_width % (16 * settings->hor_dcm);
+ if (settings->img_width == 0)
+ settings->img_width = 16 * settings->hor_dcm;
+ err0++;
+ }
+ if (settings->img_height % (8 * settings->ver_dcm) != 0) {
+ settings->img_height -= settings->img_height % (8 * settings->ver_dcm);
+ if (settings->img_height == 0)
+ settings->img_height = 8 * settings->ver_dcm;
+ err0++;
+ }
+
+ if (!try && err0) {
+ pci_err(zr->pci_dev, "%s - error in params for decimation = 0\n", __func__);
+ err++;
+ }
+ break;
+ default:
+ pci_err(zr->pci_dev, "%s - decimation = %d, must be 0, 1, 2 or 4\n",
+ __func__, settings->decimation);
+ err++;
+ break;
+ }
+
+ if (settings->jpg_comp.quality > 100)
+ settings->jpg_comp.quality = 100;
+ if (settings->jpg_comp.quality < 5)
+ settings->jpg_comp.quality = 5;
+ if (settings->jpg_comp.APPn < 0)
+ settings->jpg_comp.APPn = 0;
+ if (settings->jpg_comp.APPn > 15)
+ settings->jpg_comp.APPn = 15;
+ if (settings->jpg_comp.APP_len < 0)
+ settings->jpg_comp.APP_len = 0;
+ if (settings->jpg_comp.APP_len > 60)
+ settings->jpg_comp.APP_len = 60;
+ if (settings->jpg_comp.COM_len < 0)
+ settings->jpg_comp.COM_len = 0;
+ if (settings->jpg_comp.COM_len > 60)
+ settings->jpg_comp.COM_len = 60;
+ if (err)
+ return -EINVAL;
+ return 0;
+}
+
+void zoran_open_init_params(struct zoran *zr)
+{
+ int i;
+
+ zr->v4l_settings.width = 192;
+ zr->v4l_settings.height = 144;
+ zr->v4l_settings.format = &zoran_formats[7]; /* YUY2 - YUV-4:2:2 packed */
+ zr->v4l_settings.bytesperline = zr->v4l_settings.width *
+ ((zr->v4l_settings.format->depth + 7) / 8);
+
+ /* Set necessary params and call zoran_check_jpg_settings to set the defaults */
+ zr->jpg_settings.decimation = 1;
+ zr->jpg_settings.jpg_comp.quality = 50; /* default compression factor 8 */
+ if (zr->card.type != BUZ)
+ zr->jpg_settings.odd_even = 1;
+ else
+ zr->jpg_settings.odd_even = 0;
+ zr->jpg_settings.jpg_comp.APPn = 0;
+ zr->jpg_settings.jpg_comp.APP_len = 0; /* No APPn marker */
+ memset(zr->jpg_settings.jpg_comp.APP_data, 0,
+ sizeof(zr->jpg_settings.jpg_comp.APP_data));
+ zr->jpg_settings.jpg_comp.COM_len = 0; /* No COM marker */
+ memset(zr->jpg_settings.jpg_comp.COM_data, 0,
+ sizeof(zr->jpg_settings.jpg_comp.COM_data));
+ zr->jpg_settings.jpg_comp.jpeg_markers =
+ V4L2_JPEG_MARKER_DHT | V4L2_JPEG_MARKER_DQT;
+ i = zoran_check_jpg_settings(zr, &zr->jpg_settings, 0);
+ if (i)
+ pci_err(zr->pci_dev, "%s internal error\n", __func__);
+
+ zr->buffer_size = zr->v4l_settings.bytesperline * zr->v4l_settings.height;
+
+ clear_interrupt_counters(zr);
+}
+
+static int zr36057_init(struct zoran *zr)
+{
+ int j, err;
+
+ pci_info(zr->pci_dev, "initializing card[%d]\n", zr->id);
+
+ /* Avoid nonsense settings from user for default input/norm */
+ if (default_norm < 0 || default_norm > 2)
+ default_norm = 0;
+ if (default_norm == 0) {
+ zr->norm = V4L2_STD_PAL;
+ zr->timing = zr->card.tvn[ZR_NORM_PAL];
+ } else if (default_norm == 1) {
+ zr->norm = V4L2_STD_NTSC;
+ zr->timing = zr->card.tvn[ZR_NORM_NTSC];
+ } else {
+ zr->norm = V4L2_STD_SECAM;
+ zr->timing = zr->card.tvn[ZR_NORM_SECAM];
+ }
+ if (!zr->timing) {
+ pci_warn(zr->pci_dev, "%s - default TV standard not supported by hardware. PAL will be used.\n", __func__);
+ zr->norm = V4L2_STD_PAL;
+ zr->timing = zr->card.tvn[ZR_NORM_PAL];
+ }
+
+ if (default_input > zr->card.inputs - 1) {
+ pci_warn(zr->pci_dev, "default_input value %d out of range (0-%d)\n",
+ default_input, zr->card.inputs - 1);
+ default_input = 0;
+ }
+ zr->input = default_input;
+
+ /* default setup (will be repeated at every open) */
+ zoran_open_init_params(zr);
+
+ /* allocate memory *before* doing anything to the hardware in case allocation fails */
+ zr->video_dev = video_device_alloc();
+ if (!zr->video_dev) {
+ err = -ENOMEM;
+ goto exit;
+ }
+ zr->stat_com = dma_alloc_coherent(&zr->pci_dev->dev,
+ BUZ_NUM_STAT_COM * sizeof(u32),
+ &zr->p_sc, GFP_KERNEL);
+ if (!zr->stat_com) {
+ err = -ENOMEM;
+ goto exit_video;
+ }
+ for (j = 0; j < BUZ_NUM_STAT_COM; j++)
+ zr->stat_com[j] = cpu_to_le32(1); /* mark as unavailable to zr36057 */
+
+ zr->stat_comb = dma_alloc_coherent(&zr->pci_dev->dev,
+ BUZ_NUM_STAT_COM * sizeof(u32) * 2,
+ &zr->p_scb, GFP_KERNEL);
+ if (!zr->stat_comb) {
+ err = -ENOMEM;
+ goto exit_statcom;
+ }
+
+ /* Now add the template and register the device unit. */
+ *zr->video_dev = zoran_template;
+ zr->video_dev->v4l2_dev = &zr->v4l2_dev;
+ zr->video_dev->lock = &zr->lock;
+ zr->video_dev->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE;
+
+ strscpy(zr->video_dev->name, ZR_DEVNAME(zr), sizeof(zr->video_dev->name));
+ /*
+ * It's not a mem2mem device, but you can both capture and output from one and the same
+ * device. This should really be split up into two device nodes, but that's a job for
+ * another day.
+ */
+ zr->video_dev->vfl_dir = VFL_DIR_M2M;
+
+ zoran_queue_init(zr, &zr->vq);
+
+ err = video_register_device(zr->video_dev, VFL_TYPE_VIDEO, video_nr[zr->id]);
+ if (err < 0)
+ goto exit_statcomb;
+ video_set_drvdata(zr->video_dev, zr);
+
+ zoran_init_hardware(zr);
+ if (!pass_through) {
+ decoder_call(zr, video, s_stream, 0);
+ encoder_call(zr, video, s_routing, 2, 0, 0);
+ }
+
+ zr->initialized = 1;
+ return 0;
+
+exit_statcomb:
+ dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32) * 2, zr->stat_comb, zr->p_scb);
+exit_statcom:
+ dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32), zr->stat_com, zr->p_sc);
+exit_video:
+ kfree(zr->video_dev);
+exit:
+ return err;
+}
+
+static void zoran_remove(struct pci_dev *pdev)
+{
+ struct v4l2_device *v4l2_dev = dev_get_drvdata(&pdev->dev);
+ struct zoran *zr = to_zoran(v4l2_dev);
+
+ if (!zr->initialized)
+ goto exit_free;
+
+ zoran_queue_exit(zr);
+
+ /* unregister videocodec bus */
+ if (zr->codec)
+ videocodec_detach(zr->codec);
+ if (zr->vfe)
+ videocodec_detach(zr->vfe);
+
+ /* unregister i2c bus */
+ zoran_unregister_i2c(zr);
+ /* disable PCI bus-mastering */
+ zoran_set_pci_master(zr, 0);
+ /* put chip into reset */
+ btwrite(0, ZR36057_SPGPPCR);
+ pci_free_irq(zr->pci_dev, 0, zr);
+ /* unmap and free memory */
+ dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32), zr->stat_com, zr->p_sc);
+ dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32) * 2, zr->stat_comb, zr->p_scb);
+ pci_release_regions(pdev);
+ pci_disable_device(zr->pci_dev);
+ video_unregister_device(zr->video_dev);
+exit_free:
+ v4l2_ctrl_handler_free(&zr->hdl);
+ v4l2_device_unregister(&zr->v4l2_dev);
+}
+
+void zoran_vdev_release(struct video_device *vdev)
+{
+ kfree(vdev);
+}
+
+static struct videocodec_master *zoran_setup_videocodec(struct zoran *zr,
+ int type)
+{
+ struct videocodec_master *m = NULL;
+
+ m = devm_kmalloc(&zr->pci_dev->dev, sizeof(*m), GFP_KERNEL);
+ if (!m)
+ return m;
+
+ /*
+ * magic and type are unused for master struct. Makes sense only at codec structs.
+ * In the past, .type were initialized to the old V4L1 .hardware value,
+ * as VID_HARDWARE_ZR36067
+ */
+ m->magic = 0L;
+ m->type = 0;
+
+ m->flags = CODEC_FLAG_ENCODER | CODEC_FLAG_DECODER;
+ strscpy(m->name, ZR_DEVNAME(zr), sizeof(m->name));
+ m->data = zr;
+
+ switch (type) {
+ case CODEC_TYPE_ZR36060:
+ m->readreg = zr36060_read;
+ m->writereg = zr36060_write;
+ m->flags |= CODEC_FLAG_JPEG | CODEC_FLAG_VFE;
+ break;
+ case CODEC_TYPE_ZR36050:
+ m->readreg = zr36050_read;
+ m->writereg = zr36050_write;
+ m->flags |= CODEC_FLAG_JPEG;
+ break;
+ case CODEC_TYPE_ZR36016:
+ m->readreg = zr36016_read;
+ m->writereg = zr36016_write;
+ m->flags |= CODEC_FLAG_VFE;
+ break;
+ }
+
+ return m;
+}
+
+static void zoran_subdev_notify(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
+{
+ struct zoran *zr = to_zoran(sd->v4l2_dev);
+
+ /*
+ * Bt819 needs to reset its FIFO buffer using #FRST pin and
+ * LML33 card uses GPIO(7) for that.
+ */
+ if (cmd == BT819_FIFO_RESET_LOW)
+ GPIO(zr, 7, 0);
+ else if (cmd == BT819_FIFO_RESET_HIGH)
+ GPIO(zr, 7, 1);
+}
+
+static int zoran_video_set_ctrl(struct v4l2_ctrl *ctrl)
+{
+ struct zoran *zr = container_of(ctrl->handler, struct zoran, hdl);
+
+ switch (ctrl->id) {
+ case V4L2_CID_JPEG_COMPRESSION_QUALITY:
+ zr->jpg_settings.jpg_comp.quality = ctrl->val;
+ return zoran_check_jpg_settings(zr, &zr->jpg_settings, 0);
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static const struct v4l2_ctrl_ops zoran_video_ctrl_ops = {
+ .s_ctrl = zoran_video_set_ctrl,
+};
+
+/*
+ * Scan for a Buz card (actually for the PCI controller ZR36057),
+ * request the irq and map the io memory
+ */
+static int zoran_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+ unsigned char latency, need_latency;
+ struct zoran *zr;
+ int result;
+ struct videocodec_master *master_vfe = NULL;
+ struct videocodec_master *master_codec = NULL;
+ int card_num;
+ const char *codec_name, *vfe_name;
+ unsigned int nr;
+ int err;
+
+ err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+ if (err)
+ return -ENODEV;
+ vb2_dma_contig_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32));
+
+ nr = zoran_num++;
+ if (nr >= BUZ_MAX) {
+ pci_err(pdev, "driver limited to %d card(s) maximum\n", BUZ_MAX);
+ return -ENOENT;
+ }
+
+ zr = devm_kzalloc(&pdev->dev, sizeof(*zr), GFP_KERNEL);
+ if (!zr)
+ return -ENOMEM;
+
+ zr->v4l2_dev.notify = zoran_subdev_notify;
+ if (v4l2_device_register(&pdev->dev, &zr->v4l2_dev))
+ goto zr_free_mem;
+ zr->pci_dev = pdev;
+ zr->id = nr;
+ snprintf(ZR_DEVNAME(zr), sizeof(ZR_DEVNAME(zr)), "MJPEG[%u]", zr->id);
+ if (v4l2_ctrl_handler_init(&zr->hdl, 10))
+ goto zr_unreg;
+ zr->v4l2_dev.ctrl_handler = &zr->hdl;
+ v4l2_ctrl_new_std(&zr->hdl, &zoran_video_ctrl_ops,
+ V4L2_CID_JPEG_COMPRESSION_QUALITY, 0,
+ 100, 1, 50);
+ spin_lock_init(&zr->spinlock);
+ mutex_init(&zr->lock);
+ if (pci_enable_device(pdev))
+ goto zr_unreg;
+ zr->revision = zr->pci_dev->revision;
+
+ pci_info(zr->pci_dev, "Zoran ZR360%c7 (rev %d), irq: %d, memory: 0x%08llx\n",
+ zr->revision < 2 ? '5' : '6', zr->revision,
+ zr->pci_dev->irq, (uint64_t)pci_resource_start(zr->pci_dev, 0));
+ if (zr->revision >= 2)
+ pci_info(zr->pci_dev, "Subsystem vendor=0x%04x id=0x%04x\n",
+ zr->pci_dev->subsystem_vendor, zr->pci_dev->subsystem_device);
+
+ /* Use auto-detected card type? */
+ if (card[nr] == -1) {
+ if (zr->revision < 2) {
+ pci_err(pdev, "No card type specified, please use the card=X module parameter\n");
+ pci_err(pdev, "It is not possible to auto-detect ZR36057 based cards\n");
+ goto zr_unreg;
+ }
+
+ card_num = ent->driver_data;
+ if (card_num >= NUM_CARDS) {
+ pci_err(pdev, "Unknown card, try specifying card=X module parameter\n");
+ goto zr_unreg;
+ }
+ pci_info(zr->pci_dev, "%s() - card %s detected\n", __func__, zoran_cards[card_num].name);
+ } else {
+ card_num = card[nr];
+ if (card_num >= NUM_CARDS || card_num < 0) {
+ pci_err(pdev, "User specified card type %d out of range (0 .. %d)\n",
+ card_num, NUM_CARDS - 1);
+ goto zr_unreg;
+ }
+ }
+
+ /*
+ * even though we make this a non pointer and thus
+ * theoretically allow for making changes to this struct
+ * on a per-individual card basis at runtime, this is
+ * strongly discouraged. This structure is intended to
+ * keep general card information, no settings or anything
+ */
+ zr->card = zoran_cards[card_num];
+ snprintf(ZR_DEVNAME(zr), sizeof(ZR_DEVNAME(zr)), "%s[%u]",
+ zr->card.name, zr->id);
+
+ err = pci_request_regions(pdev, ZR_DEVNAME(zr));
+ if (err)
+ goto zr_unreg;
+
+ zr->zr36057_mem = devm_ioremap(&pdev->dev, pci_resource_start(pdev, 0), pci_resource_len(pdev, 0));
+ if (!zr->zr36057_mem) {
+ pci_err(pdev, "%s() - ioremap failed\n", __func__);
+ goto zr_pci_release;
+ }
+
+ result = pci_request_irq(pdev, 0, zoran_irq, NULL, zr, ZR_DEVNAME(zr));
+ if (result < 0) {
+ if (result == -EINVAL) {
+ pci_err(pdev, "%s - bad IRQ number or handler\n", __func__);
+ } else if (result == -EBUSY) {
+ pci_err(pdev, "%s - IRQ %d busy, change your PnP config in BIOS\n",
+ __func__, zr->pci_dev->irq);
+ } else {
+ pci_err(pdev, "%s - cannot assign IRQ, error code %d\n", __func__, result);
+ }
+ goto zr_pci_release;
+ }
+
+ /* set PCI latency timer */
+ pci_read_config_byte(zr->pci_dev, PCI_LATENCY_TIMER,
+ &latency);
+ need_latency = zr->revision > 1 ? 32 : 48;
+ if (latency != need_latency) {
+ pci_info(zr->pci_dev, "Changing PCI latency from %d to %d\n", latency, need_latency);
+ pci_write_config_byte(zr->pci_dev, PCI_LATENCY_TIMER, need_latency);
+ }
+
+ zr36057_restart(zr);
+ /* i2c */
+ pci_info(zr->pci_dev, "Initializing i2c bus...\n");
+
+ if (zoran_register_i2c(zr) < 0) {
+ pci_err(pdev, "%s - can't initialize i2c bus\n", __func__);
+ goto zr_free_irq;
+ }
+
+ zr->decoder = v4l2_i2c_new_subdev(&zr->v4l2_dev, &zr->i2c_adapter,
+ zr->card.i2c_decoder, 0,
+ zr->card.addrs_decoder);
+
+ if (zr->card.i2c_encoder)
+ zr->encoder = v4l2_i2c_new_subdev(&zr->v4l2_dev, &zr->i2c_adapter,
+ zr->card.i2c_encoder, 0,
+ zr->card.addrs_encoder);
+
+ pci_info(zr->pci_dev, "Initializing videocodec bus...\n");
+
+ if (zr->card.video_codec) {
+ codec_name = codecid_to_modulename(zr->card.video_codec);
+ if (codec_name) {
+ result = request_module(codec_name);
+ if (result)
+ pci_err(pdev, "failed to load modules %s: %d\n", codec_name, result);
+ }
+ }
+ if (zr->card.video_vfe) {
+ vfe_name = codecid_to_modulename(zr->card.video_vfe);
+ if (vfe_name) {
+ result = request_module(vfe_name);
+ if (result < 0)
+ pci_err(pdev, "failed to load modules %s: %d\n", vfe_name, result);
+ }
+ }
+
+ /* reset JPEG codec */
+ jpeg_codec_sleep(zr, 1);
+ jpeg_codec_reset(zr);
+ /* video bus enabled */
+ /* display codec revision */
+ if (zr->card.video_codec != 0) {
+ master_codec = zoran_setup_videocodec(zr, zr->card.video_codec);
+ if (!master_codec)
+ goto zr_unreg_i2c;
+ zr->codec = videocodec_attach(master_codec);
+ if (!zr->codec) {
+ pci_err(pdev, "%s - no codec found\n", __func__);
+ goto zr_unreg_i2c;
+ }
+ if (zr->codec->type != zr->card.video_codec) {
+ pci_err(pdev, "%s - wrong codec\n", __func__);
+ goto zr_detach_codec;
+ }
+ }
+ if (zr->card.video_vfe != 0) {
+ master_vfe = zoran_setup_videocodec(zr, zr->card.video_vfe);
+ if (!master_vfe)
+ goto zr_detach_codec;
+ zr->vfe = videocodec_attach(master_vfe);
+ if (!zr->vfe) {
+ pci_err(pdev, "%s - no VFE found\n", __func__);
+ goto zr_detach_codec;
+ }
+ if (zr->vfe->type != zr->card.video_vfe) {
+ pci_err(pdev, "%s = wrong VFE\n", __func__);
+ goto zr_detach_vfe;
+ }
+ }
+
+ /* take care of Natoma chipset and a revision 1 zr36057 */
+ if ((pci_pci_problems & PCIPCI_NATOMA) && zr->revision <= 1)
+ pci_info(zr->pci_dev, "ZR36057/Natoma bug, max. buffer size is 128K\n");
+
+ if (zr36057_init(zr) < 0)
+ goto zr_detach_vfe;
+
+ zr->map_mode = ZORAN_MAP_MODE_RAW;
+
+ return 0;
+
+zr_detach_vfe:
+ videocodec_detach(zr->vfe);
+zr_detach_codec:
+ videocodec_detach(zr->codec);
+zr_unreg_i2c:
+ zoran_unregister_i2c(zr);
+zr_free_irq:
+ btwrite(0, ZR36057_SPGPPCR);
+ pci_free_irq(zr->pci_dev, 0, zr);
+zr_pci_release:
+ pci_release_regions(pdev);
+zr_unreg:
+ v4l2_ctrl_handler_free(&zr->hdl);
+ v4l2_device_unregister(&zr->v4l2_dev);
+zr_free_mem:
+
+ return -ENODEV;
+}
+
+static struct pci_driver zoran_driver = {
+ .name = "zr36067",
+ .id_table = zr36067_pci_tbl,
+ .probe = zoran_probe,
+ .remove = zoran_remove,
+};
+
+static int __init zoran_init(void)
+{
+ int res;
+
+ pr_info("Zoran MJPEG board driver version %s\n", ZORAN_VERSION);
+
+ /* check the parameters we have been given, adjust if necessary */
+ if (v4l_nbufs < 2)
+ v4l_nbufs = 2;
+ if (v4l_nbufs > VIDEO_MAX_FRAME)
+ v4l_nbufs = VIDEO_MAX_FRAME;
+ /* The user specifies the in KB, we want them in byte (and page aligned) */
+ v4l_bufsize = PAGE_ALIGN(v4l_bufsize * 1024);
+ if (v4l_bufsize < 32768)
+ v4l_bufsize = 32768;
+ /* 2 MB is arbitrary but sufficient for the maximum possible images */
+ if (v4l_bufsize > 2048 * 1024)
+ v4l_bufsize = 2048 * 1024;
+ if (jpg_nbufs < 4)
+ jpg_nbufs = 4;
+ if (jpg_nbufs > BUZ_MAX_FRAME)
+ jpg_nbufs = BUZ_MAX_FRAME;
+ jpg_bufsize = PAGE_ALIGN(jpg_bufsize * 1024);
+ if (jpg_bufsize < 8192)
+ jpg_bufsize = 8192;
+ if (jpg_bufsize > (512 * 1024))
+ jpg_bufsize = 512 * 1024;
+ /* Use parameter for vidmem or try to find a video card */
+ if (vidmem)
+ pr_info("%s: Using supplied video memory base address @ 0x%lx\n", ZORAN_NAME, vidmem);
+
+ /* some mainboards might not do PCI-PCI data transfer well */
+ if (pci_pci_problems & (PCIPCI_FAIL | PCIAGP_FAIL | PCIPCI_ALIMAGIK))
+ pr_warn("%s: chipset does not support reliable PCI-PCI DMA\n", ZORAN_NAME);
+
+ res = pci_register_driver(&zoran_driver);
+ if (res) {
+ pr_err("Unable to register ZR36057 driver\n");
+ return res;
+ }
+
+ return 0;
+}
+
+static void __exit zoran_exit(void)
+{
+ pci_unregister_driver(&zoran_driver);
+}
+
+module_init(zoran_init);
+module_exit(zoran_exit);
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Zoran zr36057/zr36067 PCI controller driver, for the
+ * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
+ * Media Labs LML33/LML33R10.
+ *
+ * This part handles card-specific data and detection
+ *
+ * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
+ */
+
+#ifndef __ZORAN_CARD_H__
+#define __ZORAN_CARD_H__
+
+extern int zr36067_debug;
+
+/* Anybody who uses more than four? */
+#define BUZ_MAX 4
+
+extern const struct video_device zoran_template;
+
+extern int zoran_check_jpg_settings(struct zoran *zr,
+ struct zoran_jpg_settings *settings,
+ int try);
+extern void zoran_open_init_params(struct zoran *zr);
+extern void zoran_vdev_release(struct video_device *vdev);
+
+void zr36016_write(struct videocodec *codec, u16 reg, u32 val);
+
+#endif /* __ZORAN_CARD_H__ */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Zoran zr36057/zr36067 PCI controller driver, for the
+ * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
+ * Media Labs LML33/LML33R10.
+ *
+ * This part handles device access (PCI/I2C/codec/...)
+ *
+ * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
+ */
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#include <linux/interrupt.h>
+#include <linux/i2c.h>
+#include <linux/i2c-algo-bit.h>
+#include <linux/videodev2.h>
+#include <media/v4l2-common.h>
+#include <linux/spinlock.h>
+
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/wait.h>
+#include <linux/dma-mapping.h>
+
+#include <linux/io.h>
+
+#include "videocodec.h"
+#include "zoran.h"
+#include "zoran_device.h"
+#include "zoran_card.h"
+
+#define IRQ_MASK (ZR36057_ISR_GIRQ0 | \
+ ZR36057_ISR_GIRQ1 | \
+ ZR36057_ISR_JPEG_REP_IRQ)
+
+static bool lml33dpath; /* default = 0
+ * 1 will use digital path in capture
+ * mode instead of analog. It can be
+ * used for picture adjustments using
+ * tool like xawtv while watching image
+ * on TV monitor connected to the output.
+ * However, due to absence of 75 Ohm
+ * load on Bt819 input, there will be
+ * some image imperfections
+ */
+
+module_param(lml33dpath, bool, 0644);
+MODULE_PARM_DESC(lml33dpath, "Use digital path capture mode (on LML33 cards)");
+
+int zr_set_buf(struct zoran *zr);
+/*
+ * initialize video front end
+ */
+static void zr36057_init_vfe(struct zoran *zr)
+{
+ u32 reg;
+
+ reg = btread(ZR36057_VFESPFR);
+ reg |= ZR36057_VFESPFR_LITTLE_ENDIAN;
+ reg &= ~ZR36057_VFESPFR_VCLK_POL;
+ reg |= ZR36057_VFESPFR_EXT_FL;
+ reg |= ZR36057_VFESPFR_TOP_FIELD;
+ btwrite(reg, ZR36057_VFESPFR);
+ reg = btread(ZR36057_VDCR);
+ if (pci_pci_problems & PCIPCI_TRITON)
+ // || zr->revision < 1) // Revision 1 has also Triton support
+ reg &= ~ZR36057_VDCR_TRITON;
+ else
+ reg |= ZR36057_VDCR_TRITON;
+ btwrite(reg, ZR36057_VDCR);
+}
+
+/*
+ * General Purpose I/O and Guest bus access
+ */
+
+/*
+ * This is a bit tricky. When a board lacks a GPIO function, the corresponding
+ * GPIO bit number in the card_info structure is set to 0.
+ */
+
+void GPIO(struct zoran *zr, int bit, unsigned int value)
+{
+ u32 reg;
+ u32 mask;
+
+ /* Make sure the bit number is legal
+ * A bit number of -1 (lacking) gives a mask of 0,
+ * making it harmless
+ */
+ mask = (1 << (24 + bit)) & 0xff000000;
+ reg = btread(ZR36057_GPPGCR1) & ~mask;
+ if (value)
+ reg |= mask;
+
+ btwrite(reg, ZR36057_GPPGCR1);
+ udelay(1);
+}
+
+/*
+ * Wait til post office is no longer busy
+ */
+
+int post_office_wait(struct zoran *zr)
+{
+ u32 por;
+
+// while (((por = btread(ZR36057_POR)) & (ZR36057_POR_PO_PEN | ZR36057_POR_PO_TIME)) == ZR36057_POR_PO_PEN) {
+ while ((por = btread(ZR36057_POR)) & ZR36057_POR_PO_PEN) {
+ /* wait for something to happen */
+ /* TODO add timeout */
+ }
+ if ((por & ZR36057_POR_PO_TIME) && !zr->card.gws_not_connected) {
+ /* In LML33/BUZ \GWS line is not connected, so it has always timeout set */
+ pci_info(zr->pci_dev, "pop timeout %08x\n", por);
+ return -1;
+ }
+
+ return 0;
+}
+
+int post_office_write(struct zoran *zr, unsigned int guest,
+ unsigned int reg, unsigned int value)
+{
+ u32 por;
+
+ por =
+ ZR36057_POR_PO_DIR | ZR36057_POR_PO_TIME | ((guest & 7) << 20) |
+ ((reg & 7) << 16) | (value & 0xFF);
+ btwrite(por, ZR36057_POR);
+
+ return post_office_wait(zr);
+}
+
+int post_office_read(struct zoran *zr, unsigned int guest, unsigned int reg)
+{
+ u32 por;
+
+ por = ZR36057_POR_PO_TIME | ((guest & 7) << 20) | ((reg & 7) << 16);
+ btwrite(por, ZR36057_POR);
+ if (post_office_wait(zr) < 0)
+ return -1;
+
+ return btread(ZR36057_POR) & 0xFF;
+}
+
+/*
+ * detect guests
+ */
+
+static void dump_guests(struct zoran *zr)
+{
+ if (zr36067_debug > 2) {
+ int i, guest[8];
+
+ /* do not print random data */
+ guest[0] = 0;
+
+ for (i = 1; i < 8; i++) /* Don't read jpeg codec here */
+ guest[i] = post_office_read(zr, i, 0);
+
+ pci_info(zr->pci_dev, "Guests: %*ph\n", 8, guest);
+ }
+}
+
+void detect_guest_activity(struct zoran *zr)
+{
+ int timeout, i, j, res, guest[8], guest0[8], change[8][3];
+ ktime_t t0, t1;
+
+ /* do not print random data */
+ guest[0] = 0;
+ guest0[0] = 0;
+
+ dump_guests(zr);
+ pci_info(zr->pci_dev, "Detecting guests activity, please wait...\n");
+ for (i = 1; i < 8; i++) /* Don't read jpeg codec here */
+ guest0[i] = guest[i] = post_office_read(zr, i, 0);
+
+ timeout = 0;
+ j = 0;
+ t0 = ktime_get();
+ while (timeout < 10000) {
+ udelay(10);
+ timeout++;
+ for (i = 1; (i < 8) && (j < 8); i++) {
+ res = post_office_read(zr, i, 0);
+ if (res != guest[i]) {
+ t1 = ktime_get();
+ change[j][0] = ktime_to_us(ktime_sub(t1, t0));
+ t0 = t1;
+ change[j][1] = i;
+ change[j][2] = res;
+ j++;
+ guest[i] = res;
+ }
+ }
+ if (j >= 8)
+ break;
+ }
+
+ pci_info(zr->pci_dev, "Guests: %*ph\n", 8, guest0);
+
+ if (j == 0) {
+ pci_info(zr->pci_dev, "No activity detected.\n");
+ return;
+ }
+ for (i = 0; i < j; i++)
+ pci_info(zr->pci_dev, "%6d: %d => 0x%02x\n", change[i][0], change[i][1], change[i][2]);
+}
+
+/*
+ * JPEG Codec access
+ */
+
+void jpeg_codec_sleep(struct zoran *zr, int sleep)
+{
+ GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_SLEEP], !sleep);
+ if (!sleep) {
+ pci_dbg(zr->pci_dev, "%s() - wake GPIO=0x%08x\n", __func__, btread(ZR36057_GPPGCR1));
+ udelay(500);
+ } else {
+ pci_dbg(zr->pci_dev, "%s() - sleep GPIO=0x%08x\n", __func__, btread(ZR36057_GPPGCR1));
+ udelay(2);
+ }
+}
+
+int jpeg_codec_reset(struct zoran *zr)
+{
+ /* Take the codec out of sleep */
+ jpeg_codec_sleep(zr, 0);
+
+ if (zr->card.gpcs[GPCS_JPEG_RESET] != 0xff) {
+ post_office_write(zr, zr->card.gpcs[GPCS_JPEG_RESET], 0,
+ 0);
+ udelay(2);
+ } else {
+ GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_RESET], 0);
+ udelay(2);
+ GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_RESET], 1);
+ udelay(2);
+ }
+
+ return 0;
+}
+
+/*
+ * Set the registers for the size we have specified. Don't bother
+ * trying to understand this without the ZR36057 manual in front of
+ * you [AC].
+ */
+static void zr36057_adjust_vfe(struct zoran *zr, enum zoran_codec_mode mode)
+{
+ u32 reg;
+
+ switch (mode) {
+ case BUZ_MODE_MOTION_DECOMPRESS:
+ btand(~ZR36057_VFESPFR_EXT_FL, ZR36057_VFESPFR);
+ reg = btread(ZR36057_VFEHCR);
+ if ((reg & (1 << 10)) && zr->card.type != LML33R10)
+ reg += ((1 << 10) | 1);
+
+ btwrite(reg, ZR36057_VFEHCR);
+ break;
+ case BUZ_MODE_MOTION_COMPRESS:
+ case BUZ_MODE_IDLE:
+ default:
+ if ((zr->norm & V4L2_STD_NTSC) ||
+ (zr->card.type == LML33R10 &&
+ (zr->norm & V4L2_STD_PAL)))
+ btand(~ZR36057_VFESPFR_EXT_FL, ZR36057_VFESPFR);
+ else
+ btor(ZR36057_VFESPFR_EXT_FL, ZR36057_VFESPFR);
+ reg = btread(ZR36057_VFEHCR);
+ if (!(reg & (1 << 10)) && zr->card.type != LML33R10)
+ reg -= ((1 << 10) | 1);
+
+ btwrite(reg, ZR36057_VFEHCR);
+ break;
+ }
+}
+
+/*
+ * set geometry
+ */
+
+static void zr36057_set_vfe(struct zoran *zr, int video_width, int video_height,
+ const struct zoran_format *format)
+{
+ const struct tvnorm *tvn;
+ unsigned int h_start, HEnd, v_start, VEnd;
+ unsigned int DispMode;
+ unsigned int VidWinWid, VidWinHt;
+ unsigned int hcrop1, hcrop2, vcrop1, vcrop2;
+ unsigned int wa, We, ha, He;
+ unsigned int X, Y, hor_dcm, ver_dcm;
+ u32 reg;
+
+ tvn = zr->timing;
+
+ wa = tvn->wa;
+ ha = tvn->ha;
+
+ pci_info(zr->pci_dev, "set_vfe() - width = %d, height = %d\n", video_width, video_height);
+
+ if (video_width < BUZ_MIN_WIDTH ||
+ video_height < BUZ_MIN_HEIGHT ||
+ video_width > wa || video_height > ha) {
+ pci_err(zr->pci_dev, "set_vfe: w=%d h=%d not valid\n", video_width, video_height);
+ return;
+ }
+
+ /**** zr36057 ****/
+
+ /* horizontal */
+ VidWinWid = video_width;
+ X = DIV_ROUND_UP(VidWinWid * 64, tvn->wa);
+ We = (VidWinWid * 64) / X;
+ hor_dcm = 64 - X;
+ hcrop1 = 2 * ((tvn->wa - We) / 4);
+ hcrop2 = tvn->wa - We - hcrop1;
+ h_start = tvn->h_start ? tvn->h_start : 1;
+ /* (Ronald) Original comment:
+ * "| 1 Doesn't have any effect, tested on both a DC10 and a DC10+"
+ * this is false. It inverses chroma values on the LML33R10 (so Cr
+ * suddenly is shown as Cb and reverse, really cool effect if you
+ * want to see blue faces, not useful otherwise). So don't use |1.
+ * However, the DC10 has '0' as h_start, but does need |1, so we
+ * use a dirty check...
+ */
+ HEnd = h_start + tvn->wa - 1;
+ h_start += hcrop1;
+ HEnd -= hcrop2;
+ reg = ((h_start & ZR36057_VFEHCR_HMASK) << ZR36057_VFEHCR_H_START)
+ | ((HEnd & ZR36057_VFEHCR_HMASK) << ZR36057_VFEHCR_H_END);
+ if (zr->card.vfe_pol.hsync_pol)
+ reg |= ZR36057_VFEHCR_HS_POL;
+ btwrite(reg, ZR36057_VFEHCR);
+
+ /* Vertical */
+ DispMode = !(video_height > BUZ_MAX_HEIGHT / 2);
+ VidWinHt = DispMode ? video_height : video_height / 2;
+ Y = DIV_ROUND_UP(VidWinHt * 64 * 2, tvn->ha);
+ He = (VidWinHt * 64) / Y;
+ ver_dcm = 64 - Y;
+ vcrop1 = (tvn->ha / 2 - He) / 2;
+ vcrop2 = tvn->ha / 2 - He - vcrop1;
+ v_start = tvn->v_start;
+ VEnd = v_start + tvn->ha / 2; // - 1; FIXME SnapShot times out with -1 in 768*576 on the DC10 - LP
+ v_start += vcrop1;
+ VEnd -= vcrop2;
+ reg = ((v_start & ZR36057_VFEVCR_VMASK) << ZR36057_VFEVCR_V_START)
+ | ((VEnd & ZR36057_VFEVCR_VMASK) << ZR36057_VFEVCR_V_END);
+ if (zr->card.vfe_pol.vsync_pol)
+ reg |= ZR36057_VFEVCR_VS_POL;
+ btwrite(reg, ZR36057_VFEVCR);
+
+ /* scaler and pixel format */
+ reg = 0;
+ reg |= (hor_dcm << ZR36057_VFESPFR_HOR_DCM);
+ reg |= (ver_dcm << ZR36057_VFESPFR_VER_DCM);
+ reg |= (DispMode << ZR36057_VFESPFR_DISP_MODE);
+ /* RJ: I don't know, why the following has to be the opposite
+ * of the corresponding ZR36060 setting, but only this way
+ * we get the correct colors when uncompressing to the screen */
+ //reg |= ZR36057_VFESPFR_VCLK_POL; /**/
+ /* RJ: Don't know if that is needed for NTSC also */
+ if (!(zr->norm & V4L2_STD_NTSC))
+ reg |= ZR36057_VFESPFR_EXT_FL; // NEEDED!!!!!!! Wolfgang
+ reg |= ZR36057_VFESPFR_TOP_FIELD;
+ if (hor_dcm >= 48)
+ reg |= 3 << ZR36057_VFESPFR_H_FILTER; /* 5 tap filter */
+ else if (hor_dcm >= 32)
+ reg |= 2 << ZR36057_VFESPFR_H_FILTER; /* 4 tap filter */
+ else if (hor_dcm >= 16)
+ reg |= 1 << ZR36057_VFESPFR_H_FILTER; /* 3 tap filter */
+
+ reg |= format->vfespfr;
+ btwrite(reg, ZR36057_VFESPFR);
+
+ /* display configuration */
+ reg = (16 << ZR36057_VDCR_MIN_PIX)
+ | (VidWinHt << ZR36057_VDCR_VID_WIN_HT)
+ | (VidWinWid << ZR36057_VDCR_VID_WIN_WID);
+ if (pci_pci_problems & PCIPCI_TRITON)
+ // || zr->revision < 1) // Revision 1 has also Triton support
+ reg &= ~ZR36057_VDCR_TRITON;
+ else
+ reg |= ZR36057_VDCR_TRITON;
+ btwrite(reg, ZR36057_VDCR);
+
+ zr36057_adjust_vfe(zr, zr->codec_mode);
+}
+
+/* Enable/Disable uncompressed memory grabbing of the 36057 */
+void zr36057_set_memgrab(struct zoran *zr, int mode)
+{
+ if (mode) {
+ /* We only check SnapShot and not FrameGrab here. SnapShot==1
+ * means a capture is already in progress, but FrameGrab==1
+ * doesn't necessary mean that. It's more correct to say a 1
+ * to 0 transition indicates a capture completed. If a
+ * capture is pending when capturing is tuned off, FrameGrab
+ * will be stuck at 1 until capturing is turned back on.
+ */
+ if (btread(ZR36057_VSSFGR) & ZR36057_VSSFGR_SNAP_SHOT)
+ pci_warn(zr->pci_dev, "zr36057_set_memgrab(1) with SnapShot on!?\n");
+
+ /* switch on VSync interrupts */
+ btwrite(IRQ_MASK, ZR36057_ISR); // Clear Interrupts
+ btor(zr->card.vsync_int, ZR36057_ICR); // SW
+
+ /* enable SnapShot */
+ btor(ZR36057_VSSFGR_SNAP_SHOT, ZR36057_VSSFGR);
+
+ /* Set zr36057 video front end and enable video */
+ zr36057_set_vfe(zr, zr->v4l_settings.width,
+ zr->v4l_settings.height,
+ zr->v4l_settings.format);
+ } else {
+ /* switch off VSync interrupts */
+ btand(~zr->card.vsync_int, ZR36057_ICR); // SW
+
+ /* re-enable grabbing to screen if it was running */
+ btand(~ZR36057_VDCR_VID_EN, ZR36057_VDCR);
+ btand(~ZR36057_VSSFGR_SNAP_SHOT, ZR36057_VSSFGR);
+ }
+}
+
+/*****************************************************************************
+ * *
+ * Set up the Buz-specific MJPEG part *
+ * *
+ *****************************************************************************/
+
+static inline void set_frame(struct zoran *zr, int val)
+{
+ GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_FRAME], val);
+}
+
+static void set_videobus_dir(struct zoran *zr, int val)
+{
+ switch (zr->card.type) {
+ case LML33:
+ case LML33R10:
+ if (!lml33dpath)
+ GPIO(zr, 5, val);
+ else
+ GPIO(zr, 5, 1);
+ break;
+ default:
+ GPIO(zr, zr->card.gpio[ZR_GPIO_VID_DIR],
+ zr->card.gpio_pol[ZR_GPIO_VID_DIR] ? !val : val);
+ break;
+ }
+}
+
+static void init_jpeg_queue(struct zoran *zr)
+{
+ int i;
+
+ /* re-initialize DMA ring stuff */
+ zr->jpg_que_head = 0;
+ zr->jpg_dma_head = 0;
+ zr->jpg_dma_tail = 0;
+ zr->jpg_que_tail = 0;
+ zr->jpg_seq_num = 0;
+ zr->jpeg_error = 0;
+ zr->num_errors = 0;
+ zr->jpg_err_seq = 0;
+ zr->jpg_err_shift = 0;
+ zr->jpg_queued_num = 0;
+ for (i = 0; i < BUZ_NUM_STAT_COM; i++)
+ zr->stat_com[i] = cpu_to_le32(1); /* mark as unavailable to zr36057 */
+}
+
+static void zr36057_set_jpg(struct zoran *zr, enum zoran_codec_mode mode)
+{
+ const struct tvnorm *tvn;
+ u32 reg;
+
+ tvn = zr->timing;
+
+ /* assert P_Reset, disable code transfer, deassert Active */
+ btwrite(0, ZR36057_JPC);
+
+ /* MJPEG compression mode */
+ switch (mode) {
+ case BUZ_MODE_MOTION_COMPRESS:
+ default:
+ reg = ZR36057_JMC_MJPG_CMP_MODE;
+ break;
+
+ case BUZ_MODE_MOTION_DECOMPRESS:
+ reg = ZR36057_JMC_MJPG_EXP_MODE;
+ reg |= ZR36057_JMC_SYNC_MSTR;
+ /* RJ: The following is experimental - improves the output to screen */
+ //if(zr->jpg_settings.VFIFO_FB) reg |= ZR36057_JMC_VFIFO_FB; // No, it doesn't. SM
+ break;
+
+ case BUZ_MODE_STILL_COMPRESS:
+ reg = ZR36057_JMC_JPG_CMP_MODE;
+ break;
+
+ case BUZ_MODE_STILL_DECOMPRESS:
+ reg = ZR36057_JMC_JPG_EXP_MODE;
+ break;
+ }
+ reg |= ZR36057_JMC_JPG;
+ if (zr->jpg_settings.field_per_buff == 1)
+ reg |= ZR36057_JMC_FLD_PER_BUFF;
+ btwrite(reg, ZR36057_JMC);
+
+ /* vertical */
+ btor(ZR36057_VFEVCR_VS_POL, ZR36057_VFEVCR);
+ reg = (6 << ZR36057_VSP_VSYNC_SIZE) |
+ (tvn->ht << ZR36057_VSP_FRM_TOT);
+ btwrite(reg, ZR36057_VSP);
+ reg = ((zr->jpg_settings.img_y + tvn->v_start) << ZR36057_FVAP_NAY) |
+ (zr->jpg_settings.img_height << ZR36057_FVAP_PAY);
+ btwrite(reg, ZR36057_FVAP);
+
+ /* horizontal */
+ if (zr->card.vfe_pol.hsync_pol)
+ btor(ZR36057_VFEHCR_HS_POL, ZR36057_VFEHCR);
+ else
+ btand(~ZR36057_VFEHCR_HS_POL, ZR36057_VFEHCR);
+ reg = ((tvn->h_sync_start) << ZR36057_HSP_HSYNC_START) |
+ (tvn->wt << ZR36057_HSP_LINE_TOT);
+ btwrite(reg, ZR36057_HSP);
+ reg = ((zr->jpg_settings.img_x +
+ tvn->h_start + 4) << ZR36057_FHAP_NAX) |
+ (zr->jpg_settings.img_width << ZR36057_FHAP_PAX);
+ btwrite(reg, ZR36057_FHAP);
+
+ /* field process parameters */
+ if (zr->jpg_settings.odd_even)
+ reg = ZR36057_FPP_ODD_EVEN;
+ else
+ reg = 0;
+
+ btwrite(reg, ZR36057_FPP);
+
+ /* Set proper VCLK Polarity, else colors will be wrong during playback */
+ //btor(ZR36057_VFESPFR_VCLK_POL, ZR36057_VFESPFR);
+
+ /* code base address */
+ btwrite(zr->p_sc, ZR36057_JCBA);
+
+ /* FIFO threshold (FIFO is 160. double words) */
+ /* NOTE: decimal values here */
+ switch (mode) {
+ case BUZ_MODE_STILL_COMPRESS:
+ case BUZ_MODE_MOTION_COMPRESS:
+ if (zr->card.type != BUZ)
+ reg = 140;
+ else
+ reg = 60;
+ break;
+
+ case BUZ_MODE_STILL_DECOMPRESS:
+ case BUZ_MODE_MOTION_DECOMPRESS:
+ reg = 20;
+ break;
+
+ default:
+ reg = 80;
+ break;
+ }
+ btwrite(reg, ZR36057_JCFT);
+ zr36057_adjust_vfe(zr, mode);
+}
+
+void clear_interrupt_counters(struct zoran *zr)
+{
+ zr->intr_counter_GIRQ1 = 0;
+ zr->intr_counter_GIRQ0 = 0;
+ zr->intr_counter_cod_rep_irq = 0;
+ zr->intr_counter_jpeg_rep_irq = 0;
+ zr->field_counter = 0;
+ zr->irq1_in = 0;
+ zr->irq1_out = 0;
+ zr->jpeg_in = 0;
+ zr->jpeg_out = 0;
+ zr->JPEG_0 = 0;
+ zr->JPEG_1 = 0;
+ zr->end_event_missed = 0;
+ zr->jpeg_missed = 0;
+ zr->jpeg_max_missed = 0;
+ zr->jpeg_min_missed = 0x7fffffff;
+}
+
+static u32 count_reset_interrupt(struct zoran *zr)
+{
+ u32 isr;
+
+ isr = btread(ZR36057_ISR) & 0x78000000;
+ if (isr) {
+ if (isr & ZR36057_ISR_GIRQ1) {
+ btwrite(ZR36057_ISR_GIRQ1, ZR36057_ISR);
+ zr->intr_counter_GIRQ1++;
+ }
+ if (isr & ZR36057_ISR_GIRQ0) {
+ btwrite(ZR36057_ISR_GIRQ0, ZR36057_ISR);
+ zr->intr_counter_GIRQ0++;
+ }
+ if (isr & ZR36057_ISR_COD_REP_IRQ) {
+ btwrite(ZR36057_ISR_COD_REP_IRQ, ZR36057_ISR);
+ zr->intr_counter_cod_rep_irq++;
+ }
+ if (isr & ZR36057_ISR_JPEG_REP_IRQ) {
+ btwrite(ZR36057_ISR_JPEG_REP_IRQ, ZR36057_ISR);
+ zr->intr_counter_jpeg_rep_irq++;
+ }
+ }
+ return isr;
+}
+
+void jpeg_start(struct zoran *zr)
+{
+ int reg;
+
+ zr->frame_num = 0;
+
+ /* deassert P_reset, disable code transfer, deassert Active */
+ btwrite(ZR36057_JPC_P_RESET, ZR36057_JPC);
+ /* stop flushing the internal code buffer */
+ btand(~ZR36057_MCTCR_C_FLUSH, ZR36057_MCTCR);
+ /* enable code transfer */
+ btor(ZR36057_JPC_COD_TRNS_EN, ZR36057_JPC);
+
+ /* clear IRQs */
+ btwrite(IRQ_MASK, ZR36057_ISR);
+ /* enable the JPEG IRQs */
+ btwrite(zr->card.jpeg_int | ZR36057_ICR_JPEG_REP_IRQ | ZR36057_ICR_INT_PIN_EN,
+ ZR36057_ICR);
+
+ set_frame(zr, 0); // \FRAME
+
+ /* set the JPEG codec guest ID */
+ reg = (zr->card.gpcs[1] << ZR36057_JCGI_JPE_GUEST_ID) |
+ (0 << ZR36057_JCGI_JPE_GUEST_REG);
+ btwrite(reg, ZR36057_JCGI);
+
+ if (zr->card.video_vfe == CODEC_TYPE_ZR36016 &&
+ zr->card.video_codec == CODEC_TYPE_ZR36050) {
+ /* Enable processing on the ZR36016 */
+ if (zr->vfe)
+ zr36016_write(zr->vfe, 0, 1);
+
+ /* load the address of the GO register in the ZR36050 latch */
+ post_office_write(zr, 0, 0, 0);
+ }
+
+ /* assert Active */
+ btor(ZR36057_JPC_ACTIVE, ZR36057_JPC);
+
+ /* enable the Go generation */
+ btor(ZR36057_JMC_GO_EN, ZR36057_JMC);
+ udelay(30);
+
+ set_frame(zr, 1); // /FRAME
+
+ pci_dbg(zr->pci_dev, "jpeg_start\n");
+}
+
+void zr36057_enable_jpg(struct zoran *zr, enum zoran_codec_mode mode)
+{
+ struct vfe_settings cap;
+ int field_size = zr->buffer_size / zr->jpg_settings.field_per_buff;
+
+ zr->codec_mode = mode;
+
+ cap.x = zr->jpg_settings.img_x;
+ cap.y = zr->jpg_settings.img_y;
+ cap.width = zr->jpg_settings.img_width;
+ cap.height = zr->jpg_settings.img_height;
+ cap.decimation =
+ zr->jpg_settings.hor_dcm | (zr->jpg_settings.ver_dcm << 8);
+ cap.quality = zr->jpg_settings.jpg_comp.quality;
+
+ switch (mode) {
+ case BUZ_MODE_MOTION_COMPRESS: {
+ struct jpeg_app_marker app;
+ struct jpeg_com_marker com;
+
+ /* In motion compress mode, the decoder output must be enabled, and
+ * the video bus direction set to input.
+ */
+ set_videobus_dir(zr, 0);
+ decoder_call(zr, video, s_stream, 1);
+ encoder_call(zr, video, s_routing, 0, 0, 0);
+
+ /* Take the JPEG codec and the VFE out of sleep */
+ jpeg_codec_sleep(zr, 0);
+
+ /* set JPEG app/com marker */
+ app.appn = zr->jpg_settings.jpg_comp.APPn;
+ app.len = zr->jpg_settings.jpg_comp.APP_len;
+ memcpy(app.data, zr->jpg_settings.jpg_comp.APP_data, 60);
+ zr->codec->control(zr->codec, CODEC_S_JPEG_APP_DATA,
+ sizeof(struct jpeg_app_marker), &app);
+
+ com.len = zr->jpg_settings.jpg_comp.COM_len;
+ memcpy(com.data, zr->jpg_settings.jpg_comp.COM_data, 60);
+ zr->codec->control(zr->codec, CODEC_S_JPEG_COM_DATA,
+ sizeof(struct jpeg_com_marker), &com);
+
+ /* Setup the JPEG codec */
+ zr->codec->control(zr->codec, CODEC_S_JPEG_TDS_BYTE,
+ sizeof(int), &field_size);
+ zr->codec->set_video(zr->codec, zr->timing, &cap,
+ &zr->card.vfe_pol);
+ zr->codec->set_mode(zr->codec, CODEC_DO_COMPRESSION);
+
+ /* Setup the VFE */
+ if (zr->vfe) {
+ zr->vfe->control(zr->vfe, CODEC_S_JPEG_TDS_BYTE,
+ sizeof(int), &field_size);
+ zr->vfe->set_video(zr->vfe, zr->timing, &cap,
+ &zr->card.vfe_pol);
+ zr->vfe->set_mode(zr->vfe, CODEC_DO_COMPRESSION);
+ }
+
+ init_jpeg_queue(zr);
+ zr36057_set_jpg(zr, mode); // \P_Reset, ... Video param, FIFO
+
+ clear_interrupt_counters(zr);
+ pci_info(zr->pci_dev, "enable_jpg(MOTION_COMPRESS)\n");
+ break;
+ }
+
+ case BUZ_MODE_MOTION_DECOMPRESS:
+ /* In motion decompression mode, the decoder output must be disabled, and
+ * the video bus direction set to output.
+ */
+ decoder_call(zr, video, s_stream, 0);
+ set_videobus_dir(zr, 1);
+ encoder_call(zr, video, s_routing, 1, 0, 0);
+
+ /* Take the JPEG codec and the VFE out of sleep */
+ jpeg_codec_sleep(zr, 0);
+ /* Setup the VFE */
+ if (zr->vfe) {
+ zr->vfe->set_video(zr->vfe, zr->timing, &cap,
+ &zr->card.vfe_pol);
+ zr->vfe->set_mode(zr->vfe, CODEC_DO_EXPANSION);
+ }
+ /* Setup the JPEG codec */
+ zr->codec->set_video(zr->codec, zr->timing, &cap,
+ &zr->card.vfe_pol);
+ zr->codec->set_mode(zr->codec, CODEC_DO_EXPANSION);
+
+ init_jpeg_queue(zr);
+ zr36057_set_jpg(zr, mode); // \P_Reset, ... Video param, FIFO
+
+ clear_interrupt_counters(zr);
+ pci_info(zr->pci_dev, "enable_jpg(MOTION_DECOMPRESS)\n");
+ break;
+
+ case BUZ_MODE_IDLE:
+ default:
+ /* shut down processing */
+ btand(~(zr->card.jpeg_int | ZR36057_ICR_JPEG_REP_IRQ),
+ ZR36057_ICR);
+ btwrite(zr->card.jpeg_int | ZR36057_ICR_JPEG_REP_IRQ,
+ ZR36057_ISR);
+ btand(~ZR36057_JMC_GO_EN, ZR36057_JMC); // \Go_en
+
+ msleep(50);
+
+ set_videobus_dir(zr, 0);
+ set_frame(zr, 1); // /FRAME
+ btor(ZR36057_MCTCR_C_FLUSH, ZR36057_MCTCR); // /CFlush
+ btwrite(0, ZR36057_JPC); // \P_Reset,\CodTrnsEn,\Active
+ btand(~ZR36057_JMC_VFIFO_FB, ZR36057_JMC);
+ btand(~ZR36057_JMC_SYNC_MSTR, ZR36057_JMC);
+ jpeg_codec_reset(zr);
+ jpeg_codec_sleep(zr, 1);
+ zr36057_adjust_vfe(zr, mode);
+
+ decoder_call(zr, video, s_stream, 1);
+ encoder_call(zr, video, s_routing, 0, 0, 0);
+
+ pci_info(zr->pci_dev, "enable_jpg(IDLE)\n");
+ break;
+ }
+}
+
+/* when this is called the spinlock must be held */
+void zoran_feed_stat_com(struct zoran *zr)
+{
+ /* move frames from pending queue to DMA */
+
+ int i, max_stat_com;
+ struct zr_buffer *buf;
+ struct vb2_v4l2_buffer *vbuf;
+ dma_addr_t phys_addr = 0;
+ unsigned long flags;
+ unsigned long payload;
+
+ max_stat_com =
+ (zr->jpg_settings.tmp_dcm ==
+ 1) ? BUZ_NUM_STAT_COM : (BUZ_NUM_STAT_COM >> 1);
+
+ spin_lock_irqsave(&zr->queued_bufs_lock, flags);
+ while ((zr->jpg_dma_head - zr->jpg_dma_tail) < max_stat_com) {
+ buf = list_first_entry_or_null(&zr->queued_bufs, struct zr_buffer, queue);
+ if (!buf) {
+ pci_err(zr->pci_dev, "No buffer available to queue\n");
+ spin_unlock_irqrestore(&zr->queued_bufs_lock, flags);
+ return;
+ }
+ list_del(&buf->queue);
+ zr->buf_in_reserve--;
+ vbuf = &buf->vbuf;
+ vbuf->vb2_buf.state = VB2_BUF_STATE_ACTIVE;
+ phys_addr = vb2_dma_contig_plane_dma_addr(&vbuf->vb2_buf, 0);
+ payload = vb2_get_plane_payload(&vbuf->vb2_buf, 0);
+ if (payload == 0)
+ payload = zr->buffer_size;
+ if (zr->jpg_settings.tmp_dcm == 1) {
+ /* fill 1 stat_com entry */
+ i = (zr->jpg_dma_head -
+ zr->jpg_err_shift) & BUZ_MASK_STAT_COM;
+ if (!(zr->stat_com[i] & cpu_to_le32(1)))
+ break;
+ zr->stat_comb[i * 2] = cpu_to_le32(phys_addr);
+ zr->stat_comb[i * 2 + 1] = cpu_to_le32((payload >> 1) | 1);
+ zr->inuse[i] = buf;
+ zr->stat_com[i] = cpu_to_le32(zr->p_scb + i * 2 * 4);
+ } else {
+ /* fill 2 stat_com entries */
+ i = ((zr->jpg_dma_head -
+ zr->jpg_err_shift) & 1) * 2;
+ if (!(zr->stat_com[i] & cpu_to_le32(1)))
+ break;
+ zr->stat_com[i] = cpu_to_le32(zr->p_scb + i * 2 * 4);
+ zr->stat_com[i + 1] = cpu_to_le32(zr->p_scb + i * 2 * 4);
+
+ zr->stat_comb[i * 2] = cpu_to_le32(phys_addr);
+ zr->stat_comb[i * 2 + 1] = cpu_to_le32((payload >> 1) | 1);
+
+ zr->inuse[i] = buf;
+ zr->inuse[i + 1] = NULL;
+ }
+ zr->jpg_dma_head++;
+ }
+ spin_unlock_irqrestore(&zr->queued_bufs_lock, flags);
+ if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS)
+ zr->jpg_queued_num++;
+}
+
+/* when this is called the spinlock must be held */
+static void zoran_reap_stat_com(struct zoran *zr)
+{
+ /* move frames from DMA queue to done queue */
+
+ int i;
+ u32 stat_com;
+ unsigned int seq;
+ unsigned int dif;
+ unsigned long flags;
+ struct zr_buffer *buf;
+ unsigned int size = 0;
+ u32 fcnt;
+
+ /* In motion decompress we don't have a hardware frame counter,
+ * we just count the interrupts here */
+
+ if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS)
+ zr->jpg_seq_num++;
+
+ spin_lock_irqsave(&zr->queued_bufs_lock, flags);
+ while (zr->jpg_dma_tail < zr->jpg_dma_head) {
+ if (zr->jpg_settings.tmp_dcm == 1)
+ i = (zr->jpg_dma_tail - zr->jpg_err_shift) & BUZ_MASK_STAT_COM;
+ else
+ i = ((zr->jpg_dma_tail - zr->jpg_err_shift) & 1) * 2 + 1;
+
+ stat_com = le32_to_cpu(zr->stat_com[i]);
+ if ((stat_com & 1) == 0) {
+ spin_unlock_irqrestore(&zr->queued_bufs_lock, flags);
+ return;
+ }
+
+ fcnt = (stat_com & GENMASK(31, 24)) >> 24;
+ size = (stat_com & GENMASK(22, 1)) >> 1;
+
+ buf = zr->inuse[i];
+ buf->vbuf.vb2_buf.timestamp = ktime_get_ns();
+
+ if (zr->codec_mode == BUZ_MODE_MOTION_COMPRESS) {
+ vb2_set_plane_payload(&buf->vbuf.vb2_buf, 0, size);
+
+ /* update sequence number with the help of the counter in stat_com */
+ seq = (fcnt + zr->jpg_err_seq) & 0xff;
+ dif = (seq - zr->jpg_seq_num) & 0xff;
+ zr->jpg_seq_num += dif;
+ }
+ buf->vbuf.sequence = zr->jpg_settings.tmp_dcm ==
+ 2 ? (zr->jpg_seq_num >> 1) : zr->jpg_seq_num;
+ zr->inuse[i] = NULL;
+ if (zr->jpg_settings.tmp_dcm != 1)
+ buf->vbuf.field = zr->jpg_settings.odd_even ?
+ V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM;
+ else
+ buf->vbuf.field = zr->jpg_settings.odd_even ?
+ V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT;
+ vb2_buffer_done(&buf->vbuf.vb2_buf, VB2_BUF_STATE_DONE);
+
+ zr->jpg_dma_tail++;
+ }
+ spin_unlock_irqrestore(&zr->queued_bufs_lock, flags);
+}
+
+irqreturn_t zoran_irq(int irq, void *dev_id)
+{
+ struct zoran *zr = dev_id;
+ u32 stat, astat;
+
+ stat = count_reset_interrupt(zr);
+ astat = stat & IRQ_MASK;
+ if (astat & zr->card.vsync_int) {
+ if (zr->running == ZORAN_MAP_MODE_RAW) {
+ if ((btread(ZR36057_VSSFGR) & ZR36057_VSSFGR_SNAP_SHOT) == 0)
+ pci_warn(zr->pci_dev, "BuzIRQ with SnapShot off ???\n");
+ if ((btread(ZR36057_VSSFGR) & ZR36057_VSSFGR_FRAME_GRAB) == 0)
+ zr_set_buf(zr);
+ return IRQ_HANDLED;
+ }
+ if (astat & ZR36057_ISR_JPEG_REP_IRQ) {
+ if (zr->codec_mode != BUZ_MODE_MOTION_DECOMPRESS &&
+ zr->codec_mode != BUZ_MODE_MOTION_COMPRESS) {
+ pci_err(zr->pci_dev, "JPG IRQ when not in good mode\n");
+ return IRQ_HANDLED;
+ }
+ zr->frame_num++;
+ zoran_reap_stat_com(zr);
+ zoran_feed_stat_com(zr);
+ return IRQ_HANDLED;
+ }
+ /* unused interrupts */
+ }
+ zr->ghost_int++;
+ return IRQ_HANDLED;
+}
+
+void zoran_set_pci_master(struct zoran *zr, int set_master)
+{
+ if (set_master) {
+ pci_set_master(zr->pci_dev);
+ } else {
+ u16 command;
+
+ pci_read_config_word(zr->pci_dev, PCI_COMMAND, &command);
+ command &= ~PCI_COMMAND_MASTER;
+ pci_write_config_word(zr->pci_dev, PCI_COMMAND, command);
+ }
+}
+
+void zoran_init_hardware(struct zoran *zr)
+{
+ /* Enable bus-mastering */
+ zoran_set_pci_master(zr, 1);
+
+ /* Initialize the board */
+ if (zr->card.init)
+ zr->card.init(zr);
+
+ decoder_call(zr, core, init, 0);
+ decoder_call(zr, video, s_std, zr->norm);
+ decoder_call(zr, video, s_routing,
+ zr->card.input[zr->input].muxsel, 0, 0);
+
+ encoder_call(zr, core, init, 0);
+ encoder_call(zr, video, s_std_output, zr->norm);
+ encoder_call(zr, video, s_routing, 0, 0, 0);
+
+ /* toggle JPEG codec sleep to sync PLL */
+ jpeg_codec_sleep(zr, 1);
+ jpeg_codec_sleep(zr, 0);
+
+ /*
+ * set individual interrupt enables (without GIRQ1)
+ * but don't global enable until zoran_open()
+ */
+ zr36057_init_vfe(zr);
+
+ zr36057_enable_jpg(zr, BUZ_MODE_IDLE);
+
+ btwrite(IRQ_MASK, ZR36057_ISR); // Clears interrupts
+}
+
+void zr36057_restart(struct zoran *zr)
+{
+ btwrite(0, ZR36057_SPGPPCR);
+ udelay(1000);
+ btor(ZR36057_SPGPPCR_SOFT_RESET, ZR36057_SPGPPCR);
+ udelay(1000);
+
+ /* assert P_Reset */
+ btwrite(0, ZR36057_JPC);
+ /* set up GPIO direction - all output */
+ btwrite(ZR36057_SPGPPCR_SOFT_RESET | 0, ZR36057_SPGPPCR);
+
+ /* set up GPIO pins and guest bus timing */
+ btwrite((0x81 << 24) | 0x8888, ZR36057_GPPGCR1);
+}
+
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Zoran zr36057/zr36067 PCI controller driver, for the
+ * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
+ * Media Labs LML33/LML33R10.
+ *
+ * This part handles card-specific data and detection
+ *
+ * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
+ */
+
+#ifndef __ZORAN_DEVICE_H__
+#define __ZORAN_DEVICE_H__
+
+/* general purpose I/O */
+extern void GPIO(struct zoran *zr, int bit, unsigned int value);
+
+/* codec (or actually: guest bus) access */
+extern int post_office_wait(struct zoran *zr);
+extern int post_office_write(struct zoran *zr, unsigned int guest, unsigned int reg, unsigned int value);
+extern int post_office_read(struct zoran *zr, unsigned int guest, unsigned int reg);
+
+extern void detect_guest_activity(struct zoran *zr);
+
+extern void jpeg_codec_sleep(struct zoran *zr, int sleep);
+extern int jpeg_codec_reset(struct zoran *zr);
+
+/* zr360x7 access to raw capture */
+extern void zr36057_overlay(struct zoran *zr, int on);
+extern void write_overlay_mask(struct zoran_fh *fh, struct v4l2_clip *vp, int count);
+extern void zr36057_set_memgrab(struct zoran *zr, int mode);
+extern int wait_grab_pending(struct zoran *zr);
+
+/* interrupts */
+extern void print_interrupts(struct zoran *zr);
+extern void clear_interrupt_counters(struct zoran *zr);
+extern irqreturn_t zoran_irq(int irq, void *dev_id);
+
+/* JPEG codec access */
+extern void jpeg_start(struct zoran *zr);
+extern void zr36057_enable_jpg(struct zoran *zr,
+ enum zoran_codec_mode mode);
+extern void zoran_feed_stat_com(struct zoran *zr);
+
+/* general */
+extern void zoran_set_pci_master(struct zoran *zr, int set_master);
+extern void zoran_init_hardware(struct zoran *zr);
+extern void zr36057_restart(struct zoran *zr);
+
+extern const struct zoran_format zoran_formats[];
+
+extern int v4l_nbufs;
+extern int v4l_bufsize;
+extern int jpg_nbufs;
+extern int jpg_bufsize;
+extern int pass_through;
+
+/* i2c */
+#define decoder_call(zr, o, f, args...) \
+ v4l2_subdev_call(zr->decoder, o, f, ##args)
+#define encoder_call(zr, o, f, args...) \
+ v4l2_subdev_call(zr->encoder, o, f, ##args)
+
+#endif /* __ZORAN_DEVICE_H__ */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Zoran zr36057/zr36067 PCI controller driver, for the
+ * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
+ * Media Labs LML33/LML33R10.
+ *
+ * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
+ *
+ * Changes for BUZ by Wolfgang Scherr <scherr@net4you.net>
+ *
+ * Changes for DC10/DC30 by Laurent Pinchart <laurent.pinchart@skynet.be>
+ *
+ * Changes for LML33R10 by Maxim Yevtyushkin <max@linuxmedialabs.com>
+ *
+ * Changes for videodev2/v4l2 by Ronald Bultje <rbultje@ronald.bitfreak.net>
+ *
+ * Based on
+ *
+ * Miro DC10 driver
+ * Copyright (C) 1999 Wolfgang Scherr <scherr@net4you.net>
+ *
+ * Iomega Buz driver version 1.0
+ * Copyright (C) 1999 Rainer Johanni <Rainer@Johanni.de>
+ *
+ * buz.0.0.3
+ * Copyright (C) 1998 Dave Perks <dperks@ibm.net>
+ *
+ * bttv - Bt848 frame grabber driver
+ * Copyright (C) 1996,97,98 Ralph Metzler (rjkm@thp.uni-koeln.de)
+ * & Marcus Metzler (mocm@thp.uni-koeln.de)
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/slab.h>
+#include <linux/pci.h>
+#include <linux/wait.h>
+
+#include <linux/interrupt.h>
+#include <linux/i2c.h>
+#include <linux/i2c-algo-bit.h>
+
+#include <linux/spinlock.h>
+
+#include <linux/videodev2.h>
+#include <media/v4l2-common.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-event.h>
+#include "videocodec.h"
+
+#include <linux/io.h>
+#include <linux/uaccess.h>
+
+#include <linux/mutex.h>
+#include "zoran.h"
+#include "zoran_device.h"
+#include "zoran_card.h"
+
+const struct zoran_format zoran_formats[] = {
+ {
+ .name = "15-bit RGB LE",
+ .fourcc = V4L2_PIX_FMT_RGB555,
+ .colorspace = V4L2_COLORSPACE_SRGB,
+ .depth = 15,
+ .flags = ZORAN_FORMAT_CAPTURE,
+ .vfespfr = ZR36057_VFESPFR_RGB555 | ZR36057_VFESPFR_ERR_DIF |
+ ZR36057_VFESPFR_LITTLE_ENDIAN,
+ }, {
+ .name = "15-bit RGB BE",
+ .fourcc = V4L2_PIX_FMT_RGB555X,
+ .colorspace = V4L2_COLORSPACE_SRGB,
+ .depth = 15,
+ .flags = ZORAN_FORMAT_CAPTURE,
+ .vfespfr = ZR36057_VFESPFR_RGB555 | ZR36057_VFESPFR_ERR_DIF,
+ }, {
+ .name = "16-bit RGB LE",
+ .fourcc = V4L2_PIX_FMT_RGB565,
+ .colorspace = V4L2_COLORSPACE_SRGB,
+ .depth = 16,
+ .flags = ZORAN_FORMAT_CAPTURE,
+ .vfespfr = ZR36057_VFESPFR_RGB565 | ZR36057_VFESPFR_ERR_DIF |
+ ZR36057_VFESPFR_LITTLE_ENDIAN,
+ }, {
+ .name = "16-bit RGB BE",
+ .fourcc = V4L2_PIX_FMT_RGB565X,
+ .colorspace = V4L2_COLORSPACE_SRGB,
+ .depth = 16,
+ .flags = ZORAN_FORMAT_CAPTURE,
+ .vfespfr = ZR36057_VFESPFR_RGB565 | ZR36057_VFESPFR_ERR_DIF,
+ }, {
+ .name = "24-bit RGB",
+ .fourcc = V4L2_PIX_FMT_BGR24,
+ .colorspace = V4L2_COLORSPACE_SRGB,
+ .depth = 24,
+ .flags = ZORAN_FORMAT_CAPTURE,
+ .vfespfr = ZR36057_VFESPFR_RGB888 | ZR36057_VFESPFR_PACK24,
+ }, {
+ .name = "32-bit RGB LE",
+ .fourcc = V4L2_PIX_FMT_BGR32,
+ .colorspace = V4L2_COLORSPACE_SRGB,
+ .depth = 32,
+ .flags = ZORAN_FORMAT_CAPTURE,
+ .vfespfr = ZR36057_VFESPFR_RGB888 | ZR36057_VFESPFR_LITTLE_ENDIAN,
+ }, {
+ .name = "32-bit RGB BE",
+ .fourcc = V4L2_PIX_FMT_RGB32,
+ .colorspace = V4L2_COLORSPACE_SRGB,
+ .depth = 32,
+ .flags = ZORAN_FORMAT_CAPTURE,
+ .vfespfr = ZR36057_VFESPFR_RGB888,
+ }, {
+ .name = "4:2:2, packed, YUYV",
+ .fourcc = V4L2_PIX_FMT_YUYV,
+ .colorspace = V4L2_COLORSPACE_SMPTE170M,
+ .depth = 16,
+ .flags = ZORAN_FORMAT_CAPTURE,
+ .vfespfr = ZR36057_VFESPFR_YUV422,
+ }, {
+ .name = "4:2:2, packed, UYVY",
+ .fourcc = V4L2_PIX_FMT_UYVY,
+ .colorspace = V4L2_COLORSPACE_SMPTE170M,
+ .depth = 16,
+ .flags = ZORAN_FORMAT_CAPTURE,
+ .vfespfr = ZR36057_VFESPFR_YUV422 | ZR36057_VFESPFR_LITTLE_ENDIAN,
+ }, {
+ .name = "Hardware-encoded Motion-JPEG",
+ .fourcc = V4L2_PIX_FMT_MJPEG,
+ .colorspace = V4L2_COLORSPACE_SMPTE170M,
+ .depth = 0,
+ .flags = ZORAN_FORMAT_CAPTURE |
+ ZORAN_FORMAT_PLAYBACK |
+ ZORAN_FORMAT_COMPRESSED,
+ }
+};
+
+#define NUM_FORMATS ARRAY_SIZE(zoran_formats)
+
+ /*
+ * small helper function for calculating buffersizes for v4l2
+ * we calculate the nearest higher power-of-two, which
+ * will be the recommended buffersize
+ */
+static __u32 zoran_v4l2_calc_bufsize(struct zoran_jpg_settings *settings)
+{
+ __u8 div = settings->ver_dcm * settings->hor_dcm * settings->tmp_dcm;
+ __u32 num = (1024 * 512) / (div);
+ __u32 result = 2;
+
+ num--;
+ while (num) {
+ num >>= 1;
+ result <<= 1;
+ }
+
+ if (result > jpg_bufsize)
+ return jpg_bufsize;
+ if (result < 8192)
+ return 8192;
+
+ return result;
+}
+
+/*
+ * V4L Buffer grabbing
+ */
+static int zoran_v4l_set_format(struct zoran *zr, int width, int height,
+ const struct zoran_format *format)
+{
+ int bpp;
+
+ /* Check size and format of the grab wanted */
+
+ if (height < BUZ_MIN_HEIGHT || width < BUZ_MIN_WIDTH ||
+ height > BUZ_MAX_HEIGHT || width > BUZ_MAX_WIDTH) {
+ pci_err(zr->pci_dev, "%s - wrong frame size (%dx%d)\n", __func__, width, height);
+ return -EINVAL;
+ }
+
+ bpp = (format->depth + 7) / 8;
+
+ zr->buffer_size = height * width * bpp;
+
+ /* Check against available buffer size */
+ if (height * width * bpp > zr->buffer_size) {
+ pci_err(zr->pci_dev, "%s - video buffer size (%d kB) is too small\n",
+ __func__, zr->buffer_size >> 10);
+ return -EINVAL;
+ }
+
+ /* The video front end needs 4-byte alinged line sizes */
+
+ if ((bpp == 2 && (width & 1)) || (bpp == 3 && (width & 3))) {
+ pci_err(zr->pci_dev, "%s - wrong frame alignment\n", __func__);
+ return -EINVAL;
+ }
+
+ zr->v4l_settings.width = width;
+ zr->v4l_settings.height = height;
+ zr->v4l_settings.format = format;
+ zr->v4l_settings.bytesperline = bpp * zr->v4l_settings.width;
+
+ return 0;
+}
+
+static int zoran_set_norm(struct zoran *zr, v4l2_std_id norm)
+{
+
+ if (!(norm & zr->card.norms)) {
+ pci_err(zr->pci_dev, "%s - unsupported norm %llx\n", __func__, norm);
+ return -EINVAL;
+ }
+
+ if (norm & V4L2_STD_SECAM)
+ zr->timing = zr->card.tvn[ZR_NORM_SECAM];
+ else if (norm & V4L2_STD_NTSC)
+ zr->timing = zr->card.tvn[ZR_NORM_NTSC];
+ else
+ zr->timing = zr->card.tvn[ZR_NORM_PAL];
+
+ decoder_call(zr, video, s_std, norm);
+ encoder_call(zr, video, s_std_output, norm);
+
+ /* Make sure the changes come into effect */
+ zr->norm = norm;
+
+ return 0;
+}
+
+static int zoran_set_input(struct zoran *zr, int input)
+{
+ if (input == zr->input)
+ return 0;
+
+ if (input < 0 || input >= zr->card.inputs) {
+ pci_err(zr->pci_dev, "%s - unsupported input %d\n", __func__, input);
+ return -EINVAL;
+ }
+
+ zr->input = input;
+
+ decoder_call(zr, video, s_routing, zr->card.input[input].muxsel, 0, 0);
+
+ return 0;
+}
+
+/*
+ * ioctl routine
+ */
+
+static int zoran_querycap(struct file *file, void *__fh, struct v4l2_capability *cap)
+{
+ struct zoran *zr = video_drvdata(file);
+
+ strscpy(cap->card, ZR_DEVNAME(zr), sizeof(cap->card));
+ strscpy(cap->driver, "zoran", sizeof(cap->driver));
+ snprintf(cap->bus_info, sizeof(cap->bus_info), "PCI:%s", pci_name(zr->pci_dev));
+ cap->device_caps = zr->video_dev->device_caps;
+ cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
+ return 0;
+}
+
+static int zoran_enum_fmt(struct zoran *zr, struct v4l2_fmtdesc *fmt, int flag)
+{
+ unsigned int num, i;
+
+ if (fmt->index >= ARRAY_SIZE(zoran_formats))
+ return -EINVAL;
+ if (fmt->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+ return -EINVAL;
+
+ for (num = i = 0; i < NUM_FORMATS; i++) {
+ if (zoran_formats[i].flags & flag && num++ == fmt->index) {
+ strscpy(fmt->description, zoran_formats[i].name,
+ sizeof(fmt->description));
+ /* fmt struct pre-zeroed, so adding '\0' not needed */
+ fmt->pixelformat = zoran_formats[i].fourcc;
+ if (zoran_formats[i].flags & ZORAN_FORMAT_COMPRESSED)
+ fmt->flags |= V4L2_FMT_FLAG_COMPRESSED;
+ return 0;
+ }
+ }
+ return -EINVAL;
+}
+
+static int zoran_enum_fmt_vid_cap(struct file *file, void *__fh,
+ struct v4l2_fmtdesc *f)
+{
+ struct zoran *zr = video_drvdata(file);
+
+ return zoran_enum_fmt(zr, f, ZORAN_FORMAT_CAPTURE);
+}
+
+#if 0
+/* TODO: output does not work yet */
+static int zoran_enum_fmt_vid_out(struct file *file, void *__fh,
+ struct v4l2_fmtdesc *f)
+{
+ struct zoran *zr = video_drvdata(file);
+
+ return zoran_enum_fmt(zr, f, ZORAN_FORMAT_PLAYBACK);
+}
+#endif
+
+static int zoran_g_fmt_vid_out(struct file *file, void *__fh,
+ struct v4l2_format *fmt)
+{
+ struct zoran *zr = video_drvdata(file);
+
+ fmt->fmt.pix.width = zr->jpg_settings.img_width / zr->jpg_settings.hor_dcm;
+ fmt->fmt.pix.height = zr->jpg_settings.img_height * 2 /
+ (zr->jpg_settings.ver_dcm * zr->jpg_settings.tmp_dcm);
+ fmt->fmt.pix.sizeimage = zr->buffer_size;
+ fmt->fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG;
+ if (zr->jpg_settings.tmp_dcm == 1)
+ fmt->fmt.pix.field = (zr->jpg_settings.odd_even ?
+ V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT);
+ else
+ fmt->fmt.pix.field = (zr->jpg_settings.odd_even ?
+ V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM);
+ fmt->fmt.pix.bytesperline = 0;
+ fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
+
+ return 0;
+}
+
+static int zoran_g_fmt_vid_cap(struct file *file, void *__fh,
+ struct v4l2_format *fmt)
+{
+ struct zoran *zr = video_drvdata(file);
+
+ if (zr->map_mode != ZORAN_MAP_MODE_RAW)
+ return zoran_g_fmt_vid_out(file, __fh, fmt);
+ fmt->fmt.pix.width = zr->v4l_settings.width;
+ fmt->fmt.pix.height = zr->v4l_settings.height;
+ fmt->fmt.pix.sizeimage = zr->buffer_size;
+ fmt->fmt.pix.pixelformat = zr->v4l_settings.format->fourcc;
+ fmt->fmt.pix.colorspace = zr->v4l_settings.format->colorspace;
+ fmt->fmt.pix.bytesperline = zr->v4l_settings.bytesperline;
+ if (BUZ_MAX_HEIGHT < (zr->v4l_settings.height * 2))
+ fmt->fmt.pix.field = V4L2_FIELD_INTERLACED;
+ else
+ fmt->fmt.pix.field = V4L2_FIELD_TOP;
+ return 0;
+}
+
+static int zoran_try_fmt_vid_out(struct file *file, void *__fh,
+ struct v4l2_format *fmt)
+{
+ struct zoran *zr = video_drvdata(file);
+ struct zoran_jpg_settings settings;
+ int res = 0;
+
+ if (fmt->fmt.pix.pixelformat != V4L2_PIX_FMT_MJPEG)
+ return -EINVAL;
+
+ settings = zr->jpg_settings;
+
+ /* we actually need to set 'real' parameters now */
+ if ((fmt->fmt.pix.height * 2) > BUZ_MAX_HEIGHT)
+ settings.tmp_dcm = 1;
+ else
+ settings.tmp_dcm = 2;
+ settings.decimation = 0;
+ if (fmt->fmt.pix.height <= zr->jpg_settings.img_height / 2)
+ settings.ver_dcm = 2;
+ else
+ settings.ver_dcm = 1;
+ if (fmt->fmt.pix.width <= zr->jpg_settings.img_width / 4)
+ settings.hor_dcm = 4;
+ else if (fmt->fmt.pix.width <= zr->jpg_settings.img_width / 2)
+ settings.hor_dcm = 2;
+ else
+ settings.hor_dcm = 1;
+ if (settings.tmp_dcm == 1)
+ settings.field_per_buff = 2;
+ else
+ settings.field_per_buff = 1;
+
+ if (settings.hor_dcm > 1) {
+ settings.img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
+ settings.img_width = (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
+ } else {
+ settings.img_x = 0;
+ settings.img_width = BUZ_MAX_WIDTH;
+ }
+
+ /* check */
+ res = zoran_check_jpg_settings(zr, &settings, 1);
+ if (res)
+ return res;
+
+ /* tell the user what we actually did */
+ fmt->fmt.pix.width = settings.img_width / settings.hor_dcm;
+ fmt->fmt.pix.height = settings.img_height * 2 /
+ (settings.tmp_dcm * settings.ver_dcm);
+ if (settings.tmp_dcm == 1)
+ fmt->fmt.pix.field = (zr->jpg_settings.odd_even ?
+ V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT);
+ else
+ fmt->fmt.pix.field = (zr->jpg_settings.odd_even ?
+ V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM);
+
+ fmt->fmt.pix.sizeimage = zoran_v4l2_calc_bufsize(&settings);
+ zr->buffer_size = fmt->fmt.pix.sizeimage;
+ fmt->fmt.pix.bytesperline = 0;
+ fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
+ return res;
+}
+
+static int zoran_try_fmt_vid_cap(struct file *file, void *__fh,
+ struct v4l2_format *fmt)
+{
+ struct zoran *zr = video_drvdata(file);
+ int bpp;
+ int i;
+
+ if (fmt->fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG)
+ return zoran_try_fmt_vid_out(file, __fh, fmt);
+
+ for (i = 0; i < NUM_FORMATS; i++)
+ if (zoran_formats[i].fourcc == fmt->fmt.pix.pixelformat)
+ break;
+
+ if (i == NUM_FORMATS) {
+ /* TODO do not return here to fix the TRY_FMT cannot handle an invalid pixelformat*/
+ return -EINVAL;
+ }
+
+ fmt->fmt.pix.pixelformat = zoran_formats[i].fourcc;
+ fmt->fmt.pix.colorspace = zoran_formats[i].colorspace;
+ if (BUZ_MAX_HEIGHT < (fmt->fmt.pix.height * 2))
+ fmt->fmt.pix.field = V4L2_FIELD_INTERLACED;
+ else
+ fmt->fmt.pix.field = V4L2_FIELD_TOP;
+
+ bpp = DIV_ROUND_UP(zoran_formats[i].depth, 8);
+ v4l_bound_align_image(&fmt->fmt.pix.width, BUZ_MIN_WIDTH, BUZ_MAX_WIDTH, bpp == 2 ? 1 : 2,
+ &fmt->fmt.pix.height, BUZ_MIN_HEIGHT, BUZ_MAX_HEIGHT, 0, 0);
+ return 0;
+}
+
+static int zoran_s_fmt_vid_out(struct file *file, void *__fh,
+ struct v4l2_format *fmt)
+{
+ struct zoran *zr = video_drvdata(file);
+ __le32 printformat = __cpu_to_le32(fmt->fmt.pix.pixelformat);
+ struct zoran_jpg_settings settings;
+ int res = 0;
+
+ pci_dbg(zr->pci_dev, "size=%dx%d, fmt=0x%x (%4.4s)\n",
+ fmt->fmt.pix.width, fmt->fmt.pix.height,
+ fmt->fmt.pix.pixelformat,
+ (char *)&printformat);
+ if (fmt->fmt.pix.pixelformat != V4L2_PIX_FMT_MJPEG)
+ return -EINVAL;
+
+ if (!fmt->fmt.pix.height || !fmt->fmt.pix.width)
+ return -EINVAL;
+
+ settings = zr->jpg_settings;
+
+ /* we actually need to set 'real' parameters now */
+ if (fmt->fmt.pix.height * 2 > BUZ_MAX_HEIGHT)
+ settings.tmp_dcm = 1;
+ else
+ settings.tmp_dcm = 2;
+ settings.decimation = 0;
+ if (fmt->fmt.pix.height <= zr->jpg_settings.img_height / 2)
+ settings.ver_dcm = 2;
+ else
+ settings.ver_dcm = 1;
+ if (fmt->fmt.pix.width <= zr->jpg_settings.img_width / 4)
+ settings.hor_dcm = 4;
+ else if (fmt->fmt.pix.width <= zr->jpg_settings.img_width / 2)
+ settings.hor_dcm = 2;
+ else
+ settings.hor_dcm = 1;
+ if (settings.tmp_dcm == 1)
+ settings.field_per_buff = 2;
+ else
+ settings.field_per_buff = 1;
+
+ if (settings.hor_dcm > 1) {
+ settings.img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
+ settings.img_width = (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
+ } else {
+ settings.img_x = 0;
+ settings.img_width = BUZ_MAX_WIDTH;
+ }
+
+ /* check */
+ res = zoran_check_jpg_settings(zr, &settings, 0);
+ if (res)
+ return res;
+
+ /* it's ok, so set them */
+ zr->jpg_settings = settings;
+
+ if (fmt->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+ zr->map_mode = ZORAN_MAP_MODE_JPG_REC;
+ else
+ zr->map_mode = ZORAN_MAP_MODE_JPG_PLAY;
+
+ zr->buffer_size = zoran_v4l2_calc_bufsize(&zr->jpg_settings);
+
+ /* tell the user what we actually did */
+ fmt->fmt.pix.width = settings.img_width / settings.hor_dcm;
+ fmt->fmt.pix.height = settings.img_height * 2 /
+ (settings.tmp_dcm * settings.ver_dcm);
+ if (settings.tmp_dcm == 1)
+ fmt->fmt.pix.field = (zr->jpg_settings.odd_even ?
+ V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT);
+ else
+ fmt->fmt.pix.field = (zr->jpg_settings.odd_even ?
+ V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM);
+ fmt->fmt.pix.bytesperline = 0;
+ fmt->fmt.pix.sizeimage = zr->buffer_size;
+ fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
+ return res;
+}
+
+static int zoran_s_fmt_vid_cap(struct file *file, void *__fh,
+ struct v4l2_format *fmt)
+{
+ struct zoran *zr = video_drvdata(file);
+ struct zoran_fh *fh = __fh;
+ int i;
+ int res = 0;
+
+ if (fmt->fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG)
+ return zoran_s_fmt_vid_out(file, fh, fmt);
+
+ for (i = 0; i < NUM_FORMATS; i++)
+ if (fmt->fmt.pix.pixelformat == zoran_formats[i].fourcc)
+ break;
+ if (i == NUM_FORMATS) {
+ pci_err(zr->pci_dev, "VIDIOC_S_FMT - unknown/unsupported format 0x%x\n",
+ fmt->fmt.pix.pixelformat);
+ /* TODO do not return here to fix the TRY_FMT cannot handle an invalid pixelformat*/
+ return -EINVAL;
+ }
+
+ fmt->fmt.pix.pixelformat = zoran_formats[i].fourcc;
+ if (fmt->fmt.pix.height > BUZ_MAX_HEIGHT)
+ fmt->fmt.pix.height = BUZ_MAX_HEIGHT;
+ if (fmt->fmt.pix.width > BUZ_MAX_WIDTH)
+ fmt->fmt.pix.width = BUZ_MAX_WIDTH;
+ if (fmt->fmt.pix.height < BUZ_MIN_HEIGHT)
+ fmt->fmt.pix.height = BUZ_MIN_HEIGHT;
+ if (fmt->fmt.pix.width < BUZ_MIN_WIDTH)
+ fmt->fmt.pix.width = BUZ_MIN_WIDTH;
+
+ zr->map_mode = ZORAN_MAP_MODE_RAW;
+
+ res = zoran_v4l_set_format(zr, fmt->fmt.pix.width, fmt->fmt.pix.height,
+ &zoran_formats[i]);
+ if (res)
+ return res;
+
+ /* tell the user the results/missing stuff */
+ fmt->fmt.pix.bytesperline = zr->v4l_settings.bytesperline;
+ fmt->fmt.pix.sizeimage = zr->buffer_size;
+ fmt->fmt.pix.colorspace = zr->v4l_settings.format->colorspace;
+ if (BUZ_MAX_HEIGHT < (zr->v4l_settings.height * 2))
+ fmt->fmt.pix.field = V4L2_FIELD_INTERLACED;
+ else
+ fmt->fmt.pix.field = V4L2_FIELD_TOP;
+ return res;
+}
+
+static int zoran_g_std(struct file *file, void *__fh, v4l2_std_id *std)
+{
+ struct zoran *zr = video_drvdata(file);
+
+ *std = zr->norm;
+ return 0;
+}
+
+static int zoran_s_std(struct file *file, void *__fh, v4l2_std_id std)
+{
+ struct zoran *zr = video_drvdata(file);
+ int res = 0;
+
+ if (zr->running != ZORAN_MAP_MODE_NONE)
+ return -EBUSY;
+
+ res = zoran_set_norm(zr, std);
+ return res;
+}
+
+static int zoran_enum_input(struct file *file, void *__fh,
+ struct v4l2_input *inp)
+{
+ struct zoran *zr = video_drvdata(file);
+
+ if (inp->index >= zr->card.inputs)
+ return -EINVAL;
+
+ strscpy(inp->name, zr->card.input[inp->index].name, sizeof(inp->name));
+ inp->type = V4L2_INPUT_TYPE_CAMERA;
+ inp->std = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM;
+
+ /* Get status of video decoder */
+ decoder_call(zr, video, g_input_status, &inp->status);
+ return 0;
+}
+
+static int zoran_g_input(struct file *file, void *__fh, unsigned int *input)
+{
+ struct zoran *zr = video_drvdata(file);
+
+ *input = zr->input;
+
+ return 0;
+}
+
+static int zoran_s_input(struct file *file, void *__fh, unsigned int input)
+{
+ struct zoran *zr = video_drvdata(file);
+ int res;
+
+ if (zr->running != ZORAN_MAP_MODE_NONE)
+ return -EBUSY;
+
+ res = zoran_set_input(zr, input);
+ return res;
+}
+
+#if 0
+/* TODO: output does not work yet */
+static int zoran_enum_output(struct file *file, void *__fh,
+ struct v4l2_output *outp)
+{
+ if (outp->index != 0)
+ return -EINVAL;
+
+ outp->index = 0;
+ outp->type = V4L2_OUTPUT_TYPE_ANALOGVGAOVERLAY;
+ outp->std = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM;
+ outp->capabilities = V4L2_OUT_CAP_STD;
+ strscpy(outp->name, "Autodetect", sizeof(outp->name));
+
+ return 0;
+}
+static int zoran_g_output(struct file *file, void *__fh, unsigned int *output)
+{
+ *output = 0;
+
+ return 0;
+}
+
+static int zoran_s_output(struct file *file, void *__fh, unsigned int output)
+{
+ if (output != 0)
+ return -EINVAL;
+
+ return 0;
+}
+#endif
+
+/* cropping (sub-frame capture) */
+static int zoran_g_selection(struct file *file, void *__fh, struct v4l2_selection *sel)
+{
+ struct zoran *zr = video_drvdata(file);
+
+ if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
+ sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
+ pci_err(zr->pci_dev, "%s invalid combinaison\n", __func__);
+ return -EINVAL;
+ }
+
+ switch (sel->target) {
+ case V4L2_SEL_TGT_CROP:
+ sel->r.top = zr->jpg_settings.img_y;
+ sel->r.left = zr->jpg_settings.img_x;
+ sel->r.width = zr->jpg_settings.img_width;
+ sel->r.height = zr->jpg_settings.img_height;
+ break;
+ case V4L2_SEL_TGT_CROP_DEFAULT:
+ sel->r.top = sel->r.left = 0;
+ sel->r.width = BUZ_MIN_WIDTH;
+ sel->r.height = BUZ_MIN_HEIGHT;
+ break;
+ case V4L2_SEL_TGT_CROP_BOUNDS:
+ sel->r.top = sel->r.left = 0;
+ sel->r.width = BUZ_MAX_WIDTH;
+ sel->r.height = BUZ_MAX_HEIGHT;
+ break;
+ default:
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static int zoran_s_selection(struct file *file, void *__fh, struct v4l2_selection *sel)
+{
+ struct zoran *zr = video_drvdata(file);
+ struct zoran_jpg_settings settings;
+ int res;
+
+ if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
+ sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+ return -EINVAL;
+
+ if (!sel->r.width || !sel->r.height)
+ return -EINVAL;
+
+ if (sel->target != V4L2_SEL_TGT_CROP)
+ return -EINVAL;
+
+ if (zr->map_mode == ZORAN_MAP_MODE_RAW) {
+ pci_err(zr->pci_dev, "VIDIOC_S_SELECTION - subcapture only supported for compressed capture\n");
+ return -EINVAL;
+ }
+
+ settings = zr->jpg_settings;
+
+ /* move into a form that we understand */
+ settings.img_x = sel->r.left;
+ settings.img_y = sel->r.top;
+ settings.img_width = sel->r.width;
+ settings.img_height = sel->r.height;
+
+ /* check validity */
+ res = zoran_check_jpg_settings(zr, &settings, 0);
+ if (res)
+ return res;
+
+ /* accept */
+ zr->jpg_settings = settings;
+ return res;
+}
+
+static int zoran_g_parm(struct file *file, void *priv, struct v4l2_streamparm *parm)
+{
+ if (parm->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+ return -EINVAL;
+
+ return 0;
+}
+
+/*
+ * Output is disabled temporarily
+ * Zoran is picky about jpeg data it accepts. At least it seems to unsupport COM and APPn.
+ * So until a way to filter data will be done, disable output.
+ */
+static const struct v4l2_ioctl_ops zoran_ioctl_ops = {
+ .vidioc_querycap = zoran_querycap,
+ .vidioc_g_parm = zoran_g_parm,
+ .vidioc_s_selection = zoran_s_selection,
+ .vidioc_g_selection = zoran_g_selection,
+ .vidioc_enum_input = zoran_enum_input,
+ .vidioc_g_input = zoran_g_input,
+ .vidioc_s_input = zoran_s_input,
+/* .vidioc_enum_output = zoran_enum_output,
+ .vidioc_g_output = zoran_g_output,
+ .vidioc_s_output = zoran_s_output,*/
+ .vidioc_g_std = zoran_g_std,
+ .vidioc_s_std = zoran_s_std,
+ .vidioc_create_bufs = vb2_ioctl_create_bufs,
+ .vidioc_reqbufs = vb2_ioctl_reqbufs,
+ .vidioc_querybuf = vb2_ioctl_querybuf,
+ .vidioc_qbuf = vb2_ioctl_qbuf,
+ .vidioc_dqbuf = vb2_ioctl_dqbuf,
+ .vidioc_expbuf = vb2_ioctl_expbuf,
+ .vidioc_streamon = vb2_ioctl_streamon,
+ .vidioc_streamoff = vb2_ioctl_streamoff,
+ .vidioc_enum_fmt_vid_cap = zoran_enum_fmt_vid_cap,
+/* .vidioc_enum_fmt_vid_out = zoran_enum_fmt_vid_out,*/
+ .vidioc_g_fmt_vid_cap = zoran_g_fmt_vid_cap,
+/* .vidioc_g_fmt_vid_out = zoran_g_fmt_vid_out,*/
+ .vidioc_s_fmt_vid_cap = zoran_s_fmt_vid_cap,
+/* .vidioc_s_fmt_vid_out = zoran_s_fmt_vid_out,*/
+ .vidioc_try_fmt_vid_cap = zoran_try_fmt_vid_cap,
+/* .vidioc_try_fmt_vid_out = zoran_try_fmt_vid_out,*/
+ .vidioc_subscribe_event = v4l2_ctrl_subscribe_event,
+ .vidioc_unsubscribe_event = v4l2_event_unsubscribe,
+};
+
+static const struct v4l2_file_operations zoran_fops = {
+ .owner = THIS_MODULE,
+ .unlocked_ioctl = video_ioctl2,
+ .open = v4l2_fh_open,
+ .release = vb2_fop_release,
+ .read = vb2_fop_read,
+ .write = vb2_fop_write,
+ .mmap = vb2_fop_mmap,
+ .poll = vb2_fop_poll,
+};
+
+const struct video_device zoran_template = {
+ .name = ZORAN_NAME,
+ .fops = &zoran_fops,
+ .ioctl_ops = &zoran_ioctl_ops,
+ .release = &zoran_vdev_release,
+ .tvnorms = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM,
+};
+
+static int zr_vb2_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, unsigned int *nplanes,
+ unsigned int sizes[], struct device *alloc_devs[])
+{
+ struct zoran *zr = vb2_get_drv_priv(vq);
+ unsigned int size = zr->buffer_size;
+
+ pci_dbg(zr->pci_dev, "%s nbuf=%u nplanes=%u", __func__, *nbuffers, *nplanes);
+
+ zr->buf_in_reserve = 0;
+
+ if (*nbuffers < vq->min_buffers_needed)
+ *nbuffers = vq->min_buffers_needed;
+
+ if (*nplanes) {
+ if (sizes[0] < size)
+ return -EINVAL;
+ else
+ return 0;
+ }
+
+ *nplanes = 1;
+ sizes[0] = size;
+
+ return 0;
+}
+
+static void zr_vb2_queue(struct vb2_buffer *vb)
+{
+ struct zoran *zr = vb2_get_drv_priv(vb->vb2_queue);
+ struct zr_buffer *buf = vb2_to_zr_buffer(vb);
+ unsigned long flags;
+
+ spin_lock_irqsave(&zr->queued_bufs_lock, flags);
+ list_add_tail(&buf->queue, &zr->queued_bufs);
+ zr->buf_in_reserve++;
+ spin_unlock_irqrestore(&zr->queued_bufs_lock, flags);
+ if (zr->running == ZORAN_MAP_MODE_JPG_REC)
+ zoran_feed_stat_com(zr);
+ zr->queued++;
+}
+
+static int zr_vb2_prepare(struct vb2_buffer *vb)
+{
+ struct zoran *zr = vb2_get_drv_priv(vb->vb2_queue);
+
+ if (vb2_plane_size(vb, 0) < zr->buffer_size)
+ return -EINVAL;
+ zr->prepared++;
+
+ return 0;
+}
+
+int zr_set_buf(struct zoran *zr)
+{
+ struct zr_buffer *buf;
+ struct vb2_v4l2_buffer *vbuf;
+ dma_addr_t phys_addr;
+ unsigned long flags;
+ u32 reg;
+
+ if (zr->running == ZORAN_MAP_MODE_NONE)
+ return 0;
+
+ if (zr->inuse[0]) {
+ buf = zr->inuse[0];
+ buf->vbuf.vb2_buf.timestamp = ktime_get_ns();
+ buf->vbuf.sequence = zr->vbseq++;
+ vbuf = &buf->vbuf;
+
+ buf->vbuf.field = V4L2_FIELD_INTERLACED;
+ vb2_set_plane_payload(&buf->vbuf.vb2_buf, 0, zr->buffer_size);
+ vb2_buffer_done(&buf->vbuf.vb2_buf, VB2_BUF_STATE_DONE);
+ zr->inuse[0] = NULL;
+ }
+
+ spin_lock_irqsave(&zr->queued_bufs_lock, flags);
+ if (list_empty(&zr->queued_bufs)) {
+ btand(~ZR36057_ICR_INT_PIN_EN, ZR36057_ICR);
+ vb2_queue_error(zr->video_dev->queue);
+ spin_unlock_irqrestore(&zr->queued_bufs_lock, flags);
+ return -EINVAL;
+ }
+ buf = list_first_entry_or_null(&zr->queued_bufs, struct zr_buffer, queue);
+ if (!buf) {
+ btand(~ZR36057_ICR_INT_PIN_EN, ZR36057_ICR);
+ vb2_queue_error(zr->video_dev->queue);
+ spin_unlock_irqrestore(&zr->queued_bufs_lock, flags);
+ return -EINVAL;
+ }
+ list_del(&buf->queue);
+ spin_unlock_irqrestore(&zr->queued_bufs_lock, flags);
+
+ vbuf = &buf->vbuf;
+ vbuf->vb2_buf.state = VB2_BUF_STATE_ACTIVE;
+ phys_addr = vb2_dma_contig_plane_dma_addr(&vbuf->vb2_buf, 0);
+
+ if (!phys_addr)
+ return -EINVAL;
+
+ zr->inuse[0] = buf;
+
+ reg = phys_addr;
+ btwrite(reg, ZR36057_VDTR);
+ if (zr->v4l_settings.height > BUZ_MAX_HEIGHT / 2)
+ reg += zr->v4l_settings.bytesperline;
+ btwrite(reg, ZR36057_VDBR);
+
+ reg = 0;
+ if (zr->v4l_settings.height > BUZ_MAX_HEIGHT / 2)
+ reg += zr->v4l_settings.bytesperline;
+ reg = (reg << ZR36057_VSSFGR_DISP_STRIDE);
+ reg |= ZR36057_VSSFGR_VID_OVF;
+ reg |= ZR36057_VSSFGR_SNAP_SHOT;
+ reg |= ZR36057_VSSFGR_FRAME_GRAB;
+ btwrite(reg, ZR36057_VSSFGR);
+
+ btor(ZR36057_VDCR_VID_EN, ZR36057_VDCR);
+ return 0;
+}
+
+static int zr_vb2_start_streaming(struct vb2_queue *vq, unsigned int count)
+{
+ struct zoran *zr = vq->drv_priv;
+ int j;
+
+ for (j = 0; j < BUZ_NUM_STAT_COM; j++) {
+ zr->stat_com[j] = cpu_to_le32(1);
+ zr->inuse[j] = NULL;
+ }
+
+ if (zr->map_mode != ZORAN_MAP_MODE_RAW) {
+ pci_info(zr->pci_dev, "START JPG\n");
+ zr36057_restart(zr);
+ zoran_init_hardware(zr);
+ if (zr->map_mode == ZORAN_MAP_MODE_JPG_REC)
+ zr36057_enable_jpg(zr, BUZ_MODE_MOTION_DECOMPRESS);
+ else
+ zr36057_enable_jpg(zr, BUZ_MODE_MOTION_COMPRESS);
+ zoran_feed_stat_com(zr);
+ jpeg_start(zr);
+ zr->running = zr->map_mode;
+ btor(ZR36057_ICR_INT_PIN_EN, ZR36057_ICR);
+ return 0;
+ }
+
+ pci_info(zr->pci_dev, "START RAW\n");
+ zr36057_restart(zr);
+ zoran_init_hardware(zr);
+
+ zr36057_enable_jpg(zr, BUZ_MODE_IDLE);
+ zr36057_set_memgrab(zr, 1);
+ zr->running = zr->map_mode;
+ btor(ZR36057_ICR_INT_PIN_EN, ZR36057_ICR);
+ return 0;
+}
+
+static void zr_vb2_stop_streaming(struct vb2_queue *vq)
+{
+ struct zoran *zr = vq->drv_priv;
+ struct zr_buffer *buf;
+ unsigned long flags;
+ int j;
+
+ btand(~ZR36057_ICR_INT_PIN_EN, ZR36057_ICR);
+ if (zr->map_mode != ZORAN_MAP_MODE_RAW)
+ zr36057_enable_jpg(zr, BUZ_MODE_IDLE);
+ zr36057_set_memgrab(zr, 0);
+ zr->running = ZORAN_MAP_MODE_NONE;
+
+ zoran_set_pci_master(zr, 0);
+
+ if (!pass_through) { /* Switch to color bar */
+ decoder_call(zr, video, s_stream, 0);
+ encoder_call(zr, video, s_routing, 2, 0, 0);
+ }
+
+ for (j = 0; j < BUZ_NUM_STAT_COM; j++) {
+ zr->stat_com[j] = cpu_to_le32(1);
+ if (!zr->inuse[j])
+ continue;
+ buf = zr->inuse[j];
+ pci_dbg(zr->pci_dev, "%s clean buf %d\n", __func__, j);
+ vb2_buffer_done(&buf->vbuf.vb2_buf, VB2_BUF_STATE_ERROR);
+ zr->inuse[j] = NULL;
+ }
+
+ spin_lock_irqsave(&zr->queued_bufs_lock, flags);
+ while (!list_empty(&zr->queued_bufs)) {
+ buf = list_entry(zr->queued_bufs.next, struct zr_buffer, queue);
+ list_del(&buf->queue);
+ vb2_buffer_done(&buf->vbuf.vb2_buf, VB2_BUF_STATE_ERROR);
+ zr->buf_in_reserve--;
+ }
+ spin_unlock_irqrestore(&zr->queued_bufs_lock, flags);
+ if (zr->buf_in_reserve)
+ pci_err(zr->pci_dev, "Buffer remaining %d\n", zr->buf_in_reserve);
+ zr->map_mode = ZORAN_MAP_MODE_RAW;
+}
+
+static const struct vb2_ops zr_video_qops = {
+ .queue_setup = zr_vb2_queue_setup,
+ .buf_queue = zr_vb2_queue,
+ .buf_prepare = zr_vb2_prepare,
+ .start_streaming = zr_vb2_start_streaming,
+ .stop_streaming = zr_vb2_stop_streaming,
+ .wait_prepare = vb2_ops_wait_prepare,
+ .wait_finish = vb2_ops_wait_finish,
+};
+
+int zoran_queue_init(struct zoran *zr, struct vb2_queue *vq)
+{
+ int err;
+
+ spin_lock_init(&zr->queued_bufs_lock);
+ INIT_LIST_HEAD(&zr->queued_bufs);
+
+ vq->dev = &zr->pci_dev->dev;
+ vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+ vq->io_modes = VB2_USERPTR | VB2_DMABUF | VB2_MMAP | VB2_READ | VB2_WRITE;
+ vq->drv_priv = zr;
+ vq->buf_struct_size = sizeof(struct zr_buffer);
+ vq->ops = &zr_video_qops;
+ vq->mem_ops = &vb2_dma_contig_memops;
+ vq->gfp_flags = GFP_DMA32,
+ vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+ vq->min_buffers_needed = 9;
+ vq->lock = &zr->lock;
+ err = vb2_queue_init(vq);
+ if (err)
+ return err;
+ zr->video_dev->queue = vq;
+ return 0;
+}
+
+void zoran_queue_exit(struct zoran *zr)
+{
+ vb2_queue_release(zr->video_dev->queue);
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Zoran ZR36016 basic configuration functions
+ *
+ * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+
+/* headerfile of this module */
+#include "zr36016.h"
+
+/* codec io API */
+#include "videocodec.h"
+
+/* it doesn't make sense to have more than 20 or so,
+ just to prevent some unwanted loops */
+#define MAX_CODECS 20
+
+/* amount of chips attached via this driver */
+static int zr36016_codecs;
+
+/* debugging is available via module parameter */
+static int debug;
+module_param(debug, int, 0);
+MODULE_PARM_DESC(debug, "Debug level (0-4)");
+
+#define dprintk(num, format, args...) \
+ do { \
+ if (debug >= num) \
+ printk(format, ##args); \
+ } while (0)
+
+/* =========================================================================
+ Local hardware I/O functions:
+
+ read/write via codec layer (registers are located in the master device)
+ ========================================================================= */
+
+/* read and write functions */
+static u8 zr36016_read(struct zr36016 *ptr, u16 reg)
+{
+ u8 value = 0;
+
+ /* just in case something is wrong... */
+ if (ptr->codec->master_data->readreg)
+ value = (ptr->codec->master_data->readreg(ptr->codec, reg)) & 0xFF;
+ else
+ pr_err("%s: invalid I/O setup, nothing read!\n", ptr->name);
+
+ dprintk(4, "%s: reading from 0x%04x: %02x\n", ptr->name, reg, value);
+
+ return value;
+}
+
+static void zr36016_write(struct zr36016 *ptr, u16 reg, u8 value)
+{
+ dprintk(4, "%s: writing 0x%02x to 0x%04x\n", ptr->name, value, reg);
+
+ // just in case something is wrong...
+ if (ptr->codec->master_data->writereg)
+ ptr->codec->master_data->writereg(ptr->codec, reg, value);
+ else
+ pr_err("%s: invalid I/O setup, nothing written!\n", ptr->name);
+}
+
+/* indirect read and write functions */
+/* the 016 supports auto-addr-increment, but
+ * writing it all time cost not much and is safer... */
+static u8 zr36016_readi(struct zr36016 *ptr, u16 reg)
+{
+ u8 value = 0;
+
+ /* just in case something is wrong... */
+ if ((ptr->codec->master_data->writereg) && (ptr->codec->master_data->readreg)) {
+ ptr->codec->master_data->writereg(ptr->codec, ZR016_IADDR, reg & 0x0F); // ADDR
+ value = (ptr->codec->master_data->readreg(ptr->codec, ZR016_IDATA)) & 0xFF; // DATA
+ } else {
+ pr_err("%s: invalid I/O setup, nothing read (i)!\n", ptr->name);
+ }
+
+ dprintk(4, "%s: reading indirect from 0x%04x: %02x\n", ptr->name, reg, value);
+ return value;
+}
+
+static void zr36016_writei(struct zr36016 *ptr, u16 reg, u8 value)
+{
+ dprintk(4, "%s: writing indirect 0x%02x to 0x%04x\n", ptr->name,
+ value, reg);
+
+ /* just in case something is wrong... */
+ if (ptr->codec->master_data->writereg) {
+ ptr->codec->master_data->writereg(ptr->codec, ZR016_IADDR, reg & 0x0F); // ADDR
+ ptr->codec->master_data->writereg(ptr->codec, ZR016_IDATA, value & 0x0FF); // DATA
+ } else {
+ pr_err("%s: invalid I/O setup, nothing written (i)!\n", ptr->name);
+ }
+}
+
+/* =========================================================================
+ Local helper function:
+
+ version read
+ ========================================================================= */
+
+/* version kept in datastructure */
+static u8 zr36016_read_version(struct zr36016 *ptr)
+{
+ ptr->version = zr36016_read(ptr, 0) >> 4;
+ return ptr->version;
+}
+
+/* =========================================================================
+ Local helper function:
+
+ basic test of "connectivity", writes/reads to/from PAX-Lo register
+ ========================================================================= */
+
+static int zr36016_basic_test(struct zr36016 *ptr)
+{
+ if (debug) {
+ int i;
+
+ zr36016_writei(ptr, ZR016I_PAX_LO, 0x55);
+ dprintk(1, KERN_INFO "%s: registers: ", ptr->name);
+ for (i = 0; i <= 0x0b; i++)
+ dprintk(1, "%02x ", zr36016_readi(ptr, i));
+ dprintk(1, "\n");
+ }
+ // for testing just write 0, then the default value to a register and read
+ // it back in both cases
+ zr36016_writei(ptr, ZR016I_PAX_LO, 0x00);
+ if (zr36016_readi(ptr, ZR016I_PAX_LO) != 0x0) {
+ pr_err("%s: attach failed, can't connect to vfe processor!\n", ptr->name);
+ return -ENXIO;
+ }
+ zr36016_writei(ptr, ZR016I_PAX_LO, 0x0d0);
+ if (zr36016_readi(ptr, ZR016I_PAX_LO) != 0x0d0) {
+ pr_err("%s: attach failed, can't connect to vfe processor!\n", ptr->name);
+ return -ENXIO;
+ }
+ // we allow version numbers from 0-3, should be enough, though
+ zr36016_read_version(ptr);
+ if (ptr->version & 0x0c) {
+ pr_err("%s: attach failed, suspicious version %d found...\n", ptr->name,
+ ptr->version);
+ return -ENXIO;
+ }
+
+ return 0; /* looks good! */
+}
+
+/* =========================================================================
+ Local helper function:
+
+ simple loop for pushing the init datasets - NO USE --
+ ========================================================================= */
+
+#if 0
+static int zr36016_pushit(struct zr36016 *ptr,
+ u16 startreg,
+ u16 len,
+ const char *data)
+{
+ int i = 0;
+
+ dprintk(4, "%s: write data block to 0x%04x (len=%d)\n",
+ ptr->name, startreg, len);
+ while (i < len) {
+ zr36016_writei(ptr, startreg++, data[i++]);
+ }
+
+ return i;
+}
+#endif
+
+/* =========================================================================
+ Basic datasets & init:
+
+ //TODO//
+ ========================================================================= */
+
+static void zr36016_init(struct zr36016 *ptr)
+{
+ // stop any processing
+ zr36016_write(ptr, ZR016_GOSTOP, 0);
+
+ // mode setup (yuv422 in and out, compression/expansuon due to mode)
+ zr36016_write(ptr, ZR016_MODE,
+ ZR016_YUV422 | ZR016_YUV422_YUV422 |
+ (ptr->mode == CODEC_DO_COMPRESSION ?
+ ZR016_COMPRESSION : ZR016_EXPANSION));
+
+ // misc setup
+ zr36016_writei(ptr, ZR016I_SETUP1,
+ (ptr->xdec ? (ZR016_HRFL | ZR016_HORZ) : 0) |
+ (ptr->ydec ? ZR016_VERT : 0) | ZR016_CNTI);
+ zr36016_writei(ptr, ZR016I_SETUP2, ZR016_CCIR);
+
+ // Window setup
+ // (no extra offset for now, norm defines offset, default width height)
+ zr36016_writei(ptr, ZR016I_PAX_HI, ptr->width >> 8);
+ zr36016_writei(ptr, ZR016I_PAX_LO, ptr->width & 0xFF);
+ zr36016_writei(ptr, ZR016I_PAY_HI, ptr->height >> 8);
+ zr36016_writei(ptr, ZR016I_PAY_LO, ptr->height & 0xFF);
+ zr36016_writei(ptr, ZR016I_NAX_HI, ptr->xoff >> 8);
+ zr36016_writei(ptr, ZR016I_NAX_LO, ptr->xoff & 0xFF);
+ zr36016_writei(ptr, ZR016I_NAY_HI, ptr->yoff >> 8);
+ zr36016_writei(ptr, ZR016I_NAY_LO, ptr->yoff & 0xFF);
+
+ /* shall we continue now, please? */
+ zr36016_write(ptr, ZR016_GOSTOP, 1);
+}
+
+/* =========================================================================
+ CODEC API FUNCTIONS
+
+ this functions are accessed by the master via the API structure
+ ========================================================================= */
+
+/* set compression/expansion mode and launches codec -
+ this should be the last call from the master before starting processing */
+static int zr36016_set_mode(struct videocodec *codec, int mode)
+{
+ struct zr36016 *ptr = (struct zr36016 *)codec->data;
+
+ dprintk(2, "%s: set_mode %d call\n", ptr->name, mode);
+
+ if ((mode != CODEC_DO_EXPANSION) && (mode != CODEC_DO_COMPRESSION))
+ return -EINVAL;
+
+ ptr->mode = mode;
+ zr36016_init(ptr);
+
+ return 0;
+}
+
+/* set picture size */
+static int zr36016_set_video(struct videocodec *codec, const struct tvnorm *norm,
+ struct vfe_settings *cap, struct vfe_polarity *pol)
+{
+ struct zr36016 *ptr = (struct zr36016 *)codec->data;
+
+ dprintk(2, "%s: set_video %d.%d, %d/%d-%dx%d (0x%x) call\n",
+ ptr->name, norm->h_start, norm->v_start,
+ cap->x, cap->y, cap->width, cap->height,
+ cap->decimation);
+
+ /* if () return -EINVAL;
+ * trust the master driver that it knows what it does - so
+ * we allow invalid startx/y for now ... */
+ ptr->width = cap->width;
+ ptr->height = cap->height;
+ /* (Ronald) This is ugly. zoran_device.c, line 387
+ * already mentions what happens if h_start is even
+ * (blue faces, etc., cr/cb inversed). There's probably
+ * some good reason why h_start is 0 instead of 1, so I'm
+ * leaving it to this for now, but really... This can be
+ * done a lot simpler */
+ ptr->xoff = (norm->h_start ? norm->h_start : 1) + cap->x;
+ /* Something to note here (I don't understand it), setting
+ * v_start too high will cause the codec to 'not work'. I
+ * really don't get it. values of 16 (v_start) already break
+ * it here. Just '0' seems to work. More testing needed! */
+ ptr->yoff = norm->v_start + cap->y;
+ /* (Ronald) dzjeeh, can't this thing do hor_decimation = 4? */
+ ptr->xdec = ((cap->decimation & 0xff) == 1) ? 0 : 1;
+ ptr->ydec = (((cap->decimation >> 8) & 0xff) == 1) ? 0 : 1;
+
+ return 0;
+}
+
+/* additional control functions */
+static int zr36016_control(struct videocodec *codec, int type, int size, void *data)
+{
+ struct zr36016 *ptr = (struct zr36016 *)codec->data;
+ int *ival = (int *)data;
+
+ dprintk(2, "%s: control %d call with %d byte\n", ptr->name, type, size);
+
+ switch (type) {
+ case CODEC_G_STATUS: /* get last status - we don't know it ... */
+ if (size != sizeof(int))
+ return -EFAULT;
+ *ival = 0;
+ break;
+
+ case CODEC_G_CODEC_MODE:
+ if (size != sizeof(int))
+ return -EFAULT;
+ *ival = 0;
+ break;
+
+ case CODEC_S_CODEC_MODE:
+ if (size != sizeof(int))
+ return -EFAULT;
+ if (*ival != 0)
+ return -EINVAL;
+ /* not needed, do nothing */
+ return 0;
+
+ case CODEC_G_VFE:
+ case CODEC_S_VFE:
+ return 0;
+
+ case CODEC_S_MMAP:
+ /* not available, give an error */
+ return -ENXIO;
+
+ default:
+ return -EINVAL;
+ }
+
+ return size;
+}
+
+/* =========================================================================
+ Exit and unregister function:
+
+ Deinitializes Zoran's JPEG processor
+ ========================================================================= */
+
+static int zr36016_unset(struct videocodec *codec)
+{
+ struct zr36016 *ptr = codec->data;
+
+ if (ptr) {
+ /* do wee need some codec deinit here, too ???? */
+
+ dprintk(1, "%s: finished codec #%d\n", ptr->name, ptr->num);
+ kfree(ptr);
+ codec->data = NULL;
+
+ zr36016_codecs--;
+ return 0;
+ }
+
+ return -EFAULT;
+}
+
+/* =========================================================================
+ Setup and registry function:
+
+ Initializes Zoran's JPEG processor
+
+ Also sets pixel size, average code size, mode (compr./decompr.)
+ (the given size is determined by the processor with the video interface)
+ ========================================================================= */
+
+static int zr36016_setup(struct videocodec *codec)
+{
+ struct zr36016 *ptr;
+ int res;
+
+ dprintk(2, "zr36016: initializing VFE subsystem #%d.\n", zr36016_codecs);
+
+ if (zr36016_codecs == MAX_CODECS) {
+ pr_err("zr36016: Can't attach more codecs!\n");
+ return -ENOSPC;
+ }
+ //mem structure init
+ codec->data = ptr = kzalloc(sizeof(struct zr36016), GFP_KERNEL);
+ if (!ptr)
+ return -ENOMEM;
+
+ snprintf(ptr->name, sizeof(ptr->name), "zr36016[%d]", zr36016_codecs);
+ ptr->num = zr36016_codecs++;
+ ptr->codec = codec;
+
+ //testing
+ res = zr36016_basic_test(ptr);
+ if (res < 0) {
+ zr36016_unset(codec);
+ return res;
+ }
+ //final setup
+ ptr->mode = CODEC_DO_COMPRESSION;
+ ptr->width = 768;
+ ptr->height = 288;
+ ptr->xdec = 1;
+ ptr->ydec = 0;
+ zr36016_init(ptr);
+
+ dprintk(1, KERN_INFO "%s: codec v%d attached and running\n", ptr->name, ptr->version);
+
+ return 0;
+}
+
+static const struct videocodec zr36016_codec = {
+ .owner = THIS_MODULE,
+ .name = "zr36016",
+ .magic = 0L, /* magic not used */
+ .flags =
+ CODEC_FLAG_HARDWARE | CODEC_FLAG_VFE | CODEC_FLAG_ENCODER |
+ CODEC_FLAG_DECODER,
+ .type = CODEC_TYPE_ZR36016,
+ .setup = zr36016_setup, /* functionality */
+ .unset = zr36016_unset,
+ .set_mode = zr36016_set_mode,
+ .set_video = zr36016_set_video,
+ .control = zr36016_control,
+ /* others are not used */
+};
+
+/* =========================================================================
+ HOOK IN DRIVER AS KERNEL MODULE
+ ========================================================================= */
+
+static int __init zr36016_init_module(void)
+{
+ //dprintk(1, "ZR36016 driver %s\n",ZR016_VERSION);
+ zr36016_codecs = 0;
+ return videocodec_register(&zr36016_codec);
+}
+
+static void __exit zr36016_cleanup_module(void)
+{
+ if (zr36016_codecs) {
+ dprintk(1,
+ "zr36016: something's wrong - %d codecs left somehow.\n",
+ zr36016_codecs);
+ }
+ videocodec_unregister(&zr36016_codec);
+}
+
+module_init(zr36016_init_module);
+module_exit(zr36016_cleanup_module);
+
+MODULE_AUTHOR("Wolfgang Scherr <scherr@net4you.at>");
+MODULE_DESCRIPTION("Driver module for ZR36016 video frontends");
+MODULE_LICENSE("GPL");
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Zoran ZR36016 basic configuration functions - header file
+ *
+ * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
+ */
+
+#ifndef ZR36016_H
+#define ZR36016_H
+
+/* data stored for each zoran jpeg codec chip */
+struct zr36016 {
+ char name[32];
+ int num;
+ /* io datastructure */
+ struct videocodec *codec;
+ // coder status
+ __u8 version;
+ // actual coder setup
+ int mode;
+
+ __u16 xoff;
+ __u16 yoff;
+ __u16 width;
+ __u16 height;
+ __u16 xdec;
+ __u16 ydec;
+};
+
+/* direct register addresses */
+#define ZR016_GOSTOP 0x00
+#define ZR016_MODE 0x01
+#define ZR016_IADDR 0x02
+#define ZR016_IDATA 0x03
+
+/* indirect register addresses */
+#define ZR016I_SETUP1 0x00
+#define ZR016I_SETUP2 0x01
+#define ZR016I_NAX_LO 0x02
+#define ZR016I_NAX_HI 0x03
+#define ZR016I_PAX_LO 0x04
+#define ZR016I_PAX_HI 0x05
+#define ZR016I_NAY_LO 0x06
+#define ZR016I_NAY_HI 0x07
+#define ZR016I_PAY_LO 0x08
+#define ZR016I_PAY_HI 0x09
+#define ZR016I_NOL_LO 0x0a
+#define ZR016I_NOL_HI 0x0b
+
+/* possible values for mode register */
+#define ZR016_RGB444_YUV444 0x00
+#define ZR016_RGB444_YUV422 0x01
+#define ZR016_RGB444_YUV411 0x02
+#define ZR016_RGB444_Y400 0x03
+#define ZR016_RGB444_RGB444 0x04
+#define ZR016_YUV444_YUV444 0x08
+#define ZR016_YUV444_YUV422 0x09
+#define ZR016_YUV444_YUV411 0x0a
+#define ZR016_YUV444_Y400 0x0b
+#define ZR016_YUV444_RGB444 0x0c
+#define ZR016_YUV422_YUV422 0x11
+#define ZR016_YUV422_YUV411 0x12
+#define ZR016_YUV422_Y400 0x13
+#define ZR016_YUV411_YUV411 0x16
+#define ZR016_YUV411_Y400 0x17
+#define ZR016_4444_4444 0x19
+#define ZR016_100_100 0x1b
+
+#define ZR016_RGB444 0x00
+#define ZR016_YUV444 0x20
+#define ZR016_YUV422 0x40
+
+#define ZR016_COMPRESSION 0x80
+#define ZR016_EXPANSION 0x80
+
+/* possible values for setup 1 register */
+#define ZR016_CKRT 0x80
+#define ZR016_VERT 0x40
+#define ZR016_HORZ 0x20
+#define ZR016_HRFL 0x10
+#define ZR016_DSFL 0x08
+#define ZR016_SBFL 0x04
+#define ZR016_RSTR 0x02
+#define ZR016_CNTI 0x01
+
+/* possible values for setup 2 register */
+#define ZR016_SYEN 0x40
+#define ZR016_CCIR 0x04
+#define ZR016_SIGN 0x02
+#define ZR016_YMCS 0x01
+
+#endif /*fndef ZR36016_H */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Zoran ZR36050 basic configuration functions
+ *
+ * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
+ */
+
+#define ZR050_VERSION "v0.7.1"
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+
+#include <linux/types.h>
+#include <linux/wait.h>
+
+/* I/O commands, error codes */
+#include <asm/io.h>
+
+/* headerfile of this module */
+#include "zr36050.h"
+
+/* codec io API */
+#include "videocodec.h"
+
+/* it doesn't make sense to have more than 20 or so,
+ just to prevent some unwanted loops */
+#define MAX_CODECS 20
+
+/* amount of chips attached via this driver */
+static int zr36050_codecs;
+
+/* debugging is available via module parameter */
+static int debug;
+module_param(debug, int, 0);
+MODULE_PARM_DESC(debug, "Debug level (0-4)");
+
+#define dprintk(num, format, args...) \
+ do { \
+ if (debug >= num) \
+ printk(format, ##args); \
+ } while (0)
+
+/* =========================================================================
+ Local hardware I/O functions:
+
+ read/write via codec layer (registers are located in the master device)
+ ========================================================================= */
+
+/* read and write functions */
+static u8 zr36050_read(struct zr36050 *ptr, u16 reg)
+{
+ u8 value = 0;
+
+ /* just in case something is wrong... */
+ if (ptr->codec->master_data->readreg)
+ value = (ptr->codec->master_data->readreg(ptr->codec, reg)) & 0xFF;
+ else
+ dprintk(1,
+ KERN_ERR "%s: invalid I/O setup, nothing read!\n", ptr->name);
+
+ dprintk(4, "%s: reading from 0x%04x: %02x\n", ptr->name, reg, value);
+
+ return value;
+}
+
+static void zr36050_write(struct zr36050 *ptr, u16 reg, u8 value)
+{
+ dprintk(4, "%s: writing 0x%02x to 0x%04x\n", ptr->name, value, reg);
+
+ /* just in case something is wrong... */
+ if (ptr->codec->master_data->writereg)
+ ptr->codec->master_data->writereg(ptr->codec, reg, value);
+ else
+ dprintk(1,
+ KERN_ERR
+ "%s: invalid I/O setup, nothing written!\n",
+ ptr->name);
+}
+
+/* =========================================================================
+ Local helper function:
+
+ status read
+ ========================================================================= */
+
+/* status is kept in datastructure */
+static u8 zr36050_read_status1(struct zr36050 *ptr)
+{
+ ptr->status1 = zr36050_read(ptr, ZR050_STATUS_1);
+
+ zr36050_read(ptr, 0);
+ return ptr->status1;
+}
+
+/* =========================================================================
+ Local helper function:
+
+ scale factor read
+ ========================================================================= */
+
+/* scale factor is kept in datastructure */
+static u16 zr36050_read_scalefactor(struct zr36050 *ptr)
+{
+ ptr->scalefact = (zr36050_read(ptr, ZR050_SF_HI) << 8) |
+ (zr36050_read(ptr, ZR050_SF_LO) & 0xFF);
+
+ /* leave 0 selected for an eventually GO from master */
+ zr36050_read(ptr, 0);
+ return ptr->scalefact;
+}
+
+/* =========================================================================
+ Local helper function:
+
+ wait if codec is ready to proceed (end of processing) or time is over
+ ========================================================================= */
+
+static void zr36050_wait_end(struct zr36050 *ptr)
+{
+ int i = 0;
+
+ while (!(zr36050_read_status1(ptr) & 0x4)) {
+ udelay(1);
+ if (i++ > 200000) { // 200ms, there is for sure something wrong!!!
+ dprintk(1,
+ "%s: timeout at wait_end (last status: 0x%02x)\n",
+ ptr->name, ptr->status1);
+ break;
+ }
+ }
+}
+
+/* =========================================================================
+ Local helper function:
+
+ basic test of "connectivity", writes/reads to/from memory the SOF marker
+ ========================================================================= */
+
+static int zr36050_basic_test(struct zr36050 *ptr)
+{
+ zr36050_write(ptr, ZR050_SOF_IDX, 0x00);
+ zr36050_write(ptr, ZR050_SOF_IDX + 1, 0x00);
+ if ((zr36050_read(ptr, ZR050_SOF_IDX) |
+ zr36050_read(ptr, ZR050_SOF_IDX + 1)) != 0x0000) {
+ dprintk(1,
+ KERN_ERR
+ "%s: attach failed, can't connect to jpeg processor!\n",
+ ptr->name);
+ return -ENXIO;
+ }
+ zr36050_write(ptr, ZR050_SOF_IDX, 0xff);
+ zr36050_write(ptr, ZR050_SOF_IDX + 1, 0xc0);
+ if (((zr36050_read(ptr, ZR050_SOF_IDX) << 8) |
+ zr36050_read(ptr, ZR050_SOF_IDX + 1)) != 0xffc0) {
+ dprintk(1,
+ KERN_ERR
+ "%s: attach failed, can't connect to jpeg processor!\n",
+ ptr->name);
+ return -ENXIO;
+ }
+
+ zr36050_wait_end(ptr);
+ if ((ptr->status1 & 0x4) == 0) {
+ dprintk(1,
+ KERN_ERR
+ "%s: attach failed, jpeg processor failed (end flag)!\n",
+ ptr->name);
+ return -EBUSY;
+ }
+
+ return 0; /* looks good! */
+}
+
+/* =========================================================================
+ Local helper function:
+
+ simple loop for pushing the init datasets
+ ========================================================================= */
+
+static int zr36050_pushit(struct zr36050 *ptr, u16 startreg, u16 len, const char *data)
+{
+ int i = 0;
+
+ dprintk(4, "%s: write data block to 0x%04x (len=%d)\n", ptr->name,
+ startreg, len);
+ while (i < len)
+ zr36050_write(ptr, startreg++, data[i++]);
+
+ return i;
+}
+
+/* =========================================================================
+ Basic datasets:
+
+ jpeg baseline setup data (you find it on lots places in internet, or just
+ extract it from any regular .jpg image...)
+
+ Could be variable, but until it's not needed it they are just fixed to save
+ memory. Otherwise expand zr36050 structure with arrays, push the values to
+ it and initialize from there, as e.g. the linux zr36057/60 driver does it.
+ ========================================================================= */
+
+static const char zr36050_dqt[0x86] = {
+ 0xff, 0xdb, //Marker: DQT
+ 0x00, 0x84, //Length: 2*65+2
+ 0x00, //Pq,Tq first table
+ 0x10, 0x0b, 0x0c, 0x0e, 0x0c, 0x0a, 0x10, 0x0e,
+ 0x0d, 0x0e, 0x12, 0x11, 0x10, 0x13, 0x18, 0x28,
+ 0x1a, 0x18, 0x16, 0x16, 0x18, 0x31, 0x23, 0x25,
+ 0x1d, 0x28, 0x3a, 0x33, 0x3d, 0x3c, 0x39, 0x33,
+ 0x38, 0x37, 0x40, 0x48, 0x5c, 0x4e, 0x40, 0x44,
+ 0x57, 0x45, 0x37, 0x38, 0x50, 0x6d, 0x51, 0x57,
+ 0x5f, 0x62, 0x67, 0x68, 0x67, 0x3e, 0x4d, 0x71,
+ 0x79, 0x70, 0x64, 0x78, 0x5c, 0x65, 0x67, 0x63,
+ 0x01, //Pq,Tq second table
+ 0x11, 0x12, 0x12, 0x18, 0x15, 0x18, 0x2f, 0x1a,
+ 0x1a, 0x2f, 0x63, 0x42, 0x38, 0x42, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63
+};
+
+static const char zr36050_dht[0x1a4] = {
+ 0xff, 0xc4, //Marker: DHT
+ 0x01, 0xa2, //Length: 2*AC, 2*DC
+ 0x00, //DC first table
+ 0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+ 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
+ 0x01, //DC second table
+ 0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+ 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
+ 0x10, //AC first table
+ 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03,
+ 0x05, 0x05, 0x04, 0x04, 0x00, 0x00,
+ 0x01, 0x7D, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11,
+ 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61,
+ 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1,
+ 0x08, 0x23, 0x42, 0xB1, 0xC1, 0x15, 0x52, 0xD1, 0xF0, 0x24,
+ 0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17,
+ 0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x34,
+ 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
+ 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
+ 0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
+ 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
+ 0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88,
+ 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99,
+ 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8,
+ 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9,
+ 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8,
+ 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9,
+ 0xDA, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
+ 0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
+ 0xF8, 0xF9, 0xFA,
+ 0x11, //AC second table
+ 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04,
+ 0x07, 0x05, 0x04, 0x04, 0x00, 0x01,
+ 0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04,
+ 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71,
+ 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91,
+ 0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, 0x15, 0x62,
+ 0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25,
+ 0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, 0x27, 0x28, 0x29, 0x2A,
+ 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
+ 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
+ 0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
+ 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
+ 0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+ 0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98,
+ 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
+ 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8,
+ 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
+ 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8,
+ 0xD9, 0xDA, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
+ 0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8,
+ 0xF9, 0xFA
+};
+
+/* jpeg baseline setup, this is just fixed in this driver (YUV pictures) */
+#define NO_OF_COMPONENTS 0x3 //Y,U,V
+#define BASELINE_PRECISION 0x8 //MCU size (?)
+static const char zr36050_tq[8] = { 0, 1, 1, 0, 0, 0, 0, 0 }; //table idx's QT
+static const char zr36050_td[8] = { 0, 1, 1, 0, 0, 0, 0, 0 }; //table idx's DC
+static const char zr36050_ta[8] = { 0, 1, 1, 0, 0, 0, 0, 0 }; //table idx's AC
+
+/* horizontal 422 decimation setup (maybe we support 411 or so later, too) */
+static const char zr36050_decimation_h[8] = { 2, 1, 1, 0, 0, 0, 0, 0 };
+static const char zr36050_decimation_v[8] = { 1, 1, 1, 0, 0, 0, 0, 0 };
+
+/* =========================================================================
+ Local helper functions:
+
+ calculation and setup of parameter-dependent JPEG baseline segments
+ (needed for compression only)
+ ========================================================================= */
+
+/* ------------------------------------------------------------------------- */
+
+/* SOF (start of frame) segment depends on width, height and sampling ratio
+ of each color component */
+
+static int zr36050_set_sof(struct zr36050 *ptr)
+{
+ char sof_data[34]; // max. size of register set
+ int i;
+
+ dprintk(3, "%s: write SOF (%dx%d, %d components)\n", ptr->name,
+ ptr->width, ptr->height, NO_OF_COMPONENTS);
+ sof_data[0] = 0xff;
+ sof_data[1] = 0xc0;
+ sof_data[2] = 0x00;
+ sof_data[3] = (3 * NO_OF_COMPONENTS) + 8;
+ sof_data[4] = BASELINE_PRECISION; // only '8' possible with zr36050
+ sof_data[5] = (ptr->height) >> 8;
+ sof_data[6] = (ptr->height) & 0xff;
+ sof_data[7] = (ptr->width) >> 8;
+ sof_data[8] = (ptr->width) & 0xff;
+ sof_data[9] = NO_OF_COMPONENTS;
+ for (i = 0; i < NO_OF_COMPONENTS; i++) {
+ sof_data[10 + (i * 3)] = i; // index identifier
+ sof_data[11 + (i * 3)] = (ptr->h_samp_ratio[i] << 4) | (ptr->v_samp_ratio[i]); // sampling ratios
+ sof_data[12 + (i * 3)] = zr36050_tq[i]; // Q table selection
+ }
+ return zr36050_pushit(ptr, ZR050_SOF_IDX,
+ (3 * NO_OF_COMPONENTS) + 10, sof_data);
+}
+
+/* ------------------------------------------------------------------------- */
+
+/* SOS (start of scan) segment depends on the used scan components
+ of each color component */
+
+static int zr36050_set_sos(struct zr36050 *ptr)
+{
+ char sos_data[16]; // max. size of register set
+ int i;
+
+ dprintk(3, "%s: write SOS\n", ptr->name);
+ sos_data[0] = 0xff;
+ sos_data[1] = 0xda;
+ sos_data[2] = 0x00;
+ sos_data[3] = 2 + 1 + (2 * NO_OF_COMPONENTS) + 3;
+ sos_data[4] = NO_OF_COMPONENTS;
+ for (i = 0; i < NO_OF_COMPONENTS; i++) {
+ sos_data[5 + (i * 2)] = i; // index
+ sos_data[6 + (i * 2)] = (zr36050_td[i] << 4) | zr36050_ta[i]; // AC/DC tbl.sel.
+ }
+ sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 2] = 00; // scan start
+ sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 3] = 0x3F;
+ sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 4] = 00;
+ return zr36050_pushit(ptr, ZR050_SOS1_IDX,
+ 4 + 1 + (2 * NO_OF_COMPONENTS) + 3,
+ sos_data);
+}
+
+/* ------------------------------------------------------------------------- */
+
+/* DRI (define restart interval) */
+
+static int zr36050_set_dri(struct zr36050 *ptr)
+{
+ char dri_data[6]; // max. size of register set
+
+ dprintk(3, "%s: write DRI\n", ptr->name);
+ dri_data[0] = 0xff;
+ dri_data[1] = 0xdd;
+ dri_data[2] = 0x00;
+ dri_data[3] = 0x04;
+ dri_data[4] = ptr->dri >> 8;
+ dri_data[5] = ptr->dri & 0xff;
+ return zr36050_pushit(ptr, ZR050_DRI_IDX, 6, dri_data);
+}
+
+/* =========================================================================
+ Setup function:
+
+ Setup compression/decompression of Zoran's JPEG processor
+ ( see also zoran 36050 manual )
+
+ ... sorry for the spaghetti code ...
+ ========================================================================= */
+static void zr36050_init(struct zr36050 *ptr)
+{
+ int sum = 0;
+ long bitcnt, tmp;
+
+ if (ptr->mode == CODEC_DO_COMPRESSION) {
+ dprintk(2, "%s: COMPRESSION SETUP\n", ptr->name);
+
+ /* 050 communicates with 057 in master mode */
+ zr36050_write(ptr, ZR050_HARDWARE, ZR050_HW_MSTR);
+
+ /* encoding table preload for compression */
+ zr36050_write(ptr, ZR050_MODE,
+ ZR050_MO_COMP | ZR050_MO_TLM);
+ zr36050_write(ptr, ZR050_OPTIONS, 0);
+
+ /* disable all IRQs */
+ zr36050_write(ptr, ZR050_INT_REQ_0, 0);
+ zr36050_write(ptr, ZR050_INT_REQ_1, 3); // low 2 bits always 1
+
+ /* volume control settings */
+ /*zr36050_write(ptr, ZR050_MBCV, ptr->max_block_vol);*/
+ zr36050_write(ptr, ZR050_SF_HI, ptr->scalefact >> 8);
+ zr36050_write(ptr, ZR050_SF_LO, ptr->scalefact & 0xff);
+
+ zr36050_write(ptr, ZR050_AF_HI, 0xff);
+ zr36050_write(ptr, ZR050_AF_M, 0xff);
+ zr36050_write(ptr, ZR050_AF_LO, 0xff);
+
+ /* setup the variable jpeg tables */
+ sum += zr36050_set_sof(ptr);
+ sum += zr36050_set_sos(ptr);
+ sum += zr36050_set_dri(ptr);
+
+ /* setup the fixed jpeg tables - maybe variable, though -
+ * (see table init section above) */
+ dprintk(3, "%s: write DQT, DHT, APP\n", ptr->name);
+ sum += zr36050_pushit(ptr, ZR050_DQT_IDX,
+ sizeof(zr36050_dqt), zr36050_dqt);
+ sum += zr36050_pushit(ptr, ZR050_DHT_IDX,
+ sizeof(zr36050_dht), zr36050_dht);
+ zr36050_write(ptr, ZR050_APP_IDX, 0xff);
+ zr36050_write(ptr, ZR050_APP_IDX + 1, 0xe0 + ptr->app.appn);
+ zr36050_write(ptr, ZR050_APP_IDX + 2, 0x00);
+ zr36050_write(ptr, ZR050_APP_IDX + 3, ptr->app.len + 2);
+ sum += zr36050_pushit(ptr, ZR050_APP_IDX + 4, 60,
+ ptr->app.data) + 4;
+ zr36050_write(ptr, ZR050_COM_IDX, 0xff);
+ zr36050_write(ptr, ZR050_COM_IDX + 1, 0xfe);
+ zr36050_write(ptr, ZR050_COM_IDX + 2, 0x00);
+ zr36050_write(ptr, ZR050_COM_IDX + 3, ptr->com.len + 2);
+ sum += zr36050_pushit(ptr, ZR050_COM_IDX + 4, 60,
+ ptr->com.data) + 4;
+
+ /* do the internal huffman table preload */
+ zr36050_write(ptr, ZR050_MARKERS_EN, ZR050_ME_DHTI);
+
+ zr36050_write(ptr, ZR050_GO, 1); // launch codec
+ zr36050_wait_end(ptr);
+ dprintk(2, "%s: Status after table preload: 0x%02x\n",
+ ptr->name, ptr->status1);
+
+ if ((ptr->status1 & 0x4) == 0) {
+ pr_err("%s: init aborted!\n", ptr->name);
+ return; // something is wrong, its timed out!!!!
+ }
+
+ /* setup misc. data for compression (target code sizes) */
+
+ /* size of compressed code to reach without header data */
+ sum = ptr->real_code_vol - sum;
+ bitcnt = sum << 3; /* need the size in bits */
+
+ tmp = bitcnt >> 16;
+ dprintk(3,
+ "%s: code: csize=%d, tot=%d, bit=%ld, highbits=%ld\n",
+ ptr->name, sum, ptr->real_code_vol, bitcnt, tmp);
+ zr36050_write(ptr, ZR050_TCV_NET_HI, tmp >> 8);
+ zr36050_write(ptr, ZR050_TCV_NET_MH, tmp & 0xff);
+ tmp = bitcnt & 0xffff;
+ zr36050_write(ptr, ZR050_TCV_NET_ML, tmp >> 8);
+ zr36050_write(ptr, ZR050_TCV_NET_LO, tmp & 0xff);
+
+ bitcnt -= bitcnt >> 7; // bits without stuffing
+ bitcnt -= ((bitcnt * 5) >> 6); // bits without eob
+
+ tmp = bitcnt >> 16;
+ dprintk(3, "%s: code: nettobit=%ld, highnettobits=%ld\n",
+ ptr->name, bitcnt, tmp);
+ zr36050_write(ptr, ZR050_TCV_DATA_HI, tmp >> 8);
+ zr36050_write(ptr, ZR050_TCV_DATA_MH, tmp & 0xff);
+ tmp = bitcnt & 0xffff;
+ zr36050_write(ptr, ZR050_TCV_DATA_ML, tmp >> 8);
+ zr36050_write(ptr, ZR050_TCV_DATA_LO, tmp & 0xff);
+
+ /* compression setup with or without bitrate control */
+ zr36050_write(ptr, ZR050_MODE,
+ ZR050_MO_COMP | ZR050_MO_PASS2 |
+ (ptr->bitrate_ctrl ? ZR050_MO_BRC : 0));
+
+ /* this headers seem to deliver "valid AVI" jpeg frames */
+ zr36050_write(ptr, ZR050_MARKERS_EN,
+ ZR050_ME_DQT | ZR050_ME_DHT |
+ ((ptr->app.len > 0) ? ZR050_ME_APP : 0) |
+ ((ptr->com.len > 0) ? ZR050_ME_COM : 0));
+ } else {
+ dprintk(2, "%s: EXPANSION SETUP\n", ptr->name);
+
+ /* 050 communicates with 055 in master mode */
+ zr36050_write(ptr, ZR050_HARDWARE,
+ ZR050_HW_MSTR | ZR050_HW_CFIS_2_CLK);
+
+ /* encoding table preload */
+ zr36050_write(ptr, ZR050_MODE, ZR050_MO_TLM);
+
+ /* disable all IRQs */
+ zr36050_write(ptr, ZR050_INT_REQ_0, 0);
+ zr36050_write(ptr, ZR050_INT_REQ_1, 3); // low 2 bits always 1
+
+ dprintk(3, "%s: write DHT\n", ptr->name);
+ zr36050_pushit(ptr, ZR050_DHT_IDX, sizeof(zr36050_dht),
+ zr36050_dht);
+
+ /* do the internal huffman table preload */
+ zr36050_write(ptr, ZR050_MARKERS_EN, ZR050_ME_DHTI);
+
+ zr36050_write(ptr, ZR050_GO, 1); // launch codec
+ zr36050_wait_end(ptr);
+ dprintk(2, "%s: Status after table preload: 0x%02x\n",
+ ptr->name, ptr->status1);
+
+ if ((ptr->status1 & 0x4) == 0) {
+ pr_err("%s: init aborted!\n", ptr->name);
+ return; // something is wrong, its timed out!!!!
+ }
+
+ /* setup misc. data for expansion */
+ zr36050_write(ptr, ZR050_MODE, 0);
+ zr36050_write(ptr, ZR050_MARKERS_EN, 0);
+ }
+
+ /* adr on selected, to allow GO from master */
+ zr36050_read(ptr, 0);
+}
+
+/* =========================================================================
+ CODEC API FUNCTIONS
+
+ this functions are accessed by the master via the API structure
+ ========================================================================= */
+
+/* set compression/expansion mode and launches codec -
+ this should be the last call from the master before starting processing */
+static int zr36050_set_mode(struct videocodec *codec, int mode)
+{
+ struct zr36050 *ptr = (struct zr36050 *)codec->data;
+
+ dprintk(2, "%s: set_mode %d call\n", ptr->name, mode);
+
+ if ((mode != CODEC_DO_EXPANSION) && (mode != CODEC_DO_COMPRESSION))
+ return -EINVAL;
+
+ ptr->mode = mode;
+ zr36050_init(ptr);
+
+ return 0;
+}
+
+/* set picture size (norm is ignored as the codec doesn't know about it) */
+static int zr36050_set_video(struct videocodec *codec, const struct tvnorm *norm,
+ struct vfe_settings *cap, struct vfe_polarity *pol)
+{
+ struct zr36050 *ptr = (struct zr36050 *)codec->data;
+ int size;
+
+ dprintk(2, "%s: set_video %d.%d, %d/%d-%dx%d (0x%x) q%d call\n",
+ ptr->name, norm->h_start, norm->v_start,
+ cap->x, cap->y, cap->width, cap->height,
+ cap->decimation, cap->quality);
+ /* if () return -EINVAL;
+ * trust the master driver that it knows what it does - so
+ * we allow invalid startx/y and norm for now ... */
+ ptr->width = cap->width / (cap->decimation & 0xff);
+ ptr->height = cap->height / ((cap->decimation >> 8) & 0xff);
+
+ /* (KM) JPEG quality */
+ size = ptr->width * ptr->height;
+ size *= 16; /* size in bits */
+ /* apply quality setting */
+ size = size * cap->quality / 200;
+
+ /* Minimum: 1kb */
+ if (size < 8192)
+ size = 8192;
+ /* Maximum: 7/8 of code buffer */
+ if (size > ptr->total_code_vol * 7)
+ size = ptr->total_code_vol * 7;
+
+ ptr->real_code_vol = size >> 3; /* in bytes */
+
+ /* Set max_block_vol here (previously in zr36050_init, moved
+ * here for consistency with zr36060 code */
+ zr36050_write(ptr, ZR050_MBCV, ptr->max_block_vol);
+
+ return 0;
+}
+
+/* additional control functions */
+static int zr36050_control(struct videocodec *codec, int type, int size, void *data)
+{
+ struct zr36050 *ptr = (struct zr36050 *)codec->data;
+ int *ival = (int *)data;
+
+ dprintk(2, "%s: control %d call with %d byte\n", ptr->name, type,
+ size);
+
+ switch (type) {
+ case CODEC_G_STATUS: /* get last status */
+ if (size != sizeof(int))
+ return -EFAULT;
+ zr36050_read_status1(ptr);
+ *ival = ptr->status1;
+ break;
+
+ case CODEC_G_CODEC_MODE:
+ if (size != sizeof(int))
+ return -EFAULT;
+ *ival = CODEC_MODE_BJPG;
+ break;
+
+ case CODEC_S_CODEC_MODE:
+ if (size != sizeof(int))
+ return -EFAULT;
+ if (*ival != CODEC_MODE_BJPG)
+ return -EINVAL;
+ /* not needed, do nothing */
+ return 0;
+
+ case CODEC_G_VFE:
+ case CODEC_S_VFE:
+ /* not needed, do nothing */
+ return 0;
+
+ case CODEC_S_MMAP:
+ /* not available, give an error */
+ return -ENXIO;
+
+ case CODEC_G_JPEG_TDS_BYTE: /* get target volume in byte */
+ if (size != sizeof(int))
+ return -EFAULT;
+ *ival = ptr->total_code_vol;
+ break;
+
+ case CODEC_S_JPEG_TDS_BYTE: /* get target volume in byte */
+ if (size != sizeof(int))
+ return -EFAULT;
+ ptr->total_code_vol = *ival;
+ /* (Kieran Morrissey)
+ * code copied from zr36060.c to ensure proper bitrate */
+ ptr->real_code_vol = (ptr->total_code_vol * 6) >> 3;
+ break;
+
+ case CODEC_G_JPEG_SCALE: /* get scaling factor */
+ if (size != sizeof(int))
+ return -EFAULT;
+ *ival = zr36050_read_scalefactor(ptr);
+ break;
+
+ case CODEC_S_JPEG_SCALE: /* set scaling factor */
+ if (size != sizeof(int))
+ return -EFAULT;
+ ptr->scalefact = *ival;
+ break;
+
+ case CODEC_G_JPEG_APP_DATA: { /* get appn marker data */
+ struct jpeg_app_marker *app = data;
+
+ if (size != sizeof(struct jpeg_app_marker))
+ return -EFAULT;
+
+ *app = ptr->app;
+ break;
+ }
+
+ case CODEC_S_JPEG_APP_DATA: { /* set appn marker data */
+ struct jpeg_app_marker *app = data;
+
+ if (size != sizeof(struct jpeg_app_marker))
+ return -EFAULT;
+
+ ptr->app = *app;
+ break;
+ }
+
+ case CODEC_G_JPEG_COM_DATA: { /* get comment marker data */
+ struct jpeg_com_marker *com = data;
+
+ if (size != sizeof(struct jpeg_com_marker))
+ return -EFAULT;
+
+ *com = ptr->com;
+ break;
+ }
+
+ case CODEC_S_JPEG_COM_DATA: { /* set comment marker data */
+ struct jpeg_com_marker *com = data;
+
+ if (size != sizeof(struct jpeg_com_marker))
+ return -EFAULT;
+
+ ptr->com = *com;
+ break;
+ }
+
+ default:
+ return -EINVAL;
+ }
+
+ return size;
+}
+
+/* =========================================================================
+ Exit and unregister function:
+
+ Deinitializes Zoran's JPEG processor
+ ========================================================================= */
+
+static int zr36050_unset(struct videocodec *codec)
+{
+ struct zr36050 *ptr = codec->data;
+
+ if (ptr) {
+ /* do wee need some codec deinit here, too ???? */
+
+ dprintk(1, "%s: finished codec #%d\n", ptr->name,
+ ptr->num);
+ kfree(ptr);
+ codec->data = NULL;
+
+ zr36050_codecs--;
+ return 0;
+ }
+
+ return -EFAULT;
+}
+
+/* =========================================================================
+ Setup and registry function:
+
+ Initializes Zoran's JPEG processor
+
+ Also sets pixel size, average code size, mode (compr./decompr.)
+ (the given size is determined by the processor with the video interface)
+ ========================================================================= */
+
+static int zr36050_setup(struct videocodec *codec)
+{
+ struct zr36050 *ptr;
+ int res;
+
+ dprintk(2, "zr36050: initializing MJPEG subsystem #%d.\n",
+ zr36050_codecs);
+
+ if (zr36050_codecs == MAX_CODECS) {
+ dprintk(1,
+ KERN_ERR "zr36050: Can't attach more codecs!\n");
+ return -ENOSPC;
+ }
+ //mem structure init
+ codec->data = ptr = kzalloc(sizeof(struct zr36050), GFP_KERNEL);
+ if (!ptr)
+ return -ENOMEM;
+
+ snprintf(ptr->name, sizeof(ptr->name), "zr36050[%d]",
+ zr36050_codecs);
+ ptr->num = zr36050_codecs++;
+ ptr->codec = codec;
+
+ //testing
+ res = zr36050_basic_test(ptr);
+ if (res < 0) {
+ zr36050_unset(codec);
+ return res;
+ }
+ //final setup
+ memcpy(ptr->h_samp_ratio, zr36050_decimation_h, 8);
+ memcpy(ptr->v_samp_ratio, zr36050_decimation_v, 8);
+
+ ptr->bitrate_ctrl = 0; /* 0 or 1 - fixed file size flag
+ * (what is the difference?) */
+ ptr->mode = CODEC_DO_COMPRESSION;
+ ptr->width = 384;
+ ptr->height = 288;
+ ptr->total_code_vol = 16000;
+ ptr->max_block_vol = 240;
+ ptr->scalefact = 0x100;
+ ptr->dri = 1;
+
+ /* no app/com marker by default */
+ ptr->app.appn = 0;
+ ptr->app.len = 0;
+ ptr->com.len = 0;
+
+ zr36050_init(ptr);
+
+ dprintk(1, KERN_INFO "%s: codec attached and running\n",
+ ptr->name);
+
+ return 0;
+}
+
+static const struct videocodec zr36050_codec = {
+ .owner = THIS_MODULE,
+ .name = "zr36050",
+ .magic = 0L, // magic not used
+ .flags =
+ CODEC_FLAG_JPEG | CODEC_FLAG_HARDWARE | CODEC_FLAG_ENCODER |
+ CODEC_FLAG_DECODER,
+ .type = CODEC_TYPE_ZR36050,
+ .setup = zr36050_setup, // functionality
+ .unset = zr36050_unset,
+ .set_mode = zr36050_set_mode,
+ .set_video = zr36050_set_video,
+ .control = zr36050_control,
+ // others are not used
+};
+
+/* =========================================================================
+ HOOK IN DRIVER AS KERNEL MODULE
+ ========================================================================= */
+
+static int __init zr36050_init_module(void)
+{
+ //dprintk(1, "ZR36050 driver %s\n",ZR050_VERSION);
+ zr36050_codecs = 0;
+ return videocodec_register(&zr36050_codec);
+}
+
+static void __exit zr36050_cleanup_module(void)
+{
+ if (zr36050_codecs) {
+ dprintk(1,
+ "zr36050: something's wrong - %d codecs left somehow.\n",
+ zr36050_codecs);
+ }
+ videocodec_unregister(&zr36050_codec);
+}
+
+module_init(zr36050_init_module);
+module_exit(zr36050_cleanup_module);
+
+MODULE_AUTHOR("Wolfgang Scherr <scherr@net4you.at>");
+MODULE_DESCRIPTION("Driver module for ZR36050 jpeg processors "
+ ZR050_VERSION);
+MODULE_LICENSE("GPL");
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Zoran ZR36050 basic configuration functions - header file
+ *
+ * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
+ */
+
+#ifndef ZR36050_H
+#define ZR36050_H
+
+#include "videocodec.h"
+
+/* data stored for each zoran jpeg codec chip */
+struct zr36050 {
+ char name[32];
+ int num;
+ /* io datastructure */
+ struct videocodec *codec;
+ // last coder status
+ __u8 status1;
+ // actual coder setup
+ int mode;
+
+ __u16 width;
+ __u16 height;
+
+ __u16 bitrate_ctrl;
+
+ __u32 total_code_vol;
+ __u32 real_code_vol;
+ __u16 max_block_vol;
+
+ __u8 h_samp_ratio[8];
+ __u8 v_samp_ratio[8];
+ __u16 scalefact;
+ __u16 dri;
+
+ /* com/app marker */
+ struct jpeg_com_marker com;
+ struct jpeg_app_marker app;
+};
+
+/* zr36050 register addresses */
+#define ZR050_GO 0x000
+#define ZR050_HARDWARE 0x002
+#define ZR050_MODE 0x003
+#define ZR050_OPTIONS 0x004
+#define ZR050_MBCV 0x005
+#define ZR050_MARKERS_EN 0x006
+#define ZR050_INT_REQ_0 0x007
+#define ZR050_INT_REQ_1 0x008
+#define ZR050_TCV_NET_HI 0x009
+#define ZR050_TCV_NET_MH 0x00a
+#define ZR050_TCV_NET_ML 0x00b
+#define ZR050_TCV_NET_LO 0x00c
+#define ZR050_TCV_DATA_HI 0x00d
+#define ZR050_TCV_DATA_MH 0x00e
+#define ZR050_TCV_DATA_ML 0x00f
+#define ZR050_TCV_DATA_LO 0x010
+#define ZR050_SF_HI 0x011
+#define ZR050_SF_LO 0x012
+#define ZR050_AF_HI 0x013
+#define ZR050_AF_M 0x014
+#define ZR050_AF_LO 0x015
+#define ZR050_ACV_HI 0x016
+#define ZR050_ACV_MH 0x017
+#define ZR050_ACV_ML 0x018
+#define ZR050_ACV_LO 0x019
+#define ZR050_ACT_HI 0x01a
+#define ZR050_ACT_MH 0x01b
+#define ZR050_ACT_ML 0x01c
+#define ZR050_ACT_LO 0x01d
+#define ZR050_ACV_TURN_HI 0x01e
+#define ZR050_ACV_TURN_MH 0x01f
+#define ZR050_ACV_TURN_ML 0x020
+#define ZR050_ACV_TURN_LO 0x021
+#define ZR050_STATUS_0 0x02e
+#define ZR050_STATUS_1 0x02f
+
+#define ZR050_SOF_IDX 0x040
+#define ZR050_SOS1_IDX 0x07a
+#define ZR050_SOS2_IDX 0x08a
+#define ZR050_SOS3_IDX 0x09a
+#define ZR050_SOS4_IDX 0x0aa
+#define ZR050_DRI_IDX 0x0c0
+#define ZR050_DNL_IDX 0x0c6
+#define ZR050_DQT_IDX 0x0cc
+#define ZR050_DHT_IDX 0x1d4
+#define ZR050_APP_IDX 0x380
+#define ZR050_COM_IDX 0x3c0
+
+/* zr36050 hardware register bits */
+
+#define ZR050_HW_BSWD 0x80
+#define ZR050_HW_MSTR 0x40
+#define ZR050_HW_DMA 0x20
+#define ZR050_HW_CFIS_1_CLK 0x00
+#define ZR050_HW_CFIS_2_CLK 0x04
+#define ZR050_HW_CFIS_3_CLK 0x08
+#define ZR050_HW_CFIS_4_CLK 0x0C
+#define ZR050_HW_CFIS_5_CLK 0x10
+#define ZR050_HW_CFIS_6_CLK 0x14
+#define ZR050_HW_CFIS_7_CLK 0x18
+#define ZR050_HW_CFIS_8_CLK 0x1C
+#define ZR050_HW_BELE 0x01
+
+/* zr36050 mode register bits */
+
+#define ZR050_MO_COMP 0x80
+#define ZR050_MO_ATP 0x40
+#define ZR050_MO_PASS2 0x20
+#define ZR050_MO_TLM 0x10
+#define ZR050_MO_DCONLY 0x08
+#define ZR050_MO_BRC 0x04
+
+#define ZR050_MO_ATP 0x40
+#define ZR050_MO_PASS2 0x20
+#define ZR050_MO_TLM 0x10
+#define ZR050_MO_DCONLY 0x08
+
+/* zr36050 option register bits */
+
+#define ZR050_OP_NSCN_1 0x00
+#define ZR050_OP_NSCN_2 0x20
+#define ZR050_OP_NSCN_3 0x40
+#define ZR050_OP_NSCN_4 0x60
+#define ZR050_OP_NSCN_5 0x80
+#define ZR050_OP_NSCN_6 0xA0
+#define ZR050_OP_NSCN_7 0xC0
+#define ZR050_OP_NSCN_8 0xE0
+#define ZR050_OP_OVF 0x10
+
+/* zr36050 markers-enable register bits */
+
+#define ZR050_ME_APP 0x80
+#define ZR050_ME_COM 0x40
+#define ZR050_ME_DRI 0x20
+#define ZR050_ME_DQT 0x10
+#define ZR050_ME_DHT 0x08
+#define ZR050_ME_DNL 0x04
+#define ZR050_ME_DQTI 0x02
+#define ZR050_ME_DHTI 0x01
+
+/* zr36050 status0/1 register bit masks */
+
+#define ZR050_ST_RST_MASK 0x20
+#define ZR050_ST_SOF_MASK 0x02
+#define ZR050_ST_SOS_MASK 0x02
+#define ZR050_ST_DATRDY_MASK 0x80
+#define ZR050_ST_MRKDET_MASK 0x40
+#define ZR050_ST_RFM_MASK 0x10
+#define ZR050_ST_RFD_MASK 0x08
+#define ZR050_ST_END_MASK 0x04
+#define ZR050_ST_TCVOVF_MASK 0x02
+#define ZR050_ST_DATOVF_MASK 0x01
+
+/* pixel component idx */
+
+#define ZR050_Y_COMPONENT 0
+#define ZR050_U_COMPONENT 1
+#define ZR050_V_COMPONENT 2
+
+#endif /*fndef ZR36050_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * zr36057.h - zr36057 register offsets
+ *
+ * Copyright (C) 1998 Dave Perks <dperks@ibm.net>
+ */
+
+#ifndef _ZR36057_H_
+#define _ZR36057_H_
+
+/* Zoran ZR36057 registers */
+
+#define ZR36057_VFEHCR 0x000 /* Video Front End, Horizontal Configuration Register */
+#define ZR36057_VFEHCR_HS_POL BIT(30)
+#define ZR36057_VFEHCR_H_START 10
+#define ZR36057_VFEHCR_H_END 0
+#define ZR36057_VFEHCR_HMASK 0x3ff
+
+#define ZR36057_VFEVCR 0x004 /* Video Front End, Vertical Configuration Register */
+#define ZR36057_VFEVCR_VS_POL BIT(30)
+#define ZR36057_VFEVCR_V_START 10
+#define ZR36057_VFEVCR_V_END 0
+#define ZR36057_VFEVCR_VMASK 0x3ff
+
+#define ZR36057_VFESPFR 0x008 /* Video Front End, Scaler and Pixel Format Register */
+#define ZR36057_VFESPFR_EXT_FL BIT(26)
+#define ZR36057_VFESPFR_TOP_FIELD BIT(25)
+#define ZR36057_VFESPFR_VCLK_POL BIT(24)
+#define ZR36057_VFESPFR_H_FILTER 21
+#define ZR36057_VFESPFR_HOR_DCM 14
+#define ZR36057_VFESPFR_VER_DCM 8
+#define ZR36057_VFESPFR_DISP_MODE 6
+#define ZR36057_VFESPFR_YUV422 (0<<3)
+#define ZR36057_VFESPFR_RGB888 (1<<3)
+#define ZR36057_VFESPFR_RGB565 (2<<3)
+#define ZR36057_VFESPFR_RGB555 (3<<3)
+#define ZR36057_VFESPFR_ERR_DIF (1<<2)
+#define ZR36057_VFESPFR_PACK24 (1<<1)
+#define ZR36057_VFESPFR_LITTLE_ENDIAN (1<<0)
+
+#define ZR36057_VDTR 0x00c /* Video Display "Top" Register */
+
+#define ZR36057_VDBR 0x010 /* Video Display "Bottom" Register */
+
+#define ZR36057_VSSFGR 0x014 /* Video Stride, Status, and Frame Grab Register */
+#define ZR36057_VSSFGR_DISP_STRIDE 16
+#define ZR36057_VSSFGR_VID_OVF BIT(8)
+#define ZR36057_VSSFGR_SNAP_SHOT BIT(1)
+#define ZR36057_VSSFGR_FRAME_GRAB BIT(0)
+
+#define ZR36057_VDCR 0x018 /* Video Display Configuration Register */
+#define ZR36057_VDCR_VID_EN BIT(31)
+#define ZR36057_VDCR_MIN_PIX 24
+#define ZR36057_VDCR_TRITON BIT(24)
+#define ZR36057_VDCR_VID_WIN_HT 12
+#define ZR36057_VDCR_VID_WIN_WID 0
+
+#define ZR36057_MMTR 0x01c /* Masking Map "Top" Register */
+
+#define ZR36057_MMBR 0x020 /* Masking Map "Bottom" Register */
+
+#define ZR36057_OCR 0x024 /* Overlay Control Register */
+#define ZR36057_OCR_OVL_ENABLE BIT(15)
+#define ZR36057_OCR_MASK_STRIDE 0
+
+#define ZR36057_SPGPPCR 0x028 /* System, PCI, and General Purpose Pins Control Register */
+#define ZR36057_SPGPPCR_SOFT_RESET BIT(24)
+
+#define ZR36057_GPPGCR1 0x02c /* General Purpose Pins and GuestBus Control Register (1) */
+
+#define ZR36057_MCSAR 0x030 /* MPEG Code Source Address Register */
+
+#define ZR36057_MCTCR 0x034 /* MPEG Code Transfer Control Register */
+#define ZR36057_MCTCR_COD_TIME BIT(30)
+#define ZR36057_MCTCR_C_EMPTY BIT(29)
+#define ZR36057_MCTCR_C_FLUSH BIT(28)
+#define ZR36057_MCTCR_COD_GUEST_ID 20
+#define ZR36057_MCTCR_COD_GUEST_REG 16
+
+#define ZR36057_MCMPR 0x038 /* MPEG Code Memory Pointer Register */
+
+#define ZR36057_ISR 0x03c /* Interrupt Status Register */
+#define ZR36057_ISR_GIRQ1 BIT(30)
+#define ZR36057_ISR_GIRQ0 BIT(29)
+#define ZR36057_ISR_COD_REP_IRQ BIT(28)
+#define ZR36057_ISR_JPEG_REP_IRQ BIT(27)
+
+#define ZR36057_ICR 0x040 /* Interrupt Control Register */
+#define ZR36057_ICR_GIRQ1 BIT(30)
+#define ZR36057_ICR_GIRQ0 BIT(29)
+#define ZR36057_ICR_COD_REP_IRQ BIT(28)
+#define ZR36057_ICR_JPEG_REP_IRQ BIT(27)
+#define ZR36057_ICR_INT_PIN_EN BIT(24)
+
+#define ZR36057_I2CBR 0x044 /* I2C Bus Register */
+#define ZR36057_I2CBR_SDA BIT(1)
+#define ZR36057_I2CBR_SCL BIT(0)
+
+#define ZR36057_JMC 0x100 /* JPEG Mode and Control */
+#define ZR36057_JMC_JPG BIT(31)
+#define ZR36057_JMC_JPG_EXP_MODE (0 << 29)
+#define ZR36057_JMC_JPG_CMP_MODE BIT(29)
+#define ZR36057_JMC_MJPG_EXP_MODE (2 << 29)
+#define ZR36057_JMC_MJPG_CMP_MODE (3 << 29)
+#define ZR36057_JMC_RTBUSY_FB BIT(6)
+#define ZR36057_JMC_GO_EN BIT(5)
+#define ZR36057_JMC_SYNC_MSTR BIT(4)
+#define ZR36057_JMC_FLD_PER_BUFF BIT(3)
+#define ZR36057_JMC_VFIFO_FB BIT(2)
+#define ZR36057_JMC_CFIFO_FB BIT(1)
+#define ZR36057_JMC_STLL_LIT_ENDIAN BIT(0)
+
+#define ZR36057_JPC 0x104 /* JPEG Process Control */
+#define ZR36057_JPC_P_RESET BIT(7)
+#define ZR36057_JPC_COD_TRNS_EN BIT(5)
+#define ZR36057_JPC_ACTIVE BIT(0)
+
+#define ZR36057_VSP 0x108 /* Vertical Sync Parameters */
+#define ZR36057_VSP_VSYNC_SIZE 16
+#define ZR36057_VSP_FRM_TOT 0
+
+#define ZR36057_HSP 0x10c /* Horizontal Sync Parameters */
+#define ZR36057_HSP_HSYNC_START 16
+#define ZR36057_HSP_LINE_TOT 0
+
+#define ZR36057_FHAP 0x110 /* Field Horizontal Active Portion */
+#define ZR36057_FHAP_NAX 16
+#define ZR36057_FHAP_PAX 0
+
+#define ZR36057_FVAP 0x114 /* Field Vertical Active Portion */
+#define ZR36057_FVAP_NAY 16
+#define ZR36057_FVAP_PAY 0
+
+#define ZR36057_FPP 0x118 /* Field Process Parameters */
+#define ZR36057_FPP_ODD_EVEN BIT(0)
+
+#define ZR36057_JCBA 0x11c /* JPEG Code Base Address */
+
+#define ZR36057_JCFT 0x120 /* JPEG Code FIFO Threshold */
+
+#define ZR36057_JCGI 0x124 /* JPEG Codec Guest ID */
+#define ZR36057_JCGI_JPE_GUEST_ID 4
+#define ZR36057_JCGI_JPE_GUEST_REG 0
+
+#define ZR36057_GCR2 0x12c /* GuestBus Control Register (2) */
+
+#define ZR36057_POR 0x200 /* Post Office Register */
+#define ZR36057_POR_PO_PEN BIT(25)
+#define ZR36057_POR_PO_TIME BIT(24)
+#define ZR36057_POR_PO_DIR BIT(23)
+
+#define ZR36057_STR 0x300 /* "Still" Transfer Register */
+
+#endif
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Zoran ZR36060 basic configuration functions
+ *
+ * Copyright (C) 2002 Laurent Pinchart <laurent.pinchart@skynet.be>
+ */
+
+#define ZR060_VERSION "v0.7"
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+
+#include <linux/types.h>
+#include <linux/wait.h>
+
+/* I/O commands, error codes */
+#include <linux/io.h>
+
+/* headerfile of this module */
+#include "zr36060.h"
+
+/* codec io API */
+#include "videocodec.h"
+
+/* it doesn't make sense to have more than 20 or so, just to prevent some unwanted loops */
+#define MAX_CODECS 20
+
+/* amount of chips attached via this driver */
+static int zr36060_codecs;
+
+static bool low_bitrate;
+module_param(low_bitrate, bool, 0);
+MODULE_PARM_DESC(low_bitrate, "Buz compatibility option, halves bitrate");
+
+/* debugging is available via module parameter */
+static int debug;
+module_param(debug, int, 0);
+MODULE_PARM_DESC(debug, "Debug level (0-4)");
+
+#define dprintk(num, format, args...) \
+ do { \
+ if (debug >= num) \
+ printk(format, ##args); \
+ } while (0)
+
+/* =========================================================================
+ * Local hardware I/O functions:
+ * read/write via codec layer (registers are located in the master device)
+ * =========================================================================
+ */
+
+static u8 zr36060_read(struct zr36060 *ptr, u16 reg)
+{
+ u8 value = 0;
+
+ // just in case something is wrong...
+ if (ptr->codec->master_data->readreg)
+ value = (ptr->codec->master_data->readreg(ptr->codec, reg)) & 0xff;
+ else
+ pr_err("%s: invalid I/O setup, nothing read!\n", ptr->name);
+
+ return value;
+}
+
+static void zr36060_write(struct zr36060 *ptr, u16 reg, u8 value)
+{
+ dprintk(4, "0x%02x @0x%04x\n", value, reg);
+
+ // just in case something is wrong...
+ if (ptr->codec->master_data->writereg)
+ ptr->codec->master_data->writereg(ptr->codec, reg, value);
+ else
+ pr_err("%s: invalid I/O setup, nothing written!\n", ptr->name);
+}
+
+/* =========================================================================
+ * Local helper function:
+ * status read
+ * =========================================================================
+ */
+
+/* status is kept in datastructure */
+static u8 zr36060_read_status(struct zr36060 *ptr)
+{
+ ptr->status = zr36060_read(ptr, ZR060_CFSR);
+
+ zr36060_read(ptr, 0);
+ return ptr->status;
+}
+
+/* scale factor is kept in datastructure */
+static u16 zr36060_read_scalefactor(struct zr36060 *ptr)
+{
+ ptr->scalefact = (zr36060_read(ptr, ZR060_SF_HI) << 8) |
+ (zr36060_read(ptr, ZR060_SF_LO) & 0xFF);
+
+ /* leave 0 selected for an eventually GO from master */
+ zr36060_read(ptr, 0);
+ return ptr->scalefact;
+}
+
+/* wait if codec is ready to proceed (end of processing) or time is over */
+static void zr36060_wait_end(struct zr36060 *ptr)
+{
+ int i = 0;
+
+ while (zr36060_read_status(ptr) & ZR060_CFSR_BUSY) {
+ udelay(1);
+ if (i++ > 200000) { // 200ms, there is for sure something wrong!!!
+ dprintk(1,
+ "%s: timeout at wait_end (last status: 0x%02x)\n",
+ ptr->name, ptr->status);
+ break;
+ }
+ }
+}
+
+/* Basic test of "connectivity", writes/reads to/from memory the SOF marker */
+static int zr36060_basic_test(struct zr36060 *ptr)
+{
+ if ((zr36060_read(ptr, ZR060_IDR_DEV) != 0x33) &&
+ (zr36060_read(ptr, ZR060_IDR_REV) != 0x01)) {
+ pr_err("%s: attach failed, can't connect to jpeg processor!\n", ptr->name);
+ return -ENXIO;
+ }
+
+ zr36060_wait_end(ptr);
+ if (ptr->status & ZR060_CFSR_BUSY) {
+ pr_err("%s: attach failed, jpeg processor failed (end flag)!\n", ptr->name);
+ return -EBUSY;
+ }
+
+ return 0; /* looks good! */
+}
+
+/* simple loop for pushing the init datasets */
+static int zr36060_pushit(struct zr36060 *ptr, u16 startreg, u16 len, const char *data)
+{
+ int i = 0;
+
+ dprintk(4, "%s: write data block to 0x%04x (len=%d)\n", ptr->name,
+ startreg, len);
+ while (i < len)
+ zr36060_write(ptr, startreg++, data[i++]);
+
+ return i;
+}
+
+/* =========================================================================
+ * Basic datasets:
+ * jpeg baseline setup data (you find it on lots places in internet, or just
+ * extract it from any regular .jpg image...)
+ *
+ * Could be variable, but until it's not needed it they are just fixed to save
+ * memory. Otherwise expand zr36060 structure with arrays, push the values to
+ * it and initialize from there, as e.g. the linux zr36057/60 driver does it.
+ * =========================================================================
+ */
+static const char zr36060_dqt[0x86] = {
+ 0xff, 0xdb, //Marker: DQT
+ 0x00, 0x84, //Length: 2*65+2
+ 0x00, //Pq,Tq first table
+ 0x10, 0x0b, 0x0c, 0x0e, 0x0c, 0x0a, 0x10, 0x0e,
+ 0x0d, 0x0e, 0x12, 0x11, 0x10, 0x13, 0x18, 0x28,
+ 0x1a, 0x18, 0x16, 0x16, 0x18, 0x31, 0x23, 0x25,
+ 0x1d, 0x28, 0x3a, 0x33, 0x3d, 0x3c, 0x39, 0x33,
+ 0x38, 0x37, 0x40, 0x48, 0x5c, 0x4e, 0x40, 0x44,
+ 0x57, 0x45, 0x37, 0x38, 0x50, 0x6d, 0x51, 0x57,
+ 0x5f, 0x62, 0x67, 0x68, 0x67, 0x3e, 0x4d, 0x71,
+ 0x79, 0x70, 0x64, 0x78, 0x5c, 0x65, 0x67, 0x63,
+ 0x01, //Pq,Tq second table
+ 0x11, 0x12, 0x12, 0x18, 0x15, 0x18, 0x2f, 0x1a,
+ 0x1a, 0x2f, 0x63, 0x42, 0x38, 0x42, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
+ 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63
+};
+
+static const char zr36060_dht[0x1a4] = {
+ 0xff, 0xc4, //Marker: DHT
+ 0x01, 0xa2, //Length: 2*AC, 2*DC
+ 0x00, //DC first table
+ 0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+ 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
+ 0x01, //DC second table
+ 0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+ 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
+ 0x10, //AC first table
+ 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03,
+ 0x05, 0x05, 0x04, 0x04, 0x00, 0x00,
+ 0x01, 0x7D, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11,
+ 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61,
+ 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1,
+ 0x08, 0x23, 0x42, 0xB1, 0xC1, 0x15, 0x52, 0xD1, 0xF0, 0x24,
+ 0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17,
+ 0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x34,
+ 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
+ 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
+ 0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
+ 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
+ 0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88,
+ 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99,
+ 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8,
+ 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9,
+ 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8,
+ 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9,
+ 0xDA, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
+ 0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
+ 0xF8, 0xF9, 0xFA,
+ 0x11, //AC second table
+ 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04,
+ 0x07, 0x05, 0x04, 0x04, 0x00, 0x01,
+ 0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04,
+ 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71,
+ 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91,
+ 0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, 0x15, 0x62,
+ 0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25,
+ 0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, 0x27, 0x28, 0x29, 0x2A,
+ 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
+ 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
+ 0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
+ 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
+ 0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+ 0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98,
+ 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
+ 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8,
+ 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
+ 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8,
+ 0xD9, 0xDA, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
+ 0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8,
+ 0xF9, 0xFA
+};
+
+/* jpeg baseline setup, this is just fixed in this driver (YUV pictures) */
+#define NO_OF_COMPONENTS 0x3 //Y,U,V
+#define BASELINE_PRECISION 0x8 //MCU size (?)
+static const char zr36060_tq[8] = { 0, 1, 1, 0, 0, 0, 0, 0 }; //table idx's QT
+static const char zr36060_td[8] = { 0, 1, 1, 0, 0, 0, 0, 0 }; //table idx's DC
+static const char zr36060_ta[8] = { 0, 1, 1, 0, 0, 0, 0, 0 }; //table idx's AC
+
+/* horizontal 422 decimation setup (maybe we support 411 or so later, too) */
+static const char zr36060_decimation_h[8] = { 2, 1, 1, 0, 0, 0, 0, 0 };
+static const char zr36060_decimation_v[8] = { 1, 1, 1, 0, 0, 0, 0, 0 };
+
+/* SOF (start of frame) segment depends on width, height and sampling ratio of each color component */
+static int zr36060_set_sof(struct zr36060 *ptr)
+{
+ char sof_data[34]; // max. size of register set
+ int i;
+
+ dprintk(3, "%s: write SOF (%dx%d, %d components)\n", ptr->name,
+ ptr->width, ptr->height, NO_OF_COMPONENTS);
+ sof_data[0] = 0xff;
+ sof_data[1] = 0xc0;
+ sof_data[2] = 0x00;
+ sof_data[3] = (3 * NO_OF_COMPONENTS) + 8;
+ sof_data[4] = BASELINE_PRECISION; // only '8' possible with zr36060
+ sof_data[5] = (ptr->height) >> 8;
+ sof_data[6] = (ptr->height) & 0xff;
+ sof_data[7] = (ptr->width) >> 8;
+ sof_data[8] = (ptr->width) & 0xff;
+ sof_data[9] = NO_OF_COMPONENTS;
+ for (i = 0; i < NO_OF_COMPONENTS; i++) {
+ sof_data[10 + (i * 3)] = i; // index identifier
+ sof_data[11 + (i * 3)] = (ptr->h_samp_ratio[i] << 4) |
+ (ptr->v_samp_ratio[i]); // sampling ratios
+ sof_data[12 + (i * 3)] = zr36060_tq[i]; // Q table selection
+ }
+ return zr36060_pushit(ptr, ZR060_SOF_IDX,
+ (3 * NO_OF_COMPONENTS) + 10, sof_data);
+}
+
+/* SOS (start of scan) segment depends on the used scan components of each color component */
+static int zr36060_set_sos(struct zr36060 *ptr)
+{
+ char sos_data[16]; // max. size of register set
+ int i;
+
+ dprintk(3, "%s: write SOS\n", ptr->name);
+ sos_data[0] = 0xff;
+ sos_data[1] = 0xda;
+ sos_data[2] = 0x00;
+ sos_data[3] = 2 + 1 + (2 * NO_OF_COMPONENTS) + 3;
+ sos_data[4] = NO_OF_COMPONENTS;
+ for (i = 0; i < NO_OF_COMPONENTS; i++) {
+ sos_data[5 + (i * 2)] = i; // index
+ sos_data[6 + (i * 2)] = (zr36060_td[i] << 4) |
+ zr36060_ta[i]; // AC/DC tbl.sel.
+ }
+ sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 2] = 00; // scan start
+ sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 3] = 0x3f;
+ sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 4] = 00;
+ return zr36060_pushit(ptr, ZR060_SOS_IDX,
+ 4 + 1 + (2 * NO_OF_COMPONENTS) + 3,
+ sos_data);
+}
+
+/* DRI (define restart interval) */
+static int zr36060_set_dri(struct zr36060 *ptr)
+{
+ char dri_data[6]; // max. size of register set
+
+ dprintk(3, "%s: write DRI\n", ptr->name);
+ dri_data[0] = 0xff;
+ dri_data[1] = 0xdd;
+ dri_data[2] = 0x00;
+ dri_data[3] = 0x04;
+ dri_data[4] = (ptr->dri) >> 8;
+ dri_data[5] = (ptr->dri) & 0xff;
+ return zr36060_pushit(ptr, ZR060_DRI_IDX, 6, dri_data);
+}
+
+/* Setup compression/decompression of Zoran's JPEG processor ( see also zoran 36060 manual )
+ * ... sorry for the spaghetti code ...
+ */
+static void zr36060_init(struct zr36060 *ptr)
+{
+ int sum = 0;
+ long bitcnt, tmp;
+
+ if (ptr->mode == CODEC_DO_COMPRESSION) {
+ dprintk(2, "%s: COMPRESSION SETUP\n", ptr->name);
+
+ zr36060_write(ptr, ZR060_LOAD, ZR060_LOAD_SYNC_RST);
+
+ /* 060 communicates with 067 in master mode */
+ zr36060_write(ptr, ZR060_CIR, ZR060_CIR_CODE_MSTR);
+
+ /* Compression with or without variable scale factor */
+ /*FIXME: What about ptr->bitrate_ctrl? */
+ zr36060_write(ptr, ZR060_CMR, ZR060_CMR_COMP | ZR060_CMR_PASS2 | ZR060_CMR_BRB);
+
+ /* Must be zero */
+ zr36060_write(ptr, ZR060_MBZ, 0x00);
+ zr36060_write(ptr, ZR060_TCR_HI, 0x00);
+ zr36060_write(ptr, ZR060_TCR_LO, 0x00);
+
+ /* Disable all IRQs - no DataErr means autoreset */
+ zr36060_write(ptr, ZR060_IMR, 0);
+
+ /* volume control settings */
+ zr36060_write(ptr, ZR060_SF_HI, ptr->scalefact >> 8);
+ zr36060_write(ptr, ZR060_SF_LO, ptr->scalefact & 0xff);
+
+ zr36060_write(ptr, ZR060_AF_HI, 0xff);
+ zr36060_write(ptr, ZR060_AF_M, 0xff);
+ zr36060_write(ptr, ZR060_AF_LO, 0xff);
+
+ /* setup the variable jpeg tables */
+ sum += zr36060_set_sof(ptr);
+ sum += zr36060_set_sos(ptr);
+ sum += zr36060_set_dri(ptr);
+
+/* setup the fixed jpeg tables - maybe variable, though - (see table init section above) */
+ sum += zr36060_pushit(ptr, ZR060_DQT_IDX, sizeof(zr36060_dqt), zr36060_dqt);
+ sum += zr36060_pushit(ptr, ZR060_DHT_IDX, sizeof(zr36060_dht), zr36060_dht);
+ zr36060_write(ptr, ZR060_APP_IDX, 0xff);
+ zr36060_write(ptr, ZR060_APP_IDX + 1, 0xe0 + ptr->app.appn);
+ zr36060_write(ptr, ZR060_APP_IDX + 2, 0x00);
+ zr36060_write(ptr, ZR060_APP_IDX + 3, ptr->app.len + 2);
+ sum += zr36060_pushit(ptr, ZR060_APP_IDX + 4, 60, ptr->app.data) + 4;
+ zr36060_write(ptr, ZR060_COM_IDX, 0xff);
+ zr36060_write(ptr, ZR060_COM_IDX + 1, 0xfe);
+ zr36060_write(ptr, ZR060_COM_IDX + 2, 0x00);
+ zr36060_write(ptr, ZR060_COM_IDX + 3, ptr->com.len + 2);
+ sum += zr36060_pushit(ptr, ZR060_COM_IDX + 4, 60, ptr->com.data) + 4;
+
+ /* setup misc. data for compression (target code sizes) */
+
+ /* size of compressed code to reach without header data */
+ sum = ptr->real_code_vol - sum;
+ bitcnt = sum << 3; /* need the size in bits */
+
+ tmp = bitcnt >> 16;
+ dprintk(3,
+ "%s: code: csize=%d, tot=%d, bit=%ld, highbits=%ld\n",
+ ptr->name, sum, ptr->real_code_vol, bitcnt, tmp);
+ zr36060_write(ptr, ZR060_TCV_NET_HI, tmp >> 8);
+ zr36060_write(ptr, ZR060_TCV_NET_MH, tmp & 0xff);
+ tmp = bitcnt & 0xffff;
+ zr36060_write(ptr, ZR060_TCV_NET_ML, tmp >> 8);
+ zr36060_write(ptr, ZR060_TCV_NET_LO, tmp & 0xff);
+
+ bitcnt -= bitcnt >> 7; // bits without stuffing
+ bitcnt -= ((bitcnt * 5) >> 6); // bits without eob
+
+ tmp = bitcnt >> 16;
+ dprintk(3, "%s: code: nettobit=%ld, highnettobits=%ld\n",
+ ptr->name, bitcnt, tmp);
+ zr36060_write(ptr, ZR060_TCV_DATA_HI, tmp >> 8);
+ zr36060_write(ptr, ZR060_TCV_DATA_MH, tmp & 0xff);
+ tmp = bitcnt & 0xffff;
+ zr36060_write(ptr, ZR060_TCV_DATA_ML, tmp >> 8);
+ zr36060_write(ptr, ZR060_TCV_DATA_LO, tmp & 0xff);
+
+ /* JPEG markers to be included in the compressed stream */
+ zr36060_write(ptr, ZR060_MER,
+ ZR060_MER_DQT | ZR060_MER_DHT |
+ ((ptr->com.len > 0) ? ZR060_MER_COM : 0) |
+ ((ptr->app.len > 0) ? ZR060_MER_APP : 0));
+
+ /* Setup the Video Frontend */
+ /* Limit pixel range to 16..235 as per CCIR-601 */
+ zr36060_write(ptr, ZR060_VCR, ZR060_VCR_RANGE);
+
+ } else {
+ dprintk(2, "%s: EXPANSION SETUP\n", ptr->name);
+
+ zr36060_write(ptr, ZR060_LOAD, ZR060_LOAD_SYNC_RST);
+
+ /* 060 communicates with 067 in master mode */
+ zr36060_write(ptr, ZR060_CIR, ZR060_CIR_CODE_MSTR);
+
+ /* Decompression */
+ zr36060_write(ptr, ZR060_CMR, 0);
+
+ /* Must be zero */
+ zr36060_write(ptr, ZR060_MBZ, 0x00);
+ zr36060_write(ptr, ZR060_TCR_HI, 0x00);
+ zr36060_write(ptr, ZR060_TCR_LO, 0x00);
+
+ /* Disable all IRQs - no DataErr means autoreset */
+ zr36060_write(ptr, ZR060_IMR, 0);
+
+ /* setup misc. data for expansion */
+ zr36060_write(ptr, ZR060_MER, 0);
+
+/* setup the fixed jpeg tables - maybe variable, though - (see table init section above) */
+ zr36060_pushit(ptr, ZR060_DHT_IDX, sizeof(zr36060_dht), zr36060_dht);
+
+ /* Setup the Video Frontend */
+ //zr36060_write(ptr, ZR060_VCR, ZR060_VCR_FI_EXT);
+ //this doesn't seem right and doesn't work...
+ zr36060_write(ptr, ZR060_VCR, ZR060_VCR_RANGE);
+ }
+
+ /* Load the tables */
+ zr36060_write(ptr, ZR060_LOAD, ZR060_LOAD_SYNC_RST | ZR060_LOAD_LOAD);
+ zr36060_wait_end(ptr);
+ dprintk(2, "%s: Status after table preload: 0x%02x\n", ptr->name, ptr->status);
+
+ if (ptr->status & ZR060_CFSR_BUSY) {
+ pr_err("%s: init aborted!\n", ptr->name);
+ return; // something is wrong, its timed out!!!!
+ }
+}
+
+/* =========================================================================
+ * CODEC API FUNCTIONS
+ * this functions are accessed by the master via the API structure
+ * =========================================================================
+ */
+
+/* set compressiion/expansion mode and launches codec -
+ * this should be the last call from the master before starting processing
+ */
+static int zr36060_set_mode(struct videocodec *codec, int mode)
+{
+ struct zr36060 *ptr = (struct zr36060 *)codec->data;
+
+ dprintk(2, "%s: set_mode %d call\n", ptr->name, mode);
+
+ if (mode != CODEC_DO_EXPANSION && mode != CODEC_DO_COMPRESSION)
+ return -EINVAL;
+
+ ptr->mode = mode;
+ zr36060_init(ptr);
+
+ return 0;
+}
+
+/* set picture size (norm is ignored as the codec doesn't know about it) */
+static int zr36060_set_video(struct videocodec *codec, const struct tvnorm *norm,
+ struct vfe_settings *cap, struct vfe_polarity *pol)
+{
+ struct zr36060 *ptr = (struct zr36060 *)codec->data;
+ u32 reg;
+ int size;
+
+ dprintk(2, "%s: set_video %d/%d-%dx%d (%%%d) call\n", ptr->name,
+ cap->x, cap->y, cap->width, cap->height, cap->decimation);
+
+ /* if () return -EINVAL;
+ * trust the master driver that it knows what it does - so
+ * we allow invalid startx/y and norm for now ...
+ */
+ ptr->width = cap->width / (cap->decimation & 0xff);
+ ptr->height = cap->height / (cap->decimation >> 8);
+
+ zr36060_write(ptr, ZR060_LOAD, ZR060_LOAD_SYNC_RST);
+
+ /* Note that VSPol/HSPol bits in zr36060 have the opposite
+ * meaning of their zr360x7 counterparts with the same names
+ * N.b. for VSPol this is only true if FIVEdge = 0 (default,
+ * left unchanged here - in accordance with datasheet).
+ */
+ reg = (!pol->vsync_pol ? ZR060_VPR_VS_POL : 0)
+ | (!pol->hsync_pol ? ZR060_VPR_HS_POL : 0)
+ | (pol->field_pol ? ZR060_VPR_FI_POL : 0)
+ | (pol->blank_pol ? ZR060_VPR_BL_POL : 0)
+ | (pol->subimg_pol ? ZR060_VPR_S_IMG_POL : 0)
+ | (pol->poe_pol ? ZR060_VPR_POE_POL : 0)
+ | (pol->pvalid_pol ? ZR060_VPR_P_VAL_POL : 0)
+ | (pol->vclk_pol ? ZR060_VPR_VCLK_POL : 0);
+ zr36060_write(ptr, ZR060_VPR, reg);
+
+ reg = 0;
+ switch (cap->decimation & 0xff) {
+ default:
+ case 1:
+ break;
+
+ case 2:
+ reg |= ZR060_SR_H_SCALE2;
+ break;
+
+ case 4:
+ reg |= ZR060_SR_H_SCALE4;
+ break;
+ }
+
+ switch (cap->decimation >> 8) {
+ default:
+ case 1:
+ break;
+
+ case 2:
+ reg |= ZR060_SR_V_SCALE;
+ break;
+ }
+ zr36060_write(ptr, ZR060_SR, reg);
+
+ zr36060_write(ptr, ZR060_BCR_Y, 0x00);
+ zr36060_write(ptr, ZR060_BCR_U, 0x80);
+ zr36060_write(ptr, ZR060_BCR_V, 0x80);
+
+ /* sync generator */
+
+ reg = norm->ht - 1; /* Vtotal */
+ zr36060_write(ptr, ZR060_SGR_VTOTAL_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_SGR_VTOTAL_LO, (reg >> 0) & 0xff);
+
+ reg = norm->wt - 1; /* Htotal */
+ zr36060_write(ptr, ZR060_SGR_HTOTAL_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_SGR_HTOTAL_LO, (reg >> 0) & 0xff);
+
+ reg = 6 - 1; /* VsyncSize */
+ zr36060_write(ptr, ZR060_SGR_VSYNC, reg);
+
+ //reg = 30 - 1; /* HsyncSize */
+///*CP*/ reg = (zr->params.norm == 1 ? 57 : 68);
+ reg = 68;
+ zr36060_write(ptr, ZR060_SGR_HSYNC, reg);
+
+ reg = norm->v_start - 1; /* BVstart */
+ zr36060_write(ptr, ZR060_SGR_BVSTART, reg);
+
+ reg += norm->ha / 2; /* BVend */
+ zr36060_write(ptr, ZR060_SGR_BVEND_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_SGR_BVEND_LO, (reg >> 0) & 0xff);
+
+ reg = norm->h_start - 1; /* BHstart */
+ zr36060_write(ptr, ZR060_SGR_BHSTART, reg);
+
+ reg += norm->wa; /* BHend */
+ zr36060_write(ptr, ZR060_SGR_BHEND_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_SGR_BHEND_LO, (reg >> 0) & 0xff);
+
+ /* active area */
+ reg = cap->y + norm->v_start; /* Vstart */
+ zr36060_write(ptr, ZR060_AAR_VSTART_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_AAR_VSTART_LO, (reg >> 0) & 0xff);
+
+ reg += cap->height; /* Vend */
+ zr36060_write(ptr, ZR060_AAR_VEND_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_AAR_VEND_LO, (reg >> 0) & 0xff);
+
+ reg = cap->x + norm->h_start; /* Hstart */
+ zr36060_write(ptr, ZR060_AAR_HSTART_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_AAR_HSTART_LO, (reg >> 0) & 0xff);
+
+ reg += cap->width; /* Hend */
+ zr36060_write(ptr, ZR060_AAR_HEND_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_AAR_HEND_LO, (reg >> 0) & 0xff);
+
+ /* subimage area */
+ reg = norm->v_start - 4; /* SVstart */
+ zr36060_write(ptr, ZR060_SWR_VSTART_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_SWR_VSTART_LO, (reg >> 0) & 0xff);
+
+ reg += norm->ha / 2 + 8; /* SVend */
+ zr36060_write(ptr, ZR060_SWR_VEND_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_SWR_VEND_LO, (reg >> 0) & 0xff);
+
+ reg = norm->h_start /*+ 64 */ - 4; /* SHstart */
+ zr36060_write(ptr, ZR060_SWR_HSTART_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_SWR_HSTART_LO, (reg >> 0) & 0xff);
+
+ reg += norm->wa + 8; /* SHend */
+ zr36060_write(ptr, ZR060_SWR_HEND_HI, (reg >> 8) & 0xff);
+ zr36060_write(ptr, ZR060_SWR_HEND_LO, (reg >> 0) & 0xff);
+
+ size = ptr->width * ptr->height;
+ /* Target compressed field size in bits: */
+ size = size * 16; /* uncompressed size in bits */
+ /* (Ronald) by default, quality = 100 is a compression
+ * ratio 1:2. Setting low_bitrate (insmod option) sets
+ * it to 1:4 (instead of 1:2, zr36060 max) as limit because the
+ * buz can't handle more at decimation=1... Use low_bitrate if
+ * you have a Buz, unless you know what you're doing
+ */
+ size = size * cap->quality / (low_bitrate ? 400 : 200);
+ /* Lower limit (arbitrary, 1 KB) */
+ if (size < 8192)
+ size = 8192;
+ /* Upper limit: 7/8 of the code buffers */
+ if (size > ptr->total_code_vol * 7)
+ size = ptr->total_code_vol * 7;
+
+ ptr->real_code_vol = size >> 3; /* in bytes */
+
+ /* the MBCVR is the *maximum* block volume, according to the
+ * JPEG ISO specs, this shouldn't be used, since that allows
+ * for the best encoding quality. So set it to it's max value
+ */
+ reg = ptr->max_block_vol;
+ zr36060_write(ptr, ZR060_MBCVR, reg);
+
+ return 0;
+}
+
+/* additional control functions */
+static int zr36060_control(struct videocodec *codec, int type, int size, void *data)
+{
+ struct zr36060 *ptr = (struct zr36060 *)codec->data;
+ int *ival = (int *)data;
+
+ dprintk(2, "%s: control %d call with %d byte\n", ptr->name, type,
+ size);
+
+ switch (type) {
+ case CODEC_G_STATUS: /* get last status */
+ if (size != sizeof(int))
+ return -EFAULT;
+ zr36060_read_status(ptr);
+ *ival = ptr->status;
+ break;
+
+ case CODEC_G_CODEC_MODE:
+ if (size != sizeof(int))
+ return -EFAULT;
+ *ival = CODEC_MODE_BJPG;
+ break;
+
+ case CODEC_S_CODEC_MODE:
+ if (size != sizeof(int))
+ return -EFAULT;
+ if (*ival != CODEC_MODE_BJPG)
+ return -EINVAL;
+ /* not needed, do nothing */
+ return 0;
+
+ case CODEC_G_VFE:
+ case CODEC_S_VFE:
+ /* not needed, do nothing */
+ return 0;
+
+ case CODEC_S_MMAP:
+ /* not available, give an error */
+ return -ENXIO;
+
+ case CODEC_G_JPEG_TDS_BYTE: /* get target volume in byte */
+ if (size != sizeof(int))
+ return -EFAULT;
+ *ival = ptr->total_code_vol;
+ break;
+
+ case CODEC_S_JPEG_TDS_BYTE: /* get target volume in byte */
+ if (size != sizeof(int))
+ return -EFAULT;
+ ptr->total_code_vol = *ival;
+ ptr->real_code_vol = (ptr->total_code_vol * 6) >> 3;
+ break;
+
+ case CODEC_G_JPEG_SCALE: /* get scaling factor */
+ if (size != sizeof(int))
+ return -EFAULT;
+ *ival = zr36060_read_scalefactor(ptr);
+ break;
+
+ case CODEC_S_JPEG_SCALE: /* set scaling factor */
+ if (size != sizeof(int))
+ return -EFAULT;
+ ptr->scalefact = *ival;
+ break;
+
+ case CODEC_G_JPEG_APP_DATA: { /* get appn marker data */
+ struct jpeg_app_marker *app = data;
+
+ if (size != sizeof(struct jpeg_app_marker))
+ return -EFAULT;
+
+ *app = ptr->app;
+ break;
+ }
+
+ case CODEC_S_JPEG_APP_DATA: { /* set appn marker data */
+ struct jpeg_app_marker *app = data;
+
+ if (size != sizeof(struct jpeg_app_marker))
+ return -EFAULT;
+
+ ptr->app = *app;
+ break;
+ }
+
+ case CODEC_G_JPEG_COM_DATA: { /* get comment marker data */
+ struct jpeg_com_marker *com = data;
+
+ if (size != sizeof(struct jpeg_com_marker))
+ return -EFAULT;
+
+ *com = ptr->com;
+ break;
+ }
+
+ case CODEC_S_JPEG_COM_DATA: { /* set comment marker data */
+ struct jpeg_com_marker *com = data;
+
+ if (size != sizeof(struct jpeg_com_marker))
+ return -EFAULT;
+
+ ptr->com = *com;
+ break;
+ }
+
+ default:
+ return -EINVAL;
+ }
+
+ return size;
+}
+
+/* =========================================================================
+ * Exit and unregister function:
+ * Deinitializes Zoran's JPEG processor
+ * =========================================================================
+ */
+static int zr36060_unset(struct videocodec *codec)
+{
+ struct zr36060 *ptr = codec->data;
+
+ if (ptr) {
+ /* do wee need some codec deinit here, too ???? */
+
+ dprintk(1, "%s: finished codec #%d\n", ptr->name, ptr->num);
+ kfree(ptr);
+ codec->data = NULL;
+
+ zr36060_codecs--;
+ return 0;
+ }
+
+ return -EFAULT;
+}
+
+/* =========================================================================
+ * Setup and registry function:
+ * Initializes Zoran's JPEG processor
+ * Also sets pixel size, average code size, mode (compr./decompr.)
+ * (the given size is determined by the processor with the video interface)
+ * =========================================================================
+ */
+static int zr36060_setup(struct videocodec *codec)
+{
+ struct zr36060 *ptr;
+ int res;
+
+ dprintk(2, "zr36060: initializing MJPEG subsystem #%d.\n", zr36060_codecs);
+
+ if (zr36060_codecs == MAX_CODECS) {
+ pr_err("zr36060: Can't attach more codecs!\n");
+ return -ENOSPC;
+ }
+ //mem structure init
+ codec->data = ptr = kzalloc(sizeof(*ptr), GFP_KERNEL);
+ if (!ptr)
+ return -ENOMEM;
+
+ snprintf(ptr->name, sizeof(ptr->name), "zr36060[%d]", zr36060_codecs);
+ ptr->num = zr36060_codecs++;
+ ptr->codec = codec;
+
+ //testing
+ res = zr36060_basic_test(ptr);
+ if (res < 0) {
+ zr36060_unset(codec);
+ return res;
+ }
+ //final setup
+ memcpy(ptr->h_samp_ratio, zr36060_decimation_h, 8);
+ memcpy(ptr->v_samp_ratio, zr36060_decimation_v, 8);
+
+ ptr->bitrate_ctrl = 0; /* 0 or 1 - fixed file size flag (what is the difference?) */
+ ptr->mode = CODEC_DO_COMPRESSION;
+ ptr->width = 384;
+ ptr->height = 288;
+ ptr->total_code_vol = 16000; /* CHECKME */
+ ptr->real_code_vol = (ptr->total_code_vol * 6) >> 3;
+ ptr->max_block_vol = 240; /* CHECKME, was 120 is 240 */
+ ptr->scalefact = 0x100;
+ ptr->dri = 1; /* CHECKME, was 8 is 1 */
+
+ /* by default, no COM or APP markers - app should set those */
+ ptr->com.len = 0;
+ ptr->app.appn = 0;
+ ptr->app.len = 0;
+
+ zr36060_init(ptr);
+
+ dprintk(1, KERN_INFO "%s: codec attached and running\n", ptr->name);
+
+ return 0;
+}
+
+static const struct videocodec zr36060_codec = {
+ .owner = THIS_MODULE,
+ .name = "zr36060",
+ .magic = 0L, // magic not used
+ .flags =
+ CODEC_FLAG_JPEG | CODEC_FLAG_HARDWARE | CODEC_FLAG_ENCODER |
+ CODEC_FLAG_DECODER | CODEC_FLAG_VFE,
+ .type = CODEC_TYPE_ZR36060,
+ .setup = zr36060_setup, // functionality
+ .unset = zr36060_unset,
+ .set_mode = zr36060_set_mode,
+ .set_video = zr36060_set_video,
+ .control = zr36060_control,
+ // others are not used
+};
+
+static int __init zr36060_init_module(void)
+{
+ zr36060_codecs = 0;
+ return videocodec_register(&zr36060_codec);
+}
+
+static void __exit zr36060_cleanup_module(void)
+{
+ if (zr36060_codecs) {
+ dprintk(1,
+ "zr36060: something's wrong - %d codecs left somehow.\n",
+ zr36060_codecs);
+ }
+
+ /* however, we can't just stay alive */
+ videocodec_unregister(&zr36060_codec);
+}
+
+module_init(zr36060_init_module);
+module_exit(zr36060_cleanup_module);
+
+MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@skynet.be>");
+MODULE_DESCRIPTION("Driver module for ZR36060 jpeg processors " ZR060_VERSION);
+MODULE_LICENSE("GPL");
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Zoran ZR36060 basic configuration functions - header file
+ *
+ * Copyright (C) 2002 Laurent Pinchart <laurent.pinchart@skynet.be>
+ */
+
+#ifndef ZR36060_H
+#define ZR36060_H
+
+#include "videocodec.h"
+
+/* data stored for each zoran jpeg codec chip */
+struct zr36060 {
+ char name[32];
+ int num;
+ /* io datastructure */
+ struct videocodec *codec;
+ // last coder status
+ __u8 status;
+ // actual coder setup
+ int mode;
+
+ __u16 width;
+ __u16 height;
+
+ __u16 bitrate_ctrl;
+
+ __u32 total_code_vol;
+ __u32 real_code_vol;
+ __u16 max_block_vol;
+
+ __u8 h_samp_ratio[8];
+ __u8 v_samp_ratio[8];
+ __u16 scalefact;
+ __u16 dri;
+
+ /* app/com marker data */
+ struct jpeg_app_marker app;
+ struct jpeg_com_marker com;
+};
+
+/* ZR36060 register addresses */
+#define ZR060_LOAD 0x000
+#define ZR060_CFSR 0x001
+#define ZR060_CIR 0x002
+#define ZR060_CMR 0x003
+#define ZR060_MBZ 0x004
+#define ZR060_MBCVR 0x005
+#define ZR060_MER 0x006
+#define ZR060_IMR 0x007
+#define ZR060_ISR 0x008
+#define ZR060_TCV_NET_HI 0x009
+#define ZR060_TCV_NET_MH 0x00a
+#define ZR060_TCV_NET_ML 0x00b
+#define ZR060_TCV_NET_LO 0x00c
+#define ZR060_TCV_DATA_HI 0x00d
+#define ZR060_TCV_DATA_MH 0x00e
+#define ZR060_TCV_DATA_ML 0x00f
+#define ZR060_TCV_DATA_LO 0x010
+#define ZR060_SF_HI 0x011
+#define ZR060_SF_LO 0x012
+#define ZR060_AF_HI 0x013
+#define ZR060_AF_M 0x014
+#define ZR060_AF_LO 0x015
+#define ZR060_ACV_HI 0x016
+#define ZR060_ACV_MH 0x017
+#define ZR060_ACV_ML 0x018
+#define ZR060_ACV_LO 0x019
+#define ZR060_ACT_HI 0x01a
+#define ZR060_ACT_MH 0x01b
+#define ZR060_ACT_ML 0x01c
+#define ZR060_ACT_LO 0x01d
+#define ZR060_ACV_TURN_HI 0x01e
+#define ZR060_ACV_TURN_MH 0x01f
+#define ZR060_ACV_TURN_ML 0x020
+#define ZR060_ACV_TURN_LO 0x021
+#define ZR060_IDR_DEV 0x022
+#define ZR060_IDR_REV 0x023
+#define ZR060_TCR_HI 0x024
+#define ZR060_TCR_LO 0x025
+#define ZR060_VCR 0x030
+#define ZR060_VPR 0x031
+#define ZR060_SR 0x032
+#define ZR060_BCR_Y 0x033
+#define ZR060_BCR_U 0x034
+#define ZR060_BCR_V 0x035
+#define ZR060_SGR_VTOTAL_HI 0x036
+#define ZR060_SGR_VTOTAL_LO 0x037
+#define ZR060_SGR_HTOTAL_HI 0x038
+#define ZR060_SGR_HTOTAL_LO 0x039
+#define ZR060_SGR_VSYNC 0x03a
+#define ZR060_SGR_HSYNC 0x03b
+#define ZR060_SGR_BVSTART 0x03c
+#define ZR060_SGR_BHSTART 0x03d
+#define ZR060_SGR_BVEND_HI 0x03e
+#define ZR060_SGR_BVEND_LO 0x03f
+#define ZR060_SGR_BHEND_HI 0x040
+#define ZR060_SGR_BHEND_LO 0x041
+#define ZR060_AAR_VSTART_HI 0x042
+#define ZR060_AAR_VSTART_LO 0x043
+#define ZR060_AAR_VEND_HI 0x044
+#define ZR060_AAR_VEND_LO 0x045
+#define ZR060_AAR_HSTART_HI 0x046
+#define ZR060_AAR_HSTART_LO 0x047
+#define ZR060_AAR_HEND_HI 0x048
+#define ZR060_AAR_HEND_LO 0x049
+#define ZR060_SWR_VSTART_HI 0x04a
+#define ZR060_SWR_VSTART_LO 0x04b
+#define ZR060_SWR_VEND_HI 0x04c
+#define ZR060_SWR_VEND_LO 0x04d
+#define ZR060_SWR_HSTART_HI 0x04e
+#define ZR060_SWR_HSTART_LO 0x04f
+#define ZR060_SWR_HEND_HI 0x050
+#define ZR060_SWR_HEND_LO 0x051
+
+#define ZR060_SOF_IDX 0x060
+#define ZR060_SOS_IDX 0x07a
+#define ZR060_DRI_IDX 0x0c0
+#define ZR060_DQT_IDX 0x0cc
+#define ZR060_DHT_IDX 0x1d4
+#define ZR060_APP_IDX 0x380
+#define ZR060_COM_IDX 0x3c0
+
+/* ZR36060 LOAD register bits */
+
+#define ZR060_LOAD_LOAD BIT(7)
+#define ZR060_LOAD_SYNC_RST BIT(0)
+
+/* ZR36060 Code FIFO Status register bits */
+
+#define ZR060_CFSR_BUSY BIT(7)
+#define ZR060_CFSR_C_BUSY BIT(2)
+#define ZR060_CFSR_CFIFO (3 << 0)
+
+/* ZR36060 Code Interface register */
+
+#define ZR060_CIR_CODE16 BIT(7)
+#define ZR060_CIR_ENDIAN BIT(6)
+#define ZR060_CIR_CFIS BIT(2)
+#define ZR060_CIR_CODE_MSTR BIT(0)
+
+/* ZR36060 Codec Mode register */
+
+#define ZR060_CMR_COMP BIT(7)
+#define ZR060_CMR_ATP BIT(6)
+#define ZR060_CMR_PASS2 BIT(5)
+#define ZR060_CMR_TLM BIT(4)
+#define ZR060_CMR_BRB BIT(2)
+#define ZR060_CMR_FSF BIT(1)
+
+/* ZR36060 Markers Enable register */
+
+#define ZR060_MER_APP BIT(7)
+#define ZR060_MER_COM BIT(6)
+#define ZR060_MER_DRI BIT(5)
+#define ZR060_MER_DQT BIT(4)
+#define ZR060_MER_DHT BIT(3)
+
+/* ZR36060 Interrupt Mask register */
+
+#define ZR060_IMR_EOAV BIT(3)
+#define ZR060_IMR_EOI BIT(2)
+#define ZR060_IMR_END BIT(1)
+#define ZR060_IMR_DATA_ERR BIT(0)
+
+/* ZR36060 Interrupt Status register */
+
+#define ZR060_ISR_PRO_CNT (3 << 6)
+#define ZR060_ISR_EOAV BIT(3)
+#define ZR060_ISR_EOI BIT(2)
+#define ZR060_ISR_END BIT(1)
+#define ZR060_ISR_DATA_ERR BIT(0)
+
+/* ZR36060 Video Control register */
+
+#define ZR060_VCR_VIDEO8 BIT(7)
+#define ZR060_VCR_RANGE BIT(6)
+#define ZR060_VCR_FI_DET BIT(3)
+#define ZR060_VCR_FI_VEDGE BIT(2)
+#define ZR060_VCR_FI_EXT BIT(1)
+#define ZR060_VCR_SYNC_MSTR BIT(0)
+
+/* ZR36060 Video Polarity register */
+
+#define ZR060_VPR_VCLK_POL BIT(7)
+#define ZR060_VPR_P_VAL_POL BIT(6)
+#define ZR060_VPR_POE_POL BIT(5)
+#define ZR060_VPR_S_IMG_POL BIT(4)
+#define ZR060_VPR_BL_POL BIT(3)
+#define ZR060_VPR_FI_POL BIT(2)
+#define ZR060_VPR_HS_POL BIT(1)
+#define ZR060_VPR_VS_POL BIT(0)
+
+/* ZR36060 Scaling register */
+
+#define ZR060_SR_V_SCALE BIT(2)
+#define ZR060_SR_H_SCALE2 BIT(0)
+#define ZR060_SR_H_SCALE4 (2 << 0)
+
+#endif /*fndef ZR36060_H */
config RTLLIB_CRYPTO_TKIP
tristate "Support for rtllib TKIP crypto"
depends on RTLLIB
- select CRYPTO_ARC4
+ select CRYPTO_LIB_ARC4
select CRYPTO_MICHAEL_MIC
default y
help
config RTLLIB_CRYPTO_WEP
tristate "Support for rtllib WEP crypto"
- select CRYPTO_ARC4
+ select CRYPTO_LIB_ARC4
depends on RTLLIB
default y
help
* Copyright (c) 2003-2004, Jouni Malinen <jkmaline@cc.hut.fi>
*/
+#include <crypto/arc4.h>
#include <crypto/hash.h>
-#include <crypto/skcipher.h>
+#include <linux/fips.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/if_ether.h>
#include <linux/if_arp.h>
#include <linux/string.h>
-#include <linux/scatterlist.h>
#include <linux/crc32.h>
#include <linux/etherdevice.h>
u32 dot11RSNAStatsTKIPLocalMICFailures;
int key_idx;
- struct crypto_sync_skcipher *rx_tfm_arc4;
+ struct arc4_ctx rx_ctx_arc4;
+ struct arc4_ctx tx_ctx_arc4;
struct crypto_shash *rx_tfm_michael;
- struct crypto_sync_skcipher *tx_tfm_arc4;
struct crypto_shash *tx_tfm_michael;
/* scratch buffers for virt_to_page() (crypto API) */
u8 rx_hdr[16];
{
struct rtllib_tkip_data *priv;
+ if (fips_enabled)
+ return NULL;
+
priv = kzalloc(sizeof(*priv), GFP_ATOMIC);
if (priv == NULL)
goto fail;
priv->key_idx = key_idx;
- priv->tx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
- if (IS_ERR(priv->tx_tfm_arc4)) {
- pr_debug("Could not allocate crypto API arc4\n");
- priv->tx_tfm_arc4 = NULL;
- goto fail;
- }
priv->tx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0);
if (IS_ERR(priv->tx_tfm_michael)) {
goto fail;
}
- priv->rx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
- if (IS_ERR(priv->rx_tfm_arc4)) {
- pr_debug("Could not allocate crypto API arc4\n");
- priv->rx_tfm_arc4 = NULL;
- goto fail;
- }
-
priv->rx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0);
if (IS_ERR(priv->rx_tfm_michael)) {
pr_debug("Could not allocate crypto API michael_mic\n");
fail:
if (priv) {
crypto_free_shash(priv->tx_tfm_michael);
- crypto_free_sync_skcipher(priv->tx_tfm_arc4);
crypto_free_shash(priv->rx_tfm_michael);
- crypto_free_sync_skcipher(priv->rx_tfm_arc4);
kfree(priv);
}
if (_priv) {
crypto_free_shash(_priv->tx_tfm_michael);
- crypto_free_sync_skcipher(_priv->tx_tfm_arc4);
crypto_free_shash(_priv->rx_tfm_michael);
- crypto_free_sync_skcipher(_priv->rx_tfm_arc4);
}
- kfree(priv);
+ kzfree(priv);
}
int ret = 0;
u8 rc4key[16], *icv;
u32 crc;
- struct scatterlist sg;
if (skb_headroom(skb) < 8 || skb_tailroom(skb) < 4 ||
skb->len < hdr_len)
*pos++ = (tkey->tx_iv32 >> 24) & 0xff;
if (!tcb_desc->bHwSec) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->tx_tfm_arc4);
-
icv = skb_put(skb, 4);
crc = ~crc32_le(~0, pos, len);
icv[0] = crc;
icv[2] = crc >> 16;
icv[3] = crc >> 24;
- sg_init_one(&sg, pos, len+4);
-
-
- crypto_sync_skcipher_setkey(tkey->tx_tfm_arc4, rc4key, 16);
- skcipher_request_set_sync_tfm(req, tkey->tx_tfm_arc4);
- skcipher_request_set_callback(req, 0, NULL, NULL);
- skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL);
- ret = crypto_skcipher_encrypt(req);
- skcipher_request_zero(req);
+ arc4_setkey(&tkey->tx_ctx_arc4, rc4key, 16);
+ arc4_crypt(&tkey->tx_ctx_arc4, pos, pos, len + 4);
}
tkey->tx_iv16++;
u8 rc4key[16];
u8 icv[4];
u32 crc;
- struct scatterlist sg;
int plen;
- int err;
if (skb->len < hdr_len + 8 + 4)
return -1;
pos += 8;
if (!tcb_desc->bHwSec || (skb->cb[0] == 1)) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->rx_tfm_arc4);
-
if ((iv32 < tkey->rx_iv32 ||
(iv32 == tkey->rx_iv32 && iv16 <= tkey->rx_iv16)) &&
tkey->initialized) {
plen = skb->len - hdr_len - 12;
- sg_init_one(&sg, pos, plen+4);
-
- crypto_sync_skcipher_setkey(tkey->rx_tfm_arc4, rc4key, 16);
- skcipher_request_set_sync_tfm(req, tkey->rx_tfm_arc4);
- skcipher_request_set_callback(req, 0, NULL, NULL);
- skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL);
- err = crypto_skcipher_decrypt(req);
- skcipher_request_zero(req);
- if (err) {
- if (net_ratelimit()) {
- netdev_dbg(skb->dev,
- "Failed to decrypt received packet from %pM\n",
- hdr->addr2);
- }
- return -7;
- }
+ arc4_setkey(&tkey->rx_ctx_arc4, rc4key, 16);
+ arc4_crypt(&tkey->rx_ctx_arc4, pos, pos, plen + 4);
crc = ~crc32_le(~0, pos, plen);
icv[0] = crc;
struct rtllib_tkip_data *tkey = priv;
int keyidx;
struct crypto_shash *tfm = tkey->tx_tfm_michael;
- struct crypto_sync_skcipher *tfm2 = tkey->tx_tfm_arc4;
struct crypto_shash *tfm3 = tkey->rx_tfm_michael;
- struct crypto_sync_skcipher *tfm4 = tkey->rx_tfm_arc4;
keyidx = tkey->key_idx;
memset(tkey, 0, sizeof(*tkey));
tkey->key_idx = keyidx;
tkey->tx_tfm_michael = tfm;
- tkey->tx_tfm_arc4 = tfm2;
tkey->rx_tfm_michael = tfm3;
- tkey->rx_tfm_arc4 = tfm4;
if (len == TKIP_KEY_LEN) {
memcpy(tkey->key, key, TKIP_KEY_LEN);
* Copyright (c) 2002-2004, Jouni Malinen <jkmaline@cc.hut.fi>
*/
-#include <crypto/skcipher.h>
+#include <crypto/arc4.h>
+#include <linux/fips.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/string.h>
#include "rtllib.h"
-#include <linux/scatterlist.h>
#include <linux/crc32.h>
struct prism2_wep_data {
u8 key[WEP_KEY_LEN + 1];
u8 key_len;
u8 key_idx;
- struct crypto_sync_skcipher *tx_tfm;
- struct crypto_sync_skcipher *rx_tfm;
+ struct arc4_ctx rx_ctx_arc4;
+ struct arc4_ctx tx_ctx_arc4;
};
{
struct prism2_wep_data *priv;
+ if (fips_enabled)
+ return NULL;
+
priv = kzalloc(sizeof(*priv), GFP_ATOMIC);
if (priv == NULL)
- goto fail;
+ return NULL;
priv->key_idx = keyidx;
- priv->tx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
- if (IS_ERR(priv->tx_tfm)) {
- pr_debug("rtllib_crypt_wep: could not allocate crypto API arc4\n");
- priv->tx_tfm = NULL;
- goto fail;
- }
- priv->rx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
- if (IS_ERR(priv->rx_tfm)) {
- pr_debug("rtllib_crypt_wep: could not allocate crypto API arc4\n");
- priv->rx_tfm = NULL;
- goto fail;
- }
-
/* start WEP IV from a random value */
get_random_bytes(&priv->iv, 4);
return priv;
-
-fail:
- if (priv) {
- crypto_free_sync_skcipher(priv->tx_tfm);
- crypto_free_sync_skcipher(priv->rx_tfm);
- kfree(priv);
- }
- return NULL;
}
static void prism2_wep_deinit(void *priv)
{
- struct prism2_wep_data *_priv = priv;
-
- if (_priv) {
- crypto_free_sync_skcipher(_priv->tx_tfm);
- crypto_free_sync_skcipher(_priv->rx_tfm);
- }
- kfree(priv);
+ kzfree(priv);
}
/* Perform WEP encryption on given skb that has at least 4 bytes of headroom
MAX_DEV_ADDR_SIZE);
u32 crc;
u8 *icv;
- struct scatterlist sg;
- int err;
if (skb_headroom(skb) < 4 || skb_tailroom(skb) < 4 ||
skb->len < hdr_len){
memcpy(key + 3, wep->key, wep->key_len);
if (!tcb_desc->bHwSec) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->tx_tfm);
-
/* Append little-endian CRC32 and encrypt it to produce ICV */
crc = ~crc32_le(~0, pos, len);
icv = skb_put(skb, 4);
icv[2] = crc >> 16;
icv[3] = crc >> 24;
- sg_init_one(&sg, pos, len+4);
- crypto_sync_skcipher_setkey(wep->tx_tfm, key, klen);
- skcipher_request_set_sync_tfm(req, wep->tx_tfm);
- skcipher_request_set_callback(req, 0, NULL, NULL);
- skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL);
- err = crypto_skcipher_encrypt(req);
- skcipher_request_zero(req);
- return err;
+ arc4_setkey(&wep->tx_ctx_arc4, key, klen);
+ arc4_crypt(&wep->tx_ctx_arc4, pos, pos, len + 4);
}
return 0;
MAX_DEV_ADDR_SIZE);
u32 crc;
u8 icv[4];
- struct scatterlist sg;
- int err;
if (skb->len < hdr_len + 8)
return -1;
plen = skb->len - hdr_len - 8;
if (!tcb_desc->bHwSec) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->rx_tfm);
-
- sg_init_one(&sg, pos, plen+4);
- crypto_sync_skcipher_setkey(wep->rx_tfm, key, klen);
- skcipher_request_set_sync_tfm(req, wep->rx_tfm);
- skcipher_request_set_callback(req, 0, NULL, NULL);
- skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL);
- err = crypto_skcipher_decrypt(req);
- skcipher_request_zero(req);
- if (err)
- return -7;
+ arc4_setkey(&wep->rx_ctx_arc4, key, klen);
+ arc4_crypt(&wep->rx_ctx_arc4, pos, pos, plen + 4);
+
crc = ~crc32_le(~0, pos, plen);
icv[0] = crc;
icv[1] = crc >> 8;
select CRYPTO
select CRYPTO_AES
select CRYPTO_CCM
+ select CRYPTO_LIB_ARC4
* Copyright (c) 2003-2004, Jouni Malinen <jkmaline@cc.hut.fi>
*/
+#include <linux/fips.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/slab.h>
#include "ieee80211.h"
+#include <crypto/arc4.h>
#include <crypto/hash.h>
-#include <crypto/skcipher.h>
- #include <linux/scatterlist.h>
#include <linux/crc32.h>
MODULE_AUTHOR("Jouni Malinen");
int key_idx;
- struct crypto_sync_skcipher *rx_tfm_arc4;
+ struct arc4_ctx rx_ctx_arc4;
+ struct arc4_ctx tx_ctx_arc4;
struct crypto_shash *rx_tfm_michael;
- struct crypto_sync_skcipher *tx_tfm_arc4;
struct crypto_shash *tx_tfm_michael;
/* scratch buffers for virt_to_page() (crypto API) */
{
struct ieee80211_tkip_data *priv;
+ if (fips_enabled)
+ return NULL;
+
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
goto fail;
priv->key_idx = key_idx;
- priv->tx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
- if (IS_ERR(priv->tx_tfm_arc4)) {
- printk(KERN_DEBUG "ieee80211_crypt_tkip: could not allocate "
- "crypto API arc4\n");
- priv->tx_tfm_arc4 = NULL;
- goto fail;
- }
-
priv->tx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0);
if (IS_ERR(priv->tx_tfm_michael)) {
printk(KERN_DEBUG "ieee80211_crypt_tkip: could not allocate "
goto fail;
}
- priv->rx_tfm_arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
- if (IS_ERR(priv->rx_tfm_arc4)) {
- printk(KERN_DEBUG "ieee80211_crypt_tkip: could not allocate "
- "crypto API arc4\n");
- priv->rx_tfm_arc4 = NULL;
- goto fail;
- }
-
priv->rx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0);
if (IS_ERR(priv->rx_tfm_michael)) {
printk(KERN_DEBUG "ieee80211_crypt_tkip: could not allocate "
fail:
if (priv) {
crypto_free_shash(priv->tx_tfm_michael);
- crypto_free_sync_skcipher(priv->tx_tfm_arc4);
crypto_free_shash(priv->rx_tfm_michael);
- crypto_free_sync_skcipher(priv->rx_tfm_arc4);
kfree(priv);
}
if (_priv) {
crypto_free_shash(_priv->tx_tfm_michael);
- crypto_free_sync_skcipher(_priv->tx_tfm_arc4);
crypto_free_shash(_priv->rx_tfm_michael);
- crypto_free_sync_skcipher(_priv->rx_tfm_arc4);
}
- kfree(priv);
+ kzfree(priv);
}
u8 *pos;
struct rtl_80211_hdr_4addr *hdr;
struct cb_desc *tcb_desc = (struct cb_desc *)(skb->cb + MAX_DEV_ADDR_SIZE);
- int ret = 0;
u8 rc4key[16], *icv;
u32 crc;
- struct scatterlist sg;
if (skb_headroom(skb) < 8 || skb_tailroom(skb) < 4 ||
skb->len < hdr_len)
*pos++ = (tkey->tx_iv32 >> 24) & 0xff;
if (!tcb_desc->bHwSec) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->tx_tfm_arc4);
-
icv = skb_put(skb, 4);
crc = ~crc32_le(~0, pos, len);
icv[0] = crc;
icv[1] = crc >> 8;
icv[2] = crc >> 16;
icv[3] = crc >> 24;
- crypto_sync_skcipher_setkey(tkey->tx_tfm_arc4, rc4key, 16);
- sg_init_one(&sg, pos, len + 4);
- skcipher_request_set_sync_tfm(req, tkey->tx_tfm_arc4);
- skcipher_request_set_callback(req, 0, NULL, NULL);
- skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL);
- ret = crypto_skcipher_encrypt(req);
- skcipher_request_zero(req);
+
+ arc4_setkey(&tkey->tx_ctx_arc4, rc4key, 16);
+ arc4_crypt(&tkey->tx_ctx_arc4, pos, pos, len + 4);
}
tkey->tx_iv16++;
tkey->tx_iv32++;
}
- if (!tcb_desc->bHwSec)
- return ret;
- else
- return 0;
-
-
+ return 0;
}
static int ieee80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
u8 rc4key[16];
u8 icv[4];
u32 crc;
- struct scatterlist sg;
int plen;
- int err;
if (skb->len < hdr_len + 8 + 4)
return -1;
pos += 8;
if (!tcb_desc->bHwSec) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, tkey->rx_tfm_arc4);
-
if (iv32 < tkey->rx_iv32 ||
(iv32 == tkey->rx_iv32 && iv16 <= tkey->rx_iv16)) {
if (net_ratelimit()) {
plen = skb->len - hdr_len - 12;
- crypto_sync_skcipher_setkey(tkey->rx_tfm_arc4, rc4key, 16);
- sg_init_one(&sg, pos, plen + 4);
-
- skcipher_request_set_sync_tfm(req, tkey->rx_tfm_arc4);
- skcipher_request_set_callback(req, 0, NULL, NULL);
- skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL);
-
- err = crypto_skcipher_decrypt(req);
- skcipher_request_zero(req);
- if (err) {
- if (net_ratelimit()) {
- netdev_dbg(skb->dev, "TKIP: failed to decrypt "
- "received packet from %pM\n",
- hdr->addr2);
- }
- return -7;
- }
+ arc4_setkey(&tkey->rx_ctx_arc4, rc4key, 16);
+ arc4_crypt(&tkey->rx_ctx_arc4, pos, pos, plen + 4);
crc = ~crc32_le(~0, pos, plen);
icv[0] = crc;
struct ieee80211_tkip_data *tkey = priv;
int keyidx;
struct crypto_shash *tfm = tkey->tx_tfm_michael;
- struct crypto_sync_skcipher *tfm2 = tkey->tx_tfm_arc4;
struct crypto_shash *tfm3 = tkey->rx_tfm_michael;
- struct crypto_sync_skcipher *tfm4 = tkey->rx_tfm_arc4;
keyidx = tkey->key_idx;
memset(tkey, 0, sizeof(*tkey));
tkey->key_idx = keyidx;
tkey->tx_tfm_michael = tfm;
- tkey->tx_tfm_arc4 = tfm2;
tkey->rx_tfm_michael = tfm3;
- tkey->rx_tfm_arc4 = tfm4;
if (len == TKIP_KEY_LEN) {
memcpy(tkey->key, key, TKIP_KEY_LEN);
* Copyright (c) 2002-2004, Jouni Malinen <jkmaline@cc.hut.fi>
*/
+#include <linux/fips.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/slab.h>
#include "ieee80211.h"
-#include <crypto/skcipher.h>
-#include <linux/scatterlist.h>
+#include <crypto/arc4.h>
#include <linux/crc32.h>
MODULE_AUTHOR("Jouni Malinen");
u8 key[WEP_KEY_LEN + 1];
u8 key_len;
u8 key_idx;
- struct crypto_sync_skcipher *tx_tfm;
- struct crypto_sync_skcipher *rx_tfm;
+ struct arc4_ctx rx_ctx_arc4;
+ struct arc4_ctx tx_ctx_arc4;
};
{
struct prism2_wep_data *priv;
+ if (fips_enabled)
+ return NULL;
+
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
return NULL;
priv->key_idx = keyidx;
- priv->tx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
- if (IS_ERR(priv->tx_tfm))
- goto free_priv;
- priv->rx_tfm = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
- if (IS_ERR(priv->rx_tfm))
- goto free_tx;
-
/* start WEP IV from a random value */
get_random_bytes(&priv->iv, 4);
return priv;
-free_tx:
- crypto_free_sync_skcipher(priv->tx_tfm);
-free_priv:
- kfree(priv);
- return NULL;
}
static void prism2_wep_deinit(void *priv)
{
- struct prism2_wep_data *_priv = priv;
-
- if (_priv) {
- crypto_free_sync_skcipher(_priv->tx_tfm);
- crypto_free_sync_skcipher(_priv->rx_tfm);
- }
- kfree(priv);
+ kzfree(priv);
}
/* Perform WEP encryption on given skb that has at least 4 bytes of headroom
struct cb_desc *tcb_desc = (struct cb_desc *)(skb->cb + MAX_DEV_ADDR_SIZE);
u32 crc;
u8 *icv;
- struct scatterlist sg;
- int err;
if (skb_headroom(skb) < 4 || skb_tailroom(skb) < 4 ||
skb->len < hdr_len)
memcpy(key + 3, wep->key, wep->key_len);
if (!tcb_desc->bHwSec) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->tx_tfm);
-
/* Append little-endian CRC32 and encrypt it to produce ICV */
crc = ~crc32_le(~0, pos, len);
icv = skb_put(skb, 4);
icv[2] = crc >> 16;
icv[3] = crc >> 24;
- crypto_sync_skcipher_setkey(wep->tx_tfm, key, klen);
- sg_init_one(&sg, pos, len + 4);
-
- skcipher_request_set_sync_tfm(req, wep->tx_tfm);
- skcipher_request_set_callback(req, 0, NULL, NULL);
- skcipher_request_set_crypt(req, &sg, &sg, len + 4, NULL);
-
- err = crypto_skcipher_encrypt(req);
- skcipher_request_zero(req);
- return err;
+ arc4_setkey(&wep->tx_ctx_arc4, key, klen);
+ arc4_crypt(&wep->tx_ctx_arc4, pos, pos, len + 4);
}
return 0;
struct cb_desc *tcb_desc = (struct cb_desc *)(skb->cb + MAX_DEV_ADDR_SIZE);
u32 crc;
u8 icv[4];
- struct scatterlist sg;
- int err;
if (skb->len < hdr_len + 8)
return -1;
plen = skb->len - hdr_len - 8;
if (!tcb_desc->bHwSec) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(req, wep->rx_tfm);
-
- crypto_sync_skcipher_setkey(wep->rx_tfm, key, klen);
- sg_init_one(&sg, pos, plen + 4);
-
- skcipher_request_set_sync_tfm(req, wep->rx_tfm);
- skcipher_request_set_callback(req, 0, NULL, NULL);
- skcipher_request_set_crypt(req, &sg, &sg, plen + 4, NULL);
-
- err = crypto_skcipher_decrypt(req);
- skcipher_request_zero(req);
- if (err)
- return -7;
+ arc4_setkey(&wep->rx_ctx_arc4, key, klen);
+ arc4_crypt(&wep->rx_ctx_arc4, pos, pos, plen + 4);
crc = ~crc32_le(~0, pos, plen);
icv[0] = crc;
bl += sprintf(b + bl, " ");
if (bd) {
bl += sprintf(b + bl, "Major: %d Minor: %d %s\n",
- MAJOR(bd->bd_dev), MINOR(bd->bd_dev), (!bd->bd_contains) ?
- "" : (bd->bd_holder == ib_dev) ?
- "CLAIMED: IBLOCK" : "CLAIMED: OS");
+ MAJOR(bd->bd_dev), MINOR(bd->bd_dev),
+ "CLAIMED: IBLOCK");
} else {
bl += sprintf(b + bl, "Major: 0 Minor: 0\n");
}
* out unpacked_lun for the original se_cmd.
*/
if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) {
- if (!target_lookup_lun_from_tag(se_sess, tag, &unpacked_lun))
+ if (!target_lookup_lun_from_tag(se_sess, tag,
+ &se_cmd->orig_fe_lun))
goto failure;
}
if (error)
return error;
+ /* Probe the USB device with the driver in hand, but only
+ * defer to a generic driver in case the current USB
+ * device driver has an id_table or a match function; i.e.,
+ * when the device driver was explicitly matched against
+ * a device.
+ *
+ * If the device driver does not have either of these,
+ * then we assume that it can bind to any device and is
+ * not truly a more specialized/non-generic driver, so a
+ * return value of -ENODEV should not force the device
+ * to be handled by the generic USB driver, as there
+ * can still be another, more specialized, device driver.
+ *
+ * This accommodates the usbip driver.
+ *
+ * TODO: What if, in the future, there are multiple
+ * specialized USB device drivers for a particular device?
+ * In such cases, there is a need to try all matching
+ * specialised device drivers prior to setting the
+ * use_generic_driver bit.
+ */
error = udriver->probe(udev);
- if (error == -ENODEV && udriver != &usb_generic_driver) {
+ if (error == -ENODEV && udriver != &usb_generic_driver &&
+ (udriver->id_table || udriver->match)) {
udev->use_generic_driver = 1;
return -EPROBE_DEFER;
}
udev = to_usb_device(dev);
udrv = to_usb_device_driver(drv);
- if (udrv->id_table &&
- usb_device_match_id(udev, udrv->id_table) != NULL) {
- return 1;
- }
+ if (udrv->id_table)
+ return usb_device_match_id(udev, udrv->id_table) != NULL;
if (udrv->match)
return udrv->match(udev);
- return 0;
+
+ /* If the device driver under consideration does not have a
+ * id_table or a match function, then let the driver's probe
+ * function decide.
+ */
+ return 1;
} else if (is_usb_interface(dev)) {
struct usb_interface *intf;
return 0;
}
-static bool is_dev_usb_generic_driver(struct device *dev)
-{
- struct usb_device_driver *udd = dev->driver ?
- to_usb_device_driver(dev->driver) : NULL;
-
- return udd == &usb_generic_driver;
-}
-
static int __usb_bus_reprobe_drivers(struct device *dev, void *data)
{
struct usb_device_driver *new_udriver = data;
struct usb_device *udev;
int ret;
- if (!is_dev_usb_generic_driver(dev))
+ /* Don't reprobe if current driver isn't usb_generic_driver */
+ if (dev->driver != &usb_generic_driver.drvwrap.driver)
return 0;
udev = to_usb_device(dev);
if (usb_device_match_id(udev, new_udriver->id_table) == NULL &&
- (!new_udriver->match || new_udriver->match(udev) != 0))
+ (!new_udriver->match || new_udriver->match(udev) == 0))
return 0;
ret = device_reprobe(dev);
const struct ndp_parser_opts *opts = ncm->parser_opts;
unsigned crc_len = ncm->is_crc ? sizeof(uint32_t) : 0;
int dgram_counter;
- bool ndp_after_header;
/* dwSignature */
if (get_unaligned_le32(tmp) != opts->nth_sign) {
}
ndp_index = get_ncm(&tmp, opts->ndp_index);
- ndp_after_header = false;
/* Run through all the NDP's in the NTB */
do {
ndp_index);
goto err;
}
- if (ndp_index == opts->nth_size)
- ndp_after_header = true;
/*
* walk through NDP
index2 = get_ncm(&tmp, opts->dgram_item_len);
dg_len2 = get_ncm(&tmp, opts->dgram_item_len);
- if (index2 == 0 || dg_len2 == 0)
- break;
-
/* wDatagramIndex[1] */
- if (ndp_after_header) {
- if (index2 < opts->nth_size + opts->ndp_size) {
- INFO(port->func.config->cdev,
- "Bad index: %#X\n", index2);
- goto err;
- }
- } else {
- if (index2 < opts->nth_size + opts->dpe_size) {
- INFO(port->func.config->cdev,
- "Bad index: %#X\n", index2);
- goto err;
- }
- }
if (index2 > block_len - opts->dpe_size) {
INFO(port->func.config->cdev,
"Bad index: %#X\n", index2);
goto err;
}
- /* wDatagramLength[1] */
- if ((dg_len2 < 14 + crc_len) ||
- (dg_len2 > frame_max)) {
- INFO(port->func.config->cdev,
- "Bad dgram length: %#X\n", dg_len);
- goto err;
- }
-
/*
* Copy the data into a new skb.
* This ensures the truesize is correct
ndp_len -= 2 * (opts->dgram_item_len * 2);
dgram_counter++;
+ if (index2 == 0 || dg_len2 == 0)
+ break;
} while (ndp_len > 2 * (opts->dgram_item_len * 2));
} while (ndp_index);
return;
}
-static bool usbip_match(struct usb_device *udev)
-{
- return true;
-}
-
#ifdef CONFIG_PM
/* These functions need usb_port_suspend and usb_port_resume,
.name = "usbip-host",
.probe = stub_probe,
.disconnect = stub_disconnect,
- .match = usbip_match,
#ifdef CONFIG_PM
.suspend = stub_suspend,
.resume = stub_resume,
be called ifcvf.
config MLX5_VDPA
- bool "MLX5 VDPA support library for ConnectX devices"
- depends on MLX5_CORE
- default n
+ bool
help
Support library for Mellanox VDPA drivers. Provides code that is
common for all types of VDPA drivers. The following drivers are planned:
config MLX5_VDPA_NET
tristate "vDPA driver for ConnectX devices"
- depends on MLX5_VDPA
+ select MLX5_VDPA
+ depends on MLX5_CORE
default n
help
VDPA network driver for ConnectX6 and newer. Provides offloading
if (!mvq->initialized)
return;
- if (query_virtqueue(ndev, mvq, &attr)) {
- mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n");
- return;
- }
if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY)
return;
if (modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND))
mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n");
+
+ if (query_virtqueue(ndev, mvq, &attr)) {
+ mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n");
+ return;
+ }
+ mvq->avail_idx = attr.available_index;
}
static void suspend_vqs(struct mlx5_vdpa_net *ndev)
struct mlx5_virtq_attr attr;
int err;
- if (!mvq->initialized)
- return -EAGAIN;
+ /* If the virtq object was destroyed, use the value saved at
+ * the last minute of suspend_vq. This caters for userspace
+ * that cares about emulating the index after vq is stopped.
+ */
+ if (!mvq->initialized) {
+ state->avail_index = mvq->avail_idx;
+ return 0;
+ }
err = query_virtqueue(ndev, mvq, &attr);
if (err) {
* vhost_iotlb_itree_first - return the first overlapped range
* @iotlb: the IOTLB
* @start: start of IOVA range
- * @end: end of IOVA range
+ * @last: last byte in IOVA range
*/
struct vhost_iotlb_map *
vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last)
* vhost_iotlb_itree_next - return the next overlapped range
* @map: the starting map node
* @start: start of IOVA range
- * @end: end of IOVA range
+ * @last: last byte IOVA range
*/
struct vhost_iotlb_map *
vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last)
struct vdpa_callback cb;
struct vhost_virtqueue *vq;
struct vhost_vring_state s;
- u64 __user *featurep = argp;
- u64 features;
u32 idx;
long r;
vq->last_avail_idx = vq_state.avail_index;
break;
- case VHOST_GET_BACKEND_FEATURES:
- features = VHOST_VDPA_BACKEND_FEATURES;
- if (copy_to_user(featurep, &features, sizeof(features)))
- return -EFAULT;
- return 0;
- case VHOST_SET_BACKEND_FEATURES:
- if (copy_from_user(&features, featurep, sizeof(features)))
- return -EFAULT;
- if (features & ~VHOST_VDPA_BACKEND_FEATURES)
- return -EOPNOTSUPP;
- vhost_set_backend_features(&v->vdev, features);
- return 0;
}
r = vhost_vring_ioctl(&v->vdev, cmd, argp);
struct vhost_vdpa *v = filep->private_data;
struct vhost_dev *d = &v->vdev;
void __user *argp = (void __user *)arg;
+ u64 __user *featurep = argp;
+ u64 features;
long r;
+ if (cmd == VHOST_SET_BACKEND_FEATURES) {
+ r = copy_from_user(&features, featurep, sizeof(features));
+ if (r)
+ return r;
+ if (features & ~VHOST_VDPA_BACKEND_FEATURES)
+ return -EOPNOTSUPP;
+ vhost_set_backend_features(&v->vdev, features);
+ return 0;
+ }
+
mutex_lock(&d->mutex);
switch (cmd) {
case VHOST_VDPA_SET_CONFIG_CALL:
r = vhost_vdpa_set_config_call(v, argp);
break;
+ case VHOST_GET_BACKEND_FEATURES:
+ features = VHOST_VDPA_BACKEND_FEATURES;
+ r = copy_to_user(featurep, &features, sizeof(features));
+ break;
default:
r = vhost_dev_ioctl(&v->vdev, cmd, argp);
if (r == -ENOIOCTLCMD)
perm_to_iommu_flags(perm));
}
+ if (r)
+ vhost_iotlb_del_range(dev->iotlb, iova, iova + size - 1);
+
return r;
}
struct vhost_dev *dev = &v->vdev;
struct vhost_iotlb *iotlb = dev->iotlb;
struct page **page_list;
- unsigned long list_size = PAGE_SIZE / sizeof(struct page *);
+ struct vm_area_struct **vmas;
unsigned int gup_flags = FOLL_LONGTERM;
- unsigned long npages, cur_base, map_pfn, last_pfn = 0;
- unsigned long locked, lock_limit, pinned, i;
+ unsigned long map_pfn, last_pfn = 0;
+ unsigned long npages, lock_limit;
+ unsigned long i, nmap = 0;
u64 iova = msg->iova;
+ long pinned;
int ret = 0;
if (vhost_iotlb_itree_first(iotlb, msg->iova,
msg->iova + msg->size - 1))
return -EEXIST;
- page_list = (struct page **) __get_free_page(GFP_KERNEL);
- if (!page_list)
- return -ENOMEM;
-
if (msg->perm & VHOST_ACCESS_WO)
gup_flags |= FOLL_WRITE;
if (!npages)
return -EINVAL;
+ page_list = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
+ vmas = kvmalloc_array(npages, sizeof(struct vm_area_struct *),
+ GFP_KERNEL);
+ if (!page_list || !vmas) {
+ ret = -ENOMEM;
+ goto free;
+ }
+
mmap_read_lock(dev->mm);
- locked = atomic64_add_return(npages, &dev->mm->pinned_vm);
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
- if (locked > lock_limit) {
+ if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) {
ret = -ENOMEM;
- goto out;
+ goto unlock;
}
- cur_base = msg->uaddr & PAGE_MASK;
- iova &= PAGE_MASK;
+ pinned = pin_user_pages(msg->uaddr & PAGE_MASK, npages, gup_flags,
+ page_list, vmas);
+ if (npages != pinned) {
+ if (pinned < 0) {
+ ret = pinned;
+ } else {
+ unpin_user_pages(page_list, pinned);
+ ret = -ENOMEM;
+ }
+ goto unlock;
+ }
- while (npages) {
- pinned = min_t(unsigned long, npages, list_size);
- ret = pin_user_pages(cur_base, pinned,
- gup_flags, page_list, NULL);
- if (ret != pinned)
- goto out;
-
- if (!last_pfn)
- map_pfn = page_to_pfn(page_list[0]);
-
- for (i = 0; i < ret; i++) {
- unsigned long this_pfn = page_to_pfn(page_list[i]);
- u64 csize;
-
- if (last_pfn && (this_pfn != last_pfn + 1)) {
- /* Pin a contiguous chunk of memory */
- csize = (last_pfn - map_pfn + 1) << PAGE_SHIFT;
- if (vhost_vdpa_map(v, iova, csize,
- map_pfn << PAGE_SHIFT,
- msg->perm))
- goto out;
- map_pfn = this_pfn;
- iova += csize;
+ iova &= PAGE_MASK;
+ map_pfn = page_to_pfn(page_list[0]);
+
+ /* One more iteration to avoid extra vdpa_map() call out of loop. */
+ for (i = 0; i <= npages; i++) {
+ unsigned long this_pfn;
+ u64 csize;
+
+ /* The last chunk may have no valid PFN next to it */
+ this_pfn = i < npages ? page_to_pfn(page_list[i]) : -1UL;
+
+ if (last_pfn && (this_pfn == -1UL ||
+ this_pfn != last_pfn + 1)) {
+ /* Pin a contiguous chunk of memory */
+ csize = last_pfn - map_pfn + 1;
+ ret = vhost_vdpa_map(v, iova, csize << PAGE_SHIFT,
+ map_pfn << PAGE_SHIFT,
+ msg->perm);
+ if (ret) {
+ /*
+ * Unpin the rest chunks of memory on the
+ * flight with no corresponding vdpa_map()
+ * calls having been made yet. On the other
+ * hand, vdpa_unmap() in the failure path
+ * is in charge of accounting the number of
+ * pinned pages for its own.
+ * This asymmetrical pattern of accounting
+ * is for efficiency to pin all pages at
+ * once, while there is no other callsite
+ * of vdpa_map() than here above.
+ */
+ unpin_user_pages(&page_list[nmap],
+ npages - nmap);
+ goto out;
}
-
- last_pfn = this_pfn;
+ atomic64_add(csize, &dev->mm->pinned_vm);
+ nmap += csize;
+ iova += csize << PAGE_SHIFT;
+ map_pfn = this_pfn;
}
-
- cur_base += ret << PAGE_SHIFT;
- npages -= ret;
+ last_pfn = this_pfn;
}
- /* Pin the rest chunk */
- ret = vhost_vdpa_map(v, iova, (last_pfn - map_pfn + 1) << PAGE_SHIFT,
- map_pfn << PAGE_SHIFT, msg->perm);
+ WARN_ON(nmap != npages);
out:
- if (ret) {
+ if (ret)
vhost_vdpa_unmap(v, msg->iova, msg->size);
- atomic64_sub(npages, &dev->mm->pinned_vm);
- }
+unlock:
mmap_read_unlock(dev->mm);
- free_page((unsigned long)page_list);
+free:
+ kvfree(vmas);
+ kvfree(page_list);
return ret;
}
err_init_iotlb:
vhost_dev_cleanup(&v->vdev);
+ kfree(vqs);
err:
atomic_dec(&v->opened);
return r;
vring_used_t __user *used)
{
+ /* If an IOTLB device is present, the vring addresses are
+ * GIOVAs. Access validation occurs at prefetch time. */
+ if (vq->iotlb)
+ return true;
+
return access_ok(desc, vhost_get_desc_size(vq, num)) &&
access_ok(avail, vhost_get_avail_size(vq, num)) &&
access_ok(used, vhost_get_used_size(vq, num));
}
EXPORT_SYMBOL_GPL(vhost_log_access_ok);
+static bool vq_log_used_access_ok(struct vhost_virtqueue *vq,
+ void __user *log_base,
+ bool log_used,
+ u64 log_addr)
+{
+ /* If an IOTLB device is present, log_addr is a GIOVA that
+ * will never be logged by log_used(). */
+ if (vq->iotlb)
+ return true;
+
+ return !log_used || log_access_ok(log_base, log_addr,
+ vhost_get_used_size(vq, vq->num));
+}
+
/* Verify access for write logging. */
/* Caller should have vq mutex and device mutex */
static bool vq_log_access_ok(struct vhost_virtqueue *vq,
{
return vq_memory_access_ok(log_base, vq->umem,
vhost_has_feature(vq, VHOST_F_LOG_ALL)) &&
- (!vq->log_used || log_access_ok(log_base, vq->log_addr,
- vhost_get_used_size(vq, vq->num)));
+ vq_log_used_access_ok(vq, log_base, vq->log_used, vq->log_addr);
}
/* Can we start vq? */
if (!vq_log_access_ok(vq, vq->log_base))
return false;
- /* Access validation occurs at prefetch time with IOTLB */
- if (vq->iotlb)
- return true;
-
return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used);
}
EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
return -EINVAL;
/* Also validate log access for used ring if enabled. */
- if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
- !log_access_ok(vq->log_base, a.log_guest_addr,
- sizeof *vq->used +
- vq->num * sizeof *vq->used->ring))
+ if (!vq_log_used_access_ok(vq, vq->log_base,
+ a.flags & (0x1 << VHOST_VRING_F_LOG),
+ a.log_guest_addr))
return -EINVAL;
}
#define FONT_DATA ((unsigned char *)font_vga_8x16.data)
-/* borrowed from fbcon.c */
-#define REFCOUNT(fd) (((int *)(fd))[-1])
-#define FNTSIZE(fd) (((int *)(fd))[-2])
-#define FNTCHARCNT(fd) (((int *)(fd))[-3])
-#define FONT_EXTRA_WORDS 3
-
static unsigned char *font_data[MAX_NR_CONSOLES];
static struct newport_regs *npregs;
FNTSIZE(new_data) = size;
FNTCHARCNT(new_data) = op->charcount;
REFCOUNT(new_data) = 0; /* usage counter */
+ FNTSUM(new_data) = 0;
p = new_data;
for (i = 0; i < op->charcount; i++) {
help
Support the Permedia2 FIFO disconnect feature.
+config FB_ARMCLCD
+ tristate "ARM PrimeCell PL110 support"
+ depends on ARM || ARM64 || COMPILE_TEST
+ depends on FB && ARM_AMBA && HAS_IOMEM
+ select FB_CFB_FILLRECT
+ select FB_CFB_COPYAREA
+ select FB_CFB_IMAGEBLIT
+ select FB_MODE_HELPERS if OF
+ select VIDEOMODE_HELPERS if OF
+ select BACKLIGHT_CLASS_DEVICE if OF
+ help
+ This framebuffer device driver is for the ARM PrimeCell PL110
+ Colour LCD controller. ARM PrimeCells provide the building
+ blocks for System on a Chip devices.
+
+ If you want to compile this as a module (=code which can be
+ inserted into and removed from the running kernel), say M
+ here and read <file:Documentation/kbuild/modules.rst>. The module
+ will be called amba-clcd.
+
config FB_ACORN
bool "Acorn VIDC support"
depends on (FB = y) && ARM && ARCH_ACORN
obj-$(CONFIG_FB_ATMEL) += atmel_lcdfb.o
obj-$(CONFIG_FB_PVR2) += pvr2fb.o
obj-$(CONFIG_FB_VOODOO1) += sstfb.o
+obj-$(CONFIG_FB_ARMCLCD) += amba-clcd.o
obj-$(CONFIG_FB_GOLDFISH) += goldfishfb.o
obj-$(CONFIG_FB_68328) += 68328fb.o
obj-$(CONFIG_FB_GBE) += gbefb.o
--- /dev/null
+/*
+ * linux/drivers/video/amba-clcd.c
+ *
+ * Copyright (C) 2001 ARM Limited, by David A Rusling
+ * Updated to 2.5, Deep Blue Solutions Ltd.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of this archive
+ * for more details.
+ *
+ * ARM PrimeCell PL110 Color LCD Controller
+ */
+#include <linux/amba/bus.h>
+#include <linux/amba/clcd.h>
+#include <linux/backlight.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/fb.h>
+#include <linux/init.h>
+#include <linux/ioport.h>
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_graph.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <video/display_timing.h>
+#include <video/of_display_timing.h>
+#include <video/videomode.h>
+
+#define to_clcd(info) container_of(info, struct clcd_fb, fb)
+
+/* This is limited to 16 characters when displayed by X startup */
+static const char *clcd_name = "CLCD FB";
+
+/*
+ * Unfortunately, the enable/disable functions may be called either from
+ * process or IRQ context, and we _need_ to delay. This is _not_ good.
+ */
+static inline void clcdfb_sleep(unsigned int ms)
+{
+ if (in_atomic()) {
+ mdelay(ms);
+ } else {
+ msleep(ms);
+ }
+}
+
+static inline void clcdfb_set_start(struct clcd_fb *fb)
+{
+ unsigned long ustart = fb->fb.fix.smem_start;
+ unsigned long lstart;
+
+ ustart += fb->fb.var.yoffset * fb->fb.fix.line_length;
+ lstart = ustart + fb->fb.var.yres * fb->fb.fix.line_length / 2;
+
+ writel(ustart, fb->regs + CLCD_UBAS);
+ writel(lstart, fb->regs + CLCD_LBAS);
+}
+
+static void clcdfb_disable(struct clcd_fb *fb)
+{
+ u32 val;
+
+ if (fb->board->disable)
+ fb->board->disable(fb);
+
+ if (fb->panel->backlight) {
+ fb->panel->backlight->props.power = FB_BLANK_POWERDOWN;
+ backlight_update_status(fb->panel->backlight);
+ }
+
+ val = readl(fb->regs + fb->off_cntl);
+ if (val & CNTL_LCDPWR) {
+ val &= ~CNTL_LCDPWR;
+ writel(val, fb->regs + fb->off_cntl);
+
+ clcdfb_sleep(20);
+ }
+ if (val & CNTL_LCDEN) {
+ val &= ~CNTL_LCDEN;
+ writel(val, fb->regs + fb->off_cntl);
+ }
+
+ /*
+ * Disable CLCD clock source.
+ */
+ if (fb->clk_enabled) {
+ fb->clk_enabled = false;
+ clk_disable(fb->clk);
+ }
+}
+
+static void clcdfb_enable(struct clcd_fb *fb, u32 cntl)
+{
+ /*
+ * Enable the CLCD clock source.
+ */
+ if (!fb->clk_enabled) {
+ fb->clk_enabled = true;
+ clk_enable(fb->clk);
+ }
+
+ /*
+ * Bring up by first enabling..
+ */
+ cntl |= CNTL_LCDEN;
+ writel(cntl, fb->regs + fb->off_cntl);
+
+ clcdfb_sleep(20);
+
+ /*
+ * and now apply power.
+ */
+ cntl |= CNTL_LCDPWR;
+ writel(cntl, fb->regs + fb->off_cntl);
+
+ /*
+ * Turn on backlight
+ */
+ if (fb->panel->backlight) {
+ fb->panel->backlight->props.power = FB_BLANK_UNBLANK;
+ backlight_update_status(fb->panel->backlight);
+ }
+
+ /*
+ * finally, enable the interface.
+ */
+ if (fb->board->enable)
+ fb->board->enable(fb);
+}
+
+static int
+clcdfb_set_bitfields(struct clcd_fb *fb, struct fb_var_screeninfo *var)
+{
+ u32 caps;
+ int ret = 0;
+
+ if (fb->panel->caps && fb->board->caps)
+ caps = fb->panel->caps & fb->board->caps;
+ else {
+ /* Old way of specifying what can be used */
+ caps = fb->panel->cntl & CNTL_BGR ?
+ CLCD_CAP_BGR : CLCD_CAP_RGB;
+ /* But mask out 444 modes as they weren't supported */
+ caps &= ~CLCD_CAP_444;
+ }
+
+ /* Only TFT panels can do RGB888/BGR888 */
+ if (!(fb->panel->cntl & CNTL_LCDTFT))
+ caps &= ~CLCD_CAP_888;
+
+ memset(&var->transp, 0, sizeof(var->transp));
+
+ var->red.msb_right = 0;
+ var->green.msb_right = 0;
+ var->blue.msb_right = 0;
+
+ switch (var->bits_per_pixel) {
+ case 1:
+ case 2:
+ case 4:
+ case 8:
+ /* If we can't do 5551, reject */
+ caps &= CLCD_CAP_5551;
+ if (!caps) {
+ ret = -EINVAL;
+ break;
+ }
+
+ var->red.length = var->bits_per_pixel;
+ var->red.offset = 0;
+ var->green.length = var->bits_per_pixel;
+ var->green.offset = 0;
+ var->blue.length = var->bits_per_pixel;
+ var->blue.offset = 0;
+ break;
+
+ case 16:
+ /* If we can't do 444, 5551 or 565, reject */
+ if (!(caps & (CLCD_CAP_444 | CLCD_CAP_5551 | CLCD_CAP_565))) {
+ ret = -EINVAL;
+ break;
+ }
+
+ /*
+ * Green length can be 4, 5 or 6 depending whether
+ * we're operating in 444, 5551 or 565 mode.
+ */
+ if (var->green.length == 4 && caps & CLCD_CAP_444)
+ caps &= CLCD_CAP_444;
+ if (var->green.length == 5 && caps & CLCD_CAP_5551)
+ caps &= CLCD_CAP_5551;
+ else if (var->green.length == 6 && caps & CLCD_CAP_565)
+ caps &= CLCD_CAP_565;
+ else {
+ /*
+ * PL110 officially only supports RGB555,
+ * but may be wired up to allow RGB565.
+ */
+ if (caps & CLCD_CAP_565) {
+ var->green.length = 6;
+ caps &= CLCD_CAP_565;
+ } else if (caps & CLCD_CAP_5551) {
+ var->green.length = 5;
+ caps &= CLCD_CAP_5551;
+ } else {
+ var->green.length = 4;
+ caps &= CLCD_CAP_444;
+ }
+ }
+
+ if (var->green.length >= 5) {
+ var->red.length = 5;
+ var->blue.length = 5;
+ } else {
+ var->red.length = 4;
+ var->blue.length = 4;
+ }
+ break;
+ case 32:
+ /* If we can't do 888, reject */
+ caps &= CLCD_CAP_888;
+ if (!caps) {
+ ret = -EINVAL;
+ break;
+ }
+
+ var->red.length = 8;
+ var->green.length = 8;
+ var->blue.length = 8;
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+ /*
+ * >= 16bpp displays have separate colour component bitfields
+ * encoded in the pixel data. Calculate their position from
+ * the bitfield length defined above.
+ */
+ if (ret == 0 && var->bits_per_pixel >= 16) {
+ bool bgr, rgb;
+
+ bgr = caps & CLCD_CAP_BGR && var->blue.offset == 0;
+ rgb = caps & CLCD_CAP_RGB && var->red.offset == 0;
+
+ if (!bgr && !rgb)
+ /*
+ * The requested format was not possible, try just
+ * our capabilities. One of BGR or RGB must be
+ * supported.
+ */
+ bgr = caps & CLCD_CAP_BGR;
+
+ if (bgr) {
+ var->blue.offset = 0;
+ var->green.offset = var->blue.offset + var->blue.length;
+ var->red.offset = var->green.offset + var->green.length;
+ } else {
+ var->red.offset = 0;
+ var->green.offset = var->red.offset + var->red.length;
+ var->blue.offset = var->green.offset + var->green.length;
+ }
+ }
+
+ return ret;
+}
+
+static int clcdfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
+{
+ struct clcd_fb *fb = to_clcd(info);
+ int ret = -EINVAL;
+
+ if (fb->board->check)
+ ret = fb->board->check(fb, var);
+
+ if (ret == 0 &&
+ var->xres_virtual * var->bits_per_pixel / 8 *
+ var->yres_virtual > fb->fb.fix.smem_len)
+ ret = -EINVAL;
+
+ if (ret == 0)
+ ret = clcdfb_set_bitfields(fb, var);
+
+ return ret;
+}
+
+static int clcdfb_set_par(struct fb_info *info)
+{
+ struct clcd_fb *fb = to_clcd(info);
+ struct clcd_regs regs;
+
+ fb->fb.fix.line_length = fb->fb.var.xres_virtual *
+ fb->fb.var.bits_per_pixel / 8;
+
+ if (fb->fb.var.bits_per_pixel <= 8)
+ fb->fb.fix.visual = FB_VISUAL_PSEUDOCOLOR;
+ else
+ fb->fb.fix.visual = FB_VISUAL_TRUECOLOR;
+
+ fb->board->decode(fb, ®s);
+
+ clcdfb_disable(fb);
+
+ writel(regs.tim0, fb->regs + CLCD_TIM0);
+ writel(regs.tim1, fb->regs + CLCD_TIM1);
+ writel(regs.tim2, fb->regs + CLCD_TIM2);
+ writel(regs.tim3, fb->regs + CLCD_TIM3);
+
+ clcdfb_set_start(fb);
+
+ clk_set_rate(fb->clk, (1000000000 / regs.pixclock) * 1000);
+
+ fb->clcd_cntl = regs.cntl;
+
+ clcdfb_enable(fb, regs.cntl);
+
+#ifdef DEBUG
+ printk(KERN_INFO
+ "CLCD: Registers set to\n"
+ " %08x %08x %08x %08x\n"
+ " %08x %08x %08x %08x\n",
+ readl(fb->regs + CLCD_TIM0), readl(fb->regs + CLCD_TIM1),
+ readl(fb->regs + CLCD_TIM2), readl(fb->regs + CLCD_TIM3),
+ readl(fb->regs + CLCD_UBAS), readl(fb->regs + CLCD_LBAS),
+ readl(fb->regs + fb->off_ienb), readl(fb->regs + fb->off_cntl));
+#endif
+
+ return 0;
+}
+
+static inline u32 convert_bitfield(int val, struct fb_bitfield *bf)
+{
+ unsigned int mask = (1 << bf->length) - 1;
+
+ return (val >> (16 - bf->length) & mask) << bf->offset;
+}
+
+/*
+ * Set a single color register. The values supplied have a 16 bit
+ * magnitude. Return != 0 for invalid regno.
+ */
+static int
+clcdfb_setcolreg(unsigned int regno, unsigned int red, unsigned int green,
+ unsigned int blue, unsigned int transp, struct fb_info *info)
+{
+ struct clcd_fb *fb = to_clcd(info);
+
+ if (regno < 16)
+ fb->cmap[regno] = convert_bitfield(transp, &fb->fb.var.transp) |
+ convert_bitfield(blue, &fb->fb.var.blue) |
+ convert_bitfield(green, &fb->fb.var.green) |
+ convert_bitfield(red, &fb->fb.var.red);
+
+ if (fb->fb.fix.visual == FB_VISUAL_PSEUDOCOLOR && regno < 256) {
+ int hw_reg = CLCD_PALETTE + ((regno * 2) & ~3);
+ u32 val, mask, newval;
+
+ newval = (red >> 11) & 0x001f;
+ newval |= (green >> 6) & 0x03e0;
+ newval |= (blue >> 1) & 0x7c00;
+
+ /*
+ * 3.2.11: if we're configured for big endian
+ * byte order, the palette entries are swapped.
+ */
+ if (fb->clcd_cntl & CNTL_BEBO)
+ regno ^= 1;
+
+ if (regno & 1) {
+ newval <<= 16;
+ mask = 0x0000ffff;
+ } else {
+ mask = 0xffff0000;
+ }
+
+ val = readl(fb->regs + hw_reg) & mask;
+ writel(val | newval, fb->regs + hw_reg);
+ }
+
+ return regno > 255;
+}
+
+/*
+ * Blank the screen if blank_mode != 0, else unblank. If blank == NULL
+ * then the caller blanks by setting the CLUT (Color Look Up Table) to all
+ * black. Return 0 if blanking succeeded, != 0 if un-/blanking failed due
+ * to e.g. a video mode which doesn't support it. Implements VESA suspend
+ * and powerdown modes on hardware that supports disabling hsync/vsync:
+ * blank_mode == 2: suspend vsync
+ * blank_mode == 3: suspend hsync
+ * blank_mode == 4: powerdown
+ */
+static int clcdfb_blank(int blank_mode, struct fb_info *info)
+{
+ struct clcd_fb *fb = to_clcd(info);
+
+ if (blank_mode != 0) {
+ clcdfb_disable(fb);
+ } else {
+ clcdfb_enable(fb, fb->clcd_cntl);
+ }
+ return 0;
+}
+
+static int clcdfb_mmap(struct fb_info *info,
+ struct vm_area_struct *vma)
+{
+ struct clcd_fb *fb = to_clcd(info);
+ unsigned long len, off = vma->vm_pgoff << PAGE_SHIFT;
+ int ret = -EINVAL;
+
+ len = info->fix.smem_len;
+
+ if (off <= len && vma->vm_end - vma->vm_start <= len - off &&
+ fb->board->mmap)
+ ret = fb->board->mmap(fb, vma);
+
+ return ret;
+}
+
+static const struct fb_ops clcdfb_ops = {
+ .owner = THIS_MODULE,
+ .fb_check_var = clcdfb_check_var,
+ .fb_set_par = clcdfb_set_par,
+ .fb_setcolreg = clcdfb_setcolreg,
+ .fb_blank = clcdfb_blank,
+ .fb_fillrect = cfb_fillrect,
+ .fb_copyarea = cfb_copyarea,
+ .fb_imageblit = cfb_imageblit,
+ .fb_mmap = clcdfb_mmap,
+};
+
+static int clcdfb_register(struct clcd_fb *fb)
+{
+ int ret;
+
+ /*
+ * ARM PL111 always has IENB at 0x1c; it's only PL110
+ * which is reversed on some platforms.
+ */
+ if (amba_manf(fb->dev) == 0x41 && amba_part(fb->dev) == 0x111) {
+ fb->off_ienb = CLCD_PL111_IENB;
+ fb->off_cntl = CLCD_PL111_CNTL;
+ } else {
+ fb->off_ienb = CLCD_PL110_IENB;
+ fb->off_cntl = CLCD_PL110_CNTL;
+ }
+
+ fb->clk = clk_get(&fb->dev->dev, NULL);
+ if (IS_ERR(fb->clk)) {
+ ret = PTR_ERR(fb->clk);
+ goto out;
+ }
+
+ ret = clk_prepare(fb->clk);
+ if (ret)
+ goto free_clk;
+
+ fb->fb.device = &fb->dev->dev;
+
+ fb->fb.fix.mmio_start = fb->dev->res.start;
+ fb->fb.fix.mmio_len = resource_size(&fb->dev->res);
+
+ fb->regs = ioremap(fb->fb.fix.mmio_start, fb->fb.fix.mmio_len);
+ if (!fb->regs) {
+ printk(KERN_ERR "CLCD: unable to remap registers\n");
+ ret = -ENOMEM;
+ goto clk_unprep;
+ }
+
+ fb->fb.fbops = &clcdfb_ops;
+ fb->fb.flags = FBINFO_FLAG_DEFAULT;
+ fb->fb.pseudo_palette = fb->cmap;
+
+ strncpy(fb->fb.fix.id, clcd_name, sizeof(fb->fb.fix.id));
+ fb->fb.fix.type = FB_TYPE_PACKED_PIXELS;
+ fb->fb.fix.type_aux = 0;
+ fb->fb.fix.xpanstep = 0;
+ fb->fb.fix.ypanstep = 0;
+ fb->fb.fix.ywrapstep = 0;
+ fb->fb.fix.accel = FB_ACCEL_NONE;
+
+ fb->fb.var.xres = fb->panel->mode.xres;
+ fb->fb.var.yres = fb->panel->mode.yres;
+ fb->fb.var.xres_virtual = fb->panel->mode.xres;
+ fb->fb.var.yres_virtual = fb->panel->mode.yres;
+ fb->fb.var.bits_per_pixel = fb->panel->bpp;
+ fb->fb.var.grayscale = fb->panel->grayscale;
+ fb->fb.var.pixclock = fb->panel->mode.pixclock;
+ fb->fb.var.left_margin = fb->panel->mode.left_margin;
+ fb->fb.var.right_margin = fb->panel->mode.right_margin;
+ fb->fb.var.upper_margin = fb->panel->mode.upper_margin;
+ fb->fb.var.lower_margin = fb->panel->mode.lower_margin;
+ fb->fb.var.hsync_len = fb->panel->mode.hsync_len;
+ fb->fb.var.vsync_len = fb->panel->mode.vsync_len;
+ fb->fb.var.sync = fb->panel->mode.sync;
+ fb->fb.var.vmode = fb->panel->mode.vmode;
+ fb->fb.var.activate = FB_ACTIVATE_NOW;
+ fb->fb.var.nonstd = 0;
+ fb->fb.var.height = fb->panel->height;
+ fb->fb.var.width = fb->panel->width;
+ fb->fb.var.accel_flags = 0;
+
+ fb->fb.monspecs.hfmin = 0;
+ fb->fb.monspecs.hfmax = 100000;
+ fb->fb.monspecs.vfmin = 0;
+ fb->fb.monspecs.vfmax = 400;
+ fb->fb.monspecs.dclkmin = 1000000;
+ fb->fb.monspecs.dclkmax = 100000000;
+
+ /*
+ * Make sure that the bitfields are set appropriately.
+ */
+ clcdfb_set_bitfields(fb, &fb->fb.var);
+
+ /*
+ * Allocate colourmap.
+ */
+ ret = fb_alloc_cmap(&fb->fb.cmap, 256, 0);
+ if (ret)
+ goto unmap;
+
+ /*
+ * Ensure interrupts are disabled.
+ */
+ writel(0, fb->regs + fb->off_ienb);
+
+ fb_set_var(&fb->fb, &fb->fb.var);
+
+ dev_info(&fb->dev->dev, "%s hardware, %s display\n",
+ fb->board->name, fb->panel->mode.name);
+
+ ret = register_framebuffer(&fb->fb);
+ if (ret == 0)
+ goto out;
+
+ printk(KERN_ERR "CLCD: cannot register framebuffer (%d)\n", ret);
+
+ fb_dealloc_cmap(&fb->fb.cmap);
+ unmap:
+ iounmap(fb->regs);
+ clk_unprep:
+ clk_unprepare(fb->clk);
+ free_clk:
+ clk_put(fb->clk);
+ out:
+ return ret;
+}
+
+#ifdef CONFIG_OF
+static int clcdfb_of_get_dpi_panel_mode(struct device_node *node,
+ struct clcd_panel *clcd_panel)
+{
+ int err;
+ struct display_timing timing;
+ struct videomode video;
+
+ err = of_get_display_timing(node, "panel-timing", &timing);
+ if (err) {
+ pr_err("%pOF: problems parsing panel-timing (%d)\n", node, err);
+ return err;
+ }
+
+ videomode_from_timing(&timing, &video);
+
+ err = fb_videomode_from_videomode(&video, &clcd_panel->mode);
+ if (err)
+ return err;
+
+ /* Set up some inversion flags */
+ if (timing.flags & DISPLAY_FLAGS_PIXDATA_NEGEDGE)
+ clcd_panel->tim2 |= TIM2_IPC;
+ else if (!(timing.flags & DISPLAY_FLAGS_PIXDATA_POSEDGE))
+ /*
+ * To preserve backwards compatibility, the IPC (inverted
+ * pixel clock) flag needs to be set on any display that
+ * doesn't explicitly specify that the pixel clock is
+ * active on the negative or positive edge.
+ */
+ clcd_panel->tim2 |= TIM2_IPC;
+
+ if (timing.flags & DISPLAY_FLAGS_HSYNC_LOW)
+ clcd_panel->tim2 |= TIM2_IHS;
+
+ if (timing.flags & DISPLAY_FLAGS_VSYNC_LOW)
+ clcd_panel->tim2 |= TIM2_IVS;
+
+ if (timing.flags & DISPLAY_FLAGS_DE_LOW)
+ clcd_panel->tim2 |= TIM2_IOE;
+
+ return 0;
+}
+
+static int clcdfb_snprintf_mode(char *buf, int size, struct fb_videomode *mode)
+{
+ return snprintf(buf, size, "%ux%u@%u", mode->xres, mode->yres,
+ mode->refresh);
+}
+
+static int clcdfb_of_get_backlight(struct device *dev,
+ struct clcd_panel *clcd_panel)
+{
+ struct backlight_device *backlight;
+
+ /* Look up the optional backlight device */
+ backlight = devm_of_find_backlight(dev);
+ if (IS_ERR(backlight))
+ return PTR_ERR(backlight);
+
+ clcd_panel->backlight = backlight;
+ return 0;
+}
+
+static int clcdfb_of_get_mode(struct device *dev, struct device_node *panel,
+ struct clcd_panel *clcd_panel)
+{
+ int err;
+ struct fb_videomode *mode;
+ char *name;
+ int len;
+
+ /* Only directly connected DPI panels supported for now */
+ if (of_device_is_compatible(panel, "panel-dpi"))
+ err = clcdfb_of_get_dpi_panel_mode(panel, clcd_panel);
+ else
+ err = -ENOENT;
+ if (err)
+ return err;
+ mode = &clcd_panel->mode;
+
+ len = clcdfb_snprintf_mode(NULL, 0, mode);
+ name = devm_kzalloc(dev, len + 1, GFP_KERNEL);
+ if (!name)
+ return -ENOMEM;
+
+ clcdfb_snprintf_mode(name, len + 1, mode);
+ mode->name = name;
+
+ return 0;
+}
+
+static int clcdfb_of_init_tft_panel(struct clcd_fb *fb, u32 r0, u32 g0, u32 b0)
+{
+ static struct {
+ unsigned int part;
+ u32 r0, g0, b0;
+ u32 caps;
+ } panels[] = {
+ { 0x110, 1, 7, 13, CLCD_CAP_5551 },
+ { 0x110, 0, 8, 16, CLCD_CAP_888 },
+ { 0x110, 16, 8, 0, CLCD_CAP_888 },
+ { 0x111, 4, 14, 20, CLCD_CAP_444 },
+ { 0x111, 3, 11, 19, CLCD_CAP_444 | CLCD_CAP_5551 },
+ { 0x111, 3, 10, 19, CLCD_CAP_444 | CLCD_CAP_5551 |
+ CLCD_CAP_565 },
+ { 0x111, 0, 8, 16, CLCD_CAP_444 | CLCD_CAP_5551 |
+ CLCD_CAP_565 | CLCD_CAP_888 },
+ };
+ int i;
+
+ /* Bypass pixel clock divider */
+ fb->panel->tim2 |= TIM2_BCD;
+
+ /* TFT display, vert. comp. interrupt at the start of the back porch */
+ fb->panel->cntl |= CNTL_LCDTFT | CNTL_LCDVCOMP(1);
+
+ fb->panel->caps = 0;
+
+ /* Match the setup with known variants */
+ for (i = 0; i < ARRAY_SIZE(panels) && !fb->panel->caps; i++) {
+ if (amba_part(fb->dev) != panels[i].part)
+ continue;
+ if (g0 != panels[i].g0)
+ continue;
+ if (r0 == panels[i].r0 && b0 == panels[i].b0)
+ fb->panel->caps = panels[i].caps;
+ }
+
+ /*
+ * If we actually physically connected the R lines to B and
+ * vice versa
+ */
+ if (r0 != 0 && b0 == 0)
+ fb->panel->bgr_connection = true;
+
+ return fb->panel->caps ? 0 : -EINVAL;
+}
+
+static int clcdfb_of_init_display(struct clcd_fb *fb)
+{
+ struct device_node *endpoint, *panel;
+ int err;
+ unsigned int bpp;
+ u32 max_bandwidth;
+ u32 tft_r0b0g0[3];
+
+ fb->panel = devm_kzalloc(&fb->dev->dev, sizeof(*fb->panel), GFP_KERNEL);
+ if (!fb->panel)
+ return -ENOMEM;
+
+ /*
+ * Fetch the panel endpoint.
+ */
+ endpoint = of_graph_get_next_endpoint(fb->dev->dev.of_node, NULL);
+ if (!endpoint)
+ return -ENODEV;
+
+ panel = of_graph_get_remote_port_parent(endpoint);
+ if (!panel)
+ return -ENODEV;
+
+ err = clcdfb_of_get_backlight(&fb->dev->dev, fb->panel);
+ if (err)
+ return err;
+
+ err = clcdfb_of_get_mode(&fb->dev->dev, panel, fb->panel);
+ if (err)
+ return err;
+
+ err = of_property_read_u32(fb->dev->dev.of_node, "max-memory-bandwidth",
+ &max_bandwidth);
+ if (!err) {
+ /*
+ * max_bandwidth is in bytes per second and pixclock in
+ * pico-seconds, so the maximum allowed bits per pixel is
+ * 8 * max_bandwidth / (PICOS2KHZ(pixclock) * 1000)
+ * Rearrange this calculation to avoid overflow and then ensure
+ * result is a valid format.
+ */
+ bpp = max_bandwidth / (1000 / 8)
+ / PICOS2KHZ(fb->panel->mode.pixclock);
+ bpp = rounddown_pow_of_two(bpp);
+ if (bpp > 32)
+ bpp = 32;
+ } else
+ bpp = 32;
+ fb->panel->bpp = bpp;
+
+#ifdef CONFIG_CPU_BIG_ENDIAN
+ fb->panel->cntl |= CNTL_BEBO;
+#endif
+ fb->panel->width = -1;
+ fb->panel->height = -1;
+
+ if (of_property_read_u32_array(endpoint,
+ "arm,pl11x,tft-r0g0b0-pads",
+ tft_r0b0g0, ARRAY_SIZE(tft_r0b0g0)) != 0)
+ return -ENOENT;
+
+ return clcdfb_of_init_tft_panel(fb, tft_r0b0g0[0],
+ tft_r0b0g0[1], tft_r0b0g0[2]);
+}
+
+static int clcdfb_of_vram_setup(struct clcd_fb *fb)
+{
+ int err;
+ struct device_node *memory;
+ u64 size;
+
+ err = clcdfb_of_init_display(fb);
+ if (err)
+ return err;
+
+ memory = of_parse_phandle(fb->dev->dev.of_node, "memory-region", 0);
+ if (!memory)
+ return -ENODEV;
+
+ fb->fb.screen_base = of_iomap(memory, 0);
+ if (!fb->fb.screen_base)
+ return -ENOMEM;
+
+ fb->fb.fix.smem_start = of_translate_address(memory,
+ of_get_address(memory, 0, &size, NULL));
+ fb->fb.fix.smem_len = size;
+
+ return 0;
+}
+
+static int clcdfb_of_vram_mmap(struct clcd_fb *fb, struct vm_area_struct *vma)
+{
+ unsigned long off, user_size, kernel_size;
+
+
+ off = vma->vm_pgoff << PAGE_SHIFT;
+ user_size = vma->vm_end - vma->vm_start;
+ kernel_size = fb->fb.fix.smem_len;
+
+ if (off >= kernel_size || user_size > (kernel_size - off))
+ return -ENXIO;
+
+ return remap_pfn_range(vma, vma->vm_start,
+ __phys_to_pfn(fb->fb.fix.smem_start) + vma->vm_pgoff,
+ user_size,
+ pgprot_writecombine(vma->vm_page_prot));
+}
+
+static void clcdfb_of_vram_remove(struct clcd_fb *fb)
+{
+ iounmap(fb->fb.screen_base);
+}
+
+static int clcdfb_of_dma_setup(struct clcd_fb *fb)
+{
+ unsigned long framesize;
+ dma_addr_t dma;
+ int err;
+
+ err = clcdfb_of_init_display(fb);
+ if (err)
+ return err;
+
+ framesize = PAGE_ALIGN(fb->panel->mode.xres * fb->panel->mode.yres *
+ fb->panel->bpp / 8);
+ fb->fb.screen_base = dma_alloc_coherent(&fb->dev->dev, framesize,
+ &dma, GFP_KERNEL);
+ if (!fb->fb.screen_base)
+ return -ENOMEM;
+
+ fb->fb.fix.smem_start = dma;
+ fb->fb.fix.smem_len = framesize;
+
+ return 0;
+}
+
+static int clcdfb_of_dma_mmap(struct clcd_fb *fb, struct vm_area_struct *vma)
+{
+ return dma_mmap_wc(&fb->dev->dev, vma, fb->fb.screen_base,
+ fb->fb.fix.smem_start, fb->fb.fix.smem_len);
+}
+
+static void clcdfb_of_dma_remove(struct clcd_fb *fb)
+{
+ dma_free_coherent(&fb->dev->dev, fb->fb.fix.smem_len,
+ fb->fb.screen_base, fb->fb.fix.smem_start);
+}
+
+static struct clcd_board *clcdfb_of_get_board(struct amba_device *dev)
+{
+ struct clcd_board *board = devm_kzalloc(&dev->dev, sizeof(*board),
+ GFP_KERNEL);
+ struct device_node *node = dev->dev.of_node;
+
+ if (!board)
+ return NULL;
+
+ board->name = of_node_full_name(node);
+ board->caps = CLCD_CAP_ALL;
+ board->check = clcdfb_check;
+ board->decode = clcdfb_decode;
+ if (of_find_property(node, "memory-region", NULL)) {
+ board->setup = clcdfb_of_vram_setup;
+ board->mmap = clcdfb_of_vram_mmap;
+ board->remove = clcdfb_of_vram_remove;
+ } else {
+ board->setup = clcdfb_of_dma_setup;
+ board->mmap = clcdfb_of_dma_mmap;
+ board->remove = clcdfb_of_dma_remove;
+ }
+
+ return board;
+}
+#else
+static struct clcd_board *clcdfb_of_get_board(struct amba_device *dev)
+{
+ return NULL;
+}
+#endif
+
+static int clcdfb_probe(struct amba_device *dev, const struct amba_id *id)
+{
+ struct clcd_board *board = dev_get_platdata(&dev->dev);
+ struct clcd_fb *fb;
+ int ret;
+
+ if (!board)
+ board = clcdfb_of_get_board(dev);
+
+ if (!board)
+ return -EINVAL;
+
+ ret = dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32));
+ if (ret)
+ goto out;
+
+ ret = amba_request_regions(dev, NULL);
+ if (ret) {
+ printk(KERN_ERR "CLCD: unable to reserve regs region\n");
+ goto out;
+ }
+
+ fb = kzalloc(sizeof(*fb), GFP_KERNEL);
+ if (!fb) {
+ ret = -ENOMEM;
+ goto free_region;
+ }
+
+ fb->dev = dev;
+ fb->board = board;
+
+ dev_info(&fb->dev->dev, "PL%03x designer %02x rev%u at 0x%08llx\n",
+ amba_part(dev), amba_manf(dev), amba_rev(dev),
+ (unsigned long long)dev->res.start);
+
+ ret = fb->board->setup(fb);
+ if (ret)
+ goto free_fb;
+
+ ret = clcdfb_register(fb);
+ if (ret == 0) {
+ amba_set_drvdata(dev, fb);
+ goto out;
+ }
+
+ fb->board->remove(fb);
+ free_fb:
+ kfree(fb);
+ free_region:
+ amba_release_regions(dev);
+ out:
+ return ret;
+}
+
+static int clcdfb_remove(struct amba_device *dev)
+{
+ struct clcd_fb *fb = amba_get_drvdata(dev);
+
+ clcdfb_disable(fb);
+ unregister_framebuffer(&fb->fb);
+ if (fb->fb.cmap.len)
+ fb_dealloc_cmap(&fb->fb.cmap);
+ iounmap(fb->regs);
+ clk_unprepare(fb->clk);
+ clk_put(fb->clk);
+
+ fb->board->remove(fb);
+
+ kfree(fb);
+
+ amba_release_regions(dev);
+
+ return 0;
+}
+
+static const struct amba_id clcdfb_id_table[] = {
+ {
+ .id = 0x00041110,
+ .mask = 0x000ffffe,
+ },
+ { 0, 0 },
+};
+
+MODULE_DEVICE_TABLE(amba, clcdfb_id_table);
+
+static struct amba_driver clcd_driver = {
+ .drv = {
+ .name = "clcd-pl11x",
+ },
+ .probe = clcdfb_probe,
+ .remove = clcdfb_remove,
+ .id_table = clcdfb_id_table,
+};
+
+static int __init amba_clcdfb_init(void)
+{
+ if (fb_get_options("ambafb", NULL))
+ return -ENODEV;
+
+ return amba_driver_register(&clcd_driver);
+}
+
+module_init(amba_clcdfb_init);
+
+static void __exit amba_clcdfb_exit(void)
+{
+ amba_driver_unregister(&clcd_driver);
+}
+
+module_exit(amba_clcdfb_exit);
+
+MODULE_DESCRIPTION("ARM PrimeCell PL110 CLCD core driver");
+MODULE_LICENSE("GPL");
if (font->width <= 8) {
j = vc->vc_font.height;
+ if (font->charcount * j > FNTSIZE(fontdata))
+ return -EINVAL;
+
for (i = 0; i < font->charcount; i++) {
memcpy(data, fontdata, j);
memset(data + j, 0, 32 - j);
}
} else if (font->width <= 16) {
j = vc->vc_font.height * 2;
+ if (font->charcount * j > FNTSIZE(fontdata))
+ return -EINVAL;
+
for (i = 0; i < font->charcount; i++) {
memcpy(data, fontdata, j);
memset(data + j, 0, 64 - j);
fontdata += j;
}
} else if (font->width <= 24) {
+ if (font->charcount * (vc->vc_font.height * sizeof(u32)) > FNTSIZE(fontdata))
+ return -EINVAL;
+
for (i = 0; i < font->charcount; i++) {
for (j = 0; j < vc->vc_font.height; j++) {
*data++ = fontdata[0];
}
} else {
j = vc->vc_font.height * 4;
+ if (font->charcount * j > FNTSIZE(fontdata))
+ return -EINVAL;
+
for (i = 0; i < font->charcount; i++) {
memcpy(data, fontdata, j);
memset(data + j, 0, 128 - j);
#define attr_bgcol_ec(bgshift, vc, info) attr_col_ec(bgshift, vc, info, 0)
#define attr_fgcol_ec(fgshift, vc, info) attr_col_ec(fgshift, vc, info, 1)
-/* Font */
-#define REFCOUNT(fd) (((int *)(fd))[-1])
-#define FNTSIZE(fd) (((int *)(fd))[-2])
-#define FNTCHARCNT(fd) (((int *)(fd))[-3])
-#define FNTSUM(fd) (((int *)(fd))[-4])
-#define FONT_EXTRA_WORDS 4
-
/*
* Scroll Method
*/
#include <linux/fb.h>
#include <linux/vt_kern.h>
#include <linux/console.h>
+#include <linux/font.h>
#include <asm/types.h>
#include "fbcon.h"
#include "fbcon_rotate.h"
#include <linux/fb.h>
#include <linux/vt_kern.h>
#include <linux/console.h>
+#include <linux/font.h>
#include <asm/types.h>
#include "fbcon.h"
/* Xen will never allocate port zero for any purpose. */
#define VALID_EVTCHN(chn) ((chn) != 0)
+static struct irq_info *legacy_info_ptrs[NR_IRQS_LEGACY];
+
static struct irq_chip xen_dynamic_chip;
static struct irq_chip xen_percpu_chip;
static struct irq_chip xen_pirq_chip;
/* Get info for IRQ */
struct irq_info *info_for_irq(unsigned irq)
{
- return irq_get_chip_data(irq);
+ if (irq < nr_legacy_irqs())
+ return legacy_info_ptrs[irq];
+ else
+ return irq_get_chip_data(irq);
+}
+
+static void set_info_for_irq(unsigned int irq, struct irq_info *info)
+{
+ if (irq < nr_legacy_irqs())
+ legacy_info_ptrs[irq] = info;
+ else
+ irq_set_chip_data(irq, info);
}
/* Constructors for packed IRQ information. */
info->type = IRQT_UNBOUND;
info->refcnt = -1;
- irq_set_chip_data(irq, info);
+ set_info_for_irq(irq, info);
list_add_tail(&info->list, &xen_irq_list_head);
}
static void xen_free_irq(unsigned irq)
{
- struct irq_info *info = irq_get_chip_data(irq);
+ struct irq_info *info = info_for_irq(irq);
if (WARN_ON(!info))
return;
list_del(&info->list);
- irq_set_chip_data(irq, NULL);
+ set_info_for_irq(irq, NULL);
WARN_ON(info->refcnt > 0);
static void __unbind_from_irq(unsigned int irq)
{
evtchn_port_t evtchn = evtchn_from_irq(irq);
- struct irq_info *info = irq_get_chip_data(irq);
+ struct irq_info *info = info_for_irq(irq);
if (info->refcnt > 0) {
info->refcnt--;
void unbind_from_irqhandler(unsigned int irq, void *dev_id)
{
- struct irq_info *info = irq_get_chip_data(irq);
+ struct irq_info *info = info_for_irq(irq);
if (WARN_ON(!info))
return;
if (irq == -1)
return -ENOENT;
- info = irq_get_chip_data(irq);
+ info = info_for_irq(irq);
if (!info)
return -ENOENT;
if (irq == -1)
goto done;
- info = irq_get_chip_data(irq);
+ info = info_for_irq(irq);
if (!info)
goto done;
z->resource.name = z->name;
r = zorro_find_parent_resource(pdev, z);
error = request_resource(r, &z->resource);
- if (error)
+ if (error && !(z->rom.er_Type & ERTF_MEMLIST))
dev_err(&bus->dev,
"Address space collision on device %s %pR\n",
z->name, &z->resource);
obj-$(CONFIG_FS_ENCRYPTION) += crypto/
obj-$(CONFIG_FS_VERITY) += verity/
obj-$(CONFIG_FILE_LOCKING) += locks.o
-obj-$(CONFIG_COMPAT) += compat.o
obj-$(CONFIG_BINFMT_AOUT) += binfmt_aout.o
obj-$(CONFIG_BINFMT_EM86) += binfmt_em86.o
obj-$(CONFIG_BINFMT_MISC) += binfmt_misc.o
static void afs_setattr_success(struct afs_operation *op)
{
- struct inode *inode = &op->file[0].vnode->vfs_inode;
+ struct afs_vnode_param *vp = &op->file[0];
+ struct inode *inode = &vp->vnode->vfs_inode;
+ loff_t old_i_size = i_size_read(inode);
+
+ op->setattr.old_i_size = old_i_size;
+ afs_vnode_commit_status(op, vp);
+ /* inode->i_size has now been changed. */
+
+ if (op->setattr.attr->ia_valid & ATTR_SIZE) {
+ loff_t size = op->setattr.attr->ia_size;
+ if (size > old_i_size)
+ pagecache_isize_extended(inode, old_i_size, size);
+ }
+}
+
+static void afs_setattr_edit_file(struct afs_operation *op)
+{
+ struct afs_vnode_param *vp = &op->file[0];
+ struct inode *inode = &vp->vnode->vfs_inode;
- afs_vnode_commit_status(op, &op->file[0]);
if (op->setattr.attr->ia_valid & ATTR_SIZE) {
- loff_t i_size = inode->i_size, size = op->setattr.attr->ia_size;
- if (size > i_size)
- pagecache_isize_extended(inode, i_size, size);
- truncate_pagecache(inode, size);
+ loff_t size = op->setattr.attr->ia_size;
+ loff_t i_size = op->setattr.old_i_size;
+
+ if (size < i_size)
+ truncate_pagecache(inode, size);
}
}
.issue_afs_rpc = afs_fs_setattr,
.issue_yfs_rpc = yfs_fs_setattr,
.success = afs_setattr_success,
+ .edit_dir = afs_setattr_edit_file,
};
/*
if (S_ISREG(vnode->vfs_inode.i_mode))
filemap_write_and_wait(vnode->vfs_inode.i_mapping);
+ /* Prevent any new writebacks from starting whilst we do this. */
+ down_write(&vnode->validate_lock);
+
op = afs_alloc_operation(((attr->ia_valid & ATTR_FILE) ?
afs_file_key(attr->ia_file) : NULL),
vnode->volume);
- if (IS_ERR(op))
- return PTR_ERR(op);
+ if (IS_ERR(op)) {
+ ret = PTR_ERR(op);
+ goto out_unlock;
+ }
afs_op_set_vnode(op, 0, vnode);
op->setattr.attr = attr;
op->file[0].update_ctime = 1;
op->ops = &afs_setattr_operation;
- return afs_do_sync_operation(op);
+ ret = afs_do_sync_operation(op);
+
+out_unlock:
+ up_write(&vnode->validate_lock);
+ _leave(" = %d", ret);
+ return ret;
}
} store;
struct {
struct iattr *attr;
+ loff_t old_i_size;
} setattr;
struct afs_acl *acl;
struct yfs_acl *yacl;
int afs_writepages(struct address_space *mapping,
struct writeback_control *wbc)
{
+ struct afs_vnode *vnode = AFS_FS_I(mapping->host);
pgoff_t start, end, next;
int ret;
_enter("");
+ /* We have to be careful as we can end up racing with setattr()
+ * truncating the pagecache since the caller doesn't take a lock here
+ * to prevent it.
+ */
+ if (wbc->sync_mode == WB_SYNC_ALL)
+ down_read(&vnode->validate_lock);
+ else if (!down_read_trylock(&vnode->validate_lock))
+ return 0;
+
if (wbc->range_cyclic) {
start = mapping->writeback_index;
end = -1;
ret = afs_writepages_region(mapping, wbc, start, end, &next);
}
+ up_read(&vnode->validate_lock);
_leave(" = %d", ret);
return ret;
}
*iovec = NULL;
return ret;
}
-#ifdef CONFIG_COMPAT
- if (compat)
- return compat_import_iovec(rw, buf, len, UIO_FASTIOV, iovec,
- iter);
-#endif
- return import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter);
+
+ return __import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter, compat);
}
static inline void aio_rw_done(struct kiocb *req, ssize_t ret)
mutex_lock(&sbi->pipe_mutex);
while (bytes) {
- wr = kernel_write(file, data, bytes, &file->f_pos);
+ wr = __kernel_write(file, data, bytes, NULL);
if (wr <= 0)
break;
data += wr;
return 0;
}
-struct block_device *bdget(dev_t dev)
+static struct block_device *bdget(dev_t dev)
{
struct block_device *bdev;
struct inode *inode;
return bdev;
}
-EXPORT_SYMBOL(bdget);
-
/**
* bdgrab -- Grab a reference to an already referenced block device
* @bdev: Block device to grab a reference to.
}
EXPORT_SYMBOL(bdgrab);
+struct block_device *bdget_part(struct hd_struct *part)
+{
+ return bdget(part_devt(part));
+}
+
long nr_blockdev_pages(void)
{
struct inode *inode;
select LZO_DECOMPRESS
select ZSTD_COMPRESS
select ZSTD_DECOMPRESS
+ select FS_IOMAP
select RAID6_PQ
select XOR_BLOCKS
select SRCU
while (!list_empty(&pending_edge)) {
struct btrfs_backref_node *upper;
struct btrfs_backref_node *lower;
- struct rb_node *rb_node;
edge = list_first_entry(&pending_edge,
struct btrfs_backref_edge, list[UPPER]);
{
struct btrfs_space_info *space_info = cache->space_info;
int index = btrfs_bg_flags_to_raid_index(cache->flags);
- bool first = false;
down_write(&space_info->groups_sem);
- if (list_empty(&space_info->block_groups[index]))
- first = true;
list_add_tail(&cache->list, &space_info->block_groups[index]);
up_write(&space_info->groups_sem);
-
- if (first)
- btrfs_sysfs_add_block_group_type(cache);
}
static struct btrfs_block_group *btrfs_create_block_group_cache(
return ret;
}
-static int read_block_group_item(struct btrfs_block_group *cache,
+static void read_block_group_item(struct btrfs_block_group *cache,
struct btrfs_path *path,
const struct btrfs_key *key)
{
sizeof(bgi));
cache->used = btrfs_stack_block_group_used(&bgi);
cache->flags = btrfs_stack_block_group_flags(&bgi);
-
- return 0;
}
static int read_one_block_group(struct btrfs_fs_info *info,
if (!cache)
return -ENOMEM;
- ret = read_block_group_item(cache, path, key);
- if (ret < 0)
- goto error;
+ read_block_group_item(cache, path, key);
set_free_space_tree_thresholds(cache);
btrfs_release_path(path);
}
- rcu_read_lock();
- list_for_each_entry_rcu(space_info, &info->space_info, list) {
+ list_for_each_entry(space_info, &info->space_info, list) {
+ int i;
+
+ for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
+ if (list_empty(&space_info->block_groups[i]))
+ continue;
+ cache = list_first_entry(&space_info->block_groups[i],
+ struct btrfs_block_group,
+ list);
+ btrfs_sysfs_add_block_group_type(cache);
+ }
+
if (!(btrfs_get_alloc_profile(info, space_info->flags) &
(BTRFS_BLOCK_GROUP_RAID10 |
BTRFS_BLOCK_GROUP_RAID1_MASK |
list)
inc_block_group_ro(cache, 1);
}
- rcu_read_unlock();
btrfs_init_global_block_rsv(info);
ret = check_chunk_block_group_mappings(info);
return;
while (!list_empty(&trans->new_bgs)) {
+ int index;
+
block_group = list_first_entry(&trans->new_bgs,
struct btrfs_block_group,
bg_list);
if (ret)
goto next;
+ index = btrfs_bg_flags_to_raid_index(block_group->flags);
+
ret = insert_block_group_item(trans, block_group);
if (ret)
btrfs_abort_transaction(trans, ret);
if (ret)
btrfs_abort_transaction(trans, ret);
add_block_group_free_space(trans, block_group);
+
+ /*
+ * If we restriped during balance, we may have added a new raid
+ * type, so now add the sysfs entries when it is safe to do so.
+ * We don't have to worry about locking here as it's handled in
+ * btrfs_sysfs_add_block_group_type.
+ */
+ if (block_group->space_info->block_group_kobjs[index] == NULL)
+ btrfs_sysfs_add_block_group_type(block_group);
+
/* Already aborted the transaction if it failed. */
next:
btrfs_delayed_refs_rsv_release(fs_info, 1);
* finished yet (no block group item in the extent tree
* yet, etc). If this is the case, wait for all free
* space endio workers to finish and retry. This is a
- * a very rare case so no need for a more efficient and
+ * very rare case so no need for a more efficient and
* complex approach.
*/
if (ret == -ENOENT) {
space_info, -ram_bytes);
if (delalloc)
cache->delalloc_bytes += num_bytes;
+
+ /*
+ * Compression can use less space than we reserved, so wake
+ * tickets if that happens
+ */
+ if (num_bytes < ram_bytes)
+ btrfs_try_granting_tickets(cache->fs_info, space_info);
}
spin_unlock(&cache->lock);
spin_unlock(&space_info->lock);
if (delalloc)
cache->delalloc_bytes -= num_bytes;
spin_unlock(&cache->lock);
+
+ btrfs_try_granting_tickets(cache->fs_info, space_info);
spin_unlock(&space_info->lock);
}
struct list_head *head = &info->space_info;
struct btrfs_space_info *found;
- rcu_read_lock();
- list_for_each_entry_rcu(found, head, list) {
+ list_for_each_entry(found, head, list) {
if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
found->force_alloc = CHUNK_ALLOC_FORCE;
}
- rcu_read_unlock();
}
static int should_alloc_chunk(struct btrfs_fs_info *fs_info,
}
spin_unlock(&info->block_group_cache_lock);
- /*
- * Now that all the block groups are freed, go through and free all the
- * space_info structs. This is only called during the final stages of
- * unmount, and so we know nobody is using them. We call
- * synchronize_rcu() once before we start, just to be on the safe side.
- */
- synchronize_rcu();
-
btrfs_release_global_block_rsv(info);
while (!list_empty(&info->space_info)) {
* new data the application may have written before commit.
*/
enum {
- BTRFS_INODE_ORDERED_DATA_CLOSE,
+ BTRFS_INODE_FLUSH_ON_CLOSE,
BTRFS_INODE_DUMMY,
BTRFS_INODE_IN_DEFRAG,
BTRFS_INODE_HAS_ASYNC_EXTENT,
+ /*
+ * Always set under the VFS' inode lock, otherwise it can cause races
+ * during fsync (we start as a fast fsync and then end up in a full
+ * fsync racing with ordered extent completion).
+ */
BTRFS_INODE_NEEDS_FULL_SYNC,
BTRFS_INODE_COPY_EVERYTHING,
BTRFS_INODE_IN_DELALLOC_LIST,
- BTRFS_INODE_READDIO_NEED_LOCK,
BTRFS_INODE_HAS_PROPS,
BTRFS_INODE_SNAPSHOT_FLUSH,
};
struct inode vfs_inode;
};
+static inline u32 btrfs_inode_sectorsize(const struct btrfs_inode *inode)
+{
+ return inode->root->fs_info->sectorsize;
+}
+
static inline struct btrfs_inode *BTRFS_I(const struct inode *inode)
{
return container_of(inode, struct btrfs_inode, vfs_inode);
u8 csums[];
};
-/*
- * Disable DIO read nolock optimization, so new dio readers will be forced
- * to grab i_mutex. It is used to avoid the endless truncate due to
- * nonlocked dio read.
- */
-static inline void btrfs_inode_block_unlocked_dio(struct btrfs_inode *inode)
-{
- set_bit(BTRFS_INODE_READDIO_NEED_LOCK, &inode->runtime_flags);
- smp_mb();
-}
-
-static inline void btrfs_inode_resume_unlocked_dio(struct btrfs_inode *inode)
-{
- smp_mb__before_atomic();
- clear_bit(BTRFS_INODE_READDIO_NEED_LOCK, &inode->runtime_flags);
-}
-
/* Array of bytes with variable length, hexadecimal format 0x1234 */
#define CSUM_FMT "0x%*phN"
#define CSUM_FMT_VALUE(size, bytes) size, bytes
#include "extent_io.h"
#include "extent_map.h"
-int zlib_compress_pages(struct list_head *ws, struct address_space *mapping,
- u64 start, struct page **pages, unsigned long *out_pages,
- unsigned long *total_in, unsigned long *total_out);
-int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb);
-int zlib_decompress(struct list_head *ws, unsigned char *data_in,
- struct page *dest_page, unsigned long start_byte, size_t srclen,
- size_t destlen);
-struct list_head *zlib_alloc_workspace(unsigned int level);
-void zlib_free_workspace(struct list_head *ws);
-struct list_head *zlib_get_workspace(unsigned int level);
-
-int lzo_compress_pages(struct list_head *ws, struct address_space *mapping,
- u64 start, struct page **pages, unsigned long *out_pages,
- unsigned long *total_in, unsigned long *total_out);
-int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb);
-int lzo_decompress(struct list_head *ws, unsigned char *data_in,
- struct page *dest_page, unsigned long start_byte, size_t srclen,
- size_t destlen);
-struct list_head *lzo_alloc_workspace(unsigned int level);
-void lzo_free_workspace(struct list_head *ws);
-
-int zstd_compress_pages(struct list_head *ws, struct address_space *mapping,
- u64 start, struct page **pages, unsigned long *out_pages,
- unsigned long *total_in, unsigned long *total_out);
-int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb);
-int zstd_decompress(struct list_head *ws, unsigned char *data_in,
- struct page *dest_page, unsigned long start_byte, size_t srclen,
- size_t destlen);
-void zstd_init_workspace_manager(void);
-void zstd_cleanup_workspace_manager(void);
-struct list_head *zstd_alloc_workspace(unsigned int level);
-void zstd_free_workspace(struct list_head *ws);
-struct list_head *zstd_get_workspace(unsigned int level);
-void zstd_put_workspace(struct list_head *ws);
-
static const char* const btrfs_compress_types[] = { "", "zlib", "lzo", "zstd" };
const char* btrfs_compress_type2str(enum btrfs_compression_type type)
int btrfs_compress_heuristic(struct inode *inode, u64 start, u64 end);
+int zlib_compress_pages(struct list_head *ws, struct address_space *mapping,
+ u64 start, struct page **pages, unsigned long *out_pages,
+ unsigned long *total_in, unsigned long *total_out);
+int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb);
+int zlib_decompress(struct list_head *ws, unsigned char *data_in,
+ struct page *dest_page, unsigned long start_byte, size_t srclen,
+ size_t destlen);
+struct list_head *zlib_alloc_workspace(unsigned int level);
+void zlib_free_workspace(struct list_head *ws);
+struct list_head *zlib_get_workspace(unsigned int level);
+
+int lzo_compress_pages(struct list_head *ws, struct address_space *mapping,
+ u64 start, struct page **pages, unsigned long *out_pages,
+ unsigned long *total_in, unsigned long *total_out);
+int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb);
+int lzo_decompress(struct list_head *ws, unsigned char *data_in,
+ struct page *dest_page, unsigned long start_byte, size_t srclen,
+ size_t destlen);
+struct list_head *lzo_alloc_workspace(unsigned int level);
+void lzo_free_workspace(struct list_head *ws);
+
+int zstd_compress_pages(struct list_head *ws, struct address_space *mapping,
+ u64 start, struct page **pages, unsigned long *out_pages,
+ unsigned long *total_in, unsigned long *total_out);
+int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb);
+int zstd_decompress(struct list_head *ws, unsigned char *data_in,
+ struct page *dest_page, unsigned long start_byte, size_t srclen,
+ size_t destlen);
+void zstd_init_workspace_manager(void);
+void zstd_cleanup_workspace_manager(void);
+struct list_head *zstd_alloc_workspace(unsigned int level);
+void zstd_free_workspace(struct list_head *ws);
+struct list_head *zstd_get_workspace(unsigned int level);
+void zstd_put_workspace(struct list_head *ws);
+
#endif
btrfs_node_key(buf, &disk_key, 0);
cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid,
- &disk_key, level, buf->start, 0);
+ &disk_key, level, buf->start, 0,
+ BTRFS_NESTING_NEW_ROOT);
if (IS_ERR(cow))
return PTR_ERR(cow);
const struct btrfs_disk_key *disk_key,
int level,
u64 hint,
- u64 empty_size)
+ u64 empty_size,
+ enum btrfs_lock_nesting nest)
{
struct btrfs_fs_info *fs_info = root->fs_info;
struct extent_buffer *ret;
ret = btrfs_alloc_tree_block(trans, root, parent_start,
root->root_key.objectid, disk_key, level,
- hint, empty_size);
+ hint, empty_size, nest);
trans->can_flush_pending_bgs = true;
return ret;
struct extent_buffer *buf,
struct extent_buffer *parent, int parent_slot,
struct extent_buffer **cow_ret,
- u64 search_start, u64 empty_size)
+ u64 search_start, u64 empty_size,
+ enum btrfs_lock_nesting nest)
{
struct btrfs_fs_info *fs_info = root->fs_info;
struct btrfs_disk_key disk_key;
parent_start = parent->start;
cow = alloc_tree_block_no_bg_flush(trans, root, parent_start, &disk_key,
- level, search_start, empty_size);
+ level, search_start, empty_size, nest);
if (IS_ERR(cow))
return PTR_ERR(cow);
ret = update_ref_for_cow(trans, root, buf, cow, &last_ref);
if (ret) {
+ btrfs_tree_unlock(cow);
+ free_extent_buffer(cow);
btrfs_abort_transaction(trans, ret);
return ret;
}
if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) {
ret = btrfs_reloc_cow_block(trans, root, buf, cow);
if (ret) {
+ btrfs_tree_unlock(cow);
+ free_extent_buffer(cow);
btrfs_abort_transaction(trans, ret);
return ret;
}
if (last_ref) {
ret = tree_mod_log_free_eb(buf);
if (ret) {
+ btrfs_tree_unlock(cow);
+ free_extent_buffer(cow);
btrfs_abort_transaction(trans, ret);
return ret;
}
noinline int btrfs_cow_block(struct btrfs_trans_handle *trans,
struct btrfs_root *root, struct extent_buffer *buf,
struct extent_buffer *parent, int parent_slot,
- struct extent_buffer **cow_ret)
+ struct extent_buffer **cow_ret,
+ enum btrfs_lock_nesting nest)
{
struct btrfs_fs_info *fs_info = root->fs_info;
u64 search_start;
*/
btrfs_qgroup_trace_subtree_after_cow(trans, root, buf);
ret = __btrfs_cow_block(trans, root, buf, parent,
- parent_slot, cow_ret, search_start, 0);
+ parent_slot, cow_ret, search_start, 0, nest);
trace_btrfs_cow_block(root, buf, *cow_ret);
err = __btrfs_cow_block(trans, root, cur, parent, i,
&cur, search_start,
min(16 * blocksize,
- (end_slot - i) * blocksize));
+ (end_slot - i) * blocksize),
+ BTRFS_NESTING_COW);
if (err) {
btrfs_tree_unlock(cur);
free_extent_buffer(cur);
btrfs_tree_lock(child);
btrfs_set_lock_blocking_write(child);
- ret = btrfs_cow_block(trans, root, child, mid, 0, &child);
+ ret = btrfs_cow_block(trans, root, child, mid, 0, &child,
+ BTRFS_NESTING_COW);
if (ret) {
btrfs_tree_unlock(child);
free_extent_buffer(child);
left = NULL;
if (left) {
- btrfs_tree_lock(left);
+ __btrfs_tree_lock(left, BTRFS_NESTING_LEFT);
btrfs_set_lock_blocking_write(left);
wret = btrfs_cow_block(trans, root, left,
- parent, pslot - 1, &left);
+ parent, pslot - 1, &left,
+ BTRFS_NESTING_LEFT_COW);
if (wret) {
ret = wret;
goto enospc;
right = NULL;
if (right) {
- btrfs_tree_lock(right);
+ __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT);
btrfs_set_lock_blocking_write(right);
wret = btrfs_cow_block(trans, root, right,
- parent, pslot + 1, &right);
+ parent, pslot + 1, &right,
+ BTRFS_NESTING_RIGHT_COW);
if (wret) {
ret = wret;
goto enospc;
if (left) {
u32 left_nr;
- btrfs_tree_lock(left);
+ __btrfs_tree_lock(left, BTRFS_NESTING_LEFT);
btrfs_set_lock_blocking_write(left);
left_nr = btrfs_header_nritems(left);
wret = 1;
} else {
ret = btrfs_cow_block(trans, root, left, parent,
- pslot - 1, &left);
+ pslot - 1, &left,
+ BTRFS_NESTING_LEFT_COW);
if (ret)
wret = 1;
else {
if (right) {
u32 right_nr;
- btrfs_tree_lock(right);
+ __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT);
btrfs_set_lock_blocking_write(right);
right_nr = btrfs_header_nritems(right);
} else {
ret = btrfs_cow_block(trans, root, right,
parent, pslot + 1,
- &right);
+ &right, BTRFS_NESTING_RIGHT_COW);
if (ret)
wret = 1;
else {
* We don't know the level of the root node until we actually
* have it read locked
*/
- b = btrfs_read_lock_root_node(root);
+ b = __btrfs_read_lock_root_node(root, p->recurse);
level = btrfs_header_level(b);
if (level > write_lock_level)
goto out;
btrfs_set_path_blocking(p);
if (last_level)
err = btrfs_cow_block(trans, root, b, NULL, 0,
- &b);
+ &b,
+ BTRFS_NESTING_COW);
else
err = btrfs_cow_block(trans, root, b,
p->nodes[level + 1],
- p->slots[level + 1], &b);
+ p->slots[level + 1], &b,
+ BTRFS_NESTING_COW);
if (err) {
ret = err;
goto done;
} else {
if (!btrfs_tree_read_lock_atomic(b)) {
btrfs_set_path_blocking(p);
- btrfs_tree_read_lock(b);
+ __btrfs_tree_read_lock(b, BTRFS_NESTING_NORMAL,
+ p->recurse);
}
p->locks[level] = BTRFS_READ_LOCK;
}
fixup_low_keys(path, &disk_key, 1);
}
+/*
+ * Check key order of two sibling extent buffers.
+ *
+ * Return true if something is wrong.
+ * Return false if everything is fine.
+ *
+ * Tree-checker only works inside one tree block, thus the following
+ * corruption can not be detected by tree-checker:
+ *
+ * Leaf @left | Leaf @right
+ * --------------------------------------------------------------
+ * | 1 | 2 | 3 | 4 | 5 | f6 | | 7 | 8 |
+ *
+ * Key f6 in leaf @left itself is valid, but not valid when the next
+ * key in leaf @right is 7.
+ * This can only be checked at tree block merge time.
+ * And since tree checker has ensured all key order in each tree block
+ * is correct, we only need to bother the last key of @left and the first
+ * key of @right.
+ */
+static bool check_sibling_keys(struct extent_buffer *left,
+ struct extent_buffer *right)
+{
+ struct btrfs_key left_last;
+ struct btrfs_key right_first;
+ int level = btrfs_header_level(left);
+ int nr_left = btrfs_header_nritems(left);
+ int nr_right = btrfs_header_nritems(right);
+
+ /* No key to check in one of the tree blocks */
+ if (!nr_left || !nr_right)
+ return false;
+
+ if (level) {
+ btrfs_node_key_to_cpu(left, &left_last, nr_left - 1);
+ btrfs_node_key_to_cpu(right, &right_first, 0);
+ } else {
+ btrfs_item_key_to_cpu(left, &left_last, nr_left - 1);
+ btrfs_item_key_to_cpu(right, &right_first, 0);
+ }
+
+ if (btrfs_comp_cpu_keys(&left_last, &right_first) >= 0) {
+ btrfs_crit(left->fs_info,
+"bad key order, sibling blocks, left last (%llu %u %llu) right first (%llu %u %llu)",
+ left_last.objectid, left_last.type,
+ left_last.offset, right_first.objectid,
+ right_first.type, right_first.offset);
+ return true;
+ }
+ return false;
+}
+
/*
* try to push data from one node into the next node left in the
* tree.
} else
push_items = min(src_nritems - 8, push_items);
+ /* dst is the left eb, src is the middle eb */
+ if (check_sibling_keys(dst, src)) {
+ ret = -EUCLEAN;
+ btrfs_abort_transaction(trans, ret);
+ return ret;
+ }
ret = tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items);
if (ret) {
btrfs_abort_transaction(trans, ret);
if (max_push < push_items)
push_items = max_push;
+ /* dst is the right eb, src is the middle eb */
+ if (check_sibling_keys(src, dst)) {
+ ret = -EUCLEAN;
+ btrfs_abort_transaction(trans, ret);
+ return ret;
+ }
ret = tree_mod_log_insert_move(dst, push_items, 0, dst_nritems);
BUG_ON(ret < 0);
memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items),
btrfs_node_key(lower, &lower_key, 0);
c = alloc_tree_block_no_bg_flush(trans, root, 0, &lower_key, level,
- root->node->start, 0);
+ root->node->start, 0,
+ BTRFS_NESTING_NEW_ROOT);
if (IS_ERR(c))
return PTR_ERR(c);
btrfs_node_key(c, &disk_key, mid);
split = alloc_tree_block_no_bg_flush(trans, root, 0, &disk_key, level,
- c->start, 0);
+ c->start, 0, BTRFS_NESTING_SPLIT);
if (IS_ERR(split))
return PTR_ERR(split);
if (IS_ERR(right))
return 1;
- btrfs_tree_lock(right);
+ __btrfs_tree_lock(right, BTRFS_NESTING_RIGHT);
btrfs_set_lock_blocking_write(right);
free_space = btrfs_leaf_free_space(right);
/* cow and double check */
ret = btrfs_cow_block(trans, root, right, upper,
- slot + 1, &right);
+ slot + 1, &right, BTRFS_NESTING_RIGHT_COW);
if (ret)
goto out_unlock;
if (left_nritems == 0)
goto out_unlock;
+ if (check_sibling_keys(left, right)) {
+ ret = -EUCLEAN;
+ btrfs_tree_unlock(right);
+ free_extent_buffer(right);
+ return ret;
+ }
if (path->slots[0] == left_nritems && !empty) {
/* Key greater than all keys in the leaf, right neighbor has
* enough room for it and we're not emptying our leaf to delete
if (IS_ERR(left))
return 1;
- btrfs_tree_lock(left);
+ __btrfs_tree_lock(left, BTRFS_NESTING_LEFT);
btrfs_set_lock_blocking_write(left);
free_space = btrfs_leaf_free_space(left);
/* cow and double check */
ret = btrfs_cow_block(trans, root, left,
- path->nodes[1], slot - 1, &left);
+ path->nodes[1], slot - 1, &left,
+ BTRFS_NESTING_LEFT_COW);
if (ret) {
/* we hit -ENOSPC, but it isn't fatal here */
if (ret == -ENOSPC)
goto out;
}
+ if (check_sibling_keys(left, right)) {
+ ret = -EUCLEAN;
+ goto out;
+ }
return __push_leaf_left(path, min_data_size,
empty, left, free_space, right_nritems,
max_slot);
else
btrfs_item_key(l, &disk_key, mid);
+ /*
+ * We have to about BTRFS_NESTING_NEW_ROOT here if we've done a double
+ * split, because we're only allowed to have MAX_LOCKDEP_SUBCLASSES
+ * subclasses, which is 8 at the time of this patch, and we've maxed it
+ * out. In the future we could add a
+ * BTRFS_NESTING_SPLIT_THE_SPLITTENING if we need to, but for now just
+ * use BTRFS_NESTING_NEW_ROOT.
+ */
right = alloc_tree_block_no_bg_flush(trans, root, 0, &disk_key, 0,
- l->start, 0);
+ l->start, 0, num_doubles ?
+ BTRFS_NESTING_NEW_ROOT :
+ BTRFS_NESTING_SPLIT);
if (IS_ERR(right))
return PTR_ERR(right);
return ret;
path->slots[0]++;
- setup_items_for_insert(root, path, new_key, &item_size,
- item_size, item_size +
- sizeof(struct btrfs_item), 1);
+ setup_items_for_insert(root, path, new_key, &item_size, 1);
leaf = path->nodes[0];
memcpy_extent_buffer(leaf,
btrfs_item_ptr_offset(leaf, path->slots[0]),
}
}
-/*
- * this is a helper for btrfs_insert_empty_items, the main goal here is
- * to save stack depth by doing the bulk of the work in a function
- * that doesn't call btrfs_search_slot
+/**
+ * setup_items_for_insert - Helper called before inserting one or more items
+ * to a leaf. Main purpose is to save stack depth by doing the bulk of the work
+ * in a function that doesn't call btrfs_search_slot
+ *
+ * @root: root we are inserting items to
+ * @path: points to the leaf/slot where we are going to insert new items
+ * @cpu_key: array of keys for items to be inserted
+ * @data_size: size of the body of each item we are going to insert
+ * @nr: size of @cpu_key/@data_size arrays
*/
void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path,
const struct btrfs_key *cpu_key, u32 *data_size,
- u32 total_data, u32 total_size, int nr)
+ int nr)
{
struct btrfs_fs_info *fs_info = root->fs_info;
struct btrfs_item *item;
struct extent_buffer *leaf;
int slot;
struct btrfs_map_token token;
+ u32 total_size;
+ u32 total_data = 0;
+
+ for (i = 0; i < nr; i++)
+ total_data += data_size[i];
+ total_size = total_data + (nr * sizeof(struct btrfs_item));
if (path->slots[0] == 0) {
btrfs_cpu_key_to_disk(&disk_key, cpu_key);
if (old_data < data_end) {
btrfs_print_leaf(leaf);
- btrfs_crit(fs_info, "slot %d old_data %d data_end %d",
+ btrfs_crit(fs_info,
+ "item at slot %d with data offset %u beyond data end of leaf %u",
slot, old_data, data_end);
BUG();
}
btrfs_cpu_key_to_disk(&disk_key, cpu_key + i);
btrfs_set_item_key(leaf, &disk_key, slot + i);
item = btrfs_item_nr(slot + i);
- btrfs_set_token_item_offset(&token, item, data_end - data_size[i]);
data_end -= data_size[i];
+ btrfs_set_token_item_offset(&token, item, data_end);
btrfs_set_token_item_size(&token, item, data_size[i]);
}
slot = path->slots[0];
BUG_ON(slot < 0);
- setup_items_for_insert(root, path, cpu_key, data_size,
- total_data, total_size, nr);
+ setup_items_for_insert(root, path, cpu_key, data_size, nr);
return 0;
}
slot--;
/*
* check this node pointer against the min_trans parameters.
- * If it is too old, old, skip to the next one.
+ * If it is too old, skip to the next one.
*/
while (slot < nritems) {
u64 gen;
}
if (!ret) {
btrfs_set_path_blocking(path);
- btrfs_tree_read_lock(next);
+ __btrfs_tree_read_lock(next,
+ BTRFS_NESTING_RIGHT,
+ path->recurse);
}
next_rw_lock = BTRFS_READ_LOCK;
}
ret = btrfs_try_tree_read_lock(next);
if (!ret) {
btrfs_set_path_blocking(path);
- btrfs_tree_read_lock(next);
+ __btrfs_tree_read_lock(next,
+ BTRFS_NESTING_RIGHT,
+ path->recurse);
}
next_rw_lock = BTRFS_READ_LOCK;
}
unsigned int search_commit_root:1;
unsigned int need_commit_sem:1;
unsigned int skip_release_on_error:1;
+ unsigned int recurse:1;
};
#define BTRFS_MAX_EXTENT_ITEM_SIZE(r) ((BTRFS_LEAF_DATA_SIZE(r->fs_info) >> 4) - \
sizeof(struct btrfs_item))
ORPHAN_CLEANUP_DONE = 2,
};
-void btrfs_init_async_reclaim_work(struct work_struct *work);
+void btrfs_init_async_reclaim_work(struct btrfs_fs_info *fs_info);
/* fs_info */
struct reloc_control;
BTRFS_FS_QUOTA_OVERRIDE,
/* Used to record internally whether fs has been frozen */
BTRFS_FS_FROZEN,
- /*
- * Indicate that a whole-filesystem exclusive operation is running
- * (device replace, resize, device add/delete, balance)
- */
- BTRFS_FS_EXCL_OP,
/*
* Indicate that balance has been set up from the ioctl and is in the
* main phase. The fs_info::balance_ctl is initialized.
BTRFS_FS_DISCARD_RUNNING,
};
+/*
+ * Exclusive operations (device replace, resize, device add/remove, balance)
+ */
+enum btrfs_exclusive_operation {
+ BTRFS_EXCLOP_NONE,
+ BTRFS_EXCLOP_BALANCE,
+ BTRFS_EXCLOP_DEV_ADD,
+ BTRFS_EXCLOP_DEV_REMOVE,
+ BTRFS_EXCLOP_DEV_REPLACE,
+ BTRFS_EXCLOP_RESIZE,
+ BTRFS_EXCLOP_SWAP_ACTIVATE,
+};
+
struct btrfs_fs_info {
u8 chunk_tree_uuid[BTRFS_UUID_SIZE];
unsigned long flags;
/* Used to reclaim the metadata space in the background. */
struct work_struct async_reclaim_work;
+ struct work_struct async_data_reclaim_work;
spinlock_t unused_bgs_lock;
struct list_head unused_bgs;
*/
int send_in_progress;
+ /* Type of exclusive operation running */
+ unsigned long exclusive_operation;
+
#ifdef CONFIG_BTRFS_FS_REF_VERIFY
spinlock_t ref_verify_lock;
struct rb_root block_tree;
#endif
};
-struct btrfs_clone_extent_info {
+/*
+ * Structure that conveys information about an extent that is going to replace
+ * all the extents in a file range.
+ */
+struct btrfs_replace_extent_info {
u64 disk_offset;
u64 disk_len;
u64 data_offset;
u64 data_len;
u64 file_offset;
+ /* Pointer to a file extent item of type regular or prealloc. */
char *extent_buf;
- u32 item_size;
+ /*
+ * Set to true when attempting to replace a file range with a new extent
+ * described by this structure, set to false when attempting to clone an
+ * existing extent into a file range.
+ */
+ bool is_new_extent;
+ /* Meaningful only if is_new_extent is true. */
+ int qgroup_reserved;
+ /*
+ * Meaningful only if is_new_extent is true.
+ * Used to track how many extent items we have already inserted in a
+ * subvolume tree that refer to the extent described by this structure,
+ * so that we know when to create a new delayed ref or update an existing
+ * one.
+ */
+ int insertions;
};
struct btrfs_file_private {
void *filldir_buf;
};
-static inline u32 btrfs_inode_sectorsize(const struct inode *inode)
-{
- return btrfs_sb(inode->i_sb)->sectorsize;
-}
static inline u32 BTRFS_LEAF_DATA_SIZE(const struct btrfs_fs_info *info)
{
#define cpu_to_le8(v) (v)
#define __le8 u8
+static inline u8 get_unaligned_le8(const void *p)
+{
+ return *(u8 *)p;
+}
+
+static inline void put_unaligned_le8(u8 val, void *p)
+{
+ *(u8 *)p = val;
+}
+
#define read_eb_member(eb, ptr, type, member, result) (\
read_extent_buffer(eb, (char *)(result), \
((unsigned long)(ptr)) + \
static inline u##bits btrfs_##name(const struct extent_buffer *eb) \
{ \
const type *p = page_address(eb->pages[0]); \
- u##bits res = le##bits##_to_cpu(p->member); \
- return res; \
+ return get_unaligned_le##bits(&p->member); \
} \
static inline void btrfs_set_##name(const struct extent_buffer *eb, \
u##bits val) \
{ \
type *p = page_address(eb->pages[0]); \
- p->member = cpu_to_le##bits(val); \
+ put_unaligned_le##bits(val, &p->member); \
}
#define BTRFS_SETGET_STACK_FUNCS(name, type, member, bits) \
static inline u##bits btrfs_##name(const type *s) \
{ \
- return le##bits##_to_cpu(s->member); \
+ return get_unaligned_le##bits(&s->member); \
} \
static inline void btrfs_set_##name(type *s, u##bits val) \
{ \
- s->member = cpu_to_le##bits(val); \
+ put_unaligned_le##bits(val, &s->member); \
}
-
static inline u64 btrfs_device_total_bytes(const struct extent_buffer *eb,
struct btrfs_dev_item *s)
{
u64 parent, u64 root_objectid,
const struct btrfs_disk_key *key,
int level, u64 hint,
- u64 empty_size);
+ u64 empty_size,
+ enum btrfs_lock_nesting nest);
void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
struct extent_buffer *buf,
*
* Can be interruped by fatal signal.
*/
+ BTRFS_RESERVE_FLUSH_DATA,
+ BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE,
BTRFS_RESERVE_FLUSH_ALL,
/*
int btrfs_subvolume_reserve_metadata(struct btrfs_root *root,
struct btrfs_block_rsv *rsv,
int nitems, bool use_global_rsv);
-void btrfs_subvolume_release_metadata(struct btrfs_fs_info *fs_info,
+void btrfs_subvolume_release_metadata(struct btrfs_root *root,
struct btrfs_block_rsv *rsv);
void btrfs_delalloc_release_extents(struct btrfs_inode *inode, u64 num_bytes);
struct btrfs_path *path,
const struct btrfs_key *new_key);
struct extent_buffer *btrfs_root_node(struct btrfs_root *root);
-struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root);
-struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root);
int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path,
struct btrfs_key *key, int lowest_level,
u64 min_trans);
int btrfs_cow_block(struct btrfs_trans_handle *trans,
struct btrfs_root *root, struct extent_buffer *buf,
struct extent_buffer *parent, int parent_slot,
- struct extent_buffer **cow_ret);
+ struct extent_buffer **cow_ret,
+ enum btrfs_lock_nesting nest);
int btrfs_copy_root(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
struct extent_buffer *buf,
void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path,
const struct btrfs_key *cpu_key, u32 *data_size,
- u32 total_data, u32 total_size, int nr);
+ int nr);
int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root,
const struct btrfs_key *key, void *data, u32 data_size);
int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
u64 btrfs_file_extent_end(const struct btrfs_path *path);
/* inode.c */
+blk_status_t btrfs_submit_data_bio(struct inode *inode, struct bio *bio,
+ int mirror_num, unsigned long bio_flags);
+int btrfs_verify_data_csum(struct btrfs_io_bio *io_bio, u64 phy_offset,
+ struct page *page, u64 start, u64 end, int mirror);
struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode,
u64 start, u64 len);
noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
u32 min_type);
int btrfs_start_delalloc_snapshot(struct btrfs_root *root);
-int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int nr);
+int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, u64 nr);
int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
unsigned int extra_bits,
struct extent_state **cached_state);
void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start,
u64 end, int uptodate);
extern const struct dentry_operations btrfs_dentry_operations;
+ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
/* ioctl.c */
long btrfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
struct btrfs_ioctl_space_info *space);
void btrfs_update_ioctl_balance_args(struct btrfs_fs_info *fs_info,
struct btrfs_ioctl_balance_args *bargs);
+bool btrfs_exclop_start(struct btrfs_fs_info *fs_info,
+ enum btrfs_exclusive_operation type);
+void btrfs_exclop_finish(struct btrfs_fs_info *fs_info);
/* file.c */
int __init btrfs_auto_defrag_init(void);
int btrfs_drop_extents(struct btrfs_trans_handle *trans,
struct btrfs_root *root, struct inode *inode, u64 start,
u64 end, int drop_cache);
-int btrfs_punch_hole_range(struct inode *inode, struct btrfs_path *path,
+int btrfs_replace_file_extents(struct inode *inode, struct btrfs_path *path,
const u64 start, const u64 end,
- struct btrfs_clone_extent_info *clone_info,
+ struct btrfs_replace_extent_info *extent_info,
struct btrfs_trans_handle **trans_out);
int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
struct btrfs_inode *inode, u64 start, u64 end);
/* Sanity test specific functions */
#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
-void btrfs_test_inode_set_ops(struct inode *inode);
void btrfs_test_destroy_inode(struct inode *inode);
-
static inline int btrfs_is_testing(struct btrfs_fs_info *fs_info)
{
return test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state);
{
struct btrfs_root *root = inode->root;
struct btrfs_fs_info *fs_info = root->fs_info;
- struct btrfs_space_info *data_sinfo = fs_info->data_sinfo;
- u64 used;
- int ret = 0;
- int need_commit = 2;
- int have_pinned_space;
+ enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_DATA;
/* Make sure bytes are sectorsize aligned */
bytes = ALIGN(bytes, fs_info->sectorsize);
- if (btrfs_is_free_space_inode(inode)) {
- need_commit = 0;
- ASSERT(current->journal_info);
- }
-
-again:
- /* Make sure we have enough space to handle the data first */
- spin_lock(&data_sinfo->lock);
- used = btrfs_space_info_used(data_sinfo, true);
-
- if (used + bytes > data_sinfo->total_bytes) {
- struct btrfs_trans_handle *trans;
-
- /*
- * If we don't have enough free bytes in this space then we need
- * to alloc a new chunk.
- */
- if (!data_sinfo->full) {
- u64 alloc_target;
-
- data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
- spin_unlock(&data_sinfo->lock);
-
- alloc_target = btrfs_data_alloc_profile(fs_info);
- /*
- * It is ugly that we don't call nolock join
- * transaction for the free space inode case here.
- * But it is safe because we only do the data space
- * reservation for the free space cache in the
- * transaction context, the common join transaction
- * just increase the counter of the current transaction
- * handler, doesn't try to acquire the trans_lock of
- * the fs.
- */
- trans = btrfs_join_transaction(root);
- if (IS_ERR(trans))
- return PTR_ERR(trans);
-
- ret = btrfs_chunk_alloc(trans, alloc_target,
- CHUNK_ALLOC_NO_FORCE);
- btrfs_end_transaction(trans);
- if (ret < 0) {
- if (ret != -ENOSPC)
- return ret;
- else {
- have_pinned_space = 1;
- goto commit_trans;
- }
- }
-
- goto again;
- }
-
- /*
- * If we don't have enough pinned space to deal with this
- * allocation, and no removed chunk in current transaction,
- * don't bother committing the transaction.
- */
- have_pinned_space = __percpu_counter_compare(
- &data_sinfo->total_bytes_pinned,
- used + bytes - data_sinfo->total_bytes,
- BTRFS_TOTAL_BYTES_PINNED_BATCH);
- spin_unlock(&data_sinfo->lock);
-
- /* Commit the current transaction and try again */
-commit_trans:
- if (need_commit) {
- need_commit--;
-
- if (need_commit > 0) {
- btrfs_start_delalloc_roots(fs_info, -1);
- btrfs_wait_ordered_roots(fs_info, U64_MAX, 0,
- (u64)-1);
- }
-
- trans = btrfs_join_transaction(root);
- if (IS_ERR(trans))
- return PTR_ERR(trans);
- if (have_pinned_space >= 0 ||
- test_bit(BTRFS_TRANS_HAVE_FREE_BGS,
- &trans->transaction->flags) ||
- need_commit > 0) {
- ret = btrfs_commit_transaction(trans);
- if (ret)
- return ret;
- /*
- * The cleaner kthread might still be doing iput
- * operations. Wait for it to finish so that
- * more space is released. We don't need to
- * explicitly run the delayed iputs here because
- * the commit_transaction would have woken up
- * the cleaner.
- */
- ret = btrfs_wait_on_delayed_iputs(fs_info);
- if (ret)
- return ret;
- goto again;
- } else {
- btrfs_end_transaction(trans);
- }
- }
-
- trace_btrfs_space_reservation(fs_info,
- "space_info:enospc",
- data_sinfo->flags, bytes, 1);
- return -ENOSPC;
- }
- btrfs_space_info_update_bytes_may_use(fs_info, data_sinfo, bytes);
- spin_unlock(&data_sinfo->lock);
+ if (btrfs_is_free_space_inode(inode))
+ flush = BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE;
- return 0;
+ return btrfs_reserve_data_bytes(fs_info, bytes, flush);
}
int btrfs_check_data_free_space(struct btrfs_inode *inode,
ASSERT(IS_ALIGNED(len, fs_info->sectorsize));
data_sinfo = fs_info->data_sinfo;
- spin_lock(&data_sinfo->lock);
- btrfs_space_info_update_bytes_may_use(fs_info, data_sinfo, -len);
- spin_unlock(&data_sinfo->lock);
+ btrfs_space_info_free_bytes_may_use(fs_info, data_sinfo, len);
}
/*
*/
if (!src_rsv || (!trans->bytes_reserved &&
src_rsv->type != BTRFS_BLOCK_RSV_DELALLOC)) {
- ret = btrfs_qgroup_reserve_meta_prealloc(root,
- fs_info->nodesize, true);
+ ret = btrfs_qgroup_reserve_meta_prealloc(root, num_bytes, true);
if (ret < 0)
return ret;
ret = btrfs_block_rsv_add(root, dst_rsv, num_bytes,
}
/* insert the keys of the items */
- setup_items_for_insert(root, path, keys, data_size,
- total_data_size, total_size, nitems);
+ setup_items_for_insert(root, path, keys, data_size, nitems);
/* insert the dir index items */
slot = path->slots[0];
static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
int scrub_ret);
-static void btrfs_dev_replace_update_device_in_mapping_tree(
- struct btrfs_fs_info *fs_info,
- struct btrfs_device *srcdev,
- struct btrfs_device *tgtdev);
static int btrfs_dev_replace_kthread(void *data);
int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info)
{
struct btrfs_device *device;
struct block_device *bdev;
- struct list_head *devices;
struct rcu_string *name;
u64 devid = BTRFS_DEV_REPLACE_DEVID;
int ret = 0;
*device_out = NULL;
- if (fs_info->fs_devices->seeding) {
+ if (srcdev->fs_devices->seeding) {
btrfs_err(fs_info, "the filesystem is a seed filesystem!");
return -EINVAL;
}
sync_blockdev(bdev);
- devices = &fs_info->fs_devices->devices;
- list_for_each_entry(device, devices, dev_list) {
+ list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) {
if (device->bdev == bdev) {
btrfs_err(fs_info,
"target device is in the filesystem!");
atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0);
up_write(&dev_replace->rwsem);
- ret = btrfs_sysfs_add_devices_dir(tgt_device->fs_devices, tgt_device);
+ ret = btrfs_sysfs_add_device(tgt_device);
if (ret)
btrfs_err(fs_info, "kobj add dev failed %d", ret);
wake_up(&fs_info->dev_replace.replace_wait);
}
+/*
+ * When finishing the device replace, before swapping the source device with the
+ * target device we must update the chunk allocation state in the target device,
+ * as it is empty because replace works by directly copying the chunks and not
+ * through the normal chunk allocation path.
+ */
+static int btrfs_set_target_alloc_state(struct btrfs_device *srcdev,
+ struct btrfs_device *tgtdev)
+{
+ struct extent_state *cached_state = NULL;
+ u64 start = 0;
+ u64 found_start;
+ u64 found_end;
+ int ret = 0;
+
+ lockdep_assert_held(&srcdev->fs_info->chunk_mutex);
+
+ while (!find_first_extent_bit(&srcdev->alloc_state, start,
+ &found_start, &found_end,
+ CHUNK_ALLOCATED, &cached_state)) {
+ ret = set_extent_bits(&tgtdev->alloc_state, found_start,
+ found_end, CHUNK_ALLOCATED);
+ if (ret)
+ break;
+ start = found_end + 1;
+ }
+
+ free_extent_state(cached_state);
+ return ret;
+}
+
+static void btrfs_dev_replace_update_device_in_mapping_tree(
+ struct btrfs_fs_info *fs_info,
+ struct btrfs_device *srcdev,
+ struct btrfs_device *tgtdev)
+{
+ struct extent_map_tree *em_tree = &fs_info->mapping_tree;
+ struct extent_map *em;
+ struct map_lookup *map;
+ u64 start = 0;
+ int i;
+
+ write_lock(&em_tree->lock);
+ do {
+ em = lookup_extent_mapping(em_tree, start, (u64)-1);
+ if (!em)
+ break;
+ map = em->map_lookup;
+ for (i = 0; i < map->num_stripes; i++)
+ if (srcdev == map->stripes[i].dev)
+ map->stripes[i].dev = tgtdev;
+ start = em->start + em->len;
+ free_extent_map(em);
+ } while (start);
+ write_unlock(&em_tree->lock);
+}
+
static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
int scrub_ret)
{
* flush all outstanding I/O and inode extent mappings before the
* copy operation is declared as being finished
*/
- ret = btrfs_start_delalloc_roots(fs_info, -1);
+ ret = btrfs_start_delalloc_roots(fs_info, U64_MAX);
if (ret) {
mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
return ret;
dev_replace->time_stopped = ktime_get_real_seconds();
dev_replace->item_needs_writeback = 1;
- /* replace old device with new one in mapping tree */
+ /*
+ * Update allocation state in the new device and replace the old device
+ * with the new one in the mapping tree.
+ */
if (!scrub_ret) {
+ scrub_ret = btrfs_set_target_alloc_state(src_device, tgt_device);
+ if (scrub_ret)
+ goto error;
btrfs_dev_replace_update_device_in_mapping_tree(fs_info,
src_device,
tgt_device);
btrfs_dev_name(src_device),
src_device->devid,
rcu_str_deref(tgt_device->name), scrub_ret);
+error:
up_write(&dev_replace->rwsem);
mutex_unlock(&fs_info->chunk_mutex);
mutex_unlock(&fs_info->fs_devices->device_list_mutex);
mutex_unlock(&fs_info->fs_devices->device_list_mutex);
/* replace the sysfs entry */
- btrfs_sysfs_remove_devices_dir(fs_info->fs_devices, src_device);
+ btrfs_sysfs_remove_device(src_device);
btrfs_sysfs_update_devid(tgt_device);
- btrfs_rm_dev_replace_free_srcdev(src_device);
+ if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &src_device->dev_state))
+ btrfs_scratch_superblocks(fs_info, src_device->bdev,
+ src_device->name->str);
/* write back the superblocks */
trans = btrfs_start_transaction(root, 0);
mutex_unlock(&dev_replace->lock_finishing_cancel_unmount);
- return 0;
-}
-
-static void btrfs_dev_replace_update_device_in_mapping_tree(
- struct btrfs_fs_info *fs_info,
- struct btrfs_device *srcdev,
- struct btrfs_device *tgtdev)
-{
- struct extent_map_tree *em_tree = &fs_info->mapping_tree;
- struct extent_map *em;
- struct map_lookup *map;
- u64 start = 0;
- int i;
+ btrfs_rm_dev_replace_free_srcdev(src_device);
- write_lock(&em_tree->lock);
- do {
- em = lookup_extent_mapping(em_tree, start, (u64)-1);
- if (!em)
- break;
- map = em->map_lookup;
- for (i = 0; i < map->num_stripes; i++)
- if (srcdev == map->stripes[i].dev)
- map->stripes[i].dev = tgtdev;
- start = em->start + em->len;
- free_extent_map(em);
- } while (start);
- write_unlock(&em_tree->lock);
+ return 0;
}
/*
* should never allow both to start and pause. We don't want to allow
* dev-replace to start anyway.
*/
- if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags)) {
+ if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) {
down_write(&dev_replace->rwsem);
dev_replace->replace_state =
BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED;
ret = btrfs_dev_replace_finishing(fs_info, ret);
WARN_ON(ret && ret != -ECANCELED);
- clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
+ btrfs_exclop_finish(fs_info);
return 0;
}
BTRFS_SUPER_FLAG_METADUMP |\
BTRFS_SUPER_FLAG_METADUMP_V2)
-static const struct extent_io_ops btree_extent_io_ops;
static void end_workqueue_fn(struct btrfs_work *work);
static void btrfs_destroy_ordered_extents(struct btrfs_root *root);
static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
#endif
-/*
- * extents on the btree inode are pretty simple, there's one extent
- * that covers the entire device
- */
-struct extent_map *btree_get_extent(struct btrfs_inode *inode,
- struct page *page, size_t pg_offset,
- u64 start, u64 len)
-{
- struct extent_map_tree *em_tree = &inode->extent_tree;
- struct extent_map *em;
- int ret;
-
- read_lock(&em_tree->lock);
- em = lookup_extent_mapping(em_tree, start, len);
- if (em) {
- read_unlock(&em_tree->lock);
- goto out;
- }
- read_unlock(&em_tree->lock);
-
- em = alloc_extent_map();
- if (!em) {
- em = ERR_PTR(-ENOMEM);
- goto out;
- }
- em->start = 0;
- em->len = (u64)-1;
- em->block_len = (u64)-1;
- em->block_start = 0;
-
- write_lock(&em_tree->lock);
- ret = add_extent_mapping(em_tree, em, 0);
- if (ret == -EEXIST) {
- free_extent_map(em);
- em = lookup_extent_mapping(em_tree, start, len);
- if (!em)
- em = ERR_PTR(-EIO);
- } else if (ret) {
- free_extent_map(em);
- em = ERR_PTR(ret);
- }
- write_unlock(&em_tree->lock);
-
-out:
- return em;
-}
-
/*
* Compute the csum of a btree block and store the result to provided buffer.
*/
static int check_tree_block_fsid(struct extent_buffer *eb)
{
struct btrfs_fs_info *fs_info = eb->fs_info;
- struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
+ struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
u8 fsid[BTRFS_FSID_SIZE];
- int ret = 1;
+ u8 *metadata_uuid;
read_extent_buffer(eb, fsid, offsetof(struct btrfs_header, fsid),
BTRFS_FSID_SIZE);
- while (fs_devices) {
- u8 *metadata_uuid;
+ /*
+ * Checking the incompat flag is only valid for the current fs. For
+ * seed devices it's forbidden to have their uuid changed so reading
+ * ->fsid in this case is fine
+ */
+ if (btrfs_fs_incompat(fs_info, METADATA_UUID))
+ metadata_uuid = fs_devices->metadata_uuid;
+ else
+ metadata_uuid = fs_devices->fsid;
- /*
- * Checking the incompat flag is only valid for the current
- * fs. For seed devices it's forbidden to have their uuid
- * changed so reading ->fsid in this case is fine
- */
- if (fs_devices == fs_info->fs_devices &&
- btrfs_fs_incompat(fs_info, METADATA_UUID))
- metadata_uuid = fs_devices->metadata_uuid;
- else
- metadata_uuid = fs_devices->fsid;
+ if (!memcmp(fsid, metadata_uuid, BTRFS_FSID_SIZE))
+ return 0;
- if (!memcmp(fsid, metadata_uuid, BTRFS_FSID_SIZE)) {
- ret = 0;
- break;
- }
- fs_devices = fs_devices->seed;
- }
- return ret;
+ list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list)
+ if (!memcmp(fsid, seed_devs->fsid, BTRFS_FSID_SIZE))
+ return 0;
+
+ return 1;
}
-static int btree_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
- u64 phy_offset, struct page *page,
- u64 start, u64 end, int mirror)
+int btrfs_validate_metadata_buffer(struct btrfs_io_bio *io_bio, u64 phy_offset,
+ struct page *page, u64 start, u64 end,
+ int mirror)
{
u64 found_start;
int found_level;
csum_tree_block(eb, result);
if (memcmp_extent_buffer(eb, result, 0, csum_size)) {
- u32 val;
- u32 found = 0;
-
- memcpy(&found, result, csum_size);
+ u8 val[BTRFS_CSUM_SIZE] = { 0 };
read_extent_buffer(eb, &val, 0, csum_size);
btrfs_warn_rl(fs_info,
- "%s checksum verify failed on %llu wanted %x found %x level %d",
+ "%s checksum verify failed on %llu wanted " CSUM_FMT " found " CSUM_FMT " level %d",
fs_info->sb->s_id, eb->start,
- val, found, btrfs_header_level(eb));
+ CSUM_FMT_VALUE(csum_size, val),
+ CSUM_FMT_VALUE(csum_size, result),
+ btrfs_header_level(eb));
ret = -EUCLEAN;
goto err;
}
return 1;
}
-static blk_status_t btree_submit_bio_hook(struct inode *inode, struct bio *bio,
- int mirror_num,
- unsigned long bio_flags)
+blk_status_t btrfs_submit_metadata_bio(struct inode *inode, struct bio *bio,
+ int mirror_num, unsigned long bio_flags)
{
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
int async = check_async_write(fs_info, BTRFS_I(inode));
return btree_write_cache_pages(mapping, wbc);
}
-static int btree_readpage(struct file *file, struct page *page)
-{
- return extent_read_full_page(page, btree_get_extent, 0);
-}
-
static int btree_releasepage(struct page *page, gfp_t gfp_flags)
{
if (PageWriteback(page) || PageDirty(page))
}
static const struct address_space_operations btree_aops = {
- .readpage = btree_readpage,
.writepages = btree_writepages,
.releasepage = btree_releasepage,
.invalidatepage = btree_invalidatepage,
root->root_key.type = BTRFS_ROOT_ITEM_KEY;
root->root_key.offset = 0;
- leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0);
+ leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0,
+ BTRFS_NESTING_NORMAL);
if (IS_ERR(leaf)) {
ret = PTR_ERR(leaf);
leaf = NULL;
*/
leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID,
- NULL, 0, 0, 0);
+ NULL, 0, 0, 0, BTRFS_NESTING_NORMAL);
if (IS_ERR(leaf)) {
btrfs_put_root(root);
return ERR_CAST(leaf);
struct btrfs_root *root;
while (!list_empty(&fs_info->allocated_roots)) {
+ char buf[BTRFS_ROOT_NAME_BUF_LEN];
+
root = list_first_entry(&fs_info->allocated_roots,
struct btrfs_root, leak_list);
- btrfs_err(fs_info, "leaked root %llu-%llu refcount %d",
- root->root_key.objectid, root->root_key.offset,
+ btrfs_err(fs_info, "leaked root %s refcount %d",
+ btrfs_root_name(root->root_key.objectid, buf),
refcount_read(&root->refs));
while (refcount_read(&root->refs) > 1)
btrfs_put_root(root);
RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
extent_io_tree_init(fs_info, &BTRFS_I(inode)->io_tree,
- IO_TREE_INODE_IO, inode);
+ IO_TREE_BTREE_INODE_IO, inode);
BTRFS_I(inode)->io_tree.track_uptodate = false;
extent_map_tree_init(&BTRFS_I(inode)->extent_tree);
- BTRFS_I(inode)->io_tree.ops = &btree_extent_io_ops;
-
BTRFS_I(inode)->root = btrfs_grab_root(fs_info->tree_root);
memset(&BTRFS_I(inode)->location, 0, sizeof(struct btrfs_key));
set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
level = btrfs_super_root_level(sb);
tree_root->node = read_tree_block(fs_info, btrfs_super_root(sb),
generation, level, NULL);
- if (IS_ERR(tree_root->node) ||
- !extent_buffer_uptodate(tree_root->node)) {
+ if (IS_ERR(tree_root->node)) {
handle_error = true;
+ ret = PTR_ERR(tree_root->node);
+ tree_root->node = NULL;
+ btrfs_warn(fs_info, "couldn't read tree root");
+ continue;
- if (IS_ERR(tree_root->node)) {
- ret = PTR_ERR(tree_root->node);
- tree_root->node = NULL;
- } else if (!extent_buffer_uptodate(tree_root->node)) {
- ret = -EUCLEAN;
- }
-
- btrfs_warn(fs_info, "failed to read tree root");
+ } else if (!extent_buffer_uptodate(tree_root->node)) {
+ handle_error = true;
+ ret = -EIO;
+ btrfs_warn(fs_info, "error while reading tree root");
continue;
}
fs_info->check_integrity_print_mask = 0;
#endif
btrfs_init_balance(fs_info);
- btrfs_init_async_reclaim_work(&fs_info->async_reclaim_work);
+ btrfs_init_async_reclaim_work(fs_info);
spin_lock_init(&fs_info->block_group_cache_lock);
fs_info->block_group_cache_tree = RB_ROOT;
}
/*
- * Verify the type first, if that or the the checksum value are
+ * Verify the type first, if that or the checksum value are
* corrupted, we'll find out
*/
csum_type = btrfs_super_csum_type(disk_super);
return ERR_CAST(page);
super = page_address(page);
- if (btrfs_super_bytenr(super) != bytenr ||
- btrfs_super_magic(super) != BTRFS_MAGIC) {
+ if (btrfs_super_magic(super) != BTRFS_MAGIC) {
+ btrfs_release_disk_super(super);
+ return ERR_PTR(-ENODATA);
+ }
+
+ if (btrfs_super_bytenr(super) != bytenr) {
btrfs_release_disk_super(super);
return ERR_PTR(-EINVAL);
}
btrfs_cleanup_defrag_inodes(fs_info);
cancel_work_sync(&fs_info->async_reclaim_work);
+ cancel_work_sync(&fs_info->async_data_reclaim_work);
/* Cancel or finish ongoing discard work */
btrfs_discard_cleanup(fs_info);
return 0;
}
-
-static const struct extent_io_ops btree_extent_io_ops = {
- /* mandatory callbacks */
- .submit_bio_hook = btree_submit_bio_hook,
- .readpage_end_io_hook = btree_readpage_end_io_hook,
-};
void btrfs_btree_balance_dirty_nodelay(struct btrfs_fs_info *fs_info);
void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info,
struct btrfs_root *root);
-
+int btrfs_validate_metadata_buffer(struct btrfs_io_bio *io_bio, u64 phy_offset,
+ struct page *page, u64 start, u64 end,
+ int mirror);
+blk_status_t btrfs_submit_metadata_bio(struct inode *inode, struct bio *bio,
+ int mirror_num, unsigned long bio_flags);
#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
struct btrfs_root *btrfs_alloc_dummy_root(struct btrfs_fs_info *fs_info);
#endif
u64 objectid);
int btree_lock_page_hook(struct page *page, void *data,
void (*flush_fn)(void *));
-struct extent_map *btree_get_extent(struct btrfs_inode *inode,
- struct page *page, size_t pg_offset,
- u64 start, u64 len);
int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags);
int __init btrfs_end_io_wq_init(void);
void __cold btrfs_end_io_wq_exit(void);
enum {
IO_TREE_FS_PINNED_EXTENTS,
IO_TREE_FS_EXCLUDED_EXTENTS,
+ IO_TREE_BTREE_INODE_IO,
IO_TREE_INODE_IO,
IO_TREE_INODE_IO_FAILURE,
IO_TREE_RELOC_BLOCKS,
IO_TREE_INODE_FILE_EXTENT,
IO_TREE_LOG_CSUM_RANGE,
IO_TREE_SELFTEST,
+ IO_TREE_DEVICE_ALLOC_STATE,
};
struct extent_io_tree {
u8 owner;
spinlock_t lock;
- const struct extent_io_ops *ops;
};
struct extent_state {
num_bytes, parent, root_objectid,
owner, offset, 1);
if (ret == 0) {
- BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
+ /*
+ * We're adding refs to a tree block we already own, this
+ * should not happen at all.
+ */
+ if (owner < BTRFS_FIRST_FREE_OBJECTID) {
+ btrfs_crit(trans->fs_info,
+"adding refs to an existing tree ref, bytenr %llu num_bytes %llu root_objectid %llu",
+ bytenr, num_bytes, root_objectid);
+ if (IS_ENABLED(CONFIG_BTRFS_DEBUG)) {
+ WARN_ON(1);
+ btrfs_crit(trans->fs_info,
+ "path->slots[0]=%d path->nodes[0]:", path->slots[0]);
+ btrfs_print_leaf(path->nodes[0]);
+ }
+ return -EUCLEAN;
+ }
update_inline_extent_backref(path, iref, refs_to_add,
extent_op, NULL);
} else if (ret == -ENOENT) {
/*
* __btrfs_inc_extent_ref - insert backreference for a given extent
*
+ * The counterpart is in __btrfs_free_extent(), with examples and more details
+ * how it works.
+ *
* @trans: Handle of transaction
*
* @node: The delayed ref node used to get the bytenr/length for
len -= to_add;
}
spin_unlock(&global_rsv->lock);
- /* Add to any tickets we may have */
- if (len)
- btrfs_try_granting_tickets(fs_info,
- space_info);
}
+ /* Add to any tickets we may have */
+ if (!readonly && return_free_space && len)
+ btrfs_try_granting_tickets(fs_info, space_info);
spin_unlock(&space_info->lock);
}
return 0;
}
+/*
+ * Drop one or more refs of @node.
+ *
+ * 1. Locate the extent refs.
+ * It's either inline in EXTENT/METADATA_ITEM or in keyed SHARED_* item.
+ * Locate it, then reduce the refs number or remove the ref line completely.
+ *
+ * 2. Update the refs count in EXTENT/METADATA_ITEM
+ *
+ * Inline backref case:
+ *
+ * in extent tree we have:
+ *
+ * item 0 key (13631488 EXTENT_ITEM 1048576) itemoff 16201 itemsize 82
+ * refs 2 gen 6 flags DATA
+ * extent data backref root FS_TREE objectid 258 offset 0 count 1
+ * extent data backref root FS_TREE objectid 257 offset 0 count 1
+ *
+ * This function gets called with:
+ *
+ * node->bytenr = 13631488
+ * node->num_bytes = 1048576
+ * root_objectid = FS_TREE
+ * owner_objectid = 257
+ * owner_offset = 0
+ * refs_to_drop = 1
+ *
+ * Then we should get some like:
+ *
+ * item 0 key (13631488 EXTENT_ITEM 1048576) itemoff 16201 itemsize 82
+ * refs 1 gen 6 flags DATA
+ * extent data backref root FS_TREE objectid 258 offset 0 count 1
+ *
+ * Keyed backref case:
+ *
+ * in extent tree we have:
+ *
+ * item 0 key (13631488 EXTENT_ITEM 1048576) itemoff 3971 itemsize 24
+ * refs 754 gen 6 flags DATA
+ * [...]
+ * item 2 key (13631488 EXTENT_DATA_REF <HASH>) itemoff 3915 itemsize 28
+ * extent data backref root FS_TREE objectid 866 offset 0 count 1
+ *
+ * This function get called with:
+ *
+ * node->bytenr = 13631488
+ * node->num_bytes = 1048576
+ * root_objectid = FS_TREE
+ * owner_objectid = 866
+ * owner_offset = 0
+ * refs_to_drop = 1
+ *
+ * Then we should get some like:
+ *
+ * item 0 key (13631488 EXTENT_ITEM 1048576) itemoff 3971 itemsize 24
+ * refs 753 gen 6 flags DATA
+ *
+ * And that (13631488 EXTENT_DATA_REF <HASH>) gets removed.
+ */
static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
struct btrfs_delayed_ref_node *node, u64 parent,
u64 root_objectid, u64 owner_objectid,
path->leave_spinning = 1;
is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
- BUG_ON(!is_data && refs_to_drop != 1);
+
+ if (!is_data && refs_to_drop != 1) {
+ btrfs_crit(info,
+"invalid refs_to_drop, dropping more than 1 refs for tree block %llu refs_to_drop %u",
+ node->bytenr, refs_to_drop);
+ ret = -EINVAL;
+ btrfs_abort_transaction(trans, ret);
+ goto out;
+ }
if (is_data)
skinny_metadata = false;
parent, root_objectid, owner_objectid,
owner_offset);
if (ret == 0) {
+ /*
+ * Either the inline backref or the SHARED_DATA_REF/
+ * SHARED_BLOCK_REF is found
+ *
+ * Here is a quick path to locate EXTENT/METADATA_ITEM.
+ * It's possible the EXTENT/METADATA_ITEM is near current slot.
+ */
extent_slot = path->slots[0];
while (extent_slot >= 0) {
btrfs_item_key_to_cpu(path->nodes[0], &key,
found_extent = 1;
break;
}
+
+ /* Quick path didn't find the EXTEMT/METADATA_ITEM */
if (path->slots[0] - extent_slot > 5)
break;
extent_slot--;
}
if (!found_extent) {
- BUG_ON(iref);
+ if (iref) {
+ btrfs_crit(info,
+"invalid iref, no EXTENT/METADATA_ITEM found but has inline extent ref");
+ btrfs_abort_transaction(trans, -EUCLEAN);
+ goto err_dump;
+ }
+ /* Must be SHARED_* item, remove the backref first */
ret = remove_extent_backref(trans, path, NULL,
refs_to_drop,
is_data, &last_ref);
btrfs_release_path(path);
path->leave_spinning = 1;
+ /* Slow path to locate EXTENT/METADATA_ITEM */
key.objectid = bytenr;
key.type = BTRFS_EXTENT_ITEM_KEY;
key.offset = num_bytes;
if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID &&
key.type == BTRFS_EXTENT_ITEM_KEY) {
struct btrfs_tree_block_info *bi;
- BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
+ if (item_size < sizeof(*ei) + sizeof(*bi)) {
+ btrfs_crit(info,
+"invalid extent item size for key (%llu, %u, %llu) owner %llu, has %u expect >= %lu",
+ key.objectid, key.type, key.offset,
+ owner_objectid, item_size,
+ sizeof(*ei) + sizeof(*bi));
+ btrfs_abort_transaction(trans, -EUCLEAN);
+ goto err_dump;
+ }
bi = (struct btrfs_tree_block_info *)(ei + 1);
WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
}
refs = btrfs_extent_refs(leaf, ei);
if (refs < refs_to_drop) {
- btrfs_err(info,
- "trying to drop %d refs but we only have %Lu for bytenr %Lu",
+ btrfs_crit(info,
+ "trying to drop %d refs but we only have %llu for bytenr %llu",
refs_to_drop, refs, bytenr);
- ret = -EINVAL;
- btrfs_abort_transaction(trans, ret);
- goto out;
+ btrfs_abort_transaction(trans, -EUCLEAN);
+ goto err_dump;
}
refs -= refs_to_drop;
* be updated by remove_extent_backref
*/
if (iref) {
- BUG_ON(!found_extent);
+ if (!found_extent) {
+ btrfs_crit(info,
+"invalid iref, got inlined extent ref but no EXTENT/METADATA_ITEM found");
+ btrfs_abort_transaction(trans, -EUCLEAN);
+ goto err_dump;
+ }
} else {
btrfs_set_extent_refs(leaf, ei, refs);
btrfs_mark_buffer_dirty(leaf);
}
}
} else {
+ /* In this branch refs == 1 */
if (found_extent) {
- BUG_ON(is_data && refs_to_drop !=
- extent_data_ref_count(path, iref));
+ if (is_data && refs_to_drop !=
+ extent_data_ref_count(path, iref)) {
+ btrfs_crit(info,
+ "invalid refs_to_drop, current refs %u refs_to_drop %u",
+ extent_data_ref_count(path, iref),
+ refs_to_drop);
+ btrfs_abort_transaction(trans, -EUCLEAN);
+ goto err_dump;
+ }
if (iref) {
- BUG_ON(path->slots[0] != extent_slot);
+ if (path->slots[0] != extent_slot) {
+ btrfs_crit(info,
+"invalid iref, extent item key (%llu %u %llu) doesn't have wanted iref",
+ key.objectid, key.type,
+ key.offset);
+ btrfs_abort_transaction(trans, -EUCLEAN);
+ goto err_dump;
+ }
} else {
- BUG_ON(path->slots[0] != extent_slot + 1);
+ /*
+ * No inline ref, we must be at SHARED_* item,
+ * And it's single ref, it must be:
+ * | extent_slot ||extent_slot + 1|
+ * [ EXTENT/METADATA_ITEM ][ SHARED_* ITEM ]
+ */
+ if (path->slots[0] != extent_slot + 1) {
+ btrfs_crit(info,
+ "invalid SHARED_* item, previous item is not EXTENT/METADATA_ITEM");
+ btrfs_abort_transaction(trans, -EUCLEAN);
+ goto err_dump;
+ }
path->slots[0] = extent_slot;
num_to_del = 2;
}
out:
btrfs_free_path(path);
return ret;
+err_dump:
+ /*
+ * Leaf dump can take up a lot of log buffer, so we only do full leaf
+ * dump for debug build.
+ */
+ if (IS_ENABLED(CONFIG_BTRFS_DEBUG)) {
+ btrfs_crit(info, "path->slots[0]=%d extent_slot=%d",
+ path->slots[0], extent_slot);
+ btrfs_print_leaf(path->nodes[0]);
+ }
+
+ btrfs_free_path(path);
+ return -EUCLEAN;
}
/*
* |- Push harder to find free extents
* |- If not found, re-iterate all block groups
*/
-static noinline int find_free_extent(struct btrfs_fs_info *fs_info,
+static noinline int find_free_extent(struct btrfs_root *root,
u64 ram_bytes, u64 num_bytes, u64 empty_size,
u64 hint_byte_orig, struct btrfs_key *ins,
u64 flags, int delalloc)
{
+ struct btrfs_fs_info *fs_info = root->fs_info;
int ret = 0;
int cache_block_group_error = 0;
struct btrfs_block_group *block_group = NULL;
ins->objectid = 0;
ins->offset = 0;
- trace_find_free_extent(fs_info, num_bytes, empty_size, flags);
+ trace_find_free_extent(root, num_bytes, empty_size, flags);
space_info = btrfs_find_space_info(fs_info, flags);
if (!space_info) {
flags = get_alloc_profile_by_root(root, is_data);
again:
WARN_ON(num_bytes < fs_info->sectorsize);
- ret = find_free_extent(fs_info, ram_bytes, num_bytes, empty_size,
+ ret = find_free_extent(root, ram_bytes, num_bytes, empty_size,
hint_byte, ins, flags, delalloc);
if (!ret && !is_data) {
btrfs_dec_block_group_reservations(fs_info, ins->objectid);
static struct extent_buffer *
btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root,
- u64 bytenr, int level, u64 owner)
+ u64 bytenr, int level, u64 owner,
+ enum btrfs_lock_nesting nest)
{
struct btrfs_fs_info *fs_info = root->fs_info;
struct extent_buffer *buf;
}
btrfs_set_buffer_lockdep_class(owner, buf, level);
- btrfs_tree_lock(buf);
+ __btrfs_tree_lock(buf, nest);
btrfs_clean_tree_block(buf);
clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
u64 parent, u64 root_objectid,
const struct btrfs_disk_key *key,
int level, u64 hint,
- u64 empty_size)
+ u64 empty_size,
+ enum btrfs_lock_nesting nest)
{
struct btrfs_fs_info *fs_info = root->fs_info;
struct btrfs_key ins;
#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
if (btrfs_is_testing(fs_info)) {
buf = btrfs_init_new_buffer(trans, root, root->alloc_bytenr,
- level, root_objectid);
+ level, root_objectid, nest);
if (!IS_ERR(buf))
root->alloc_bytenr += blocksize;
return buf;
goto out_unuse;
buf = btrfs_init_new_buffer(trans, root, ins.objectid, level,
- root_objectid);
+ root_objectid, nest);
if (IS_ERR(buf)) {
ret = PTR_ERR(buf);
goto out_free_reserved;
return ret;
}
-static int __must_check submit_one_bio(struct bio *bio, int mirror_num,
- unsigned long bio_flags)
+int __must_check submit_one_bio(struct bio *bio, int mirror_num,
+ unsigned long bio_flags)
{
blk_status_t ret = 0;
struct extent_io_tree *tree = bio->bi_private;
bio->bi_private = NULL;
- if (tree->ops)
- ret = tree->ops->submit_bio_hook(tree->private_data, bio,
- mirror_num, bio_flags);
+ if (is_data_inode(tree->private_data))
+ ret = btrfs_submit_data_bio(tree->private_data, bio, mirror_num,
+ bio_flags);
else
- btrfsic_submit_bio(bio);
+ ret = btrfs_submit_metadata_bio(tree->private_data, bio,
+ mirror_num, bio_flags);
return blk_status_to_errno(ret);
}
{
tree->fs_info = fs_info;
tree->state = RB_ROOT;
- tree->ops = NULL;
tree->dirty_bytes = 0;
spin_lock_init(&tree->lock);
tree->private_data = private_data;
struct page *page = bvec->bv_page;
struct inode *inode = page->mapping->host;
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- bool data_inode = btrfs_ino(BTRFS_I(inode))
- != BTRFS_BTREE_INODE_OBJECTID;
btrfs_debug(fs_info,
"end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
mirror = io_bio->mirror_num;
if (likely(uptodate)) {
- ret = tree->ops->readpage_end_io_hook(io_bio, offset,
- page, start, end,
- mirror);
+ if (is_data_inode(inode))
+ ret = btrfs_verify_data_csum(io_bio, offset, page,
+ start, end, mirror);
+ else
+ ret = btrfs_validate_metadata_buffer(io_bio,
+ offset, page, start, end, mirror);
if (ret)
uptodate = 0;
else
if (likely(uptodate))
goto readpage_ok;
- if (data_inode) {
+ if (is_data_inode(inode)) {
/*
* The generic bio_readpage_error handles errors the
if (!btrfs_submit_read_repair(inode, bio, offset, page,
start - page_offset(page),
start, end, mirror,
- tree->ops->submit_bio_hook)) {
+ btrfs_submit_data_bio)) {
uptodate = !bio->bi_status;
offset += len;
continue;
else
contig = bio_end_sector(bio) == sector;
- ASSERT(tree->ops);
if (btrfs_bio_fits_in_stripe(page, page_size, bio, bio_flags))
can_merge = false;
static struct extent_map *
__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
- u64 start, u64 len, get_extent_t *get_extent,
- struct extent_map **em_cached)
+ u64 start, u64 len, struct extent_map **em_cached)
{
struct extent_map *em;
*em_cached = NULL;
}
- em = get_extent(BTRFS_I(inode), page, pg_offset, start, len);
+ em = btrfs_get_extent(BTRFS_I(inode), page, pg_offset, start, len);
if (em_cached && !IS_ERR_OR_NULL(em)) {
BUG_ON(*em_cached);
refcount_inc(&em->refs);
* XXX JDM: This needs looking at to ensure proper page locking
* return 0 on success, otherwise return error
*/
-static int __do_readpage(struct page *page,
- get_extent_t *get_extent,
- struct extent_map **em_cached,
- struct bio **bio, int mirror_num,
- unsigned long *bio_flags, unsigned int read_flags,
- u64 *prev_em_start)
+int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
+ struct bio **bio, unsigned long *bio_flags,
+ unsigned int read_flags, u64 *prev_em_start)
{
struct inode *inode = page->mapping->host;
u64 start = page_offset(page);
break;
}
em = __get_extent_map(inode, page, pg_offset, cur,
- end - cur + 1, get_extent, em_cached);
+ end - cur + 1, em_cached);
if (IS_ERR_OR_NULL(em)) {
SetPageError(page);
unlock_extent(tree, cur, end);
/*
* If we have a file range that points to a compressed extent
- * and it's followed by a consecutive file range that points to
+ * and it's followed by a consecutive file range that points
* to the same compressed extent (possibly with a different
* offset and/or length, so it either points to the whole extent
* or only part of it), we must make sure we do not submit a
ret = submit_extent_page(REQ_OP_READ | read_flags, NULL,
page, offset, disk_io_size,
pg_offset, bio,
- end_bio_extent_readpage, mirror_num,
+ end_bio_extent_readpage, 0,
*bio_flags,
this_bio_flag,
force_bio_submit);
btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
for (index = 0; index < nr_pages; index++) {
- __do_readpage(pages[index], btrfs_get_extent, em_cached,
- bio, 0, bio_flags, REQ_RAHEAD, prev_em_start);
+ btrfs_do_readpage(pages[index], em_cached, bio, bio_flags,
+ REQ_RAHEAD, prev_em_start);
put_page(pages[index]);
}
}
-static int __extent_read_full_page(struct page *page,
- get_extent_t *get_extent,
- struct bio **bio, int mirror_num,
- unsigned long *bio_flags,
- unsigned int read_flags)
-{
- struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
- u64 start = page_offset(page);
- u64 end = start + PAGE_SIZE - 1;
- int ret;
-
- btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
-
- ret = __do_readpage(page, get_extent, NULL, bio, mirror_num,
- bio_flags, read_flags, NULL);
- return ret;
-}
-
-int extent_read_full_page(struct page *page, get_extent_t *get_extent,
- int mirror_num)
-{
- struct bio *bio = NULL;
- unsigned long bio_flags = 0;
- int ret;
-
- ret = __extent_read_full_page(page, get_extent, &bio, mirror_num,
- &bio_flags, 0);
- if (bio)
- ret = submit_one_bio(bio, mirror_num, bio_flags);
- return ret;
-}
-
static void update_nr_written(struct writeback_control *wbc,
unsigned long nr_written)
{
* helper function for fiemap, which doesn't want to see any holes.
* This maps until we find something past 'last'
*/
-static struct extent_map *get_extent_skip_holes(struct inode *inode,
+static struct extent_map *get_extent_skip_holes(struct btrfs_inode *inode,
u64 offset, u64 last)
{
u64 sectorsize = btrfs_inode_sectorsize(inode);
if (len == 0)
break;
len = ALIGN(len, sectorsize);
- em = btrfs_get_extent_fiemap(BTRFS_I(inode), offset, len);
+ em = btrfs_get_extent_fiemap(inode, offset, len);
if (IS_ERR_OR_NULL(em))
return em;
return ret;
}
-int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
+int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo,
u64 start, u64 len)
{
int ret = 0;
u64 last;
u64 last_for_get_extent = 0;
u64 disko = 0;
- u64 isize = i_size_read(inode);
+ u64 isize = i_size_read(&inode->vfs_inode);
struct btrfs_key found_key;
struct extent_map *em = NULL;
struct extent_state *cached_state = NULL;
struct btrfs_path *path;
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_root *root = inode->root;
struct fiemap_cache cache = { 0 };
struct ulist *roots;
struct ulist *tmp_ulist;
* lookup the last file extent. We're not using i_size here
* because there might be preallocation past i_size
*/
- ret = btrfs_lookup_file_extent(NULL, root, path,
- btrfs_ino(BTRFS_I(inode)), -1, 0);
+ ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), -1,
+ 0);
if (ret < 0) {
goto out_free_ulist;
} else {
found_type = found_key.type;
/* No extents, but there might be delalloc bits */
- if (found_key.objectid != btrfs_ino(BTRFS_I(inode)) ||
+ if (found_key.objectid != btrfs_ino(inode) ||
found_type != BTRFS_EXTENT_DATA_KEY) {
/* have to trust i_size as the end */
last = (u64)-1;
last_for_get_extent = isize;
}
- lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1,
+ lock_extent_bits(&inode->io_tree, start, start + len - 1,
&cached_state);
em = get_extent_skip_holes(inode, start, last_for_get_extent);
* then we're just getting a count and we can skip the
* lookup stuff.
*/
- ret = btrfs_check_shared(root,
- btrfs_ino(BTRFS_I(inode)),
+ ret = btrfs_check_shared(root, btrfs_ino(inode),
bytenr, roots, tmp_ulist);
if (ret < 0)
goto out_free;
ret = emit_last_fiemap_cache(fieinfo, &cache);
free_extent_map(em);
out:
- unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
+ unlock_extent_cached(&inode->io_tree, start, start + len - 1,
&cached_state);
out_free_ulist:
rwlock_init(&eb->lock);
atomic_set(&eb->blocking_readers, 0);
eb->blocking_writers = 0;
- eb->lock_nested = false;
+ eb->lock_recursed = false;
init_waitqueue_head(&eb->write_lock_wq);
init_waitqueue_head(&eb->read_lock_wq);
}
ClearPageError(page);
- err = __extent_read_full_page(page,
- btree_get_extent, &bio,
- mirror_num, &bio_flags,
- REQ_META);
+ err = submit_extent_page(REQ_OP_READ | REQ_META, NULL,
+ page, page_offset(page), PAGE_SIZE, 0,
+ &bio, end_bio_extent_readpage,
+ mirror_num, 0, 0, false);
if (err) {
- ret = err;
/*
- * We use &bio in above __extent_read_full_page,
- * so we ensure that if it returns error, the
- * current page fails to add itself to bio and
- * it's been unlocked.
- *
- * We must dec io_pages by ourselves.
+ * We failed to submit the bio so it's the
+ * caller's responsibility to perform cleanup
+ * i.e unlock page/set error bit.
*/
+ ret = err;
+ SetPageError(page);
+ unlock_page(page);
atomic_dec(&eb->io_pages);
}
} else {
return ret;
}
+static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
+ unsigned long len)
+{
+ btrfs_warn(eb->fs_info,
+ "access to eb bytenr %llu len %lu out of range start %lu len %lu",
+ eb->start, eb->len, start, len);
+ WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
+
+ return true;
+}
+
+/*
+ * Check if the [start, start + len) range is valid before reading/writing
+ * the eb.
+ * NOTE: @start and @len are offset inside the eb, not logical address.
+ *
+ * Caller should not touch the dst/src memory if this function returns error.
+ */
+static inline int check_eb_range(const struct extent_buffer *eb,
+ unsigned long start, unsigned long len)
+{
+ unsigned long offset;
+
+ /* start, start + len should not go beyond eb->len nor overflow */
+ if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
+ return report_eb_range(eb, start, len);
+
+ return false;
+}
+
void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
unsigned long start, unsigned long len)
{
char *dst = (char *)dstv;
unsigned long i = start >> PAGE_SHIFT;
- if (start + len > eb->len) {
- WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
- eb->start, eb->len, start, len);
- memset(dst, 0, len);
+ if (check_eb_range(eb, start, len))
return;
- }
offset = offset_in_page(start);
unsigned long i = start >> PAGE_SHIFT;
int ret = 0;
- WARN_ON(start > eb->len);
- WARN_ON(start + len > eb->start + eb->len);
+ if (check_eb_range(eb, start, len))
+ return -EINVAL;
offset = offset_in_page(start);
char *src = (char *)srcv;
unsigned long i = start >> PAGE_SHIFT;
- WARN_ON(start > eb->len);
- WARN_ON(start + len > eb->start + eb->len);
+ if (check_eb_range(eb, start, len))
+ return;
offset = offset_in_page(start);
char *kaddr;
unsigned long i = start >> PAGE_SHIFT;
- WARN_ON(start > eb->len);
- WARN_ON(start + len > eb->start + eb->len);
+ if (check_eb_range(eb, start, len))
+ return;
offset = offset_in_page(start);
char *kaddr;
unsigned long i = dst_offset >> PAGE_SHIFT;
+ if (check_eb_range(dst, dst_offset, len) ||
+ check_eb_range(src, src_offset, len))
+ return;
+
WARN_ON(src->len != dst_len);
offset = offset_in_page(dst_offset);
unsigned long dst_offset, unsigned long src_offset,
unsigned long len)
{
- struct btrfs_fs_info *fs_info = dst->fs_info;
size_t cur;
size_t dst_off_in_page;
size_t src_off_in_page;
unsigned long dst_i;
unsigned long src_i;
- if (src_offset + len > dst->len) {
- btrfs_err(fs_info,
- "memmove bogus src_offset %lu move len %lu dst len %lu",
- src_offset, len, dst->len);
- BUG();
- }
- if (dst_offset + len > dst->len) {
- btrfs_err(fs_info,
- "memmove bogus dst_offset %lu move len %lu dst len %lu",
- dst_offset, len, dst->len);
- BUG();
- }
+ if (check_eb_range(dst, dst_offset, len) ||
+ check_eb_range(dst, src_offset, len))
+ return;
while (len > 0) {
dst_off_in_page = offset_in_page(dst_offset);
unsigned long dst_offset, unsigned long src_offset,
unsigned long len)
{
- struct btrfs_fs_info *fs_info = dst->fs_info;
size_t cur;
size_t dst_off_in_page;
size_t src_off_in_page;
unsigned long dst_i;
unsigned long src_i;
- if (src_offset + len > dst->len) {
- btrfs_err(fs_info,
- "memmove bogus src_offset %lu move len %lu len %lu",
- src_offset, len, dst->len);
- BUG();
- }
- if (dst_offset + len > dst->len) {
- btrfs_err(fs_info,
- "memmove bogus dst_offset %lu move len %lu len %lu",
- dst_offset, len, dst->len);
- BUG();
- }
+ if (check_eb_range(dst, dst_offset, len) ||
+ check_eb_range(dst, src_offset, len))
+ return;
if (dst_offset < src_offset) {
memcpy_extent_buffer(dst, dst_offset, src_offset, len);
return;
typedef blk_status_t (extent_submit_bio_start_t)(void *private_data,
struct bio *bio, u64 bio_offset);
-struct extent_io_ops {
- /*
- * The following callbacks must be always defined, the function
- * pointer will be called unconditionally.
- */
- submit_bio_hook_t *submit_bio_hook;
- int (*readpage_end_io_hook)(struct btrfs_io_bio *io_bio, u64 phy_offset,
- struct page *page, u64 start, u64 end,
- int mirror);
-};
-
-
#define INLINE_EXTENT_BUFFER_PAGES 16
#define MAX_INLINE_EXTENT_BUFFER_SIZE (INLINE_EXTENT_BUFFER_PAGES * PAGE_SIZE)
struct extent_buffer {
int blocking_writers;
atomic_t blocking_readers;
- bool lock_nested;
+ bool lock_recursed;
/* >= 0 if eb belongs to a log tree, -1 otherwise */
short log_index;
int try_release_extent_mapping(struct page *page, gfp_t mask);
int try_release_extent_buffer(struct page *page);
-int extent_read_full_page(struct page *page, get_extent_t *get_extent,
- int mirror_num);
+int __must_check submit_one_bio(struct bio *bio, int mirror_num,
+ unsigned long bio_flags);
+int btrfs_do_readpage(struct page *page, struct extent_map **em_cached,
+ struct bio **bio, unsigned long *bio_flags,
+ unsigned int read_flags, u64 *prev_em_start);
int extent_write_full_page(struct page *page, struct writeback_control *wbc);
int extent_write_locked_range(struct inode *inode, u64 start, u64 end,
int mode);
int btree_write_cache_pages(struct address_space *mapping,
struct writeback_control *wbc);
void extent_readahead(struct readahead_control *rac);
-int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
+int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo,
u64 start, u64 len);
void set_page_extent_mapped(struct page *page);
if (page_offsets)
offset = page_offset(bvec.bv_page) + bvec.bv_offset;
- count = btrfs_find_ordered_sum(inode, offset, disk_bytenr,
- csum, nblocks);
+ count = btrfs_find_ordered_sum(BTRFS_I(inode), offset,
+ disk_bytenr, csum, nblocks);
if (count)
goto found;
if (btrfs_comp_cpu_keys(&key, &slot_key) > 0)
path->slots[0]++;
}
- setup_items_for_insert(root, path, &key,
- &extent_item_size,
- extent_item_size,
- sizeof(struct btrfs_item) +
- extent_item_size, 1);
+ setup_items_for_insert(root, path, &key, &extent_item_size, 1);
*key_inserted = 1;
}
int ret = 0;
start_pos = round_down(pos, fs_info->sectorsize);
- last_pos = start_pos
- + round_up(pos + write_bytes - start_pos,
- fs_info->sectorsize) - 1;
+ last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1;
if (start_pos < inode->vfs_inode.i_size) {
struct btrfs_ordered_extent *ordered;
unlock_page(pages[i]);
put_page(pages[i]);
}
- btrfs_start_ordered_extent(&inode->vfs_inode,
- ordered, 1);
+ btrfs_start_ordered_extent(ordered, 1);
btrfs_put_ordered_extent(ordered);
return -EAGAIN;
}
loff_t endbyte;
int err;
- written = generic_file_direct_write(iocb, from);
+ written = btrfs_direct_IO(iocb, from);
if (written < 0 || !iov_iter_count(from))
return written;
atomic_inc(&BTRFS_I(inode)->sync_writers);
if (iocb->ki_flags & IOCB_DIRECT) {
+ /*
+ * 1. We must always clear IOCB_DSYNC in order to not deadlock
+ * in iomap, as it calls generic_write_sync() in this case.
+ * 2. If we are async, we can call iomap_dio_complete() either
+ * in
+ *
+ * 2.1. A worker thread from the last bio completed. In this
+ * case we need to mark the btrfs_dio_data that it is
+ * async in order to call generic_write_sync() properly.
+ * This is handled by setting BTRFS_DIO_SYNC_STUB in the
+ * current->journal_info.
+ * 2.2 The submitter context, because all IO completed
+ * before we exited iomap_dio_rw(). In this case we can
+ * just re-set the IOCB_DSYNC on the iocb and we'll do
+ * the sync below. If our ->end_io() gets called and
+ * current->journal_info is set, then we know we're in
+ * our current context and we will clear
+ * current->journal_info to indicate that we need to
+ * sync below.
+ */
+ if (sync) {
+ ASSERT(current->journal_info == NULL);
+ iocb->ki_flags &= ~IOCB_DSYNC;
+ current->journal_info = BTRFS_DIO_SYNC_STUB;
+ }
num_written = __btrfs_direct_write(iocb, from);
+
+ /*
+ * As stated above, we cleared journal_info, so we need to do
+ * the sync ourselves.
+ */
+ if (sync && current->journal_info == NULL)
+ iocb->ki_flags |= IOCB_DSYNC;
+ current->journal_info = NULL;
} else {
num_written = btrfs_buffered_write(iocb, from);
if (num_written > 0)
filp->private_data = NULL;
/*
- * ordered_data_close is set by setattr when we are about to truncate
- * a file from a non-zero size to a zero size. This tries to
- * flush down new bytes that may have been written if the
- * application were using truncate to replace a file in place.
+ * Set by setattr when we are about to truncate a file from a non-zero
+ * size to a zero size. This tries to flush down new bytes that may
+ * have been written if the application were using truncate to replace
+ * a file in place.
*/
- if (test_and_clear_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
+ if (test_and_clear_bit(BTRFS_INODE_FLUSH_ON_CLOSE,
&BTRFS_I(inode)->runtime_flags))
filemap_flush(inode->i_mapping);
return 0;
struct btrfs_trans_handle *trans;
struct btrfs_log_ctx ctx;
int ret = 0, err;
+ u64 len;
+ bool full_sync;
trace_btrfs_sync_file(file, datasync);
btrfs_init_log_ctx(&ctx, inode);
/*
- * Set the range to full if the NO_HOLES feature is not enabled.
- * This is to avoid missing file extent items representing holes after
- * replaying the log.
+ * Always set the range to a full range, otherwise we can get into
+ * several problems, from missing file extent items to represent holes
+ * when not using the NO_HOLES feature, to log tree corruption due to
+ * races between hole detection during logging and completion of ordered
+ * extents outside the range, to missing checksums due to ordered extents
+ * for which we flushed only a subset of their pages.
*/
- if (!btrfs_fs_incompat(fs_info, NO_HOLES)) {
- start = 0;
- end = LLONG_MAX;
- }
+ start = 0;
+ end = LLONG_MAX;
+ len = (u64)LLONG_MAX + 1;
/*
* We write the dirty pages in the range and wait until they complete
atomic_inc(&root->log_batch);
/*
- * If the inode needs a full sync, make sure we use a full range to
- * avoid log tree corruption, due to hole detection racing with ordered
- * extent completion for adjacent ranges and races between logging and
- * completion of ordered extents for adjancent ranges - both races
- * could lead to file extent items in the log with overlapping ranges.
- * Do this while holding the inode lock, to avoid races with other
- * tasks.
+ * Always check for the full sync flag while holding the inode's lock,
+ * to avoid races with other tasks. The flag must be either set all the
+ * time during logging or always off all the time while logging.
*/
- if (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
- &BTRFS_I(inode)->runtime_flags)) {
- start = 0;
- end = LLONG_MAX;
- }
+ full_sync = test_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
+ &BTRFS_I(inode)->runtime_flags);
/*
* Before we acquired the inode's lock, someone may have dirtied more
* We have to do this here to avoid the priority inversion of waiting on
* IO of a lower priority task while holding a transaction open.
*
- * Also, the range length can be represented by u64, we have to do the
- * typecasts to avoid signed overflow if it's [0, LLONG_MAX].
+ * For a full fsync we wait for the ordered extents to complete while
+ * for a fast fsync we wait just for writeback to complete, and then
+ * attach the ordered extents to the transaction so that a transaction
+ * commit waits for their completion, to avoid data loss if we fsync,
+ * the current transaction commits before the ordered extents complete
+ * and a power failure happens right after that.
*/
- ret = btrfs_wait_ordered_range(inode, start, (u64)end - (u64)start + 1);
- if (ret) {
- up_write(&BTRFS_I(inode)->dio_sem);
- inode_unlock(inode);
- goto out;
+ if (full_sync) {
+ ret = btrfs_wait_ordered_range(inode, start, len);
+ } else {
+ /*
+ * Get our ordered extents as soon as possible to avoid doing
+ * checksum lookups in the csum tree, and use instead the
+ * checksums attached to the ordered extents.
+ */
+ btrfs_get_ordered_extents_for_logging(BTRFS_I(inode),
+ &ctx.ordered_extents);
+ ret = filemap_fdatawait_range(inode->i_mapping, start, end);
}
+
+ if (ret)
+ goto out_release_extents;
+
atomic_inc(&root->log_batch);
+ /*
+ * If we are doing a fast fsync we can not bail out if the inode's
+ * last_trans is <= then the last committed transaction, because we only
+ * update the last_trans of the inode during ordered extent completion,
+ * and for a fast fsync we don't wait for that, we only wait for the
+ * writeback to complete.
+ */
smp_mb();
if (btrfs_inode_in_log(BTRFS_I(inode), fs_info->generation) ||
- BTRFS_I(inode)->last_trans <= fs_info->last_trans_committed) {
+ (BTRFS_I(inode)->last_trans <= fs_info->last_trans_committed &&
+ (full_sync || list_empty(&ctx.ordered_extents)))) {
/*
* We've had everything committed since the last time we were
* modified so clear this flag in case it was set for whatever
* checked called fsync.
*/
ret = filemap_check_wb_err(inode->i_mapping, file->f_wb_err);
- up_write(&BTRFS_I(inode)->dio_sem);
- inode_unlock(inode);
- goto out;
+ goto out_release_extents;
}
/*
trans = btrfs_start_transaction(root, 0);
if (IS_ERR(trans)) {
ret = PTR_ERR(trans);
- up_write(&BTRFS_I(inode)->dio_sem);
- inode_unlock(inode);
- goto out;
+ goto out_release_extents;
}
- ret = btrfs_log_dentry_safe(trans, dentry, start, end, &ctx);
+ ret = btrfs_log_dentry_safe(trans, dentry, &ctx);
+ btrfs_release_log_ctx_extents(&ctx);
if (ret < 0) {
/* Fallthrough and commit/free transaction. */
ret = 1;
goto out;
}
}
+ if (!full_sync) {
+ ret = btrfs_wait_ordered_range(inode, start, len);
+ if (ret) {
+ btrfs_end_transaction(trans);
+ goto out;
+ }
+ }
ret = btrfs_commit_transaction(trans);
} else {
ret = btrfs_end_transaction(trans);
if (!ret)
ret = err;
return ret > 0 ? -EIO : ret;
+
+out_release_extents:
+ btrfs_release_log_ctx_extents(&ctx);
+ up_write(&BTRFS_I(inode)->dio_sem);
+ inode_unlock(inode);
+ goto out;
}
static const struct vm_operations_struct btrfs_file_vm_ops = {
lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
cached_state);
- ordered = btrfs_lookup_first_ordered_extent(inode, lockend);
+ ordered = btrfs_lookup_first_ordered_extent(BTRFS_I(inode),
+ lockend);
/*
* We need to make sure we have no ordered extents in this range
return 0;
}
-static int btrfs_insert_clone_extent(struct btrfs_trans_handle *trans,
+static int btrfs_insert_replace_extent(struct btrfs_trans_handle *trans,
struct inode *inode,
struct btrfs_path *path,
- struct btrfs_clone_extent_info *clone_info,
- const u64 clone_len)
+ struct btrfs_replace_extent_info *extent_info,
+ const u64 replace_len)
{
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
struct btrfs_root *root = BTRFS_I(inode)->root;
struct btrfs_key key;
int slot;
struct btrfs_ref ref = { 0 };
- u64 ref_offset;
int ret;
- if (clone_len == 0)
+ if (replace_len == 0)
return 0;
- if (clone_info->disk_offset == 0 &&
+ if (extent_info->disk_offset == 0 &&
btrfs_fs_incompat(fs_info, NO_HOLES))
return 0;
key.objectid = btrfs_ino(BTRFS_I(inode));
key.type = BTRFS_EXTENT_DATA_KEY;
- key.offset = clone_info->file_offset;
+ key.offset = extent_info->file_offset;
ret = btrfs_insert_empty_item(trans, root, path, &key,
- clone_info->item_size);
+ sizeof(struct btrfs_file_extent_item));
if (ret)
return ret;
leaf = path->nodes[0];
slot = path->slots[0];
- write_extent_buffer(leaf, clone_info->extent_buf,
+ write_extent_buffer(leaf, extent_info->extent_buf,
btrfs_item_ptr_offset(leaf, slot),
- clone_info->item_size);
+ sizeof(struct btrfs_file_extent_item));
extent = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
- btrfs_set_file_extent_offset(leaf, extent, clone_info->data_offset);
- btrfs_set_file_extent_num_bytes(leaf, extent, clone_len);
+ ASSERT(btrfs_file_extent_type(leaf, extent) != BTRFS_FILE_EXTENT_INLINE);
+ btrfs_set_file_extent_offset(leaf, extent, extent_info->data_offset);
+ btrfs_set_file_extent_num_bytes(leaf, extent, replace_len);
+ if (extent_info->is_new_extent)
+ btrfs_set_file_extent_generation(leaf, extent, trans->transid);
btrfs_mark_buffer_dirty(leaf);
btrfs_release_path(path);
ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode),
- clone_info->file_offset, clone_len);
+ extent_info->file_offset, replace_len);
if (ret)
return ret;
/* If it's a hole, nothing more needs to be done. */
- if (clone_info->disk_offset == 0)
+ if (extent_info->disk_offset == 0)
return 0;
- inode_add_bytes(inode, clone_len);
- btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF,
- clone_info->disk_offset,
- clone_info->disk_len, 0);
- ref_offset = clone_info->file_offset - clone_info->data_offset;
- btrfs_init_data_ref(&ref, root->root_key.objectid,
- btrfs_ino(BTRFS_I(inode)), ref_offset);
- ret = btrfs_inc_extent_ref(trans, &ref);
+ inode_add_bytes(inode, replace_len);
+
+ if (extent_info->is_new_extent && extent_info->insertions == 0) {
+ key.objectid = extent_info->disk_offset;
+ key.type = BTRFS_EXTENT_ITEM_KEY;
+ key.offset = extent_info->disk_len;
+ ret = btrfs_alloc_reserved_file_extent(trans, root,
+ btrfs_ino(BTRFS_I(inode)),
+ extent_info->file_offset,
+ extent_info->qgroup_reserved,
+ &key);
+ } else {
+ u64 ref_offset;
+
+ btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF,
+ extent_info->disk_offset,
+ extent_info->disk_len, 0);
+ ref_offset = extent_info->file_offset - extent_info->data_offset;
+ btrfs_init_data_ref(&ref, root->root_key.objectid,
+ btrfs_ino(BTRFS_I(inode)), ref_offset);
+ ret = btrfs_inc_extent_ref(trans, &ref);
+ }
+
+ extent_info->insertions++;
return ret;
}
/*
* The respective range must have been previously locked, as well as the inode.
* The end offset is inclusive (last byte of the range).
- * @clone_info is NULL for fallocate's hole punching and non-NULL for extent
- * cloning.
- * When cloning, we don't want to end up in a state where we dropped extents
- * without inserting a new one, so we must abort the transaction to avoid a
- * corruption.
+ * @extent_info is NULL for fallocate's hole punching and non-NULL when replacing
+ * the file range with an extent.
+ * When not punching a hole, we don't want to end up in a state where we dropped
+ * extents without inserting a new one, so we must abort the transaction to avoid
+ * a corruption.
*/
-int btrfs_punch_hole_range(struct inode *inode, struct btrfs_path *path,
+int btrfs_replace_file_extents(struct inode *inode, struct btrfs_path *path,
const u64 start, const u64 end,
- struct btrfs_clone_extent_info *clone_info,
+ struct btrfs_replace_extent_info *extent_info,
struct btrfs_trans_handle **trans_out)
{
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
/*
* 1 - update the inode
* 1 - removing the extents in the range
- * 1 - adding the hole extent if no_holes isn't set or if we are cloning
- * an extent
+ * 1 - adding the hole extent if no_holes isn't set or if we are
+ * replacing the range with a new extent
*/
- if (!btrfs_fs_incompat(fs_info, NO_HOLES) || clone_info)
+ if (!btrfs_fs_incompat(fs_info, NO_HOLES) || extent_info)
rsv_count = 3;
else
rsv_count = 2;
* returned by __btrfs_drop_extents() without having
* changed anything in the file.
*/
- if (clone_info && ret && ret != -EOPNOTSUPP)
+ if (extent_info && !extent_info->is_new_extent &&
+ ret && ret != -EOPNOTSUPP)
btrfs_abort_transaction(trans, ret);
break;
}
trans->block_rsv = &fs_info->trans_block_rsv;
- if (!clone_info && cur_offset < drop_end &&
+ if (!extent_info && cur_offset < drop_end &&
cur_offset < ino_size) {
ret = fill_holes(trans, BTRFS_I(inode), path,
cur_offset, drop_end);
btrfs_abort_transaction(trans, ret);
break;
}
- } else if (!clone_info && cur_offset < drop_end) {
+ } else if (!extent_info && cur_offset < drop_end) {
/*
* We are past the i_size here, but since we didn't
* insert holes we need to clear the mapped area so we
}
}
- if (clone_info && drop_end > clone_info->file_offset) {
- u64 clone_len = drop_end - clone_info->file_offset;
+ if (extent_info && drop_end > extent_info->file_offset) {
+ u64 replace_len = drop_end - extent_info->file_offset;
- ret = btrfs_insert_clone_extent(trans, inode, path,
- clone_info, clone_len);
+ ret = btrfs_insert_replace_extent(trans, inode, path,
+ extent_info, replace_len);
if (ret) {
btrfs_abort_transaction(trans, ret);
break;
}
- clone_info->data_len -= clone_len;
- clone_info->data_offset += clone_len;
- clone_info->file_offset += clone_len;
+ extent_info->data_len -= replace_len;
+ extent_info->data_offset += replace_len;
+ extent_info->file_offset += replace_len;
}
cur_offset = drop_end;
BUG_ON(ret); /* shouldn't happen */
trans->block_rsv = rsv;
- if (!clone_info) {
+ if (!extent_info) {
ret = find_first_non_hole(inode, &cur_offset, &len);
if (unlikely(ret < 0))
break;
* than 16Mb would force the full fsync any way (when
* try_release_extent_mapping() is invoked during page cache truncation.
*/
- if (clone_info)
+ if (extent_info && !extent_info->is_new_extent)
set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
&BTRFS_I(inode)->runtime_flags);
* (because it's useless) or if it represents a 0 bytes range (when
* cur_offset == drop_end).
*/
- if (!clone_info && cur_offset < ino_size && cur_offset < drop_end) {
+ if (!extent_info && cur_offset < ino_size && cur_offset < drop_end) {
ret = fill_holes(trans, BTRFS_I(inode), path,
cur_offset, drop_end);
if (ret) {
btrfs_abort_transaction(trans, ret);
goto out_trans;
}
- } else if (!clone_info && cur_offset < drop_end) {
+ } else if (!extent_info && cur_offset < drop_end) {
/* See the comment in the loop above for the reasoning here. */
ret = btrfs_inode_clear_file_extent_range(BTRFS_I(inode),
cur_offset, drop_end - cur_offset);
}
}
- if (clone_info) {
- ret = btrfs_insert_clone_extent(trans, inode, path, clone_info,
- clone_info->data_len);
+ if (extent_info) {
+ ret = btrfs_insert_replace_extent(trans, inode, path, extent_info,
+ extent_info->data_len);
if (ret) {
btrfs_abort_transaction(trans, ret);
goto out_trans;
goto out_only_mutex;
}
- lockstart = round_up(offset, btrfs_inode_sectorsize(inode));
+ lockstart = round_up(offset, btrfs_inode_sectorsize(BTRFS_I(inode)));
lockend = round_down(offset + len,
- btrfs_inode_sectorsize(inode)) - 1;
+ btrfs_inode_sectorsize(BTRFS_I(inode))) - 1;
same_block = (BTRFS_BYTES_TO_BLKS(fs_info, offset))
== (BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1));
/*
goto out;
}
- ret = btrfs_punch_hole_range(inode, path, lockstart, lockend, NULL,
+ ret = btrfs_replace_file_extents(inode, path, lockstart, lockend, NULL,
&trans);
btrfs_free_path(path);
if (ret)
RANGE_BOUNDARY_HOLE,
};
-static int btrfs_zero_range_check_range_boundary(struct inode *inode,
+static int btrfs_zero_range_check_range_boundary(struct btrfs_inode *inode,
u64 offset)
{
const u64 sectorsize = btrfs_inode_sectorsize(inode);
int ret;
offset = round_down(offset, sectorsize);
- em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize);
+ em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize);
if (IS_ERR(em))
return PTR_ERR(em);
struct extent_changeset *data_reserved = NULL;
int ret;
u64 alloc_hint = 0;
- const u64 sectorsize = btrfs_inode_sectorsize(inode);
+ const u64 sectorsize = btrfs_inode_sectorsize(BTRFS_I(inode));
u64 alloc_start = round_down(offset, sectorsize);
u64 alloc_end = round_up(offset + len, sectorsize);
u64 bytes_to_reserve = 0;
* to cover them.
*/
if (!IS_ALIGNED(offset, sectorsize)) {
- ret = btrfs_zero_range_check_range_boundary(inode, offset);
+ ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
+ offset);
if (ret < 0)
goto out;
if (ret == RANGE_BOUNDARY_HOLE) {
}
if (!IS_ALIGNED(offset + len, sectorsize)) {
- ret = btrfs_zero_range_check_range_boundary(inode,
+ ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
offset + len);
if (ret < 0)
goto out;
u64 locked_end;
u64 actual_end = 0;
struct extent_map *em;
- int blocksize = btrfs_inode_sectorsize(inode);
+ int blocksize = btrfs_inode_sectorsize(BTRFS_I(inode));
int ret;
alloc_start = round_down(offset, blocksize);
*/
lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
locked_end, &cached_state);
- ordered = btrfs_lookup_first_ordered_extent(inode, locked_end);
+ ordered = btrfs_lookup_first_ordered_extent(BTRFS_I(inode),
+ locked_end);
if (ordered &&
ordered->file_offset + ordered->num_bytes > alloc_start &&
return generic_file_open(inode, filp);
}
+static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
+{
+ ssize_t ret = 0;
+
+ if (iocb->ki_flags & IOCB_DIRECT) {
+ struct inode *inode = file_inode(iocb->ki_filp);
+
+ inode_lock_shared(inode);
+ ret = btrfs_direct_IO(iocb, to);
+ inode_unlock_shared(inode);
+ if (ret < 0)
+ return ret;
+ }
+
+ return generic_file_buffered_read(iocb, to, ret);
+}
+
const struct file_operations btrfs_file_operations = {
.llseek = btrfs_file_llseek,
- .read_iter = generic_file_read_iter,
+ .read_iter = btrfs_file_read_iter,
.splice_read = generic_file_splice_read,
.write_iter = btrfs_file_write_iter,
.splice_write = iter_file_splice_write,
static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation)
{
- __le64 *val;
-
io_ctl_map_page(io_ctl, 1);
/*
io_ctl->size -= sizeof(u64) * 2;
}
- val = io_ctl->cur;
- *val = cpu_to_le64(generation);
+ put_unaligned_le64(generation, io_ctl->cur);
io_ctl->cur += sizeof(u64);
}
static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation)
{
- __le64 *gen;
+ u64 cache_gen;
/*
* Skip the crc area. If we don't check crcs then we just have a 64bit
io_ctl->size -= sizeof(u64) * 2;
}
- gen = io_ctl->cur;
- if (le64_to_cpu(*gen) != generation) {
+ cache_gen = get_unaligned_le64(io_ctl->cur);
+ if (cache_gen != generation) {
btrfs_err_rl(io_ctl->fs_info,
"space cache generation (%llu) does not match inode (%llu)",
- *gen, generation);
+ cache_gen, generation);
io_ctl_unmap_page(io_ctl);
return -EIO;
}
return -ENOSPC;
entry = io_ctl->cur;
- entry->offset = cpu_to_le64(offset);
- entry->bytes = cpu_to_le64(bytes);
+ put_unaligned_le64(offset, &entry->offset);
+ put_unaligned_le64(bytes, &entry->bytes);
entry->type = (bitmap) ? BTRFS_FREE_SPACE_BITMAP :
BTRFS_FREE_SPACE_EXTENT;
io_ctl->cur += sizeof(struct btrfs_free_space_entry);
}
e = io_ctl->cur;
- entry->offset = le64_to_cpu(e->offset);
- entry->bytes = le64_to_cpu(e->bytes);
+ entry->offset = get_unaligned_le64(&e->offset);
+ entry->bytes = get_unaligned_le64(&e->bytes);
*type = e->type;
io_ctl->cur += sizeof(struct btrfs_free_space_entry);
io_ctl->size -= sizeof(struct btrfs_free_space_entry);
/*
* at this point the pages are under IO and we're happy,
- * The caller is responsible for waiting on them and updating the
+ * The caller is responsible for waiting on them and updating
* the cache and the inode
*/
io_ctl->entries = entries;
#include <crypto/hash.h>
#include <linux/kernel.h>
#include <linux/bio.h>
-#include <linux/buffer_head.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/migrate.h>
#include <linux/sched/mm.h>
+#include <linux/iomap.h>
#include <asm/unaligned.h>
#include "misc.h"
#include "ctree.h"
struct btrfs_dio_data {
u64 reserve;
- u64 unsubmitted_oe_range_start;
- u64 unsubmitted_oe_range_end;
- int overwrite;
+ loff_t length;
+ ssize_t submitted;
+ struct extent_changeset *data_reserved;
+ bool sync;
};
static const struct inode_operations btrfs_dir_inode_operations;
static const struct inode_operations btrfs_file_inode_operations;
static const struct address_space_operations btrfs_aops;
static const struct file_operations btrfs_dir_file_operations;
-static const struct extent_io_ops btrfs_extent_io_ops;
static struct kmem_cache *btrfs_inode_cachep;
struct kmem_cache *btrfs_trans_handle_cachep;
static int btrfs_dirty_inode(struct inode *inode);
-#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
-void btrfs_test_inode_set_ops(struct inode *inode)
-{
- BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
-}
-#endif
-
static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
struct inode *inode, struct inode *dir,
const struct qstr *qstr)
*
* c-3) otherwise: async submit
*/
-static blk_status_t btrfs_submit_bio_hook(struct inode *inode, struct bio *bio,
- int mirror_num,
- unsigned long bio_flags)
+blk_status_t btrfs_submit_data_bio(struct inode *inode, struct bio *bio,
+ int mirror_num, unsigned long bio_flags)
{
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
* given a list of ordered sums record them in the inode. This happens
* at IO completion time based on sums calculated at bio submission time.
*/
-static noinline int add_pending_csums(struct btrfs_trans_handle *trans,
- struct inode *inode, struct list_head *list)
+static int add_pending_csums(struct btrfs_trans_handle *trans,
+ struct list_head *list)
{
struct btrfs_ordered_sum *sum;
int ret;
list_for_each_entry(sum, list, list) {
trans->adding_csums = true;
- ret = btrfs_csum_file_blocks(trans,
- BTRFS_I(inode)->root->fs_info->csum_root, sum);
+ ret = btrfs_csum_file_blocks(trans, trans->fs_info->csum_root, sum);
trans->adding_csums = false;
if (ret)
return ret;
unlock_extent_cached(&inode->io_tree, page_start, page_end,
&cached_state);
unlock_page(page);
- btrfs_start_ordered_extent(&inode->vfs_inode, ordered, 1);
+ btrfs_start_ordered_extent(ordered, 1);
btrfs_put_ordered_extent(ordered);
goto again;
}
}
static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans,
- struct inode *inode,
struct btrfs_ordered_extent *oe)
{
struct btrfs_file_extent_item stack_fi;
btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type);
/* Encryption and other encoding is reserved and all 0 */
- return insert_reserved_file_extent(trans, BTRFS_I(inode), oe->file_offset,
- &stack_fi, oe->qgroup_rsv);
+ return insert_reserved_file_extent(trans, BTRFS_I(oe->inode),
+ oe->file_offset, &stack_fi,
+ oe->qgroup_rsv);
}
/*
logical_len);
} else {
BUG_ON(root == fs_info->tree_root);
- ret = insert_ordered_extent_file_extent(trans, inode,
- ordered_extent);
+ ret = insert_ordered_extent_file_extent(trans, ordered_extent);
if (!ret) {
clear_reserved_extent = false;
btrfs_release_delalloc_bytes(fs_info,
goto out;
}
- ret = add_pending_csums(trans, inode, &ordered_extent->list);
+ ret = add_pending_csums(trans, &ordered_extent->list);
if (ret) {
btrfs_abort_transaction(trans, ret);
goto out;
* This needs to be done to make sure anybody waiting knows we are done
* updating everything for this ordered extent.
*/
- btrfs_remove_ordered_extent(inode, ordered_extent);
+ btrfs_remove_ordered_extent(BTRFS_I(inode), ordered_extent);
/* once for us */
btrfs_put_ordered_extent(ordered_extent);
void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start,
u64 end, int uptodate)
{
- struct inode *inode = page->mapping->host;
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct btrfs_ordered_extent *ordered_extent = NULL;
struct btrfs_workqueue *wq;
end - start + 1, uptodate))
return;
- if (btrfs_is_free_space_inode(BTRFS_I(inode)))
+ if (btrfs_is_free_space_inode(inode))
wq = fs_info->endio_freespace_worker;
else
wq = fs_info->endio_write_workers;
* if there's a match, we allow the bio to finish. If not, the code in
* extent_io.c will try to find good copies for us.
*/
-static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
- u64 phy_offset, struct page *page,
- u64 start, u64 end, int mirror)
+int btrfs_verify_data_csum(struct btrfs_io_bio *io_bio, u64 phy_offset,
+ struct page *page, u64 start, u64 end, int mirror)
{
size_t offset = start - page_offset(page);
struct inode *inode = page->mapping->host;
if (ret == -ENOENT && root == fs_info->tree_root) {
struct btrfs_root *dead_root;
- struct btrfs_fs_info *fs_info = root->fs_info;
int is_dead_root = 0;
/*
switch (inode->i_mode & S_IFMT) {
case S_IFREG:
inode->i_mapping->a_ops = &btrfs_aops;
- BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
inode->i_fop = &btrfs_file_operations;
inode->i_op = &btrfs_file_inode_operations;
break;
err = ret;
inode->i_flags |= S_DEAD;
out_release:
- btrfs_subvolume_release_metadata(fs_info, &block_rsv);
+ btrfs_subvolume_release_metadata(root, &block_rsv);
out_up_write:
up_write(&fs_info->subvol_sem);
if (err) {
&cached_state);
unlock_page(page);
put_page(page);
- btrfs_start_ordered_extent(inode, ordered, 1);
+ btrfs_start_ordered_extent(ordered, 1);
btrfs_put_ordered_extent(ordered);
goto again;
}
/*
* We're truncating a file that used to have good data down to
- * zero. Make sure it gets into the ordered flush list so that
- * any new writes get down to disk quickly.
+ * zero. Make sure any new writes to the file get on disk
+ * on close.
*/
if (newsize == 0)
- set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
+ set_bit(BTRFS_INODE_FLUSH_ON_CLOSE,
&BTRFS_I(inode)->runtime_flags);
truncate_setsize(inode, newsize);
- /* Disable nonlocked read DIO to avoid the endless truncate */
- btrfs_inode_block_unlocked_dio(BTRFS_I(inode));
inode_dio_wait(inode);
- btrfs_inode_resume_unlocked_dio(BTRFS_I(inode));
ret = btrfs_truncate(inode, newsize == oldsize);
if (ret && inode->i_nlink) {
spin_unlock(&root->inode_lock);
}
-static void inode_tree_del(struct inode *inode)
+static void inode_tree_del(struct btrfs_inode *inode)
{
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_root *root = inode->root;
int empty = 0;
spin_lock(&root->inode_lock);
- if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) {
- rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree);
- RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
+ if (!RB_EMPTY_NODE(&inode->rb_node)) {
+ rb_erase(&inode->rb_node, &root->inode_tree);
+ RB_CLEAR_NODE(&inode->rb_node);
empty = RB_EMPTY_ROOT(&root->inode_tree);
}
spin_unlock(&root->inode_lock);
if (err)
goto out_unlock;
- BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
d_instantiate_new(dentry, inode);
out_unlock:
drop_inode = 1;
} else {
struct dentry *parent = dentry->d_parent;
- int ret;
err = btrfs_update_inode(trans, root, inode);
if (err)
goto fail;
}
d_instantiate(dentry, inode);
- ret = btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent,
- true, NULL);
- if (ret == BTRFS_NEED_TRANS_COMMIT) {
- err = btrfs_commit_transaction(trans);
- trans = NULL;
- }
+ btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent);
}
fail:
u64 start, u64 len)
{
struct btrfs_fs_info *fs_info = inode->root->fs_info;
- int ret;
- int err = 0;
+ int ret = 0;
u64 extent_start = 0;
u64 extent_end = 0;
u64 objectid = btrfs_ino(inode);
}
em = alloc_extent_map();
if (!em) {
- err = -ENOMEM;
+ ret = -ENOMEM;
goto out;
}
em->start = EXTENT_MAP_HOLE;
path = btrfs_alloc_path();
if (!path) {
- err = -ENOMEM;
+ ret = -ENOMEM;
goto out;
}
*/
path->leave_spinning = 1;
+ path->recurse = btrfs_is_free_space_inode(inode);
+
ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0);
if (ret < 0) {
- err = ret;
goto out;
} else if (ret > 0) {
if (path->slots[0] == 0)
goto not_found;
path->slots[0]--;
+ ret = 0;
}
leaf = path->nodes[0];
extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
/* Only regular file could have regular/prealloc extent */
if (!S_ISREG(inode->vfs_inode.i_mode)) {
- err = -EUCLEAN;
+ ret = -EUCLEAN;
btrfs_crit(fs_info,
"regular/prealloc extent found for non-regular inode %llu",
btrfs_ino(inode));
path->slots[0]++;
if (path->slots[0] >= btrfs_header_nritems(leaf)) {
ret = btrfs_next_leaf(root, path);
- if (ret < 0) {
- err = ret;
+ if (ret < 0)
goto out;
- } else if (ret > 0) {
+ else if (ret > 0)
goto not_found;
- }
+
leaf = path->nodes[0];
}
btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
BTRFS_COMPRESS_NONE) {
ret = uncompress_inline(path, page, pg_offset,
extent_offset, item);
- if (ret) {
- err = ret;
+ if (ret)
goto out;
- }
} else {
map = kmap(page);
read_extent_buffer(leaf, map + pg_offset, ptr,
em->len = len;
em->block_start = EXTENT_MAP_HOLE;
insert:
+ ret = 0;
btrfs_release_path(path);
if (em->start > start || extent_map_end(em) <= start) {
btrfs_err(fs_info,
"bad extent! em: [%llu %llu] passed [%llu %llu]",
em->start, em->len, start, len);
- err = -EIO;
+ ret = -EIO;
goto out;
}
- err = 0;
write_lock(&em_tree->lock);
- err = btrfs_add_extent_mapping(fs_info, em_tree, &em, start, len);
+ ret = btrfs_add_extent_mapping(fs_info, em_tree, &em, start, len);
write_unlock(&em_tree->lock);
out:
btrfs_free_path(path);
trace_btrfs_get_extent(root, inode, em);
- if (err) {
+ if (ret) {
free_extent_map(em);
- return ERR_PTR(err);
+ return ERR_PTR(ret);
}
- BUG_ON(!em); /* Error is always set */
return em;
}
}
static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
- struct extent_state **cached_state, int writing)
+ struct extent_state **cached_state, bool writing)
{
struct btrfs_ordered_extent *ordered;
int ret = 0;
*/
if (writing ||
test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))
- btrfs_start_ordered_extent(inode, ordered, 1);
+ btrfs_start_ordered_extent(ordered, 1);
else
ret = -ENOTBLK;
btrfs_put_ordered_extent(ordered);
}
-static int btrfs_get_blocks_direct_read(struct extent_map *em,
- struct buffer_head *bh_result,
- struct inode *inode,
- u64 start, u64 len)
-{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
-
- if (em->block_start == EXTENT_MAP_HOLE ||
- test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
- return -ENOENT;
-
- len = min(len, em->len - (start - em->start));
-
- bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
- inode->i_blkbits;
- bh_result->b_size = len;
- bh_result->b_bdev = fs_info->fs_devices->latest_bdev;
- set_buffer_mapped(bh_result);
-
- return 0;
-}
-
static int btrfs_get_blocks_direct_write(struct extent_map **map,
- struct buffer_head *bh_result,
struct inode *inode,
struct btrfs_dio_data *dio_data,
u64 start, u64 len)
}
/* this will cow the extent */
- len = bh_result->b_size;
free_extent_map(em);
*map = em = btrfs_new_extent_direct(BTRFS_I(inode), start, len);
if (IS_ERR(em)) {
len = min(len, em->len - (start - em->start));
skip_cow:
- bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
- inode->i_blkbits;
- bh_result->b_size = len;
- bh_result->b_bdev = fs_info->fs_devices->latest_bdev;
- set_buffer_mapped(bh_result);
-
- if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
- set_buffer_new(bh_result);
-
/*
* Need to update the i_size under the extent lock so buffered
* readers will get the updated i_size when we unlock.
*/
- if (!dio_data->overwrite && start + len > i_size_read(inode))
+ if (start + len > i_size_read(inode))
i_size_write(inode, start + len);
- WARN_ON(dio_data->reserve < len);
dio_data->reserve -= len;
- dio_data->unsubmitted_oe_range_end = start + len;
- current->journal_info = dio_data;
out:
return ret;
}
-static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock,
- struct buffer_head *bh_result, int create)
+static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start,
+ loff_t length, unsigned int flags, struct iomap *iomap,
+ struct iomap *srcmap)
{
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
struct extent_map *em;
struct extent_state *cached_state = NULL;
struct btrfs_dio_data *dio_data = NULL;
- u64 start = iblock << inode->i_blkbits;
u64 lockstart, lockend;
- u64 len = bh_result->b_size;
+ const bool write = !!(flags & IOMAP_WRITE);
int ret = 0;
+ u64 len = length;
+ bool unlock_extents = false;
+ bool sync = (current->journal_info == BTRFS_DIO_SYNC_STUB);
+
+ /*
+ * We used current->journal_info here to see if we were sync, but
+ * there's a lot of tests in the enospc machinery to not do flushing if
+ * we have a journal_info set, so we need to clear this out and re-set
+ * it in iomap_end.
+ */
+ ASSERT(current->journal_info == NULL ||
+ current->journal_info == BTRFS_DIO_SYNC_STUB);
+ current->journal_info = NULL;
- if (!create)
+ if (!write)
len = min_t(u64, len, fs_info->sectorsize);
lockstart = start;
lockend = start + len - 1;
- if (current->journal_info) {
- /*
- * Need to pull our outstanding extents and set journal_info to NULL so
- * that anything that needs to check if there's a transaction doesn't get
- * confused.
- */
- dio_data = current->journal_info;
- current->journal_info = NULL;
+ /*
+ * The generic stuff only does filemap_write_and_wait_range, which
+ * isn't enough if we've written compressed pages to this area, so we
+ * need to flush the dirty pages again to make absolutely sure that any
+ * outstanding dirty pages are on disk.
+ */
+ if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
+ &BTRFS_I(inode)->runtime_flags)) {
+ ret = filemap_fdatawrite_range(inode->i_mapping, start,
+ start + length - 1);
+ if (ret)
+ return ret;
+ }
+
+ dio_data = kzalloc(sizeof(*dio_data), GFP_NOFS);
+ if (!dio_data)
+ return -ENOMEM;
+
+ dio_data->sync = sync;
+ dio_data->length = length;
+ if (write) {
+ dio_data->reserve = round_up(length, fs_info->sectorsize);
+ ret = btrfs_delalloc_reserve_space(BTRFS_I(inode),
+ &dio_data->data_reserved,
+ start, dio_data->reserve);
+ if (ret) {
+ extent_changeset_free(dio_data->data_reserved);
+ kfree(dio_data);
+ return ret;
+ }
}
+ iomap->private = dio_data;
+
/*
* If this errors out it's because we couldn't invalidate pagecache for
* this range and we need to fallback to buffered.
*/
- if (lock_extent_direct(inode, lockstart, lockend, &cached_state,
- create)) {
+ if (lock_extent_direct(inode, lockstart, lockend, &cached_state, write)) {
ret = -ENOTBLK;
goto err;
}
goto unlock_err;
}
- if (create) {
- ret = btrfs_get_blocks_direct_write(&em, bh_result, inode,
- dio_data, start, len);
+ len = min(len, em->len - (start - em->start));
+ if (write) {
+ ret = btrfs_get_blocks_direct_write(&em, inode, dio_data,
+ start, len);
if (ret < 0)
goto unlock_err;
-
- unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
- lockend, &cached_state);
+ unlock_extents = true;
+ /* Recalc len in case the new em is smaller than requested */
+ len = min(len, em->len - (start - em->start));
} else {
- ret = btrfs_get_blocks_direct_read(em, bh_result, inode,
- start, len);
- /* Can be negative only if we read from a hole */
- if (ret < 0) {
- ret = 0;
- free_extent_map(em);
- goto unlock_err;
- }
/*
* We need to unlock only the end area that we aren't using.
* The rest is going to be unlocked by the endio routine.
*/
- lockstart = start + bh_result->b_size;
- if (lockstart < lockend) {
- unlock_extent_cached(&BTRFS_I(inode)->io_tree,
- lockstart, lockend, &cached_state);
- } else {
- free_extent_state(cached_state);
- }
+ lockstart = start + len;
+ if (lockstart < lockend)
+ unlock_extents = true;
+ }
+
+ if (unlock_extents)
+ unlock_extent_cached(&BTRFS_I(inode)->io_tree,
+ lockstart, lockend, &cached_state);
+ else
+ free_extent_state(cached_state);
+
+ /*
+ * Translate extent map information to iomap.
+ * We trim the extents (and move the addr) even though iomap code does
+ * that, since we have locked only the parts we are performing I/O in.
+ */
+ if ((em->block_start == EXTENT_MAP_HOLE) ||
+ (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) && !write)) {
+ iomap->addr = IOMAP_NULL_ADDR;
+ iomap->type = IOMAP_HOLE;
+ } else {
+ iomap->addr = em->block_start + (start - em->start);
+ iomap->type = IOMAP_MAPPED;
}
+ iomap->offset = start;
+ iomap->bdev = fs_info->fs_devices->latest_bdev;
+ iomap->length = len;
free_extent_map(em);
unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
&cached_state);
err:
- if (dio_data)
- current->journal_info = dio_data;
+ if (dio_data) {
+ btrfs_delalloc_release_space(BTRFS_I(inode),
+ dio_data->data_reserved, start,
+ dio_data->reserve, true);
+ btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->reserve);
+ extent_changeset_free(dio_data->data_reserved);
+ kfree(dio_data);
+ }
+ return ret;
+}
+
+static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length,
+ ssize_t written, unsigned int flags, struct iomap *iomap)
+{
+ int ret = 0;
+ struct btrfs_dio_data *dio_data = iomap->private;
+ size_t submitted = dio_data->submitted;
+ const bool write = !!(flags & IOMAP_WRITE);
+
+ if (!write && (iomap->type == IOMAP_HOLE)) {
+ /* If reading from a hole, unlock and return */
+ unlock_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1);
+ goto out;
+ }
+
+ if (submitted < length) {
+ pos += submitted;
+ length -= submitted;
+ if (write)
+ __endio_write_update_ordered(BTRFS_I(inode), pos,
+ length, false);
+ else
+ unlock_extent(&BTRFS_I(inode)->io_tree, pos,
+ pos + length - 1);
+ ret = -ENOTBLK;
+ }
+
+ if (write) {
+ if (dio_data->reserve)
+ btrfs_delalloc_release_space(BTRFS_I(inode),
+ dio_data->data_reserved, pos,
+ dio_data->reserve, true);
+ btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->length);
+ extent_changeset_free(dio_data->data_reserved);
+ }
+out:
+ /*
+ * We're all done, we can re-set the current->journal_info now safely
+ * for our endio.
+ */
+ if (dio_data->sync) {
+ ASSERT(current->journal_info == NULL);
+ current->journal_info = BTRFS_DIO_SYNC_STUB;
+ }
+ kfree(dio_data);
+ iomap->private = NULL;
+
return ret;
}
dip->logical_offset + dip->bytes - 1);
}
- dio_end_io(dip->dio_bio);
+ bio_endio(dip->dio_bio);
kfree(dip);
}
dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
dip->dio_bio = dio_bio;
refcount_set(&dip->refs, 1);
-
- if (write) {
- struct btrfs_dio_data *dio_data = current->journal_info;
-
- /*
- * Setting range start and end to the same value means that
- * no cleanup will happen in btrfs_direct_IO
- */
- dio_data->unsubmitted_oe_range_end = dip->logical_offset +
- dip->bytes;
- dio_data->unsubmitted_oe_range_start =
- dio_data->unsubmitted_oe_range_end;
- }
return dip;
}
-static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode,
- loff_t file_offset)
+static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap,
+ struct bio *dio_bio, loff_t file_offset)
{
const bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM);
int ret;
blk_status_t status;
struct btrfs_io_geometry geom;
+ struct btrfs_dio_data *dio_data = iomap->private;
dip = btrfs_create_dio_private(dio_bio, inode, file_offset);
if (!dip) {
file_offset + dio_bio->bi_iter.bi_size - 1);
}
dio_bio->bi_status = BLK_STS_RESOURCE;
- dio_end_io(dio_bio);
- return;
+ bio_endio(dio_bio);
+ return BLK_QC_T_NONE;
}
if (!write && csum) {
goto out_err;
}
+ dio_data->submitted += clone_len;
clone_offset += clone_len;
start_sector += clone_len >> 9;
file_offset += clone_len;
} while (submit_len > 0);
- return;
+ return BLK_QC_T_NONE;
out_err:
dip->dio_bio->bi_status = status;
btrfs_dio_private_put(dip);
+ return BLK_QC_T_NONE;
}
static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
return retval;
}
-static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
+static inline int btrfs_maybe_fsync_end_io(struct kiocb *iocb, ssize_t size,
+ int error, unsigned flags)
+{
+ /*
+ * Now if we're still in the context of our submitter we know we can't
+ * safely run generic_write_sync(), so clear our flag here so that the
+ * caller knows to follow up with a sync.
+ */
+ if (current->journal_info == BTRFS_DIO_SYNC_STUB) {
+ current->journal_info = NULL;
+ return error;
+ }
+
+ if (error)
+ return error;
+
+ if (size) {
+ iocb->ki_flags |= IOCB_DSYNC;
+ return generic_write_sync(iocb, size);
+ }
+
+ return 0;
+}
+
+static const struct iomap_ops btrfs_dio_iomap_ops = {
+ .iomap_begin = btrfs_dio_iomap_begin,
+ .iomap_end = btrfs_dio_iomap_end,
+};
+
+static const struct iomap_dio_ops btrfs_dio_ops = {
+ .submit_io = btrfs_submit_direct,
+};
+
+static const struct iomap_dio_ops btrfs_sync_dops = {
+ .submit_io = btrfs_submit_direct,
+ .end_io = btrfs_maybe_fsync_end_io,
+};
+
+ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
{
struct file *file = iocb->ki_filp;
struct inode *inode = file->f_mapping->host;
struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
- struct btrfs_dio_data dio_data = { 0 };
struct extent_changeset *data_reserved = NULL;
loff_t offset = iocb->ki_pos;
size_t count = 0;
- int flags = 0;
- bool wakeup = true;
bool relock = false;
ssize_t ret;
if (check_direct_IO(fs_info, iter, offset))
return 0;
- inode_dio_begin(inode);
-
- /*
- * The generic stuff only does filemap_write_and_wait_range, which
- * isn't enough if we've written compressed pages to this area, so
- * we need to flush the dirty pages again to make absolutely sure
- * that any outstanding dirty pages are on disk.
- */
count = iov_iter_count(iter);
- if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
- &BTRFS_I(inode)->runtime_flags))
- filemap_fdatawrite_range(inode->i_mapping, offset,
- offset + count - 1);
-
if (iov_iter_rw(iter) == WRITE) {
/*
* If the write DIO is beyond the EOF, we need update
* not unlock the i_mutex at this case.
*/
if (offset + count <= inode->i_size) {
- dio_data.overwrite = 1;
inode_unlock(inode);
relock = true;
}
- ret = btrfs_delalloc_reserve_space(BTRFS_I(inode), &data_reserved,
- offset, count);
- if (ret)
- goto out;
-
- /*
- * We need to know how many extents we reserved so that we can
- * do the accounting properly if we go over the number we
- * originally calculated. Abuse current->journal_info for this.
- */
- dio_data.reserve = round_up(count,
- fs_info->sectorsize);
- dio_data.unsubmitted_oe_range_start = (u64)offset;
- dio_data.unsubmitted_oe_range_end = (u64)offset;
- current->journal_info = &dio_data;
down_read(&BTRFS_I(inode)->dio_sem);
- } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
- &BTRFS_I(inode)->runtime_flags)) {
- inode_dio_end(inode);
- flags = DIO_LOCKING | DIO_SKIP_HOLES;
- wakeup = false;
}
- ret = __blockdev_direct_IO(iocb, inode,
- fs_info->fs_devices->latest_bdev,
- iter, btrfs_get_blocks_direct, NULL,
- btrfs_submit_direct, flags);
- if (iov_iter_rw(iter) == WRITE) {
+ /*
+ * We have are actually a sync iocb, so we need our fancy endio to know
+ * if we need to sync.
+ */
+ if (current->journal_info)
+ ret = iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops,
+ &btrfs_sync_dops, is_sync_kiocb(iocb));
+ else
+ ret = iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops,
+ &btrfs_dio_ops, is_sync_kiocb(iocb));
+
+ if (ret == -ENOTBLK)
+ ret = 0;
+
+ if (iov_iter_rw(iter) == WRITE)
up_read(&BTRFS_I(inode)->dio_sem);
- current->journal_info = NULL;
- if (ret < 0 && ret != -EIOCBQUEUED) {
- if (dio_data.reserve)
- btrfs_delalloc_release_space(BTRFS_I(inode),
- data_reserved, offset, dio_data.reserve,
- true);
- /*
- * On error we might have left some ordered extents
- * without submitting corresponding bios for them, so
- * cleanup them up to avoid other tasks getting them
- * and waiting for them to complete forever.
- */
- if (dio_data.unsubmitted_oe_range_start <
- dio_data.unsubmitted_oe_range_end)
- __endio_write_update_ordered(BTRFS_I(inode),
- dio_data.unsubmitted_oe_range_start,
- dio_data.unsubmitted_oe_range_end -
- dio_data.unsubmitted_oe_range_start,
- false);
- } else if (ret >= 0 && (size_t)ret < count)
- btrfs_delalloc_release_space(BTRFS_I(inode), data_reserved,
- offset, count - (size_t)ret, true);
- btrfs_delalloc_release_extents(BTRFS_I(inode), count);
- }
-out:
- if (wakeup)
- inode_dio_end(inode);
+
if (relock)
inode_lock(inode);
if (ret)
return ret;
- return extent_fiemap(inode, fieinfo, start, len);
+ return extent_fiemap(BTRFS_I(inode), fieinfo, start, len);
}
int btrfs_readpage(struct file *file, struct page *page)
{
- return extent_read_full_page(page, btrfs_get_extent, 0);
+ struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
+ u64 start = page_offset(page);
+ u64 end = start + PAGE_SIZE - 1;
+ unsigned long bio_flags = 0;
+ struct bio *bio = NULL;
+ int ret;
+
+ btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
+
+ ret = btrfs_do_readpage(page, NULL, &bio, &bio_flags, 0, NULL);
+ if (bio)
+ ret = submit_one_bio(bio, 0, bio_flags);
+ return ret;
}
static int btrfs_writepage(struct page *page, struct writeback_control *wbc)
static void btrfs_invalidatepage(struct page *page, unsigned int offset,
unsigned int length)
{
- struct inode *inode = page->mapping->host;
- struct extent_io_tree *tree;
+ struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
+ struct extent_io_tree *tree = &inode->io_tree;
struct btrfs_ordered_extent *ordered;
struct extent_state *cached_state = NULL;
u64 page_start = page_offset(page);
u64 page_end = page_start + PAGE_SIZE - 1;
u64 start;
u64 end;
- int inode_evicting = inode->i_state & I_FREEING;
+ int inode_evicting = inode->vfs_inode.i_state & I_FREEING;
/*
* we have the page locked, so new writeback can't start,
*/
wait_on_page_writeback(page);
- tree = &BTRFS_I(inode)->io_tree;
if (offset) {
btrfs_releasepage(page, GFP_NOFS);
return;
lock_extent_bits(tree, page_start, page_end, &cached_state);
again:
start = page_start;
- ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
- page_end - start + 1);
+ ordered = btrfs_lookup_ordered_range(inode, start, page_end - start + 1);
if (ordered) {
end = min(page_end,
ordered->file_offset + ordered->num_bytes - 1);
struct btrfs_ordered_inode_tree *tree;
u64 new_len;
- tree = &BTRFS_I(inode)->ordered_tree;
+ tree = &inode->ordered_tree;
spin_lock_irq(&tree->lock);
set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
* bit of its io_tree, and free the qgroup reserved data space.
* Since the IO will never happen for this page.
*/
- btrfs_qgroup_free_data(BTRFS_I(inode), NULL, page_start, PAGE_SIZE);
+ btrfs_qgroup_free_data(inode, NULL, page_start, PAGE_SIZE);
if (!inode_evicting) {
clear_extent_bit(tree, page_start, page_end, EXTENT_LOCKED |
EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
unlock_extent_cached(io_tree, page_start, page_end,
&cached_state);
unlock_page(page);
- btrfs_start_ordered_extent(inode, ordered, 1);
+ btrfs_start_ordered_extent(ordered, 1);
btrfs_put_ordered_extent(ordered);
goto again;
}
kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
}
-void btrfs_destroy_inode(struct inode *inode)
+void btrfs_destroy_inode(struct inode *vfs_inode)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
struct btrfs_ordered_extent *ordered;
- struct btrfs_root *root = BTRFS_I(inode)->root;
+ struct btrfs_inode *inode = BTRFS_I(vfs_inode);
+ struct btrfs_root *root = inode->root;
- WARN_ON(!hlist_empty(&inode->i_dentry));
- WARN_ON(inode->i_data.nrpages);
- WARN_ON(BTRFS_I(inode)->block_rsv.reserved);
- WARN_ON(BTRFS_I(inode)->block_rsv.size);
- WARN_ON(BTRFS_I(inode)->outstanding_extents);
- WARN_ON(BTRFS_I(inode)->delalloc_bytes);
- WARN_ON(BTRFS_I(inode)->new_delalloc_bytes);
- WARN_ON(BTRFS_I(inode)->csum_bytes);
- WARN_ON(BTRFS_I(inode)->defrag_bytes);
+ WARN_ON(!hlist_empty(&vfs_inode->i_dentry));
+ WARN_ON(vfs_inode->i_data.nrpages);
+ WARN_ON(inode->block_rsv.reserved);
+ WARN_ON(inode->block_rsv.size);
+ WARN_ON(inode->outstanding_extents);
+ WARN_ON(inode->delalloc_bytes);
+ WARN_ON(inode->new_delalloc_bytes);
+ WARN_ON(inode->csum_bytes);
+ WARN_ON(inode->defrag_bytes);
/*
* This can happen where we create an inode, but somebody else also
if (!ordered)
break;
else {
- btrfs_err(fs_info,
+ btrfs_err(root->fs_info,
"found ordered extent %llu %llu on inode cleanup",
ordered->file_offset, ordered->num_bytes);
btrfs_remove_ordered_extent(inode, ordered);
btrfs_put_ordered_extent(ordered);
}
}
- btrfs_qgroup_check_reserved_leak(BTRFS_I(inode));
+ btrfs_qgroup_check_reserved_leak(inode);
inode_tree_del(inode);
- btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
- btrfs_inode_clear_file_extent_range(BTRFS_I(inode), 0, (u64)-1);
- btrfs_put_root(BTRFS_I(inode)->root);
+ btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
+ btrfs_inode_clear_file_extent_range(inode, 0, (u64)-1);
+ btrfs_put_root(inode->root);
}
int btrfs_drop_inode(struct inode *inode)
struct inode *new_inode = new_dentry->d_inode;
struct inode *old_inode = old_dentry->d_inode;
struct timespec64 ctime = current_time(old_inode);
- struct dentry *parent;
u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
u64 new_ino = btrfs_ino(BTRFS_I(new_inode));
u64 old_idx = 0;
u64 new_idx = 0;
int ret;
+ int ret2;
bool root_log_pinned = false;
bool dest_log_pinned = false;
- struct btrfs_log_ctx ctx_root;
- struct btrfs_log_ctx ctx_dest;
- bool sync_log_root = false;
- bool sync_log_dest = false;
- bool commit_transaction = false;
/* we only allow rename subvolume link between subvolumes */
if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
return -EXDEV;
- btrfs_init_log_ctx(&ctx_root, old_inode);
- btrfs_init_log_ctx(&ctx_dest, new_inode);
-
/* close the race window with snapshot create/destroy ioctl */
if (old_ino == BTRFS_FIRST_FREE_OBJECTID ||
new_ino == BTRFS_FIRST_FREE_OBJECTID)
BTRFS_I(new_inode)->dir_index = new_idx;
if (root_log_pinned) {
- parent = new_dentry->d_parent;
- ret = btrfs_log_new_name(trans, BTRFS_I(old_inode),
- BTRFS_I(old_dir), parent,
- false, &ctx_root);
- if (ret == BTRFS_NEED_LOG_SYNC)
- sync_log_root = true;
- else if (ret == BTRFS_NEED_TRANS_COMMIT)
- commit_transaction = true;
- ret = 0;
+ btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
+ new_dentry->d_parent);
btrfs_end_log_trans(root);
root_log_pinned = false;
}
if (dest_log_pinned) {
- if (!commit_transaction) {
- parent = old_dentry->d_parent;
- ret = btrfs_log_new_name(trans, BTRFS_I(new_inode),
- BTRFS_I(new_dir), parent,
- false, &ctx_dest);
- if (ret == BTRFS_NEED_LOG_SYNC)
- sync_log_dest = true;
- else if (ret == BTRFS_NEED_TRANS_COMMIT)
- commit_transaction = true;
- ret = 0;
- }
+ btrfs_log_new_name(trans, BTRFS_I(new_inode), BTRFS_I(new_dir),
+ old_dentry->d_parent);
btrfs_end_log_trans(dest);
dest_log_pinned = false;
}
dest_log_pinned = false;
}
}
- if (!ret && sync_log_root && !commit_transaction) {
- ret = btrfs_sync_log(trans, BTRFS_I(old_inode)->root,
- &ctx_root);
- if (ret)
- commit_transaction = true;
- }
- if (!ret && sync_log_dest && !commit_transaction) {
- ret = btrfs_sync_log(trans, BTRFS_I(new_inode)->root,
- &ctx_dest);
- if (ret)
- commit_transaction = true;
- }
- if (commit_transaction) {
- /*
- * We may have set commit_transaction when logging the new name
- * in the destination root, in which case we left the source
- * root context in the list of log contextes. So make sure we
- * remove it to avoid invalid memory accesses, since the context
- * was allocated in our stack frame.
- */
- if (sync_log_root) {
- mutex_lock(&root->log_mutex);
- list_del_init(&ctx_root.list);
- mutex_unlock(&root->log_mutex);
- }
- ret = btrfs_commit_transaction(trans);
- } else {
- int ret2;
-
- ret2 = btrfs_end_transaction(trans);
- ret = ret ? ret : ret2;
- }
+ ret2 = btrfs_end_transaction(trans);
+ ret = ret ? ret : ret2;
out_notrans:
if (new_ino == BTRFS_FIRST_FREE_OBJECTID ||
old_ino == BTRFS_FIRST_FREE_OBJECTID)
up_read(&fs_info->subvol_sem);
- ASSERT(list_empty(&ctx_root.list));
- ASSERT(list_empty(&ctx_dest.list));
-
return ret;
}
struct inode *old_inode = d_inode(old_dentry);
u64 index = 0;
int ret;
+ int ret2;
u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
bool log_pinned = false;
- struct btrfs_log_ctx ctx;
- bool sync_log = false;
- bool commit_transaction = false;
if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
return -EPERM;
BTRFS_I(old_inode)->dir_index = index;
if (log_pinned) {
- struct dentry *parent = new_dentry->d_parent;
-
- btrfs_init_log_ctx(&ctx, old_inode);
- ret = btrfs_log_new_name(trans, BTRFS_I(old_inode),
- BTRFS_I(old_dir), parent,
- false, &ctx);
- if (ret == BTRFS_NEED_LOG_SYNC)
- sync_log = true;
- else if (ret == BTRFS_NEED_TRANS_COMMIT)
- commit_transaction = true;
- ret = 0;
+ btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
+ new_dentry->d_parent);
btrfs_end_log_trans(root);
log_pinned = false;
}
btrfs_end_log_trans(root);
log_pinned = false;
}
- if (!ret && sync_log) {
- ret = btrfs_sync_log(trans, BTRFS_I(old_inode)->root, &ctx);
- if (ret)
- commit_transaction = true;
- } else if (sync_log) {
- mutex_lock(&root->log_mutex);
- list_del(&ctx.list);
- mutex_unlock(&root->log_mutex);
- }
- if (commit_transaction) {
- ret = btrfs_commit_transaction(trans);
- } else {
- int ret2;
-
- ret2 = btrfs_end_transaction(trans);
- ret = ret ? ret : ret2;
- }
+ ret2 = btrfs_end_transaction(trans);
+ ret = ret ? ret : ret2;
out_notrans:
if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
up_read(&fs_info->subvol_sem);
* some fairly slow code that needs optimization. This walks the list
* of all the inodes with pending delalloc and forces them to disk.
*/
-static int start_delalloc_inodes(struct btrfs_root *root, int nr, bool snapshot)
+static int start_delalloc_inodes(struct btrfs_root *root, u64 *nr, bool snapshot)
{
struct btrfs_inode *binode;
struct inode *inode;
list_add_tail(&work->list, &works);
btrfs_queue_work(root->fs_info->flush_workers,
&work->work);
- ret++;
- if (nr != -1 && ret >= nr)
- goto out;
+ if (*nr != U64_MAX) {
+ (*nr)--;
+ if (*nr == 0)
+ goto out;
+ }
cond_resched();
spin_lock(&root->delalloc_lock);
}
int btrfs_start_delalloc_snapshot(struct btrfs_root *root)
{
struct btrfs_fs_info *fs_info = root->fs_info;
- int ret;
+ u64 nr = U64_MAX;
if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
return -EROFS;
- ret = start_delalloc_inodes(root, -1, true);
- if (ret > 0)
- ret = 0;
- return ret;
+ return start_delalloc_inodes(root, &nr, true);
}
-int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int nr)
+int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, u64 nr)
{
struct btrfs_root *root;
struct list_head splice;
&fs_info->delalloc_roots);
spin_unlock(&fs_info->delalloc_root_lock);
- ret = start_delalloc_inodes(root, nr, false);
+ ret = start_delalloc_inodes(root, &nr, false);
btrfs_put_root(root);
if (ret < 0)
goto out;
-
- if (nr != -1) {
- nr -= ret;
- WARN_ON(nr < 0);
- }
spin_lock(&fs_info->delalloc_root_lock);
}
spin_unlock(&fs_info->delalloc_root_lock);
inode->i_fop = &btrfs_file_operations;
inode->i_op = &btrfs_file_inode_operations;
inode->i_mapping->a_ops = &btrfs_aops;
- BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
if (err)
return err;
}
-static int insert_prealloc_file_extent(struct btrfs_trans_handle *trans,
+static struct btrfs_trans_handle *insert_prealloc_file_extent(
+ struct btrfs_trans_handle *trans_in,
struct inode *inode, struct btrfs_key *ins,
u64 file_offset)
{
struct btrfs_file_extent_item stack_fi;
+ struct btrfs_replace_extent_info extent_info;
+ struct btrfs_trans_handle *trans = trans_in;
+ struct btrfs_path *path;
u64 start = ins->objectid;
u64 len = ins->offset;
int ret;
ret = btrfs_qgroup_release_data(BTRFS_I(inode), file_offset, len);
if (ret < 0)
- return ret;
- return insert_reserved_file_extent(trans, BTRFS_I(inode), file_offset,
- &stack_fi, ret);
+ return ERR_PTR(ret);
+
+ if (trans) {
+ ret = insert_reserved_file_extent(trans, BTRFS_I(inode),
+ file_offset, &stack_fi, ret);
+ if (ret)
+ return ERR_PTR(ret);
+ return trans;
+ }
+
+ extent_info.disk_offset = start;
+ extent_info.disk_len = len;
+ extent_info.data_offset = 0;
+ extent_info.data_len = len;
+ extent_info.file_offset = file_offset;
+ extent_info.extent_buf = (char *)&stack_fi;
+ extent_info.is_new_extent = true;
+ extent_info.qgroup_reserved = ret;
+ extent_info.insertions = 0;
+
+ path = btrfs_alloc_path();
+ if (!path)
+ return ERR_PTR(-ENOMEM);
+
+ ret = btrfs_replace_file_extents(inode, path, file_offset,
+ file_offset + len - 1, &extent_info,
+ &trans);
+ btrfs_free_path(path);
+ if (ret)
+ return ERR_PTR(ret);
+
+ return trans;
}
+
static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
u64 start, u64 num_bytes, u64 min_size,
loff_t actual_len, u64 *alloc_hint,
if (trans)
own_trans = false;
while (num_bytes > 0) {
- if (own_trans) {
- trans = btrfs_start_transaction(root, 3);
- if (IS_ERR(trans)) {
- ret = PTR_ERR(trans);
- break;
- }
- }
-
cur_bytes = min_t(u64, num_bytes, SZ_256M);
cur_bytes = max(cur_bytes, min_size);
/*
cur_bytes = min(cur_bytes, last_alloc);
ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
min_size, 0, *alloc_hint, &ins, 1, 0);
- if (ret) {
- if (own_trans)
- btrfs_end_transaction(trans);
+ if (ret)
break;
- }
/*
* We've reserved this space, and thus converted it from
btrfs_dec_block_group_reservations(fs_info, ins.objectid);
last_alloc = ins.offset;
- ret = insert_prealloc_file_extent(trans, inode, &ins, cur_offset);
- if (ret) {
+ trans = insert_prealloc_file_extent(trans, inode, &ins, cur_offset);
+ if (IS_ERR(trans)) {
+ ret = PTR_ERR(trans);
btrfs_free_reserved_extent(fs_info, ins.objectid,
ins.offset, 0);
- btrfs_abort_transaction(trans, ret);
- if (own_trans)
- btrfs_end_transaction(trans);
break;
}
break;
}
- if (own_trans)
+ if (own_trans) {
btrfs_end_transaction(trans);
+ trans = NULL;
+ }
}
if (clear_offset < end)
btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, clear_offset,
inode->i_op = &btrfs_file_inode_operations;
inode->i_mapping->a_ops = &btrfs_aops;
- BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
ret = btrfs_init_inode_security(trans, inode, dir, NULL);
if (ret)
/*
* Balance or device remove/replace/resize can move stuff around from
- * under us. The EXCL_OP flag makes sure they aren't running/won't run
- * concurrently while we are mapping the swap extents, and
- * fs_info->swapfile_pins prevents them from running while the swap file
- * is active and moving the extents. Note that this also prevents a
- * concurrent device add which isn't actually necessary, but it's not
+ * under us. The exclop protection makes sure they aren't running/won't
+ * run concurrently while we are mapping the swap extents, and
+ * fs_info->swapfile_pins prevents them from running while the swap
+ * file is active and moving the extents. Note that this also prevents
+ * a concurrent device add which isn't actually necessary, but it's not
* really worth the trouble to allow it.
*/
- if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags)) {
+ if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_SWAP_ACTIVATE)) {
btrfs_warn(fs_info,
"cannot activate swapfile while exclusive operation is running");
return -EBUSY;
if (ret)
btrfs_swap_deactivate(file);
- clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
+ btrfs_exclop_finish(fs_info);
if (ret)
return ret;
.fsync = btrfs_sync_file,
};
-static const struct extent_io_ops btrfs_extent_io_ops = {
- /* mandatory callbacks */
- .submit_bio_hook = btrfs_submit_bio_hook,
- .readpage_end_io_hook = btrfs_readpage_end_io_hook,
-};
-
/*
* btrfs doesn't support the bmap operation because swapfiles
* use bmap to make a mapping of extents in the file. They assume
.writepage = btrfs_writepage,
.writepages = btrfs_writepages,
.readahead = btrfs_readahead,
- .direct_IO = btrfs_direct_IO,
+ .direct_IO = noop_direct_IO,
.invalidatepage = btrfs_invalidatepage,
.releasepage = btrfs_releasepage,
#ifdef CONFIG_MIGRATION
return 0;
}
+bool btrfs_exclop_start(struct btrfs_fs_info *fs_info,
+ enum btrfs_exclusive_operation type)
+{
+ return !cmpxchg(&fs_info->exclusive_operation, BTRFS_EXCLOP_NONE, type);
+}
+
+void btrfs_exclop_finish(struct btrfs_fs_info *fs_info)
+{
+ WRITE_ONCE(fs_info->exclusive_operation, BTRFS_EXCLOP_NONE);
+ sysfs_notify(&fs_info->fs_devices->fsid_kobj, NULL, "exclusive_operation");
+}
+
/*
* Set the xflags from the internal inode flags. The remaining items of fsxattr
* are zeroed.
trans = btrfs_start_transaction(root, 0);
if (IS_ERR(trans)) {
ret = PTR_ERR(trans);
- btrfs_subvolume_release_metadata(fs_info, &block_rsv);
+ btrfs_subvolume_release_metadata(root, &block_rsv);
goto fail_free;
}
trans->block_rsv = &block_rsv;
if (ret)
goto fail;
- leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0);
+ leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0,
+ BTRFS_NESTING_NORMAL);
if (IS_ERR(leaf)) {
ret = PTR_ERR(leaf);
goto fail;
kfree(root_item);
trans->block_rsv = NULL;
trans->bytes_reserved = 0;
- btrfs_subvolume_release_metadata(fs_info, &block_rsv);
+ btrfs_subvolume_release_metadata(root, &block_rsv);
err = btrfs_commit_transaction(trans);
if (err && !ret)
if (ret && pending_snapshot->snap)
pending_snapshot->snap->anon_dev = 0;
btrfs_put_root(pending_snapshot->snap);
- btrfs_subvolume_release_metadata(fs_info, &pending_snapshot->block_rsv);
+ btrfs_subvolume_release_metadata(root, &pending_snapshot->block_rsv);
free_pending:
if (pending_snapshot->anon_dev)
free_anon_bdev(pending_snapshot->anon_dev);
break;
unlock_page(page);
- btrfs_start_ordered_extent(inode, ordered, 1);
+ btrfs_start_ordered_extent(ordered, 1);
btrfs_put_ordered_extent(ordered);
lock_page(page);
/*
if (ret)
return ret;
- if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags)) {
+ if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_RESIZE)) {
mnt_drop_write_file(file);
return BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS;
}
out_free:
kfree(vol_args);
out:
- clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
+ btrfs_exclop_finish(fs_info);
mnt_drop_write_file(file);
return ret;
}
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
- if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags))
+ if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_ADD))
return BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS;
vol_args = memdup_user(arg, sizeof(*vol_args));
kfree(vol_args);
out:
- clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
+ btrfs_exclop_finish(fs_info);
return ret;
}
goto out;
}
- if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags)) {
+ if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REMOVE)) {
ret = BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS;
goto out;
}
vol_args->name[BTRFS_SUBVOL_NAME_MAX] = '\0';
ret = btrfs_rm_device(fs_info, vol_args->name, 0);
}
- clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
+ btrfs_exclop_finish(fs_info);
if (!ret) {
if (vol_args->flags & BTRFS_DEVICE_SPEC_BY_ID)
if (ret)
return ret;
- if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags)) {
+ if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REMOVE)) {
ret = BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS;
goto out_drop_write;
}
btrfs_info(fs_info, "disk deleted %s", vol_args->name);
kfree(vol_args);
out:
- clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
+ btrfs_exclop_finish(fs_info);
out_drop_write:
mnt_drop_write_file(file);
struct btrfs_space_info *tmp;
info = NULL;
- rcu_read_lock();
- list_for_each_entry_rcu(tmp, &fs_info->space_info,
- list) {
+ list_for_each_entry(tmp, &fs_info->space_info, list) {
if (tmp->flags == types[i]) {
info = tmp;
break;
}
}
- rcu_read_unlock();
if (!info)
continue;
break;
info = NULL;
- rcu_read_lock();
- list_for_each_entry_rcu(tmp, &fs_info->space_info,
- list) {
+ list_for_each_entry(tmp, &fs_info->space_info, list) {
if (tmp->flags == types[i]) {
info = tmp;
break;
}
}
- rcu_read_unlock();
if (!info)
continue;
ret = -EROFS;
goto out;
}
- if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags)) {
+ if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) {
ret = BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS;
} else {
ret = btrfs_dev_replace_by_ioctl(fs_info, p);
- clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
+ btrfs_exclop_finish(fs_info);
}
break;
case BTRFS_IOCTL_DEV_REPLACE_CMD_STATUS:
return ret;
again:
- if (!test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags)) {
+ if (btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE)) {
mutex_lock(&fs_info->balance_mutex);
need_unlock = true;
goto locked;
}
locked:
- BUG_ON(!test_bit(BTRFS_FS_EXCL_OP, &fs_info->flags));
if (arg) {
bargs = memdup_user(arg, sizeof(*bargs));
do_balance:
/*
- * Ownership of bctl and filesystem flag BTRFS_FS_EXCL_OP goes to
- * btrfs_balance. bctl is freed in reset_balance_state, or, if
- * restriper was paused all the way until unmount, in free_fs_info.
- * The flag should be cleared after reset_balance_state.
+ * Ownership of bctl and exclusive operation goes to btrfs_balance.
+ * bctl is freed in reset_balance_state, or, if restriper was paused
+ * all the way until unmount, in free_fs_info. The flag should be
+ * cleared after reset_balance_state.
*/
need_unlock = false;
out_unlock:
mutex_unlock(&fs_info->balance_mutex);
if (need_unlock)
- clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
+ btrfs_exclop_finish(fs_info);
out:
mnt_drop_write_file(file);
return ret;
case BTRFS_IOC_SYNC: {
int ret;
- ret = btrfs_start_delalloc_roots(fs_info, -1);
+ ret = btrfs_start_delalloc_roots(fs_info, U64_MAX);
if (ret)
return ret;
ret = btrfs_sync_fs(inode->i_sb, 1);
* performance reasons.
*
*
- * Lock nesting
- * ------------
+ * Lock recursion
+ * --------------
*
* A write operation on a tree might indirectly start a look up on the same
* tree. This can happen when btrfs_cow_block locks the tree and needs to
* lock, but it won't change to or away from us. If we have the write
* lock, we are the owner and it'll never change.
*/
- if (eb->lock_nested && current->pid == eb->lock_owner)
+ if (eb->lock_recursed && current->pid == eb->lock_owner)
return;
btrfs_assert_tree_read_locked(eb);
atomic_inc(&eb->blocking_readers);
* lock, but it won't change to or away from us. If we have the write
* lock, we are the owner and it'll never change.
*/
- if (eb->lock_nested && current->pid == eb->lock_owner)
+ if (eb->lock_recursed && current->pid == eb->lock_owner)
return;
if (eb->blocking_writers == 0) {
btrfs_assert_spinning_writers_put(eb);
*
* The rwlock is held upon exit.
*/
-void btrfs_tree_read_lock(struct extent_buffer *eb)
+void __btrfs_tree_read_lock(struct extent_buffer *eb, enum btrfs_lock_nesting nest,
+ bool recurse)
{
u64 start_ns = 0;
* depends on this as it may be called on a partly
* (write-)locked tree.
*/
- BUG_ON(eb->lock_nested);
- eb->lock_nested = true;
+ WARN_ON(!recurse);
+ BUG_ON(eb->lock_recursed);
+ eb->lock_recursed = true;
read_unlock(&eb->lock);
trace_btrfs_tree_read_lock(eb, start_ns);
return;
trace_btrfs_tree_read_lock(eb, start_ns);
}
+void btrfs_tree_read_lock(struct extent_buffer *eb)
+{
+ __btrfs_tree_read_lock(eb, BTRFS_NESTING_NORMAL, false);
+}
+
/*
* Lock extent buffer for read, optimistically expecting that there are no
* contending blocking writers. If there are, don't wait.
/*
* if we're nested, we have the write lock. No new locking
* is needed as long as we are the lock owner.
- * The write unlock will do a barrier for us, and the lock_nested
+ * The write unlock will do a barrier for us, and the lock_recursed
* field only matters to the lock owner.
*/
- if (eb->lock_nested && current->pid == eb->lock_owner) {
- eb->lock_nested = false;
+ if (eb->lock_recursed && current->pid == eb->lock_owner) {
+ eb->lock_recursed = false;
return;
}
btrfs_assert_tree_read_locked(eb);
/*
* if we're nested, we have the write lock. No new locking
* is needed as long as we are the lock owner.
- * The write unlock will do a barrier for us, and the lock_nested
+ * The write unlock will do a barrier for us, and the lock_recursed
* field only matters to the lock owner.
*/
- if (eb->lock_nested && current->pid == eb->lock_owner) {
- eb->lock_nested = false;
+ if (eb->lock_recursed && current->pid == eb->lock_owner) {
+ eb->lock_recursed = false;
return;
}
btrfs_assert_tree_read_locked(eb);
*
* The rwlock is held for write upon exit.
*/
-void btrfs_tree_lock(struct extent_buffer *eb)
+void __btrfs_tree_lock(struct extent_buffer *eb, enum btrfs_lock_nesting nest)
__acquires(&eb->lock)
{
u64 start_ns = 0;
trace_btrfs_tree_lock(eb, start_ns);
}
+void btrfs_tree_lock(struct extent_buffer *eb)
+{
+ __btrfs_tree_lock(eb, BTRFS_NESTING_NORMAL);
+}
+
/*
* Release the write lock, either blocking or spinning (ie. there's no need
* for an explicit blocking unlock, like btrfs_tree_read_unlock_blocking).
*
* Return: root extent buffer with read lock held
*/
-struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root)
+struct extent_buffer *__btrfs_read_lock_root_node(struct btrfs_root *root,
+ bool recurse)
{
struct extent_buffer *eb;
while (1) {
eb = btrfs_root_node(root);
- btrfs_tree_read_lock(eb);
+ __btrfs_tree_read_lock(eb, BTRFS_NESTING_NORMAL, recurse);
if (eb == root->node)
break;
btrfs_tree_read_unlock(eb);
#define BTRFS_WRITE_LOCK_BLOCKING 3
#define BTRFS_READ_LOCK_BLOCKING 4
+/*
+ * We are limited in number of subclasses by MAX_LOCKDEP_SUBCLASSES, which at
+ * the time of this patch is 8, which is how many we use. Keep this in mind if
+ * you decide you want to add another subclass.
+ */
+enum btrfs_lock_nesting {
+ BTRFS_NESTING_NORMAL,
+
+ /*
+ * When we COW a block we are holding the lock on the original block,
+ * and since our lockdep maps are rootid+level, this confuses lockdep
+ * when we lock the newly allocated COW'd block. Handle this by having
+ * a subclass for COW'ed blocks so that lockdep doesn't complain.
+ */
+ BTRFS_NESTING_COW,
+
+ /*
+ * Oftentimes we need to lock adjacent nodes on the same level while
+ * still holding the lock on the original node we searched to, such as
+ * for searching forward or for split/balance.
+ *
+ * Because of this we need to indicate to lockdep that this is
+ * acceptable by having a different subclass for each of these
+ * operations.
+ */
+ BTRFS_NESTING_LEFT,
+ BTRFS_NESTING_RIGHT,
+
+ /*
+ * When splitting we will be holding a lock on the left/right node when
+ * we need to cow that node, thus we need a new set of subclasses for
+ * these two operations.
+ */
+ BTRFS_NESTING_LEFT_COW,
+ BTRFS_NESTING_RIGHT_COW,
+
+ /*
+ * When splitting we may push nodes to the left or right, but still use
+ * the subsequent nodes in our path, keeping our locks on those adjacent
+ * blocks. Thus when we go to allocate a new split block we've already
+ * used up all of our available subclasses, so this subclass exists to
+ * handle this case where we need to allocate a new split block.
+ */
+ BTRFS_NESTING_SPLIT,
+
+ /*
+ * When promoting a new block to a root we need to have a special
+ * subclass so we don't confuse lockdep, as it will appear that we are
+ * locking a higher level node before a lower level one. Copying also
+ * has this problem as it appears we're locking the same block again
+ * when we make a snapshot of an existing root.
+ */
+ BTRFS_NESTING_NEW_ROOT,
+
+ /*
+ * We are limited to MAX_LOCKDEP_SUBLCLASSES number of subclasses, so
+ * add this in here and add a static_assert to keep us from going over
+ * the limit. As of this writing we're limited to 8, and we're
+ * definitely using 8, hence this check to keep us from messing up in
+ * the future.
+ */
+ BTRFS_NESTING_MAX,
+};
+
+static_assert(BTRFS_NESTING_MAX <= MAX_LOCKDEP_SUBCLASSES,
+ "too many lock subclasses defined");
+
struct btrfs_path;
+void __btrfs_tree_lock(struct extent_buffer *eb, enum btrfs_lock_nesting nest);
void btrfs_tree_lock(struct extent_buffer *eb);
void btrfs_tree_unlock(struct extent_buffer *eb);
+void __btrfs_tree_read_lock(struct extent_buffer *eb, enum btrfs_lock_nesting nest,
+ bool recurse);
void btrfs_tree_read_lock(struct extent_buffer *eb);
void btrfs_tree_read_unlock(struct extent_buffer *eb);
void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb);
int btrfs_try_tree_read_lock(struct extent_buffer *eb);
int btrfs_try_tree_write_lock(struct extent_buffer *eb);
int btrfs_tree_read_lock_atomic(struct extent_buffer *eb);
+struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root);
+struct extent_buffer *__btrfs_read_lock_root_node(struct btrfs_root *root,
+ bool recurse);
+
+static inline struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root)
+{
+ return __btrfs_read_lock_root_node(root, false);
+}
#ifdef CONFIG_BTRFS_DEBUG
static inline void btrfs_assert_tree_locked(struct extent_buffer *eb) {
refcount_set(&entry->refs, 1);
init_waitqueue_head(&entry->wait);
INIT_LIST_HEAD(&entry->list);
+ INIT_LIST_HEAD(&entry->log_list);
INIT_LIST_HEAD(&entry->root_extent_list);
INIT_LIST_HEAD(&entry->work_list);
init_completion(&entry->completion);
- trace_btrfs_ordered_extent_add(&inode->vfs_inode, entry);
+ trace_btrfs_ordered_extent_add(inode, entry);
spin_lock_irq(&tree->lock);
node = tree_insert(&tree->tree, file_offset,
* test_and_set_bit on a flag in the struct btrfs_ordered_extent is used
* to make sure this function only returns 1 once for a given ordered extent.
*/
-int btrfs_dec_test_ordered_pending(struct inode *inode,
+int btrfs_dec_test_ordered_pending(struct btrfs_inode *inode,
struct btrfs_ordered_extent **cached,
u64 file_offset, u64 io_size, int uptodate)
{
- struct btrfs_ordered_inode_tree *tree;
+ struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree;
struct rb_node *node;
struct btrfs_ordered_extent *entry = NULL;
unsigned long flags;
int ret;
- tree = &BTRFS_I(inode)->ordered_tree;
spin_lock_irqsave(&tree->lock, flags);
if (cached && *cached) {
entry = *cached;
}
if (io_size > entry->bytes_left) {
- btrfs_crit(BTRFS_I(inode)->root->fs_info,
+ btrfs_crit(inode->root->fs_info,
"bad ordered accounting left %llu size %llu",
entry->bytes_left, io_size);
}
struct list_head *cur;
struct btrfs_ordered_sum *sum;
- trace_btrfs_ordered_extent_put(entry->inode, entry);
+ trace_btrfs_ordered_extent_put(BTRFS_I(entry->inode), entry);
if (refcount_dec_and_test(&entry->refs)) {
ASSERT(list_empty(&entry->root_extent_list));
+ ASSERT(list_empty(&entry->log_list));
ASSERT(RB_EMPTY_NODE(&entry->rb_node));
if (entry->inode)
btrfs_add_delayed_iput(entry->inode);
* remove an ordered extent from the tree. No references are dropped
* and waiters are woken up.
*/
-void btrfs_remove_ordered_extent(struct inode *inode,
+void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode,
struct btrfs_ordered_extent *entry)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
struct btrfs_ordered_inode_tree *tree;
- struct btrfs_inode *btrfs_inode = BTRFS_I(inode);
struct btrfs_root *root = btrfs_inode->root;
+ struct btrfs_fs_info *fs_info = root->fs_info;
struct rb_node *node;
+ bool pending;
/* This is paired with btrfs_add_ordered_extent. */
spin_lock(&btrfs_inode->lock);
if (tree->last == node)
tree->last = NULL;
set_bit(BTRFS_ORDERED_COMPLETE, &entry->flags);
+ pending = test_and_clear_bit(BTRFS_ORDERED_PENDING, &entry->flags);
spin_unlock_irq(&tree->lock);
+ /*
+ * The current running transaction is waiting on us, we need to let it
+ * know that we're complete and wake it up.
+ */
+ if (pending) {
+ struct btrfs_transaction *trans;
+
+ /*
+ * The checks for trans are just a formality, it should be set,
+ * but if it isn't we don't want to deref/assert under the spin
+ * lock, so be nice and check if trans is set, but ASSERT() so
+ * if it isn't set a developer will notice.
+ */
+ spin_lock(&fs_info->trans_lock);
+ trans = fs_info->running_transaction;
+ if (trans)
+ refcount_inc(&trans->use_count);
+ spin_unlock(&fs_info->trans_lock);
+
+ ASSERT(trans);
+ if (trans) {
+ if (atomic_dec_and_test(&trans->pending_ordered))
+ wake_up(&trans->pending_wait);
+ btrfs_put_transaction(trans);
+ }
+ }
+
spin_lock(&root->ordered_extent_lock);
list_del_init(&entry->root_extent_list);
root->nr_ordered_extents--;
- trace_btrfs_ordered_extent_remove(inode, entry);
+ trace_btrfs_ordered_extent_remove(btrfs_inode, entry);
if (!root->nr_ordered_extents) {
spin_lock(&fs_info->ordered_root_lock);
struct btrfs_ordered_extent *ordered;
ordered = container_of(work, struct btrfs_ordered_extent, flush_work);
- btrfs_start_ordered_extent(ordered->inode, ordered, 1);
+ btrfs_start_ordered_extent(ordered, 1);
complete(&ordered->completion);
}
* in the extent, and it waits on the io completion code to insert
* metadata into the btree corresponding to the extent
*/
-void btrfs_start_ordered_extent(struct inode *inode,
- struct btrfs_ordered_extent *entry,
- int wait)
+void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry, int wait)
{
u64 start = entry->file_offset;
u64 end = start + entry->num_bytes - 1;
+ struct btrfs_inode *inode = BTRFS_I(entry->inode);
trace_btrfs_ordered_extent_start(inode, entry);
* for the flusher thread to find them
*/
if (!test_bit(BTRFS_ORDERED_DIRECT, &entry->flags))
- filemap_fdatawrite_range(inode->i_mapping, start, end);
+ filemap_fdatawrite_range(inode->vfs_inode.i_mapping, start, end);
if (wait) {
wait_event(entry->wait, test_bit(BTRFS_ORDERED_COMPLETE,
&entry->flags));
end = orig_end;
while (1) {
- ordered = btrfs_lookup_first_ordered_extent(inode, end);
+ ordered = btrfs_lookup_first_ordered_extent(BTRFS_I(inode), end);
if (!ordered)
break;
if (ordered->file_offset > orig_end) {
btrfs_put_ordered_extent(ordered);
break;
}
- btrfs_start_ordered_extent(inode, ordered, 1);
+ btrfs_start_ordered_extent(ordered, 1);
end = ordered->file_offset;
/*
* If the ordered extent had an error save the error but don't
return entry;
}
+/*
+ * Adds all ordered extents to the given list. The list ends up sorted by the
+ * file_offset of the ordered extents.
+ */
+void btrfs_get_ordered_extents_for_logging(struct btrfs_inode *inode,
+ struct list_head *list)
+{
+ struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree;
+ struct rb_node *n;
+
+ ASSERT(inode_is_locked(&inode->vfs_inode));
+
+ spin_lock_irq(&tree->lock);
+ for (n = rb_first(&tree->tree); n; n = rb_next(n)) {
+ struct btrfs_ordered_extent *ordered;
+
+ ordered = rb_entry(n, struct btrfs_ordered_extent, rb_node);
+
+ if (test_bit(BTRFS_ORDERED_LOGGED, &ordered->flags))
+ continue;
+
+ ASSERT(list_empty(&ordered->log_list));
+ list_add_tail(&ordered->log_list, list);
+ refcount_inc(&ordered->refs);
+ }
+ spin_unlock_irq(&tree->lock);
+}
+
/*
* lookup and return any extent before 'file_offset'. NULL is returned
* if none is found
*/
struct btrfs_ordered_extent *
-btrfs_lookup_first_ordered_extent(struct inode *inode, u64 file_offset)
+btrfs_lookup_first_ordered_extent(struct btrfs_inode *inode, u64 file_offset)
{
struct btrfs_ordered_inode_tree *tree;
struct rb_node *node;
struct btrfs_ordered_extent *entry = NULL;
- tree = &BTRFS_I(inode)->ordered_tree;
+ tree = &inode->ordered_tree;
spin_lock_irq(&tree->lock);
node = tree_search(tree, file_offset);
if (!node)
* try to find a checksum. This is used because we allow pages to
* be reclaimed before their checksum is actually put into the btree
*/
-int btrfs_find_ordered_sum(struct inode *inode, u64 offset, u64 disk_bytenr,
- u8 *sum, int len)
+int btrfs_find_ordered_sum(struct btrfs_inode *inode, u64 offset,
+ u64 disk_bytenr, u8 *sum, int len)
{
- struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
+ struct btrfs_fs_info *fs_info = inode->root->fs_info;
struct btrfs_ordered_sum *ordered_sum;
struct btrfs_ordered_extent *ordered;
- struct btrfs_ordered_inode_tree *tree = &BTRFS_I(inode)->ordered_tree;
+ struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree;
unsigned long num_sectors;
unsigned long i;
u32 sectorsize = btrfs_inode_sectorsize(inode);
+ const u8 blocksize_bits = inode->vfs_inode.i_sb->s_blocksize_bits;
const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
int index = 0;
- ordered = btrfs_lookup_ordered_extent(BTRFS_I(inode), offset);
+ ordered = btrfs_lookup_ordered_extent(inode, offset);
if (!ordered)
return 0;
list_for_each_entry_reverse(ordered_sum, &ordered->list, list) {
if (disk_bytenr >= ordered_sum->bytenr &&
disk_bytenr < ordered_sum->bytenr + ordered_sum->len) {
- i = (disk_bytenr - ordered_sum->bytenr) >>
- inode->i_sb->s_blocksize_bits;
- num_sectors = ordered_sum->len >>
- inode->i_sb->s_blocksize_bits;
+ i = (disk_bytenr - ordered_sum->bytenr) >> blocksize_bits;
+ num_sectors = ordered_sum->len >> blocksize_bits;
num_sectors = min_t(int, len - index, num_sectors - i);
memcpy(sum + index, ordered_sum->sums + i * csum_size,
num_sectors * csum_size);
break;
}
unlock_extent_cached(&inode->io_tree, start, end, cachedp);
- btrfs_start_ordered_extent(&inode->vfs_inode, ordered, 1);
+ btrfs_start_ordered_extent(ordered, 1);
btrfs_put_ordered_extent(ordered);
}
}
BTRFS_ORDERED_TRUNCATED,
/* Regular IO for COW */
BTRFS_ORDERED_REGULAR,
+ /* Used during fsync to track already logged extents */
+ BTRFS_ORDERED_LOGGED,
+ /* We have already logged all the csums of the ordered extent */
+ BTRFS_ORDERED_LOGGED_CSUM,
+ /* We wait for this extent to complete in the current transaction */
+ BTRFS_ORDERED_PENDING,
};
struct btrfs_ordered_extent {
/* list of checksums for insertion when the extent io is done */
struct list_head list;
+ /* used for fast fsyncs */
+ struct list_head log_list;
+
/* used to wait for the BTRFS_ORDERED_COMPLETE bit */
wait_queue_head_t wait;
}
void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry);
-void btrfs_remove_ordered_extent(struct inode *inode,
+void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode,
struct btrfs_ordered_extent *entry);
-int btrfs_dec_test_ordered_pending(struct inode *inode,
+int btrfs_dec_test_ordered_pending(struct btrfs_inode *inode,
struct btrfs_ordered_extent **cached,
u64 file_offset, u64 io_size, int uptodate);
int btrfs_dec_test_first_ordered_pending(struct btrfs_inode *inode,
struct btrfs_ordered_sum *sum);
struct btrfs_ordered_extent *btrfs_lookup_ordered_extent(struct btrfs_inode *inode,
u64 file_offset);
-void btrfs_start_ordered_extent(struct inode *inode,
- struct btrfs_ordered_extent *entry, int wait);
+void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry, int wait);
int btrfs_wait_ordered_range(struct inode *inode, u64 start, u64 len);
struct btrfs_ordered_extent *
-btrfs_lookup_first_ordered_extent(struct inode * inode, u64 file_offset);
+btrfs_lookup_first_ordered_extent(struct btrfs_inode *inode, u64 file_offset);
struct btrfs_ordered_extent *btrfs_lookup_ordered_range(
struct btrfs_inode *inode,
u64 file_offset,
u64 len);
-int btrfs_find_ordered_sum(struct inode *inode, u64 offset, u64 disk_bytenr,
- u8 *sum, int len);
+void btrfs_get_ordered_extents_for_logging(struct btrfs_inode *inode,
+ struct list_head *list);
+int btrfs_find_ordered_sum(struct btrfs_inode *inode, u64 offset,
+ u64 disk_bytenr, u8 *sum, int len);
u64 btrfs_wait_ordered_extents(struct btrfs_root *root, u64 nr,
const u64 range_start, const u64 range_len);
void btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr,
#include "disk-io.h"
#include "print-tree.h"
+struct root_name_map {
+ u64 id;
+ char name[16];
+};
+
+static const struct root_name_map root_map[] = {
+ { BTRFS_ROOT_TREE_OBJECTID, "ROOT_TREE" },
+ { BTRFS_EXTENT_TREE_OBJECTID, "EXTENT_TREE" },
+ { BTRFS_CHUNK_TREE_OBJECTID, "CHUNK_TREE" },
+ { BTRFS_DEV_TREE_OBJECTID, "DEV_TREE" },
+ { BTRFS_FS_TREE_OBJECTID, "FS_TREE" },
+ { BTRFS_CSUM_TREE_OBJECTID, "CSUM_TREE" },
+ { BTRFS_TREE_LOG_OBJECTID, "TREE_LOG" },
+ { BTRFS_QUOTA_TREE_OBJECTID, "QUOTA_TREE" },
+ { BTRFS_UUID_TREE_OBJECTID, "UUID_TREE" },
+ { BTRFS_FREE_SPACE_TREE_OBJECTID, "FREE_SPACE_TREE" },
+ { BTRFS_DATA_RELOC_TREE_OBJECTID, "DATA_RELOC_TREE" },
+};
+
+const char *btrfs_root_name(u64 objectid, char *buf)
+{
+ int i;
+
+ if (objectid == BTRFS_TREE_RELOC_OBJECTID) {
+ snprintf(buf, BTRFS_ROOT_NAME_BUF_LEN,
+ "TREE_RELOC offset=%llu", objectid);
+ return buf;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(root_map); i++) {
+ if (root_map[i].id == objectid)
+ return root_map[i].name;
+ }
+
+ snprintf(buf, BTRFS_ROOT_NAME_BUF_LEN, "%llu", objectid);
+ return buf;
+}
+
static void print_chunk(struct extent_buffer *eb, struct btrfs_chunk *chunk)
{
int num_stripes = btrfs_chunk_num_stripes(eb, chunk);
#ifndef BTRFS_PRINT_TREE_H
#define BTRFS_PRINT_TREE_H
+/* Buffer size to contain tree name and possibly additional data (offset) */
+#define BTRFS_ROOT_NAME_BUF_LEN 48
+
void btrfs_print_leaf(struct extent_buffer *l);
void btrfs_print_tree(struct extent_buffer *c, bool follow);
+const char *btrfs_root_name(u64 objectid, char *buf);
#endif
* Update qgroup rfer/excl counters.
* Rfer update is easy, codes can explain themselves.
*
- * Excl update is tricky, the update is split into 2 part.
+ * Excl update is tricky, the update is split into 2 parts.
* Part 1: Possible exclusive <-> sharing detect:
* | A | !A |
* -------------------------------------
kfree(rmw);
}
-static void __reada_start_machine(struct btrfs_fs_info *fs_info)
+/* Try to start up to 10k READA requests for a group of devices */
+static int reada_start_for_fsdevs(struct btrfs_fs_devices *fs_devices)
{
- struct btrfs_device *device;
- struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
u64 enqueued;
u64 total = 0;
- int i;
+ struct btrfs_device *device;
-again:
do {
enqueued = 0;
- mutex_lock(&fs_devices->device_list_mutex);
list_for_each_entry(device, &fs_devices->devices, dev_list) {
if (atomic_read(&device->reada_in_flight) <
MAX_IN_FLIGHT)
enqueued += reada_start_machine_dev(device);
}
- mutex_unlock(&fs_devices->device_list_mutex);
total += enqueued;
} while (enqueued && total < 10000);
- if (fs_devices->seed) {
- fs_devices = fs_devices->seed;
- goto again;
- }
+ return total;
+}
+
+static void __reada_start_machine(struct btrfs_fs_info *fs_info)
+{
+ struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
+ int i;
+ u64 enqueued = 0;
+
+ mutex_lock(&fs_devices->device_list_mutex);
+
+ enqueued += reada_start_for_fsdevs(fs_devices);
+ list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list)
+ enqueued += reada_start_for_fsdevs(seed_devs);
+
+ mutex_unlock(&fs_devices->device_list_mutex);
if (enqueued == 0)
return;
return ret;
}
-static int copy_inline_to_page(struct inode *inode,
+static int copy_inline_to_page(struct btrfs_inode *inode,
const u64 file_offset,
char *inline_data,
const u64 size,
char *data_start = inline_data + btrfs_file_extent_calc_inline_size(0);
struct extent_changeset *data_reserved = NULL;
struct page *page = NULL;
+ struct address_space *mapping = inode->vfs_inode.i_mapping;
int ret;
ASSERT(IS_ALIGNED(file_offset, block_size));
* reservation here. Also we must not do the reservation while holding
* a transaction open, otherwise we would deadlock.
*/
- ret = btrfs_delalloc_reserve_space(BTRFS_I(inode), &data_reserved,
- file_offset, block_size);
+ ret = btrfs_delalloc_reserve_space(inode, &data_reserved, file_offset,
+ block_size);
if (ret)
goto out;
- page = find_or_create_page(inode->i_mapping, file_offset >> PAGE_SHIFT,
- btrfs_alloc_write_mask(inode->i_mapping));
+ page = find_or_create_page(mapping, file_offset >> PAGE_SHIFT,
+ btrfs_alloc_write_mask(mapping));
if (!page) {
ret = -ENOMEM;
goto out_unlock;
}
set_page_extent_mapped(page);
- clear_extent_bit(&BTRFS_I(inode)->io_tree, file_offset, range_end,
+ clear_extent_bit(&inode->io_tree, file_offset, range_end,
EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
0, 0, NULL);
- ret = btrfs_set_extent_delalloc(BTRFS_I(inode), file_offset, range_end,
- 0, NULL);
+ ret = btrfs_set_extent_delalloc(inode, file_offset, range_end, 0, NULL);
if (ret)
goto out_unlock;
put_page(page);
}
if (ret)
- btrfs_delalloc_release_space(BTRFS_I(inode), data_reserved,
- file_offset, block_size, true);
- btrfs_delalloc_release_extents(BTRFS_I(inode), block_size);
+ btrfs_delalloc_release_space(inode, data_reserved, file_offset,
+ block_size, true);
+ btrfs_delalloc_release_extents(inode, block_size);
out:
extent_changeset_free(data_reserved);
struct btrfs_key key;
if (new_key->offset > 0) {
- ret = copy_inline_to_page(dst, new_key->offset, inline_data,
- size, datal, comp_type);
+ ret = copy_inline_to_page(BTRFS_I(dst), new_key->offset,
+ inline_data, size, datal, comp_type);
goto out;
}
* inline extent's data to the page.
*/
ASSERT(key.offset > 0);
- ret = copy_inline_to_page(dst, new_key->offset,
+ ret = copy_inline_to_page(BTRFS_I(dst), new_key->offset,
inline_data, size, datal,
comp_type);
goto out;
BTRFS_FILE_EXTENT_INLINE)
goto copy_inline_extent;
- ret = copy_inline_to_page(dst, new_key->offset, inline_data,
- size, datal, comp_type);
+ ret = copy_inline_to_page(BTRFS_I(dst), new_key->offset,
+ inline_data, size, datal, comp_type);
goto out;
}
* clone. Deal with all these cases by copying the inline extent
* data into the respective page at the destination inode.
*/
- ret = copy_inline_to_page(dst, new_key->offset, inline_data,
- size, datal, comp_type);
+ ret = copy_inline_to_page(BTRFS_I(dst), new_key->offset,
+ inline_data, size, datal, comp_type);
goto out;
}
if (type == BTRFS_FILE_EXTENT_REG ||
type == BTRFS_FILE_EXTENT_PREALLOC) {
- struct btrfs_clone_extent_info clone_info;
+ struct btrfs_replace_extent_info clone_info;
/*
* a | --- range to clone ---| b
clone_info.data_len = datal;
clone_info.file_offset = new_key.offset;
clone_info.extent_buf = buf;
- clone_info.item_size = size;
- ret = btrfs_punch_hole_range(inode, path, drop_start,
+ clone_info.is_new_extent = false;
+ ret = btrfs_replace_file_extents(inode, path, drop_start,
new_key.offset + datal - 1, &clone_info,
&trans);
if (ret)
ret = -EINTR;
goto out;
}
+
+ cond_resched();
}
ret = 0;
btrfs_release_path(path);
path->leave_spinning = 0;
- ret = btrfs_punch_hole_range(inode, path, last_dest_end,
+ ret = btrfs_replace_file_extents(inode, path, last_dest_end,
destoff + len - 1, NULL, &trans);
if (ret)
goto out;
}
if (cow) {
- ret = btrfs_cow_block(trans, dest, eb, NULL, 0, &eb);
+ ret = btrfs_cow_block(trans, dest, eb, NULL, 0, &eb,
+ BTRFS_NESTING_COW);
BUG_ON(ret);
}
btrfs_set_lock_blocking_write(eb);
btrfs_tree_lock(eb);
if (cow) {
ret = btrfs_cow_block(trans, dest, eb, parent,
- slot, &eb);
+ slot, &eb,
+ BTRFS_NESTING_COW);
BUG_ON(ret);
}
btrfs_set_lock_blocking_write(eb);
* relocated and the block is tree root.
*/
leaf = btrfs_lock_root_node(root);
- ret = btrfs_cow_block(trans, root, leaf, NULL, 0, &leaf);
+ ret = btrfs_cow_block(trans, root, leaf, NULL, 0, &leaf,
+ BTRFS_NESTING_COW);
btrfs_tree_unlock(leaf);
free_extent_buffer(leaf);
if (ret < 0)
if (!node->eb) {
ret = btrfs_cow_block(trans, root, eb, upper->eb,
- slot, &eb);
+ slot, &eb, BTRFS_NESTING_COW);
btrfs_tree_unlock(eb);
free_extent_buffer(eb);
if (ret < 0) {
if (ret && qgroup_num_bytes)
btrfs_qgroup_free_meta_prealloc(root, qgroup_num_bytes);
+ if (!ret) {
+ spin_lock(&rsv->lock);
+ rsv->qgroup_rsv_reserved += qgroup_num_bytes;
+ spin_unlock(&rsv->lock);
+ }
return ret;
}
-void btrfs_subvolume_release_metadata(struct btrfs_fs_info *fs_info,
+void btrfs_subvolume_release_metadata(struct btrfs_root *root,
struct btrfs_block_rsv *rsv)
{
- btrfs_block_rsv_release(fs_info, rsv, (u64)-1, NULL);
+ struct btrfs_fs_info *fs_info = root->fs_info;
+ u64 qgroup_to_release;
+
+ btrfs_block_rsv_release(fs_info, rsv, (u64)-1, &qgroup_to_release);
+ btrfs_qgroup_convert_reserved_meta(root, qgroup_to_release);
}
int success;
bool full_stripe_locked;
unsigned int nofs_flag;
- static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
+ static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
DEFAULT_RATELIMIT_BURST);
BUG_ON(sblock_to_check->page_count < 1);
spin_lock(&sctx->stat_lock);
sctx->stat.read_errors++;
spin_unlock(&sctx->stat_lock);
- if (__ratelimit(&_rs))
+ if (__ratelimit(&rs))
scrub_print_warning("i/o error", sblock_to_check);
btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
} else if (sblock_bad->checksum_error) {
spin_lock(&sctx->stat_lock);
sctx->stat.csum_errors++;
spin_unlock(&sctx->stat_lock);
- if (__ratelimit(&_rs))
+ if (__ratelimit(&rs))
scrub_print_warning("checksum error", sblock_to_check);
btrfs_dev_stat_inc_and_print(dev,
BTRFS_DEV_STAT_CORRUPTION_ERRS);
spin_lock(&sctx->stat_lock);
sctx->stat.verify_errors++;
spin_unlock(&sctx->stat_lock);
- if (__ratelimit(&_rs))
+ if (__ratelimit(&rs))
scrub_print_warning("checksum/header error",
sblock_to_check);
if (sblock_bad->generation_error)
struct file_ra_state ra;
- char *read_buf;
-
/*
* We process inodes by their increasing order, so if before an
* incremental send we reverse the parent/child relationship of
BTRFS_COMPARE_TREE_CHANGED,
BTRFS_COMPARE_TREE_SAME,
};
-typedef int (*btrfs_changed_cb_t)(struct btrfs_path *left_path,
- struct btrfs_path *right_path,
- struct btrfs_key *key,
- enum btrfs_compare_tree_result result,
- void *ctx);
__cold
static void inconsistent_snapshot_error(struct send_ctx *sctx,
return -EOVERFLOW;
hdr = (struct btrfs_tlv_header *) (sctx->send_buf + sctx->send_size);
- hdr->tlv_type = cpu_to_le16(attr);
- hdr->tlv_len = cpu_to_le16(len);
+ put_unaligned_le16(attr, &hdr->tlv_type);
+ put_unaligned_le16(len, &hdr->tlv_len);
memcpy(hdr + 1, data, len);
sctx->send_size += total_len;
sctx->send_size += sizeof(*hdr);
hdr = (struct btrfs_cmd_header *)sctx->send_buf;
- hdr->cmd = cpu_to_le16(cmd);
+ put_unaligned_le16(cmd, &hdr->cmd);
return 0;
}
u32 crc;
hdr = (struct btrfs_cmd_header *)sctx->send_buf;
- hdr->len = cpu_to_le32(sctx->send_size - sizeof(*hdr));
- hdr->crc = 0;
+ put_unaligned_le32(sctx->send_size - sizeof(*hdr), &hdr->len);
+ put_unaligned_le32(0, &hdr->crc);
crc = btrfs_crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size);
- hdr->crc = cpu_to_le32(crc);
+ put_unaligned_le32(crc, &hdr->crc);
ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size,
&sctx->send_off);
sctx->total_send_size += sctx->send_size;
- sctx->cmd_send_size[le16_to_cpu(hdr->cmd)] += sctx->send_size;
+ sctx->cmd_send_size[get_unaligned_le16(&hdr->cmd)] += sctx->send_size;
sctx->send_size = 0;
return ret;
return 0;
}
+/*
+ * When processing the new references for an inode we may orphanize an existing
+ * directory inode because its old name conflicts with one of the new references
+ * of the current inode. Later, when processing another new reference of our
+ * inode, we might need to orphanize another inode, but the path we have in the
+ * reference reflects the pre-orphanization name of the directory we previously
+ * orphanized. For example:
+ *
+ * parent snapshot looks like:
+ *
+ * . (ino 256)
+ * |----- f1 (ino 257)
+ * |----- f2 (ino 258)
+ * |----- d1/ (ino 259)
+ * |----- d2/ (ino 260)
+ *
+ * send snapshot looks like:
+ *
+ * . (ino 256)
+ * |----- d1 (ino 258)
+ * |----- f2/ (ino 259)
+ * |----- f2_link/ (ino 260)
+ * | |----- f1 (ino 257)
+ * |
+ * |----- d2 (ino 258)
+ *
+ * When processing inode 257 we compute the name for inode 259 as "d1", and we
+ * cache it in the name cache. Later when we start processing inode 258, when
+ * collecting all its new references we set a full path of "d1/d2" for its new
+ * reference with name "d2". When we start processing the new references we
+ * start by processing the new reference with name "d1", and this results in
+ * orphanizing inode 259, since its old reference causes a conflict. Then we
+ * move on the next new reference, with name "d2", and we find out we must
+ * orphanize inode 260, as its old reference conflicts with ours - but for the
+ * orphanization we use a source path corresponding to the path we stored in the
+ * new reference, which is "d1/d2" and not "o259-6-0/d2" - this makes the
+ * receiver fail since the path component "d1/" no longer exists, it was renamed
+ * to "o259-6-0/" when processing the previous new reference. So in this case we
+ * must recompute the path in the new reference and use it for the new
+ * orphanization operation.
+ */
+static int refresh_ref_path(struct send_ctx *sctx, struct recorded_ref *ref)
+{
+ char *name;
+ int ret;
+
+ name = kmemdup(ref->name, ref->name_len, GFP_KERNEL);
+ if (!name)
+ return -ENOMEM;
+
+ fs_path_reset(ref->full_path);
+ ret = get_cur_path(sctx, ref->dir, ref->dir_gen, ref->full_path);
+ if (ret < 0)
+ goto out;
+
+ ret = fs_path_add(ref->full_path, name, ref->name_len);
+ if (ret < 0)
+ goto out;
+
+ /* Update the reference's base name pointer. */
+ set_ref_path(ref, ref->full_path);
+out:
+ kfree(name);
+ return ret;
+}
+
/*
* This does all the move/link/unlink/rmdir magic.
*/
goto out;
}
+ /*
+ * Before doing any rename and link operations, do a first pass on the
+ * new references to orphanize any unprocessed inodes that may have a
+ * reference that conflicts with one of the new references of the current
+ * inode. This needs to happen first because a new reference may conflict
+ * with the old reference of a parent directory, so we must make sure
+ * that the path used for link and rename commands don't use an
+ * orphanized name when an ancestor was not yet orphanized.
+ *
+ * Example:
+ *
+ * Parent snapshot:
+ *
+ * . (ino 256)
+ * |----- testdir/ (ino 259)
+ * | |----- a (ino 257)
+ * |
+ * |----- b (ino 258)
+ *
+ * Send snapshot:
+ *
+ * . (ino 256)
+ * |----- testdir_2/ (ino 259)
+ * | |----- a (ino 260)
+ * |
+ * |----- testdir (ino 257)
+ * |----- b (ino 257)
+ * |----- b2 (ino 258)
+ *
+ * Processing the new reference for inode 257 with name "b" may happen
+ * before processing the new reference with name "testdir". If so, we
+ * must make sure that by the time we send a link command to create the
+ * hard link "b", inode 259 was already orphanized, since the generated
+ * path in "valid_path" already contains the orphanized name for 259.
+ * We are processing inode 257, so only later when processing 259 we do
+ * the rename operation to change its temporary (orphanized) name to
+ * "testdir_2".
+ */
list_for_each_entry(cur, &sctx->new_refs, list) {
- /*
- * We may have refs where the parent directory does not exist
- * yet. This happens if the parent directories inum is higher
- * than the current inum. To handle this case, we create the
- * parent directory out of order. But we need to check if this
- * did already happen before due to other refs in the same dir.
- */
ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen);
if (ret < 0)
goto out;
- if (ret == inode_state_will_create) {
- ret = 0;
- /*
- * First check if any of the current inodes refs did
- * already create the dir.
- */
- list_for_each_entry(cur2, &sctx->new_refs, list) {
- if (cur == cur2)
- break;
- if (cur2->dir == cur->dir) {
- ret = 1;
- break;
- }
- }
-
- /*
- * If that did not happen, check if a previous inode
- * did already create the dir.
- */
- if (!ret)
- ret = did_create_dir(sctx, cur->dir);
- if (ret < 0)
- goto out;
- if (!ret) {
- ret = send_create_inode(sctx, cur->dir);
- if (ret < 0)
- goto out;
- }
- }
+ if (ret == inode_state_will_create)
+ continue;
/*
- * Check if this new ref would overwrite the first ref of
- * another unprocessed inode. If yes, orphanize the
- * overwritten inode. If we find an overwritten ref that is
- * not the first ref, simply unlink it.
+ * Check if this new ref would overwrite the first ref of another
+ * unprocessed inode. If yes, orphanize the overwritten inode.
+ * If we find an overwritten ref that is not the first ref,
+ * simply unlink it.
*/
ret = will_overwrite_ref(sctx, cur->dir, cur->dir_gen,
cur->name, cur->name_len,
struct name_cache_entry *nce;
struct waiting_dir_move *wdm;
+ if (orphanized_dir) {
+ ret = refresh_ref_path(sctx, cur);
+ if (ret < 0)
+ goto out;
+ }
+
ret = orphanize_inode(sctx, ow_inode, ow_gen,
cur->full_path);
if (ret < 0)
}
}
+ }
+
+ list_for_each_entry(cur, &sctx->new_refs, list) {
+ /*
+ * We may have refs where the parent directory does not exist
+ * yet. This happens if the parent directories inum is higher
+ * than the current inum. To handle this case, we create the
+ * parent directory out of order. But we need to check if this
+ * did already happen before due to other refs in the same dir.
+ */
+ ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen);
+ if (ret < 0)
+ goto out;
+ if (ret == inode_state_will_create) {
+ ret = 0;
+ /*
+ * First check if any of the current inodes refs did
+ * already create the dir.
+ */
+ list_for_each_entry(cur2, &sctx->new_refs, list) {
+ if (cur == cur2)
+ break;
+ if (cur2->dir == cur->dir) {
+ ret = 1;
+ break;
+ }
+ }
+
+ /*
+ * If that did not happen, check if a previous inode
+ * did already create the dir.
+ */
+ if (!ret)
+ ret = did_create_dir(sctx, cur->dir);
+ if (ret < 0)
+ goto out;
+ if (!ret) {
+ ret = send_create_inode(sctx, cur->dir);
+ if (ret < 0)
+ goto out;
+ }
+ }
+
if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root) {
ret = wait_for_dest_dir_move(sctx, cur, is_orphan);
if (ret < 0)
return ret;
}
-static ssize_t fill_read_buf(struct send_ctx *sctx, u64 offset, u32 len)
+static inline u64 max_send_read_size(const struct send_ctx *sctx)
+{
+ return sctx->send_max_size - SZ_16K;
+}
+
+static int put_data_header(struct send_ctx *sctx, u32 len)
+{
+ struct btrfs_tlv_header *hdr;
+
+ if (sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len)
+ return -EOVERFLOW;
+ hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size);
+ put_unaligned_le16(BTRFS_SEND_A_DATA, &hdr->tlv_type);
+ put_unaligned_le16(len, &hdr->tlv_len);
+ sctx->send_size += sizeof(*hdr);
+ return 0;
+}
+
+static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len)
{
struct btrfs_root *root = sctx->send_root;
struct btrfs_fs_info *fs_info = root->fs_info;
pgoff_t index = offset >> PAGE_SHIFT;
pgoff_t last_index;
unsigned pg_offset = offset_in_page(offset);
- ssize_t ret = 0;
+ int ret;
+
+ ret = put_data_header(sctx, len);
+ if (ret)
+ return ret;
inode = btrfs_iget(fs_info->sb, sctx->cur_ino, root);
if (IS_ERR(inode))
return PTR_ERR(inode);
- if (offset + len > i_size_read(inode)) {
- if (offset > i_size_read(inode))
- len = 0;
- else
- len = offset - i_size_read(inode);
- }
- if (len == 0)
- goto out;
-
last_index = (offset + len - 1) >> PAGE_SHIFT;
/* initial readahead */
}
addr = kmap(page);
- memcpy(sctx->read_buf + ret, addr + pg_offset, cur_len);
+ memcpy(sctx->send_buf + sctx->send_size, addr + pg_offset,
+ cur_len);
kunmap(page);
unlock_page(page);
put_page(page);
index++;
pg_offset = 0;
len -= cur_len;
- ret += cur_len;
+ sctx->send_size += cur_len;
}
-out:
iput(inode);
return ret;
}
struct btrfs_fs_info *fs_info = sctx->send_root->fs_info;
int ret = 0;
struct fs_path *p;
- ssize_t num_read = 0;
p = fs_path_alloc();
if (!p)
btrfs_debug(fs_info, "send_write offset=%llu, len=%d", offset, len);
- num_read = fill_read_buf(sctx, offset, len);
- if (num_read <= 0) {
- if (num_read < 0)
- ret = num_read;
- goto out;
- }
-
ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
if (ret < 0)
goto out;
TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
- TLV_PUT(sctx, BTRFS_SEND_A_DATA, sctx->read_buf, num_read);
+ ret = put_file_data(sctx, offset, len);
+ if (ret < 0)
+ goto out;
ret = send_cmd(sctx);
tlv_put_failure:
out:
fs_path_free(p);
- if (ret < 0)
- return ret;
- return num_read;
+ return ret;
}
/*
static int send_hole(struct send_ctx *sctx, u64 end)
{
struct fs_path *p = NULL;
+ u64 read_size = max_send_read_size(sctx);
u64 offset = sctx->cur_inode_last_extent;
- u64 len;
int ret = 0;
/*
ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p);
if (ret < 0)
goto tlv_put_failure;
- memset(sctx->read_buf, 0, BTRFS_SEND_READ_SIZE);
while (offset < end) {
- len = min_t(u64, end - offset, BTRFS_SEND_READ_SIZE);
+ u64 len = min(end - offset, read_size);
ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE);
if (ret < 0)
break;
TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p);
TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset);
- TLV_PUT(sctx, BTRFS_SEND_A_DATA, sctx->read_buf, len);
+ ret = put_data_header(sctx, len);
+ if (ret < 0)
+ break;
+ memset(sctx->send_buf + sctx->send_size, 0, len);
+ sctx->send_size += len;
ret = send_cmd(sctx);
if (ret < 0)
break;
const u64 offset,
const u64 len)
{
+ u64 read_size = max_send_read_size(sctx);
u64 sent = 0;
if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA)
return send_update_extent(sctx, offset, len);
while (sent < len) {
- u64 size = len - sent;
+ u64 size = min(len - sent, read_size);
int ret;
- if (size > BTRFS_SEND_READ_SIZE)
- size = BTRFS_SEND_READ_SIZE;
ret = send_write(sctx, offset + sent, size);
if (ret < 0)
return ret;
- if (!ret)
- break;
- sent += ret;
+ sent += size;
}
return 0;
}
struct clone_root *clone_root)
{
int ret = 0;
- struct btrfs_file_extent_item *ei;
u64 offset = key->offset;
- u64 len;
- u8 type;
+ u64 end;
u64 bs = sctx->send_root->fs_info->sb->s_blocksize;
- ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
- struct btrfs_file_extent_item);
- type = btrfs_file_extent_type(path->nodes[0], ei);
- if (type == BTRFS_FILE_EXTENT_INLINE) {
- len = btrfs_file_extent_ram_bytes(path->nodes[0], ei);
- /*
- * it is possible the inline item won't cover the whole page,
- * but there may be items after this page. Make
- * sure to send the whole thing
- */
- len = PAGE_ALIGN(len);
- } else {
- len = btrfs_file_extent_num_bytes(path->nodes[0], ei);
- }
-
- if (offset >= sctx->cur_inode_size) {
- ret = 0;
- goto out;
- }
- if (offset + len > sctx->cur_inode_size)
- len = sctx->cur_inode_size - offset;
- if (len == 0) {
- ret = 0;
- goto out;
- }
+ end = min_t(u64, btrfs_file_extent_end(path), sctx->cur_inode_size);
+ if (offset >= end)
+ return 0;
- if (clone_root && IS_ALIGNED(offset + len, bs)) {
+ if (clone_root && IS_ALIGNED(end, bs)) {
+ struct btrfs_file_extent_item *ei;
u64 disk_byte;
u64 data_offset;
+ ei = btrfs_item_ptr(path->nodes[0], path->slots[0],
+ struct btrfs_file_extent_item);
disk_byte = btrfs_file_extent_disk_bytenr(path->nodes[0], ei);
data_offset = btrfs_file_extent_offset(path->nodes[0], ei);
ret = clone_range(sctx, clone_root, disk_byte, data_offset,
- offset, len);
+ offset, end - offset);
} else {
- ret = send_extent_data(sctx, offset, len);
+ ret = send_extent_data(sctx, offset, end - offset);
}
- sctx->cur_inode_next_write_offset = offset + len;
-out:
+ sctx->cur_inode_next_write_offset = end;
return ret;
}
* If it detects a change, it aborts immediately.
*/
static int btrfs_compare_trees(struct btrfs_root *left_root,
- struct btrfs_root *right_root,
- btrfs_changed_cb_t changed_cb, void *ctx)
+ struct btrfs_root *right_root, void *ctx)
{
struct btrfs_fs_info *fs_info = left_root->fs_info;
int ret;
goto out;
if (sctx->parent_root) {
- ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root,
- changed_cb, sctx);
+ ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root, sctx);
if (ret < 0)
goto out;
ret = finish_inode_if_needed(sctx, 1);
u32 i;
u64 *clone_sources_tmp = NULL;
int clone_sources_to_rollback = 0;
- unsigned alloc_size;
+ size_t alloc_size;
int sort_clone_roots = 0;
if (!capable(CAP_SYS_ADMIN))
goto out;
}
- sctx->read_buf = kvmalloc(BTRFS_SEND_READ_SIZE, GFP_KERNEL);
- if (!sctx->read_buf) {
- ret = -ENOMEM;
- goto out;
- }
-
sctx->pending_dir_moves = RB_ROOT;
sctx->waiting_dir_moves = RB_ROOT;
sctx->orphan_dirs = RB_ROOT;
- alloc_size = sizeof(struct clone_root) * (arg->clone_sources_count + 1);
-
- sctx->clone_roots = kzalloc(alloc_size, GFP_KERNEL);
+ sctx->clone_roots = kvcalloc(sizeof(*sctx->clone_roots),
+ arg->clone_sources_count + 1,
+ GFP_KERNEL);
if (!sctx->clone_roots) {
ret = -ENOMEM;
goto out;
}
- alloc_size = arg->clone_sources_count * sizeof(*arg->clone_sources);
+ alloc_size = array_size(sizeof(*arg->clone_sources),
+ arg->clone_sources_count);
if (arg->clone_sources_count) {
clone_sources_tmp = kvmalloc(alloc_size, GFP_KERNEL);
kvfree(sctx->clone_roots);
kvfree(sctx->send_buf);
- kvfree(sctx->read_buf);
name_cache_free(sctx);
#define BTRFS_SEND_STREAM_VERSION 1
#define BTRFS_SEND_BUF_SIZE SZ_64K
-#define BTRFS_SEND_READ_SIZE (48 * SZ_1K)
enum btrfs_tlv_type {
BTRFS_TLV_U8,
struct list_head *head = &info->space_info;
struct btrfs_space_info *found;
- rcu_read_lock();
- list_for_each_entry_rcu(found, head, list)
+ list_for_each_entry(found, head, list)
found->full = 0;
- rcu_read_unlock();
}
static int create_space_info(struct btrfs_fs_info *info, u64 flags)
if (ret)
return ret;
- list_add_rcu(&space_info->list, &info->space_info);
+ list_add(&space_info->list, &info->space_info);
if (flags & BTRFS_BLOCK_GROUP_DATA)
info->data_sinfo = space_info;
flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
- rcu_read_lock();
- list_for_each_entry_rcu(found, head, list) {
- if (found->flags & flags) {
- rcu_read_unlock();
+ list_for_each_entry(found, head, list) {
+ if (found->flags & flags)
return found;
- }
}
- rcu_read_unlock();
return NULL;
}
-static inline u64 calc_global_rsv_need_space(struct btrfs_block_rsv *global)
-{
- return (global->size << 1);
-}
-
static u64 calc_available_free_space(struct btrfs_fs_info *fs_info,
struct btrfs_space_info *space_info,
enum btrfs_reserve_flush_enum flush)
up_read(&info->groups_sem);
}
-static void btrfs_writeback_inodes_sb_nr(struct btrfs_fs_info *fs_info,
- unsigned long nr_pages, int nr_items)
-{
- struct super_block *sb = fs_info->sb;
-
- if (down_read_trylock(&sb->s_umount)) {
- writeback_inodes_sb_nr(sb, nr_pages, WB_REASON_FS_FREE_SPACE);
- up_read(&sb->s_umount);
- } else {
- /*
- * We needn't worry the filesystem going from r/w to r/o though
- * we don't acquire ->s_umount mutex, because the filesystem
- * should guarantee the delalloc inodes list be empty after
- * the filesystem is readonly(all dirty pages are written to
- * the disk).
- */
- btrfs_start_delalloc_roots(fs_info, nr_items);
- if (!current->journal_info)
- btrfs_wait_ordered_roots(fs_info, nr_items, 0, (u64)-1);
- }
-}
-
static inline u64 calc_reclaim_items_nr(struct btrfs_fs_info *fs_info,
u64 to_reclaim)
{
/*
* shrink metadata reservation for delalloc
*/
-static void shrink_delalloc(struct btrfs_fs_info *fs_info, u64 to_reclaim,
- u64 orig, bool wait_ordered)
+static void shrink_delalloc(struct btrfs_fs_info *fs_info,
+ struct btrfs_space_info *space_info,
+ u64 to_reclaim, bool wait_ordered)
{
- struct btrfs_space_info *space_info;
struct btrfs_trans_handle *trans;
u64 delalloc_bytes;
u64 dio_bytes;
- u64 async_pages;
u64 items;
long time_left;
- unsigned long nr_pages;
int loops;
/* Calc the number of the pages we need flush for space reservation */
- items = calc_reclaim_items_nr(fs_info, to_reclaim);
- to_reclaim = items * EXTENT_SIZE_PER_ITEM;
+ if (to_reclaim == U64_MAX) {
+ items = U64_MAX;
+ } else {
+ /*
+ * to_reclaim is set to however much metadata we need to
+ * reclaim, but reclaiming that much data doesn't really track
+ * exactly, so increase the amount to reclaim by 2x in order to
+ * make sure we're flushing enough delalloc to hopefully reclaim
+ * some metadata reservations.
+ */
+ items = calc_reclaim_items_nr(fs_info, to_reclaim) * 2;
+ to_reclaim = items * EXTENT_SIZE_PER_ITEM;
+ }
trans = (struct btrfs_trans_handle *)current->journal_info;
- space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
delalloc_bytes = percpu_counter_sum_positive(
&fs_info->delalloc_bytes);
loops = 0;
while ((delalloc_bytes || dio_bytes) && loops < 3) {
- nr_pages = min(delalloc_bytes, to_reclaim) >> PAGE_SHIFT;
-
- /*
- * Triggers inode writeback for up to nr_pages. This will invoke
- * ->writepages callback and trigger delalloc filling
- * (btrfs_run_delalloc_range()).
- */
- btrfs_writeback_inodes_sb_nr(fs_info, nr_pages, items);
+ btrfs_start_delalloc_roots(fs_info, items);
- /*
- * We need to wait for the compressed pages to start before
- * we continue.
- */
- async_pages = atomic_read(&fs_info->async_delalloc_pages);
- if (!async_pages)
- goto skip_async;
-
- /*
- * Calculate how many compressed pages we want to be written
- * before we continue. I.e if there are more async pages than we
- * require wait_event will wait until nr_pages are written.
- */
- if (async_pages <= nr_pages)
- async_pages = 0;
- else
- async_pages -= nr_pages;
+ loops++;
+ if (wait_ordered && !trans) {
+ btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1);
+ } else {
+ time_left = schedule_timeout_killable(1);
+ if (time_left)
+ break;
+ }
- wait_event(fs_info->async_submit_wait,
- atomic_read(&fs_info->async_delalloc_pages) <=
- (int)async_pages);
-skip_async:
spin_lock(&space_info->lock);
if (list_empty(&space_info->tickets) &&
list_empty(&space_info->priority_tickets)) {
}
spin_unlock(&space_info->lock);
- loops++;
- if (wait_ordered && !trans) {
- btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1);
- } else {
- time_left = schedule_timeout_killable(1);
- if (time_left)
- break;
- }
delalloc_bytes = percpu_counter_sum_positive(
&fs_info->delalloc_bytes);
dio_bytes = percpu_counter_sum_positive(&fs_info->dio_bytes);
struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv;
struct btrfs_block_rsv *trans_rsv = &fs_info->trans_block_rsv;
struct btrfs_trans_handle *trans;
- u64 bytes_needed;
u64 reclaim_bytes = 0;
+ u64 bytes_needed = 0;
u64 cur_free_bytes = 0;
trans = (struct btrfs_trans_handle *)current->journal_info;
else if (!list_empty(&space_info->tickets))
ticket = list_first_entry(&space_info->tickets,
struct reserve_ticket, list);
- bytes_needed = (ticket) ? ticket->bytes : 0;
+ if (ticket)
+ bytes_needed = ticket->bytes;
if (bytes_needed > cur_free_bytes)
bytes_needed -= cur_free_bytes;
goto commit;
/*
- * See if there is some space in the delayed insertion reservation for
- * this reservation.
+ * See if there is some space in the delayed insertion reserve for this
+ * reservation. If the space_info's don't match (like for DATA or
+ * SYSTEM) then just go enospc, reclaiming this space won't recover any
+ * space to satisfy those reservations.
*/
if (space_info != delayed_rsv->space_info)
goto enospc;
break;
case FLUSH_DELALLOC:
case FLUSH_DELALLOC_WAIT:
- shrink_delalloc(fs_info, num_bytes * 2, num_bytes,
+ shrink_delalloc(fs_info, space_info, num_bytes,
state == FLUSH_DELALLOC_WAIT);
break;
case FLUSH_DELAYED_REFS_NR:
break;
}
ret = btrfs_chunk_alloc(trans,
- btrfs_metadata_alloc_profile(fs_info),
+ btrfs_get_alloc_profile(fs_info, space_info->flags),
(state == ALLOC_CHUNK) ? CHUNK_ALLOC_NO_FORCE :
CHUNK_ALLOC_FORCE);
btrfs_end_transaction(trans);
} while (flush_state <= COMMIT_TRANS);
}
-void btrfs_init_async_reclaim_work(struct work_struct *work)
+/*
+ * FLUSH_DELALLOC_WAIT:
+ * Space is freed from flushing delalloc in one of two ways.
+ *
+ * 1) compression is on and we allocate less space than we reserved
+ * 2) we are overwriting existing space
+ *
+ * For #1 that extra space is reclaimed as soon as the delalloc pages are
+ * COWed, by way of btrfs_add_reserved_bytes() which adds the actual extent
+ * length to ->bytes_reserved, and subtracts the reserved space from
+ * ->bytes_may_use.
+ *
+ * For #2 this is trickier. Once the ordered extent runs we will drop the
+ * extent in the range we are overwriting, which creates a delayed ref for
+ * that freed extent. This however is not reclaimed until the transaction
+ * commits, thus the next stages.
+ *
+ * RUN_DELAYED_IPUTS
+ * If we are freeing inodes, we want to make sure all delayed iputs have
+ * completed, because they could have been on an inode with i_nlink == 0, and
+ * thus have been truncated and freed up space. But again this space is not
+ * immediately re-usable, it comes in the form of a delayed ref, which must be
+ * run and then the transaction must be committed.
+ *
+ * FLUSH_DELAYED_REFS
+ * The above two cases generate delayed refs that will affect
+ * ->total_bytes_pinned. However this counter can be inconsistent with
+ * reality if there are outstanding delayed refs. This is because we adjust
+ * the counter based solely on the current set of delayed refs and disregard
+ * any on-disk state which might include more refs. So for example, if we
+ * have an extent with 2 references, but we only drop 1, we'll see that there
+ * is a negative delayed ref count for the extent and assume that the space
+ * will be freed, and thus increase ->total_bytes_pinned.
+ *
+ * Running the delayed refs gives us the actual real view of what will be
+ * freed at the transaction commit time. This stage will not actually free
+ * space for us, it just makes sure that may_commit_transaction() has all of
+ * the information it needs to make the right decision.
+ *
+ * COMMIT_TRANS
+ * This is where we reclaim all of the pinned space generated by the previous
+ * two stages. We will not commit the transaction if we don't think we're
+ * likely to satisfy our request, which means if our current free space +
+ * total_bytes_pinned < reservation we will not commit. This is why the
+ * previous states are actually important, to make sure we know for sure
+ * whether committing the transaction will allow us to make progress.
+ *
+ * ALLOC_CHUNK_FORCE
+ * For data we start with alloc chunk force, however we could have been full
+ * before, and then the transaction commit could have freed new block groups,
+ * so if we now have space to allocate do the force chunk allocation.
+ */
+static const enum btrfs_flush_state data_flush_states[] = {
+ FLUSH_DELALLOC_WAIT,
+ RUN_DELAYED_IPUTS,
+ FLUSH_DELAYED_REFS,
+ COMMIT_TRANS,
+ ALLOC_CHUNK_FORCE,
+};
+
+static void btrfs_async_reclaim_data_space(struct work_struct *work)
+{
+ struct btrfs_fs_info *fs_info;
+ struct btrfs_space_info *space_info;
+ u64 last_tickets_id;
+ int flush_state = 0;
+
+ fs_info = container_of(work, struct btrfs_fs_info, async_data_reclaim_work);
+ space_info = fs_info->data_sinfo;
+
+ spin_lock(&space_info->lock);
+ if (list_empty(&space_info->tickets)) {
+ space_info->flush = 0;
+ spin_unlock(&space_info->lock);
+ return;
+ }
+ last_tickets_id = space_info->tickets_id;
+ spin_unlock(&space_info->lock);
+
+ while (!space_info->full) {
+ flush_space(fs_info, space_info, U64_MAX, ALLOC_CHUNK_FORCE);
+ spin_lock(&space_info->lock);
+ if (list_empty(&space_info->tickets)) {
+ space_info->flush = 0;
+ spin_unlock(&space_info->lock);
+ return;
+ }
+ last_tickets_id = space_info->tickets_id;
+ spin_unlock(&space_info->lock);
+ }
+
+ while (flush_state < ARRAY_SIZE(data_flush_states)) {
+ flush_space(fs_info, space_info, U64_MAX,
+ data_flush_states[flush_state]);
+ spin_lock(&space_info->lock);
+ if (list_empty(&space_info->tickets)) {
+ space_info->flush = 0;
+ spin_unlock(&space_info->lock);
+ return;
+ }
+
+ if (last_tickets_id == space_info->tickets_id) {
+ flush_state++;
+ } else {
+ last_tickets_id = space_info->tickets_id;
+ flush_state = 0;
+ }
+
+ if (flush_state >= ARRAY_SIZE(data_flush_states)) {
+ if (space_info->full) {
+ if (maybe_fail_all_tickets(fs_info, space_info))
+ flush_state = 0;
+ else
+ space_info->flush = 0;
+ } else {
+ flush_state = 0;
+ }
+ }
+ spin_unlock(&space_info->lock);
+ }
+}
+
+void btrfs_init_async_reclaim_work(struct btrfs_fs_info *fs_info)
{
- INIT_WORK(work, btrfs_async_reclaim_metadata_space);
+ INIT_WORK(&fs_info->async_reclaim_work, btrfs_async_reclaim_metadata_space);
+ INIT_WORK(&fs_info->async_data_reclaim_work, btrfs_async_reclaim_data_space);
}
static const enum btrfs_flush_state priority_flush_states[] = {
} while (flush_state < states_nr);
}
+static void priority_reclaim_data_space(struct btrfs_fs_info *fs_info,
+ struct btrfs_space_info *space_info,
+ struct reserve_ticket *ticket)
+{
+ while (!space_info->full) {
+ flush_space(fs_info, space_info, U64_MAX, ALLOC_CHUNK_FORCE);
+ spin_lock(&space_info->lock);
+ if (ticket->bytes == 0) {
+ spin_unlock(&space_info->lock);
+ return;
+ }
+ spin_unlock(&space_info->lock);
+ }
+}
+
static void wait_reserve_ticket(struct btrfs_fs_info *fs_info,
struct btrfs_space_info *space_info,
struct reserve_ticket *ticket)
int ret;
switch (flush) {
+ case BTRFS_RESERVE_FLUSH_DATA:
case BTRFS_RESERVE_FLUSH_ALL:
case BTRFS_RESERVE_FLUSH_ALL_STEAL:
wait_reserve_ticket(fs_info, space_info, ticket);
evict_flush_states,
ARRAY_SIZE(evict_flush_states));
break;
+ case BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE:
+ priority_reclaim_data_space(fs_info, space_info, ticket);
+ break;
default:
ASSERT(0);
break;
* regain reservations will be made and this will fail if there is not enough
* space already.
*/
-static int __reserve_metadata_bytes(struct btrfs_fs_info *fs_info,
- struct btrfs_space_info *space_info,
- u64 orig_bytes,
- enum btrfs_reserve_flush_enum flush)
+static int __reserve_bytes(struct btrfs_fs_info *fs_info,
+ struct btrfs_space_info *space_info, u64 orig_bytes,
+ enum btrfs_reserve_flush_enum flush)
{
+ struct work_struct *async_work;
struct reserve_ticket ticket;
u64 used;
int ret = 0;
ASSERT(orig_bytes);
ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_ALL);
+ if (flush == BTRFS_RESERVE_FLUSH_DATA)
+ async_work = &fs_info->async_data_reclaim_work;
+ else
+ async_work = &fs_info->async_reclaim_work;
+
spin_lock(&space_info->lock);
ret = -ENOSPC;
used = btrfs_space_info_used(space_info, true);
init_waitqueue_head(&ticket.wait);
ticket.steal = (flush == BTRFS_RESERVE_FLUSH_ALL_STEAL);
if (flush == BTRFS_RESERVE_FLUSH_ALL ||
- flush == BTRFS_RESERVE_FLUSH_ALL_STEAL) {
+ flush == BTRFS_RESERVE_FLUSH_ALL_STEAL ||
+ flush == BTRFS_RESERVE_FLUSH_DATA) {
list_add_tail(&ticket.list, &space_info->tickets);
if (!space_info->flush) {
space_info->flush = 1;
space_info->flags,
orig_bytes, flush,
"enospc");
- queue_work(system_unbound_wq,
- &fs_info->async_reclaim_work);
+ queue_work(system_unbound_wq, async_work);
}
} else {
list_add_tail(&ticket.list,
struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
int ret;
- ret = __reserve_metadata_bytes(fs_info, block_rsv->space_info,
- orig_bytes, flush);
+ ret = __reserve_bytes(fs_info, block_rsv->space_info, orig_bytes, flush);
if (ret == -ENOSPC &&
unlikely(root->orphan_cleanup_state == ORPHAN_CLEANUP_STARTED)) {
if (block_rsv != global_rsv &&
}
return ret;
}
+
+/**
+ * btrfs_reserve_data_bytes - try to reserve data bytes for an allocation
+ * @fs_info - the filesystem
+ * @bytes - the number of bytes we need
+ * @flush - how we are allowed to flush
+ *
+ * This will reserve bytes from the data space info. If there is not enough
+ * space then we will attempt to flush space as specified by flush.
+ */
+int btrfs_reserve_data_bytes(struct btrfs_fs_info *fs_info, u64 bytes,
+ enum btrfs_reserve_flush_enum flush)
+{
+ struct btrfs_space_info *data_sinfo = fs_info->data_sinfo;
+ int ret;
+
+ ASSERT(flush == BTRFS_RESERVE_FLUSH_DATA ||
+ flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE);
+ ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_DATA);
+
+ ret = __reserve_bytes(fs_info, data_sinfo, bytes, flush);
+ if (ret == -ENOSPC) {
+ trace_btrfs_space_reservation(fs_info, "space_info:enospc",
+ data_sinfo->flags, bytes, 1);
+ if (btrfs_test_opt(fs_info, ENOSPC_DEBUG))
+ btrfs_dump_space_info(fs_info, data_sinfo, bytes, 0);
+ }
+ return ret;
+}
btrfs_try_granting_tickets(fs_info, space_info);
spin_unlock(&space_info->lock);
}
+int btrfs_reserve_data_bytes(struct btrfs_fs_info *fs_info, u64 bytes,
+ enum btrfs_reserve_flush_enum flush);
#endif /* BTRFS_SPACE_INFO_H */
#include "ctree.h"
-static inline u8 get_unaligned_le8(const void *p)
-{
- return *(u8 *)p;
-}
-
-static inline void put_unaligned_le8(u8 val, void *p)
-{
- *(u8 *)p = val;
-}
-
static bool check_setget_bounds(const struct extent_buffer *eb,
const void *ptr, unsigned off, int size)
{
* the filesystem is busy.
*/
cancel_work_sync(&fs_info->async_reclaim_work);
+ cancel_work_sync(&fs_info->async_data_reclaim_work);
btrfs_discard_cleanup(fs_info);
u64 thresh = 0;
int mixed = 0;
- rcu_read_lock();
- list_for_each_entry_rcu(found, &fs_info->space_info, list) {
+ list_for_each_entry(found, &fs_info->space_info, list) {
if (found->flags & BTRFS_BLOCK_GROUP_DATA) {
int i;
total_used += found->disk_used;
}
- rcu_read_unlock();
-
buf->f_blocks = div_u64(btrfs_super_total_bytes(disk_super), factor);
buf->f_blocks >>= bits;
buf->f_bfree = buf->f_blocks - (div_u64(total_used, factor) >> bits);
#include "ctree.h"
#include "discard.h"
#include "disk-io.h"
+#include "send.h"
#include "transaction.h"
#include "sysfs.h"
#include "volumes.h"
}
BTRFS_ATTR(static_feature, supported_checksums, supported_checksums_show);
+static ssize_t send_stream_version_show(struct kobject *kobj,
+ struct kobj_attribute *ka, char *buf)
+{
+ return snprintf(buf, PAGE_SIZE, "%d\n", BTRFS_SEND_STREAM_VERSION);
+}
+BTRFS_ATTR(static_feature, send_stream_version, send_stream_version_show);
+
static struct attribute *btrfs_supported_static_feature_attrs[] = {
BTRFS_ATTR_PTR(static_feature, rmdir_subvol),
BTRFS_ATTR_PTR(static_feature, supported_checksums),
+ BTRFS_ATTR_PTR(static_feature, send_stream_version),
NULL
};
BTRFS_ATTR(, checksum, btrfs_checksum_show);
+static ssize_t btrfs_exclusive_operation_show(struct kobject *kobj,
+ struct kobj_attribute *a, char *buf)
+{
+ struct btrfs_fs_info *fs_info = to_fs_info(kobj);
+ const char *str;
+
+ switch (READ_ONCE(fs_info->exclusive_operation)) {
+ case BTRFS_EXCLOP_NONE:
+ str = "none\n";
+ break;
+ case BTRFS_EXCLOP_BALANCE:
+ str = "balance\n";
+ break;
+ case BTRFS_EXCLOP_DEV_ADD:
+ str = "device add\n";
+ break;
+ case BTRFS_EXCLOP_DEV_REMOVE:
+ str = "device remove\n";
+ break;
+ case BTRFS_EXCLOP_DEV_REPLACE:
+ str = "device replace\n";
+ break;
+ case BTRFS_EXCLOP_RESIZE:
+ str = "resize\n";
+ break;
+ case BTRFS_EXCLOP_SWAP_ACTIVATE:
+ str = "swap activate\n";
+ break;
+ default:
+ str = "UNKNOWN\n";
+ break;
+ }
+ return scnprintf(buf, PAGE_SIZE, "%s", str);
+}
+BTRFS_ATTR(, exclusive_operation, btrfs_exclusive_operation_show);
+
static const struct attribute *btrfs_attrs[] = {
BTRFS_ATTR_PTR(, label),
BTRFS_ATTR_PTR(, nodesize),
BTRFS_ATTR_PTR(, quota_override),
BTRFS_ATTR_PTR(, metadata_uuid),
BTRFS_ATTR_PTR(, checksum),
+ BTRFS_ATTR_PTR(, exclusive_operation),
NULL,
};
}
}
+static void btrfs_sysfs_remove_fs_devices(struct btrfs_fs_devices *fs_devices)
+{
+ struct btrfs_device *device;
+ struct btrfs_fs_devices *seed;
+
+ list_for_each_entry(device, &fs_devices->devices, dev_list)
+ btrfs_sysfs_remove_device(device);
+
+ list_for_each_entry(seed, &fs_devices->seed_list, seed_list) {
+ list_for_each_entry(device, &seed->devices, dev_list)
+ btrfs_sysfs_remove_device(device);
+ }
+}
+
void btrfs_sysfs_remove_mounted(struct btrfs_fs_info *fs_info)
{
struct kobject *fsid_kobj = &fs_info->fs_devices->fsid_kobj;
- btrfs_reset_fs_info_ptr(fs_info);
-
sysfs_remove_link(fsid_kobj, "bdi");
if (fs_info->space_info_kobj) {
addrm_unknown_feature_attrs(fs_info, false);
sysfs_remove_group(fsid_kobj, &btrfs_feature_attr_group);
sysfs_remove_files(fsid_kobj, btrfs_attrs);
- btrfs_sysfs_remove_devices_dir(fs_info->fs_devices, NULL);
+ btrfs_sysfs_remove_fs_devices(fs_info->fs_devices);
}
static const char * const btrfs_feature_set_names[FEAT_MAX] = {
[FEAT_INCOMPAT] = "incompat",
};
-const char * const btrfs_feature_set_name(enum btrfs_feature_set set)
+const char *btrfs_feature_set_name(enum btrfs_feature_set set)
{
return btrfs_feature_set_names[set];
}
rkobj->flags = cache->flags;
kobject_init(&rkobj->kobj, &btrfs_raid_ktype);
+
+ /*
+ * We call this either on mount, or if we've created a block group for a
+ * new index type while running (i.e. when restriping). The running
+ * case is tricky because we could race with other threads, so we need
+ * to have this check to make sure we didn't already init the kobject.
+ *
+ * We don't have to protect on the free side because it only happens on
+ * unmount.
+ */
+ spin_lock(&space_info->lock);
+ if (space_info->block_group_kobjs[index]) {
+ spin_unlock(&space_info->lock);
+ kobject_put(&rkobj->kobj);
+ return;
+ } else {
+ space_info->block_group_kobjs[index] = &rkobj->kobj;
+ }
+ spin_unlock(&space_info->lock);
+
ret = kobject_add(&rkobj->kobj, &space_info->kobj, "%s",
btrfs_bg_type_to_raid_name(rkobj->flags));
memalloc_nofs_restore(nofs_flag);
if (ret) {
+ spin_lock(&space_info->lock);
+ space_info->block_group_kobjs[index] = NULL;
+ spin_unlock(&space_info->lock);
kobject_put(&rkobj->kobj);
btrfs_warn(fs_info,
"failed to add kobject for block cache, ignoring");
return;
}
-
- space_info->block_group_kobjs[index] = &rkobj->kobj;
}
/*
return 0;
}
-/* when one_device is NULL, it removes all device links */
-
-int btrfs_sysfs_remove_devices_dir(struct btrfs_fs_devices *fs_devices,
- struct btrfs_device *one_device)
+void btrfs_sysfs_remove_device(struct btrfs_device *device)
{
struct hd_struct *disk;
struct kobject *disk_kobj;
+ struct kobject *devices_kobj;
- if (!fs_devices->devices_kobj)
- return -EINVAL;
-
- if (one_device) {
- if (one_device->bdev) {
- disk = one_device->bdev->bd_part;
- disk_kobj = &part_to_dev(disk)->kobj;
- sysfs_remove_link(fs_devices->devices_kobj,
- disk_kobj->name);
- }
-
- kobject_del(&one_device->devid_kobj);
- kobject_put(&one_device->devid_kobj);
-
- wait_for_completion(&one_device->kobj_unregister);
+ /*
+ * Seed fs_devices devices_kobj aren't used, fetch kobject from the
+ * fs_info::fs_devices.
+ */
+ devices_kobj = device->fs_info->fs_devices->devices_kobj;
+ ASSERT(devices_kobj);
- return 0;
+ if (device->bdev) {
+ disk = device->bdev->bd_part;
+ disk_kobj = &part_to_dev(disk)->kobj;
+ sysfs_remove_link(devices_kobj, disk_kobj->name);
}
- list_for_each_entry(one_device, &fs_devices->devices, dev_list) {
-
- if (one_device->bdev) {
- disk = one_device->bdev->bd_part;
- disk_kobj = &part_to_dev(disk)->kobj;
- sysfs_remove_link(fs_devices->devices_kobj,
- disk_kobj->name);
- }
- kobject_del(&one_device->devid_kobj);
- kobject_put(&one_device->devid_kobj);
-
- wait_for_completion(&one_device->kobj_unregister);
+ if (device->devid_kobj.state_initialized) {
+ kobject_del(&device->devid_kobj);
+ kobject_put(&device->devid_kobj);
+ wait_for_completion(&device->kobj_unregister);
}
-
- return 0;
}
static ssize_t btrfs_devinfo_in_fs_metadata_show(struct kobject *kobj,
.release = btrfs_release_devid_kobj,
};
-int btrfs_sysfs_add_devices_dir(struct btrfs_fs_devices *fs_devices,
- struct btrfs_device *one_device)
+int btrfs_sysfs_add_device(struct btrfs_device *device)
{
- int error = 0;
- struct btrfs_device *dev;
+ int ret;
unsigned int nofs_flag;
+ struct kobject *devices_kobj;
+ struct kobject *devinfo_kobj;
- nofs_flag = memalloc_nofs_save();
- list_for_each_entry(dev, &fs_devices->devices, dev_list) {
+ /*
+ * Make sure we use the fs_info::fs_devices to fetch the kobjects even
+ * for the seed fs_devices
+ */
+ devices_kobj = device->fs_info->fs_devices->devices_kobj;
+ devinfo_kobj = device->fs_info->fs_devices->devinfo_kobj;
+ ASSERT(devices_kobj);
+ ASSERT(devinfo_kobj);
- if (one_device && one_device != dev)
- continue;
+ nofs_flag = memalloc_nofs_save();
- if (dev->bdev) {
- struct hd_struct *disk;
- struct kobject *disk_kobj;
+ if (device->bdev) {
+ struct hd_struct *disk;
+ struct kobject *disk_kobj;
- disk = dev->bdev->bd_part;
- disk_kobj = &part_to_dev(disk)->kobj;
+ disk = device->bdev->bd_part;
+ disk_kobj = &part_to_dev(disk)->kobj;
- error = sysfs_create_link(fs_devices->devices_kobj,
- disk_kobj, disk_kobj->name);
- if (error)
- break;
+ ret = sysfs_create_link(devices_kobj, disk_kobj, disk_kobj->name);
+ if (ret) {
+ btrfs_warn(device->fs_info,
+ "creating sysfs device link for devid %llu failed: %d",
+ device->devid, ret);
+ goto out;
}
+ }
- init_completion(&dev->kobj_unregister);
- error = kobject_init_and_add(&dev->devid_kobj, &devid_ktype,
- fs_devices->devinfo_kobj, "%llu",
- dev->devid);
- if (error) {
- kobject_put(&dev->devid_kobj);
- break;
- }
+ init_completion(&device->kobj_unregister);
+ ret = kobject_init_and_add(&device->devid_kobj, &devid_ktype,
+ devinfo_kobj, "%llu", device->devid);
+ if (ret) {
+ kobject_put(&device->devid_kobj);
+ btrfs_warn(device->fs_info,
+ "devinfo init for devid %llu failed: %d",
+ device->devid, ret);
}
+
+out:
memalloc_nofs_restore(nofs_flag);
+ return ret;
+}
- return error;
+static int btrfs_sysfs_add_fs_devices(struct btrfs_fs_devices *fs_devices)
+{
+ int ret;
+ struct btrfs_device *device;
+ struct btrfs_fs_devices *seed;
+
+ list_for_each_entry(device, &fs_devices->devices, dev_list) {
+ ret = btrfs_sysfs_add_device(device);
+ if (ret)
+ goto fail;
+ }
+
+ list_for_each_entry(seed, &fs_devices->seed_list, seed_list) {
+ list_for_each_entry(device, &seed->devices, dev_list) {
+ ret = btrfs_sysfs_add_device(device);
+ if (ret)
+ goto fail;
+ }
+ }
+
+ return 0;
+
+fail:
+ btrfs_sysfs_remove_fs_devices(fs_devices);
+ return ret;
}
void btrfs_kobject_uevent(struct block_device *bdev, enum kobject_action action)
&disk_to_dev(bdev->bd_disk)->kobj);
}
-void btrfs_sysfs_update_sprout_fsid(struct btrfs_fs_devices *fs_devices,
- const u8 *fsid)
+void btrfs_sysfs_update_sprout_fsid(struct btrfs_fs_devices *fs_devices)
+
{
char fsid_buf[BTRFS_UUID_UNPARSED_SIZE];
* Sprouting changes fsid of the mounted filesystem, rename the fsid
* directory
*/
- snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU", fsid);
+ snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU", fs_devices->fsid);
if (kobject_rename(&fs_devices->fsid_kobj, fsid_buf))
btrfs_warn(fs_devices->fs_info,
"sysfs: failed to create fsid for sprout");
struct btrfs_fs_devices *fs_devs = fs_info->fs_devices;
struct kobject *fsid_kobj = &fs_devs->fsid_kobj;
- btrfs_set_fs_info_ptr(fs_info);
-
- error = btrfs_sysfs_add_devices_dir(fs_devs, NULL);
+ error = btrfs_sysfs_add_fs_devices(fs_devs);
if (error)
return error;
error = sysfs_create_files(fsid_kobj, btrfs_attrs);
if (error) {
- btrfs_sysfs_remove_devices_dir(fs_devs, NULL);
+ btrfs_sysfs_remove_fs_devices(fs_devs);
return error;
}
{
struct btrfs_fs_devices *fs_devs;
struct kobject *fsid_kobj;
- u64 features;
- int ret;
+ u64 __maybe_unused features;
+ int __maybe_unused ret;
if (!fs_info)
return;
+ /*
+ * See 14e46e04958df74 and e410e34fad913dd, feature bit updates are not
+ * safe when called from some contexts (eg. balance)
+ */
features = get_features(fs_info, set);
ASSERT(bit & supported_feature_masks[set]);
};
char *btrfs_printable_features(enum btrfs_feature_set set, u64 flags);
-const char * const btrfs_feature_set_name(enum btrfs_feature_set set);
-int btrfs_sysfs_add_devices_dir(struct btrfs_fs_devices *fs_devices,
- struct btrfs_device *one_device);
-int btrfs_sysfs_remove_devices_dir(struct btrfs_fs_devices *fs_devices,
- struct btrfs_device *one_device);
+const char *btrfs_feature_set_name(enum btrfs_feature_set set);
+int btrfs_sysfs_add_device(struct btrfs_device *device);
+void btrfs_sysfs_remove_device(struct btrfs_device *device);
int btrfs_sysfs_add_fsid(struct btrfs_fs_devices *fs_devs);
void btrfs_sysfs_remove_fsid(struct btrfs_fs_devices *fs_devs);
-void btrfs_sysfs_update_sprout_fsid(struct btrfs_fs_devices *fs_devices,
- const u8 *fsid);
+void btrfs_sysfs_update_sprout_fsid(struct btrfs_fs_devices *fs_devices);
void btrfs_sysfs_feature_update(struct btrfs_fs_info *fs_info,
u64 bit, enum btrfs_feature_set set);
void btrfs_kobject_uevent(struct block_device *bdev, enum kobject_action action);
key.type = BTRFS_EXTENT_CSUM_KEY;
key.offset = 0;
- setup_items_for_insert(root, path, &key, &value_len, value_len,
- value_len + sizeof(struct btrfs_item), 1);
+ setup_items_for_insert(root, path, &key, &value_len, 1);
item = btrfs_item_nr(0);
write_extent_buffer(eb, value, btrfs_item_ptr_offset(eb, 0),
value_len);
key.type = BTRFS_EXTENT_DATA_KEY;
key.offset = start;
- setup_items_for_insert(root, &path, &key, &value_len, value_len,
- value_len + sizeof(struct btrfs_item), 1);
+ setup_items_for_insert(root, &path, &key, &value_len, 1);
fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
btrfs_set_file_extent_generation(leaf, fi, 1);
btrfs_set_file_extent_type(leaf, fi, type);
key.type = BTRFS_INODE_ITEM_KEY;
key.offset = 0;
- setup_items_for_insert(root, &path, &key, &value_len, value_len,
- value_len + sizeof(struct btrfs_item), 1);
+ setup_items_for_insert(root, &path, &key, &value_len, 1);
}
/*
}
BTRFS_I(inode)->root = root;
- btrfs_test_inode_set_ops(inode);
/* [BTRFS_MAX_EXTENT_SIZE] */
ret = btrfs_set_extent_delalloc(BTRFS_I(inode), 0,
}
cur_trans->fs_info = fs_info;
+ atomic_set(&cur_trans->pending_ordered, 0);
+ init_waitqueue_head(&cur_trans->pending_wait);
atomic_set(&cur_trans->num_writers, 1);
extwriter_counter_init(cur_trans, type);
init_waitqueue_head(&cur_trans->writer_wait);
eb = btrfs_lock_root_node(fs_info->tree_root);
ret = btrfs_cow_block(trans, fs_info->tree_root, eb, NULL,
- 0, &eb);
+ 0, &eb, BTRFS_NESTING_COW);
btrfs_tree_unlock(eb);
free_extent_buffer(eb);
btrfs_set_root_otransid(new_root_item, trans->transid);
old = btrfs_lock_root_node(root);
- ret = btrfs_cow_block(trans, root, old, NULL, 0, &old);
+ ret = btrfs_cow_block(trans, root, old, NULL, 0, &old,
+ BTRFS_NESTING_COW);
if (ret) {
btrfs_tree_unlock(old);
free_extent_buffer(old);
btrfs_wait_delalloc_flush(trans);
+ /*
+ * Wait for all ordered extents started by a fast fsync that joined this
+ * transaction. Otherwise if this transaction commits before the ordered
+ * extents complete we lose logged data after a power failure.
+ */
+ wait_event(cur_trans->pending_wait,
+ atomic_read(&cur_trans->pending_ordered) == 0);
+
btrfs_scrub_pause(fs_info);
/*
* Ok now we need to make sure to block out any other joins while we
spinlock_t dropped_roots_lock;
struct btrfs_delayed_ref_root delayed_refs;
struct btrfs_fs_info *fs_info;
+
+ /*
+ * Number of ordered extents the transaction must wait for before
+ * committing. These are ordered extents started by a fast fsync.
+ */
+ atomic_t pending_ordered;
+ wait_queue_head_t pending_wait;
};
#define __TRANS_FREEZABLE (1U << 0)
#define TRANS_EXTWRITERS (__TRANS_START | __TRANS_ATTACH)
#define BTRFS_SEND_TRANS_STUB ((void *)1)
+#define BTRFS_DIO_SYNC_STUB ((void *)2)
struct btrfs_trans_handle {
u64 transid;
int slot)
{
struct btrfs_fs_info *fs_info = leaf->fs_info;
- struct btrfs_root_item ri;
+ struct btrfs_root_item ri = { 0 };
const u64 valid_root_flags = BTRFS_ROOT_SUBVOL_RDONLY |
BTRFS_ROOT_SUBVOL_DEAD;
int ret;
if (ret < 0)
return ret;
- if (btrfs_item_size_nr(leaf, slot) != sizeof(ri)) {
+ if (btrfs_item_size_nr(leaf, slot) != sizeof(ri) &&
+ btrfs_item_size_nr(leaf, slot) != btrfs_legacy_root_item_size()) {
generic_err(leaf, slot,
- "invalid root item size, have %u expect %zu",
- btrfs_item_size_nr(leaf, slot), sizeof(ri));
+ "invalid root item size, have %u expect %zu or %u",
+ btrfs_item_size_nr(leaf, slot), sizeof(ri),
+ btrfs_legacy_root_item_size());
}
+ /*
+ * For legacy root item, the members starting at generation_v2 will be
+ * all filled with 0.
+ * And since we allow geneartion_v2 as 0, it will still pass the check.
+ */
read_extent_buffer(leaf, &ri, btrfs_item_ptr_offset(leaf, slot),
- sizeof(ri));
+ btrfs_item_size_nr(leaf, slot));
/* Generation related */
if (btrfs_root_generation(&ri) >
static int btrfs_log_inode(struct btrfs_trans_handle *trans,
struct btrfs_root *root, struct btrfs_inode *inode,
int inode_only,
- const loff_t start,
- const loff_t end,
struct btrfs_log_ctx *ctx);
static int link_to_fixup_dir(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
atomic_inc(&root->log_batch);
atomic_inc(&root->log_writers);
- if (ctx) {
+ if (ctx && !ctx->logging_new_name) {
int index = root->log_transid % 2;
list_add_tail(&ctx->list, &root->log_ctxs[index]);
ctx->log_transid = root->log_transid;
*/
void btrfs_pin_log_trans(struct btrfs_root *root)
{
- mutex_lock(&root->log_mutex);
atomic_inc(&root->log_writers);
- mutex_unlock(&root->log_mutex);
}
/*
* search and this search we'll not find the key again and can just
* bail.
*/
+search:
ret = btrfs_search_slot(NULL, root, &min_key, path, 0, 0);
if (ret != 0)
goto done;
if (min_key.objectid != ino || min_key.type != key_type)
goto done;
+
+ if (need_resched()) {
+ btrfs_release_path(path);
+ cond_resched();
+ goto search;
+ }
+
ret = overwrite_item(trans, log, dst_path, src, i,
&min_key);
if (ret) {
static int log_extent_csums(struct btrfs_trans_handle *trans,
struct btrfs_inode *inode,
struct btrfs_root *log_root,
- const struct extent_map *em)
+ const struct extent_map *em,
+ struct btrfs_log_ctx *ctx)
{
+ struct btrfs_ordered_extent *ordered;
u64 csum_offset;
u64 csum_len;
+ u64 mod_start = em->mod_start;
+ u64 mod_len = em->mod_len;
LIST_HEAD(ordered_sums);
int ret = 0;
em->block_start == EXTENT_MAP_HOLE)
return 0;
+ list_for_each_entry(ordered, &ctx->ordered_extents, log_list) {
+ const u64 ordered_end = ordered->file_offset + ordered->num_bytes;
+ const u64 mod_end = mod_start + mod_len;
+ struct btrfs_ordered_sum *sums;
+
+ if (mod_len == 0)
+ break;
+
+ if (ordered_end <= mod_start)
+ continue;
+ if (mod_end <= ordered->file_offset)
+ break;
+
+ /*
+ * We are going to copy all the csums on this ordered extent, so
+ * go ahead and adjust mod_start and mod_len in case this ordered
+ * extent has already been logged.
+ */
+ if (ordered->file_offset > mod_start) {
+ if (ordered_end >= mod_end)
+ mod_len = ordered->file_offset - mod_start;
+ /*
+ * If we have this case
+ *
+ * |--------- logged extent ---------|
+ * |----- ordered extent ----|
+ *
+ * Just don't mess with mod_start and mod_len, we'll
+ * just end up logging more csums than we need and it
+ * will be ok.
+ */
+ } else {
+ if (ordered_end < mod_end) {
+ mod_len = mod_end - ordered_end;
+ mod_start = ordered_end;
+ } else {
+ mod_len = 0;
+ }
+ }
+
+ /*
+ * To keep us from looping for the above case of an ordered
+ * extent that falls inside of the logged extent.
+ */
+ if (test_and_set_bit(BTRFS_ORDERED_LOGGED_CSUM, &ordered->flags))
+ continue;
+
+ list_for_each_entry(sums, &ordered->list, list) {
+ ret = log_csums(trans, inode, log_root, sums);
+ if (ret)
+ return ret;
+ }
+ }
+
+ /* We're done, found all csums in the ordered extents. */
+ if (mod_len == 0)
+ return 0;
+
/* If we're compressed we have to save the entire range of csums. */
if (em->compress_type) {
csum_offset = 0;
csum_len = max(em->block_len, em->orig_block_len);
} else {
- csum_offset = em->mod_start - em->start;
- csum_len = em->mod_len;
+ csum_offset = mod_start - em->start;
+ csum_len = mod_len;
}
/* block start is already adjusted for the file extent offset. */
int ret;
int extent_inserted = 0;
- ret = log_extent_csums(trans, inode, log, em);
+ ret = log_extent_csums(trans, inode, log, em, ctx);
if (ret)
return ret;
struct btrfs_root *root,
struct btrfs_inode *inode,
struct btrfs_path *path,
- struct btrfs_log_ctx *ctx,
- const u64 start,
- const u64 end)
+ struct btrfs_log_ctx *ctx)
{
+ struct btrfs_ordered_extent *ordered;
+ struct btrfs_ordered_extent *tmp;
struct extent_map *em, *n;
struct list_head extents;
struct extent_map_tree *tree = &inode->extent_tree;
test_gen = root->fs_info->last_trans_committed;
list_for_each_entry_safe(em, n, &tree->modified_extents, list) {
- /*
- * Skip extents outside our logging range. It's important to do
- * it for correctness because if we don't ignore them, we may
- * log them before their ordered extent completes, and therefore
- * we could log them without logging their respective checksums
- * (the checksum items are added to the csum tree at the very
- * end of btrfs_finish_ordered_io()). Also leave such extents
- * outside of our range in the list, since we may have another
- * ranged fsync in the near future that needs them. If an extent
- * outside our range corresponds to a hole, log it to avoid
- * leaving gaps between extents (fsck will complain when we are
- * not using the NO_HOLES feature).
- */
- if ((em->start > end || em->start + em->len <= start) &&
- em->block_start != EXTENT_MAP_HOLE)
- continue;
-
list_del_init(&em->list);
/*
* Just an arbitrary number, this can be really CPU intensive
btrfs_release_path(path);
if (!ret)
ret = btrfs_log_prealloc_extents(trans, inode, path);
+ if (ret)
+ return ret;
- return ret;
+ /*
+ * We have logged all extents successfully, now make sure the commit of
+ * the current transaction waits for the ordered extents to complete
+ * before it commits and wipes out the log trees, otherwise we would
+ * lose data if an ordered extents completes after the transaction
+ * commits and a power failure happens after the transaction commit.
+ */
+ list_for_each_entry_safe(ordered, tmp, &ctx->ordered_extents, log_list) {
+ list_del_init(&ordered->log_list);
+ set_bit(BTRFS_ORDERED_LOGGED, &ordered->flags);
+
+ if (!test_bit(BTRFS_ORDERED_COMPLETE, &ordered->flags)) {
+ spin_lock_irq(&inode->ordered_tree.lock);
+ if (!test_bit(BTRFS_ORDERED_COMPLETE, &ordered->flags)) {
+ set_bit(BTRFS_ORDERED_PENDING, &ordered->flags);
+ atomic_inc(&trans->transaction->pending_ordered);
+ }
+ spin_unlock_irq(&inode->ordered_tree.lock);
+ }
+ btrfs_put_ordered_extent(ordered);
+ }
+
+ return 0;
}
static int logged_inode_size(struct btrfs_root *log, struct btrfs_inode *inode,
ret = btrfs_log_inode(trans, root,
BTRFS_I(inode),
LOG_OTHER_INODE_ALL,
- 0, LLONG_MAX, ctx);
+ ctx);
btrfs_add_delayed_iput(inode);
}
}
* Check the inode's logged_trans only instead of
* btrfs_inode_in_log(). This is because the last_log_commit of
* the inode is not updated when we only log that it exists and
- * and it has the full sync bit set (see btrfs_log_inode()).
+ * it has the full sync bit set (see btrfs_log_inode()).
*/
if (BTRFS_I(inode)->logged_trans == trans->transid) {
spin_unlock(&BTRFS_I(inode)->lock);
* log with the new name before we unpin it.
*/
ret = btrfs_log_inode(trans, root, BTRFS_I(inode),
- LOG_OTHER_INODE, 0, LLONG_MAX, ctx);
+ LOG_OTHER_INODE, ctx);
if (ret) {
btrfs_add_delayed_iput(inode);
continue;
static int btrfs_log_inode(struct btrfs_trans_handle *trans,
struct btrfs_root *root, struct btrfs_inode *inode,
int inode_only,
- const loff_t start,
- const loff_t end,
struct btrfs_log_ctx *ctx)
{
struct btrfs_path *path;
}
if (fast_search) {
ret = btrfs_log_changed_extents(trans, root, inode, dst_path,
- ctx, start, end);
+ ctx);
if (ret) {
err = ret;
goto out_unlock;
struct extent_map *em, *n;
write_lock(&em_tree->lock);
- /*
- * We can't just remove every em if we're called for a ranged
- * fsync - that is, one that doesn't cover the whole possible
- * file range (0 to LLONG_MAX). This is because we can have
- * em's that fall outside the range we're logging and therefore
- * their ordered operations haven't completed yet
- * (btrfs_finish_ordered_io() not invoked yet). This means we
- * didn't get their respective file extent item in the fs/subvol
- * tree yet, and need to let the next fast fsync (one which
- * consults the list of modified extent maps) find the em so
- * that it logs a matching file extent item and waits for the
- * respective ordered operation to complete (if it's still
- * running).
- *
- * Removing every em outside the range we're logging would make
- * the next fast fsync not log their matching file extent items,
- * therefore making us lose data after a log replay.
- */
- list_for_each_entry_safe(em, n, &em_tree->modified_extents,
- list) {
- const u64 mod_end = em->mod_start + em->mod_len - 1;
-
- if (em->mod_start >= start && mod_end <= end)
- list_del_init(&em->list);
- }
+ list_for_each_entry_safe(em, n, &em_tree->modified_extents, list)
+ list_del_init(&em->list);
write_unlock(&em_tree->lock);
}
}
/*
- * Don't update last_log_commit if we logged that an inode exists after
- * it was loaded to memory (full_sync bit set).
- * This is to prevent data loss when we do a write to the inode, then
- * the inode gets evicted after all delalloc was flushed, then we log
- * it exists (due to a rename for example) and then fsync it. This last
- * fsync would do nothing (not logging the extents previously written).
+ * If we are logging that an ancestor inode exists as part of logging a
+ * new name from a link or rename operation, don't mark the inode as
+ * logged - otherwise if an explicit fsync is made against an ancestor,
+ * the fsync considers the inode in the log and doesn't sync the log,
+ * resulting in the ancestor missing after a power failure unless the
+ * log was synced as part of an fsync against any other unrelated inode.
+ * So keep it simple for this case and just don't flag the ancestors as
+ * logged.
*/
- spin_lock(&inode->lock);
- inode->logged_trans = trans->transid;
- if (inode_only != LOG_INODE_EXISTS ||
- !test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags))
- inode->last_log_commit = inode->last_sub_trans;
- spin_unlock(&inode->lock);
+ if (!ctx ||
+ !(S_ISDIR(inode->vfs_inode.i_mode) && ctx->logging_new_name &&
+ &inode->vfs_inode != ctx->inode)) {
+ spin_lock(&inode->lock);
+ inode->logged_trans = trans->transid;
+ /*
+ * Don't update last_log_commit if we logged that an inode exists
+ * after it was loaded to memory (full_sync bit set).
+ * This is to prevent data loss when we do a write to the inode,
+ * then the inode gets evicted after all delalloc was flushed,
+ * then we log it exists (due to a rename for example) and then
+ * fsync it. This last fsync would do nothing (not logging the
+ * extents previously written).
+ */
+ if (inode_only != LOG_INODE_EXISTS ||
+ !test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags))
+ inode->last_log_commit = inode->last_sub_trans;
+ spin_unlock(&inode->lock);
+ }
out_unlock:
mutex_unlock(&inode->log_mutex);
if (type == BTRFS_FT_DIR || type == BTRFS_FT_SYMLINK)
log_mode = LOG_INODE_ALL;
ret = btrfs_log_inode(trans, root, BTRFS_I(di_inode),
- log_mode, 0, LLONG_MAX, ctx);
+ log_mode, ctx);
if (!ret &&
btrfs_must_commit_transaction(trans, BTRFS_I(di_inode)))
ret = 1;
if (ctx)
ctx->log_new_dentries = false;
ret = btrfs_log_inode(trans, root, BTRFS_I(dir_inode),
- LOG_INODE_ALL, 0, LLONG_MAX, ctx);
+ LOG_INODE_ALL, ctx);
if (!ret &&
btrfs_must_commit_transaction(trans, BTRFS_I(dir_inode)))
ret = 1;
if (BTRFS_I(inode)->generation > last_committed)
ret = btrfs_log_inode(trans, root, BTRFS_I(inode),
- LOG_INODE_EXISTS,
- 0, LLONG_MAX, ctx);
+ LOG_INODE_EXISTS, ctx);
btrfs_add_delayed_iput(inode);
if (ret)
return ret;
if (inode->generation > fs_info->last_trans_committed) {
ret = btrfs_log_inode(trans, root, inode,
- LOG_INODE_EXISTS, 0, LLONG_MAX, ctx);
+ LOG_INODE_EXISTS, ctx);
if (ret)
break;
}
static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans,
struct btrfs_inode *inode,
struct dentry *parent,
- const loff_t start,
- const loff_t end,
int inode_only,
struct btrfs_log_ctx *ctx)
{
if (ret)
goto end_no_trans;
- ret = btrfs_log_inode(trans, root, inode, inode_only, start, end, ctx);
+ ret = btrfs_log_inode(trans, root, inode, inode_only, ctx);
if (ret)
goto end_trans;
*/
int btrfs_log_dentry_safe(struct btrfs_trans_handle *trans,
struct dentry *dentry,
- const loff_t start,
- const loff_t end,
struct btrfs_log_ctx *ctx)
{
struct dentry *parent = dget_parent(dentry);
int ret;
ret = btrfs_log_inode_parent(trans, BTRFS_I(d_inode(dentry)), parent,
- start, end, LOG_INODE_ALL, ctx);
+ LOG_INODE_ALL, ctx);
dput(parent);
return ret;
/*
* Call this after adding a new name for a file and it will properly
* update the log to reflect the new name.
- *
- * @ctx can not be NULL when @sync_log is false, and should be NULL when it's
- * true (because it's not used).
- *
- * Return value depends on whether @sync_log is true or false.
- * When true: returns BTRFS_NEED_TRANS_COMMIT if the transaction needs to be
- * committed by the caller, and BTRFS_DONT_NEED_TRANS_COMMIT
- * otherwise.
- * When false: returns BTRFS_DONT_NEED_LOG_SYNC if the caller does not need to
- * to sync the log, BTRFS_NEED_LOG_SYNC if it needs to sync the log,
- * or BTRFS_NEED_TRANS_COMMIT if the transaction needs to be
- * committed (without attempting to sync the log).
*/
-int btrfs_log_new_name(struct btrfs_trans_handle *trans,
+void btrfs_log_new_name(struct btrfs_trans_handle *trans,
struct btrfs_inode *inode, struct btrfs_inode *old_dir,
- struct dentry *parent,
- bool sync_log, struct btrfs_log_ctx *ctx)
+ struct dentry *parent)
{
struct btrfs_fs_info *fs_info = trans->fs_info;
- int ret;
+ struct btrfs_log_ctx ctx;
/*
* this will force the logging code to walk the dentry chain
*/
if (inode->logged_trans <= fs_info->last_trans_committed &&
(!old_dir || old_dir->logged_trans <= fs_info->last_trans_committed))
- return sync_log ? BTRFS_DONT_NEED_TRANS_COMMIT :
- BTRFS_DONT_NEED_LOG_SYNC;
-
- if (sync_log) {
- struct btrfs_log_ctx ctx2;
-
- btrfs_init_log_ctx(&ctx2, &inode->vfs_inode);
- ret = btrfs_log_inode_parent(trans, inode, parent, 0, LLONG_MAX,
- LOG_INODE_EXISTS, &ctx2);
- if (ret == BTRFS_NO_LOG_SYNC)
- return BTRFS_DONT_NEED_TRANS_COMMIT;
- else if (ret)
- return BTRFS_NEED_TRANS_COMMIT;
-
- ret = btrfs_sync_log(trans, inode->root, &ctx2);
- if (ret)
- return BTRFS_NEED_TRANS_COMMIT;
- return BTRFS_DONT_NEED_TRANS_COMMIT;
- }
-
- ASSERT(ctx);
- ret = btrfs_log_inode_parent(trans, inode, parent, 0, LLONG_MAX,
- LOG_INODE_EXISTS, ctx);
- if (ret == BTRFS_NO_LOG_SYNC)
- return BTRFS_DONT_NEED_LOG_SYNC;
- else if (ret)
- return BTRFS_NEED_TRANS_COMMIT;
+ return;
- return BTRFS_NEED_LOG_SYNC;
+ btrfs_init_log_ctx(&ctx, &inode->vfs_inode);
+ ctx.logging_new_name = true;
+ /*
+ * We don't care about the return value. If we fail to log the new name
+ * then we know the next attempt to sync the log will fallback to a full
+ * transaction commit (due to a call to btrfs_set_log_full_commit()), so
+ * we don't need to worry about getting a log committed that has an
+ * inconsistent state after a rename operation.
+ */
+ btrfs_log_inode_parent(trans, inode, parent, LOG_INODE_EXISTS, &ctx);
}
int log_ret;
int log_transid;
bool log_new_dentries;
+ bool logging_new_name;
struct inode *inode;
struct list_head list;
+ /* Only used for fast fsyncs. */
+ struct list_head ordered_extents;
};
static inline void btrfs_init_log_ctx(struct btrfs_log_ctx *ctx,
ctx->log_ret = 0;
ctx->log_transid = 0;
ctx->log_new_dentries = false;
+ ctx->logging_new_name = false;
ctx->inode = inode;
INIT_LIST_HEAD(&ctx->list);
+ INIT_LIST_HEAD(&ctx->ordered_extents);
+}
+
+static inline void btrfs_release_log_ctx_extents(struct btrfs_log_ctx *ctx)
+{
+ struct btrfs_ordered_extent *ordered;
+ struct btrfs_ordered_extent *tmp;
+
+ ASSERT(inode_is_locked(ctx->inode));
+
+ list_for_each_entry_safe(ordered, tmp, &ctx->ordered_extents, log_list) {
+ list_del_init(&ordered->log_list);
+ btrfs_put_ordered_extent(ordered);
+ }
}
static inline void btrfs_set_log_full_commit(struct btrfs_trans_handle *trans)
int btrfs_recover_log_trees(struct btrfs_root *tree_root);
int btrfs_log_dentry_safe(struct btrfs_trans_handle *trans,
struct dentry *dentry,
- const loff_t start,
- const loff_t end,
struct btrfs_log_ctx *ctx);
int btrfs_del_dir_entries_in_log(struct btrfs_trans_handle *trans,
struct btrfs_root *root,
int for_rename);
void btrfs_record_snapshot_destroy(struct btrfs_trans_handle *trans,
struct btrfs_inode *dir);
-/* Return values for btrfs_log_new_name() */
-enum {
- BTRFS_DONT_NEED_TRANS_COMMIT,
- BTRFS_NEED_TRANS_COMMIT,
- BTRFS_DONT_NEED_LOG_SYNC,
- BTRFS_NEED_LOG_SYNC,
-};
-int btrfs_log_new_name(struct btrfs_trans_handle *trans,
+void btrfs_log_new_name(struct btrfs_trans_handle *trans,
struct btrfs_inode *inode, struct btrfs_inode *old_dir,
- struct dentry *parent,
- bool sync_log, struct btrfs_log_ctx *ctx);
+ struct dentry *parent);
#endif
* balance_mutex
*
*
- * Exclusive operations, BTRFS_FS_EXCL_OP
- * ======================================
+ * Exclusive operations
+ * ====================
*
* Maintains the exclusivity of the following operations that apply to the
* whole filesystem and cannot run in parallel.
* - system power-cycle and filesystem mounted as read-only
* - filesystem or device errors leading to forced read-only
*
- * BTRFS_FS_EXCL_OP flag is set and cleared using atomic operations.
- * During the course of Paused state, the BTRFS_FS_EXCL_OP remains set.
+ * The status of exclusive operation is set and cleared atomically.
+ * During the course of Paused state, fs_info::exclusive_operation remains set.
* A device operation in Paused or Running state can be canceled or resumed
* either by ioctl (Balance only) or when remounted as read-write.
- * BTRFS_FS_EXCL_OP flag is cleared when the device operation is canceled or
+ * The exclusive status is cleared when the device operation is canceled or
* completed.
*/
INIT_LIST_HEAD(&fs_devs->devices);
INIT_LIST_HEAD(&fs_devs->alloc_list);
INIT_LIST_HEAD(&fs_devs->fs_list);
+ INIT_LIST_HEAD(&fs_devs->seed_list);
if (fsid)
memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
* Returned struct is not linked onto any lists and must be destroyed using
* btrfs_free_device.
*/
-static struct btrfs_device *__alloc_device(void)
+static struct btrfs_device *__alloc_device(struct btrfs_fs_info *fs_info)
{
struct btrfs_device *dev;
btrfs_device_data_ordered_init(dev);
INIT_RADIX_TREE(&dev->reada_zones, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
INIT_RADIX_TREE(&dev->reada_extents, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
- extent_io_tree_init(NULL, &dev->alloc_state, 0, NULL);
+ extent_io_tree_init(fs_info, &dev->alloc_state,
+ IO_TREE_DEVICE_ALLOC_STATE, NULL);
return dev;
}
btrfs_free_device(device);
ret = 0;
- if (fs_devices->num_devices == 0)
- break;
}
mutex_unlock(&fs_devices->device_list_mutex);
bdput(path_bdev);
mutex_unlock(&fs_devices->device_list_mutex);
btrfs_warn_in_rcu(device->fs_info,
- "duplicate device fsid:devid for %pU:%llu old:%s new:%s",
- disk_super->fsid, devid,
- rcu_str_deref(device->name), path);
+ "duplicate device %s devid %llu generation %llu scanned by %s (%d)",
+ path, devid, found_transid,
+ current->comm,
+ task_pid_nr(current));
return ERR_PTR(-EEXIST);
}
bdput(path_bdev);
btrfs_info_in_rcu(device->fs_info,
- "device fsid %pU devid %llu moved old:%s new:%s",
- disk_super->fsid, devid,
- rcu_str_deref(device->name), path);
+ "devid %llu device path %s changed to %s scanned by %s (%d)",
+ devid, rcu_str_deref(device->name),
+ path, current->comm,
+ task_pid_nr(current));
}
name = rcu_string_strdup(path, GFP_NOFS);
return ERR_PTR(ret);
}
-/*
- * After we have read the system tree and know devids belonging to
- * this filesystem, remove the device which does not belong there.
- */
-void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices, int step)
+static void __btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices,
+ int step, struct btrfs_device **latest_dev)
{
struct btrfs_device *device, *next;
- struct btrfs_device *latest_dev = NULL;
- mutex_lock(&uuid_mutex);
-again:
/* This is the initialized path, it is safe to release the devices. */
list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
- if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
- &device->dev_state)) {
+ if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state)) {
if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
- &device->dev_state) &&
+ &device->dev_state) &&
!test_bit(BTRFS_DEV_STATE_MISSING,
&device->dev_state) &&
- (!latest_dev ||
- device->generation > latest_dev->generation)) {
- latest_dev = device;
+ (!*latest_dev ||
+ device->generation > (*latest_dev)->generation)) {
+ *latest_dev = device;
}
continue;
}
btrfs_free_device(device);
}
- if (fs_devices->seed) {
- fs_devices = fs_devices->seed;
- goto again;
- }
+}
+
+/*
+ * After we have read the system tree and know devids belonging to this
+ * filesystem, remove the device which does not belong there.
+ */
+void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices, int step)
+{
+ struct btrfs_device *latest_dev = NULL;
+ struct btrfs_fs_devices *seed_dev;
+
+ mutex_lock(&uuid_mutex);
+ __btrfs_free_extra_devids(fs_devices, step, &latest_dev);
+
+ list_for_each_entry(seed_dev, &fs_devices->seed_list, seed_list)
+ __btrfs_free_extra_devids(seed_dev, step, &latest_dev);
fs_devices->latest_bdev = latest_dev->bdev;
ASSERT(atomic_read(&device->reada_in_flight) == 0);
}
-static int close_fs_devices(struct btrfs_fs_devices *fs_devices)
+static void close_fs_devices(struct btrfs_fs_devices *fs_devices)
{
struct btrfs_device *device, *tmp;
+ lockdep_assert_held(&uuid_mutex);
+
if (--fs_devices->opened > 0)
- return 0;
+ return;
- mutex_lock(&fs_devices->device_list_mutex);
- list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list) {
+ list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list)
btrfs_close_one_device(device);
- }
- mutex_unlock(&fs_devices->device_list_mutex);
WARN_ON(fs_devices->open_devices);
WARN_ON(fs_devices->rw_devices);
fs_devices->opened = 0;
fs_devices->seeding = false;
-
- return 0;
+ fs_devices->fs_info = NULL;
}
-int btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
+void btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
{
- struct btrfs_fs_devices *seed_devices = NULL;
- int ret;
+ LIST_HEAD(list);
+ struct btrfs_fs_devices *tmp;
mutex_lock(&uuid_mutex);
- ret = close_fs_devices(fs_devices);
- if (!fs_devices->opened) {
- seed_devices = fs_devices->seed;
- fs_devices->seed = NULL;
- }
- mutex_unlock(&uuid_mutex);
+ close_fs_devices(fs_devices);
+ if (!fs_devices->opened)
+ list_splice_init(&fs_devices->seed_list, &list);
- while (seed_devices) {
- fs_devices = seed_devices;
- seed_devices = fs_devices->seed;
+ list_for_each_entry_safe(fs_devices, tmp, &list, seed_list) {
close_fs_devices(fs_devices);
+ list_del(&fs_devices->seed_list);
free_fs_devices(fs_devices);
}
- return ret;
+ mutex_unlock(&uuid_mutex);
}
static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
{
struct btrfs_device *device;
struct btrfs_device *latest_dev = NULL;
+ struct btrfs_device *tmp_device;
flags |= FMODE_EXCL;
- list_for_each_entry(device, &fs_devices->devices, dev_list) {
- /* Just open everything we can; ignore failures here */
- if (btrfs_open_one_device(fs_devices, device, flags, holder))
- continue;
+ list_for_each_entry_safe(device, tmp_device, &fs_devices->devices,
+ dev_list) {
+ int ret;
- if (!latest_dev ||
- device->generation > latest_dev->generation)
+ ret = btrfs_open_one_device(fs_devices, device, flags, holder);
+ if (ret == 0 &&
+ (!latest_dev || device->generation > latest_dev->generation)) {
latest_dev = device;
+ } else if (ret == -ENODATA) {
+ fs_devices->num_devices--;
+ list_del(&device->dev_list);
+ btrfs_free_device(device);
+ }
}
if (fs_devices->open_devices == 0)
return -EINVAL;
* this_dev) which is active.
*/
void __cold btrfs_assign_next_active_device(struct btrfs_device *device,
- struct btrfs_device *this_dev)
+ struct btrfs_device *next_device)
{
struct btrfs_fs_info *fs_info = device->fs_info;
- struct btrfs_device *next_device;
- if (this_dev)
- next_device = this_dev;
- else
+ if (!next_device)
next_device = btrfs_find_next_active_device(fs_info->fs_devices,
- device);
+ device);
ASSERT(next_device);
if (fs_info->sb->s_bdev &&
return num_devices;
}
-static void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info,
- struct block_device *bdev,
- const char *device_path)
+void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info,
+ struct block_device *bdev,
+ const char *device_path)
{
struct btrfs_super_block *disk_super;
int copy_num;
}
int btrfs_rm_device(struct btrfs_fs_info *fs_info, const char *device_path,
- u64 devid)
+ u64 devid)
{
struct btrfs_device *device;
struct btrfs_fs_devices *cur_devices;
if (device->bdev) {
cur_devices->open_devices--;
/* remove sysfs entry */
- btrfs_sysfs_remove_devices_dir(fs_devices, device);
+ btrfs_sysfs_remove_device(device);
}
num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1;
btrfs_free_device(device);
if (cur_devices->open_devices == 0) {
- while (fs_devices) {
- if (fs_devices->seed == cur_devices) {
- fs_devices->seed = cur_devices->seed;
- break;
- }
- fs_devices = fs_devices->seed;
- }
- cur_devices->seed = NULL;
+ list_del_init(&cur_devices->seed_list);
close_fs_devices(cur_devices);
free_fs_devices(cur_devices);
}
void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev)
{
- struct btrfs_fs_info *fs_info = srcdev->fs_info;
struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
- if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state)) {
- /* zero out the old super if it is writable */
- btrfs_scratch_superblocks(fs_info, srcdev->bdev,
- srcdev->name->str);
- }
+ mutex_lock(&uuid_mutex);
btrfs_close_bdev(srcdev);
synchronize_rcu();
/* if this is no devs we rather delete the fs_devices */
if (!fs_devices->num_devices) {
- struct btrfs_fs_devices *tmp_fs_devices;
-
/*
* On a mounted FS, num_devices can't be zero unless it's a
* seed. In case of a seed device being replaced, the replace
*/
ASSERT(fs_devices->seeding);
- tmp_fs_devices = fs_info->fs_devices;
- while (tmp_fs_devices) {
- if (tmp_fs_devices->seed == fs_devices) {
- tmp_fs_devices->seed = fs_devices->seed;
- break;
- }
- tmp_fs_devices = tmp_fs_devices->seed;
- }
- fs_devices->seed = NULL;
+ list_del_init(&fs_devices->seed_list);
close_fs_devices(fs_devices);
free_fs_devices(fs_devices);
}
+ mutex_unlock(&uuid_mutex);
}
void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
mutex_lock(&fs_devices->device_list_mutex);
- btrfs_sysfs_remove_devices_dir(fs_devices, tgtdev);
+ btrfs_sysfs_remove_device(tgtdev);
if (tgtdev->bdev)
fs_devices->open_devices--;
if (!fs_devices->seeding)
return -EINVAL;
+ /*
+ * Private copy of the seed devices, anchored at
+ * fs_info->fs_devices->seed_list
+ */
seed_devices = alloc_fs_devices(NULL, NULL);
if (IS_ERR(seed_devices))
return PTR_ERR(seed_devices);
+ /*
+ * It's necessary to retain a copy of the original seed fs_devices in
+ * fs_uuids so that filesystems which have been seeded can successfully
+ * reference the seed device from open_seed_devices. This also supports
+ * multiple fs seed.
+ */
old_devices = clone_fs_devices(fs_devices);
if (IS_ERR(old_devices)) {
kfree(seed_devices);
list_for_each_entry(device, &seed_devices->devices, dev_list)
device->fs_devices = seed_devices;
- mutex_lock(&fs_info->chunk_mutex);
- list_splice_init(&fs_devices->alloc_list, &seed_devices->alloc_list);
- mutex_unlock(&fs_info->chunk_mutex);
-
fs_devices->seeding = false;
fs_devices->num_devices = 0;
fs_devices->open_devices = 0;
fs_devices->missing_devices = 0;
fs_devices->rotating = false;
- fs_devices->seed = seed_devices;
+ list_add(&seed_devices->seed_list, &fs_devices->seed_list);
generate_random_uuid(fs_devices->fsid);
memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE);
u64 orig_super_num_devices;
int seeding_dev = 0;
int ret = 0;
- bool unlocked = false;
+ bool locked = false;
if (sb_rdonly(sb) && !fs_devices->seeding)
return -EROFS;
seeding_dev = 1;
down_write(&sb->s_umount);
mutex_lock(&uuid_mutex);
+ locked = true;
}
- filemap_write_and_wait(bdev->bd_inode->i_mapping);
+ sync_blockdev(bdev);
- mutex_lock(&fs_devices->device_list_mutex);
- list_for_each_entry(device, &fs_devices->devices, dev_list) {
+ rcu_read_lock();
+ list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) {
if (device->bdev == bdev) {
ret = -EEXIST;
- mutex_unlock(
- &fs_devices->device_list_mutex);
+ rcu_read_unlock();
goto error;
}
}
- mutex_unlock(&fs_devices->device_list_mutex);
+ rcu_read_unlock();
device = btrfs_alloc_device(fs_info, NULL, NULL);
if (IS_ERR(device)) {
btrfs_set_super_num_devices(fs_info->super_copy,
orig_super_num_devices + 1);
- /* add sysfs device entry */
- btrfs_sysfs_add_devices_dir(fs_devices, device);
-
/*
* we've got more storage, clear any full flags on the space
* infos
btrfs_clear_space_info_full(fs_info);
mutex_unlock(&fs_info->chunk_mutex);
+
+ /* Add sysfs device entry */
+ btrfs_sysfs_add_device(device);
+
mutex_unlock(&fs_devices->device_list_mutex);
if (seeding_dev) {
goto error_sysfs;
}
- btrfs_sysfs_update_sprout_fsid(fs_devices,
- fs_info->fs_devices->fsid);
+ /*
+ * fs_devices now represents the newly sprouted filesystem and
+ * its fsid has been changed by btrfs_prepare_sprout
+ */
+ btrfs_sysfs_update_sprout_fsid(fs_devices);
}
ret = btrfs_commit_transaction(trans);
if (seeding_dev) {
mutex_unlock(&uuid_mutex);
up_write(&sb->s_umount);
- unlocked = true;
+ locked = false;
if (ret) /* transaction commit */
return ret;
return ret;
error_sysfs:
- btrfs_sysfs_remove_devices_dir(fs_devices, device);
+ btrfs_sysfs_remove_device(device);
mutex_lock(&fs_info->fs_devices->device_list_mutex);
mutex_lock(&fs_info->chunk_mutex);
list_del_rcu(&device->dev_list);
btrfs_free_device(device);
error:
blkdev_put(bdev, FMODE_EXCL);
- if (seeding_dev && !unlocked) {
+ if (locked) {
mutex_unlock(&uuid_mutex);
up_write(&sb->s_umount);
}
/*
* rw_devices will not change at the moment, device add/delete/replace
- * are excluded by EXCL_OP
+ * are exclusive
*/
num_devices = fs_info->fs_devices->rw_devices;
if ((ret && ret != -ECANCELED && ret != -ENOSPC) ||
balance_need_close(fs_info)) {
reset_balance_state(fs_info);
- clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
+ btrfs_exclop_finish(fs_info);
}
wake_up(&fs_info->balance_wait_q);
reset_balance_state(fs_info);
else
kfree(bctl);
- clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
+ btrfs_exclop_finish(fs_info);
return ret;
}
* is in a paused state and must have fs_info::balance_ctl properly
* set up.
*/
- if (test_and_set_bit(BTRFS_FS_EXCL_OP, &fs_info->flags))
+ if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE))
btrfs_warn(fs_info,
"balance: cannot set exclusive op status, resume manually");
if (fs_info->balance_ctl) {
reset_balance_state(fs_info);
- clear_bit(BTRFS_FS_EXCL_OP, &fs_info->flags);
+ btrfs_exclop_finish(fs_info);
btrfs_info(fs_info, "balance: canceled");
}
}
bool seed)
{
struct btrfs_device *device;
+ struct btrfs_fs_devices *seed_devs;
+
+ if (!fsid || !memcmp(fs_devices->metadata_uuid, fsid, BTRFS_FSID_SIZE)) {
+ list_for_each_entry(device, &fs_devices->devices, dev_list) {
+ if (device->devid == devid &&
+ (!uuid || memcmp(device->uuid, uuid,
+ BTRFS_UUID_SIZE) == 0))
+ return device;
+ }
+ }
- while (fs_devices) {
+ list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
if (!fsid ||
- !memcmp(fs_devices->metadata_uuid, fsid, BTRFS_FSID_SIZE)) {
- list_for_each_entry(device, &fs_devices->devices,
+ !memcmp(seed_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE)) {
+ list_for_each_entry(device, &seed_devs->devices,
dev_list) {
if (device->devid == devid &&
(!uuid || memcmp(device->uuid, uuid,
return device;
}
}
- if (seed)
- fs_devices = fs_devices->seed;
- else
- return NULL;
}
+
return NULL;
}
if (WARN_ON(!devid && !fs_info))
return ERR_PTR(-EINVAL);
- dev = __alloc_device();
+ dev = __alloc_device(fs_info);
if (IS_ERR(dev))
return dev;
lockdep_assert_held(&uuid_mutex);
ASSERT(fsid);
- fs_devices = fs_info->fs_devices->seed;
- while (fs_devices) {
+ /* This will match only for multi-device seed fs */
+ list_for_each_entry(fs_devices, &fs_info->fs_devices->seed_list, seed_list)
if (!memcmp(fs_devices->fsid, fsid, BTRFS_FSID_SIZE))
return fs_devices;
- fs_devices = fs_devices->seed;
- }
fs_devices = find_fsid(fsid, NULL);
if (!fs_devices) {
return fs_devices;
}
+ /*
+ * Upon first call for a seed fs fsid, just create a private copy of the
+ * respective fs_devices and anchor it at fs_info->fs_devices->seed_list
+ */
fs_devices = clone_fs_devices(fs_devices);
if (IS_ERR(fs_devices))
return fs_devices;
ret = open_fs_devices(fs_devices, FMODE_READ, fs_info->bdev_holder);
if (ret) {
free_fs_devices(fs_devices);
- fs_devices = ERR_PTR(ret);
- goto out;
+ return ERR_PTR(ret);
}
if (!fs_devices->seeding) {
close_fs_devices(fs_devices);
free_fs_devices(fs_devices);
- fs_devices = ERR_PTR(-EINVAL);
- goto out;
+ return ERR_PTR(-EINVAL);
}
- fs_devices->seed = fs_info->fs_devices->seed;
- fs_info->fs_devices->seed = fs_devices;
-out:
+ list_add(&fs_devices->seed_list, &fs_info->fs_devices->seed_list);
+
return fs_devices;
}
void btrfs_init_devices_late(struct btrfs_fs_info *fs_info)
{
- struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
+ struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
struct btrfs_device *device;
- while (fs_devices) {
- mutex_lock(&fs_devices->device_list_mutex);
- list_for_each_entry(device, &fs_devices->devices, dev_list)
+ fs_devices->fs_info = fs_info;
+
+ mutex_lock(&fs_devices->device_list_mutex);
+ list_for_each_entry(device, &fs_devices->devices, dev_list)
+ device->fs_info = fs_info;
+
+ list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
+ list_for_each_entry(device, &seed_devs->devices, dev_list)
device->fs_info = fs_info;
- mutex_unlock(&fs_devices->device_list_mutex);
- fs_devices = fs_devices->seed;
+ seed_devs->fs_info = fs_info;
}
+ mutex_unlock(&fs_devices->device_list_mutex);
}
static u64 btrfs_dev_stats_value(const struct extent_buffer *eb,
sizeof(val));
}
-int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info)
+static int btrfs_device_init_dev_stats(struct btrfs_device *device,
+ struct btrfs_path *path)
{
- struct btrfs_key key;
- struct btrfs_root *dev_root = fs_info->dev_root;
- struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
+ struct btrfs_dev_stats_item *ptr;
struct extent_buffer *eb;
- int slot;
- int ret = 0;
+ struct btrfs_key key;
+ int item_size;
+ int i, ret, slot;
+
+ key.objectid = BTRFS_DEV_STATS_OBJECTID;
+ key.type = BTRFS_PERSISTENT_ITEM_KEY;
+ key.offset = device->devid;
+ ret = btrfs_search_slot(NULL, device->fs_info->dev_root, &key, path, 0, 0);
+ if (ret) {
+ for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
+ btrfs_dev_stat_set(device, i, 0);
+ device->dev_stats_valid = 1;
+ btrfs_release_path(path);
+ return ret < 0 ? ret : 0;
+ }
+ slot = path->slots[0];
+ eb = path->nodes[0];
+ item_size = btrfs_item_size_nr(eb, slot);
+
+ ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_stats_item);
+
+ for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
+ if (item_size >= (1 + i) * sizeof(__le64))
+ btrfs_dev_stat_set(device, i,
+ btrfs_dev_stats_value(eb, ptr, i));
+ else
+ btrfs_dev_stat_set(device, i, 0);
+ }
+
+ device->dev_stats_valid = 1;
+ btrfs_dev_stat_print_on_load(device);
+ btrfs_release_path(path);
+
+ return 0;
+}
+
+int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info)
+{
+ struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
struct btrfs_device *device;
struct btrfs_path *path = NULL;
- int i;
+ int ret = 0;
path = btrfs_alloc_path();
if (!path)
mutex_lock(&fs_devices->device_list_mutex);
list_for_each_entry(device, &fs_devices->devices, dev_list) {
- int item_size;
- struct btrfs_dev_stats_item *ptr;
-
- key.objectid = BTRFS_DEV_STATS_OBJECTID;
- key.type = BTRFS_PERSISTENT_ITEM_KEY;
- key.offset = device->devid;
- ret = btrfs_search_slot(NULL, dev_root, &key, path, 0, 0);
- if (ret) {
- for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
- btrfs_dev_stat_set(device, i, 0);
- device->dev_stats_valid = 1;
- btrfs_release_path(path);
- continue;
- }
- slot = path->slots[0];
- eb = path->nodes[0];
- item_size = btrfs_item_size_nr(eb, slot);
-
- ptr = btrfs_item_ptr(eb, slot,
- struct btrfs_dev_stats_item);
-
- for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
- if (item_size >= (1 + i) * sizeof(__le64))
- btrfs_dev_stat_set(device, i,
- btrfs_dev_stats_value(eb, ptr, i));
- else
- btrfs_dev_stat_set(device, i, 0);
+ ret = btrfs_device_init_dev_stats(device, path);
+ if (ret)
+ goto out;
+ }
+ list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
+ list_for_each_entry(device, &seed_devs->devices, dev_list) {
+ ret = btrfs_device_init_dev_stats(device, path);
+ if (ret)
+ goto out;
}
-
- device->dev_stats_valid = 1;
- btrfs_dev_stat_print_on_load(device);
- btrfs_release_path(path);
}
+out:
mutex_unlock(&fs_devices->device_list_mutex);
btrfs_free_path(path);
- return ret < 0 ? ret : 0;
+ return ret;
}
static int update_dev_stat_item(struct btrfs_trans_handle *trans,
mutex_unlock(&trans->fs_info->chunk_mutex);
}
-void btrfs_set_fs_info_ptr(struct btrfs_fs_info *fs_info)
-{
- struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
- while (fs_devices) {
- fs_devices->fs_info = fs_info;
- fs_devices = fs_devices->seed;
- }
-}
-
-void btrfs_reset_fs_info_ptr(struct btrfs_fs_info *fs_info)
-{
- struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
- while (fs_devices) {
- fs_devices->fs_info = NULL;
- fs_devices = fs_devices->seed;
- }
-}
-
/*
* Multiplicity factor for simple profiles: DUP, RAID1-like and RAID10.
*/
/* It's possible this device is a dummy for seed device */
if (dev->disk_total_bytes == 0) {
- dev = btrfs_find_device(fs_info->fs_devices->seed, devid, NULL,
- NULL, false);
+ struct btrfs_fs_devices *devs;
+
+ devs = list_first_entry(&fs_info->fs_devices->seed_list,
+ struct btrfs_fs_devices, seed_list);
+ dev = btrfs_find_device(devs, devid, NULL, NULL, false);
if (!dev) {
btrfs_err(fs_info, "failed to find seed devid %llu",
devid);
struct btrfs_fs_devices *fs_devices;
struct btrfs_fs_info *fs_info;
- struct rcu_string *name;
+ struct rcu_string __rcu *name;
u64 generation;
*/
struct list_head alloc_list;
- struct btrfs_fs_devices *seed;
+ struct list_head seed_list;
bool seeding;
int opened;
struct btrfs_device *btrfs_scan_one_device(const char *path,
fmode_t flags, void *holder);
int btrfs_forget_devices(const char *path);
-int btrfs_close_devices(struct btrfs_fs_devices *fs_devices);
+void btrfs_close_devices(struct btrfs_fs_devices *fs_devices);
void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices, int step);
void btrfs_assign_next_active_device(struct btrfs_device *device,
struct btrfs_device *this_dev);
void btrfs_commit_device_sizes(struct btrfs_transaction *trans);
struct list_head * __attribute_const__ btrfs_get_fs_uuids(void);
-void btrfs_set_fs_info_ptr(struct btrfs_fs_info *fs_info);
-void btrfs_reset_fs_info_ptr(struct btrfs_fs_info *fs_info);
bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info,
struct btrfs_device *failing_dev);
+void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info,
+ struct block_device *bdev,
+ const char *device_path);
int btrfs_bg_type_to_factor(u64 flags);
const char *btrfs_bg_type_to_raid_name(u64 flags);
rqst[1].rq_iov = si_iov;
rqst[1].rq_nvec = 1;
- len = sizeof(ea) + ea_name_len + ea_value_len + 1;
+ len = sizeof(*ea) + ea_name_len + ea_value_len + 1;
ea = kzalloc(len, GFP_KERNEL);
if (ea == NULL) {
rc = -ENOMEM;
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * linux/fs/compat.c
- *
- * Kernel compatibililty routines for e.g. 32 bit syscall support
- * on 64 bit kernels.
- *
- * Copyright (C) 2002 Stephen Rothwell, IBM Corporation
- * Copyright (C) 1997-2000 Jakub Jelinek (jakub@redhat.com)
- * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be)
- * Copyright (C) 2001,2002 Andi Kleen, SuSE Labs
- * Copyright (C) 2003 Pavel Machek (pavel@ucw.cz)
- */
-
-#include <linux/compat.h>
-#include <linux/nfs4_mount.h>
-#include <linux/syscalls.h>
-#include <linux/slab.h>
-#include <linux/uaccess.h>
-#include "internal.h"
-
-struct compat_nfs_string {
- compat_uint_t len;
- compat_uptr_t data;
-};
-
-static inline void compat_nfs_string(struct nfs_string *dst,
- struct compat_nfs_string *src)
-{
- dst->data = compat_ptr(src->data);
- dst->len = src->len;
-}
-
-struct compat_nfs4_mount_data_v1 {
- compat_int_t version;
- compat_int_t flags;
- compat_int_t rsize;
- compat_int_t wsize;
- compat_int_t timeo;
- compat_int_t retrans;
- compat_int_t acregmin;
- compat_int_t acregmax;
- compat_int_t acdirmin;
- compat_int_t acdirmax;
- struct compat_nfs_string client_addr;
- struct compat_nfs_string mnt_path;
- struct compat_nfs_string hostname;
- compat_uint_t host_addrlen;
- compat_uptr_t host_addr;
- compat_int_t proto;
- compat_int_t auth_flavourlen;
- compat_uptr_t auth_flavours;
-};
-
-static int do_nfs4_super_data_conv(void *raw_data)
-{
- int version = *(compat_uint_t *) raw_data;
-
- if (version == 1) {
- struct compat_nfs4_mount_data_v1 *raw = raw_data;
- struct nfs4_mount_data *real = raw_data;
-
- /* copy the fields backwards */
- real->auth_flavours = compat_ptr(raw->auth_flavours);
- real->auth_flavourlen = raw->auth_flavourlen;
- real->proto = raw->proto;
- real->host_addr = compat_ptr(raw->host_addr);
- real->host_addrlen = raw->host_addrlen;
- compat_nfs_string(&real->hostname, &raw->hostname);
- compat_nfs_string(&real->mnt_path, &raw->mnt_path);
- compat_nfs_string(&real->client_addr, &raw->client_addr);
- real->acdirmax = raw->acdirmax;
- real->acdirmin = raw->acdirmin;
- real->acregmax = raw->acregmax;
- real->acregmin = raw->acregmin;
- real->retrans = raw->retrans;
- real->timeo = raw->timeo;
- real->wsize = raw->wsize;
- real->rsize = raw->rsize;
- real->flags = raw->flags;
- real->version = raw->version;
- }
-
- return 0;
-}
-
-#define NFS4_NAME "nfs4"
-
-COMPAT_SYSCALL_DEFINE5(mount, const char __user *, dev_name,
- const char __user *, dir_name,
- const char __user *, type, compat_ulong_t, flags,
- const void __user *, data)
-{
- char *kernel_type;
- void *options;
- char *kernel_dev;
- int retval;
-
- kernel_type = copy_mount_string(type);
- retval = PTR_ERR(kernel_type);
- if (IS_ERR(kernel_type))
- goto out;
-
- kernel_dev = copy_mount_string(dev_name);
- retval = PTR_ERR(kernel_dev);
- if (IS_ERR(kernel_dev))
- goto out1;
-
- options = copy_mount_options(data);
- retval = PTR_ERR(options);
- if (IS_ERR(options))
- goto out2;
-
- if (kernel_type && options) {
- if (!strcmp(kernel_type, NFS4_NAME)) {
- retval = -EINVAL;
- if (do_nfs4_super_data_conv(options))
- goto out3;
- }
- }
-
- retval = do_mount(kernel_dev, dir_name, kernel_type, flags, options);
-
- out3:
- kfree(options);
- out2:
- kfree(kernel_dev);
- out1:
- kfree(kernel_type);
- out:
- return retval;
-}
va_start(args, fmt);
vaf.fmt = fmt;
vaf.va = &args;
- if (inode)
+ if (inode && inode->i_ino)
printk("%sfscrypt (%s, inode %lu): %pV\n",
level, inode->i_sb->s_id, inode->i_ino, &vaf);
+ else if (inode)
+ printk("%sfscrypt (%s): %pV\n", level, inode->i_sb->s_id, &vaf);
else
printk("%sfscrypt: %pV\n", level, &vaf);
va_end(args);
*/
#define FSCRYPT_NOKEY_NAME_MAX offsetofend(struct fscrypt_nokey_name, sha256)
-static void fscrypt_do_sha256(const u8 *data, unsigned int data_len, u8 *result)
-{
- struct sha256_state sctx;
-
- sha256_init(&sctx);
- sha256_update(&sctx, data, data_len);
- sha256_final(&sctx, result);
-}
-
static inline bool fscrypt_is_dot_dotdot(const struct qstr *str)
{
if (str->len == 1 && str->name[0] == '.')
return cp - dst;
}
-bool fscrypt_fname_encrypted_size(const struct inode *inode, u32 orig_len,
- u32 max_len, u32 *encrypted_len_ret)
+bool fscrypt_fname_encrypted_size(const union fscrypt_policy *policy,
+ u32 orig_len, u32 max_len,
+ u32 *encrypted_len_ret)
{
- const struct fscrypt_info *ci = inode->i_crypt_info;
- int padding = 4 << (fscrypt_policy_flags(&ci->ci_policy) &
+ int padding = 4 << (fscrypt_policy_flags(policy) &
FSCRYPT_POLICY_FLAGS_PAD_MASK);
u32 encrypted_len;
/**
* fscrypt_fname_alloc_buffer() - allocate a buffer for presented filenames
- * @inode: inode of the parent directory (for regular filenames)
- * or of the symlink (for symlink targets)
* @max_encrypted_len: maximum length of encrypted filenames the buffer will be
* used to present
* @crypto_str: (output) buffer to allocate
*
* Return: 0 on success, -errno on failure
*/
-int fscrypt_fname_alloc_buffer(const struct inode *inode,
- u32 max_encrypted_len,
+int fscrypt_fname_alloc_buffer(u32 max_encrypted_len,
struct fscrypt_str *crypto_str)
{
const u32 max_encoded_len = BASE64_CHARS(FSCRYPT_NOKEY_NAME_MAX);
} else {
memcpy(nokey_name.bytes, iname->name, sizeof(nokey_name.bytes));
/* Compute strong hash of remaining part of name. */
- fscrypt_do_sha256(&iname->name[sizeof(nokey_name.bytes)],
- iname->len - sizeof(nokey_name.bytes),
- nokey_name.sha256);
+ sha256(&iname->name[sizeof(nokey_name.bytes)],
+ iname->len - sizeof(nokey_name.bytes),
+ nokey_name.sha256);
size = FSCRYPT_NOKEY_NAME_MAX;
}
oname->len = base64_encode((const u8 *)&nokey_name, size, oname->name);
* directory's encryption key, then @iname is the plaintext, so we encrypt it to
* get the disk_name.
*
- * Else, for keyless @lookup operations, @iname is the presented ciphertext, so
- * we decode it to get the fscrypt_nokey_name. Non-@lookup operations will be
- * impossible in this case, so we fail them with ENOKEY.
+ * Else, for keyless @lookup operations, @iname should be a no-key name, so we
+ * decode it to get the struct fscrypt_nokey_name. Non-@lookup operations will
+ * be impossible in this case, so we fail them with ENOKEY.
*
* If successful, fscrypt_free_filename() must be called later to clean up.
*
return ret;
if (fscrypt_has_encryption_key(dir)) {
- if (!fscrypt_fname_encrypted_size(dir, iname->len,
+ if (!fscrypt_fname_encrypted_size(&dir->i_crypt_info->ci_policy,
+ iname->len,
dir->i_sb->s_cop->max_namelen,
&fname->crypto_buf.len))
return -ENAMETOOLONG;
}
if (!lookup)
return -ENOKEY;
- fname->is_ciphertext_name = true;
+ fname->is_nokey_name = true;
/*
* We don't have the key and we are doing a lookup; decode the
{
const struct fscrypt_nokey_name *nokey_name =
(const void *)fname->crypto_buf.name;
- u8 sha256[SHA256_DIGEST_SIZE];
+ u8 digest[SHA256_DIGEST_SIZE];
if (likely(fname->disk_name.name)) {
if (de_name_len != fname->disk_name.len)
return false;
if (memcmp(de_name, nokey_name->bytes, sizeof(nokey_name->bytes)))
return false;
- fscrypt_do_sha256(&de_name[sizeof(nokey_name->bytes)],
- de_name_len - sizeof(nokey_name->bytes), sha256);
- return !memcmp(sha256, nokey_name->sha256, sizeof(sha256));
+ sha256(&de_name[sizeof(nokey_name->bytes)],
+ de_name_len - sizeof(nokey_name->bytes), digest);
+ return !memcmp(digest, nokey_name->sha256, sizeof(digest));
}
EXPORT_SYMBOL_GPL(fscrypt_match_name);
* Validate dentries in encrypted directories to make sure we aren't potentially
* caching stale dentries after a key has been added.
*/
-static int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags)
+int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags)
{
struct dentry *dir;
int err;
/*
* Plaintext names are always valid, since fscrypt doesn't support
- * reverting to ciphertext names without evicting the directory's inode
+ * reverting to no-key names without evicting the directory's inode
* -- which implies eviction of the dentries in the directory.
*/
- if (!(dentry->d_flags & DCACHE_ENCRYPTED_NAME))
+ if (!(dentry->d_flags & DCACHE_NOKEY_NAME))
return 1;
/*
- * Ciphertext name; valid if the directory's key is still unavailable.
+ * No-key name; valid if the directory's key is still unavailable.
*
- * Although fscrypt forbids rename() on ciphertext names, we still must
- * use dget_parent() here rather than use ->d_parent directly. That's
+ * Although fscrypt forbids rename() on no-key names, we still must use
+ * dget_parent() here rather than use ->d_parent directly. That's
* because a corrupted fs image may contain directory hard links, which
* the VFS handles by moving the directory's dentry tree in the dcache
* each time ->lookup() finds the directory and it already has a dentry
return valid;
}
+EXPORT_SYMBOL_GPL(fscrypt_d_revalidate);
const struct dentry_operations fscrypt_d_ops = {
.d_revalidate = fscrypt_d_revalidate,
return NULL;
}
-#undef fscrypt_policy
union fscrypt_policy {
u8 version;
struct fscrypt_policy_v1 v1;
/* fname.c */
int fscrypt_fname_encrypt(const struct inode *inode, const struct qstr *iname,
u8 *out, unsigned int olen);
-bool fscrypt_fname_encrypted_size(const struct inode *inode, u32 orig_len,
- u32 max_len, u32 *encrypted_len_ret);
+bool fscrypt_fname_encrypted_size(const union fscrypt_policy *policy,
+ u32 orig_len, u32 max_len,
+ u32 *encrypted_len_ret);
extern const struct dentry_operations fscrypt_d_ops;
/* hkdf.c */
int fscrypt_derive_dirhash_key(struct fscrypt_info *ci,
const struct fscrypt_master_key *mk);
+void fscrypt_hash_inode_number(struct fscrypt_info *ci,
+ const struct fscrypt_master_key *mk);
+
/* keysetup_v1.c */
void fscrypt_put_direct_key(struct fscrypt_direct_key *dk);
int fscrypt_policy_from_context(union fscrypt_policy *policy_u,
const union fscrypt_context *ctx_u,
int ctx_size);
+const union fscrypt_policy *fscrypt_policy_to_inherit(struct inode *dir);
#endif /* _FSCRYPT_PRIVATE_H */
if (err)
return err;
- /* ... in case we looked up ciphertext name before key was added */
- if (dentry->d_flags & DCACHE_ENCRYPTED_NAME)
+ /* ... in case we looked up no-key name before key was added */
+ if (dentry->d_flags & DCACHE_NOKEY_NAME)
return -ENOKEY;
if (!fscrypt_has_permitted_context(dir, inode))
if (err)
return err;
- /* ... in case we looked up ciphertext name(s) before key was added */
- if ((old_dentry->d_flags | new_dentry->d_flags) &
- DCACHE_ENCRYPTED_NAME)
+ /* ... in case we looked up no-key name(s) before key was added */
+ if ((old_dentry->d_flags | new_dentry->d_flags) & DCACHE_NOKEY_NAME)
return -ENOKEY;
if (old_dir != new_dir) {
if (err && err != -ENOENT)
return err;
- if (fname->is_ciphertext_name) {
+ if (fname->is_nokey_name) {
spin_lock(&dentry->d_lock);
- dentry->d_flags |= DCACHE_ENCRYPTED_NAME;
+ dentry->d_flags |= DCACHE_NOKEY_NAME;
spin_unlock(&dentry->d_lock);
d_set_d_op(dentry, &fscrypt_d_ops);
}
return 0;
}
-int __fscrypt_prepare_symlink(struct inode *dir, unsigned int len,
- unsigned int max_len,
- struct fscrypt_str *disk_link)
+/**
+ * fscrypt_prepare_symlink() - prepare to create a possibly-encrypted symlink
+ * @dir: directory in which the symlink is being created
+ * @target: plaintext symlink target
+ * @len: length of @target excluding null terminator
+ * @max_len: space the filesystem has available to store the symlink target
+ * @disk_link: (out) the on-disk symlink target being prepared
+ *
+ * This function computes the size the symlink target will require on-disk,
+ * stores it in @disk_link->len, and validates it against @max_len. An
+ * encrypted symlink may be longer than the original.
+ *
+ * Additionally, @disk_link->name is set to @target if the symlink will be
+ * unencrypted, but left NULL if the symlink will be encrypted. For encrypted
+ * symlinks, the filesystem must call fscrypt_encrypt_symlink() to create the
+ * on-disk target later. (The reason for the two-step process is that some
+ * filesystems need to know the size of the symlink target before creating the
+ * inode, e.g. to determine whether it will be a "fast" or "slow" symlink.)
+ *
+ * Return: 0 on success, -ENAMETOOLONG if the symlink target is too long,
+ * -ENOKEY if the encryption key is missing, or another -errno code if a problem
+ * occurred while setting up the encryption key.
+ */
+int fscrypt_prepare_symlink(struct inode *dir, const char *target,
+ unsigned int len, unsigned int max_len,
+ struct fscrypt_str *disk_link)
{
- int err;
+ const union fscrypt_policy *policy;
/*
* To calculate the size of the encrypted symlink target we need to know
* the amount of NUL padding, which is determined by the flags set in
* the encryption policy which will be inherited from the directory.
- * The easiest way to get access to this is to just load the directory's
- * fscrypt_info, since we'll need it to create the dir_entry anyway.
- *
- * Note: in test_dummy_encryption mode, @dir may be unencrypted.
*/
- err = fscrypt_get_encryption_info(dir);
- if (err)
- return err;
- if (!fscrypt_has_encryption_key(dir))
- return -ENOKEY;
+ policy = fscrypt_policy_to_inherit(dir);
+ if (policy == NULL) {
+ /* Not encrypted */
+ disk_link->name = (unsigned char *)target;
+ disk_link->len = len + 1;
+ if (disk_link->len > max_len)
+ return -ENAMETOOLONG;
+ return 0;
+ }
+ if (IS_ERR(policy))
+ return PTR_ERR(policy);
/*
* Calculate the size of the encrypted symlink and verify it won't
* counting it (even though it is meaningless for ciphertext) is simpler
* for now since filesystems will assume it is there and subtract it.
*/
- if (!fscrypt_fname_encrypted_size(dir, len,
+ if (!fscrypt_fname_encrypted_size(policy, len,
max_len - sizeof(struct fscrypt_symlink_data),
&disk_link->len))
return -ENAMETOOLONG;
disk_link->name = NULL;
return 0;
}
-EXPORT_SYMBOL_GPL(__fscrypt_prepare_symlink);
+EXPORT_SYMBOL_GPL(fscrypt_prepare_symlink);
int __fscrypt_encrypt_symlink(struct inode *inode, const char *target,
unsigned int len, struct fscrypt_str *disk_link)
struct fscrypt_symlink_data *sd;
unsigned int ciphertext_len;
- err = fscrypt_require_key(inode);
- if (err)
- return err;
+ /*
+ * fscrypt_prepare_new_inode() should have already set up the new
+ * symlink inode's encryption key. We don't wait until now to do it,
+ * since we may be in a filesystem transaction now.
+ */
+ if (WARN_ON_ONCE(!fscrypt_has_encryption_key(inode)))
+ return -ENOKEY;
if (disk_link->name) {
/* filesystem-provided buffer */
if (cstr.len + sizeof(*sd) - 1 > max_size)
return ERR_PTR(-EUCLEAN);
- err = fscrypt_fname_alloc_buffer(inode, cstr.len, &pstr);
+ err = fscrypt_fname_alloc_buffer(cstr.len, &pstr);
if (err)
return ERR_PTR(err);
crypto_cfg.data_unit_size = sb->s_blocksize;
crypto_cfg.dun_bytes = fscrypt_get_dun_bytes(ci);
num_devs = fscrypt_get_num_devices(sb);
- devs = kmalloc_array(num_devs, sizeof(*devs), GFP_NOFS);
+ devs = kmalloc_array(num_devs, sizeof(*devs), GFP_KERNEL);
if (!devs)
return -ENOMEM;
fscrypt_get_devices(sb, num_devs, devs);
struct fscrypt_blk_crypto_key *blk_key;
int err;
int i;
- unsigned int flags;
- blk_key = kzalloc(struct_size(blk_key, devs, num_devs), GFP_NOFS);
+ blk_key = kzalloc(struct_size(blk_key, devs, num_devs), GFP_KERNEL);
if (!blk_key)
return -ENOMEM;
}
queue_refs++;
- flags = memalloc_nofs_save();
err = blk_crypto_start_using_key(&blk_key->base,
blk_key->devs[i]);
- memalloc_nofs_restore(flags);
if (err) {
fscrypt_err(inode,
"error %d starting to use blk-crypto", err);
struct list_head *pos;
size_t busy_count = 0;
unsigned long ino;
+ char ino_str[50] = "";
spin_lock(&mk->mk_decrypted_inodes_lock);
}
spin_unlock(&mk->mk_decrypted_inodes_lock);
+ /* If the inode is currently being created, ino may still be 0. */
+ if (ino)
+ snprintf(ino_str, sizeof(ino_str), ", including ino %lu", ino);
+
fscrypt_warn(NULL,
- "%s: %zu inode(s) still busy after removing key with %s %*phN, including ino %lu",
+ "%s: %zu inode(s) still busy after removing key with %s %*phN%s",
sb->s_id, busy_count, master_key_spec_type(&mk->mk_spec),
master_key_spec_len(&mk->mk_spec), (u8 *)&mk->mk_spec.u,
- ino);
+ ino_str);
return -EBUSY;
}
#include <crypto/skcipher.h>
#include <linux/key.h>
+#include <linux/random.h>
#include "fscrypt_private.h"
return 0;
}
+void fscrypt_hash_inode_number(struct fscrypt_info *ci,
+ const struct fscrypt_master_key *mk)
+{
+ WARN_ON(ci->ci_inode->i_ino == 0);
+ WARN_ON(!mk->mk_ino_hash_key_initialized);
+
+ ci->ci_hashed_ino = (u32)siphash_1u64(ci->ci_inode->i_ino,
+ &mk->mk_ino_hash_key);
+}
+
static int fscrypt_setup_iv_ino_lblk_32_key(struct fscrypt_info *ci,
struct fscrypt_master_key *mk)
{
return err;
}
- ci->ci_hashed_ino = (u32)siphash_1u64(ci->ci_inode->i_ino,
- &mk->mk_ino_hash_key);
+ /*
+ * New inodes may not have an inode number assigned yet.
+ * Hashing their inode number is delayed until later.
+ */
+ if (ci->ci_inode->i_ino == 0)
+ WARN_ON(!(ci->ci_inode->i_state & I_CREATING));
+ else
+ fscrypt_hash_inode_number(ci, mk);
return 0;
}
static int fscrypt_setup_v2_file_key(struct fscrypt_info *ci,
- struct fscrypt_master_key *mk)
+ struct fscrypt_master_key *mk,
+ bool need_dirhash_key)
{
int err;
return err;
/* Derive a secret dirhash key for directories that need it. */
- if (S_ISDIR(ci->ci_inode->i_mode) && IS_CASEFOLDED(ci->ci_inode)) {
+ if (need_dirhash_key) {
err = fscrypt_derive_dirhash_key(ci, mk);
if (err)
return err;
* key being removed with a new inode starting to use it.
*/
static int setup_file_encryption_key(struct fscrypt_info *ci,
+ bool need_dirhash_key,
struct key **master_key_ret)
{
struct key *key;
err = fscrypt_setup_v1_file_key(ci, mk->mk_secret.raw);
break;
case FSCRYPT_POLICY_V2:
- err = fscrypt_setup_v2_file_key(ci, mk);
+ err = fscrypt_setup_v2_file_key(ci, mk, need_dirhash_key);
break;
default:
WARN_ON(1);
kmem_cache_free(fscrypt_info_cachep, ci);
}
-int fscrypt_get_encryption_info(struct inode *inode)
+static int
+fscrypt_setup_encryption_info(struct inode *inode,
+ const union fscrypt_policy *policy,
+ const u8 nonce[FSCRYPT_FILE_NONCE_SIZE],
+ bool need_dirhash_key)
{
struct fscrypt_info *crypt_info;
- union fscrypt_context ctx;
struct fscrypt_mode *mode;
struct key *master_key = NULL;
int res;
- if (fscrypt_has_encryption_key(inode))
- return 0;
-
res = fscrypt_initialize(inode->i_sb->s_cop->flags);
if (res)
return res;
- res = inode->i_sb->s_cop->get_context(inode, &ctx, sizeof(ctx));
- if (res < 0) {
- const union fscrypt_context *dummy_ctx =
- fscrypt_get_dummy_context(inode->i_sb);
-
- if (IS_ENCRYPTED(inode) || !dummy_ctx) {
- fscrypt_warn(inode,
- "Error %d getting encryption context",
- res);
- return res;
- }
- /* Fake up a context for an unencrypted directory */
- res = fscrypt_context_size(dummy_ctx);
- memcpy(&ctx, dummy_ctx, res);
- }
-
- crypt_info = kmem_cache_zalloc(fscrypt_info_cachep, GFP_NOFS);
+ crypt_info = kmem_cache_zalloc(fscrypt_info_cachep, GFP_KERNEL);
if (!crypt_info)
return -ENOMEM;
crypt_info->ci_inode = inode;
-
- res = fscrypt_policy_from_context(&crypt_info->ci_policy, &ctx, res);
- if (res) {
- fscrypt_warn(inode,
- "Unrecognized or corrupt encryption context");
- goto out;
- }
-
- memcpy(crypt_info->ci_nonce, fscrypt_context_nonce(&ctx),
- FSCRYPT_FILE_NONCE_SIZE);
-
- if (!fscrypt_supported_policy(&crypt_info->ci_policy, inode)) {
- res = -EINVAL;
- goto out;
- }
+ crypt_info->ci_policy = *policy;
+ memcpy(crypt_info->ci_nonce, nonce, FSCRYPT_FILE_NONCE_SIZE);
mode = select_encryption_mode(&crypt_info->ci_policy, inode);
if (IS_ERR(mode)) {
WARN_ON(mode->ivsize > FSCRYPT_MAX_IV_SIZE);
crypt_info->ci_mode = mode;
- res = setup_file_encryption_key(crypt_info, &master_key);
+ res = setup_file_encryption_key(crypt_info, need_dirhash_key,
+ &master_key);
if (res)
goto out;
/*
- * Multiple tasks may race to set ->i_crypt_info, so use
- * cmpxchg_release(). This pairs with the smp_load_acquire() in
+ * For existing inodes, multiple tasks may race to set ->i_crypt_info.
+ * So use cmpxchg_release(). This pairs with the smp_load_acquire() in
* fscrypt_get_info(). I.e., here we publish ->i_crypt_info with a
* RELEASE barrier so that other tasks can ACQUIRE it.
*/
up_read(&mk->mk_secret_sem);
key_put(master_key);
}
+ put_crypt_info(crypt_info);
+ return res;
+}
+
+/**
+ * fscrypt_get_encryption_info() - set up an inode's encryption key
+ * @inode: the inode to set up the key for. Must be encrypted.
+ *
+ * Set up ->i_crypt_info, if it hasn't already been done.
+ *
+ * Note: unless ->i_crypt_info is already set, this isn't %GFP_NOFS-safe. So
+ * generally this shouldn't be called from within a filesystem transaction.
+ *
+ * Return: 0 if ->i_crypt_info was set or was already set, *or* if the
+ * encryption key is unavailable. (Use fscrypt_has_encryption_key() to
+ * distinguish these cases.) Also can return another -errno code.
+ */
+int fscrypt_get_encryption_info(struct inode *inode)
+{
+ int res;
+ union fscrypt_context ctx;
+ union fscrypt_policy policy;
+
+ if (fscrypt_has_encryption_key(inode))
+ return 0;
+
+ res = inode->i_sb->s_cop->get_context(inode, &ctx, sizeof(ctx));
+ if (res < 0) {
+ fscrypt_warn(inode, "Error %d getting encryption context", res);
+ return res;
+ }
+
+ res = fscrypt_policy_from_context(&policy, &ctx, res);
+ if (res) {
+ fscrypt_warn(inode,
+ "Unrecognized or corrupt encryption context");
+ return res;
+ }
+
+ if (!fscrypt_supported_policy(&policy, inode))
+ return -EINVAL;
+
+ res = fscrypt_setup_encryption_info(inode, &policy,
+ fscrypt_context_nonce(&ctx),
+ IS_CASEFOLDED(inode) &&
+ S_ISDIR(inode->i_mode));
if (res == -ENOKEY)
res = 0;
- put_crypt_info(crypt_info);
return res;
}
EXPORT_SYMBOL(fscrypt_get_encryption_info);
+/**
+ * fscrypt_prepare_new_inode() - prepare to create a new inode in a directory
+ * @dir: a possibly-encrypted directory
+ * @inode: the new inode. ->i_mode must be set already.
+ * ->i_ino doesn't need to be set yet.
+ * @encrypt_ret: (output) set to %true if the new inode will be encrypted
+ *
+ * If the directory is encrypted, set up its ->i_crypt_info in preparation for
+ * encrypting the name of the new file. Also, if the new inode will be
+ * encrypted, set up its ->i_crypt_info and set *encrypt_ret=true.
+ *
+ * This isn't %GFP_NOFS-safe, and therefore it should be called before starting
+ * any filesystem transaction to create the inode. For this reason, ->i_ino
+ * isn't required to be set yet, as the filesystem may not have set it yet.
+ *
+ * This doesn't persist the new inode's encryption context. That still needs to
+ * be done later by calling fscrypt_set_context().
+ *
+ * Return: 0 on success, -ENOKEY if the encryption key is missing, or another
+ * -errno code
+ */
+int fscrypt_prepare_new_inode(struct inode *dir, struct inode *inode,
+ bool *encrypt_ret)
+{
+ const union fscrypt_policy *policy;
+ u8 nonce[FSCRYPT_FILE_NONCE_SIZE];
+
+ policy = fscrypt_policy_to_inherit(dir);
+ if (policy == NULL)
+ return 0;
+ if (IS_ERR(policy))
+ return PTR_ERR(policy);
+
+ if (WARN_ON_ONCE(inode->i_mode == 0))
+ return -EINVAL;
+
+ /*
+ * Only regular files, directories, and symlinks are encrypted.
+ * Special files like device nodes and named pipes aren't.
+ */
+ if (!S_ISREG(inode->i_mode) &&
+ !S_ISDIR(inode->i_mode) &&
+ !S_ISLNK(inode->i_mode))
+ return 0;
+
+ *encrypt_ret = true;
+
+ get_random_bytes(nonce, FSCRYPT_FILE_NONCE_SIZE);
+ return fscrypt_setup_encryption_info(inode, policy, nonce,
+ IS_CASEFOLDED(dir) &&
+ S_ISDIR(inode->i_mode));
+}
+EXPORT_SYMBOL_GPL(fscrypt_prepare_new_inode);
+
/**
* fscrypt_put_encryption_info() - free most of an inode's fscrypt data
* @inode: an inode being evicted
goto out;
}
crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
- req = skcipher_request_alloc(tfm, GFP_NOFS);
+ req = skcipher_request_alloc(tfm, GFP_KERNEL);
if (!req) {
res = -ENOMEM;
goto out;
const struct user_key_payload *ukp;
const struct fscrypt_key *payload;
- description = kasprintf(GFP_NOFS, "%s%*phN", prefix,
+ description = kasprintf(GFP_KERNEL, "%s%*phN", prefix,
FSCRYPT_KEY_DESCRIPTOR_SIZE, descriptor);
if (!description)
return ERR_PTR(-ENOMEM);
return dk;
/* Nope, allocate one. */
- dk = kzalloc(sizeof(*dk), GFP_NOFS);
+ dk = kzalloc(sizeof(*dk), GFP_KERNEL);
if (!dk)
return ERR_PTR(-ENOMEM);
refcount_set(&dk->dk_refcount, 1);
* This cannot be a stack buffer because it will be passed to the
* scatterlist crypto API during derive_key_aes().
*/
- derived_key = kmalloc(ci->ci_mode->keysize, GFP_NOFS);
+ derived_key = kmalloc(ci->ci_mode->keysize, GFP_KERNEL);
if (!derived_key)
return -ENOMEM;
return !memcmp(policy1, policy2, fscrypt_policy_size(policy1));
}
+static const union fscrypt_policy *
+fscrypt_get_dummy_policy(struct super_block *sb)
+{
+ if (!sb->s_cop->get_dummy_policy)
+ return NULL;
+ return sb->s_cop->get_dummy_policy(sb);
+}
+
static bool fscrypt_valid_enc_modes(u32 contents_mode, u32 filenames_mode)
{
if (contents_mode == FSCRYPT_MODE_AES_256_XTS &&
32, 32))
return false;
+ /*
+ * IV_INO_LBLK_32 hashes the inode number, so in principle it can
+ * support any ino_bits. However, currently the inode number is gotten
+ * from inode::i_ino which is 'unsigned long'. So for now the
+ * implementation limit is 32 bits.
+ */
if ((policy->flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) &&
- /* This uses hashed inode numbers, so ino_bits doesn't matter. */
!supported_iv_ino_lblk_policy(policy, inode, "IV_INO_LBLK_32",
- INT_MAX, 32))
+ 32, 32))
return false;
if (memchr_inv(policy->__reserved, 0, sizeof(policy->__reserved))) {
}
/**
- * fscrypt_new_context_from_policy() - create a new fscrypt_context from
- * an fscrypt_policy
+ * fscrypt_new_context() - create a new fscrypt_context
* @ctx_u: output context
* @policy_u: input policy
+ * @nonce: nonce to use
*
* Create an fscrypt_context for an inode that is being assigned the given
- * encryption policy. A new nonce is randomly generated.
+ * encryption policy. @nonce must be a new random nonce.
*
* Return: the size of the new context in bytes.
*/
-static int fscrypt_new_context_from_policy(union fscrypt_context *ctx_u,
- const union fscrypt_policy *policy_u)
+static int fscrypt_new_context(union fscrypt_context *ctx_u,
+ const union fscrypt_policy *policy_u,
+ const u8 nonce[FSCRYPT_FILE_NONCE_SIZE])
{
memset(ctx_u, 0, sizeof(*ctx_u));
memcpy(ctx->master_key_descriptor,
policy->master_key_descriptor,
sizeof(ctx->master_key_descriptor));
- get_random_bytes(ctx->nonce, sizeof(ctx->nonce));
+ memcpy(ctx->nonce, nonce, FSCRYPT_FILE_NONCE_SIZE);
return sizeof(*ctx);
}
case FSCRYPT_POLICY_V2: {
memcpy(ctx->master_key_identifier,
policy->master_key_identifier,
sizeof(ctx->master_key_identifier));
- get_random_bytes(ctx->nonce, sizeof(ctx->nonce));
+ memcpy(ctx->nonce, nonce, FSCRYPT_FILE_NONCE_SIZE);
return sizeof(*ctx);
}
}
static int set_encryption_policy(struct inode *inode,
const union fscrypt_policy *policy)
{
+ u8 nonce[FSCRYPT_FILE_NONCE_SIZE];
union fscrypt_context ctx;
int ctxsize;
int err;
return -EINVAL;
}
- ctxsize = fscrypt_new_context_from_policy(&ctx, policy);
+ get_random_bytes(nonce, FSCRYPT_FILE_NONCE_SIZE);
+ ctxsize = fscrypt_new_context(&ctx, policy, nonce);
return inode->i_sb->s_cop->set_context(inode, &ctx, ctxsize, NULL);
}
}
EXPORT_SYMBOL(fscrypt_has_permitted_context);
+/*
+ * Return the encryption policy that new files in the directory will inherit, or
+ * NULL if none, or an ERR_PTR() on error. If the directory is encrypted, also
+ * ensure that its key is set up, so that the new filename can be encrypted.
+ */
+const union fscrypt_policy *fscrypt_policy_to_inherit(struct inode *dir)
+{
+ int err;
+
+ if (IS_ENCRYPTED(dir)) {
+ err = fscrypt_require_key(dir);
+ if (err)
+ return ERR_PTR(err);
+ return &dir->i_crypt_info->ci_policy;
+ }
+
+ return fscrypt_get_dummy_policy(dir->i_sb);
+}
+
/**
- * fscrypt_inherit_context() - Sets a child context from its parent
- * @parent: Parent inode from which the context is inherited.
- * @child: Child inode that inherits the context from @parent.
- * @fs_data: private data given by FS.
- * @preload: preload child i_crypt_info if true
+ * fscrypt_set_context() - Set the fscrypt context of a new inode
+ * @inode: a new inode
+ * @fs_data: private data given by FS and passed to ->set_context()
+ *
+ * This should be called after fscrypt_prepare_new_inode(), generally during a
+ * filesystem transaction. Everything here must be %GFP_NOFS-safe.
*
* Return: 0 on success, -errno on failure
*/
-int fscrypt_inherit_context(struct inode *parent, struct inode *child,
- void *fs_data, bool preload)
+int fscrypt_set_context(struct inode *inode, void *fs_data)
{
+ struct fscrypt_info *ci = inode->i_crypt_info;
union fscrypt_context ctx;
int ctxsize;
- struct fscrypt_info *ci;
- int res;
-
- res = fscrypt_get_encryption_info(parent);
- if (res < 0)
- return res;
- ci = fscrypt_get_info(parent);
- if (ci == NULL)
+ /* fscrypt_prepare_new_inode() should have set up the key already. */
+ if (WARN_ON_ONCE(!ci))
return -ENOKEY;
- ctxsize = fscrypt_new_context_from_policy(&ctx, &ci->ci_policy);
-
BUILD_BUG_ON(sizeof(ctx) != FSCRYPT_SET_CONTEXT_MAX_SIZE);
- res = parent->i_sb->s_cop->set_context(child, &ctx, ctxsize, fs_data);
- if (res)
- return res;
- return preload ? fscrypt_get_encryption_info(child): 0;
+ ctxsize = fscrypt_new_context(&ctx, &ci->ci_policy, ci->ci_nonce);
+
+ /*
+ * This may be the first time the inode number is available, so do any
+ * delayed key setup that requires the inode number.
+ */
+ if (ci->ci_policy.version == FSCRYPT_POLICY_V2 &&
+ (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) {
+ const struct fscrypt_master_key *mk =
+ ci->ci_master_key->payload.data[0];
+
+ fscrypt_hash_inode_number(ci, mk);
+ }
+
+ return inode->i_sb->s_cop->set_context(inode, &ctx, ctxsize, fs_data);
}
-EXPORT_SYMBOL(fscrypt_inherit_context);
+EXPORT_SYMBOL_GPL(fscrypt_set_context);
/**
* fscrypt_set_test_dummy_encryption() - handle '-o test_dummy_encryption'
* @sb: the filesystem on which test_dummy_encryption is being specified
- * @arg: the argument to the test_dummy_encryption option.
- * If no argument was specified, then @arg->from == NULL.
- * @dummy_ctx: the filesystem's current dummy context (input/output, see below)
+ * @arg: the argument to the test_dummy_encryption option. May be NULL.
+ * @dummy_policy: the filesystem's current dummy policy (input/output, see
+ * below)
*
* Handle the test_dummy_encryption mount option by creating a dummy encryption
- * context, saving it in @dummy_ctx, and adding the corresponding dummy
- * encryption key to the filesystem. If the @dummy_ctx is already set, then
+ * policy, saving it in @dummy_policy, and adding the corresponding dummy
+ * encryption key to the filesystem. If the @dummy_policy is already set, then
* instead validate that it matches @arg. Don't support changing it via
* remount, as that is difficult to do safely.
*
- * The reason we use an fscrypt_context rather than an fscrypt_policy is because
- * we mustn't generate a new nonce each time we access a dummy-encrypted
- * directory, as that would change the way filenames are encrypted.
- *
- * Return: 0 on success (dummy context set, or the same context is already set);
- * -EEXIST if a different dummy context is already set;
+ * Return: 0 on success (dummy policy set, or the same policy is already set);
+ * -EEXIST if a different dummy policy is already set;
* or another -errno value.
*/
-int fscrypt_set_test_dummy_encryption(struct super_block *sb,
- const substring_t *arg,
- struct fscrypt_dummy_context *dummy_ctx)
+int fscrypt_set_test_dummy_encryption(struct super_block *sb, const char *arg,
+ struct fscrypt_dummy_policy *dummy_policy)
{
- const char *argstr = "v2";
- const char *argstr_to_free = NULL;
struct fscrypt_key_specifier key_spec = { 0 };
int version;
- union fscrypt_context *ctx = NULL;
+ union fscrypt_policy *policy = NULL;
int err;
- if (arg->from) {
- argstr = argstr_to_free = match_strdup(arg);
- if (!argstr)
- return -ENOMEM;
- }
+ if (!arg)
+ arg = "v2";
- if (!strcmp(argstr, "v1")) {
- version = FSCRYPT_CONTEXT_V1;
+ if (!strcmp(arg, "v1")) {
+ version = FSCRYPT_POLICY_V1;
key_spec.type = FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR;
memset(key_spec.u.descriptor, 0x42,
FSCRYPT_KEY_DESCRIPTOR_SIZE);
- } else if (!strcmp(argstr, "v2")) {
- version = FSCRYPT_CONTEXT_V2;
+ } else if (!strcmp(arg, "v2")) {
+ version = FSCRYPT_POLICY_V2;
key_spec.type = FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER;
/* key_spec.u.identifier gets filled in when adding the key */
} else {
goto out;
}
- if (dummy_ctx->ctx) {
- /*
- * Note: if we ever make test_dummy_encryption support
- * specifying other encryption settings, such as the encryption
- * modes, we'll need to compare those settings here.
- */
- if (dummy_ctx->ctx->version == version)
- err = 0;
- else
- err = -EEXIST;
- goto out;
- }
-
- ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
- if (!ctx) {
+ policy = kzalloc(sizeof(*policy), GFP_KERNEL);
+ if (!policy) {
err = -ENOMEM;
goto out;
}
if (err)
goto out;
- ctx->version = version;
- switch (ctx->version) {
- case FSCRYPT_CONTEXT_V1:
- ctx->v1.contents_encryption_mode = FSCRYPT_MODE_AES_256_XTS;
- ctx->v1.filenames_encryption_mode = FSCRYPT_MODE_AES_256_CTS;
- memcpy(ctx->v1.master_key_descriptor, key_spec.u.descriptor,
+ policy->version = version;
+ switch (policy->version) {
+ case FSCRYPT_POLICY_V1:
+ policy->v1.contents_encryption_mode = FSCRYPT_MODE_AES_256_XTS;
+ policy->v1.filenames_encryption_mode = FSCRYPT_MODE_AES_256_CTS;
+ memcpy(policy->v1.master_key_descriptor, key_spec.u.descriptor,
FSCRYPT_KEY_DESCRIPTOR_SIZE);
break;
- case FSCRYPT_CONTEXT_V2:
- ctx->v2.contents_encryption_mode = FSCRYPT_MODE_AES_256_XTS;
- ctx->v2.filenames_encryption_mode = FSCRYPT_MODE_AES_256_CTS;
- memcpy(ctx->v2.master_key_identifier, key_spec.u.identifier,
+ case FSCRYPT_POLICY_V2:
+ policy->v2.contents_encryption_mode = FSCRYPT_MODE_AES_256_XTS;
+ policy->v2.filenames_encryption_mode = FSCRYPT_MODE_AES_256_CTS;
+ memcpy(policy->v2.master_key_identifier, key_spec.u.identifier,
FSCRYPT_KEY_IDENTIFIER_SIZE);
break;
default:
err = -EINVAL;
goto out;
}
- dummy_ctx->ctx = ctx;
- ctx = NULL;
+
+ if (dummy_policy->policy) {
+ if (fscrypt_policies_equal(policy, dummy_policy->policy))
+ err = 0;
+ else
+ err = -EEXIST;
+ goto out;
+ }
+ dummy_policy->policy = policy;
+ policy = NULL;
err = 0;
out:
- kfree(ctx);
- kfree(argstr_to_free);
+ kfree(policy);
return err;
}
EXPORT_SYMBOL_GPL(fscrypt_set_test_dummy_encryption);
void fscrypt_show_test_dummy_encryption(struct seq_file *seq, char sep,
struct super_block *sb)
{
- const union fscrypt_context *ctx = fscrypt_get_dummy_context(sb);
+ const union fscrypt_policy *policy = fscrypt_get_dummy_policy(sb);
+ int vers;
- if (!ctx)
+ if (!policy)
return;
- seq_printf(seq, "%ctest_dummy_encryption=v%d", sep, ctx->version);
+
+ vers = policy->version;
+ if (vers == FSCRYPT_POLICY_V1) /* Handle numbering quirk */
+ vers = 1;
+
+ seq_printf(seq, "%ctest_dummy_encryption=v%d", sep, vers);
}
EXPORT_SYMBOL_GPL(fscrypt_show_test_dummy_encryption);
spin_unlock_irqrestore(&dio->bio_lock, flags);
}
-/**
- * dio_end_io - handle the end io action for the given bio
- * @bio: The direct io bio thats being completed
- *
- * This is meant to be called by any filesystem that uses their own dio_submit_t
- * so that the DIO specific endio actions are dealt with after the filesystem
- * has done it's completion work.
- */
-void dio_end_io(struct bio *bio)
-{
- struct dio *dio = bio->bi_private;
-
- if (dio->is_async)
- dio_bio_end_aio(bio);
- else
- dio_bio_end_io(bio);
-}
-EXPORT_SYMBOL_GPL(dio_end_io);
-
static inline void
dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,
struct block_device *bdev,
depends on INET
depends on SYSFS && CONFIGFS_FS && (IPV6 || IPV6=n)
select IP_SCTP
+ select SRCU
help
A general purpose distributed lock manager for kernel or userspace
applications.
CONFIGFS_ATTR(cluster_, cluster_name);
static ssize_t cluster_set(struct dlm_cluster *cl, unsigned int *cl_field,
- int *info_field, int check_zero,
+ int *info_field, bool (*check_cb)(unsigned int x),
const char *buf, size_t len)
{
unsigned int x;
if (rc)
return rc;
- if (check_zero && !x)
+ if (check_cb && check_cb(x))
return -EINVAL;
*cl_field = x;
return len;
}
-#define CLUSTER_ATTR(name, check_zero) \
+#define CLUSTER_ATTR(name, check_cb) \
static ssize_t cluster_##name##_store(struct config_item *item, \
const char *buf, size_t len) \
{ \
struct dlm_cluster *cl = config_item_to_cluster(item); \
return cluster_set(cl, &cl->cl_##name, &dlm_config.ci_##name, \
- check_zero, buf, len); \
+ check_cb, buf, len); \
} \
static ssize_t cluster_##name##_show(struct config_item *item, char *buf) \
{ \
} \
CONFIGFS_ATTR(cluster_, name);
-CLUSTER_ATTR(tcp_port, 1);
-CLUSTER_ATTR(buffer_size, 1);
-CLUSTER_ATTR(rsbtbl_size, 1);
-CLUSTER_ATTR(recover_timer, 1);
-CLUSTER_ATTR(toss_secs, 1);
-CLUSTER_ATTR(scan_secs, 1);
-CLUSTER_ATTR(log_debug, 0);
-CLUSTER_ATTR(log_info, 0);
-CLUSTER_ATTR(protocol, 0);
-CLUSTER_ATTR(mark, 0);
-CLUSTER_ATTR(timewarn_cs, 1);
-CLUSTER_ATTR(waitwarn_us, 0);
-CLUSTER_ATTR(new_rsb_count, 0);
-CLUSTER_ATTR(recover_callbacks, 0);
+static bool dlm_check_zero(unsigned int x)
+{
+ return !x;
+}
+
+static bool dlm_check_buffer_size(unsigned int x)
+{
+ return (x < DEFAULT_BUFFER_SIZE);
+}
+
+CLUSTER_ATTR(tcp_port, dlm_check_zero);
+CLUSTER_ATTR(buffer_size, dlm_check_buffer_size);
+CLUSTER_ATTR(rsbtbl_size, dlm_check_zero);
+CLUSTER_ATTR(recover_timer, dlm_check_zero);
+CLUSTER_ATTR(toss_secs, dlm_check_zero);
+CLUSTER_ATTR(scan_secs, dlm_check_zero);
+CLUSTER_ATTR(log_debug, NULL);
+CLUSTER_ATTR(log_info, NULL);
+CLUSTER_ATTR(protocol, NULL);
+CLUSTER_ATTR(mark, NULL);
+CLUSTER_ATTR(timewarn_cs, dlm_check_zero);
+CLUSTER_ATTR(waitwarn_us, NULL);
+CLUSTER_ATTR(new_rsb_count, NULL);
+CLUSTER_ATTR(recover_callbacks, NULL);
static struct configfs_attribute *cluster_attrs[] = {
[CLUSTER_ATTR_TCP_PORT] = &cluster_attr_tcp_port,
struct list_head members;
struct mutex members_lock;
int members_count;
+ struct dlm_nodes *nds;
};
struct dlm_comms {
INIT_LIST_HEAD(&sp->members);
mutex_init(&sp->members_lock);
sp->members_count = 0;
+ sp->nds = nds;
return &sp->group;
fail:
static void release_space(struct config_item *i)
{
struct dlm_space *sp = config_item_to_space(i);
+ kfree(sp->nds);
kfree(sp);
}
return 0;
}
-int dlm_comm_mark(int nodeid, unsigned int *mark)
+void dlm_comm_mark(int nodeid, unsigned int *mark)
{
struct dlm_comm *cm;
cm = get_comm(nodeid);
- if (!cm)
- return -ENOENT;
+ if (!cm) {
+ *mark = dlm_config.ci_mark;
+ return;
+ }
- *mark = cm->mark;
- put_comm(cm);
+ if (cm->mark)
+ *mark = cm->mark;
+ else
+ *mark = dlm_config.ci_mark;
- return 0;
+ put_comm(cm);
}
int dlm_our_nodeid(void)
/* Config file defaults */
#define DEFAULT_TCP_PORT 21064
-#define DEFAULT_BUFFER_SIZE 4096
#define DEFAULT_RSBTBL_SIZE 1024
#define DEFAULT_RECOVER_TIMER 5
#define DEFAULT_TOSS_SECS 10
#ifndef __CONFIG_DOT_H__
#define __CONFIG_DOT_H__
+#define DEFAULT_BUFFER_SIZE 4096
+
struct dlm_config_node {
int nodeid;
int weight;
int dlm_config_nodes(char *lsname, struct dlm_config_node **nodes_out,
int *count_out);
int dlm_comm_seq(int nodeid, uint32_t *seq);
-int dlm_comm_mark(int nodeid, unsigned int *mark);
+void dlm_comm_mark(int nodeid, unsigned int *mark);
int dlm_our_nodeid(void);
int dlm_our_addr(struct sockaddr_storage *addr, int num);
#define MAX_SEND_MSG_COUNT 25
#define DLM_SHUTDOWN_WAIT_TIMEOUT msecs_to_jiffies(10000)
-struct cbuf {
- unsigned int base;
- unsigned int len;
- unsigned int mask;
-};
-
-static void cbuf_add(struct cbuf *cb, int n)
-{
- cb->len += n;
-}
-
-static int cbuf_data(struct cbuf *cb)
-{
- return ((cb->base + cb->len) & cb->mask);
-}
-
-static void cbuf_init(struct cbuf *cb, int size)
-{
- cb->base = cb->len = 0;
- cb->mask = size-1;
-}
-
-static void cbuf_eat(struct cbuf *cb, int n)
-{
- cb->len -= n;
- cb->base += n;
- cb->base &= cb->mask;
-}
-
-static bool cbuf_empty(struct cbuf *cb)
-{
- return cb->len == 0;
-}
-
struct connection {
struct socket *sock; /* NULL if not connected */
uint32_t nodeid; /* So we know who we are in the list */
int (*rx_action) (struct connection *); /* What to do when active */
void (*connect_action) (struct connection *); /* What to do to connect */
void (*shutdown_action)(struct connection *con); /* What to do to shutdown */
- struct page *rx_page;
- struct cbuf cb;
int retries;
#define MAX_CONNECT_RETRIES 3
struct hlist_node list;
struct work_struct rwork; /* Receive workqueue */
struct work_struct swork; /* Send workqueue */
wait_queue_head_t shutdown_wait; /* wait for graceful shutdown */
+ unsigned char *rx_buf;
+ int rx_buflen;
+ int rx_leftover;
+ struct rcu_head rcu;
};
#define sock2con(x) ((struct connection *)(x)->sk_user_data)
static struct workqueue_struct *send_workqueue;
static struct hlist_head connection_hash[CONN_HASH_SIZE];
-static DEFINE_MUTEX(connections_lock);
-static struct kmem_cache *con_cache;
+static DEFINE_SPINLOCK(connections_lock);
+DEFINE_STATIC_SRCU(connections_srcu);
static void process_recv_sockets(struct work_struct *work);
static void process_send_sockets(struct work_struct *work);
static struct connection *__find_con(int nodeid)
{
- int r;
+ int r, idx;
struct connection *con;
r = nodeid_hash(nodeid);
- hlist_for_each_entry(con, &connection_hash[r], list) {
- if (con->nodeid == nodeid)
+ idx = srcu_read_lock(&connections_srcu);
+ hlist_for_each_entry_rcu(con, &connection_hash[r], list) {
+ if (con->nodeid == nodeid) {
+ srcu_read_unlock(&connections_srcu, idx);
return con;
+ }
}
+ srcu_read_unlock(&connections_srcu, idx);
+
return NULL;
}
* If 'allocation' is zero then we don't attempt to create a new
* connection structure for this node.
*/
-static struct connection *__nodeid2con(int nodeid, gfp_t alloc)
+static struct connection *nodeid2con(int nodeid, gfp_t alloc)
{
- struct connection *con = NULL;
+ struct connection *con, *tmp;
int r;
con = __find_con(nodeid);
if (con || !alloc)
return con;
- con = kmem_cache_zalloc(con_cache, alloc);
+ con = kzalloc(sizeof(*con), alloc);
if (!con)
return NULL;
- r = nodeid_hash(nodeid);
- hlist_add_head(&con->list, &connection_hash[r]);
+ con->rx_buflen = dlm_config.ci_buffer_size;
+ con->rx_buf = kmalloc(con->rx_buflen, GFP_NOFS);
+ if (!con->rx_buf) {
+ kfree(con);
+ return NULL;
+ }
con->nodeid = nodeid;
mutex_init(&con->sock_mutex);
con->rx_action = zerocon->rx_action;
}
+ r = nodeid_hash(nodeid);
+
+ spin_lock(&connections_lock);
+ /* Because multiple workqueues/threads calls this function it can
+ * race on multiple cpu's. Instead of locking hot path __find_con()
+ * we just check in rare cases of recently added nodes again
+ * under protection of connections_lock. If this is the case we
+ * abort our connection creation and return the existing connection.
+ */
+ tmp = __find_con(nodeid);
+ if (tmp) {
+ spin_unlock(&connections_lock);
+ kfree(con->rx_buf);
+ kfree(con);
+ return tmp;
+ }
+
+ hlist_add_head_rcu(&con->list, &connection_hash[r]);
+ spin_unlock(&connections_lock);
+
return con;
}
/* Loop round all connections */
static void foreach_conn(void (*conn_func)(struct connection *c))
{
- int i;
- struct hlist_node *n;
+ int i, idx;
struct connection *con;
+ idx = srcu_read_lock(&connections_srcu);
for (i = 0; i < CONN_HASH_SIZE; i++) {
- hlist_for_each_entry_safe(con, n, &connection_hash[i], list)
+ hlist_for_each_entry_rcu(con, &connection_hash[i], list)
conn_func(con);
}
-}
-
-static struct connection *nodeid2con(int nodeid, gfp_t allocation)
-{
- struct connection *con;
-
- mutex_lock(&connections_lock);
- con = __nodeid2con(nodeid, allocation);
- mutex_unlock(&connections_lock);
-
- return con;
+ srcu_read_unlock(&connections_srcu, idx);
}
static struct dlm_node_addr *find_node_addr(int nodeid)
/* Will only re-enter once. */
close_connection(con->othercon, false, true, true);
}
- if (con->rx_page) {
- __free_page(con->rx_page);
- con->rx_page = NULL;
- }
+ con->rx_leftover = 0;
con->retries = 0;
mutex_unlock(&con->sock_mutex);
clear_bit(CF_CLOSING, &con->flags);
shutdown_connection(con);
}
+static int con_realloc_receive_buf(struct connection *con, int newlen)
+{
+ unsigned char *newbuf;
+
+ newbuf = kmalloc(newlen, GFP_NOFS);
+ if (!newbuf)
+ return -ENOMEM;
+
+ /* copy any leftover from last receive */
+ if (con->rx_leftover)
+ memmove(newbuf, con->rx_buf, con->rx_leftover);
+
+ /* swap to new buffer space */
+ kfree(con->rx_buf);
+ con->rx_buflen = newlen;
+ con->rx_buf = newbuf;
+
+ return 0;
+}
+
/* Data received from remote end */
static int receive_from_sock(struct connection *con)
{
- int ret = 0;
- struct msghdr msg = {};
- struct kvec iov[2];
- unsigned len;
- int r;
int call_again_soon = 0;
- int nvec;
+ struct msghdr msg;
+ struct kvec iov;
+ int ret, buflen;
mutex_lock(&con->sock_mutex);
ret = -EAGAIN;
goto out_close;
}
+
if (con->nodeid == 0) {
ret = -EINVAL;
goto out_close;
}
- if (con->rx_page == NULL) {
- /*
- * This doesn't need to be atomic, but I think it should
- * improve performance if it is.
- */
- con->rx_page = alloc_page(GFP_ATOMIC);
- if (con->rx_page == NULL)
+ /* realloc if we get new buffer size to read out */
+ buflen = dlm_config.ci_buffer_size;
+ if (con->rx_buflen != buflen && con->rx_leftover <= buflen) {
+ ret = con_realloc_receive_buf(con, buflen);
+ if (ret < 0)
goto out_resched;
- cbuf_init(&con->cb, PAGE_SIZE);
}
- /*
- * iov[0] is the bit of the circular buffer between the current end
- * point (cb.base + cb.len) and the end of the buffer.
- */
- iov[0].iov_len = con->cb.base - cbuf_data(&con->cb);
- iov[0].iov_base = page_address(con->rx_page) + cbuf_data(&con->cb);
- iov[1].iov_len = 0;
- nvec = 1;
-
- /*
- * iov[1] is the bit of the circular buffer between the start of the
- * buffer and the start of the currently used section (cb.base)
+ /* calculate new buffer parameter regarding last receive and
+ * possible leftover bytes
*/
- if (cbuf_data(&con->cb) >= con->cb.base) {
- iov[0].iov_len = PAGE_SIZE - cbuf_data(&con->cb);
- iov[1].iov_len = con->cb.base;
- iov[1].iov_base = page_address(con->rx_page);
- nvec = 2;
- }
- len = iov[0].iov_len + iov[1].iov_len;
- iov_iter_kvec(&msg.msg_iter, READ, iov, nvec, len);
+ iov.iov_base = con->rx_buf + con->rx_leftover;
+ iov.iov_len = con->rx_buflen - con->rx_leftover;
- r = ret = sock_recvmsg(con->sock, &msg, MSG_DONTWAIT | MSG_NOSIGNAL);
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL;
+ ret = kernel_recvmsg(con->sock, &msg, &iov, 1, iov.iov_len,
+ msg.msg_flags);
if (ret <= 0)
goto out_close;
- else if (ret == len)
+ else if (ret == iov.iov_len)
call_again_soon = 1;
- cbuf_add(&con->cb, ret);
- ret = dlm_process_incoming_buffer(con->nodeid,
- page_address(con->rx_page),
- con->cb.base, con->cb.len,
- PAGE_SIZE);
- if (ret < 0) {
- log_print("lowcomms err %d: addr=%p, base=%u, len=%u, read=%d",
- ret, page_address(con->rx_page), con->cb.base,
- con->cb.len, r);
- cbuf_eat(&con->cb, r);
- } else {
- cbuf_eat(&con->cb, ret);
- }
+ /* new buflen according readed bytes and leftover from last receive */
+ buflen = ret + con->rx_leftover;
+ ret = dlm_process_incoming_buffer(con->nodeid, con->rx_buf, buflen);
+ if (ret < 0)
+ goto out_close;
- if (cbuf_empty(&con->cb) && !call_again_soon) {
- __free_page(con->rx_page);
- con->rx_page = NULL;
+ /* calculate leftover bytes from process and put it into begin of
+ * the receive buffer, so next receive we have the full message
+ * at the start address of the receive buffer.
+ */
+ con->rx_leftover = buflen - ret;
+ if (con->rx_leftover) {
+ memmove(con->rx_buf, con->rx_buf + ret,
+ con->rx_leftover);
+ call_again_soon = true;
}
if (call_again_soon)
goto out_resched;
+
mutex_unlock(&con->sock_mutex);
return 0;
int nodeid;
struct connection *newcon;
struct connection *addcon;
+ unsigned int mark;
- mutex_lock(&connections_lock);
if (!dlm_allow_conn) {
- mutex_unlock(&connections_lock);
return -1;
}
- mutex_unlock(&connections_lock);
mutex_lock_nested(&con->sock_mutex, 0);
return -1;
}
+ dlm_comm_mark(nodeid, &mark);
+ sock_set_mark(newsock->sk, mark);
+
log_print("got connection from %d", nodeid);
/* Check to see if we already have a connection to this node. This
struct connection *othercon = newcon->othercon;
if (!othercon) {
- othercon = kmem_cache_zalloc(con_cache, GFP_NOFS);
+ othercon = kzalloc(sizeof(*othercon), GFP_NOFS);
if (!othercon) {
log_print("failed to allocate incoming socket");
mutex_unlock(&newcon->sock_mutex);
result = -ENOMEM;
goto accept_err;
}
+
+ othercon->rx_buflen = dlm_config.ci_buffer_size;
+ othercon->rx_buf = kmalloc(othercon->rx_buflen, GFP_NOFS);
+ if (!othercon->rx_buf) {
+ mutex_unlock(&newcon->sock_mutex);
+ kfree(othercon);
+ log_print("failed to allocate incoming socket receive buffer");
+ result = -ENOMEM;
+ goto accept_err;
+ }
+
othercon->nodeid = nodeid;
othercon->rx_action = receive_from_sock;
mutex_init(&othercon->sock_mutex);
return;
}
+ dlm_comm_mark(con->nodeid, &mark);
+
mutex_lock(&con->sock_mutex);
/* Some odd races can cause double-connects, ignore them */
if (result < 0)
goto socket_err;
- /* set skb mark */
- result = dlm_comm_mark(con->nodeid, &mark);
- if (result < 0)
- goto bind_err;
-
sock_set_mark(sock->sk, mark);
con->rx_action = receive_from_sock;
return;
}
+ dlm_comm_mark(con->nodeid, &mark);
+
mutex_lock(&con->sock_mutex);
if (con->retries++ > MAX_CONNECT_RETRIES)
goto out;
if (result < 0)
goto out_err;
- /* set skb mark */
- result = dlm_comm_mark(con->nodeid, &mark);
- if (result < 0)
- goto out_err;
-
sock_set_mark(sock->sk, mark);
memset(&saddr, 0, sizeof(saddr));
}
}
+static void deinit_local(void)
+{
+ int i;
+
+ for (i = 0; i < dlm_local_count; i++)
+ kfree(dlm_local_addr[i]);
+}
+
/* Initialise SCTP socket and bind to all interfaces */
static int sctp_listen_for_all(void)
{
send_to_sock(con);
}
-
-/* Discard all entries on the write queues */
-static void clean_writequeues(void)
-{
- foreach_conn(clean_one_writequeue);
-}
-
static void work_stop(void)
{
if (recv_workqueue)
con->shutdown_action(con);
}
+static void connection_release(struct rcu_head *rcu)
+{
+ struct connection *con = container_of(rcu, struct connection, rcu);
+
+ kfree(con->rx_buf);
+ kfree(con);
+}
+
static void free_conn(struct connection *con)
{
close_connection(con, true, true, true);
- if (con->othercon)
- kmem_cache_free(con_cache, con->othercon);
- hlist_del(&con->list);
- kmem_cache_free(con_cache, con);
+ spin_lock(&connections_lock);
+ hlist_del_rcu(&con->list);
+ spin_unlock(&connections_lock);
+ if (con->othercon) {
+ clean_one_writequeue(con->othercon);
+ call_rcu(&con->othercon->rcu, connection_release);
+ }
+ clean_one_writequeue(con);
+ call_rcu(&con->rcu, connection_release);
}
static void work_flush(void)
{
- int ok;
+ int ok, idx;
int i;
- struct hlist_node *n;
struct connection *con;
- if (recv_workqueue)
- flush_workqueue(recv_workqueue);
- if (send_workqueue)
- flush_workqueue(send_workqueue);
do {
ok = 1;
foreach_conn(stop_conn);
flush_workqueue(recv_workqueue);
if (send_workqueue)
flush_workqueue(send_workqueue);
+ idx = srcu_read_lock(&connections_srcu);
for (i = 0; i < CONN_HASH_SIZE && ok; i++) {
- hlist_for_each_entry_safe(con, n,
- &connection_hash[i], list) {
+ hlist_for_each_entry_rcu(con, &connection_hash[i],
+ list) {
ok &= test_bit(CF_READ_PENDING, &con->flags);
ok &= test_bit(CF_WRITE_PENDING, &con->flags);
if (con->othercon) {
}
}
}
+ srcu_read_unlock(&connections_srcu, idx);
} while (!ok);
}
/* Set all the flags to prevent any
socket activity.
*/
- mutex_lock(&connections_lock);
dlm_allow_conn = 0;
- mutex_unlock(&connections_lock);
+
+ if (recv_workqueue)
+ flush_workqueue(recv_workqueue);
+ if (send_workqueue)
+ flush_workqueue(send_workqueue);
+
foreach_conn(shutdown_conn);
work_flush();
- clean_writequeues();
foreach_conn(free_conn);
work_stop();
-
- kmem_cache_destroy(con_cache);
+ deinit_local();
}
int dlm_lowcomms_start(void)
goto fail;
}
- error = -ENOMEM;
- con_cache = kmem_cache_create("dlm_conn", sizeof(struct connection),
- __alignof__(struct connection), 0,
- NULL);
- if (!con_cache)
- goto fail;
-
error = work_start();
if (error)
- goto fail_destroy;
+ goto fail;
dlm_allow_conn = 1;
fail_unlisten:
dlm_allow_conn = 0;
con = nodeid2con(0,0);
- if (con) {
- close_connection(con, false, true, true);
- kmem_cache_free(con_cache, con);
- }
-fail_destroy:
- kmem_cache_destroy(con_cache);
+ if (con)
+ free_conn(con);
fail:
return error;
}
* into packets and sends them to the comms layer.
*/
+#include <asm/unaligned.h>
+
#include "dlm_internal.h"
#include "lowcomms.h"
#include "config.h"
#include "lock.h"
#include "midcomms.h"
-
-static void copy_from_cb(void *dst, const void *base, unsigned offset,
- unsigned len, unsigned limit)
-{
- unsigned copy = len;
-
- if ((copy + offset) > limit)
- copy = limit - offset;
- memcpy(dst, base + offset, copy);
- len -= copy;
- if (len)
- memcpy(dst + copy, base, len);
-}
-
/*
* Called from the low-level comms layer to process a buffer of
* commands.
- *
- * Only complete messages are processed here, any "spare" bytes from
- * the end of a buffer are saved and tacked onto the front of the next
- * message that comes in. I doubt this will happen very often but we
- * need to be able to cope with it and I don't want the task to be waiting
- * for packets to come in when there is useful work to be done.
*/
-int dlm_process_incoming_buffer(int nodeid, const void *base,
- unsigned offset, unsigned len, unsigned limit)
+int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int len)
{
- union {
- unsigned char __buf[DLM_INBUF_LEN];
- /* this is to force proper alignment on some arches */
- union dlm_packet p;
- } __tmp;
- union dlm_packet *p = &__tmp.p;
- int ret = 0;
- int err = 0;
+ const unsigned char *ptr = buf;
+ const struct dlm_header *hd;
uint16_t msglen;
- uint32_t lockspace;
-
- while (len > sizeof(struct dlm_header)) {
-
- /* Copy just the header to check the total length. The
- message may wrap around the end of the buffer back to the
- start, so we need to use a temp buffer and copy_from_cb. */
-
- copy_from_cb(p, base, offset, sizeof(struct dlm_header),
- limit);
-
- msglen = le16_to_cpu(p->header.h_length);
- lockspace = p->header.h_lockspace;
+ int ret = 0;
- err = -EINVAL;
- if (msglen < sizeof(struct dlm_header))
- break;
- if (p->header.h_cmd == DLM_MSG) {
- if (msglen < sizeof(struct dlm_message))
- break;
- } else {
- if (msglen < sizeof(struct dlm_rcom))
- break;
- }
- err = -E2BIG;
- if (msglen > dlm_config.ci_buffer_size) {
- log_print("message size %d from %d too big, buf len %d",
- msglen, nodeid, len);
- break;
+ while (len >= sizeof(struct dlm_header)) {
+ hd = (struct dlm_header *)ptr;
+
+ /* no message should be more than this otherwise we
+ * cannot deliver this message to upper layers
+ */
+ msglen = get_unaligned_le16(&hd->h_length);
+ if (msglen > DEFAULT_BUFFER_SIZE) {
+ log_print("received invalid length header: %u, will abort message parsing",
+ msglen);
+ return -EBADMSG;
}
- err = 0;
-
- /* If only part of the full message is contained in this
- buffer, then do nothing and wait for lowcomms to call
- us again later with more data. We return 0 meaning
- we've consumed none of the input buffer. */
+ /* caller will take care that leftover
+ * will be parsed next call with more data
+ */
if (msglen > len)
break;
- /* Allocate a larger temp buffer if the full message won't fit
- in the buffer on the stack (which should work for most
- ordinary messages). */
-
- if (msglen > sizeof(__tmp) && p == &__tmp.p) {
- p = kmalloc(dlm_config.ci_buffer_size, GFP_NOFS);
- if (p == NULL)
- return ret;
- }
+ switch (hd->h_cmd) {
+ case DLM_MSG:
+ if (msglen < sizeof(struct dlm_message)) {
+ log_print("dlm msg too small: %u, will skip this message",
+ msglen);
+ goto skip;
+ }
- copy_from_cb(p, base, offset, msglen, limit);
+ break;
+ case DLM_RCOM:
+ if (msglen < sizeof(struct dlm_rcom)) {
+ log_print("dlm rcom msg too small: %u, will skip this message",
+ msglen);
+ goto skip;
+ }
- BUG_ON(lockspace != p->header.h_lockspace);
+ break;
+ default:
+ log_print("unsupported h_cmd received: %u, will skip this message",
+ hd->h_cmd);
+ goto skip;
+ }
+ /* for aligned memory access, we just copy current message
+ * to begin of the buffer which contains already parsed buffer
+ * data and should provide align access for upper layers
+ * because the start address of the buffer has a aligned
+ * address. This memmove can be removed when the upperlayer
+ * is capable of unaligned memory access.
+ */
+ memmove(buf, ptr, msglen);
+ dlm_receive_buffer((union dlm_packet *)buf, nodeid);
+
+skip:
ret += msglen;
- offset += msglen;
- offset &= (limit - 1);
len -= msglen;
-
- dlm_receive_buffer(p, nodeid);
+ ptr += msglen;
}
- if (p != &__tmp.p)
- kfree(p);
-
- return err ? err : ret;
+ return ret;
}
#ifndef __MIDCOMMS_DOT_H__
#define __MIDCOMMS_DOT_H__
-int dlm_process_incoming_buffer(int nodeid, const void *base, unsigned offset,
- unsigned len, unsigned limit);
+int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int buflen);
#endif /* __MIDCOMMS_DOT_H__ */
name[len + EFI_VARIABLE_GUID_LEN+1] = '\0';
+ /* replace invalid slashes like kobject_set_name_vargs does for /sys/firmware/efi/vars. */
+ strreplace(name, '/', '!');
+
inode = efivarfs_get_inode(sb, d_inode(root), S_IFREG | 0644, 0,
is_removable);
if (!inode)
bio_set_dev(bio, sb->s_bdev);
bio->bi_iter.bi_sector = (sector_t)blknr <<
LOG_SECTORS_PER_BLOCK;
- bio->bi_opf = REQ_OP_READ;
+ bio->bi_opf = REQ_OP_READ | (ra ? REQ_RAHEAD : 0);
}
err = bio_add_page(bio, page, PAGE_SIZE, 0);
enum {
Opt_user_xattr,
- Opt_nouser_xattr,
Opt_acl,
- Opt_noacl,
Opt_cache_strategy,
Opt_err
};
return -EOPNOTSUPP;
break;
case EROFS_XATTR_INDEX_TRUSTED:
- if (!capable(CAP_SYS_ADMIN))
- return -EPERM;
break;
case EROFS_XATTR_INDEX_SECURITY:
break;
struct z_erofs_collector clt;
struct erofs_map_blocks map;
+ bool readahead;
/* used for applying cache strategy on the fly */
bool backmost;
erofs_off_t headoffset;
static void preload_compressed_pages(struct z_erofs_collector *clt,
struct address_space *mc,
- enum z_erofs_cache_alloctype type,
- struct list_head *pagepool)
+ enum z_erofs_cache_alloctype type)
{
const struct z_erofs_pcluster *pcl = clt->pcl;
const unsigned int clusterpages = BIT(pcl->clusterbits);
}
static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe,
- struct page *page,
- struct list_head *pagepool)
+ struct page *page)
{
struct inode *const inode = fe->inode;
struct erofs_sb_info *const sbi = EROFS_I_SB(inode);
else
cache_strategy = DONTALLOC;
- preload_compressed_pages(clt, MNGD_MAPPING(sbi),
- cache_strategy, pagepool);
+ preload_compressed_pages(clt, MNGD_MAPPING(sbi), cache_strategy);
hitted:
/*
/* should allocate an additional staging page for pagevec */
if (err == -EAGAIN) {
struct page *const newpage =
- erofs_allocpage(pagepool, GFP_NOFS | __GFP_NOFAIL);
+ alloc_page(GFP_NOFS | __GFP_NOFAIL);
newpage->mapping = Z_EROFS_MAPPING_STAGING;
err = z_erofs_attach_page(clt, newpage,
}
static void z_erofs_submit_queue(struct super_block *sb,
- z_erofs_next_pcluster_t owned_head,
+ struct z_erofs_decompress_frontend *f,
struct list_head *pagepool,
struct z_erofs_decompressqueue *fgq,
bool *force_fg)
z_erofs_next_pcluster_t qtail[NR_JOBQUEUES];
struct z_erofs_decompressqueue *q[NR_JOBQUEUES];
void *bi_private;
+ z_erofs_next_pcluster_t owned_head = f->clt.owned_head;
/* since bio will be NULL, no need to initialize last_index */
pgoff_t last_index;
unsigned int nr_bios = 0;
do {
struct page *page;
- int err;
page = pickup_page_for_submission(pcl, i++, pagepool,
MNGD_MAPPING(sbi),
LOG_SECTORS_PER_BLOCK;
bio->bi_private = bi_private;
bio->bi_opf = REQ_OP_READ;
+ if (f->readahead)
+ bio->bi_opf |= REQ_RAHEAD;
++nr_bios;
}
- err = bio_add_page(bio, page, PAGE_SIZE, 0);
- if (err < PAGE_SIZE)
+ if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
goto submit_bio_retry;
last_index = cur;
}
static void z_erofs_runqueue(struct super_block *sb,
- struct z_erofs_collector *clt,
+ struct z_erofs_decompress_frontend *f,
struct list_head *pagepool, bool force_fg)
{
struct z_erofs_decompressqueue io[NR_JOBQUEUES];
- if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)
+ if (f->clt.owned_head == Z_EROFS_PCLUSTER_TAIL)
return;
- z_erofs_submit_queue(sb, clt->owned_head, pagepool, io, &force_fg);
+ z_erofs_submit_queue(sb, f, pagepool, io, &force_fg);
/* handle bypass queue (no i/o pclusters) immediately */
z_erofs_decompress_queue(&io[JQ_BYPASS], pagepool);
f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT;
- err = z_erofs_do_read_page(&f, page, &pagepool);
+ err = z_erofs_do_read_page(&f, page);
(void)z_erofs_collector_end(&f.clt);
/* if some compressed cluster ready, need submit them anyway */
- z_erofs_runqueue(inode->i_sb, &f.clt, &pagepool, true);
+ z_erofs_runqueue(inode->i_sb, &f, &pagepool, true);
if (err)
erofs_err(inode->i_sb, "failed to read, err [%d]", err);
return err;
}
-static bool should_decompress_synchronously(struct erofs_sb_info *sbi,
- unsigned int nr)
-{
- return nr <= sbi->ctx.max_sync_decompress_pages;
-}
-
static void z_erofs_readahead(struct readahead_control *rac)
{
struct inode *const inode = rac->mapping->host;
struct erofs_sb_info *const sbi = EROFS_I_SB(inode);
- bool sync = should_decompress_synchronously(sbi, readahead_count(rac));
+ unsigned int nr_pages = readahead_count(rac);
+ bool sync = (nr_pages <= sbi->ctx.max_sync_decompress_pages);
struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode);
struct page *page, *head = NULL;
LIST_HEAD(pagepool);
- trace_erofs_readpages(inode, readahead_index(rac),
- readahead_count(rac), false);
+ trace_erofs_readpages(inode, readahead_index(rac), nr_pages, false);
+ f.readahead = true;
f.headoffset = readahead_pos(rac);
while ((page = readahead_page(rac))) {
/* traversal in reverse order */
head = (void *)page_private(page);
- err = z_erofs_do_read_page(&f, page, &pagepool);
+ err = z_erofs_do_read_page(&f, page);
if (err)
erofs_err(inode->i_sb,
"readahead error at page %lu @ nid %llu",
(void)z_erofs_collector_end(&f.clt);
- z_erofs_runqueue(inode->i_sb, &f.clt, &pagepool, sync);
+ z_erofs_runqueue(inode->i_sb, &f, &pagepool, sync);
if (f.map.mpage)
put_page(f.map.mpage);
struct file *file;
/* used to optimize loop detection check */
- struct list_head visited_list_link;
- int visited;
+ u64 gen;
#ifdef CONFIG_NET_RX_BUSY_POLL
/* used to track busy poll napi_id */
*/
static DEFINE_MUTEX(epmutex);
+static u64 loop_check_gen = 0;
+
/* Used to check for epoll file descriptor inclusion loops */
static struct nested_calls poll_loop_ncalls;
/* Slab cache used to allocate "struct eppoll_entry" */
static struct kmem_cache *pwq_cache __read_mostly;
-/* Visited nodes during ep_loop_check(), so we can unset them when we finish */
-static LIST_HEAD(visited_list);
-
/*
* List of files with newly added links, where we may need to limit the number
* of emanating paths. Protected by the epmutex.
static int ep_create_wakeup_source(struct epitem *epi)
{
- const char *name;
+ struct name_snapshot n;
struct wakeup_source *ws;
if (!epi->ep->ws) {
return -ENOMEM;
}
- name = epi->ffd.file->f_path.dentry->d_name.name;
- ws = wakeup_source_register(NULL, name);
+ take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry);
+ ws = wakeup_source_register(NULL, n.name.name);
+ release_dentry_name_snapshot(&n);
if (!ws)
return -ENOMEM;
RCU_INIT_POINTER(epi->ws, NULL);
}
+ /* Add the current item to the list of active epoll hook for this file */
+ spin_lock(&tfile->f_lock);
+ list_add_tail_rcu(&epi->fllink, &tfile->f_ep_links);
+ spin_unlock(&tfile->f_lock);
+
+ /*
+ * Add the current item to the RB tree. All RB tree operations are
+ * protected by "mtx", and ep_insert() is called with "mtx" held.
+ */
+ ep_rbtree_insert(ep, epi);
+
+ /* now check if we've created too many backpaths */
+ error = -EINVAL;
+ if (full_check && reverse_path_check())
+ goto error_remove_epi;
+
/* Initialize the poll table using the queue callback */
epq.epi = epi;
init_poll_funcptr(&epq.pt, ep_ptable_queue_proc);
if (epi->nwait < 0)
goto error_unregister;
- /* Add the current item to the list of active epoll hook for this file */
- spin_lock(&tfile->f_lock);
- list_add_tail_rcu(&epi->fllink, &tfile->f_ep_links);
- spin_unlock(&tfile->f_lock);
-
- /*
- * Add the current item to the RB tree. All RB tree operations are
- * protected by "mtx", and ep_insert() is called with "mtx" held.
- */
- ep_rbtree_insert(ep, epi);
-
- /* now check if we've created too many backpaths */
- error = -EINVAL;
- if (full_check && reverse_path_check())
- goto error_remove_epi;
-
/* We have to drop the new item inside our item list to keep track of it */
write_lock_irq(&ep->lock);
return 0;
+error_unregister:
+ ep_unregister_pollwait(ep, epi);
error_remove_epi:
spin_lock(&tfile->f_lock);
list_del_rcu(&epi->fllink);
rb_erase_cached(&epi->rbn, &ep->rbr);
-error_unregister:
- ep_unregister_pollwait(ep, epi);
-
/*
* We need to do this because an event could have been arrived on some
* allocated wait queue. Note that we don't care about the ep->ovflist
struct epitem *epi;
mutex_lock_nested(&ep->mtx, call_nests + 1);
- ep->visited = 1;
- list_add(&ep->visited_list_link, &visited_list);
+ ep->gen = loop_check_gen;
for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
epi = rb_entry(rbp, struct epitem, rbn);
if (unlikely(is_file_epoll(epi->ffd.file))) {
ep_tovisit = epi->ffd.file->private_data;
- if (ep_tovisit->visited)
+ if (ep_tovisit->gen == loop_check_gen)
continue;
error = ep_call_nested(&poll_loop_ncalls,
ep_loop_check_proc, epi->ffd.file,
*/
static int ep_loop_check(struct eventpoll *ep, struct file *file)
{
- int ret;
- struct eventpoll *ep_cur, *ep_next;
-
- ret = ep_call_nested(&poll_loop_ncalls,
+ return ep_call_nested(&poll_loop_ncalls,
ep_loop_check_proc, file, ep, current);
- /* clear visited list */
- list_for_each_entry_safe(ep_cur, ep_next, &visited_list,
- visited_list_link) {
- ep_cur->visited = 0;
- list_del(&ep_cur->visited_list_link);
- }
- return ret;
}
static void clear_tfile_check_list(void)
goto error_tgt_fput;
if (op == EPOLL_CTL_ADD) {
if (!list_empty(&f.file->f_ep_links) ||
+ ep->gen == loop_check_gen ||
is_file_epoll(tf.file)) {
mutex_unlock(&ep->mtx);
error = epoll_mutex_lock(&epmutex, 0, nonblock);
if (error)
goto error_tgt_fput;
+ loop_check_gen++;
full_check = 1;
if (is_file_epoll(tf.file)) {
error = -ELOOP;
error_tgt_fput:
if (full_check) {
clear_tfile_check_list();
+ loop_check_gen++;
mutex_unlock(&epmutex);
}
#include <linux/oom.h>
#include <linux/compat.h>
#include <linux/vmalloc.h>
+#include <linux/io_uring.h>
#include <linux/uaccess.h>
#include <asm/mmu_context.h>
struct files_struct *displaced;
int retval;
+ /*
+ * Cancel any io_uring activity across execve
+ */
+ io_uring_task_cancel();
+
retval = unshare_files(&displaced);
if (retval)
return retval;
#include "exfat_raw.h"
#include "exfat_fs.h"
-#define EXFAT_CACHE_VALID 0
#define EXFAT_MAX_CACHE 16
struct exfat_cache {
kmem_cache_destroy(exfat_cachep);
}
-void exfat_cache_init_inode(struct inode *inode)
-{
- struct exfat_inode_info *ei = EXFAT_I(inode);
-
- spin_lock_init(&ei->cache_lru_lock);
- ei->nr_caches = 0;
- ei->cache_valid_id = EXFAT_CACHE_VALID + 1;
- INIT_LIST_HEAD(&ei->cache_lru);
-}
-
static inline struct exfat_cache *exfat_cache_alloc(void)
{
return kmem_cache_alloc(exfat_cachep, GFP_NOFS);
struct rcu_head rcu;
};
+#define EXFAT_CACHE_VALID 0
+
/*
* EXFAT file system inode in-memory data
*/
/* cache.c */
int exfat_cache_init(void);
void exfat_cache_shutdown(void);
-void exfat_cache_init_inode(struct inode *inode);
void exfat_cache_inval_inode(struct inode *inode);
int exfat_get_cluster(struct inode *inode, unsigned int cluster,
unsigned int *fclus, unsigned int *dclus,
ei->i_crtime = info->crtime;
inode->i_atime = info->atime;
- exfat_cache_init_inode(inode);
-
return 0;
}
i_pos = exfat_make_i_pos(&info);
inode = exfat_build_inode(sb, &info, i_pos);
- if (IS_ERR(inode))
+ err = PTR_ERR_OR_ZERO(inode);
+ if (err)
goto unlock;
inode_inc_iversion(inode);
i_pos = exfat_make_i_pos(&info);
inode = exfat_build_inode(sb, &info, i_pos);
- if (IS_ERR(inode)) {
- err = PTR_ERR(inode);
+ err = PTR_ERR_OR_ZERO(inode);
+ if (err)
goto unlock;
- }
i_mode = inode->i_mode;
alias = d_find_alias(inode);
i_pos = exfat_make_i_pos(&info);
inode = exfat_build_inode(sb, &info, i_pos);
- if (IS_ERR(inode)) {
- err = PTR_ERR(inode);
+ err = PTR_ERR_OR_ZERO(inode);
+ if (err)
goto unlock;
- }
inode_inc_iversion(inode);
inode->i_mtime = inode->i_atime = inode->i_ctime =
inode->i_mtime = inode->i_atime = inode->i_ctime = ei->i_crtime =
current_time(inode);
exfat_truncate_atime(&inode->i_atime);
- exfat_cache_init_inode(inode);
return 0;
}
{
struct exfat_inode_info *ei = (struct exfat_inode_info *)foo;
+ spin_lock_init(&ei->cache_lru_lock);
+ ei->nr_caches = 0;
+ ei->cache_valid_id = EXFAT_CACHE_VALID + 1;
+ INIT_LIST_HEAD(&ei->cache_lru);
INIT_HLIST_NODE(&ei->i_hash_fat);
inode_init_once(&ei->vfs_inode);
}
}
if (IS_ENCRYPTED(inode)) {
- err = fscrypt_fname_alloc_buffer(inode, EXT4_NAME_LEN, &fstr);
+ err = fscrypt_fname_alloc_buffer(EXT4_NAME_LEN, &fstr);
if (err < 0)
return err;
}
#define EXT4_MF_FS_ABORTED 0x0002 /* Fatal error detected */
#ifdef CONFIG_FS_ENCRYPTION
-#define DUMMY_ENCRYPTION_ENABLED(sbi) ((sbi)->s_dummy_enc_ctx.ctx != NULL)
+#define DUMMY_ENCRYPTION_ENABLED(sbi) ((sbi)->s_dummy_enc_policy.policy != NULL)
#else
#define DUMMY_ENCRYPTION_ENABLED(sbi) (0)
#endif
atomic_t s_warning_count;
atomic_t s_msg_count;
- /* Encryption context for '-o test_dummy_encryption' */
- struct fscrypt_dummy_context s_dummy_enc_ctx;
+ /* Encryption policy for '-o test_dummy_encryption' */
+ struct fscrypt_dummy_policy s_dummy_enc_policy;
/*
* Barrier between writepages ops and changing any inode's JOURNAL_DATA
return 1;
}
+static int ext4_xattr_credits_for_new_inode(struct inode *dir, mode_t mode,
+ bool encrypt)
+{
+ struct super_block *sb = dir->i_sb;
+ int nblocks = 0;
+#ifdef CONFIG_EXT4_FS_POSIX_ACL
+ struct posix_acl *p = get_acl(dir, ACL_TYPE_DEFAULT);
+
+ if (IS_ERR(p))
+ return PTR_ERR(p);
+ if (p) {
+ int acl_size = p->a_count * sizeof(ext4_acl_entry);
+
+ nblocks += (S_ISDIR(mode) ? 2 : 1) *
+ __ext4_xattr_set_credits(sb, NULL /* inode */,
+ NULL /* block_bh */, acl_size,
+ true /* is_create */);
+ posix_acl_release(p);
+ }
+#endif
+
+#ifdef CONFIG_SECURITY
+ {
+ int num_security_xattrs = 1;
+
+#ifdef CONFIG_INTEGRITY
+ num_security_xattrs++;
+#endif
+ /*
+ * We assume that security xattrs are never more than 1k.
+ * In practice they are under 128 bytes.
+ */
+ nblocks += num_security_xattrs *
+ __ext4_xattr_set_credits(sb, NULL /* inode */,
+ NULL /* block_bh */, 1024,
+ true /* is_create */);
+ }
+#endif
+ if (encrypt)
+ nblocks += __ext4_xattr_set_credits(sb,
+ NULL /* inode */,
+ NULL /* block_bh */,
+ FSCRYPT_SET_CONTEXT_MAX_SIZE,
+ true /* is_create */);
+ return nblocks;
+}
+
/*
* There are two policies for allocating an inode. If the new inode is
* a directory, then a forward search is made for a block group with both
ext4_group_t i;
ext4_group_t flex_group;
struct ext4_group_info *grp;
- int encrypt = 0;
+ bool encrypt = false;
/* Cannot create files in a deleted directory */
if (!dir || !dir->i_nlink)
if (unlikely(ext4_forced_shutdown(sbi)))
return ERR_PTR(-EIO);
- if ((IS_ENCRYPTED(dir) || DUMMY_ENCRYPTION_ENABLED(sbi)) &&
- (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode)) &&
- !(i_flags & EXT4_EA_INODE_FL)) {
- err = fscrypt_get_encryption_info(dir);
- if (err)
- return ERR_PTR(err);
- if (!fscrypt_has_encryption_key(dir))
- return ERR_PTR(-ENOKEY);
- encrypt = 1;
- }
-
- if (!handle && sbi->s_journal && !(i_flags & EXT4_EA_INODE_FL)) {
-#ifdef CONFIG_EXT4_FS_POSIX_ACL
- struct posix_acl *p = get_acl(dir, ACL_TYPE_DEFAULT);
-
- if (IS_ERR(p))
- return ERR_CAST(p);
- if (p) {
- int acl_size = p->a_count * sizeof(ext4_acl_entry);
-
- nblocks += (S_ISDIR(mode) ? 2 : 1) *
- __ext4_xattr_set_credits(sb, NULL /* inode */,
- NULL /* block_bh */, acl_size,
- true /* is_create */);
- posix_acl_release(p);
- }
-#endif
-
-#ifdef CONFIG_SECURITY
- {
- int num_security_xattrs = 1;
-
-#ifdef CONFIG_INTEGRITY
- num_security_xattrs++;
-#endif
- /*
- * We assume that security xattrs are never
- * more than 1k. In practice they are under
- * 128 bytes.
- */
- nblocks += num_security_xattrs *
- __ext4_xattr_set_credits(sb, NULL /* inode */,
- NULL /* block_bh */, 1024,
- true /* is_create */);
- }
-#endif
- if (encrypt)
- nblocks += __ext4_xattr_set_credits(sb,
- NULL /* inode */, NULL /* block_bh */,
- FSCRYPT_SET_CONTEXT_MAX_SIZE,
- true /* is_create */);
- }
-
ngroups = ext4_get_groups_count(sb);
trace_ext4_request_inode(dir, mode);
inode = new_inode(sb);
else
ei->i_projid = make_kprojid(&init_user_ns, EXT4_DEF_PROJID);
+ if (!(i_flags & EXT4_EA_INODE_FL)) {
+ err = fscrypt_prepare_new_inode(dir, inode, &encrypt);
+ if (err)
+ goto out;
+ }
+
err = dquot_initialize(inode);
if (err)
goto out;
+ if (!handle && sbi->s_journal && !(i_flags & EXT4_EA_INODE_FL)) {
+ ret2 = ext4_xattr_credits_for_new_inode(dir, mode, encrypt);
+ if (ret2 < 0) {
+ err = ret2;
+ goto out;
+ }
+ nblocks += ret2;
+ }
+
if (!goal)
goal = sbi->s_inode_goal;
* prevent its deduplication.
*/
if (encrypt) {
- err = fscrypt_inherit_context(dir, inode, handle, true);
+ err = fscrypt_set_context(inode, handle);
if (err)
goto fail_free_drop;
}
/* Directory is encrypted */
res = fscrypt_fname_alloc_buffer(
- dir, len,
- &fname_crypto_str);
+ len, &fname_crypto_str);
if (res)
printk(KERN_WARNING "Error "
"allocating crypto "
brelse(bh);
return err;
}
- err = fscrypt_fname_alloc_buffer(dir, EXT4_NAME_LEN,
- &fname_crypto_str);
+ err = fscrypt_fname_alloc_buffer(EXT4_NAME_LEN,
+ &fname_crypto_str);
if (err < 0) {
brelse(bh);
return err;
crypto_free_shash(sbi->s_chksum_driver);
kfree(sbi->s_blockgroup_lock);
fs_put_dax(sbi->s_daxdev);
- fscrypt_free_dummy_context(&sbi->s_dummy_enc_ctx);
+ fscrypt_free_dummy_policy(&sbi->s_dummy_enc_policy);
#ifdef CONFIG_UNICODE
utf8_unload(sbi->s_encoding);
#endif
return res;
}
-static const union fscrypt_context *
-ext4_get_dummy_context(struct super_block *sb)
+static const union fscrypt_policy *ext4_get_dummy_policy(struct super_block *sb)
{
- return EXT4_SB(sb)->s_dummy_enc_ctx.ctx;
+ return EXT4_SB(sb)->s_dummy_enc_policy.policy;
}
static bool ext4_has_stable_inodes(struct super_block *sb)
.key_prefix = "ext4:",
.get_context = ext4_get_context,
.set_context = ext4_set_context,
- .get_dummy_context = ext4_get_dummy_context,
+ .get_dummy_policy = ext4_get_dummy_policy,
.empty_dir = ext4_empty_dir,
.max_namelen = EXT4_NAME_LEN,
.has_stable_inodes = ext4_has_stable_inodes,
* needed to allow it to be set or changed during remount. We do allow
* it to be specified during remount, but only if there is no change.
*/
- if (is_remount && !sbi->s_dummy_enc_ctx.ctx) {
+ if (is_remount && !sbi->s_dummy_enc_policy.policy) {
ext4_msg(sb, KERN_WARNING,
"Can't set test_dummy_encryption on remount");
return -1;
}
- err = fscrypt_set_test_dummy_encryption(sb, arg, &sbi->s_dummy_enc_ctx);
+ err = fscrypt_set_test_dummy_encryption(sb, arg->from,
+ &sbi->s_dummy_enc_policy);
if (err) {
if (err == -EEXIST)
ext4_msg(sb, KERN_WARNING,
for (i = 0; i < EXT4_MAXQUOTAS; i++)
kfree(get_qf_name(sb, sbi, i));
#endif
- fscrypt_free_dummy_context(&sbi->s_dummy_enc_ctx);
+ fscrypt_free_dummy_policy(&sbi->s_dummy_enc_policy);
ext4_blkdev_remove(sbi);
brelse(bh);
out_fail:
#ifdef CONFIG_FS_ENCRYPTION
fname->crypto_buf = crypt_name->crypto_buf;
#endif
- if (crypt_name->is_ciphertext_name) {
+ if (crypt_name->is_nokey_name) {
/* hash was decoded from the no-key name */
fname->hash = cpu_to_le32(crypt_name->hash);
} else {
goto put_error;
if (IS_ENCRYPTED(inode)) {
- err = fscrypt_inherit_context(dir, inode, page, false);
+ err = fscrypt_set_context(inode, page);
if (err)
goto put_error;
}
if (err)
goto out;
- err = fscrypt_fname_alloc_buffer(inode, F2FS_NAME_LEN, &fstr);
+ err = fscrypt_fname_alloc_buffer(F2FS_NAME_LEN, &fstr);
if (err < 0)
goto out;
}
int fsync_mode; /* fsync policy */
int fs_mode; /* fs mode: LFS or ADAPTIVE */
int bggc_mode; /* bggc mode: off, on or sync */
- struct fscrypt_dummy_context dummy_enc_ctx; /* test dummy encryption */
+ struct fscrypt_dummy_policy dummy_enc_policy; /* test dummy encryption */
block_t unusable_cap_perc; /* percentage for cap */
block_t unusable_cap; /* Amount of space allowed to be
* unusable when disabling checkpoint
#define IS_IO_TRACED_PAGE(page) (0)
#endif
-#ifdef CONFIG_FS_ENCRYPTION
-#define DUMMY_ENCRYPTION_ENABLED(sbi) \
- (unlikely(F2FS_OPTION(sbi).dummy_enc_ctx.ctx != NULL))
-#else
-#define DUMMY_ENCRYPTION_ENABLED(sbi) (0)
-#endif
-
/* For compression */
enum compress_algorithm_type {
COMPRESS_LZO,
return F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS;
}
-static inline bool f2fs_may_encrypt(struct inode *dir, struct inode *inode)
-{
-#ifdef CONFIG_FS_ENCRYPTION
- struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
- umode_t mode = inode->i_mode;
-
- /*
- * If the directory encrypted or dummy encryption enabled,
- * then we should encrypt the inode.
- */
- if (IS_ENCRYPTED(dir) || DUMMY_ENCRYPTION_ENABLED(sbi))
- return (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode));
-#endif
- return false;
-}
-
static inline bool f2fs_may_compress(struct inode *inode)
{
if (IS_SWAPFILE(inode) || f2fs_is_pinned_file(inode) ||
nid_t ino;
struct inode *inode;
bool nid_free = false;
+ bool encrypt = false;
int xattr_size = 0;
int err;
F2FS_I(inode)->i_projid = make_kprojid(&init_user_ns,
F2FS_DEF_PROJID);
+ err = fscrypt_prepare_new_inode(dir, inode, &encrypt);
+ if (err)
+ goto fail_drop;
+
err = dquot_initialize(inode);
if (err)
goto fail_drop;
set_inode_flag(inode, FI_NEW_INODE);
- if (f2fs_may_encrypt(dir, inode))
+ if (encrypt)
f2fs_set_encrypted_inode(inode);
if (f2fs_sb_has_extra_attr(sbi)) {
* needed to allow it to be set or changed during remount. We do allow
* it to be specified during remount, but only if there is no change.
*/
- if (is_remount && !F2FS_OPTION(sbi).dummy_enc_ctx.ctx) {
+ if (is_remount && !F2FS_OPTION(sbi).dummy_enc_policy.policy) {
f2fs_warn(sbi, "Can't set test_dummy_encryption on remount");
return -EINVAL;
}
err = fscrypt_set_test_dummy_encryption(
- sb, arg, &F2FS_OPTION(sbi).dummy_enc_ctx);
+ sb, arg->from, &F2FS_OPTION(sbi).dummy_enc_policy);
if (err) {
if (err == -EEXIST)
f2fs_warn(sbi,
for (i = 0; i < MAXQUOTAS; i++)
kfree(F2FS_OPTION(sbi).s_qf_names[i]);
#endif
- fscrypt_free_dummy_context(&F2FS_OPTION(sbi).dummy_enc_ctx);
+ fscrypt_free_dummy_policy(&F2FS_OPTION(sbi).dummy_enc_policy);
destroy_percpu_info(sbi);
for (i = 0; i < NR_PAGE_TYPE; i++)
kvfree(sbi->write_io[i]);
ctx, len, fs_data, XATTR_CREATE);
}
-static const union fscrypt_context *
-f2fs_get_dummy_context(struct super_block *sb)
+static const union fscrypt_policy *f2fs_get_dummy_policy(struct super_block *sb)
{
- return F2FS_OPTION(F2FS_SB(sb)).dummy_enc_ctx.ctx;
+ return F2FS_OPTION(F2FS_SB(sb)).dummy_enc_policy.policy;
}
static bool f2fs_has_stable_inodes(struct super_block *sb)
.key_prefix = "f2fs:",
.get_context = f2fs_get_context,
.set_context = f2fs_set_context,
- .get_dummy_context = f2fs_get_dummy_context,
+ .get_dummy_policy = f2fs_get_dummy_policy,
.empty_dir = f2fs_empty_dir,
.max_namelen = F2FS_NAME_LEN,
.has_stable_inodes = f2fs_has_stable_inodes,
for (i = 0; i < MAXQUOTAS; i++)
kfree(F2FS_OPTION(sbi).s_qf_names[i]);
#endif
- fscrypt_free_dummy_context(&F2FS_OPTION(sbi).dummy_enc_ctx);
+ fscrypt_free_dummy_policy(&F2FS_OPTION(sbi).dummy_enc_policy);
kvfree(options);
free_sb_buf:
kfree(raw_super);
#include <linux/rcupdate.h>
#include <linux/close_range.h>
#include <net/sock.h>
+#include <linux/io_uring.h>
unsigned int sysctl_nr_open __read_mostly = 1024*1024;
unsigned int sysctl_nr_open_min = BITS_PER_LONG;
struct files_struct * files = tsk->files;
if (files) {
+ io_uring_files_cancel(files);
task_lock(tsk);
tsk->files = NULL;
task_unlock(tsk);
/*
* namespace.c
*/
-extern void *copy_mount_options(const void __user *);
-extern char *copy_mount_string(const void __user *);
-
extern struct vfsmount *lookup_mnt(const struct path *);
extern int finish_automount(struct vfsmount *, struct path *);
#include <linux/rculist_nulls.h>
#include <linux/fs_struct.h>
#include <linux/task_work.h>
+#include <linux/blk-cgroup.h>
#include "io-wq.h"
IO_WORKER_F_UP = 1, /* up and active */
IO_WORKER_F_RUNNING = 2, /* account as running */
IO_WORKER_F_FREE = 4, /* worker on free list */
- IO_WORKER_F_EXITING = 8, /* worker exiting */
- IO_WORKER_F_FIXED = 16, /* static idle worker */
- IO_WORKER_F_BOUND = 32, /* is doing bounded work */
+ IO_WORKER_F_FIXED = 8, /* static idle worker */
+ IO_WORKER_F_BOUND = 16, /* is doing bounded work */
};
enum {
struct rcu_head rcu;
struct mm_struct *mm;
+#ifdef CONFIG_BLK_CGROUP
+ struct cgroup_subsys_state *blkcg_css;
+#endif
const struct cred *cur_creds;
const struct cred *saved_creds;
struct files_struct *restore_files;
+ struct nsproxy *restore_nsproxy;
struct fs_struct *restore_fs;
};
*/
struct io_wqe {
struct {
- spinlock_t lock;
+ raw_spinlock_t lock;
struct io_wq_work_list work_list;
unsigned long hash_map;
unsigned flags;
if (current->files != worker->restore_files) {
__acquire(&wqe->lock);
- spin_unlock_irq(&wqe->lock);
+ raw_spin_unlock_irq(&wqe->lock);
dropped_lock = true;
task_lock(current);
current->files = worker->restore_files;
+ current->nsproxy = worker->restore_nsproxy;
task_unlock(current);
}
if (worker->mm) {
if (!dropped_lock) {
__acquire(&wqe->lock);
- spin_unlock_irq(&wqe->lock);
+ raw_spin_unlock_irq(&wqe->lock);
dropped_lock = true;
}
__set_current_state(TASK_RUNNING);
worker->mm = NULL;
}
+#ifdef CONFIG_BLK_CGROUP
+ if (worker->blkcg_css) {
+ kthread_associate_blkcg(NULL);
+ worker->blkcg_css = NULL;
+ }
+#endif
+
return dropped_lock;
}
{
struct io_wqe *wqe = worker->wqe;
struct io_wqe_acct *acct = io_wqe_get_acct(wqe, worker);
- unsigned nr_workers;
/*
* If we're not at zero, someone else is holding a brief reference
worker->flags = 0;
preempt_enable();
- spin_lock_irq(&wqe->lock);
+ raw_spin_lock_irq(&wqe->lock);
hlist_nulls_del_rcu(&worker->nulls_node);
list_del_rcu(&worker->all_list);
if (__io_worker_unuse(wqe, worker)) {
__release(&wqe->lock);
- spin_lock_irq(&wqe->lock);
+ raw_spin_lock_irq(&wqe->lock);
}
acct->nr_workers--;
- nr_workers = wqe->acct[IO_WQ_ACCT_BOUND].nr_workers +
- wqe->acct[IO_WQ_ACCT_UNBOUND].nr_workers;
- spin_unlock_irq(&wqe->lock);
-
- /* all workers gone, wq exit can proceed */
- if (!nr_workers && refcount_dec_and_test(&wqe->wq->refs))
- complete(&wqe->wq->done);
+ raw_spin_unlock_irq(&wqe->lock);
kfree_rcu(worker, rcu);
+ if (refcount_dec_and_test(&wqe->wq->refs))
+ complete(&wqe->wq->done);
}
static inline bool io_wqe_run_queue(struct io_wqe *wqe)
worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING);
worker->restore_files = current->files;
+ worker->restore_nsproxy = current->nsproxy;
worker->restore_fs = current->fs;
io_wqe_inc_running(wqe, worker);
}
work->flags |= IO_WQ_WORK_CANCEL;
}
+static inline void io_wq_switch_blkcg(struct io_worker *worker,
+ struct io_wq_work *work)
+{
+#ifdef CONFIG_BLK_CGROUP
+ if (work->blkcg_css != worker->blkcg_css) {
+ kthread_associate_blkcg(work->blkcg_css);
+ worker->blkcg_css = work->blkcg_css;
+ }
+#endif
+}
+
static void io_wq_switch_creds(struct io_worker *worker,
struct io_wq_work *work)
{
if (work->files && current->files != work->files) {
task_lock(current);
current->files = work->files;
+ current->nsproxy = work->nsproxy;
task_unlock(current);
}
if (work->fs && current->fs != work->fs)
if (worker->cur_creds != work->creds)
io_wq_switch_creds(worker, work);
current->signal->rlim[RLIMIT_FSIZE].rlim_cur = work->fsize;
+ io_wq_switch_blkcg(worker, work);
}
static void io_assign_current_work(struct io_worker *worker,
else if (!wq_list_empty(&wqe->work_list))
wqe->flags |= IO_WQE_FLAG_STALLED;
- spin_unlock_irq(&wqe->lock);
+ raw_spin_unlock_irq(&wqe->lock);
if (!work)
break;
io_assign_current_work(worker, work);
io_wqe_enqueue(wqe, linked);
if (hash != -1U && !next_hashed) {
- spin_lock_irq(&wqe->lock);
+ raw_spin_lock_irq(&wqe->lock);
wqe->hash_map &= ~BIT_ULL(hash);
wqe->flags &= ~IO_WQE_FLAG_STALLED;
/* skip unnecessary unlock-lock wqe->lock */
if (!work)
goto get_next;
- spin_unlock_irq(&wqe->lock);
+ raw_spin_unlock_irq(&wqe->lock);
}
} while (work);
- spin_lock_irq(&wqe->lock);
+ raw_spin_lock_irq(&wqe->lock);
} while (1);
}
while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) {
set_current_state(TASK_INTERRUPTIBLE);
loop:
- spin_lock_irq(&wqe->lock);
+ raw_spin_lock_irq(&wqe->lock);
if (io_wqe_run_queue(wqe)) {
__set_current_state(TASK_RUNNING);
io_worker_handle_work(worker);
__release(&wqe->lock);
goto loop;
}
- spin_unlock_irq(&wqe->lock);
+ raw_spin_unlock_irq(&wqe->lock);
if (signal_pending(current))
flush_signals(current);
if (schedule_timeout(WORKER_IDLE_TIMEOUT))
}
if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) {
- spin_lock_irq(&wqe->lock);
+ raw_spin_lock_irq(&wqe->lock);
if (!wq_list_empty(&wqe->work_list))
io_worker_handle_work(worker);
else
- spin_unlock_irq(&wqe->lock);
+ raw_spin_unlock_irq(&wqe->lock);
}
io_worker_exit(worker);
worker->flags &= ~IO_WORKER_F_RUNNING;
- spin_lock_irq(&wqe->lock);
+ raw_spin_lock_irq(&wqe->lock);
io_wqe_dec_running(wqe, worker);
- spin_unlock_irq(&wqe->lock);
+ raw_spin_unlock_irq(&wqe->lock);
}
static bool create_io_worker(struct io_wq *wq, struct io_wqe *wqe, int index)
{
- struct io_wqe_acct *acct =&wqe->acct[index];
+ struct io_wqe_acct *acct = &wqe->acct[index];
struct io_worker *worker;
worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, wqe->node);
return false;
}
- spin_lock_irq(&wqe->lock);
+ raw_spin_lock_irq(&wqe->lock);
hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list);
list_add_tail_rcu(&worker->all_list, &wqe->all_list);
worker->flags |= IO_WORKER_F_FREE;
if (!acct->nr_workers && (worker->flags & IO_WORKER_F_BOUND))
worker->flags |= IO_WORKER_F_FIXED;
acct->nr_workers++;
- spin_unlock_irq(&wqe->lock);
+ raw_spin_unlock_irq(&wqe->lock);
if (index == IO_WQ_ACCT_UNBOUND)
atomic_inc(&wq->user->processes);
+ refcount_inc(&wq->refs);
wake_up_process(worker->task);
return true;
}
return acct->nr_workers < acct->max_workers;
}
+static bool io_wqe_worker_send_sig(struct io_worker *worker, void *data)
+{
+ send_sig(SIGINT, worker->task, 1);
+ return false;
+}
+
+/*
+ * Iterate the passed in list and call the specific function for each
+ * worker that isn't exiting
+ */
+static bool io_wq_for_each_worker(struct io_wqe *wqe,
+ bool (*func)(struct io_worker *, void *),
+ void *data)
+{
+ struct io_worker *worker;
+ bool ret = false;
+
+ list_for_each_entry_rcu(worker, &wqe->all_list, all_list) {
+ if (io_worker_get(worker)) {
+ /* no task if node is/was offline */
+ if (worker->task)
+ ret = func(worker, data);
+ io_worker_release(worker);
+ if (ret)
+ break;
+ }
+ }
+
+ return ret;
+}
+
+static bool io_wq_worker_wake(struct io_worker *worker, void *data)
+{
+ wake_up_process(worker->task);
+ return false;
+}
+
/*
* Manager thread. Tasked with creating new workers, if we need them.
*/
static int io_wq_manager(void *data)
{
struct io_wq *wq = data;
- int workers_to_create = num_possible_nodes();
int node;
/* create fixed workers */
- refcount_set(&wq->refs, workers_to_create);
+ refcount_set(&wq->refs, 1);
for_each_node(node) {
if (!node_online(node))
continue;
- if (!create_io_worker(wq, wq->wqes[node], IO_WQ_ACCT_BOUND))
- goto err;
- workers_to_create--;
+ if (create_io_worker(wq, wq->wqes[node], IO_WQ_ACCT_BOUND))
+ continue;
+ set_bit(IO_WQ_BIT_ERROR, &wq->state);
+ set_bit(IO_WQ_BIT_EXIT, &wq->state);
+ goto out;
}
- while (workers_to_create--)
- refcount_dec(&wq->refs);
-
complete(&wq->done);
while (!kthread_should_stop()) {
if (!node_online(node))
continue;
- spin_lock_irq(&wqe->lock);
+ raw_spin_lock_irq(&wqe->lock);
if (io_wqe_need_worker(wqe, IO_WQ_ACCT_BOUND))
fork_worker[IO_WQ_ACCT_BOUND] = true;
if (io_wqe_need_worker(wqe, IO_WQ_ACCT_UNBOUND))
fork_worker[IO_WQ_ACCT_UNBOUND] = true;
- spin_unlock_irq(&wqe->lock);
+ raw_spin_unlock_irq(&wqe->lock);
if (fork_worker[IO_WQ_ACCT_BOUND])
create_io_worker(wq, wqe, IO_WQ_ACCT_BOUND);
if (fork_worker[IO_WQ_ACCT_UNBOUND])
if (current->task_works)
task_work_run();
- return 0;
-err:
- set_bit(IO_WQ_BIT_ERROR, &wq->state);
- set_bit(IO_WQ_BIT_EXIT, &wq->state);
- if (refcount_sub_and_test(workers_to_create, &wq->refs))
+out:
+ if (refcount_dec_and_test(&wq->refs)) {
complete(&wq->done);
+ return 0;
+ }
+ /* if ERROR is set and we get here, we have workers to wake */
+ if (test_bit(IO_WQ_BIT_ERROR, &wq->state)) {
+ rcu_read_lock();
+ for_each_node(node)
+ io_wq_for_each_worker(wq->wqes[node], io_wq_worker_wake, NULL);
+ rcu_read_unlock();
+ }
return 0;
}
}
work_flags = work->flags;
- spin_lock_irqsave(&wqe->lock, flags);
+ raw_spin_lock_irqsave(&wqe->lock, flags);
io_wqe_insert_work(wqe, work);
wqe->flags &= ~IO_WQE_FLAG_STALLED;
- spin_unlock_irqrestore(&wqe->lock, flags);
+ raw_spin_unlock_irqrestore(&wqe->lock, flags);
if ((work_flags & IO_WQ_WORK_CONCURRENT) ||
!atomic_read(&acct->nr_running))
work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT));
}
-static bool io_wqe_worker_send_sig(struct io_worker *worker, void *data)
-{
- send_sig(SIGINT, worker->task, 1);
- return false;
-}
-
-/*
- * Iterate the passed in list and call the specific function for each
- * worker that isn't exiting
- */
-static bool io_wq_for_each_worker(struct io_wqe *wqe,
- bool (*func)(struct io_worker *, void *),
- void *data)
-{
- struct io_worker *worker;
- bool ret = false;
-
- list_for_each_entry_rcu(worker, &wqe->all_list, all_list) {
- if (io_worker_get(worker)) {
- /* no task if node is/was offline */
- if (worker->task)
- ret = func(worker, data);
- io_worker_release(worker);
- if (ret)
- break;
- }
- }
-
- return ret;
-}
-
void io_wq_cancel_all(struct io_wq *wq)
{
int node;
unsigned long flags;
retry:
- spin_lock_irqsave(&wqe->lock, flags);
+ raw_spin_lock_irqsave(&wqe->lock, flags);
wq_list_for_each(node, prev, &wqe->work_list) {
work = container_of(node, struct io_wq_work, list);
if (!match->fn(work, match->data))
continue;
io_wqe_remove_pending(wqe, work, prev);
- spin_unlock_irqrestore(&wqe->lock, flags);
+ raw_spin_unlock_irqrestore(&wqe->lock, flags);
io_run_cancel(work, wqe);
match->nr_pending++;
if (!match->cancel_all)
/* not safe to continue after unlock */
goto retry;
}
- spin_unlock_irqrestore(&wqe->lock, flags);
+ raw_spin_unlock_irqrestore(&wqe->lock, flags);
}
static void io_wqe_cancel_running_work(struct io_wqe *wqe,
}
atomic_set(&wqe->acct[IO_WQ_ACCT_UNBOUND].nr_running, 0);
wqe->wq = wq;
- spin_lock_init(&wqe->lock);
+ raw_spin_lock_init(&wqe->lock);
INIT_WQ_LIST(&wqe->work_list);
INIT_HLIST_NULLS_HEAD(&wqe->free_list, 0);
INIT_LIST_HEAD(&wqe->all_list);
return refcount_inc_not_zero(&wq->use_refs);
}
-static bool io_wq_worker_wake(struct io_worker *worker, void *data)
-{
- wake_up_process(worker->task);
- return false;
-}
-
static void __io_wq_destroy(struct io_wq *wq)
{
int node;
struct io_wq_work_node list;
struct files_struct *files;
struct mm_struct *mm;
+#ifdef CONFIG_BLK_CGROUP
+ struct cgroup_subsys_state *blkcg_css;
+#endif
const struct cred *creds;
+ struct nsproxy *nsproxy;
struct fs_struct *fs;
unsigned long fsize;
unsigned flags;
#include <linux/splice.h>
#include <linux/task_work.h>
#include <linux/pagemap.h>
+#include <linux/io_uring.h>
+#include <linux/blk-cgroup.h>
#define CREATE_TRACE_POINTS
#include <trace/events/io_uring.h>
#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
+#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
+ IORING_REGISTER_LAST + IORING_OP_LAST)
struct io_uring {
u32 head ____cacheline_aligned_in_smp;
size_t len;
struct bio_vec *bvec;
unsigned int nr_bvecs;
+ unsigned long acct_pages;
};
struct fixed_file_table {
struct fixed_file_table *table;
struct io_ring_ctx *ctx;
- struct percpu_ref *cur_refs;
+ struct fixed_file_ref_node *node;
struct percpu_ref refs;
struct completion done;
struct list_head ref_list;
__u16 bid;
};
+struct io_restriction {
+ DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
+ DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
+ u8 sqe_flags_allowed;
+ u8 sqe_flags_required;
+ bool registered;
+};
+
+struct io_sq_data {
+ refcount_t refs;
+ struct mutex lock;
+
+ /* ctx's that are using this sqd */
+ struct list_head ctx_list;
+ struct list_head ctx_new_list;
+ struct mutex ctx_lock;
+
+ struct task_struct *thread;
+ struct wait_queue_head wait;
+};
+
struct io_ring_ctx {
struct {
struct percpu_ref refs;
unsigned int cq_overflow_flushed: 1;
unsigned int drain_next: 1;
unsigned int eventfd_async: 1;
+ unsigned int restricted: 1;
/*
* Ring buffer of indices into array of io_uring_sqe, which is
/* IO offload */
struct io_wq *io_wq;
- struct task_struct *sqo_thread; /* if using sq thread polling */
- struct mm_struct *sqo_mm;
- wait_queue_head_t sqo_wait;
+
+ /*
+ * For SQPOLL usage - we hold a reference to the parent task, so we
+ * have access to the ->files
+ */
+ struct task_struct *sqo_task;
+
+ /* Only used for accounting purposes */
+ struct mm_struct *mm_account;
+
+#ifdef CONFIG_BLK_CGROUP
+ struct cgroup_subsys_state *sqo_blkcg_css;
+#endif
+
+ struct io_sq_data *sq_data; /* if using sq thread polling */
+
+ struct wait_queue_head sqo_sq_wait;
+ struct wait_queue_entry sqo_wait_entry;
+ struct list_head sqd_list;
/*
* If used, fixed file set. Writers must ensure that ->refs is dead,
*/
struct fixed_file_data *file_data;
unsigned nr_user_files;
- int ring_fd;
- struct file *ring_file;
/* if used, fixed mapped user buffers */
unsigned nr_user_bufs;
struct llist_head file_put_llist;
struct work_struct exit_work;
+ struct io_restriction restrictions;
};
/*
struct io_timeout {
struct file *file;
- u64 addr;
- int flags;
u32 off;
u32 target_seq;
struct list_head list;
};
+struct io_timeout_rem {
+ struct file *file;
+ u64 addr;
+};
+
struct io_rw {
/* NOTE: kiocb has the file as the first member, so don't do it here */
struct kiocb kiocb;
struct wait_page_queue wpq;
};
-struct io_async_ctx {
- union {
- struct io_async_rw rw;
- struct io_async_msghdr msg;
- struct io_async_connect connect;
- struct io_timeout_data timeout;
- };
-};
-
enum {
REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
REQ_F_BUFFER_SELECTED_BIT,
REQ_F_NO_FILE_TABLE_BIT,
REQ_F_WORK_INITIALIZED_BIT,
- REQ_F_TASK_PINNED_BIT,
/* not a real bit, just to check we're not overflowing the space */
__REQ_F_LAST_BIT,
REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT),
/* io_wq_work is initialized */
REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT),
- /* req->task is refcounted */
- REQ_F_TASK_PINNED = BIT(REQ_F_TASK_PINNED_BIT),
};
struct async_poll {
struct io_sync sync;
struct io_cancel cancel;
struct io_timeout timeout;
+ struct io_timeout_rem timeout_rem;
struct io_connect connect;
struct io_sr_msg sr_msg;
struct io_open open;
struct io_completion compl;
};
- struct io_async_ctx *io;
+ /* opcode allocated if it needs to store data for async defer */
+ void *async_data;
u8 opcode;
/* polled IO has completed */
u8 iopoll_completed;
};
struct io_op_def {
- /* needs req->io allocated for deferral/async */
- unsigned async_ctx : 1;
/* needs current->mm setup, does mm access */
unsigned needs_mm : 1;
/* needs req->file assigned */
unsigned pollout : 1;
/* op supports buffer selection */
unsigned buffer_select : 1;
+ /* needs rlimit(RLIMIT_FSIZE) assigned */
unsigned needs_fsize : 1;
+ /* must always have async data allocated */
+ unsigned needs_async_data : 1;
+ /* needs blkcg context, issues async io potentially */
+ unsigned needs_blkcg : 1;
+ /* size of async data needed, if any */
+ unsigned short async_size;
};
-static const struct io_op_def io_op_defs[] = {
+static const struct io_op_def io_op_defs[] __read_mostly = {
[IORING_OP_NOP] = {},
[IORING_OP_READV] = {
- .async_ctx = 1,
.needs_mm = 1,
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.buffer_select = 1,
+ .needs_async_data = 1,
+ .needs_blkcg = 1,
+ .async_size = sizeof(struct io_async_rw),
},
[IORING_OP_WRITEV] = {
- .async_ctx = 1,
.needs_mm = 1,
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.needs_fsize = 1,
+ .needs_async_data = 1,
+ .needs_blkcg = 1,
+ .async_size = sizeof(struct io_async_rw),
},
[IORING_OP_FSYNC] = {
.needs_file = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_READ_FIXED] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
+ .needs_blkcg = 1,
+ .async_size = sizeof(struct io_async_rw),
},
[IORING_OP_WRITE_FIXED] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.needs_fsize = 1,
+ .needs_blkcg = 1,
+ .async_size = sizeof(struct io_async_rw),
},
[IORING_OP_POLL_ADD] = {
.needs_file = 1,
[IORING_OP_POLL_REMOVE] = {},
[IORING_OP_SYNC_FILE_RANGE] = {
.needs_file = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_SENDMSG] = {
- .async_ctx = 1,
.needs_mm = 1,
.needs_file = 1,
.unbound_nonreg_file = 1,
.needs_fs = 1,
.pollout = 1,
+ .needs_async_data = 1,
+ .needs_blkcg = 1,
+ .async_size = sizeof(struct io_async_msghdr),
},
[IORING_OP_RECVMSG] = {
- .async_ctx = 1,
.needs_mm = 1,
.needs_file = 1,
.unbound_nonreg_file = 1,
.needs_fs = 1,
.pollin = 1,
.buffer_select = 1,
+ .needs_async_data = 1,
+ .needs_blkcg = 1,
+ .async_size = sizeof(struct io_async_msghdr),
},
[IORING_OP_TIMEOUT] = {
- .async_ctx = 1,
.needs_mm = 1,
+ .needs_async_data = 1,
+ .async_size = sizeof(struct io_timeout_data),
},
[IORING_OP_TIMEOUT_REMOVE] = {},
[IORING_OP_ACCEPT] = {
},
[IORING_OP_ASYNC_CANCEL] = {},
[IORING_OP_LINK_TIMEOUT] = {
- .async_ctx = 1,
.needs_mm = 1,
+ .needs_async_data = 1,
+ .async_size = sizeof(struct io_timeout_data),
},
[IORING_OP_CONNECT] = {
- .async_ctx = 1,
.needs_mm = 1,
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
+ .needs_async_data = 1,
+ .async_size = sizeof(struct io_async_connect),
},
[IORING_OP_FALLOCATE] = {
.needs_file = 1,
.needs_fsize = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_OPENAT] = {
.file_table = 1,
.needs_fs = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_CLOSE] = {
.needs_file = 1,
.needs_file_no_error = 1,
.file_table = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_FILES_UPDATE] = {
.needs_mm = 1,
.needs_mm = 1,
.needs_fs = 1,
.file_table = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_READ] = {
.needs_mm = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.buffer_select = 1,
+ .needs_blkcg = 1,
+ .async_size = sizeof(struct io_async_rw),
},
[IORING_OP_WRITE] = {
.needs_mm = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.needs_fsize = 1,
+ .needs_blkcg = 1,
+ .async_size = sizeof(struct io_async_rw),
},
[IORING_OP_FADVISE] = {
.needs_file = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_MADVISE] = {
.needs_mm = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_SEND] = {
.needs_mm = 1,
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_RECV] = {
.needs_mm = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.buffer_select = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_OPENAT2] = {
.file_table = 1,
.needs_fs = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_EPOLL_CTL] = {
.unbound_nonreg_file = 1,
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
+ .needs_blkcg = 1,
},
[IORING_OP_PROVIDE_BUFFERS] = {},
[IORING_OP_REMOVE_BUFFERS] = {},
static int __io_sqe_files_update(struct io_ring_ctx *ctx,
struct io_uring_files_update *ip,
unsigned nr_args);
-static int io_prep_work_files(struct io_kiocb *req);
static void __io_clean_op(struct io_kiocb *req);
-static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
- int fd, struct file **out_file, bool fixed);
-static void __io_queue_sqe(struct io_kiocb *req,
- const struct io_uring_sqe *sqe,
- struct io_comp_state *cs);
+static struct file *io_file_get(struct io_submit_state *state,
+ struct io_kiocb *req, int fd, bool fixed);
+static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs);
static void io_file_put_work(struct work_struct *work);
static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
static struct kmem_cache *req_cachep;
-static const struct file_operations io_uring_fops;
+static const struct file_operations io_uring_fops __read_mostly;
struct sock *io_uring_get_socket(struct file *file)
{
}
EXPORT_SYMBOL(io_uring_get_socket);
-static void io_get_req_task(struct io_kiocb *req)
-{
- if (req->flags & REQ_F_TASK_PINNED)
- return;
- get_task_struct(req->task);
- req->flags |= REQ_F_TASK_PINNED;
-}
-
static inline void io_clean_op(struct io_kiocb *req)
{
if (req->flags & (REQ_F_NEED_CLEANUP | REQ_F_BUFFER_SELECTED |
__io_clean_op(req);
}
-/* not idempotent -- it doesn't clear REQ_F_TASK_PINNED */
-static void __io_put_req_task(struct io_kiocb *req)
-{
- if (req->flags & REQ_F_TASK_PINNED)
- put_task_struct(req->task);
-}
-
static void io_sq_thread_drop_mm(void)
{
struct mm_struct *mm = current->mm;
{
if (!current->mm) {
if (unlikely(!(ctx->flags & IORING_SETUP_SQPOLL) ||
- !mmget_not_zero(ctx->sqo_mm)))
+ !ctx->sqo_task->mm ||
+ !mmget_not_zero(ctx->sqo_task->mm)))
return -EFAULT;
- kthread_use_mm(ctx->sqo_mm);
+ kthread_use_mm(ctx->sqo_task->mm);
}
return 0;
return __io_sq_thread_acquire_mm(ctx);
}
+static void io_sq_thread_associate_blkcg(struct io_ring_ctx *ctx,
+ struct cgroup_subsys_state **cur_css)
+
+{
+#ifdef CONFIG_BLK_CGROUP
+ /* puts the old one when swapping */
+ if (*cur_css != ctx->sqo_blkcg_css) {
+ kthread_associate_blkcg(ctx->sqo_blkcg_css);
+ *cur_css = ctx->sqo_blkcg_css;
+ }
+#endif
+}
+
+static void io_sq_thread_unassociate_blkcg(void)
+{
+#ifdef CONFIG_BLK_CGROUP
+ kthread_associate_blkcg(NULL);
+#endif
+}
+
static inline void req_set_fail_links(struct io_kiocb *req)
{
if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
goto err;
ctx->flags = p->flags;
- init_waitqueue_head(&ctx->sqo_wait);
+ init_waitqueue_head(&ctx->sqo_sq_wait);
+ INIT_LIST_HEAD(&ctx->sqd_list);
init_waitqueue_head(&ctx->cq_wait);
INIT_LIST_HEAD(&ctx->cq_overflow_list);
init_completion(&ctx->ref_comp);
mmdrop(req->work.mm);
req->work.mm = NULL;
}
+#ifdef CONFIG_BLK_CGROUP
+ if (req->work.blkcg_css)
+ css_put(req->work.blkcg_css);
+#endif
if (req->work.creds) {
put_cred(req->work.creds);
req->work.creds = NULL;
static void io_prep_async_work(struct io_kiocb *req)
{
const struct io_op_def *def = &io_op_defs[req->opcode];
+ struct io_ring_ctx *ctx = req->ctx;
io_req_init_async(req);
if (req->flags & REQ_F_ISREG) {
- if (def->hash_reg_file || (req->ctx->flags & IORING_SETUP_IOPOLL))
+ if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))
io_wq_hash_work(&req->work, file_inode(req->file));
} else {
if (def->unbound_nonreg_file)
req->work.flags |= IO_WQ_WORK_UNBOUND;
}
+ if (!req->work.files && io_op_defs[req->opcode].file_table &&
+ !(req->flags & REQ_F_NO_FILE_TABLE)) {
+ req->work.files = get_files_struct(current);
+ get_nsproxy(current->nsproxy);
+ req->work.nsproxy = current->nsproxy;
+ req->flags |= REQ_F_INFLIGHT;
+
+ spin_lock_irq(&ctx->inflight_lock);
+ list_add(&req->inflight_entry, &ctx->inflight_list);
+ spin_unlock_irq(&ctx->inflight_lock);
+ }
if (!req->work.mm && def->needs_mm) {
mmgrab(current->mm);
req->work.mm = current->mm;
}
+#ifdef CONFIG_BLK_CGROUP
+ if (!req->work.blkcg_css && def->needs_blkcg) {
+ rcu_read_lock();
+ req->work.blkcg_css = blkcg_css();
+ /*
+ * This should be rare, either the cgroup is dying or the task
+ * is moving cgroups. Just punt to root for the handful of ios.
+ */
+ if (!css_tryget_online(req->work.blkcg_css))
+ req->work.blkcg_css = NULL;
+ rcu_read_unlock();
+ }
+#endif
if (!req->work.creds)
req->work.creds = get_current_cred();
if (!req->work.fs && def->needs_fs) {
static void io_kill_timeout(struct io_kiocb *req)
{
+ struct io_timeout_data *io = req->async_data;
int ret;
- ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
+ ret = hrtimer_try_to_cancel(&io->timer);
if (ret != -1) {
atomic_set(&req->ctx->cq_timeouts,
atomic_read(&req->ctx->cq_timeouts) + 1);
}
}
-static void io_kill_timeouts(struct io_ring_ctx *ctx)
+static bool io_task_match(struct io_kiocb *req, struct task_struct *tsk)
+{
+ struct io_ring_ctx *ctx = req->ctx;
+
+ if (!tsk || req->task == tsk)
+ return true;
+ if (ctx->flags & IORING_SETUP_SQPOLL) {
+ if (ctx->sq_data && req->task == ctx->sq_data->thread)
+ return true;
+ }
+ return false;
+}
+
+/*
+ * Returns true if we found and killed one or more timeouts
+ */
+static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk)
{
struct io_kiocb *req, *tmp;
+ int canceled = 0;
spin_lock_irq(&ctx->completion_lock);
- list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list)
- io_kill_timeout(req);
+ list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
+ if (io_task_match(req, tsk)) {
+ io_kill_timeout(req);
+ canceled++;
+ }
+ }
spin_unlock_irq(&ctx->completion_lock);
+ return canceled != 0;
}
static void __io_queue_deferred(struct io_ring_ctx *ctx)
__io_queue_deferred(ctx);
}
+static inline bool io_sqring_full(struct io_ring_ctx *ctx)
+{
+ struct io_rings *r = ctx->rings;
+
+ return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == r->sq_ring_entries;
+}
+
static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
{
struct io_rings *rings = ctx->rings;
{
if (waitqueue_active(&ctx->wait))
wake_up(&ctx->wait);
- if (waitqueue_active(&ctx->sqo_wait))
- wake_up(&ctx->sqo_wait);
+ if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait))
+ wake_up(&ctx->sq_data->wait);
if (io_should_trigger_evfd(ctx))
eventfd_signal(ctx->cq_ev_fd, 1);
}
}
}
+static inline bool io_match_files(struct io_kiocb *req,
+ struct files_struct *files)
+{
+ if (!files)
+ return true;
+ if (req->flags & REQ_F_WORK_INITIALIZED)
+ return req->work.files == files;
+ return false;
+}
+
/* Returns true if there are no backlogged entries after the flush */
-static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
+static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force,
+ struct task_struct *tsk,
+ struct files_struct *files)
{
struct io_rings *rings = ctx->rings;
+ struct io_kiocb *req, *tmp;
struct io_uring_cqe *cqe;
- struct io_kiocb *req;
unsigned long flags;
LIST_HEAD(list);
ctx->cq_overflow_flushed = 1;
cqe = NULL;
- while (!list_empty(&ctx->cq_overflow_list)) {
+ list_for_each_entry_safe(req, tmp, &ctx->cq_overflow_list, compl.list) {
+ if (tsk && req->task != tsk)
+ continue;
+ if (!io_match_files(req, files))
+ continue;
+
cqe = io_get_cqring(ctx);
if (!cqe && !force)
break;
- req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
- compl.list);
list_move(&req->compl.list, &list);
if (cqe) {
WRITE_ONCE(cqe->user_data, req->user_data);
WRITE_ONCE(cqe->user_data, req->user_data);
WRITE_ONCE(cqe->res, res);
WRITE_ONCE(cqe->flags, cflags);
- } else if (ctx->cq_overflow_flushed) {
+ } else if (ctx->cq_overflow_flushed || req->task->io_uring->in_idle) {
+ /*
+ * If we're in ring overflow flush mode, or in task cancel mode,
+ * then we cannot store the request for later flushing, we need
+ * to drop it on the floor.
+ */
WRITE_ONCE(ctx->rings->cq_overflow,
atomic_inc_return(&ctx->cached_cq_overflow));
} else {
static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx,
struct io_submit_state *state)
{
- gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
- struct io_kiocb *req;
-
if (!state->free_reqs) {
+ gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
size_t sz;
int ret;
goto fallback;
ret = 1;
}
- state->free_reqs = ret - 1;
- req = state->reqs[ret - 1];
- } else {
- state->free_reqs--;
- req = state->reqs[state->free_reqs];
+ state->free_reqs = ret;
}
- return req;
+ state->free_reqs--;
+ return state->reqs[state->free_reqs];
fallback:
return io_get_fallback_req(ctx);
}
{
io_clean_op(req);
- if (req->io)
- kfree(req->io);
+ if (req->async_data)
+ kfree(req->async_data);
if (req->file)
io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
static void __io_free_req_finish(struct io_kiocb *req)
{
+ struct io_uring_task *tctx = req->task->io_uring;
struct io_ring_ctx *ctx = req->ctx;
- __io_put_req_task(req);
+ atomic_long_inc(&tctx->req_complete);
+ if (tctx->in_idle)
+ wake_up(&tctx->wait);
+ put_task_struct(req->task);
+
if (likely(!io_is_fallback_req(req)))
kmem_cache_free(req_cachep, req);
else
static bool io_link_cancel_timeout(struct io_kiocb *req)
{
+ struct io_timeout_data *io = req->async_data;
struct io_ring_ctx *ctx = req->ctx;
int ret;
- ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
+ ret = hrtimer_try_to_cancel(&io->timer);
if (ret != -1) {
io_cqring_fill_event(req, -ECANCELED);
io_commit_cqring(ctx);
return __io_req_find_next(req);
}
-static int io_req_task_work_add(struct io_kiocb *req, struct callback_head *cb,
- bool twa_signal_ok)
+static int io_req_task_work_add(struct io_kiocb *req, bool twa_signal_ok)
{
struct task_struct *tsk = req->task;
struct io_ring_ctx *ctx = req->ctx;
if (!(ctx->flags & IORING_SETUP_SQPOLL) && twa_signal_ok)
notify = TWA_SIGNAL;
- ret = task_work_add(tsk, cb, notify);
+ ret = task_work_add(tsk, &req->task_work, notify);
if (!ret)
wake_up_process(tsk);
if (!__io_sq_thread_acquire_mm(ctx)) {
mutex_lock(&ctx->uring_lock);
- __io_queue_sqe(req, NULL, NULL);
+ __io_queue_sqe(req, NULL);
mutex_unlock(&ctx->uring_lock);
} else {
__io_req_task_cancel(req, -EFAULT);
init_task_work(&req->task_work, io_req_task_submit);
percpu_ref_get(&req->ctx->refs);
- ret = io_req_task_work_add(req, &req->task_work, true);
+ ret = io_req_task_work_add(req, true);
if (unlikely(ret)) {
struct task_struct *tsk;
if (rb->to_free)
__io_req_free_batch_flush(ctx, rb);
if (rb->task) {
+ atomic_long_add(rb->task_refs, &rb->task->io_uring->req_complete);
put_task_struct_many(rb->task, rb->task_refs);
rb->task = NULL;
}
if (req->flags & REQ_F_LINK_HEAD)
io_queue_next(req);
- if (req->flags & REQ_F_TASK_PINNED) {
- if (req->task != rb->task) {
- if (rb->task)
- put_task_struct_many(rb->task, rb->task_refs);
- rb->task = req->task;
- rb->task_refs = 0;
+ if (req->task != rb->task) {
+ if (rb->task) {
+ atomic_long_add(rb->task_refs, &rb->task->io_uring->req_complete);
+ put_task_struct_many(rb->task, rb->task_refs);
}
- rb->task_refs++;
- req->flags &= ~REQ_F_TASK_PINNED;
+ rb->task = req->task;
+ rb->task_refs = 0;
}
+ rb->task_refs++;
WARN_ON_ONCE(io_dismantle_req(req));
rb->reqs[rb->to_free++] = req;
if (noflush && !list_empty(&ctx->cq_overflow_list))
return -1U;
- io_cqring_overflow_flush(ctx, false);
+ io_cqring_overflow_flush(ctx, false, NULL, NULL);
}
/* See comment at the top of this file */
goto end_req;
}
- if (!req->io) {
+ if (!req->async_data) {
ret = io_import_iovec(rw, req, &iovec, &iter, false);
if (ret < 0)
goto end_req;
list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
if ((ctx->flags & IORING_SETUP_SQPOLL) &&
- wq_has_sleeper(&ctx->sqo_wait))
- wake_up(&ctx->sqo_wait);
+ wq_has_sleeper(&ctx->sq_data->wait))
+ wake_up(&ctx->sq_data->wait);
}
static void __io_state_file_put(struct io_submit_state *state)
if (state->file) {
if (state->fd == fd) {
state->has_refs--;
- state->ios_left--;
return state->file;
}
__io_state_file_put(state);
return NULL;
state->fd = fd;
- state->ios_left--;
- state->has_refs = state->ios_left;
+ state->has_refs = state->ios_left - 1;
return state->file;
}
return file->f_op->write_iter != NULL;
}
-static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
- bool force_nonblock)
+static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
struct io_ring_ctx *ctx = req->ctx;
struct kiocb *kiocb = &req->rw.kiocb;
if (kiocb->ki_flags & IOCB_NOWAIT)
req->flags |= REQ_F_NOWAIT;
- if (kiocb->ki_flags & IOCB_DIRECT)
- io_get_req_task(req);
-
- if (force_nonblock)
- kiocb->ki_flags |= IOCB_NOWAIT;
-
if (ctx->flags & IORING_SETUP_IOPOLL) {
if (!(kiocb->ki_flags & IOCB_DIRECT) ||
!kiocb->ki_filp->f_op->iopoll)
kiocb->ki_flags |= IOCB_HIPRI;
kiocb->ki_complete = io_complete_rw_iopoll;
req->iopoll_completed = 0;
- io_get_req_task(req);
} else {
if (kiocb->ki_flags & IOCB_HIPRI)
return -EINVAL;
struct io_comp_state *cs)
{
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
+ struct io_async_rw *io = req->async_data;
/* add previously done IO, if any */
- if (req->io && req->io->rw.bytes_done > 0) {
+ if (io && io->bytes_done > 0) {
if (ret < 0)
- ret = req->io->rw.bytes_done;
+ ret = io->bytes_done;
else
- ret += req->io->rw.bytes_done;
+ ret += io->bytes_done;
}
if (req->flags & REQ_F_CUR_POS)
struct io_ring_ctx *ctx = req->ctx;
size_t len = req->rw.len;
struct io_mapped_ubuf *imu;
- u16 index, buf_index;
+ u16 index, buf_index = req->buf_index;
size_t offset;
u64 buf_addr;
- /* attempt to use fixed buffers without having provided iovecs */
- if (unlikely(!ctx->user_bufs))
- return -EFAULT;
-
- buf_index = req->buf_index;
if (unlikely(buf_index >= ctx->nr_user_bufs))
return -EFAULT;
-
index = array_index_nospec(buf_index, ctx->nr_user_bufs);
imu = &ctx->user_bufs[index];
buf_addr = req->rw.addr;
return ret;
}
-#ifdef CONFIG_COMPAT
- if (req->ctx->compat)
- return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
- iovec, iter);
-#endif
-
- return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
+ return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter,
+ req->ctx->compat);
}
static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
struct iovec **iovec, struct iov_iter *iter,
bool needs_lock)
{
- if (!req->io)
+ struct io_async_rw *iorw = req->async_data;
+
+ if (!iorw)
return __io_import_iovec(rw, req, iovec, iter, needs_lock);
*iovec = NULL;
- return iov_iter_count(&req->io->rw.iter);
+ return iov_iter_count(&iorw->iter);
}
static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
{
- return kiocb->ki_filp->f_mode & FMODE_STREAM ? NULL : &kiocb->ki_pos;
+ return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
}
/*
static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
const struct iovec *fast_iov, struct iov_iter *iter)
{
- struct io_async_rw *rw = &req->io->rw;
+ struct io_async_rw *rw = req->async_data;
memcpy(&rw->iter, iter, sizeof(*iter));
- rw->free_iovec = NULL;
+ rw->free_iovec = iovec;
rw->bytes_done = 0;
/* can only be fixed buffers, no need to do anything */
if (iter->type == ITER_BVEC)
memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
sizeof(struct iovec) * iter->nr_segs);
} else {
- rw->free_iovec = iovec;
req->flags |= REQ_F_NEED_CLEANUP;
}
}
-static inline int __io_alloc_async_ctx(struct io_kiocb *req)
+static inline int __io_alloc_async_data(struct io_kiocb *req)
{
- req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
- return req->io == NULL;
+ WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
+ req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
+ return req->async_data == NULL;
}
-static int io_alloc_async_ctx(struct io_kiocb *req)
+static int io_alloc_async_data(struct io_kiocb *req)
{
- if (!io_op_defs[req->opcode].async_ctx)
+ if (!io_op_defs[req->opcode].needs_async_data)
return 0;
- return __io_alloc_async_ctx(req);
+ return __io_alloc_async_data(req);
}
static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
const struct iovec *fast_iov,
struct iov_iter *iter, bool force)
{
- if (!force && !io_op_defs[req->opcode].async_ctx)
+ if (!force && !io_op_defs[req->opcode].needs_async_data)
return 0;
- if (!req->io) {
- if (__io_alloc_async_ctx(req))
+ if (!req->async_data) {
+ if (__io_alloc_async_data(req))
return -ENOMEM;
io_req_map_rw(req, iovec, fast_iov, iter);
return 0;
}
-static inline int io_rw_prep_async(struct io_kiocb *req, int rw,
- bool force_nonblock)
+static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
{
- struct io_async_rw *iorw = &req->io->rw;
- struct iovec *iov;
+ struct io_async_rw *iorw = req->async_data;
+ struct iovec *iov = iorw->fast_iov;
ssize_t ret;
- iorw->iter.iov = iov = iorw->fast_iov;
- ret = __io_import_iovec(rw, req, &iov, &iorw->iter, !force_nonblock);
+ ret = __io_import_iovec(rw, req, &iov, &iorw->iter, false);
if (unlikely(ret < 0))
return ret;
- iorw->iter.iov = iov;
- io_req_map_rw(req, iorw->iter.iov, iorw->fast_iov, &iorw->iter);
+ iorw->bytes_done = 0;
+ iorw->free_iovec = iov;
+ if (iov)
+ req->flags |= REQ_F_NEED_CLEANUP;
return 0;
}
-static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
- bool force_nonblock)
+static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
ssize_t ret;
- ret = io_prep_rw(req, sqe, force_nonblock);
+ ret = io_prep_rw(req, sqe);
if (ret)
return ret;
return -EBADF;
/* either don't need iovec imported or already have it */
- if (!req->io || req->flags & REQ_F_NEED_CLEANUP)
+ if (!req->async_data)
return 0;
- return io_rw_prep_async(req, READ, force_nonblock);
+ return io_rw_prep_async(req, READ);
}
/*
if (!wake_page_match(wpq, key))
return 0;
+ req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;
list_del_init(&wait->entry);
init_task_work(&req->task_work, io_req_task_submit);
/* submit ref gets dropped, acquire a new one */
refcount_inc(&req->refs);
- ret = io_req_task_work_add(req, &req->task_work, true);
+ ret = io_req_task_work_add(req, true);
if (unlikely(ret)) {
struct task_struct *tsk;
*/
static bool io_rw_should_retry(struct io_kiocb *req)
{
- struct wait_page_queue *wait = &req->io->rw.wpq;
+ struct io_async_rw *rw = req->async_data;
+ struct wait_page_queue *wait = &rw->wpq;
struct kiocb *kiocb = &req->rw.kiocb;
/* never retry for NOWAIT, we just complete with -EAGAIN */
wait->wait.flags = 0;
INIT_LIST_HEAD(&wait->wait.entry);
kiocb->ki_flags |= IOCB_WAITQ;
+ kiocb->ki_flags &= ~IOCB_NOWAIT;
kiocb->ki_waitq = wait;
-
- io_get_req_task(req);
return true;
}
struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
struct kiocb *kiocb = &req->rw.kiocb;
struct iov_iter __iter, *iter = &__iter;
+ struct io_async_rw *rw = req->async_data;
ssize_t io_size, ret, ret2;
size_t iov_count;
bool no_async;
- if (req->io)
- iter = &req->io->rw.iter;
+ if (rw)
+ iter = &rw->iter;
ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
if (ret < 0)
/* Ensure we clear previously set non-block flag */
if (!force_nonblock)
kiocb->ki_flags &= ~IOCB_NOWAIT;
+ else
+ kiocb->ki_flags |= IOCB_NOWAIT;
+
/* If the file doesn't support async, just async punt */
no_async = force_nonblock && !io_file_supports_async(req->file, READ);
goto done;
/* some cases will consume bytes even on error returns */
iov_iter_revert(iter, iov_count - iov_iter_count(iter));
- ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
- if (ret)
- goto out_free;
- return -EAGAIN;
+ ret = 0;
+ goto copy_iov;
} else if (ret < 0) {
/* make sure -ERESTARTSYS -> -EINTR is done */
goto done;
}
if (no_async)
return -EAGAIN;
+ rw = req->async_data;
/* it's copied and will be cleaned with ->io */
iovec = NULL;
/* now use our persistent iterator, if we aren't already */
- iter = &req->io->rw.iter;
+ iter = &rw->iter;
retry:
- req->io->rw.bytes_done += ret;
+ rw->bytes_done += ret;
/* if we can retry, do so with the callbacks armed */
if (!io_rw_should_retry(req)) {
kiocb->ki_flags &= ~IOCB_WAITQ;
return ret;
}
-static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
- bool force_nonblock)
+static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
ssize_t ret;
- ret = io_prep_rw(req, sqe, force_nonblock);
+ ret = io_prep_rw(req, sqe);
if (ret)
return ret;
return -EBADF;
/* either don't need iovec imported or already have it */
- if (!req->io || req->flags & REQ_F_NEED_CLEANUP)
+ if (!req->async_data)
return 0;
- return io_rw_prep_async(req, WRITE, force_nonblock);
+ return io_rw_prep_async(req, WRITE);
}
static int io_write(struct io_kiocb *req, bool force_nonblock,
struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
struct kiocb *kiocb = &req->rw.kiocb;
struct iov_iter __iter, *iter = &__iter;
+ struct io_async_rw *rw = req->async_data;
size_t iov_count;
ssize_t ret, ret2, io_size;
- if (req->io)
- iter = &req->io->rw.iter;
+ if (rw)
+ iter = &rw->iter;
ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
if (ret < 0)
/* Ensure we clear previously set non-block flag */
if (!force_nonblock)
- req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
+ kiocb->ki_flags &= ~IOCB_NOWAIT;
+ else
+ kiocb->ki_flags |= IOCB_NOWAIT;
/* If the file doesn't support async, just async punt */
if (force_nonblock && !io_file_supports_async(req->file, WRITE))
{
struct io_splice* sp = &req->splice;
unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
- int ret;
- if (req->flags & REQ_F_NEED_CLEANUP)
- return 0;
if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
return -EINVAL;
if (unlikely(sp->flags & ~valid_flags))
return -EINVAL;
- ret = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in), &sp->file_in,
- (sp->flags & SPLICE_F_FD_IN_FIXED));
- if (ret)
- return ret;
+ sp->file_in = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in),
+ (sp->flags & SPLICE_F_FD_IN_FIXED));
+ if (!sp->file_in)
+ return -EBADF;
req->flags |= REQ_F_NEED_CLEANUP;
if (!S_ISREG(file_inode(sp->file_in)->i_mode)) {
if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
return -EINVAL;
- if (req->flags & REQ_F_NEED_CLEANUP)
- return 0;
mode = READ_ONCE(sqe->len);
flags = READ_ONCE(sqe->open_flags);
req->open.how = build_open_how(flags, mode);
if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
return -EINVAL;
- if (req->flags & REQ_F_NEED_CLEANUP)
- return 0;
how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
len = READ_ONCE(sqe->len);
if (len < OPEN_HOW_SIZE_VER0)
return -EBADF;
req->close.fd = READ_ONCE(sqe->fd);
- if ((req->file && req->file->f_op == &io_uring_fops) ||
- req->close.fd == req->ctx->ring_fd)
+ if ((req->file && req->file->f_op == &io_uring_fops))
return -EBADF;
req->close.put_file = NULL;
static int io_setup_async_msg(struct io_kiocb *req,
struct io_async_msghdr *kmsg)
{
- if (req->io)
+ struct io_async_msghdr *async_msg = req->async_data;
+
+ if (async_msg)
return -EAGAIN;
- if (io_alloc_async_ctx(req)) {
+ if (io_alloc_async_data(req)) {
if (kmsg->iov != kmsg->fast_iov)
kfree(kmsg->iov);
return -ENOMEM;
}
+ async_msg = req->async_data;
req->flags |= REQ_F_NEED_CLEANUP;
- memcpy(&req->io->msg, kmsg, sizeof(*kmsg));
+ memcpy(async_msg, kmsg, sizeof(*kmsg));
return -EAGAIN;
}
static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
+ struct io_async_msghdr *async_msg = req->async_data;
struct io_sr_msg *sr = &req->sr_msg;
- struct io_async_ctx *io = req->io;
int ret;
if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
sr->msg_flags |= MSG_CMSG_COMPAT;
#endif
- if (!io || req->opcode == IORING_OP_SEND)
- return 0;
- /* iovec is already imported */
- if (req->flags & REQ_F_NEED_CLEANUP)
+ if (!async_msg || !io_op_defs[req->opcode].needs_async_data)
return 0;
-
- ret = io_sendmsg_copy_hdr(req, &io->msg);
+ ret = io_sendmsg_copy_hdr(req, async_msg);
if (!ret)
req->flags |= REQ_F_NEED_CLEANUP;
return ret;
if (unlikely(!sock))
return ret;
- if (req->io) {
- kmsg = &req->io->msg;
- kmsg->msg.msg_name = &req->io->msg.addr;
+ if (req->async_data) {
+ kmsg = req->async_data;
+ kmsg->msg.msg_name = &kmsg->addr;
/* if iov is set, it's allocated already */
if (!kmsg->iov)
kmsg->iov = kmsg->fast_iov;
ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
if (unlikely(ret))
- return ret;;
+ return ret;
msg.msg_name = NULL;
msg.msg_control = NULL;
sr->len);
iomsg->iov = NULL;
} else {
- ret = import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
- &iomsg->iov, &iomsg->msg.msg_iter);
+ ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
+ &iomsg->iov, &iomsg->msg.msg_iter,
+ false);
if (ret > 0)
ret = 0;
}
sr->len = iomsg->iov[0].iov_len;
iomsg->iov = NULL;
} else {
- ret = compat_import_iovec(READ, uiov, len, UIO_FASTIOV,
- &iomsg->iov,
- &iomsg->msg.msg_iter);
+ ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
+ UIO_FASTIOV, &iomsg->iov,
+ &iomsg->msg.msg_iter, true);
if (ret < 0)
return ret;
}
static int io_recvmsg_prep(struct io_kiocb *req,
const struct io_uring_sqe *sqe)
{
+ struct io_async_msghdr *async_msg = req->async_data;
struct io_sr_msg *sr = &req->sr_msg;
- struct io_async_ctx *io = req->io;
int ret;
if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
sr->msg_flags |= MSG_CMSG_COMPAT;
#endif
- if (!io || req->opcode == IORING_OP_RECV)
+ if (!async_msg || !io_op_defs[req->opcode].needs_async_data)
return 0;
- /* iovec is already imported */
- if (req->flags & REQ_F_NEED_CLEANUP)
- return 0;
-
- ret = io_recvmsg_copy_hdr(req, &io->msg);
+ ret = io_recvmsg_copy_hdr(req, async_msg);
if (!ret)
req->flags |= REQ_F_NEED_CLEANUP;
return ret;
if (unlikely(!sock))
return ret;
- if (req->io) {
- kmsg = &req->io->msg;
- kmsg->msg.msg_name = &req->io->msg.addr;
+ if (req->async_data) {
+ kmsg = req->async_data;
+ kmsg->msg.msg_name = &kmsg->addr;
/* if iov is set, it's allocated already */
if (!kmsg->iov)
kmsg->iov = kmsg->fast_iov;
static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
struct io_connect *conn = &req->connect;
- struct io_async_ctx *io = req->io;
+ struct io_async_connect *io = req->async_data;
if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
return -EINVAL;
return 0;
return move_addr_to_kernel(conn->addr, conn->addr_len,
- &io->connect.address);
+ &io->address);
}
static int io_connect(struct io_kiocb *req, bool force_nonblock,
struct io_comp_state *cs)
{
- struct io_async_ctx __io, *io;
+ struct io_async_connect __io, *io;
unsigned file_flags;
int ret;
- if (req->io) {
- io = req->io;
+ if (req->async_data) {
+ io = req->async_data;
} else {
ret = move_addr_to_kernel(req->connect.addr,
req->connect.addr_len,
- &__io.connect.address);
+ &__io.address);
if (ret)
goto out;
io = &__io;
file_flags = force_nonblock ? O_NONBLOCK : 0;
- ret = __sys_connect_file(req->file, &io->connect.address,
+ ret = __sys_connect_file(req->file, &io->address,
req->connect.addr_len, file_flags);
if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
- if (req->io)
+ if (req->async_data)
return -EAGAIN;
- if (io_alloc_async_ctx(req)) {
+ if (io_alloc_async_data(req)) {
ret = -ENOMEM;
goto out;
}
- memcpy(&req->io->connect, &__io.connect, sizeof(__io.connect));
+ io = req->async_data;
+ memcpy(req->async_data, &__io, sizeof(__io));
return -EAGAIN;
}
if (ret == -ERESTARTSYS)
* of executing it. We can't safely execute it anyway, as we may not
* have the needed state needed for it anyway.
*/
- ret = io_req_task_work_add(req, &req->task_work, twa_signal_ok);
+ ret = io_req_task_work_add(req, twa_signal_ok);
if (unlikely(ret)) {
struct task_struct *tsk;
static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
{
- /* pure poll stashes this in ->io, poll driven retry elsewhere */
+ /* pure poll stashes this in ->async_data, poll driven retry elsewhere */
if (req->opcode == IORING_OP_POLL_ADD)
- return (struct io_poll_iocb *) req->io;
+ return req->async_data;
return req->apoll->double_poll;
}
if (mask && !(mask & poll->events))
return 0;
+ list_del_init(&wait->entry);
+
if (poll && poll->head) {
bool done;
apoll->double_poll = NULL;
req->flags |= REQ_F_POLLED;
- io_get_req_task(req);
req->apoll = apoll;
INIT_HLIST_NODE(&req->hash_node);
return do_complete;
}
-static void io_poll_remove_all(struct io_ring_ctx *ctx)
+/*
+ * Returns true if we found and killed one or more poll requests
+ */
+static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk)
{
struct hlist_node *tmp;
struct io_kiocb *req;
struct hlist_head *list;
list = &ctx->cancel_hash[i];
- hlist_for_each_entry_safe(req, tmp, list, hash_node)
- posted += io_poll_remove_one(req);
+ hlist_for_each_entry_safe(req, tmp, list, hash_node) {
+ if (io_task_match(req, tsk))
+ posted += io_poll_remove_one(req);
+ }
}
spin_unlock_irq(&ctx->completion_lock);
if (posted)
io_cqring_ev_posted(ctx);
+
+ return posted != 0;
}
static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
{
struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
- __io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->io);
+ __io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->async_data);
}
static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
#endif
poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP |
(events & EPOLLEXCLUSIVE);
-
- io_get_req_task(req);
return 0;
}
unsigned long flags;
spin_lock_irqsave(&ctx->completion_lock, flags);
+ list_del_init(&req->timeout.list);
atomic_set(&req->ctx->cq_timeouts,
atomic_read(&req->ctx->cq_timeouts) + 1);
- /*
- * We could be racing with timeout deletion. If the list is empty,
- * then timeout lookup already found it and will be handling it.
- */
- if (!list_empty(&req->timeout.list))
- list_del_init(&req->timeout.list);
-
io_cqring_fill_event(req, -ETIME);
io_commit_cqring(ctx);
spin_unlock_irqrestore(&ctx->completion_lock, flags);
static int __io_timeout_cancel(struct io_kiocb *req)
{
+ struct io_timeout_data *io = req->async_data;
int ret;
- list_del_init(&req->timeout.list);
-
- ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
+ ret = hrtimer_try_to_cancel(&io->timer);
if (ret == -1)
return -EALREADY;
+ list_del_init(&req->timeout.list);
req_set_fail_links(req);
req->flags |= REQ_F_COMP_LOCKED;
return -EINVAL;
if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
return -EINVAL;
- if (sqe->ioprio || sqe->buf_index || sqe->len)
- return -EINVAL;
-
- req->timeout.addr = READ_ONCE(sqe->addr);
- req->timeout.flags = READ_ONCE(sqe->timeout_flags);
- if (req->timeout.flags)
+ if (sqe->ioprio || sqe->buf_index || sqe->len || sqe->timeout_flags)
return -EINVAL;
+ req->timeout_rem.addr = READ_ONCE(sqe->addr);
return 0;
}
int ret;
spin_lock_irq(&ctx->completion_lock);
- ret = io_timeout_cancel(ctx, req->timeout.addr);
+ ret = io_timeout_cancel(ctx, req->timeout_rem.addr);
io_cqring_fill_event(req, ret);
io_commit_cqring(ctx);
req->timeout.off = off;
- if (!req->io && io_alloc_async_ctx(req))
+ if (!req->async_data && io_alloc_async_data(req))
return -ENOMEM;
- data = &req->io->timeout;
+ data = req->async_data;
data->req = req;
if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
static int io_timeout(struct io_kiocb *req)
{
struct io_ring_ctx *ctx = req->ctx;
- struct io_timeout_data *data = &req->io->timeout;
+ struct io_timeout_data *data = req->async_data;
struct list_head *entry;
u32 tail, off = req->timeout.off;
return 0;
}
-static int io_req_defer_prep(struct io_kiocb *req,
- const struct io_uring_sqe *sqe)
+static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
- ssize_t ret = 0;
-
- if (!sqe)
- return 0;
-
- if (io_alloc_async_ctx(req))
- return -EAGAIN;
- ret = io_prep_work_files(req);
- if (unlikely(ret))
- return ret;
-
- io_prep_async_work(req);
-
switch (req->opcode) {
case IORING_OP_NOP:
- break;
+ return 0;
case IORING_OP_READV:
case IORING_OP_READ_FIXED:
case IORING_OP_READ:
- ret = io_read_prep(req, sqe, true);
- break;
+ return io_read_prep(req, sqe);
case IORING_OP_WRITEV:
case IORING_OP_WRITE_FIXED:
case IORING_OP_WRITE:
- ret = io_write_prep(req, sqe, true);
- break;
+ return io_write_prep(req, sqe);
case IORING_OP_POLL_ADD:
- ret = io_poll_add_prep(req, sqe);
- break;
+ return io_poll_add_prep(req, sqe);
case IORING_OP_POLL_REMOVE:
- ret = io_poll_remove_prep(req, sqe);
- break;
+ return io_poll_remove_prep(req, sqe);
case IORING_OP_FSYNC:
- ret = io_prep_fsync(req, sqe);
- break;
+ return io_prep_fsync(req, sqe);
case IORING_OP_SYNC_FILE_RANGE:
- ret = io_prep_sfr(req, sqe);
- break;
+ return io_prep_sfr(req, sqe);
case IORING_OP_SENDMSG:
case IORING_OP_SEND:
- ret = io_sendmsg_prep(req, sqe);
- break;
+ return io_sendmsg_prep(req, sqe);
case IORING_OP_RECVMSG:
case IORING_OP_RECV:
- ret = io_recvmsg_prep(req, sqe);
- break;
+ return io_recvmsg_prep(req, sqe);
case IORING_OP_CONNECT:
- ret = io_connect_prep(req, sqe);
- break;
+ return io_connect_prep(req, sqe);
case IORING_OP_TIMEOUT:
- ret = io_timeout_prep(req, sqe, false);
- break;
+ return io_timeout_prep(req, sqe, false);
case IORING_OP_TIMEOUT_REMOVE:
- ret = io_timeout_remove_prep(req, sqe);
- break;
+ return io_timeout_remove_prep(req, sqe);
case IORING_OP_ASYNC_CANCEL:
- ret = io_async_cancel_prep(req, sqe);
- break;
+ return io_async_cancel_prep(req, sqe);
case IORING_OP_LINK_TIMEOUT:
- ret = io_timeout_prep(req, sqe, true);
- break;
+ return io_timeout_prep(req, sqe, true);
case IORING_OP_ACCEPT:
- ret = io_accept_prep(req, sqe);
- break;
+ return io_accept_prep(req, sqe);
case IORING_OP_FALLOCATE:
- ret = io_fallocate_prep(req, sqe);
- break;
+ return io_fallocate_prep(req, sqe);
case IORING_OP_OPENAT:
- ret = io_openat_prep(req, sqe);
- break;
+ return io_openat_prep(req, sqe);
case IORING_OP_CLOSE:
- ret = io_close_prep(req, sqe);
- break;
+ return io_close_prep(req, sqe);
case IORING_OP_FILES_UPDATE:
- ret = io_files_update_prep(req, sqe);
- break;
+ return io_files_update_prep(req, sqe);
case IORING_OP_STATX:
- ret = io_statx_prep(req, sqe);
- break;
+ return io_statx_prep(req, sqe);
case IORING_OP_FADVISE:
- ret = io_fadvise_prep(req, sqe);
- break;
+ return io_fadvise_prep(req, sqe);
case IORING_OP_MADVISE:
- ret = io_madvise_prep(req, sqe);
- break;
+ return io_madvise_prep(req, sqe);
case IORING_OP_OPENAT2:
- ret = io_openat2_prep(req, sqe);
- break;
+ return io_openat2_prep(req, sqe);
case IORING_OP_EPOLL_CTL:
- ret = io_epoll_ctl_prep(req, sqe);
- break;
+ return io_epoll_ctl_prep(req, sqe);
case IORING_OP_SPLICE:
- ret = io_splice_prep(req, sqe);
- break;
+ return io_splice_prep(req, sqe);
case IORING_OP_PROVIDE_BUFFERS:
- ret = io_provide_buffers_prep(req, sqe);
- break;
+ return io_provide_buffers_prep(req, sqe);
case IORING_OP_REMOVE_BUFFERS:
- ret = io_remove_buffers_prep(req, sqe);
- break;
+ return io_remove_buffers_prep(req, sqe);
case IORING_OP_TEE:
- ret = io_tee_prep(req, sqe);
- break;
- default:
- printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
- req->opcode);
- ret = -EINVAL;
- break;
+ return io_tee_prep(req, sqe);
}
- return ret;
+ printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
+ req->opcode);
+ return-EINVAL;
+}
+
+static int io_req_defer_prep(struct io_kiocb *req,
+ const struct io_uring_sqe *sqe)
+{
+ if (!sqe)
+ return 0;
+ if (io_alloc_async_data(req))
+ return -EAGAIN;
+ return io_req_prep(req, sqe);
}
static u32 io_get_sequence(struct io_kiocb *req)
if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
return 0;
- if (!req->io) {
+ if (!req->async_data) {
ret = io_req_defer_prep(req, sqe);
if (ret)
return ret;
return -EIOCBQUEUED;
}
-static void __io_clean_op(struct io_kiocb *req)
+static void io_req_drop_files(struct io_kiocb *req)
{
- struct io_async_ctx *io = req->io;
+ struct io_ring_ctx *ctx = req->ctx;
+ unsigned long flags;
+
+ spin_lock_irqsave(&ctx->inflight_lock, flags);
+ list_del(&req->inflight_entry);
+ if (waitqueue_active(&ctx->inflight_wait))
+ wake_up(&ctx->inflight_wait);
+ spin_unlock_irqrestore(&ctx->inflight_lock, flags);
+ req->flags &= ~REQ_F_INFLIGHT;
+ put_files_struct(req->work.files);
+ put_nsproxy(req->work.nsproxy);
+ req->work.files = NULL;
+}
+static void __io_clean_op(struct io_kiocb *req)
+{
if (req->flags & REQ_F_BUFFER_SELECTED) {
switch (req->opcode) {
case IORING_OP_READV:
case IORING_OP_READ:
case IORING_OP_WRITEV:
case IORING_OP_WRITE_FIXED:
- case IORING_OP_WRITE:
- if (io->rw.free_iovec)
- kfree(io->rw.free_iovec);
+ case IORING_OP_WRITE: {
+ struct io_async_rw *io = req->async_data;
+ if (io->free_iovec)
+ kfree(io->free_iovec);
break;
+ }
case IORING_OP_RECVMSG:
- case IORING_OP_SENDMSG:
- if (io->msg.iov != io->msg.fast_iov)
- kfree(io->msg.iov);
+ case IORING_OP_SENDMSG: {
+ struct io_async_msghdr *io = req->async_data;
+ if (io->iov != io->fast_iov)
+ kfree(io->iov);
break;
+ }
case IORING_OP_SPLICE:
case IORING_OP_TEE:
io_put_file(req, req->splice.file_in,
(req->splice.flags & SPLICE_F_FD_IN_FIXED));
break;
+ case IORING_OP_OPENAT:
+ case IORING_OP_OPENAT2:
+ if (req->open.filename)
+ putname(req->open.filename);
+ break;
}
req->flags &= ~REQ_F_NEED_CLEANUP;
}
- if (req->flags & REQ_F_INFLIGHT) {
- struct io_ring_ctx *ctx = req->ctx;
- unsigned long flags;
-
- spin_lock_irqsave(&ctx->inflight_lock, flags);
- list_del(&req->inflight_entry);
- if (waitqueue_active(&ctx->inflight_wait))
- wake_up(&ctx->inflight_wait);
- spin_unlock_irqrestore(&ctx->inflight_lock, flags);
- req->flags &= ~REQ_F_INFLIGHT;
- }
+ if (req->flags & REQ_F_INFLIGHT)
+ io_req_drop_files(req);
}
-static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
- bool force_nonblock, struct io_comp_state *cs)
+static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock,
+ struct io_comp_state *cs)
{
struct io_ring_ctx *ctx = req->ctx;
int ret;
case IORING_OP_READV:
case IORING_OP_READ_FIXED:
case IORING_OP_READ:
- if (sqe) {
- ret = io_read_prep(req, sqe, force_nonblock);
- if (ret < 0)
- break;
- }
ret = io_read(req, force_nonblock, cs);
break;
case IORING_OP_WRITEV:
case IORING_OP_WRITE_FIXED:
case IORING_OP_WRITE:
- if (sqe) {
- ret = io_write_prep(req, sqe, force_nonblock);
- if (ret < 0)
- break;
- }
ret = io_write(req, force_nonblock, cs);
break;
case IORING_OP_FSYNC:
- if (sqe) {
- ret = io_prep_fsync(req, sqe);
- if (ret < 0)
- break;
- }
ret = io_fsync(req, force_nonblock);
break;
case IORING_OP_POLL_ADD:
- if (sqe) {
- ret = io_poll_add_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_poll_add(req);
break;
case IORING_OP_POLL_REMOVE:
- if (sqe) {
- ret = io_poll_remove_prep(req, sqe);
- if (ret < 0)
- break;
- }
ret = io_poll_remove(req);
break;
case IORING_OP_SYNC_FILE_RANGE:
- if (sqe) {
- ret = io_prep_sfr(req, sqe);
- if (ret < 0)
- break;
- }
ret = io_sync_file_range(req, force_nonblock);
break;
case IORING_OP_SENDMSG:
+ ret = io_sendmsg(req, force_nonblock, cs);
+ break;
case IORING_OP_SEND:
- if (sqe) {
- ret = io_sendmsg_prep(req, sqe);
- if (ret < 0)
- break;
- }
- if (req->opcode == IORING_OP_SENDMSG)
- ret = io_sendmsg(req, force_nonblock, cs);
- else
- ret = io_send(req, force_nonblock, cs);
+ ret = io_send(req, force_nonblock, cs);
break;
case IORING_OP_RECVMSG:
+ ret = io_recvmsg(req, force_nonblock, cs);
+ break;
case IORING_OP_RECV:
- if (sqe) {
- ret = io_recvmsg_prep(req, sqe);
- if (ret)
- break;
- }
- if (req->opcode == IORING_OP_RECVMSG)
- ret = io_recvmsg(req, force_nonblock, cs);
- else
- ret = io_recv(req, force_nonblock, cs);
+ ret = io_recv(req, force_nonblock, cs);
break;
case IORING_OP_TIMEOUT:
- if (sqe) {
- ret = io_timeout_prep(req, sqe, false);
- if (ret)
- break;
- }
ret = io_timeout(req);
break;
case IORING_OP_TIMEOUT_REMOVE:
- if (sqe) {
- ret = io_timeout_remove_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_timeout_remove(req);
break;
case IORING_OP_ACCEPT:
- if (sqe) {
- ret = io_accept_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_accept(req, force_nonblock, cs);
break;
case IORING_OP_CONNECT:
- if (sqe) {
- ret = io_connect_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_connect(req, force_nonblock, cs);
break;
case IORING_OP_ASYNC_CANCEL:
- if (sqe) {
- ret = io_async_cancel_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_async_cancel(req);
break;
case IORING_OP_FALLOCATE:
- if (sqe) {
- ret = io_fallocate_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_fallocate(req, force_nonblock);
break;
case IORING_OP_OPENAT:
- if (sqe) {
- ret = io_openat_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_openat(req, force_nonblock);
break;
case IORING_OP_CLOSE:
- if (sqe) {
- ret = io_close_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_close(req, force_nonblock, cs);
break;
case IORING_OP_FILES_UPDATE:
- if (sqe) {
- ret = io_files_update_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_files_update(req, force_nonblock, cs);
break;
case IORING_OP_STATX:
- if (sqe) {
- ret = io_statx_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_statx(req, force_nonblock);
break;
case IORING_OP_FADVISE:
- if (sqe) {
- ret = io_fadvise_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_fadvise(req, force_nonblock);
break;
case IORING_OP_MADVISE:
- if (sqe) {
- ret = io_madvise_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_madvise(req, force_nonblock);
break;
case IORING_OP_OPENAT2:
- if (sqe) {
- ret = io_openat2_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_openat2(req, force_nonblock);
break;
case IORING_OP_EPOLL_CTL:
- if (sqe) {
- ret = io_epoll_ctl_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_epoll_ctl(req, force_nonblock, cs);
break;
case IORING_OP_SPLICE:
- if (sqe) {
- ret = io_splice_prep(req, sqe);
- if (ret < 0)
- break;
- }
ret = io_splice(req, force_nonblock);
break;
case IORING_OP_PROVIDE_BUFFERS:
- if (sqe) {
- ret = io_provide_buffers_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_provide_buffers(req, force_nonblock, cs);
break;
case IORING_OP_REMOVE_BUFFERS:
- if (sqe) {
- ret = io_remove_buffers_prep(req, sqe);
- if (ret)
- break;
- }
ret = io_remove_buffers(req, force_nonblock, cs);
break;
case IORING_OP_TEE:
- if (sqe) {
- ret = io_tee_prep(req, sqe);
- if (ret < 0)
- break;
- }
ret = io_tee(req, force_nonblock);
break;
default:
if (!ret) {
do {
- ret = io_issue_sqe(req, NULL, false, NULL);
+ ret = io_issue_sqe(req, false, NULL);
/*
* We can get EAGAIN for polled IO even though we're
* forcing a sync submission from here, since we can't
return table->files[index & IORING_FILE_TABLE_MASK];
}
-static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
- int fd, struct file **out_file, bool fixed)
+static struct file *io_file_get(struct io_submit_state *state,
+ struct io_kiocb *req, int fd, bool fixed)
{
struct io_ring_ctx *ctx = req->ctx;
struct file *file;
if (fixed) {
- if (unlikely(!ctx->file_data ||
- (unsigned) fd >= ctx->nr_user_files))
- return -EBADF;
+ if (unlikely((unsigned int)fd >= ctx->nr_user_files))
+ return NULL;
fd = array_index_nospec(fd, ctx->nr_user_files);
file = io_file_from_index(ctx, fd);
if (file) {
- req->fixed_file_refs = ctx->file_data->cur_refs;
+ req->fixed_file_refs = &ctx->file_data->node->refs;
percpu_ref_get(req->fixed_file_refs);
}
} else {
file = __io_file_get(state, fd);
}
- if (file || io_op_defs[req->opcode].needs_file_no_error) {
- *out_file = file;
- return 0;
- }
- return -EBADF;
+ return file;
}
static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,
if (unlikely(!fixed && io_async_submit(req->ctx)))
return -EBADF;
- return io_file_get(state, req, fd, &req->file, fixed);
-}
-
-static int io_grab_files(struct io_kiocb *req)
-{
- int ret = -EBADF;
- struct io_ring_ctx *ctx = req->ctx;
-
- io_req_init_async(req);
-
- if (req->work.files || (req->flags & REQ_F_NO_FILE_TABLE))
- return 0;
- if (!ctx->ring_file)
- return -EBADF;
-
- rcu_read_lock();
- spin_lock_irq(&ctx->inflight_lock);
- /*
- * We use the f_ops->flush() handler to ensure that we can flush
- * out work accessing these files if the fd is closed. Check if
- * the fd has changed since we started down this path, and disallow
- * this operation if it has.
- */
- if (fcheck(ctx->ring_fd) == ctx->ring_file) {
- list_add(&req->inflight_entry, &ctx->inflight_list);
- req->flags |= REQ_F_INFLIGHT;
- req->work.files = current->files;
- ret = 0;
- }
- spin_unlock_irq(&ctx->inflight_lock);
- rcu_read_unlock();
-
- return ret;
-}
-
-static inline int io_prep_work_files(struct io_kiocb *req)
-{
- if (!io_op_defs[req->opcode].file_table)
+ req->file = io_file_get(state, req, fd, fixed);
+ if (req->file || io_op_defs[req->opcode].needs_file_no_error)
return 0;
- return io_grab_files(req);
+ return -EBADF;
}
static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
* we got a chance to setup the timer
*/
if (!list_empty(&req->link_list)) {
- struct io_timeout_data *data = &req->io->timeout;
+ struct io_timeout_data *data = req->async_data;
data->timer.function = io_link_timeout_fn;
hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
return nxt;
}
-static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
- struct io_comp_state *cs)
+static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs)
{
struct io_kiocb *linked_timeout;
struct io_kiocb *nxt;
old_creds = override_creds(req->work.creds);
}
- ret = io_issue_sqe(req, sqe, true, cs);
+ ret = io_issue_sqe(req, true, cs);
/*
* We async punt it if the file wasn't marked NOWAIT, or if the file
if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
if (!io_arm_poll_handler(req)) {
punt:
- ret = io_prep_work_files(req);
- if (unlikely(ret))
- goto err;
/*
* Queued up for async execution, worker will release
* submit reference when the iocb is actually submitted.
}
if (unlikely(ret)) {
-err:
/* un-prep timeout, so it'll be killed as any other linked */
req->flags &= ~REQ_F_LINK_TIMEOUT;
req_set_fail_links(req);
io_req_complete(req, ret);
}
} else if (req->flags & REQ_F_FORCE_ASYNC) {
- if (!req->io) {
+ if (!req->async_data) {
ret = io_req_defer_prep(req, sqe);
if (unlikely(ret))
goto fail_req;
req->work.flags |= IO_WQ_WORK_CONCURRENT;
io_queue_async_work(req);
} else {
- __io_queue_sqe(req, sqe, cs);
+ if (sqe) {
+ ret = io_req_prep(req, sqe);
+ if (unlikely(ret))
+ goto fail_req;
+ }
+ __io_queue_sqe(req, cs);
}
}
return ret;
}
trace_io_uring_link(ctx, req, head);
- io_get_req_task(req);
list_add_tail(&req->link_list, &head->link_list);
/* last request of a link, enqueue the link */
struct io_ring_ctx *ctx, unsigned int max_ios)
{
blk_start_plug(&state->plug);
-#ifdef CONFIG_BLOCK
- state->plug.nowait = true;
-#endif
state->comp.nr = 0;
INIT_LIST_HEAD(&state->comp.list);
state->comp.ctx = ctx;
ctx->cached_sq_head++;
}
+/*
+ * Check SQE restrictions (opcode and flags).
+ *
+ * Returns 'true' if SQE is allowed, 'false' otherwise.
+ */
+static inline bool io_check_restriction(struct io_ring_ctx *ctx,
+ struct io_kiocb *req,
+ unsigned int sqe_flags)
+{
+ if (!ctx->restricted)
+ return true;
+
+ if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
+ return false;
+
+ if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
+ ctx->restrictions.sqe_flags_required)
+ return false;
+
+ if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
+ ctx->restrictions.sqe_flags_required))
+ return false;
+
+ return true;
+}
+
#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
IOSQE_BUFFER_SELECT)
struct io_submit_state *state)
{
unsigned int sqe_flags;
- int id;
+ int id, ret;
req->opcode = READ_ONCE(sqe->opcode);
req->user_data = READ_ONCE(sqe->user_data);
- req->io = NULL;
+ req->async_data = NULL;
req->file = NULL;
req->ctx = ctx;
req->flags = 0;
if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
return -EINVAL;
+ if (unlikely(!io_check_restriction(ctx, req, sqe_flags)))
+ return -EACCES;
+
if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
!io_op_defs[req->opcode].buffer_select)
return -EOPNOTSUPP;
if (!io_op_defs[req->opcode].needs_file)
return 0;
- return io_req_set_file(state, req, READ_ONCE(sqe->fd));
+ ret = io_req_set_file(state, req, READ_ONCE(sqe->fd));
+ state->ios_left--;
+ return ret;
}
-static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
- struct file *ring_file, int ring_fd)
+static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
{
struct io_submit_state state;
struct io_kiocb *link = NULL;
/* if we have a backlog and couldn't flush it all, return BUSY */
if (test_bit(0, &ctx->sq_check_overflow)) {
if (!list_empty(&ctx->cq_overflow_list) &&
- !io_cqring_overflow_flush(ctx, false))
+ !io_cqring_overflow_flush(ctx, false, NULL, NULL))
return -EBUSY;
}
if (!percpu_ref_tryget_many(&ctx->refs, nr))
return -EAGAIN;
- io_submit_state_start(&state, ctx, nr);
+ atomic_long_add(nr, ¤t->io_uring->req_issue);
+ refcount_add(nr, ¤t->usage);
- ctx->ring_fd = ring_fd;
- ctx->ring_file = ring_file;
+ io_submit_state_start(&state, ctx, nr);
for (i = 0; i < nr; i++) {
const struct io_uring_sqe *sqe;
submitted = -EAGAIN;
break;
}
-
- err = io_init_req(ctx, req, sqe, &state);
io_consume_sqe(ctx);
/* will complete beyond this point, count as submitted */
submitted++;
+ err = io_init_req(ctx, req, sqe, &state);
if (unlikely(err)) {
fail_req:
io_put_req(req);
int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
percpu_ref_put_many(&ctx->refs, nr - ref_used);
+ atomic_long_sub(nr - ref_used, ¤t->io_uring->req_issue);
+ put_task_struct_many(current, nr - ref_used);
}
if (link)
io_queue_link_head(link, &state.comp);
spin_unlock_irq(&ctx->completion_lock);
}
-static int io_sq_thread(void *data)
+static int io_sq_wake_function(struct wait_queue_entry *wqe, unsigned mode,
+ int sync, void *key)
{
- struct io_ring_ctx *ctx = data;
- const struct cred *old_cred;
- DEFINE_WAIT(wait);
- unsigned long timeout;
+ struct io_ring_ctx *ctx = container_of(wqe, struct io_ring_ctx, sqo_wait_entry);
+ int ret;
+
+ ret = autoremove_wake_function(wqe, mode, sync, key);
+ if (ret) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&ctx->completion_lock, flags);
+ ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
+ spin_unlock_irqrestore(&ctx->completion_lock, flags);
+ }
+ return ret;
+}
+
+enum sq_ret {
+ SQT_IDLE = 1,
+ SQT_SPIN = 2,
+ SQT_DID_WORK = 4,
+};
+
+static enum sq_ret __io_sq_thread(struct io_ring_ctx *ctx,
+ unsigned long start_jiffies, bool cap_entries)
+{
+ unsigned long timeout = start_jiffies + ctx->sq_thread_idle;
+ struct io_sq_data *sqd = ctx->sq_data;
+ unsigned int to_submit;
int ret = 0;
- complete(&ctx->sq_thread_comp);
+again:
+ if (!list_empty(&ctx->iopoll_list)) {
+ unsigned nr_events = 0;
- old_cred = override_creds(ctx->creds);
+ mutex_lock(&ctx->uring_lock);
+ if (!list_empty(&ctx->iopoll_list) && !need_resched())
+ io_do_iopoll(ctx, &nr_events, 0);
+ mutex_unlock(&ctx->uring_lock);
+ }
- timeout = jiffies + ctx->sq_thread_idle;
- while (!kthread_should_park()) {
- unsigned int to_submit;
+ to_submit = io_sqring_entries(ctx);
- if (!list_empty(&ctx->iopoll_list)) {
- unsigned nr_events = 0;
+ /*
+ * If submit got -EBUSY, flag us as needing the application
+ * to enter the kernel to reap and flush events.
+ */
+ if (!to_submit || ret == -EBUSY || need_resched()) {
+ /*
+ * Drop cur_mm before scheduling, we can't hold it for
+ * long periods (or over schedule()). Do this before
+ * adding ourselves to the waitqueue, as the unuse/drop
+ * may sleep.
+ */
+ io_sq_thread_drop_mm();
- mutex_lock(&ctx->uring_lock);
- if (!list_empty(&ctx->iopoll_list) && !need_resched())
- io_do_iopoll(ctx, &nr_events, 0);
- else
- timeout = jiffies + ctx->sq_thread_idle;
- mutex_unlock(&ctx->uring_lock);
+ /*
+ * We're polling. If we're within the defined idle
+ * period, then let us spin without work before going
+ * to sleep. The exception is if we got EBUSY doing
+ * more IO, we should wait for the application to
+ * reap events and wake us up.
+ */
+ if (!list_empty(&ctx->iopoll_list) || need_resched() ||
+ (!time_after(jiffies, timeout) && ret != -EBUSY &&
+ !percpu_ref_is_dying(&ctx->refs)))
+ return SQT_SPIN;
+
+ prepare_to_wait(&sqd->wait, &ctx->sqo_wait_entry,
+ TASK_INTERRUPTIBLE);
+
+ /*
+ * While doing polled IO, before going to sleep, we need
+ * to check if there are new reqs added to iopoll_list,
+ * it is because reqs may have been punted to io worker
+ * and will be added to iopoll_list later, hence check
+ * the iopoll_list again.
+ */
+ if ((ctx->flags & IORING_SETUP_IOPOLL) &&
+ !list_empty_careful(&ctx->iopoll_list)) {
+ finish_wait(&sqd->wait, &ctx->sqo_wait_entry);
+ goto again;
}
to_submit = io_sqring_entries(ctx);
+ if (!to_submit || ret == -EBUSY)
+ return SQT_IDLE;
+ }
+
+ finish_wait(&sqd->wait, &ctx->sqo_wait_entry);
+ io_ring_clear_wakeup_flag(ctx);
+
+ /* if we're handling multiple rings, cap submit size for fairness */
+ if (cap_entries && to_submit > 8)
+ to_submit = 8;
+
+ mutex_lock(&ctx->uring_lock);
+ if (likely(!percpu_ref_is_dying(&ctx->refs)))
+ ret = io_submit_sqes(ctx, to_submit);
+ mutex_unlock(&ctx->uring_lock);
+
+ if (!io_sqring_full(ctx) && wq_has_sleeper(&ctx->sqo_sq_wait))
+ wake_up(&ctx->sqo_sq_wait);
+
+ return SQT_DID_WORK;
+}
+
+static void io_sqd_init_new(struct io_sq_data *sqd)
+{
+ struct io_ring_ctx *ctx;
+
+ while (!list_empty(&sqd->ctx_new_list)) {
+ ctx = list_first_entry(&sqd->ctx_new_list, struct io_ring_ctx, sqd_list);
+ init_wait(&ctx->sqo_wait_entry);
+ ctx->sqo_wait_entry.func = io_sq_wake_function;
+ list_move_tail(&ctx->sqd_list, &sqd->ctx_list);
+ complete(&ctx->sq_thread_comp);
+ }
+}
+
+static int io_sq_thread(void *data)
+{
+ struct cgroup_subsys_state *cur_css = NULL;
+ const struct cred *old_cred = NULL;
+ struct io_sq_data *sqd = data;
+ struct io_ring_ctx *ctx;
+ unsigned long start_jiffies;
+
+ start_jiffies = jiffies;
+ while (!kthread_should_stop()) {
+ enum sq_ret ret = 0;
+ bool cap_entries;
/*
- * If submit got -EBUSY, flag us as needing the application
- * to enter the kernel to reap and flush events.
+ * Any changes to the sqd lists are synchronized through the
+ * kthread parking. This synchronizes the thread vs users,
+ * the users are synchronized on the sqd->ctx_lock.
*/
- if (!to_submit || ret == -EBUSY || need_resched()) {
- /*
- * Drop cur_mm before scheduling, we can't hold it for
- * long periods (or over schedule()). Do this before
- * adding ourselves to the waitqueue, as the unuse/drop
- * may sleep.
- */
- io_sq_thread_drop_mm();
+ if (kthread_should_park())
+ kthread_parkme();
- /*
- * We're polling. If we're within the defined idle
- * period, then let us spin without work before going
- * to sleep. The exception is if we got EBUSY doing
- * more IO, we should wait for the application to
- * reap events and wake us up.
- */
- if (!list_empty(&ctx->iopoll_list) || need_resched() ||
- (!time_after(jiffies, timeout) && ret != -EBUSY &&
- !percpu_ref_is_dying(&ctx->refs))) {
- io_run_task_work();
- cond_resched();
- continue;
- }
+ if (unlikely(!list_empty(&sqd->ctx_new_list)))
+ io_sqd_init_new(sqd);
- prepare_to_wait(&ctx->sqo_wait, &wait,
- TASK_INTERRUPTIBLE);
+ cap_entries = !list_is_singular(&sqd->ctx_list);
- /*
- * While doing polled IO, before going to sleep, we need
- * to check if there are new reqs added to iopoll_list,
- * it is because reqs may have been punted to io worker
- * and will be added to iopoll_list later, hence check
- * the iopoll_list again.
- */
- if ((ctx->flags & IORING_SETUP_IOPOLL) &&
- !list_empty_careful(&ctx->iopoll_list)) {
- finish_wait(&ctx->sqo_wait, &wait);
- continue;
+ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
+ if (current->cred != ctx->creds) {
+ if (old_cred)
+ revert_creds(old_cred);
+ old_cred = override_creds(ctx->creds);
}
+ io_sq_thread_associate_blkcg(ctx, &cur_css);
- io_ring_set_wakeup_flag(ctx);
+ ret |= __io_sq_thread(ctx, start_jiffies, cap_entries);
- to_submit = io_sqring_entries(ctx);
- if (!to_submit || ret == -EBUSY) {
- if (kthread_should_park()) {
- finish_wait(&ctx->sqo_wait, &wait);
- break;
- }
- if (io_run_task_work()) {
- finish_wait(&ctx->sqo_wait, &wait);
- io_ring_clear_wakeup_flag(ctx);
- continue;
- }
- if (signal_pending(current))
- flush_signals(current);
- schedule();
- finish_wait(&ctx->sqo_wait, &wait);
+ io_sq_thread_drop_mm();
+ }
- io_ring_clear_wakeup_flag(ctx);
- ret = 0;
+ if (ret & SQT_SPIN) {
+ io_run_task_work();
+ cond_resched();
+ } else if (ret == SQT_IDLE) {
+ if (kthread_should_park())
continue;
- }
- finish_wait(&ctx->sqo_wait, &wait);
-
- io_ring_clear_wakeup_flag(ctx);
+ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
+ io_ring_set_wakeup_flag(ctx);
+ schedule();
+ start_jiffies = jiffies;
+ list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
+ io_ring_clear_wakeup_flag(ctx);
}
-
- mutex_lock(&ctx->uring_lock);
- if (likely(!percpu_ref_is_dying(&ctx->refs)))
- ret = io_submit_sqes(ctx, to_submit, NULL, -1);
- mutex_unlock(&ctx->uring_lock);
- timeout = jiffies + ctx->sq_thread_idle;
}
io_run_task_work();
- io_sq_thread_drop_mm();
- revert_creds(old_cred);
+ if (cur_css)
+ io_sq_thread_unassociate_blkcg();
+ if (old_cred)
+ revert_creds(old_cred);
kthread_parkme();
return autoremove_wake_function(curr, mode, wake_flags, key);
}
+static int io_run_task_work_sig(void)
+{
+ if (io_run_task_work())
+ return 1;
+ if (!signal_pending(current))
+ return 0;
+ if (current->jobctl & JOBCTL_TASK_WORK) {
+ spin_lock_irq(¤t->sighand->siglock);
+ current->jobctl &= ~JOBCTL_TASK_WORK;
+ recalc_sigpending();
+ spin_unlock_irq(¤t->sighand->siglock);
+ return 1;
+ }
+ return -EINTR;
+}
+
/*
* Wait until events become available, if we don't already have some. The
* application must reap them itself, as they reside on the shared cq ring.
prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
TASK_INTERRUPTIBLE);
/* make sure we run task_work before checking for signals */
- if (io_run_task_work())
+ ret = io_run_task_work_sig();
+ if (ret > 0)
continue;
- if (signal_pending(current)) {
- if (current->jobctl & JOBCTL_TASK_WORK) {
- spin_lock_irq(¤t->sighand->siglock);
- current->jobctl &= ~JOBCTL_TASK_WORK;
- recalc_sigpending();
- spin_unlock_irq(¤t->sighand->siglock);
- continue;
- }
- ret = -EINTR;
+ else if (ret < 0)
break;
- }
if (io_should_wake(&iowq, false))
break;
schedule();
return 0;
}
-static void io_sq_thread_stop(struct io_ring_ctx *ctx)
+static void io_put_sq_data(struct io_sq_data *sqd)
{
- if (ctx->sqo_thread) {
- wait_for_completion(&ctx->sq_thread_comp);
+ if (refcount_dec_and_test(&sqd->refs)) {
/*
* The park is a bit of a work-around, without it we get
* warning spews on shutdown with SQPOLL set and affinity
* set to a single CPU.
*/
- kthread_park(ctx->sqo_thread);
- kthread_stop(ctx->sqo_thread);
- ctx->sqo_thread = NULL;
+ if (sqd->thread) {
+ kthread_park(sqd->thread);
+ kthread_stop(sqd->thread);
+ }
+
+ kfree(sqd);
+ }
+}
+
+static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
+{
+ struct io_ring_ctx *ctx_attach;
+ struct io_sq_data *sqd;
+ struct fd f;
+
+ f = fdget(p->wq_fd);
+ if (!f.file)
+ return ERR_PTR(-ENXIO);
+ if (f.file->f_op != &io_uring_fops) {
+ fdput(f);
+ return ERR_PTR(-EINVAL);
+ }
+
+ ctx_attach = f.file->private_data;
+ sqd = ctx_attach->sq_data;
+ if (!sqd) {
+ fdput(f);
+ return ERR_PTR(-EINVAL);
+ }
+
+ refcount_inc(&sqd->refs);
+ fdput(f);
+ return sqd;
+}
+
+static struct io_sq_data *io_get_sq_data(struct io_uring_params *p)
+{
+ struct io_sq_data *sqd;
+
+ if (p->flags & IORING_SETUP_ATTACH_WQ)
+ return io_attach_sq_data(p);
+
+ sqd = kzalloc(sizeof(*sqd), GFP_KERNEL);
+ if (!sqd)
+ return ERR_PTR(-ENOMEM);
+
+ refcount_set(&sqd->refs, 1);
+ INIT_LIST_HEAD(&sqd->ctx_list);
+ INIT_LIST_HEAD(&sqd->ctx_new_list);
+ mutex_init(&sqd->ctx_lock);
+ mutex_init(&sqd->lock);
+ init_waitqueue_head(&sqd->wait);
+ return sqd;
+}
+
+static void io_sq_thread_unpark(struct io_sq_data *sqd)
+ __releases(&sqd->lock)
+{
+ if (!sqd->thread)
+ return;
+ kthread_unpark(sqd->thread);
+ mutex_unlock(&sqd->lock);
+}
+
+static void io_sq_thread_park(struct io_sq_data *sqd)
+ __acquires(&sqd->lock)
+{
+ if (!sqd->thread)
+ return;
+ mutex_lock(&sqd->lock);
+ kthread_park(sqd->thread);
+}
+
+static void io_sq_thread_stop(struct io_ring_ctx *ctx)
+{
+ struct io_sq_data *sqd = ctx->sq_data;
+
+ if (sqd) {
+ if (sqd->thread) {
+ /*
+ * We may arrive here from the error branch in
+ * io_sq_offload_create() where the kthread is created
+ * without being waked up, thus wake it up now to make
+ * sure the wait will complete.
+ */
+ wake_up_process(sqd->thread);
+ wait_for_completion(&ctx->sq_thread_comp);
+
+ io_sq_thread_park(sqd);
+ }
+
+ mutex_lock(&sqd->ctx_lock);
+ list_del(&ctx->sqd_list);
+ mutex_unlock(&sqd->ctx_lock);
+
+ if (sqd->thread) {
+ finish_wait(&sqd->wait, &ctx->sqo_wait_entry);
+ io_sq_thread_unpark(sqd);
+ }
+
+ io_put_sq_data(sqd);
+ ctx->sq_data = NULL;
}
}
}
#endif
-static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
- unsigned nr_files)
+static int io_sqe_alloc_file_tables(struct fixed_file_data *file_data,
+ unsigned nr_tables, unsigned nr_files)
{
int i;
for (i = 0; i < nr_tables; i++) {
- struct fixed_file_table *table = &ctx->file_data->table[i];
+ struct fixed_file_table *table = &file_data->table[i];
unsigned this_files;
this_files = min(nr_files, IORING_MAX_FILES_TABLE);
return 0;
for (i = 0; i < nr_tables; i++) {
- struct fixed_file_table *table = &ctx->file_data->table[i];
+ struct fixed_file_table *table = &file_data->table[i];
kfree(table->files);
}
return 1;
unsigned nr_args)
{
__s32 __user *fds = (__s32 __user *) arg;
- unsigned nr_tables;
+ unsigned nr_tables, i;
struct file *file;
- int fd, ret = 0;
- unsigned i;
+ int fd, ret = -ENOMEM;
struct fixed_file_ref_node *ref_node;
+ struct fixed_file_data *file_data;
if (ctx->file_data)
return -EBUSY;
if (nr_args > IORING_MAX_FIXED_FILES)
return -EMFILE;
- ctx->file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL);
- if (!ctx->file_data)
+ file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL);
+ if (!file_data)
return -ENOMEM;
- ctx->file_data->ctx = ctx;
- init_completion(&ctx->file_data->done);
- INIT_LIST_HEAD(&ctx->file_data->ref_list);
- spin_lock_init(&ctx->file_data->lock);
+ file_data->ctx = ctx;
+ init_completion(&file_data->done);
+ INIT_LIST_HEAD(&file_data->ref_list);
+ spin_lock_init(&file_data->lock);
nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
- ctx->file_data->table = kcalloc(nr_tables,
- sizeof(struct fixed_file_table),
- GFP_KERNEL);
- if (!ctx->file_data->table) {
- kfree(ctx->file_data);
- ctx->file_data = NULL;
- return -ENOMEM;
- }
+ file_data->table = kcalloc(nr_tables, sizeof(file_data->table),
+ GFP_KERNEL);
+ if (!file_data->table)
+ goto out_free;
- if (percpu_ref_init(&ctx->file_data->refs, io_file_ref_kill,
- PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
- kfree(ctx->file_data->table);
- kfree(ctx->file_data);
- ctx->file_data = NULL;
- return -ENOMEM;
- }
+ if (percpu_ref_init(&file_data->refs, io_file_ref_kill,
+ PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
+ goto out_free;
- if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
- percpu_ref_exit(&ctx->file_data->refs);
- kfree(ctx->file_data->table);
- kfree(ctx->file_data);
- ctx->file_data = NULL;
- return -ENOMEM;
- }
+ if (io_sqe_alloc_file_tables(file_data, nr_tables, nr_args))
+ goto out_ref;
for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
struct fixed_file_table *table;
unsigned index;
- ret = -EFAULT;
- if (copy_from_user(&fd, &fds[i], sizeof(fd)))
- break;
+ if (copy_from_user(&fd, &fds[i], sizeof(fd))) {
+ ret = -EFAULT;
+ goto out_fput;
+ }
/* allow sparse sets */
- if (fd == -1) {
- ret = 0;
+ if (fd == -1)
continue;
- }
- table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
- index = i & IORING_FILE_TABLE_MASK;
file = fget(fd);
-
ret = -EBADF;
if (!file)
- break;
+ goto out_fput;
/*
* Don't allow io_uring instances to be registered. If UNIX
*/
if (file->f_op == &io_uring_fops) {
fput(file);
- break;
+ goto out_fput;
}
- ret = 0;
+ table = &file_data->table[i >> IORING_FILE_TABLE_SHIFT];
+ index = i & IORING_FILE_TABLE_MASK;
table->files[index] = file;
}
- if (ret) {
- for (i = 0; i < ctx->nr_user_files; i++) {
- file = io_file_from_index(ctx, i);
- if (file)
- fput(file);
- }
- for (i = 0; i < nr_tables; i++)
- kfree(ctx->file_data->table[i].files);
-
- percpu_ref_exit(&ctx->file_data->refs);
- kfree(ctx->file_data->table);
- kfree(ctx->file_data);
- ctx->file_data = NULL;
- ctx->nr_user_files = 0;
- return ret;
- }
-
+ ctx->file_data = file_data;
ret = io_sqe_files_scm(ctx);
if (ret) {
io_sqe_files_unregister(ctx);
return PTR_ERR(ref_node);
}
- ctx->file_data->cur_refs = &ref_node->refs;
- spin_lock(&ctx->file_data->lock);
- list_add(&ref_node->node, &ctx->file_data->ref_list);
- spin_unlock(&ctx->file_data->lock);
- percpu_ref_get(&ctx->file_data->refs);
+ file_data->node = ref_node;
+ spin_lock(&file_data->lock);
+ list_add(&ref_node->node, &file_data->ref_list);
+ spin_unlock(&file_data->lock);
+ percpu_ref_get(&file_data->refs);
+ return ret;
+out_fput:
+ for (i = 0; i < ctx->nr_user_files; i++) {
+ file = io_file_from_index(ctx, i);
+ if (file)
+ fput(file);
+ }
+ for (i = 0; i < nr_tables; i++)
+ kfree(file_data->table[i].files);
+ ctx->nr_user_files = 0;
+out_ref:
+ percpu_ref_exit(&file_data->refs);
+out_free:
+ kfree(file_data->table);
+ kfree(file_data);
return ret;
}
struct file *file)
{
struct io_file_put *pfile;
- struct percpu_ref *refs = data->cur_refs;
- struct fixed_file_ref_node *ref_node;
+ struct fixed_file_ref_node *ref_node = data->node;
pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
if (!pfile)
return -ENOMEM;
- ref_node = container_of(refs, struct fixed_file_ref_node, refs);
pfile->file = file;
list_add(&pfile->list, &ref_node->file_list);
}
if (needs_switch) {
- percpu_ref_kill(data->cur_refs);
+ percpu_ref_kill(&data->node->refs);
spin_lock(&data->lock);
list_add(&ref_node->node, &data->ref_list);
- data->cur_refs = &ref_node->refs;
+ data->node = ref_node;
spin_unlock(&data->lock);
percpu_ref_get(&ctx->file_data->refs);
} else
return ret;
}
-static int io_sq_offload_start(struct io_ring_ctx *ctx,
- struct io_uring_params *p)
+static int io_uring_alloc_task_context(struct task_struct *task)
+{
+ struct io_uring_task *tctx;
+
+ tctx = kmalloc(sizeof(*tctx), GFP_KERNEL);
+ if (unlikely(!tctx))
+ return -ENOMEM;
+
+ xa_init(&tctx->xa);
+ init_waitqueue_head(&tctx->wait);
+ tctx->last = NULL;
+ tctx->in_idle = 0;
+ atomic_long_set(&tctx->req_issue, 0);
+ atomic_long_set(&tctx->req_complete, 0);
+ task->io_uring = tctx;
+ return 0;
+}
+
+void __io_uring_free(struct task_struct *tsk)
+{
+ struct io_uring_task *tctx = tsk->io_uring;
+
+ WARN_ON_ONCE(!xa_empty(&tctx->xa));
+ kfree(tctx);
+ tsk->io_uring = NULL;
+}
+
+static int io_sq_offload_create(struct io_ring_ctx *ctx,
+ struct io_uring_params *p)
{
int ret;
if (ctx->flags & IORING_SETUP_SQPOLL) {
+ struct io_sq_data *sqd;
+
ret = -EPERM;
if (!capable(CAP_SYS_ADMIN))
goto err;
+ sqd = io_get_sq_data(p);
+ if (IS_ERR(sqd)) {
+ ret = PTR_ERR(sqd);
+ goto err;
+ }
+
+ ctx->sq_data = sqd;
+ io_sq_thread_park(sqd);
+ mutex_lock(&sqd->ctx_lock);
+ list_add(&ctx->sqd_list, &sqd->ctx_new_list);
+ mutex_unlock(&sqd->ctx_lock);
+ io_sq_thread_unpark(sqd);
+
ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
if (!ctx->sq_thread_idle)
ctx->sq_thread_idle = HZ;
+ if (sqd->thread)
+ goto done;
+
if (p->flags & IORING_SETUP_SQ_AFF) {
int cpu = p->sq_thread_cpu;
if (!cpu_online(cpu))
goto err;
- ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
- ctx, cpu,
- "io_uring-sq");
+ sqd->thread = kthread_create_on_cpu(io_sq_thread, sqd,
+ cpu, "io_uring-sq");
} else {
- ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
+ sqd->thread = kthread_create(io_sq_thread, sqd,
"io_uring-sq");
}
- if (IS_ERR(ctx->sqo_thread)) {
- ret = PTR_ERR(ctx->sqo_thread);
- ctx->sqo_thread = NULL;
+ if (IS_ERR(sqd->thread)) {
+ ret = PTR_ERR(sqd->thread);
+ sqd->thread = NULL;
goto err;
}
- wake_up_process(ctx->sqo_thread);
+ ret = io_uring_alloc_task_context(sqd->thread);
+ if (ret)
+ goto err;
} else if (p->flags & IORING_SETUP_SQ_AFF) {
/* Can't have SQ_AFF without SQPOLL */
ret = -EINVAL;
goto err;
}
+done:
ret = io_init_wq_offload(ctx, p);
if (ret)
goto err;
return ret;
}
+static void io_sq_offload_start(struct io_ring_ctx *ctx)
+{
+ struct io_sq_data *sqd = ctx->sq_data;
+
+ if ((ctx->flags & IORING_SETUP_SQPOLL) && sqd->thread)
+ wake_up_process(sqd->thread);
+}
+
static inline void __io_unaccount_mem(struct user_struct *user,
unsigned long nr_pages)
{
if (ctx->limit_mem)
__io_unaccount_mem(ctx->user, nr_pages);
- if (ctx->sqo_mm) {
+ if (ctx->mm_account) {
if (acct == ACCT_LOCKED)
- ctx->sqo_mm->locked_vm -= nr_pages;
+ ctx->mm_account->locked_vm -= nr_pages;
else if (acct == ACCT_PINNED)
- atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
+ atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
}
}
return ret;
}
- if (ctx->sqo_mm) {
+ if (ctx->mm_account) {
if (acct == ACCT_LOCKED)
- ctx->sqo_mm->locked_vm += nr_pages;
+ ctx->mm_account->locked_vm += nr_pages;
else if (acct == ACCT_PINNED)
- atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
+ atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
}
return 0;
for (j = 0; j < imu->nr_bvecs; j++)
unpin_user_page(imu->bvec[j].bv_page);
- io_unaccount_mem(ctx, imu->nr_bvecs, ACCT_PINNED);
+ if (imu->acct_pages)
+ io_unaccount_mem(ctx, imu->acct_pages, ACCT_PINNED);
kvfree(imu->bvec);
imu->nr_bvecs = 0;
}
return 0;
}
+/*
+ * Not super efficient, but this is just a registration time. And we do cache
+ * the last compound head, so generally we'll only do a full search if we don't
+ * match that one.
+ *
+ * We check if the given compound head page has already been accounted, to
+ * avoid double accounting it. This allows us to account the full size of the
+ * page, not just the constituent pages of a huge page.
+ */
+static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
+ int nr_pages, struct page *hpage)
+{
+ int i, j;
+
+ /* check current page array */
+ for (i = 0; i < nr_pages; i++) {
+ if (!PageCompound(pages[i]))
+ continue;
+ if (compound_head(pages[i]) == hpage)
+ return true;
+ }
+
+ /* check previously registered pages */
+ for (i = 0; i < ctx->nr_user_bufs; i++) {
+ struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
+
+ for (j = 0; j < imu->nr_bvecs; j++) {
+ if (!PageCompound(imu->bvec[j].bv_page))
+ continue;
+ if (compound_head(imu->bvec[j].bv_page) == hpage)
+ return true;
+ }
+ }
+
+ return false;
+}
+
+static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
+ int nr_pages, struct io_mapped_ubuf *imu,
+ struct page **last_hpage)
+{
+ int i, ret;
+
+ for (i = 0; i < nr_pages; i++) {
+ if (!PageCompound(pages[i])) {
+ imu->acct_pages++;
+ } else {
+ struct page *hpage;
+
+ hpage = compound_head(pages[i]);
+ if (hpage == *last_hpage)
+ continue;
+ *last_hpage = hpage;
+ if (headpage_already_acct(ctx, pages, i, hpage))
+ continue;
+ imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
+ }
+ }
+
+ if (!imu->acct_pages)
+ return 0;
+
+ ret = io_account_mem(ctx, imu->acct_pages, ACCT_PINNED);
+ if (ret)
+ imu->acct_pages = 0;
+ return ret;
+}
+
static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
unsigned nr_args)
{
struct vm_area_struct **vmas = NULL;
struct page **pages = NULL;
+ struct page *last_hpage = NULL;
int i, j, got_pages = 0;
int ret = -EINVAL;
start = ubuf >> PAGE_SHIFT;
nr_pages = end - start;
- ret = io_account_mem(ctx, nr_pages, ACCT_PINNED);
- if (ret)
- goto err;
-
ret = 0;
if (!pages || nr_pages > got_pages) {
kvfree(vmas);
GFP_KERNEL);
if (!pages || !vmas) {
ret = -ENOMEM;
- io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
goto err;
}
got_pages = nr_pages;
imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
GFP_KERNEL);
ret = -ENOMEM;
- if (!imu->bvec) {
- io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
+ if (!imu->bvec)
goto err;
- }
ret = 0;
mmap_read_lock(current->mm);
*/
if (pret > 0)
unpin_user_pages(pages, pret);
- io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
+ kvfree(imu->bvec);
+ goto err;
+ }
+
+ ret = io_buffer_account_pin(ctx, pages, pret, imu, &last_hpage);
+ if (ret) {
+ unpin_user_pages(pages, pret);
kvfree(imu->bvec);
goto err;
}
{
io_finish_async(ctx);
io_sqe_buffer_unregister(ctx);
- if (ctx->sqo_mm) {
- mmdrop(ctx->sqo_mm);
- ctx->sqo_mm = NULL;
+
+ if (ctx->sqo_task) {
+ put_task_struct(ctx->sqo_task);
+ ctx->sqo_task = NULL;
+ mmdrop(ctx->mm_account);
+ ctx->mm_account = NULL;
}
+#ifdef CONFIG_BLK_CGROUP
+ if (ctx->sqo_blkcg_css)
+ css_put(ctx->sqo_blkcg_css);
+#endif
+
io_sqe_files_unregister(ctx);
io_eventfd_unregister(ctx);
io_destroy_buffers(ctx);
* io_commit_cqring
*/
smp_rmb();
- if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
- ctx->rings->sq_ring_entries)
+ if (!io_sqring_full(ctx))
mask |= EPOLLOUT | EPOLLWRNORM;
if (io_cqring_events(ctx, false))
mask |= EPOLLIN | EPOLLRDNORM;
*/
do {
if (ctx->rings)
- io_cqring_overflow_flush(ctx, true);
+ io_cqring_overflow_flush(ctx, true, NULL, NULL);
io_iopoll_try_reap_events(ctx);
} while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
io_ring_ctx_free(ctx);
percpu_ref_kill(&ctx->refs);
mutex_unlock(&ctx->uring_lock);
- io_kill_timeouts(ctx);
- io_poll_remove_all(ctx);
+ io_kill_timeouts(ctx, NULL);
+ io_poll_remove_all(ctx, NULL);
if (ctx->io_wq)
io_wq_cancel_all(ctx->io_wq);
/* if we failed setting up the ctx, we might not have any rings */
if (ctx->rings)
- io_cqring_overflow_flush(ctx, true);
+ io_cqring_overflow_flush(ctx, true, NULL, NULL);
io_iopoll_try_reap_events(ctx);
idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
{
struct files_struct *files = data;
- return work->files == files;
+ return !files || work->files == files;
}
/*
return false;
}
-static inline bool io_match_files(struct io_kiocb *req,
- struct files_struct *files)
-{
- return (req->flags & REQ_F_WORK_INITIALIZED) && req->work.files == files;
-}
-
static bool io_match_link_files(struct io_kiocb *req,
struct files_struct *files)
{
}
}
-static void io_uring_cancel_files(struct io_ring_ctx *ctx,
+/*
+ * Returns true if we found and killed one or more files pinning requests
+ */
+static bool io_uring_cancel_files(struct io_ring_ctx *ctx,
struct files_struct *files)
{
if (list_empty_careful(&ctx->inflight_list))
- return;
+ return false;
io_cancel_defer_files(ctx, files);
/* cancel all at once, should be faster than doing it one by one*/
spin_lock_irq(&ctx->inflight_lock);
list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
- if (req->work.files != files)
+ if (files && req->work.files != files)
continue;
/* req is being completed, ignore */
if (!refcount_inc_not_zero(&req->refs))
schedule();
finish_wait(&ctx->inflight_wait, &wait);
}
+
+ return true;
}
static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
struct task_struct *task = data;
- return req->task == task;
+ return io_task_match(req, task);
+}
+
+static bool __io_uring_cancel_task_requests(struct io_ring_ctx *ctx,
+ struct task_struct *task,
+ struct files_struct *files)
+{
+ bool ret;
+
+ ret = io_uring_cancel_files(ctx, files);
+ if (!files) {
+ enum io_wq_cancel cret;
+
+ cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, task, true);
+ if (cret != IO_WQ_CANCEL_NOTFOUND)
+ ret = true;
+
+ /* SQPOLL thread does its own polling */
+ if (!(ctx->flags & IORING_SETUP_SQPOLL)) {
+ while (!list_empty_careful(&ctx->iopoll_list)) {
+ io_iopoll_try_reap_events(ctx);
+ ret = true;
+ }
+ }
+
+ ret |= io_poll_remove_all(ctx, task);
+ ret |= io_kill_timeouts(ctx, task);
+ }
+
+ return ret;
+}
+
+/*
+ * We need to iteratively cancel requests, in case a request has dependent
+ * hard links. These persist even for failure of cancelations, hence keep
+ * looping until none are found.
+ */
+static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx,
+ struct files_struct *files)
+{
+ struct task_struct *task = current;
+
+ if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data)
+ task = ctx->sq_data->thread;
+
+ io_cqring_overflow_flush(ctx, true, task, files);
+
+ while (__io_uring_cancel_task_requests(ctx, task, files)) {
+ io_run_task_work();
+ cond_resched();
+ }
+}
+
+/*
+ * Note that this task has used io_uring. We use it for cancelation purposes.
+ */
+static int io_uring_add_task_file(struct file *file)
+{
+ struct io_uring_task *tctx = current->io_uring;
+
+ if (unlikely(!tctx)) {
+ int ret;
+
+ ret = io_uring_alloc_task_context(current);
+ if (unlikely(ret))
+ return ret;
+ tctx = current->io_uring;
+ }
+ if (tctx->last != file) {
+ void *old = xa_load(&tctx->xa, (unsigned long)file);
+
+ if (!old) {
+ get_file(file);
+ xa_store(&tctx->xa, (unsigned long)file, file, GFP_KERNEL);
+ }
+ tctx->last = file;
+ }
+
+ return 0;
+}
+
+/*
+ * Remove this io_uring_file -> task mapping.
+ */
+static void io_uring_del_task_file(struct file *file)
+{
+ struct io_uring_task *tctx = current->io_uring;
+
+ if (tctx->last == file)
+ tctx->last = NULL;
+ file = xa_erase(&tctx->xa, (unsigned long)file);
+ if (file)
+ fput(file);
+}
+
+static void __io_uring_attempt_task_drop(struct file *file)
+{
+ struct file *old = xa_load(¤t->io_uring->xa, (unsigned long)file);
+
+ if (old == file)
+ io_uring_del_task_file(file);
+}
+
+/*
+ * Drop task note for this file if we're the only ones that hold it after
+ * pending fput()
+ */
+static void io_uring_attempt_task_drop(struct file *file, bool exiting)
+{
+ if (!current->io_uring)
+ return;
+ /*
+ * fput() is pending, will be 2 if the only other ref is our potential
+ * task file note. If the task is exiting, drop regardless of count.
+ */
+ if (!exiting && atomic_long_read(&file->f_count) != 2)
+ return;
+
+ __io_uring_attempt_task_drop(file);
+}
+
+void __io_uring_files_cancel(struct files_struct *files)
+{
+ struct io_uring_task *tctx = current->io_uring;
+ struct file *file;
+ unsigned long index;
+
+ /* make sure overflow events are dropped */
+ tctx->in_idle = true;
+
+ xa_for_each(&tctx->xa, index, file) {
+ struct io_ring_ctx *ctx = file->private_data;
+
+ io_uring_cancel_task_requests(ctx, files);
+ if (files)
+ io_uring_del_task_file(file);
+ }
+}
+
+static inline bool io_uring_task_idle(struct io_uring_task *tctx)
+{
+ return atomic_long_read(&tctx->req_issue) ==
+ atomic_long_read(&tctx->req_complete);
+}
+
+/*
+ * Find any io_uring fd that this task has registered or done IO on, and cancel
+ * requests.
+ */
+void __io_uring_task_cancel(void)
+{
+ struct io_uring_task *tctx = current->io_uring;
+ DEFINE_WAIT(wait);
+ long completions;
+
+ /* make sure overflow events are dropped */
+ tctx->in_idle = true;
+
+ while (!io_uring_task_idle(tctx)) {
+ /* read completions before cancelations */
+ completions = atomic_long_read(&tctx->req_complete);
+ __io_uring_files_cancel(NULL);
+
+ prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
+
+ /*
+ * If we've seen completions, retry. This avoids a race where
+ * a completion comes in before we did prepare_to_wait().
+ */
+ if (completions != atomic_long_read(&tctx->req_complete))
+ continue;
+ if (io_uring_task_idle(tctx))
+ break;
+ schedule();
+ }
+
+ finish_wait(&tctx->wait, &wait);
+ tctx->in_idle = false;
}
static int io_uring_flush(struct file *file, void *data)
{
struct io_ring_ctx *ctx = file->private_data;
- io_uring_cancel_files(ctx, data);
-
/*
* If the task is going away, cancel work it may have pending
*/
if (fatal_signal_pending(current) || (current->flags & PF_EXITING))
- io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, current, true);
+ data = NULL;
+ io_uring_cancel_task_requests(ctx, data);
+ io_uring_attempt_task_drop(file, !data);
return 0;
}
#endif /* !CONFIG_MMU */
+static void io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
+{
+ DEFINE_WAIT(wait);
+
+ do {
+ if (!io_sqring_full(ctx))
+ break;
+
+ prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE);
+
+ if (!io_sqring_full(ctx))
+ break;
+
+ schedule();
+ } while (!signal_pending(current));
+
+ finish_wait(&ctx->sqo_sq_wait, &wait);
+}
+
SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
u32, min_complete, u32, flags, const sigset_t __user *, sig,
size_t, sigsz)
io_run_task_work();
- if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
+ if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
+ IORING_ENTER_SQ_WAIT))
return -EINVAL;
f = fdget(fd);
if (!percpu_ref_tryget(&ctx->refs))
goto out_fput;
+ ret = -EBADFD;
+ if (ctx->flags & IORING_SETUP_R_DISABLED)
+ goto out;
+
/*
* For SQ polling, the thread will do all submissions and completions.
* Just return the requested submit count, and wake the thread if
ret = 0;
if (ctx->flags & IORING_SETUP_SQPOLL) {
if (!list_empty_careful(&ctx->cq_overflow_list))
- io_cqring_overflow_flush(ctx, false);
+ io_cqring_overflow_flush(ctx, false, NULL, NULL);
if (flags & IORING_ENTER_SQ_WAKEUP)
- wake_up(&ctx->sqo_wait);
+ wake_up(&ctx->sq_data->wait);
+ if (flags & IORING_ENTER_SQ_WAIT)
+ io_sqpoll_wait_sq(ctx);
submitted = to_submit;
} else if (to_submit) {
+ ret = io_uring_add_task_file(f.file);
+ if (unlikely(ret))
+ goto out;
mutex_lock(&ctx->uring_lock);
- submitted = io_submit_sqes(ctx, to_submit, f.file, fd);
+ submitted = io_submit_sqes(ctx, to_submit);
mutex_unlock(&ctx->uring_lock);
if (submitted != to_submit)
static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
{
+ struct io_sq_data *sq = NULL;
+ bool has_lock;
int i;
- mutex_lock(&ctx->uring_lock);
+ /*
+ * Avoid ABBA deadlock between the seq lock and the io_uring mutex,
+ * since fdinfo case grabs it in the opposite direction of normal use
+ * cases. If we fail to get the lock, we just don't iterate any
+ * structures that could be going away outside the io_uring mutex.
+ */
+ has_lock = mutex_trylock(&ctx->uring_lock);
+
+ if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL))
+ sq = ctx->sq_data;
+
+ seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1);
+ seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1);
seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
- for (i = 0; i < ctx->nr_user_files; i++) {
+ for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
struct fixed_file_table *table;
struct file *f;
seq_printf(m, "%5u: <none>\n", i);
}
seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
- for (i = 0; i < ctx->nr_user_bufs; i++) {
+ for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
struct io_mapped_ubuf *buf = &ctx->user_bufs[i];
seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
(unsigned int) buf->len);
}
- if (!idr_is_empty(&ctx->personality_idr)) {
+ if (has_lock && !idr_is_empty(&ctx->personality_idr)) {
seq_printf(m, "Personalities:\n");
idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
}
req->task->task_works != NULL);
}
spin_unlock_irq(&ctx->completion_lock);
- mutex_unlock(&ctx->uring_lock);
+ if (has_lock)
+ mutex_unlock(&ctx->uring_lock);
}
static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
O_RDWR | O_CLOEXEC);
if (IS_ERR(file)) {
+err_fd:
put_unused_fd(ret);
ret = PTR_ERR(file);
goto err;
#if defined(CONFIG_UNIX)
ctx->ring_sock->file = file;
#endif
+ if (unlikely(io_uring_add_task_file(file))) {
+ file = ERR_PTR(-ENOMEM);
+ goto err_fd;
+ }
fd_install(ret, file);
return ret;
err:
ctx->user = user;
ctx->creds = get_current_cred();
+ ctx->sqo_task = get_task_struct(current);
+
+ /*
+ * This is just grabbed for accounting purposes. When a process exits,
+ * the mm is exited and dropped before the files, hence we need to hang
+ * on to this mm purely for the purposes of being able to unaccount
+ * memory (locked/pinned vm). It's not used for anything else.
+ */
mmgrab(current->mm);
- ctx->sqo_mm = current->mm;
+ ctx->mm_account = current->mm;
+
+#ifdef CONFIG_BLK_CGROUP
+ /*
+ * The sq thread will belong to the original cgroup it was inited in.
+ * If the cgroup goes offline (e.g. disabling the io controller), then
+ * issued bios will be associated with the closest cgroup later in the
+ * block layer.
+ */
+ rcu_read_lock();
+ ctx->sqo_blkcg_css = blkcg_css();
+ ret = css_tryget_online(ctx->sqo_blkcg_css);
+ rcu_read_unlock();
+ if (!ret) {
+ /* don't init against a dying cgroup, have the user try again */
+ ctx->sqo_blkcg_css = NULL;
+ ret = -ENODEV;
+ goto err;
+ }
+#endif
/*
* Account memory _before_ installing the file descriptor. Once
if (ret)
goto err;
- ret = io_sq_offload_start(ctx, p);
+ ret = io_sq_offload_create(ctx, p);
if (ret)
goto err;
+ if (!(p->flags & IORING_SETUP_R_DISABLED))
+ io_sq_offload_start(ctx);
+
memset(&p->sq_off, 0, sizeof(p->sq_off));
p->sq_off.head = offsetof(struct io_rings, sq.head);
p->sq_off.tail = offsetof(struct io_rings, sq.tail);
if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
- IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))
+ IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
+ IORING_SETUP_R_DISABLED))
return -EINVAL;
return io_uring_create(entries, &p, params);
return -EINVAL;
}
+static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
+ unsigned int nr_args)
+{
+ struct io_uring_restriction *res;
+ size_t size;
+ int i, ret;
+
+ /* Restrictions allowed only if rings started disabled */
+ if (!(ctx->flags & IORING_SETUP_R_DISABLED))
+ return -EBADFD;
+
+ /* We allow only a single restrictions registration */
+ if (ctx->restrictions.registered)
+ return -EBUSY;
+
+ if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
+ return -EINVAL;
+
+ size = array_size(nr_args, sizeof(*res));
+ if (size == SIZE_MAX)
+ return -EOVERFLOW;
+
+ res = memdup_user(arg, size);
+ if (IS_ERR(res))
+ return PTR_ERR(res);
+
+ ret = 0;
+
+ for (i = 0; i < nr_args; i++) {
+ switch (res[i].opcode) {
+ case IORING_RESTRICTION_REGISTER_OP:
+ if (res[i].register_op >= IORING_REGISTER_LAST) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ __set_bit(res[i].register_op,
+ ctx->restrictions.register_op);
+ break;
+ case IORING_RESTRICTION_SQE_OP:
+ if (res[i].sqe_op >= IORING_OP_LAST) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
+ break;
+ case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
+ ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
+ break;
+ case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
+ ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
+ break;
+ default:
+ ret = -EINVAL;
+ goto out;
+ }
+ }
+
+out:
+ /* Reset all restrictions if an error happened */
+ if (ret != 0)
+ memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
+ else
+ ctx->restrictions.registered = true;
+
+ kfree(res);
+ return ret;
+}
+
+static int io_register_enable_rings(struct io_ring_ctx *ctx)
+{
+ if (!(ctx->flags & IORING_SETUP_R_DISABLED))
+ return -EBADFD;
+
+ if (ctx->restrictions.registered)
+ ctx->restricted = 1;
+
+ ctx->flags &= ~IORING_SETUP_R_DISABLED;
+
+ io_sq_offload_start(ctx);
+
+ return 0;
+}
+
static bool io_register_op_must_quiesce(int op)
{
switch (op) {
* after we've killed the percpu ref.
*/
mutex_unlock(&ctx->uring_lock);
- ret = wait_for_completion_interruptible(&ctx->ref_comp);
+ do {
+ ret = wait_for_completion_interruptible(&ctx->ref_comp);
+ if (!ret)
+ break;
+ ret = io_run_task_work_sig();
+ if (ret < 0)
+ break;
+ } while (1);
+
mutex_lock(&ctx->uring_lock);
+
if (ret) {
percpu_ref_resurrect(&ctx->refs);
- ret = -EINTR;
+ goto out_quiesce;
+ }
+ }
+
+ if (ctx->restricted) {
+ if (opcode >= IORING_REGISTER_LAST) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (!test_bit(opcode, ctx->restrictions.register_op)) {
+ ret = -EACCES;
goto out;
}
}
break;
ret = io_unregister_personality(ctx, nr_args);
break;
+ case IORING_REGISTER_ENABLE_RINGS:
+ ret = -EINVAL;
+ if (arg || nr_args)
+ break;
+ ret = io_register_enable_rings(ctx);
+ break;
+ case IORING_REGISTER_RESTRICTIONS:
+ ret = io_register_restrictions(ctx, arg, nr_args);
+ break;
default:
ret = -EINVAL;
break;
}
+out:
if (io_register_op_must_quiesce(opcode)) {
/* bring the ctx back to life */
percpu_ref_reinit(&ctx->refs);
-out:
+out_quiesce:
reinit_completion(&ctx->ref_comp);
}
return ret;
}
}
-void *copy_mount_options(const void __user * data)
+static void *copy_mount_options(const void __user * data)
{
char *copy;
- unsigned size;
+ unsigned left, offset;
if (!data)
return NULL;
if (!copy)
return ERR_PTR(-ENOMEM);
- size = PAGE_SIZE - offset_in_page(data);
+ left = copy_from_user(copy, data, PAGE_SIZE);
- if (copy_from_user(copy, data, size)) {
+ /*
+ * Not all architectures have an exact copy_from_user(). Resort to
+ * byte at a time.
+ */
+ offset = PAGE_SIZE - left;
+ while (left) {
+ char c;
+ if (get_user(c, (const char __user *)data + offset))
+ break;
+ copy[offset] = c;
+ left--;
+ offset++;
+ }
+
+ if (left == PAGE_SIZE) {
kfree(copy);
return ERR_PTR(-EFAULT);
}
- if (size != PAGE_SIZE) {
- if (copy_from_user(copy + size, data + size, PAGE_SIZE - size))
- memset(copy + size, 0, PAGE_SIZE - size);
- }
+
return copy;
}
-char *copy_mount_string(const void __user *data)
+static char *copy_mount_string(const void __user *data)
{
return data ? strndup_user(data, PATH_MAX) : NULL;
}
xdr_set_scratch_buffer(&stream, page_address(scratch), PAGE_SIZE);
do {
+ if (entry->label)
+ entry->label->len = NFS4_MAXLABELLEN;
+
status = xdr_decode(desc, entry, &stream);
if (status != 0) {
if (status == -EAGAIN)
}
static void
-ff_layout_mark_ds_unreachable(struct pnfs_layout_segment *lseg, int idx)
+ff_layout_mark_ds_unreachable(struct pnfs_layout_segment *lseg, u32 idx)
{
struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
}
static void
-ff_layout_mark_ds_reachable(struct pnfs_layout_segment *lseg, int idx)
+ff_layout_mark_ds_reachable(struct pnfs_layout_segment *lseg, u32 idx)
{
struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
static struct nfs4_pnfs_ds *
ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg,
- int start_idx, int *best_idx,
+ u32 start_idx, u32 *best_idx,
bool check_device)
{
struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg);
struct nfs4_ff_layout_mirror *mirror;
struct nfs4_pnfs_ds *ds;
bool fail_return = false;
- int idx;
+ u32 idx;
/* mirrors are initially sorted by efficiency */
for (idx = start_idx; idx < fls->mirror_array_cnt; idx++) {
static struct nfs4_pnfs_ds *
ff_layout_choose_any_ds_for_read(struct pnfs_layout_segment *lseg,
- int start_idx, int *best_idx)
+ u32 start_idx, u32 *best_idx)
{
return ff_layout_choose_ds_for_read(lseg, start_idx, best_idx, false);
}
static struct nfs4_pnfs_ds *
ff_layout_choose_valid_ds_for_read(struct pnfs_layout_segment *lseg,
- int start_idx, int *best_idx)
+ u32 start_idx, u32 *best_idx)
{
return ff_layout_choose_ds_for_read(lseg, start_idx, best_idx, true);
}
static struct nfs4_pnfs_ds *
ff_layout_choose_best_ds_for_read(struct pnfs_layout_segment *lseg,
- int start_idx, int *best_idx)
+ u32 start_idx, u32 *best_idx)
{
struct nfs4_pnfs_ds *ds;
}
static struct nfs4_pnfs_ds *
-ff_layout_get_ds_for_read(struct nfs_pageio_descriptor *pgio, int *best_idx)
+ff_layout_get_ds_for_read(struct nfs_pageio_descriptor *pgio,
+ u32 *best_idx)
{
struct pnfs_layout_segment *lseg = pgio->pg_lseg;
struct nfs4_pnfs_ds *ds;
struct nfs_pgio_mirror *pgm;
struct nfs4_ff_layout_mirror *mirror;
struct nfs4_pnfs_ds *ds;
- int ds_idx;
+ u32 ds_idx, i;
retry:
ff_layout_pg_check_layout(pgio, req);
goto retry;
}
- mirror = FF_LAYOUT_COMP(pgio->pg_lseg, ds_idx);
+ for (i = 0; i < pgio->pg_mirror_count; i++) {
+ mirror = FF_LAYOUT_COMP(pgio->pg_lseg, i);
+ pgm = &pgio->pg_mirrors[i];
+ pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].rsize;
+ }
pgio->pg_mirror_idx = ds_idx;
- /* read always uses only one mirror - idx 0 for pgio layer */
- pgm = &pgio->pg_mirrors[0];
- pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].rsize;
-
if (NFS_SERVER(pgio->pg_inode)->flags &
(NFS_MOUNT_SOFT|NFS_MOUNT_SOFTERR))
pgio->pg_maxretrans = io_maxretrans;
struct nfs4_ff_layout_mirror *mirror;
struct nfs_pgio_mirror *pgm;
struct nfs4_pnfs_ds *ds;
- int i;
+ u32 i;
retry:
ff_layout_pg_check_layout(pgio, req);
static void ff_layout_resend_pnfs_read(struct nfs_pgio_header *hdr)
{
u32 idx = hdr->pgio_mirror_idx + 1;
- int new_idx = 0;
+ u32 new_idx = 0;
if (ff_layout_choose_any_ds_for_read(hdr->lseg, idx + 1, &new_idx))
ff_layout_send_layouterror(hdr->lseg);
struct nfs4_state *state,
struct nfs_client *clp,
struct pnfs_layout_segment *lseg,
- int idx)
+ u32 idx)
{
struct pnfs_layout_hdr *lo = lseg->pls_layout;
struct inode *inode = lo->plh_inode;
/* Retry all errors through either pNFS or MDS except for -EJUKEBOX */
static int ff_layout_async_handle_error_v3(struct rpc_task *task,
struct pnfs_layout_segment *lseg,
- int idx)
+ u32 idx)
{
struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
struct nfs4_state *state,
struct nfs_client *clp,
struct pnfs_layout_segment *lseg,
- int idx)
+ u32 idx)
{
int vers = clp->cl_nfs_mod->rpc_vers->number;
}
static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg,
- int idx, u64 offset, u64 length,
+ u32 idx, u64 offset, u64 length,
u32 *op_status, int opnum, int error)
{
struct nfs4_ff_layout_mirror *mirror;
loff_t offset = hdr->args.offset;
int vers;
struct nfs_fh *fh;
- int idx = hdr->pgio_mirror_idx;
+ u32 idx = hdr->pgio_mirror_idx;
mirror = FF_LAYOUT_COMP(lseg, idx);
ds = nfs4_ff_layout_prepare_ds(lseg, mirror, true);
}
#if IS_ENABLED(CONFIG_NFS_V4)
+struct compat_nfs_string {
+ compat_uint_t len;
+ compat_uptr_t data;
+};
+
+static inline void compat_nfs_string(struct nfs_string *dst,
+ struct compat_nfs_string *src)
+{
+ dst->data = compat_ptr(src->data);
+ dst->len = src->len;
+}
+
+struct compat_nfs4_mount_data_v1 {
+ compat_int_t version;
+ compat_int_t flags;
+ compat_int_t rsize;
+ compat_int_t wsize;
+ compat_int_t timeo;
+ compat_int_t retrans;
+ compat_int_t acregmin;
+ compat_int_t acregmax;
+ compat_int_t acdirmin;
+ compat_int_t acdirmax;
+ struct compat_nfs_string client_addr;
+ struct compat_nfs_string mnt_path;
+ struct compat_nfs_string hostname;
+ compat_uint_t host_addrlen;
+ compat_uptr_t host_addr;
+ compat_int_t proto;
+ compat_int_t auth_flavourlen;
+ compat_uptr_t auth_flavours;
+};
+
+static void nfs4_compat_mount_data_conv(struct nfs4_mount_data *data)
+{
+ struct compat_nfs4_mount_data_v1 *compat =
+ (struct compat_nfs4_mount_data_v1 *)data;
+
+ /* copy the fields backwards */
+ data->auth_flavours = compat_ptr(compat->auth_flavours);
+ data->auth_flavourlen = compat->auth_flavourlen;
+ data->proto = compat->proto;
+ data->host_addr = compat_ptr(compat->host_addr);
+ data->host_addrlen = compat->host_addrlen;
+ compat_nfs_string(&data->hostname, &compat->hostname);
+ compat_nfs_string(&data->mnt_path, &compat->mnt_path);
+ compat_nfs_string(&data->client_addr, &compat->client_addr);
+ data->acdirmax = compat->acdirmax;
+ data->acdirmin = compat->acdirmin;
+ data->acregmax = compat->acregmax;
+ data->acregmin = compat->acregmin;
+ data->retrans = compat->retrans;
+ data->timeo = compat->timeo;
+ data->wsize = compat->wsize;
+ data->rsize = compat->rsize;
+ data->flags = compat->flags;
+ data->version = compat->version;
+}
+
/*
* Validate NFSv4 mount options
*/
struct sockaddr *sap = (struct sockaddr *)&ctx->nfs_server.address;
char *c;
- if (data == NULL)
- goto out_no_data;
+ if (!data) {
+ if (is_remount_fc(fc))
+ goto done;
+ return nfs_invalf(fc,
+ "NFS4: mount program didn't pass any mount data");
+ }
ctx->version = 4;
- switch (data->version) {
- case 1:
- if (data->host_addrlen > sizeof(ctx->nfs_server.address))
- goto out_no_address;
- if (data->host_addrlen == 0)
- goto out_no_address;
- ctx->nfs_server.addrlen = data->host_addrlen;
- if (copy_from_user(sap, data->host_addr, data->host_addrlen))
- return -EFAULT;
- if (!nfs_verify_server_address(sap))
- goto out_no_address;
- ctx->nfs_server.port = ntohs(((struct sockaddr_in *)sap)->sin_port);
-
- if (data->auth_flavourlen) {
- rpc_authflavor_t pseudoflavor;
- if (data->auth_flavourlen > 1)
- goto out_inval_auth;
- if (copy_from_user(&pseudoflavor,
- data->auth_flavours,
- sizeof(pseudoflavor)))
- return -EFAULT;
- ctx->selected_flavor = pseudoflavor;
- } else
- ctx->selected_flavor = RPC_AUTH_UNIX;
-
- c = strndup_user(data->hostname.data, NFS4_MAXNAMLEN);
- if (IS_ERR(c))
- return PTR_ERR(c);
- ctx->nfs_server.hostname = c;
+ if (data->version != 1)
+ return generic_parse_monolithic(fc, data);
- c = strndup_user(data->mnt_path.data, NFS4_MAXPATHLEN);
- if (IS_ERR(c))
- return PTR_ERR(c);
- ctx->nfs_server.export_path = c;
- dfprintk(MOUNT, "NFS: MNTPATH: '%s'\n", c);
+ if (in_compat_syscall())
+ nfs4_compat_mount_data_conv(data);
- c = strndup_user(data->client_addr.data, 16);
- if (IS_ERR(c))
- return PTR_ERR(c);
- ctx->client_address = c;
-
- /*
- * Translate to nfs_fs_context, which nfs_fill_super
- * can deal with.
- */
+ if (data->host_addrlen > sizeof(ctx->nfs_server.address))
+ goto out_no_address;
+ if (data->host_addrlen == 0)
+ goto out_no_address;
+ ctx->nfs_server.addrlen = data->host_addrlen;
+ if (copy_from_user(sap, data->host_addr, data->host_addrlen))
+ return -EFAULT;
+ if (!nfs_verify_server_address(sap))
+ goto out_no_address;
+ ctx->nfs_server.port = ntohs(((struct sockaddr_in *)sap)->sin_port);
- ctx->flags = data->flags & NFS4_MOUNT_FLAGMASK;
- ctx->rsize = data->rsize;
- ctx->wsize = data->wsize;
- ctx->timeo = data->timeo;
- ctx->retrans = data->retrans;
- ctx->acregmin = data->acregmin;
- ctx->acregmax = data->acregmax;
- ctx->acdirmin = data->acdirmin;
- ctx->acdirmax = data->acdirmax;
- ctx->nfs_server.protocol = data->proto;
- nfs_validate_transport_protocol(ctx);
- if (ctx->nfs_server.protocol == XPRT_TRANSPORT_UDP)
- goto out_invalid_transport_udp;
+ if (data->auth_flavourlen) {
+ rpc_authflavor_t pseudoflavor;
- break;
- default:
- goto generic;
+ if (data->auth_flavourlen > 1)
+ goto out_inval_auth;
+ if (copy_from_user(&pseudoflavor, data->auth_flavours,
+ sizeof(pseudoflavor)))
+ return -EFAULT;
+ ctx->selected_flavor = pseudoflavor;
+ } else {
+ ctx->selected_flavor = RPC_AUTH_UNIX;
}
+ c = strndup_user(data->hostname.data, NFS4_MAXNAMLEN);
+ if (IS_ERR(c))
+ return PTR_ERR(c);
+ ctx->nfs_server.hostname = c;
+
+ c = strndup_user(data->mnt_path.data, NFS4_MAXPATHLEN);
+ if (IS_ERR(c))
+ return PTR_ERR(c);
+ ctx->nfs_server.export_path = c;
+ dfprintk(MOUNT, "NFS: MNTPATH: '%s'\n", c);
+
+ c = strndup_user(data->client_addr.data, 16);
+ if (IS_ERR(c))
+ return PTR_ERR(c);
+ ctx->client_address = c;
+
+ /*
+ * Translate to nfs_fs_context, which nfs_fill_super
+ * can deal with.
+ */
+
+ ctx->flags = data->flags & NFS4_MOUNT_FLAGMASK;
+ ctx->rsize = data->rsize;
+ ctx->wsize = data->wsize;
+ ctx->timeo = data->timeo;
+ ctx->retrans = data->retrans;
+ ctx->acregmin = data->acregmin;
+ ctx->acregmax = data->acregmax;
+ ctx->acdirmin = data->acdirmin;
+ ctx->acdirmax = data->acdirmax;
+ ctx->nfs_server.protocol = data->proto;
+ nfs_validate_transport_protocol(ctx);
+ if (ctx->nfs_server.protocol == XPRT_TRANSPORT_UDP)
+ goto out_invalid_transport_udp;
+done:
ctx->skip_reconfig_option_check = true;
return 0;
-generic:
- return generic_parse_monolithic(fc, data);
-
-out_no_data:
- if (is_remount_fc(fc)) {
- ctx->skip_reconfig_option_check = true;
- return 0;
- }
- return nfs_invalf(fc, "NFS4: mount program didn't pass any mount data");
-
out_inval_auth:
return nfs_invalf(fc, "NFS4: Invalid number of RPC auth flavours %d",
data->auth_flavourlen);
truncate_pagecache_range(dst_inode, pos_dst,
pos_dst + res->write_res.count);
-
+ spin_lock(&dst_inode->i_lock);
+ NFS_I(dst_inode)->cache_validity |= (NFS_INO_REVAL_PAGECACHE |
+ NFS_INO_REVAL_FORCED | NFS_INO_INVALID_SIZE |
+ NFS_INO_INVALID_ATTR | NFS_INO_INVALID_DATA);
+ spin_unlock(&dst_inode->i_lock);
+ spin_lock(&src_inode->i_lock);
+ NFS_I(src_inode)->cache_validity |= (NFS_INO_REVAL_PAGECACHE |
+ NFS_INO_REVAL_FORCED | NFS_INO_INVALID_ATIME);
+ spin_unlock(&src_inode->i_lock);
status = res->write_res.count;
out:
if (args->sync)
struct nfs4_client *clp,
struct nfsd4_getdeviceinfo *gdp)
{
- if (sb->s_bdev != sb->s_bdev->bd_contains)
+ if (bdev_is_partition(sb->s_bdev))
return nfserr_inval;
return nfserrno(nfsd4_block_get_device_info_simple(sb, gdp));
}
struct nfs4_client *clp,
struct nfsd4_getdeviceinfo *gdp)
{
- if (sb->s_bdev != sb->s_bdev->bd_contains)
+ if (bdev_is_partition(sb->s_bdev))
return nfserr_inval;
return nfserrno(nfsd4_block_get_device_info_scsi(sb, clp, gdp));
}
}
}
-/* Drop the inode semaphore and wait for a pipe event, atomically */
-void pipe_wait(struct pipe_inode_info *pipe)
-{
- DEFINE_WAIT(rdwait);
- DEFINE_WAIT(wrwait);
-
- /*
- * Pipes are system-local resources, so sleeping on them
- * is considered a noninteractive wait:
- */
- prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
- prepare_to_wait(&pipe->wr_wait, &wrwait, TASK_INTERRUPTIBLE);
- pipe_unlock(pipe);
- schedule();
- finish_wait(&pipe->rd_wait, &rdwait);
- finish_wait(&pipe->wr_wait, &wrwait);
- pipe_lock(pipe);
-}
-
static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
struct pipe_buffer *buf)
{
{
struct inode *inode = get_pipe_inode();
struct file *f;
+ int error;
if (!inode)
return -ENFILE;
if (flags & O_NOTIFICATION_PIPE) {
-#ifdef CONFIG_WATCH_QUEUE
- if (watch_queue_init(inode->i_pipe) < 0) {
+ error = watch_queue_init(inode->i_pipe);
+ if (error) {
+ free_pipe_info(inode->i_pipe);
iput(inode);
- return -ENOMEM;
+ return error;
}
-#else
- return -ENOPKG;
-#endif
}
f = alloc_file_pseudo(inode, pipe_mnt, "",
return do_pipe2(fildes, 0);
}
+/*
+ * This is the stupid "wait for pipe to be readable or writable"
+ * model.
+ *
+ * See pipe_read/write() for the proper kind of exclusive wait,
+ * but that requires that we wake up any other readers/writers
+ * if we then do not end up reading everything (ie the whole
+ * "wake_next_reader/writer" logic in pipe_read/write()).
+ */
+void pipe_wait_readable(struct pipe_inode_info *pipe)
+{
+ pipe_unlock(pipe);
+ wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));
+ pipe_lock(pipe);
+}
+
+void pipe_wait_writable(struct pipe_inode_info *pipe)
+{
+ pipe_unlock(pipe);
+ wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));
+ pipe_lock(pipe);
+}
+
+/*
+ * This depends on both the wait (here) and the wakeup (wake_up_partner)
+ * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
+ * race with the count check and waitqueue prep.
+ *
+ * Normally in order to avoid races, you'd do the prepare_to_wait() first,
+ * then check the condition you're waiting for, and only then sleep. But
+ * because of the pipe lock, we can check the condition before being on
+ * the wait queue.
+ *
+ * We use the 'rd_wait' waitqueue for pipe partner waiting.
+ */
static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
{
+ DEFINE_WAIT(rdwait);
int cur = *cnt;
while (cur == *cnt) {
- pipe_wait(pipe);
+ prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
+ pipe_unlock(pipe);
+ schedule();
+ finish_wait(&pipe->rd_wait, &rdwait);
+ pipe_lock(pipe);
if (signal_pending(current))
break;
}
static void wake_up_partner(struct pipe_inode_info *pipe)
{
wake_up_interruptible_all(&pipe->rd_wait);
- wake_up_interruptible_all(&pipe->wr_wait);
}
static int fifo_open(struct inode *inode, struct file *filp)
u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2);
u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1);
u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1);
+#ifdef CONFIG_64BIT
+ u |= kpf_copy_bit(k, KPF_ARCH_2, PG_arch_2);
+#endif
return u;
};
[ilog2(VM_MERGEABLE)] = "mg",
[ilog2(VM_UFFD_MISSING)]= "um",
[ilog2(VM_UFFD_WP)] = "uw",
+#ifdef CONFIG_ARM64_MTE
+ [ilog2(VM_MTE)] = "mt",
+ [ilog2(VM_MTE_ALLOWED)] = "",
+#endif
#ifdef CONFIG_ARCH_HAS_PKEYS
/* These come out via ProtectionKey: */
[ilog2(VM_PKEY_BIT0)] = "",
config QUOTACTL
bool
default n
-
-config QUOTACTL_COMPAT
- bool
- depends on QUOTACTL && COMPAT_FOR_U64_ALIGNMENT
- default y
obj-$(CONFIG_QFMT_V2) += quota_v2.o
obj-$(CONFIG_QUOTA_TREE) += quota_tree.o
obj-$(CONFIG_QUOTACTL) += quota.o kqid.o
-obj-$(CONFIG_QUOTACTL_COMPAT) += compat.o
obj-$(CONFIG_QUOTA_NETLINK_INTERFACE) += netlink.o
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-
-#include <linux/syscalls.h>
-#include <linux/compat.h>
-#include <linux/quotaops.h>
-
-/*
- * This code works only for 32 bit quota tools over 64 bit OS (x86_64, ia64)
- * and is necessary due to alignment problems.
- */
-struct compat_if_dqblk {
- compat_u64 dqb_bhardlimit;
- compat_u64 dqb_bsoftlimit;
- compat_u64 dqb_curspace;
- compat_u64 dqb_ihardlimit;
- compat_u64 dqb_isoftlimit;
- compat_u64 dqb_curinodes;
- compat_u64 dqb_btime;
- compat_u64 dqb_itime;
- compat_uint_t dqb_valid;
-};
-
-/* XFS structures */
-struct compat_fs_qfilestat {
- compat_u64 dqb_bhardlimit;
- compat_u64 qfs_nblks;
- compat_uint_t qfs_nextents;
-};
-
-struct compat_fs_quota_stat {
- __s8 qs_version;
- __u16 qs_flags;
- __s8 qs_pad;
- struct compat_fs_qfilestat qs_uquota;
- struct compat_fs_qfilestat qs_gquota;
- compat_uint_t qs_incoredqs;
- compat_int_t qs_btimelimit;
- compat_int_t qs_itimelimit;
- compat_int_t qs_rtbtimelimit;
- __u16 qs_bwarnlimit;
- __u16 qs_iwarnlimit;
-};
-
-COMPAT_SYSCALL_DEFINE4(quotactl32, unsigned int, cmd,
- const char __user *, special, qid_t, id,
- void __user *, addr)
-{
- unsigned int cmds;
- struct if_dqblk __user *dqblk;
- struct compat_if_dqblk __user *compat_dqblk;
- struct fs_quota_stat __user *fsqstat;
- struct compat_fs_quota_stat __user *compat_fsqstat;
- compat_uint_t data;
- u16 xdata;
- long ret;
-
- cmds = cmd >> SUBCMDSHIFT;
-
- switch (cmds) {
- case Q_GETQUOTA:
- dqblk = compat_alloc_user_space(sizeof(struct if_dqblk));
- compat_dqblk = addr;
- ret = kernel_quotactl(cmd, special, id, dqblk);
- if (ret)
- break;
- if (copy_in_user(compat_dqblk, dqblk, sizeof(*compat_dqblk)) ||
- get_user(data, &dqblk->dqb_valid) ||
- put_user(data, &compat_dqblk->dqb_valid))
- ret = -EFAULT;
- break;
- case Q_SETQUOTA:
- dqblk = compat_alloc_user_space(sizeof(struct if_dqblk));
- compat_dqblk = addr;
- ret = -EFAULT;
- if (copy_in_user(dqblk, compat_dqblk, sizeof(*compat_dqblk)) ||
- get_user(data, &compat_dqblk->dqb_valid) ||
- put_user(data, &dqblk->dqb_valid))
- break;
- ret = kernel_quotactl(cmd, special, id, dqblk);
- break;
- case Q_XGETQSTAT:
- fsqstat = compat_alloc_user_space(sizeof(struct fs_quota_stat));
- compat_fsqstat = addr;
- ret = kernel_quotactl(cmd, special, id, fsqstat);
- if (ret)
- break;
- ret = -EFAULT;
- /* Copying qs_version, qs_flags, qs_pad */
- if (copy_in_user(compat_fsqstat, fsqstat,
- offsetof(struct compat_fs_quota_stat, qs_uquota)))
- break;
- /* Copying qs_uquota */
- if (copy_in_user(&compat_fsqstat->qs_uquota,
- &fsqstat->qs_uquota,
- sizeof(compat_fsqstat->qs_uquota)) ||
- get_user(data, &fsqstat->qs_uquota.qfs_nextents) ||
- put_user(data, &compat_fsqstat->qs_uquota.qfs_nextents))
- break;
- /* Copying qs_gquota */
- if (copy_in_user(&compat_fsqstat->qs_gquota,
- &fsqstat->qs_gquota,
- sizeof(compat_fsqstat->qs_gquota)) ||
- get_user(data, &fsqstat->qs_gquota.qfs_nextents) ||
- put_user(data, &compat_fsqstat->qs_gquota.qfs_nextents))
- break;
- /* Copying the rest */
- if (copy_in_user(&compat_fsqstat->qs_incoredqs,
- &fsqstat->qs_incoredqs,
- sizeof(struct compat_fs_quota_stat) -
- offsetof(struct compat_fs_quota_stat, qs_incoredqs)) ||
- get_user(xdata, &fsqstat->qs_iwarnlimit) ||
- put_user(xdata, &compat_fsqstat->qs_iwarnlimit))
- break;
- ret = 0;
- break;
- default:
- ret = kernel_quotactl(cmd, special, id, addr);
- }
- return ret;
-}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/compat.h>
+
+struct compat_if_dqblk {
+ compat_u64 dqb_bhardlimit;
+ compat_u64 dqb_bsoftlimit;
+ compat_u64 dqb_curspace;
+ compat_u64 dqb_ihardlimit;
+ compat_u64 dqb_isoftlimit;
+ compat_u64 dqb_curinodes;
+ compat_u64 dqb_btime;
+ compat_u64 dqb_itime;
+ compat_uint_t dqb_valid;
+};
+
+struct compat_fs_qfilestat {
+ compat_u64 dqb_bhardlimit;
+ compat_u64 qfs_nblks;
+ compat_uint_t qfs_nextents;
+};
+
+struct compat_fs_quota_stat {
+ __s8 qs_version;
+ __u16 qs_flags;
+ __s8 qs_pad;
+ struct compat_fs_qfilestat qs_uquota;
+ struct compat_fs_qfilestat qs_gquota;
+ compat_uint_t qs_incoredqs;
+ compat_int_t qs_btimelimit;
+ compat_int_t qs_itimelimit;
+ compat_int_t qs_rtbtimelimit;
+ __u16 qs_bwarnlimit;
+ __u16 qs_iwarnlimit;
+};
#include <linux/types.h>
#include <linux/writeback.h>
#include <linux/nospec.h>
+#include "compat.h"
static int check_quotactl_permission(struct super_block *sb, int type, int cmd,
qid_t id)
if (ret)
return ret;
copy_to_if_dqblk(&idq, &fdq);
- if (copy_to_user(addr, &idq, sizeof(idq)))
- return -EFAULT;
+
+ if (compat_need_64bit_alignment_fixup()) {
+ struct compat_if_dqblk __user *compat_dqblk = addr;
+
+ if (copy_to_user(compat_dqblk, &idq, sizeof(*compat_dqblk)))
+ return -EFAULT;
+ if (put_user(idq.dqb_valid, &compat_dqblk->dqb_valid))
+ return -EFAULT;
+ } else {
+ if (copy_to_user(addr, &idq, sizeof(idq)))
+ return -EFAULT;
+ }
return 0;
}
struct if_dqblk idq;
struct kqid qid;
- if (copy_from_user(&idq, addr, sizeof(idq)))
- return -EFAULT;
+ if (compat_need_64bit_alignment_fixup()) {
+ struct compat_if_dqblk __user *compat_dqblk = addr;
+
+ if (copy_from_user(&idq, compat_dqblk, sizeof(*compat_dqblk)) ||
+ get_user(idq.dqb_valid, &compat_dqblk->dqb_valid))
+ return -EFAULT;
+ } else {
+ if (copy_from_user(&idq, addr, sizeof(idq)))
+ return -EFAULT;
+ }
if (!sb->s_qcop->set_dqblk)
return -ENOSYS;
qid = make_kqid(current_user_ns(), type, id);
return 0;
}
+static int compat_copy_fs_qfilestat(struct compat_fs_qfilestat __user *to,
+ struct fs_qfilestat *from)
+{
+ if (copy_to_user(to, from, sizeof(*to)) ||
+ put_user(from->qfs_nextents, &to->qfs_nextents))
+ return -EFAULT;
+ return 0;
+}
+
+static int compat_copy_fs_quota_stat(struct compat_fs_quota_stat __user *to,
+ struct fs_quota_stat *from)
+{
+ if (put_user(from->qs_version, &to->qs_version) ||
+ put_user(from->qs_flags, &to->qs_flags) ||
+ put_user(from->qs_pad, &to->qs_pad) ||
+ compat_copy_fs_qfilestat(&to->qs_uquota, &from->qs_uquota) ||
+ compat_copy_fs_qfilestat(&to->qs_gquota, &from->qs_gquota) ||
+ put_user(from->qs_incoredqs, &to->qs_incoredqs) ||
+ put_user(from->qs_btimelimit, &to->qs_btimelimit) ||
+ put_user(from->qs_itimelimit, &to->qs_itimelimit) ||
+ put_user(from->qs_rtbtimelimit, &to->qs_rtbtimelimit) ||
+ put_user(from->qs_bwarnlimit, &to->qs_bwarnlimit) ||
+ put_user(from->qs_iwarnlimit, &to->qs_iwarnlimit))
+ return -EFAULT;
+ return 0;
+}
+
static int quota_getxstate(struct super_block *sb, int type, void __user *addr)
{
struct fs_quota_stat fqs;
if (!sb->s_qcop->get_state)
return -ENOSYS;
ret = quota_getstate(sb, type, &fqs);
- if (!ret && copy_to_user(addr, &fqs, sizeof(fqs)))
+ if (ret)
+ return ret;
+
+ if (compat_need_64bit_alignment_fixup())
+ return compat_copy_fs_quota_stat(addr, &fqs);
+ if (copy_to_user(addr, &fqs, sizeof(fqs)))
return -EFAULT;
- return ret;
+ return 0;
}
static int quota_getstatev(struct super_block *sb, int type,
* calls. Maybe we need to add the process quotas etc. in the future,
* but we probably should use rlimits for that.
*/
-int kernel_quotactl(unsigned int cmd, const char __user *special,
- qid_t id, void __user *addr)
+SYSCALL_DEFINE4(quotactl, unsigned int, cmd, const char __user *, special,
+ qid_t, id, void __user *, addr)
{
uint cmds, type;
struct super_block *sb = NULL;
path_put(pathp);
return ret;
}
-
-SYSCALL_DEFINE4(quotactl, unsigned int, cmd, const char __user *, special,
- qid_t, id, void __user *, addr)
-{
- return kernel_quotactl(cmd, special, id, addr);
-}
inc_syscw(current);
return ret;
}
+/*
+ * This "EXPORT_SYMBOL_GPL()" is more of a "EXPORT_SYMBOL_DONTUSE()",
+ * but autofs is one of the few internal kernel users that actually
+ * wants this _and_ can be built as a module. So we need to export
+ * this symbol for autofs, even though it really isn't appropriate
+ * for any other kernel modules.
+ */
+EXPORT_SYMBOL_GPL(__kernel_write);
ssize_t kernel_write(struct file *file, const void *buf, size_t count,
loff_t *pos)
return ret;
}
-/**
- * rw_copy_check_uvector() - Copy an array of &struct iovec from userspace
- * into the kernel and check that it is valid.
- *
- * @type: One of %CHECK_IOVEC_ONLY, %READ, or %WRITE.
- * @uvector: Pointer to the userspace array.
- * @nr_segs: Number of elements in userspace array.
- * @fast_segs: Number of elements in @fast_pointer.
- * @fast_pointer: Pointer to (usually small on-stack) kernel array.
- * @ret_pointer: (output parameter) Pointer to a variable that will point to
- * either @fast_pointer, a newly allocated kernel array, or NULL,
- * depending on which array was used.
- *
- * This function copies an array of &struct iovec of @nr_segs from
- * userspace into the kernel and checks that each element is valid (e.g.
- * it does not point to a kernel address or cause overflow by being too
- * large, etc.).
- *
- * As an optimization, the caller may provide a pointer to a small
- * on-stack array in @fast_pointer, typically %UIO_FASTIOV elements long
- * (the size of this array, or 0 if unused, should be given in @fast_segs).
- *
- * @ret_pointer will always point to the array that was used, so the
- * caller must take care not to call kfree() on it e.g. in case the
- * @fast_pointer array was used and it was allocated on the stack.
- *
- * Return: The total number of bytes covered by the iovec array on success
- * or a negative error code on error.
- */
-ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
- unsigned long nr_segs, unsigned long fast_segs,
- struct iovec *fast_pointer,
- struct iovec **ret_pointer)
-{
- unsigned long seg;
- ssize_t ret;
- struct iovec *iov = fast_pointer;
-
- /*
- * SuS says "The readv() function *may* fail if the iovcnt argument
- * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
- * traditionally returned zero for zero segments, so...
- */
- if (nr_segs == 0) {
- ret = 0;
- goto out;
- }
-
- /*
- * First get the "struct iovec" from user memory and
- * verify all the pointers
- */
- if (nr_segs > UIO_MAXIOV) {
- ret = -EINVAL;
- goto out;
- }
- if (nr_segs > fast_segs) {
- iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
- if (iov == NULL) {
- ret = -ENOMEM;
- goto out;
- }
- }
- if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
- ret = -EFAULT;
- goto out;
- }
-
- /*
- * According to the Single Unix Specification we should return EINVAL
- * if an element length is < 0 when cast to ssize_t or if the
- * total length would overflow the ssize_t return value of the
- * system call.
- *
- * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
- * overflow case.
- */
- ret = 0;
- for (seg = 0; seg < nr_segs; seg++) {
- void __user *buf = iov[seg].iov_base;
- ssize_t len = (ssize_t)iov[seg].iov_len;
-
- /* see if we we're about to use an invalid len or if
- * it's about to overflow ssize_t */
- if (len < 0) {
- ret = -EINVAL;
- goto out;
- }
- if (type >= 0
- && unlikely(!access_ok(buf, len))) {
- ret = -EFAULT;
- goto out;
- }
- if (len > MAX_RW_COUNT - ret) {
- len = MAX_RW_COUNT - ret;
- iov[seg].iov_len = len;
- }
- ret += len;
- }
-out:
- *ret_pointer = iov;
- return ret;
-}
-
-#ifdef CONFIG_COMPAT
-ssize_t compat_rw_copy_check_uvector(int type,
- const struct compat_iovec __user *uvector, unsigned long nr_segs,
- unsigned long fast_segs, struct iovec *fast_pointer,
- struct iovec **ret_pointer)
-{
- compat_ssize_t tot_len;
- struct iovec *iov = *ret_pointer = fast_pointer;
- ssize_t ret = 0;
- int seg;
-
- /*
- * SuS says "The readv() function *may* fail if the iovcnt argument
- * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
- * traditionally returned zero for zero segments, so...
- */
- if (nr_segs == 0)
- goto out;
-
- ret = -EINVAL;
- if (nr_segs > UIO_MAXIOV)
- goto out;
- if (nr_segs > fast_segs) {
- ret = -ENOMEM;
- iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
- if (iov == NULL)
- goto out;
- }
- *ret_pointer = iov;
-
- ret = -EFAULT;
- if (!access_ok(uvector, nr_segs*sizeof(*uvector)))
- goto out;
-
- /*
- * Single unix specification:
- * We should -EINVAL if an element length is not >= 0 and fitting an
- * ssize_t.
- *
- * In Linux, the total length is limited to MAX_RW_COUNT, there is
- * no overflow possibility.
- */
- tot_len = 0;
- ret = -EINVAL;
- for (seg = 0; seg < nr_segs; seg++) {
- compat_uptr_t buf;
- compat_ssize_t len;
-
- if (__get_user(len, &uvector->iov_len) ||
- __get_user(buf, &uvector->iov_base)) {
- ret = -EFAULT;
- goto out;
- }
- if (len < 0) /* size_t not fitting in compat_ssize_t .. */
- goto out;
- if (type >= 0 &&
- !access_ok(compat_ptr(buf), len)) {
- ret = -EFAULT;
- goto out;
- }
- if (len > MAX_RW_COUNT - tot_len)
- len = MAX_RW_COUNT - tot_len;
- tot_len += len;
- iov->iov_base = compat_ptr(buf);
- iov->iov_len = (compat_size_t) len;
- uvector++;
- iov++;
- }
- ret = tot_len;
-
-out:
- return ret;
-}
-#endif
-
static ssize_t do_iter_read(struct file *file, struct iov_iter *iter,
loff_t *pos, rwf_t flags)
{
return do_pwritev(fd, vec, vlen, pos, flags);
}
+/*
+ * Various compat syscalls. Note that they all pretend to take a native
+ * iovec - import_iovec will properly treat those as compat_iovecs based on
+ * in_compat_syscall().
+ */
#ifdef CONFIG_COMPAT
-static size_t compat_readv(struct file *file,
- const struct compat_iovec __user *vec,
- unsigned long vlen, loff_t *pos, rwf_t flags)
-{
- struct iovec iovstack[UIO_FASTIOV];
- struct iovec *iov = iovstack;
- struct iov_iter iter;
- ssize_t ret;
-
- ret = compat_import_iovec(READ, vec, vlen, UIO_FASTIOV, &iov, &iter);
- if (ret >= 0) {
- ret = do_iter_read(file, &iter, pos, flags);
- kfree(iov);
- }
- if (ret > 0)
- add_rchar(current, ret);
- inc_syscr(current);
- return ret;
-}
-
-static size_t do_compat_readv(compat_ulong_t fd,
- const struct compat_iovec __user *vec,
- compat_ulong_t vlen, rwf_t flags)
-{
- struct fd f = fdget_pos(fd);
- ssize_t ret;
- loff_t pos;
-
- if (!f.file)
- return -EBADF;
- pos = f.file->f_pos;
- ret = compat_readv(f.file, vec, vlen, &pos, flags);
- if (ret >= 0)
- f.file->f_pos = pos;
- fdput_pos(f);
- return ret;
-
-}
-
-COMPAT_SYSCALL_DEFINE3(readv, compat_ulong_t, fd,
- const struct compat_iovec __user *,vec,
- compat_ulong_t, vlen)
-{
- return do_compat_readv(fd, vec, vlen, 0);
-}
-
-static long do_compat_preadv64(unsigned long fd,
- const struct compat_iovec __user *vec,
- unsigned long vlen, loff_t pos, rwf_t flags)
-{
- struct fd f;
- ssize_t ret;
-
- if (pos < 0)
- return -EINVAL;
- f = fdget(fd);
- if (!f.file)
- return -EBADF;
- ret = -ESPIPE;
- if (f.file->f_mode & FMODE_PREAD)
- ret = compat_readv(f.file, vec, vlen, &pos, flags);
- fdput(f);
- return ret;
-}
-
#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
- const struct compat_iovec __user *,vec,
+ const struct iovec __user *, vec,
unsigned long, vlen, loff_t, pos)
{
- return do_compat_preadv64(fd, vec, vlen, pos, 0);
+ return do_preadv(fd, vec, vlen, pos, 0);
}
#endif
COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
- const struct compat_iovec __user *,vec,
+ const struct iovec __user *, vec,
compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
{
loff_t pos = ((loff_t)pos_high << 32) | pos_low;
- return do_compat_preadv64(fd, vec, vlen, pos, 0);
+ return do_preadv(fd, vec, vlen, pos, 0);
}
#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
- const struct compat_iovec __user *,vec,
+ const struct iovec __user *, vec,
unsigned long, vlen, loff_t, pos, rwf_t, flags)
{
if (pos == -1)
- return do_compat_readv(fd, vec, vlen, flags);
-
- return do_compat_preadv64(fd, vec, vlen, pos, flags);
+ return do_readv(fd, vec, vlen, flags);
+ return do_preadv(fd, vec, vlen, pos, flags);
}
#endif
COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
- const struct compat_iovec __user *,vec,
+ const struct iovec __user *, vec,
compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
rwf_t, flags)
{
loff_t pos = ((loff_t)pos_high << 32) | pos_low;
if (pos == -1)
- return do_compat_readv(fd, vec, vlen, flags);
-
- return do_compat_preadv64(fd, vec, vlen, pos, flags);
-}
-
-static size_t compat_writev(struct file *file,
- const struct compat_iovec __user *vec,
- unsigned long vlen, loff_t *pos, rwf_t flags)
-{
- struct iovec iovstack[UIO_FASTIOV];
- struct iovec *iov = iovstack;
- struct iov_iter iter;
- ssize_t ret;
-
- ret = compat_import_iovec(WRITE, vec, vlen, UIO_FASTIOV, &iov, &iter);
- if (ret >= 0) {
- file_start_write(file);
- ret = do_iter_write(file, &iter, pos, flags);
- file_end_write(file);
- kfree(iov);
- }
- if (ret > 0)
- add_wchar(current, ret);
- inc_syscw(current);
- return ret;
-}
-
-static size_t do_compat_writev(compat_ulong_t fd,
- const struct compat_iovec __user* vec,
- compat_ulong_t vlen, rwf_t flags)
-{
- struct fd f = fdget_pos(fd);
- ssize_t ret;
- loff_t pos;
-
- if (!f.file)
- return -EBADF;
- pos = f.file->f_pos;
- ret = compat_writev(f.file, vec, vlen, &pos, flags);
- if (ret >= 0)
- f.file->f_pos = pos;
- fdput_pos(f);
- return ret;
-}
-
-COMPAT_SYSCALL_DEFINE3(writev, compat_ulong_t, fd,
- const struct compat_iovec __user *, vec,
- compat_ulong_t, vlen)
-{
- return do_compat_writev(fd, vec, vlen, 0);
-}
-
-static long do_compat_pwritev64(unsigned long fd,
- const struct compat_iovec __user *vec,
- unsigned long vlen, loff_t pos, rwf_t flags)
-{
- struct fd f;
- ssize_t ret;
-
- if (pos < 0)
- return -EINVAL;
- f = fdget(fd);
- if (!f.file)
- return -EBADF;
- ret = -ESPIPE;
- if (f.file->f_mode & FMODE_PWRITE)
- ret = compat_writev(f.file, vec, vlen, &pos, flags);
- fdput(f);
- return ret;
+ return do_readv(fd, vec, vlen, flags);
+ return do_preadv(fd, vec, vlen, pos, flags);
}
#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
- const struct compat_iovec __user *,vec,
+ const struct iovec __user *, vec,
unsigned long, vlen, loff_t, pos)
{
- return do_compat_pwritev64(fd, vec, vlen, pos, 0);
+ return do_pwritev(fd, vec, vlen, pos, 0);
}
#endif
COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
- const struct compat_iovec __user *,vec,
+ const struct iovec __user *,vec,
compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
{
loff_t pos = ((loff_t)pos_high << 32) | pos_low;
- return do_compat_pwritev64(fd, vec, vlen, pos, 0);
+ return do_pwritev(fd, vec, vlen, pos, 0);
}
#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
- const struct compat_iovec __user *,vec,
+ const struct iovec __user *, vec,
unsigned long, vlen, loff_t, pos, rwf_t, flags)
{
if (pos == -1)
- return do_compat_writev(fd, vec, vlen, flags);
-
- return do_compat_pwritev64(fd, vec, vlen, pos, flags);
+ return do_writev(fd, vec, vlen, flags);
+ return do_pwritev(fd, vec, vlen, pos, flags);
}
#endif
COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
- const struct compat_iovec __user *,vec,
+ const struct iovec __user *,vec,
compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags)
{
loff_t pos = ((loff_t)pos_high << 32) | pos_low;
if (pos == -1)
- return do_compat_writev(fd, vec, vlen, flags);
-
- return do_compat_pwritev64(fd, vec, vlen, pos, flags);
+ return do_writev(fd, vec, vlen, flags);
+ return do_pwritev(fd, vec, vlen, pos, flags);
}
-
-#endif
+#endif /* CONFIG_COMPAT */
static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
size_t count, loff_t max)
#include <linux/security.h>
#include <linux/gfp.h>
#include <linux/socket.h>
-#include <linux/compat.h>
#include <linux/sched/signal.h>
#include "internal.h"
return 1;
}
+/* We know we have a pipe buffer, but maybe it's empty? */
+static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)
+{
+ unsigned int tail = pipe->tail;
+ unsigned int mask = pipe->ring_size - 1;
+ struct pipe_buffer *buf = &pipe->bufs[tail & mask];
+
+ if (unlikely(!buf->len)) {
+ pipe_buf_release(pipe, buf);
+ pipe->tail = tail+1;
+ return true;
+ }
+
+ return false;
+}
+
/**
* splice_from_pipe_next - wait for some data to splice from
* @pipe: pipe to splice from
if (signal_pending(current))
return -ERESTARTSYS;
+repeat:
while (pipe_empty(pipe->head, pipe->tail)) {
if (!pipe->writers)
return 0;
sd->need_wakeup = false;
}
- pipe_wait(pipe);
+ pipe_wait_readable(pipe);
}
+ if (eat_empty_buffer(pipe))
+ goto repeat;
+
return 1;
}
return -EAGAIN;
if (signal_pending(current))
return -ERESTARTSYS;
- pipe_wait(pipe);
+ pipe_wait_writable(pipe);
}
}
* Currently we punt and implement it as a normal copy, see pipe_to_user().
*
*/
-static long do_vmsplice(struct file *f, struct iov_iter *iter, unsigned int flags)
-{
- if (unlikely(flags & ~SPLICE_F_ALL))
- return -EINVAL;
-
- if (!iov_iter_count(iter))
- return 0;
-
- if (iov_iter_rw(iter) == WRITE)
- return vmsplice_to_pipe(f, iter, flags);
- else
- return vmsplice_to_user(f, iter, flags);
-}
-
SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,
unsigned long, nr_segs, unsigned int, flags)
{
struct fd f;
int type;
+ if (unlikely(flags & ~SPLICE_F_ALL))
+ return -EINVAL;
+
f = fdget(fd);
error = vmsplice_type(f, &type);
if (error)
error = import_iovec(type, uiov, nr_segs,
ARRAY_SIZE(iovstack), &iov, &iter);
- if (error >= 0) {
- error = do_vmsplice(f.file, &iter, flags);
- kfree(iov);
- }
- fdput(f);
- return error;
-}
-
-#ifdef CONFIG_COMPAT
-COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32,
- unsigned int, nr_segs, unsigned int, flags)
-{
- struct iovec iovstack[UIO_FASTIOV];
- struct iovec *iov = iovstack;
- struct iov_iter iter;
- ssize_t error;
- struct fd f;
- int type;
+ if (error < 0)
+ goto out_fdput;
- f = fdget(fd);
- error = vmsplice_type(f, &type);
- if (error)
- return error;
+ if (!iov_iter_count(&iter))
+ error = 0;
+ else if (iov_iter_rw(&iter) == WRITE)
+ error = vmsplice_to_pipe(f.file, &iter, flags);
+ else
+ error = vmsplice_to_user(f.file, &iter, flags);
- error = compat_import_iovec(type, iov32, nr_segs,
- ARRAY_SIZE(iovstack), &iov, &iter);
- if (error >= 0) {
- error = do_vmsplice(f.file, &iter, flags);
- kfree(iov);
- }
+ kfree(iov);
+out_fdput:
fdput(f);
return error;
}
-#endif
SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
int, fd_out, loff_t __user *, off_out,
ret = -EAGAIN;
break;
}
- pipe_wait(pipe);
+ pipe_wait_readable(pipe);
}
pipe_unlock(pipe);
ret = -ERESTARTSYS;
break;
}
- pipe_wait(pipe);
+ pipe_wait_writable(pipe);
}
pipe_unlock(pipe);
struct ubifs_inode *ui;
bool encrypted = false;
- if (IS_ENCRYPTED(dir)) {
- err = fscrypt_get_encryption_info(dir);
- if (err) {
- ubifs_err(c, "fscrypt_get_encryption_info failed: %i", err);
- return ERR_PTR(err);
- }
-
- if (!fscrypt_has_encryption_key(dir))
- return ERR_PTR(-EPERM);
-
- encrypted = true;
- }
-
inode = new_inode(c->vfs_sb);
ui = ubifs_inode(inode);
if (!inode)
current_time(inode);
inode->i_mapping->nrpages = 0;
+ err = fscrypt_prepare_new_inode(dir, inode, &encrypted);
+ if (err) {
+ ubifs_err(c, "fscrypt_prepare_new_inode failed: %i", err);
+ goto out_iput;
+ }
+
switch (mode & S_IFMT) {
case S_IFREG:
inode->i_mapping->a_ops = &ubifs_file_address_operations;
case S_IFBLK:
case S_IFCHR:
inode->i_op = &ubifs_file_inode_operations;
- encrypted = false;
break;
default:
BUG();
if (c->highest_inum >= INUM_WATERMARK) {
spin_unlock(&c->cnt_lock);
ubifs_err(c, "out of inode numbers");
- make_bad_inode(inode);
- iput(inode);
- return ERR_PTR(-EINVAL);
+ err = -EINVAL;
+ goto out_iput;
}
ubifs_warn(c, "running out of inode numbers (current %lu, max %u)",
(unsigned long)c->highest_inum, INUM_WATERMARK);
spin_unlock(&c->cnt_lock);
if (encrypted) {
- err = fscrypt_inherit_context(dir, inode, &encrypted, true);
+ err = fscrypt_set_context(inode, NULL);
if (err) {
- ubifs_err(c, "fscrypt_inherit_context failed: %i", err);
- make_bad_inode(inode);
- iput(inode);
- return ERR_PTR(err);
+ ubifs_err(c, "fscrypt_set_context failed: %i", err);
+ goto out_iput;
}
}
return inode;
+
+out_iput:
+ make_bad_inode(inode);
+ iput(inode);
+ return ERR_PTR(err);
}
static int dbg_check_name(const struct ubifs_info *c,
if (err)
return err;
- err = fscrypt_fname_alloc_buffer(dir, UBIFS_MAX_NLEN, &fstr);
+ err = fscrypt_fname_alloc_buffer(UBIFS_MAX_NLEN, &fstr);
if (err)
return err;
static __always_inline void
atomic_add(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic_add(i, v);
}
#define atomic_add atomic_add
static __always_inline int
atomic_add_return(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_add_return(i, v);
}
#define atomic_add_return atomic_add_return
static __always_inline int
atomic_add_return_acquire(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_add_return_acquire(i, v);
}
#define atomic_add_return_acquire atomic_add_return_acquire
static __always_inline int
atomic_add_return_release(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_add_return_release(i, v);
}
#define atomic_add_return_release atomic_add_return_release
static __always_inline int
atomic_add_return_relaxed(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_add_return_relaxed(i, v);
}
#define atomic_add_return_relaxed atomic_add_return_relaxed
static __always_inline int
atomic_fetch_add(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_add(i, v);
}
#define atomic_fetch_add atomic_fetch_add
static __always_inline int
atomic_fetch_add_acquire(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_add_acquire(i, v);
}
#define atomic_fetch_add_acquire atomic_fetch_add_acquire
static __always_inline int
atomic_fetch_add_release(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_add_release(i, v);
}
#define atomic_fetch_add_release atomic_fetch_add_release
static __always_inline int
atomic_fetch_add_relaxed(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_add_relaxed(i, v);
}
#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
static __always_inline void
atomic_sub(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic_sub(i, v);
}
#define atomic_sub atomic_sub
static __always_inline int
atomic_sub_return(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_sub_return(i, v);
}
#define atomic_sub_return atomic_sub_return
static __always_inline int
atomic_sub_return_acquire(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_sub_return_acquire(i, v);
}
#define atomic_sub_return_acquire atomic_sub_return_acquire
static __always_inline int
atomic_sub_return_release(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_sub_return_release(i, v);
}
#define atomic_sub_return_release atomic_sub_return_release
static __always_inline int
atomic_sub_return_relaxed(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_sub_return_relaxed(i, v);
}
#define atomic_sub_return_relaxed atomic_sub_return_relaxed
static __always_inline int
atomic_fetch_sub(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_sub(i, v);
}
#define atomic_fetch_sub atomic_fetch_sub
static __always_inline int
atomic_fetch_sub_acquire(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_sub_acquire(i, v);
}
#define atomic_fetch_sub_acquire atomic_fetch_sub_acquire
static __always_inline int
atomic_fetch_sub_release(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_sub_release(i, v);
}
#define atomic_fetch_sub_release atomic_fetch_sub_release
static __always_inline int
atomic_fetch_sub_relaxed(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_sub_relaxed(i, v);
}
#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
static __always_inline void
atomic_inc(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic_inc(v);
}
#define atomic_inc atomic_inc
static __always_inline int
atomic_inc_return(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_inc_return(v);
}
#define atomic_inc_return atomic_inc_return
static __always_inline int
atomic_inc_return_acquire(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_inc_return_acquire(v);
}
#define atomic_inc_return_acquire atomic_inc_return_acquire
static __always_inline int
atomic_inc_return_release(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_inc_return_release(v);
}
#define atomic_inc_return_release atomic_inc_return_release
static __always_inline int
atomic_inc_return_relaxed(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_inc_return_relaxed(v);
}
#define atomic_inc_return_relaxed atomic_inc_return_relaxed
static __always_inline int
atomic_fetch_inc(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_inc(v);
}
#define atomic_fetch_inc atomic_fetch_inc
static __always_inline int
atomic_fetch_inc_acquire(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_inc_acquire(v);
}
#define atomic_fetch_inc_acquire atomic_fetch_inc_acquire
static __always_inline int
atomic_fetch_inc_release(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_inc_release(v);
}
#define atomic_fetch_inc_release atomic_fetch_inc_release
static __always_inline int
atomic_fetch_inc_relaxed(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_inc_relaxed(v);
}
#define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed
static __always_inline void
atomic_dec(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic_dec(v);
}
#define atomic_dec atomic_dec
static __always_inline int
atomic_dec_return(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_dec_return(v);
}
#define atomic_dec_return atomic_dec_return
static __always_inline int
atomic_dec_return_acquire(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_dec_return_acquire(v);
}
#define atomic_dec_return_acquire atomic_dec_return_acquire
static __always_inline int
atomic_dec_return_release(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_dec_return_release(v);
}
#define atomic_dec_return_release atomic_dec_return_release
static __always_inline int
atomic_dec_return_relaxed(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_dec_return_relaxed(v);
}
#define atomic_dec_return_relaxed atomic_dec_return_relaxed
static __always_inline int
atomic_fetch_dec(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_dec(v);
}
#define atomic_fetch_dec atomic_fetch_dec
static __always_inline int
atomic_fetch_dec_acquire(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_dec_acquire(v);
}
#define atomic_fetch_dec_acquire atomic_fetch_dec_acquire
static __always_inline int
atomic_fetch_dec_release(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_dec_release(v);
}
#define atomic_fetch_dec_release atomic_fetch_dec_release
static __always_inline int
atomic_fetch_dec_relaxed(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_dec_relaxed(v);
}
#define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed
static __always_inline void
atomic_and(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic_and(i, v);
}
#define atomic_and atomic_and
static __always_inline int
atomic_fetch_and(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_and(i, v);
}
#define atomic_fetch_and atomic_fetch_and
static __always_inline int
atomic_fetch_and_acquire(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_and_acquire(i, v);
}
#define atomic_fetch_and_acquire atomic_fetch_and_acquire
static __always_inline int
atomic_fetch_and_release(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_and_release(i, v);
}
#define atomic_fetch_and_release atomic_fetch_and_release
static __always_inline int
atomic_fetch_and_relaxed(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_and_relaxed(i, v);
}
#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
static __always_inline void
atomic_andnot(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic_andnot(i, v);
}
#define atomic_andnot atomic_andnot
static __always_inline int
atomic_fetch_andnot(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_andnot(i, v);
}
#define atomic_fetch_andnot atomic_fetch_andnot
static __always_inline int
atomic_fetch_andnot_acquire(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_andnot_acquire(i, v);
}
#define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
static __always_inline int
atomic_fetch_andnot_release(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_andnot_release(i, v);
}
#define atomic_fetch_andnot_release atomic_fetch_andnot_release
static __always_inline int
atomic_fetch_andnot_relaxed(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_andnot_relaxed(i, v);
}
#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed
static __always_inline void
atomic_or(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic_or(i, v);
}
#define atomic_or atomic_or
static __always_inline int
atomic_fetch_or(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_or(i, v);
}
#define atomic_fetch_or atomic_fetch_or
static __always_inline int
atomic_fetch_or_acquire(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_or_acquire(i, v);
}
#define atomic_fetch_or_acquire atomic_fetch_or_acquire
static __always_inline int
atomic_fetch_or_release(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_or_release(i, v);
}
#define atomic_fetch_or_release atomic_fetch_or_release
static __always_inline int
atomic_fetch_or_relaxed(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_or_relaxed(i, v);
}
#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed
static __always_inline void
atomic_xor(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic_xor(i, v);
}
#define atomic_xor atomic_xor
static __always_inline int
atomic_fetch_xor(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_xor(i, v);
}
#define atomic_fetch_xor atomic_fetch_xor
static __always_inline int
atomic_fetch_xor_acquire(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_xor_acquire(i, v);
}
#define atomic_fetch_xor_acquire atomic_fetch_xor_acquire
static __always_inline int
atomic_fetch_xor_release(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_xor_release(i, v);
}
#define atomic_fetch_xor_release atomic_fetch_xor_release
static __always_inline int
atomic_fetch_xor_relaxed(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_xor_relaxed(i, v);
}
#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
static __always_inline int
atomic_xchg(atomic_t *v, int i)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_xchg(v, i);
}
#define atomic_xchg atomic_xchg
static __always_inline int
atomic_xchg_acquire(atomic_t *v, int i)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_xchg_acquire(v, i);
}
#define atomic_xchg_acquire atomic_xchg_acquire
static __always_inline int
atomic_xchg_release(atomic_t *v, int i)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_xchg_release(v, i);
}
#define atomic_xchg_release atomic_xchg_release
static __always_inline int
atomic_xchg_relaxed(atomic_t *v, int i)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_xchg_relaxed(v, i);
}
#define atomic_xchg_relaxed atomic_xchg_relaxed
static __always_inline int
atomic_cmpxchg(atomic_t *v, int old, int new)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_cmpxchg(v, old, new);
}
#define atomic_cmpxchg atomic_cmpxchg
static __always_inline int
atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_cmpxchg_acquire(v, old, new);
}
#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire
static __always_inline int
atomic_cmpxchg_release(atomic_t *v, int old, int new)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_cmpxchg_release(v, old, new);
}
#define atomic_cmpxchg_release atomic_cmpxchg_release
static __always_inline int
atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_cmpxchg_relaxed(v, old, new);
}
#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed
static __always_inline bool
atomic_try_cmpxchg(atomic_t *v, int *old, int new)
{
- instrument_atomic_write(v, sizeof(*v));
- instrument_atomic_write(old, sizeof(*old));
+ instrument_atomic_read_write(v, sizeof(*v));
+ instrument_atomic_read_write(old, sizeof(*old));
return arch_atomic_try_cmpxchg(v, old, new);
}
#define atomic_try_cmpxchg atomic_try_cmpxchg
static __always_inline bool
atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
{
- instrument_atomic_write(v, sizeof(*v));
- instrument_atomic_write(old, sizeof(*old));
+ instrument_atomic_read_write(v, sizeof(*v));
+ instrument_atomic_read_write(old, sizeof(*old));
return arch_atomic_try_cmpxchg_acquire(v, old, new);
}
#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire
static __always_inline bool
atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
{
- instrument_atomic_write(v, sizeof(*v));
- instrument_atomic_write(old, sizeof(*old));
+ instrument_atomic_read_write(v, sizeof(*v));
+ instrument_atomic_read_write(old, sizeof(*old));
return arch_atomic_try_cmpxchg_release(v, old, new);
}
#define atomic_try_cmpxchg_release atomic_try_cmpxchg_release
static __always_inline bool
atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
{
- instrument_atomic_write(v, sizeof(*v));
- instrument_atomic_write(old, sizeof(*old));
+ instrument_atomic_read_write(v, sizeof(*v));
+ instrument_atomic_read_write(old, sizeof(*old));
return arch_atomic_try_cmpxchg_relaxed(v, old, new);
}
#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed
static __always_inline bool
atomic_sub_and_test(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_sub_and_test(i, v);
}
#define atomic_sub_and_test atomic_sub_and_test
static __always_inline bool
atomic_dec_and_test(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_dec_and_test(v);
}
#define atomic_dec_and_test atomic_dec_and_test
static __always_inline bool
atomic_inc_and_test(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_inc_and_test(v);
}
#define atomic_inc_and_test atomic_inc_and_test
static __always_inline bool
atomic_add_negative(int i, atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_add_negative(i, v);
}
#define atomic_add_negative atomic_add_negative
static __always_inline int
atomic_fetch_add_unless(atomic_t *v, int a, int u)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_fetch_add_unless(v, a, u);
}
#define atomic_fetch_add_unless atomic_fetch_add_unless
static __always_inline bool
atomic_add_unless(atomic_t *v, int a, int u)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_add_unless(v, a, u);
}
#define atomic_add_unless atomic_add_unless
static __always_inline bool
atomic_inc_not_zero(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_inc_not_zero(v);
}
#define atomic_inc_not_zero atomic_inc_not_zero
static __always_inline bool
atomic_inc_unless_negative(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_inc_unless_negative(v);
}
#define atomic_inc_unless_negative atomic_inc_unless_negative
static __always_inline bool
atomic_dec_unless_positive(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_dec_unless_positive(v);
}
#define atomic_dec_unless_positive atomic_dec_unless_positive
static __always_inline int
atomic_dec_if_positive(atomic_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic_dec_if_positive(v);
}
#define atomic_dec_if_positive atomic_dec_if_positive
static __always_inline void
atomic64_add(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic64_add(i, v);
}
#define atomic64_add atomic64_add
static __always_inline s64
atomic64_add_return(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_add_return(i, v);
}
#define atomic64_add_return atomic64_add_return
static __always_inline s64
atomic64_add_return_acquire(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_add_return_acquire(i, v);
}
#define atomic64_add_return_acquire atomic64_add_return_acquire
static __always_inline s64
atomic64_add_return_release(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_add_return_release(i, v);
}
#define atomic64_add_return_release atomic64_add_return_release
static __always_inline s64
atomic64_add_return_relaxed(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_add_return_relaxed(i, v);
}
#define atomic64_add_return_relaxed atomic64_add_return_relaxed
static __always_inline s64
atomic64_fetch_add(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_add(i, v);
}
#define atomic64_fetch_add atomic64_fetch_add
static __always_inline s64
atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_add_acquire(i, v);
}
#define atomic64_fetch_add_acquire atomic64_fetch_add_acquire
static __always_inline s64
atomic64_fetch_add_release(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_add_release(i, v);
}
#define atomic64_fetch_add_release atomic64_fetch_add_release
static __always_inline s64
atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_add_relaxed(i, v);
}
#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
static __always_inline void
atomic64_sub(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic64_sub(i, v);
}
#define atomic64_sub atomic64_sub
static __always_inline s64
atomic64_sub_return(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_sub_return(i, v);
}
#define atomic64_sub_return atomic64_sub_return
static __always_inline s64
atomic64_sub_return_acquire(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_sub_return_acquire(i, v);
}
#define atomic64_sub_return_acquire atomic64_sub_return_acquire
static __always_inline s64
atomic64_sub_return_release(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_sub_return_release(i, v);
}
#define atomic64_sub_return_release atomic64_sub_return_release
static __always_inline s64
atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_sub_return_relaxed(i, v);
}
#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
static __always_inline s64
atomic64_fetch_sub(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_sub(i, v);
}
#define atomic64_fetch_sub atomic64_fetch_sub
static __always_inline s64
atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_sub_acquire(i, v);
}
#define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire
static __always_inline s64
atomic64_fetch_sub_release(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_sub_release(i, v);
}
#define atomic64_fetch_sub_release atomic64_fetch_sub_release
static __always_inline s64
atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_sub_relaxed(i, v);
}
#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
static __always_inline void
atomic64_inc(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic64_inc(v);
}
#define atomic64_inc atomic64_inc
static __always_inline s64
atomic64_inc_return(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_inc_return(v);
}
#define atomic64_inc_return atomic64_inc_return
static __always_inline s64
atomic64_inc_return_acquire(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_inc_return_acquire(v);
}
#define atomic64_inc_return_acquire atomic64_inc_return_acquire
static __always_inline s64
atomic64_inc_return_release(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_inc_return_release(v);
}
#define atomic64_inc_return_release atomic64_inc_return_release
static __always_inline s64
atomic64_inc_return_relaxed(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_inc_return_relaxed(v);
}
#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
static __always_inline s64
atomic64_fetch_inc(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_inc(v);
}
#define atomic64_fetch_inc atomic64_fetch_inc
static __always_inline s64
atomic64_fetch_inc_acquire(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_inc_acquire(v);
}
#define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire
static __always_inline s64
atomic64_fetch_inc_release(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_inc_release(v);
}
#define atomic64_fetch_inc_release atomic64_fetch_inc_release
static __always_inline s64
atomic64_fetch_inc_relaxed(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_inc_relaxed(v);
}
#define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed
static __always_inline void
atomic64_dec(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic64_dec(v);
}
#define atomic64_dec atomic64_dec
static __always_inline s64
atomic64_dec_return(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_dec_return(v);
}
#define atomic64_dec_return atomic64_dec_return
static __always_inline s64
atomic64_dec_return_acquire(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_dec_return_acquire(v);
}
#define atomic64_dec_return_acquire atomic64_dec_return_acquire
static __always_inline s64
atomic64_dec_return_release(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_dec_return_release(v);
}
#define atomic64_dec_return_release atomic64_dec_return_release
static __always_inline s64
atomic64_dec_return_relaxed(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_dec_return_relaxed(v);
}
#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
static __always_inline s64
atomic64_fetch_dec(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_dec(v);
}
#define atomic64_fetch_dec atomic64_fetch_dec
static __always_inline s64
atomic64_fetch_dec_acquire(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_dec_acquire(v);
}
#define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire
static __always_inline s64
atomic64_fetch_dec_release(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_dec_release(v);
}
#define atomic64_fetch_dec_release atomic64_fetch_dec_release
static __always_inline s64
atomic64_fetch_dec_relaxed(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_dec_relaxed(v);
}
#define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed
static __always_inline void
atomic64_and(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic64_and(i, v);
}
#define atomic64_and atomic64_and
static __always_inline s64
atomic64_fetch_and(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_and(i, v);
}
#define atomic64_fetch_and atomic64_fetch_and
static __always_inline s64
atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_and_acquire(i, v);
}
#define atomic64_fetch_and_acquire atomic64_fetch_and_acquire
static __always_inline s64
atomic64_fetch_and_release(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_and_release(i, v);
}
#define atomic64_fetch_and_release atomic64_fetch_and_release
static __always_inline s64
atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_and_relaxed(i, v);
}
#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
static __always_inline void
atomic64_andnot(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic64_andnot(i, v);
}
#define atomic64_andnot atomic64_andnot
static __always_inline s64
atomic64_fetch_andnot(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_andnot(i, v);
}
#define atomic64_fetch_andnot atomic64_fetch_andnot
static __always_inline s64
atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_andnot_acquire(i, v);
}
#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
static __always_inline s64
atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_andnot_release(i, v);
}
#define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
static __always_inline s64
atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_andnot_relaxed(i, v);
}
#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed
static __always_inline void
atomic64_or(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic64_or(i, v);
}
#define atomic64_or atomic64_or
static __always_inline s64
atomic64_fetch_or(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_or(i, v);
}
#define atomic64_fetch_or atomic64_fetch_or
static __always_inline s64
atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_or_acquire(i, v);
}
#define atomic64_fetch_or_acquire atomic64_fetch_or_acquire
static __always_inline s64
atomic64_fetch_or_release(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_or_release(i, v);
}
#define atomic64_fetch_or_release atomic64_fetch_or_release
static __always_inline s64
atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_or_relaxed(i, v);
}
#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed
static __always_inline void
atomic64_xor(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
arch_atomic64_xor(i, v);
}
#define atomic64_xor atomic64_xor
static __always_inline s64
atomic64_fetch_xor(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_xor(i, v);
}
#define atomic64_fetch_xor atomic64_fetch_xor
static __always_inline s64
atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_xor_acquire(i, v);
}
#define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire
static __always_inline s64
atomic64_fetch_xor_release(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_xor_release(i, v);
}
#define atomic64_fetch_xor_release atomic64_fetch_xor_release
static __always_inline s64
atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_xor_relaxed(i, v);
}
#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
static __always_inline s64
atomic64_xchg(atomic64_t *v, s64 i)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_xchg(v, i);
}
#define atomic64_xchg atomic64_xchg
static __always_inline s64
atomic64_xchg_acquire(atomic64_t *v, s64 i)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_xchg_acquire(v, i);
}
#define atomic64_xchg_acquire atomic64_xchg_acquire
static __always_inline s64
atomic64_xchg_release(atomic64_t *v, s64 i)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_xchg_release(v, i);
}
#define atomic64_xchg_release atomic64_xchg_release
static __always_inline s64
atomic64_xchg_relaxed(atomic64_t *v, s64 i)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_xchg_relaxed(v, i);
}
#define atomic64_xchg_relaxed atomic64_xchg_relaxed
static __always_inline s64
atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_cmpxchg(v, old, new);
}
#define atomic64_cmpxchg atomic64_cmpxchg
static __always_inline s64
atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_cmpxchg_acquire(v, old, new);
}
#define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire
static __always_inline s64
atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_cmpxchg_release(v, old, new);
}
#define atomic64_cmpxchg_release atomic64_cmpxchg_release
static __always_inline s64
atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_cmpxchg_relaxed(v, old, new);
}
#define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed
static __always_inline bool
atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
{
- instrument_atomic_write(v, sizeof(*v));
- instrument_atomic_write(old, sizeof(*old));
+ instrument_atomic_read_write(v, sizeof(*v));
+ instrument_atomic_read_write(old, sizeof(*old));
return arch_atomic64_try_cmpxchg(v, old, new);
}
#define atomic64_try_cmpxchg atomic64_try_cmpxchg
static __always_inline bool
atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
{
- instrument_atomic_write(v, sizeof(*v));
- instrument_atomic_write(old, sizeof(*old));
+ instrument_atomic_read_write(v, sizeof(*v));
+ instrument_atomic_read_write(old, sizeof(*old));
return arch_atomic64_try_cmpxchg_acquire(v, old, new);
}
#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire
static __always_inline bool
atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
{
- instrument_atomic_write(v, sizeof(*v));
- instrument_atomic_write(old, sizeof(*old));
+ instrument_atomic_read_write(v, sizeof(*v));
+ instrument_atomic_read_write(old, sizeof(*old));
return arch_atomic64_try_cmpxchg_release(v, old, new);
}
#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release
static __always_inline bool
atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
{
- instrument_atomic_write(v, sizeof(*v));
- instrument_atomic_write(old, sizeof(*old));
+ instrument_atomic_read_write(v, sizeof(*v));
+ instrument_atomic_read_write(old, sizeof(*old));
return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
}
#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed
static __always_inline bool
atomic64_sub_and_test(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_sub_and_test(i, v);
}
#define atomic64_sub_and_test atomic64_sub_and_test
static __always_inline bool
atomic64_dec_and_test(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_dec_and_test(v);
}
#define atomic64_dec_and_test atomic64_dec_and_test
static __always_inline bool
atomic64_inc_and_test(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_inc_and_test(v);
}
#define atomic64_inc_and_test atomic64_inc_and_test
static __always_inline bool
atomic64_add_negative(s64 i, atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_add_negative(i, v);
}
#define atomic64_add_negative atomic64_add_negative
static __always_inline s64
atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_fetch_add_unless(v, a, u);
}
#define atomic64_fetch_add_unless atomic64_fetch_add_unless
static __always_inline bool
atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_add_unless(v, a, u);
}
#define atomic64_add_unless atomic64_add_unless
static __always_inline bool
atomic64_inc_not_zero(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_inc_not_zero(v);
}
#define atomic64_inc_not_zero atomic64_inc_not_zero
static __always_inline bool
atomic64_inc_unless_negative(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_inc_unless_negative(v);
}
#define atomic64_inc_unless_negative atomic64_inc_unless_negative
static __always_inline bool
atomic64_dec_unless_positive(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_dec_unless_positive(v);
}
#define atomic64_dec_unless_positive atomic64_dec_unless_positive
static __always_inline s64
atomic64_dec_if_positive(atomic64_t *v)
{
- instrument_atomic_write(v, sizeof(*v));
+ instrument_atomic_read_write(v, sizeof(*v));
return arch_atomic64_dec_if_positive(v);
}
#define atomic64_dec_if_positive atomic64_dec_if_positive
})
#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */
-// 89bf97f3a7509b740845e51ddf31055b48a81f40
+// 9d5e6a315fb1335d02f0ccd3655a91c3dafcc63e
*/
static inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
{
- instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
+ instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
return arch_test_and_set_bit(nr, addr);
}
*/
static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
{
- instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
+ instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
return arch_test_and_clear_bit(nr, addr);
}
*/
static inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
{
- instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
+ instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
return arch_test_and_change_bit(nr, addr);
}
*/
static inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr)
{
- instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long));
+ instrument_atomic_read_write(addr + BIT_WORD(nr), sizeof(long));
return arch_test_and_set_bit_lock(nr, addr);
}
arch___change_bit(nr, addr);
}
+static inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
+{
+ if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
+ /*
+ * We treat non-atomic read-write bitops a little more special.
+ * Given the operations here only modify a single bit, assuming
+ * non-atomicity of the writer is sufficient may be reasonable
+ * for certain usage (and follows the permissible nature of the
+ * assume-plain-writes-atomic rule):
+ * 1. report read-modify-write races -> check read;
+ * 2. do not report races with marked readers, but do report
+ * races with unmarked readers -> check "atomic" write.
+ */
+ kcsan_check_read(addr + BIT_WORD(nr), sizeof(long));
+ /*
+ * Use generic write instrumentation, in case other sanitizers
+ * or tools are enabled alongside KCSAN.
+ */
+ instrument_write(addr + BIT_WORD(nr), sizeof(long));
+ } else {
+ instrument_read_write(addr + BIT_WORD(nr), sizeof(long));
+ }
+}
+
/**
* __test_and_set_bit - Set a bit and return its old value
* @nr: Bit to set
*/
static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
{
- instrument_write(addr + BIT_WORD(nr), sizeof(long));
+ __instrument_read_write_bitop(nr, addr);
return arch___test_and_set_bit(nr, addr);
}
*/
static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
{
- instrument_write(addr + BIT_WORD(nr), sizeof(long));
+ __instrument_read_write_bitop(nr, addr);
return arch___test_and_clear_bit(nr, addr);
}
*/
static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
{
- instrument_write(addr + BIT_WORD(nr), sizeof(long));
+ __instrument_read_write_bitop(nr, addr);
return arch___test_and_change_bit(nr, addr);
}
*/
extern __wsum csum_partial(const void *buff, int len, __wsum sum);
-/*
- * the same as csum_partial, but copies from src while it
- * checksums
- *
- * here even more important to align src and dst on a 32-bit (or even
- * better 64-bit) boundary
- */
-#ifndef csum_partial_copy_nocheck
-__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len,
- __wsum sum);
-#endif
-
#ifndef ip_fast_csum
/*
* This is a version of ip_compute_csum() optimized for IP headers,
typedef u32 compat_uptr_t;
typedef u32 compat_aio_context_t;
+#ifdef CONFIG_COMPAT_FOR_U64_ALIGNMENT
+typedef s64 __attribute__((aligned(4))) compat_s64;
+typedef u64 __attribute__((aligned(4))) compat_u64;
+#else
+typedef s64 compat_s64;
+typedef u64 compat_u64;
+#endif
+
#endif
*
* STABS_DEBUG
* DWARF_DEBUG
+ * ELF_DETAILS
*
* DISCARDS // must be the last
* }
KEEP(*(__jump_table)) \
__stop___jump_table = .;
+#define STATIC_CALL_DATA \
+ . = ALIGN(8); \
+ __start_static_call_sites = .; \
+ KEEP(*(.static_call_sites)) \
+ __stop_static_call_sites = .;
+
/*
* Allow architectures to handle ro_after_init data on their
* own by defining an empty RO_AFTER_INIT_DATA.
__start_ro_after_init = .; \
*(.data..ro_after_init) \
JUMP_TABLE_DATA \
+ STATIC_CALL_DATA \
__end_ro_after_init = .;
#endif
*/
#define TEXT_TEXT \
ALIGN_FUNCTION(); \
- *(.text.hot TEXT_MAIN .text.fixup .text.unlikely) \
+ *(.text.hot .text.hot.*) \
+ *(TEXT_MAIN .text.fixup) \
+ *(.text.unlikely .text.unlikely.*) \
+ *(.text.unknown .text.unknown.*) \
NOINSTR_TEXT \
*(.text..refcount) \
*(.ref.text) \
*(.softirqentry.text) \
__softirqentry_text_end = .;
+#define STATIC_CALL_TEXT \
+ ALIGN_FUNCTION(); \
+ __static_call_text_start = .; \
+ *(.static_call.text) \
+ __static_call_text_end = .;
+
/* Section used for early init (in .S files) */
#define HEAD_TEXT KEEP(*(.head.text))
#define BTF \
.BTF : AT(ADDR(.BTF) - LOAD_OFFSET) { \
__start_BTF = .; \
- *(.BTF) \
+ KEEP(*(.BTF)) \
__stop_BTF = .; \
} \
. = ALIGN(4); \
.debug_macro 0 : { *(.debug_macro) } \
.debug_addr 0 : { *(.debug_addr) }
- /* Stabs debugging sections. */
+/* Stabs debugging sections. */
#define STABS_DEBUG \
.stab 0 : { *(.stab) } \
.stabstr 0 : { *(.stabstr) } \
.stab.excl 0 : { *(.stab.excl) } \
.stab.exclstr 0 : { *(.stab.exclstr) } \
.stab.index 0 : { *(.stab.index) } \
- .stab.indexstr 0 : { *(.stab.indexstr) } \
- .comment 0 : { *(.comment) }
+ .stab.indexstr 0 : { *(.stab.indexstr) }
+
+/* Required sections not related to debugging. */
+#define ELF_DETAILS \
+ .comment 0 : { *(.comment) } \
+ .symtab 0 : { *(.symtab) } \
+ .strtab 0 : { *(.strtab) } \
+ .shstrtab 0 : { *(.shstrtab) }
#ifdef CONFIG_GENERIC_BUG
#define BUG_TABLE \
EXIT_DATA
#endif
+/*
+ * Clang's -fsanitize=kernel-address and -fsanitize=thread produce
+ * unwanted sections (.eh_frame and .init_array.*), but
+ * CONFIG_CONSTRUCTORS wants to keep any .init_array.* sections.
+ * https://bugs.llvm.org/show_bug.cgi?id=46478
+ */
+#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN)
+# ifdef CONFIG_CONSTRUCTORS
+# define SANITIZER_DISCARDS \
+ *(.eh_frame)
+# else
+# define SANITIZER_DISCARDS \
+ *(.init_array) *(.init_array.*) \
+ *(.eh_frame)
+# endif
+#else
+# define SANITIZER_DISCARDS
+#endif
+
+#define COMMON_DISCARDS \
+ SANITIZER_DISCARDS \
+ *(.discard) \
+ *(.discard.*) \
+ *(.modinfo) \
+ /* ld.bfd warns about .gnu.version* even when not emitted */ \
+ *(.gnu.version*) \
+
#define DISCARDS \
/DISCARD/ : { \
EXIT_DISCARDS \
EXIT_CALL \
- *(.discard) \
- *(.discard.*) \
- *(.modinfo) \
+ COMMON_DISCARDS \
}
/**
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __CLKSOURCE_TIMER_SP804_H
-#define __CLKSOURCE_TIMER_SP804_H
-
-struct clk;
-
-int __sp804_clocksource_and_sched_clock_init(void __iomem *,
- const char *, struct clk *, int);
-int __sp804_clockevents_init(void __iomem *, unsigned int,
- struct clk *, const char *);
-void sp804_timer_disable(void __iomem *);
-
-static inline void sp804_clocksource_init(void __iomem *base, const char *name)
-{
- __sp804_clocksource_and_sched_clock_init(base, name, NULL, 0);
-}
-
-static inline void sp804_clocksource_and_sched_clock_init(void __iomem *base,
- const char *name)
-{
- __sp804_clocksource_and_sched_clock_init(base, name, NULL, 1);
-}
-
-static inline void sp804_clockevents_init(void __iomem *base, unsigned int irq, const char *name)
-{
- __sp804_clockevents_init(base, irq, NULL, name);
-
-}
-#endif
#include <linux/crypto.h>
#include <linux/list.h>
#include <linux/kernel.h>
-#include <linux/skbuff.h>
/*
* Maximum values for blocksize and alignmask, used to allocate
struct module;
struct rtattr;
struct seq_file;
+struct sk_buff;
struct crypto_type {
unsigned int (*ctxsize)(struct crypto_alg *alg, u32 type, u32 mask);
return __crypto_memneq(a, b, size) != 0UL ? 1 : 0;
}
-static inline void crypto_yield(u32 flags)
-{
- if (flags & CRYPTO_TFM_REQ_MAY_SLEEP)
- cond_resched();
-}
-
int crypto_register_notifier(struct notifier_block *nb);
int crypto_unregister_notifier(struct notifier_block *nb);
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * CBC: Cipher Block Chaining mode
- *
- * Copyright (c) 2016 Herbert Xu <herbert@gondor.apana.org.au>
- */
-
-#ifndef _CRYPTO_CBC_H
-#define _CRYPTO_CBC_H
-
-#include <crypto/internal/skcipher.h>
-#include <linux/string.h>
-#include <linux/types.h>
-
-static inline int crypto_cbc_encrypt_segment(
- struct skcipher_walk *walk, struct crypto_skcipher *tfm,
- void (*fn)(struct crypto_skcipher *, const u8 *, u8 *))
-{
- unsigned int bsize = crypto_skcipher_blocksize(tfm);
- unsigned int nbytes = walk->nbytes;
- u8 *src = walk->src.virt.addr;
- u8 *dst = walk->dst.virt.addr;
- u8 *iv = walk->iv;
-
- do {
- crypto_xor(iv, src, bsize);
- fn(tfm, iv, dst);
- memcpy(iv, dst, bsize);
-
- src += bsize;
- dst += bsize;
- } while ((nbytes -= bsize) >= bsize);
-
- return nbytes;
-}
-
-static inline int crypto_cbc_encrypt_inplace(
- struct skcipher_walk *walk, struct crypto_skcipher *tfm,
- void (*fn)(struct crypto_skcipher *, const u8 *, u8 *))
-{
- unsigned int bsize = crypto_skcipher_blocksize(tfm);
- unsigned int nbytes = walk->nbytes;
- u8 *src = walk->src.virt.addr;
- u8 *iv = walk->iv;
-
- do {
- crypto_xor(src, iv, bsize);
- fn(tfm, src, src);
- iv = src;
-
- src += bsize;
- } while ((nbytes -= bsize) >= bsize);
-
- memcpy(walk->iv, iv, bsize);
-
- return nbytes;
-}
-
-static inline int crypto_cbc_encrypt_walk(struct skcipher_request *req,
- void (*fn)(struct crypto_skcipher *,
- const u8 *, u8 *))
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct skcipher_walk walk;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
-
- while (walk.nbytes) {
- if (walk.src.virt.addr == walk.dst.virt.addr)
- err = crypto_cbc_encrypt_inplace(&walk, tfm, fn);
- else
- err = crypto_cbc_encrypt_segment(&walk, tfm, fn);
- err = skcipher_walk_done(&walk, err);
- }
-
- return err;
-}
-
-static inline int crypto_cbc_decrypt_segment(
- struct skcipher_walk *walk, struct crypto_skcipher *tfm,
- void (*fn)(struct crypto_skcipher *, const u8 *, u8 *))
-{
- unsigned int bsize = crypto_skcipher_blocksize(tfm);
- unsigned int nbytes = walk->nbytes;
- u8 *src = walk->src.virt.addr;
- u8 *dst = walk->dst.virt.addr;
- u8 *iv = walk->iv;
-
- do {
- fn(tfm, src, dst);
- crypto_xor(dst, iv, bsize);
- iv = src;
-
- src += bsize;
- dst += bsize;
- } while ((nbytes -= bsize) >= bsize);
-
- memcpy(walk->iv, iv, bsize);
-
- return nbytes;
-}
-
-static inline int crypto_cbc_decrypt_inplace(
- struct skcipher_walk *walk, struct crypto_skcipher *tfm,
- void (*fn)(struct crypto_skcipher *, const u8 *, u8 *))
-{
- unsigned int bsize = crypto_skcipher_blocksize(tfm);
- unsigned int nbytes = walk->nbytes;
- u8 *src = walk->src.virt.addr;
- u8 last_iv[MAX_CIPHER_BLOCKSIZE];
-
- /* Start of the last block. */
- src += nbytes - (nbytes & (bsize - 1)) - bsize;
- memcpy(last_iv, src, bsize);
-
- for (;;) {
- fn(tfm, src, src);
- if ((nbytes -= bsize) < bsize)
- break;
- crypto_xor(src, src - bsize, bsize);
- src -= bsize;
- }
-
- crypto_xor(src, walk->iv, bsize);
- memcpy(walk->iv, last_iv, bsize);
-
- return nbytes;
-}
-
-static inline int crypto_cbc_decrypt_blocks(
- struct skcipher_walk *walk, struct crypto_skcipher *tfm,
- void (*fn)(struct crypto_skcipher *, const u8 *, u8 *))
-{
- if (walk->src.virt.addr == walk->dst.virt.addr)
- return crypto_cbc_decrypt_inplace(walk, tfm, fn);
- else
- return crypto_cbc_decrypt_segment(walk, tfm, fn);
-}
-
-#endif /* _CRYPTO_CBC_H */
void *__ctx[] CRYPTO_MINALIGN_ATTR;
};
-#define AHASH_REQUEST_ON_STACK(name, ahash) \
- char __##name##_desc[sizeof(struct ahash_request) + \
- crypto_ahash_reqsize(ahash)] CRYPTO_MINALIGN_ATTR; \
- struct ahash_request *name = (void *)__##name##_desc
-
/**
* struct ahash_alg - asynchronous message digest definition
* @init: **[mandatory]** Initialize the transformation context. Intended only to initialize the
* data so the transformation can continue from this point onward. No
* data processing happens at this point. Driver must not use
* req->result.
+ * @init_tfm: Initialize the cryptographic transformation object.
+ * This function is called only once at the instantiation
+ * time, right after the transformation context was
+ * allocated. In case the cryptographic hardware has
+ * some special requirements which need to be handled
+ * by software, this function shall check for the precise
+ * requirement of the transformation and put any software
+ * fallbacks in place.
+ * @exit_tfm: Deinitialize the cryptographic transformation object.
+ * This is a counterpart to @init_tfm, used to remove
+ * various changes set in @init_tfm.
* @halg: see struct hash_alg_common
*/
struct ahash_alg {
int (*import)(struct ahash_request *req, const void *in);
int (*setkey)(struct crypto_ahash *tfm, const u8 *key,
unsigned int keylen);
+ int (*init_tfm)(struct crypto_ahash *tfm);
+ void (*exit_tfm)(struct crypto_ahash *tfm);
struct hash_alg_common halg;
};
void *(*bind)(const char *name, u32 type, u32 mask);
void (*release)(void *private);
int (*setkey)(void *private, const u8 *key, unsigned int keylen);
+ int (*setentropy)(void *private, sockptr_t entropy, unsigned int len);
int (*accept)(void *private, struct sock *sk);
int (*accept_nokey)(void *private, struct sock *sk);
int (*setauthsize)(void *private, unsigned int authsize);
int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err);
int crypto_hash_walk_first(struct ahash_request *req,
struct crypto_hash_walk *walk);
-int crypto_ahash_walk_first(struct ahash_request *req,
- struct crypto_hash_walk *walk);
-
-static inline int crypto_ahash_walk_done(struct crypto_hash_walk *walk,
- int err)
-{
- return crypto_hash_walk_done(walk, err);
-}
static inline int crypto_hash_walk_last(struct crypto_hash_walk *walk)
{
return !(walk->entrylen | walk->total);
}
-static inline int crypto_ahash_walk_last(struct crypto_hash_walk *walk)
-{
- return crypto_hash_walk_last(walk);
-}
-
int crypto_register_ahash(struct ahash_alg *alg);
void crypto_unregister_ahash(struct ahash_alg *alg);
int crypto_register_ahashes(struct ahash_alg *algs, int count);
return container_of(inst, struct ahash_instance, s.base);
}
+static inline struct ahash_instance *ahash_alg_instance(
+ struct crypto_ahash *ahash)
+{
+ return ahash_instance(crypto_tfm_alg_instance(&ahash->base));
+}
+
static inline void *ahash_instance_ctx(struct ahash_instance *inst)
{
return crypto_instance_ctx(ahash_crypto_instance(inst));
#include <linux/keyctl.h>
#include <linux/oid_registry.h>
+#include <crypto/akcipher.h>
/*
* Cryptographic data for the public-key subtype of the asymmetric key type.
const char *pkey_algo;
const char *hash_algo;
const char *encoding;
+ const void *data;
+ unsigned int data_size;
};
extern void public_key_signature_free(struct public_key_signature *sig);
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * sm2.h - SM2 asymmetric public-key algorithm
+ * as specified by OSCCA GM/T 0003.1-2012 -- 0003.5-2012 SM2 and
+ * described at https://tools.ietf.org/html/draft-shen-sm2-ecdsa-02
+ *
+ * Copyright (c) 2020, Alibaba Group.
+ * Written by Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
+ */
+
+#ifndef _CRYPTO_SM2_H
+#define _CRYPTO_SM2_H
+
+#include <crypto/sm3.h>
+#include <crypto/akcipher.h>
+
+/* The default user id as specified in GM/T 0009-2012 */
+#define SM2_DEFAULT_USERID "1234567812345678"
+#define SM2_DEFAULT_USERID_LEN 16
+
+extern int sm2_compute_z_digest(struct crypto_akcipher *tfm,
+ const unsigned char *id, size_t id_len,
+ unsigned char dgst[SM3_DIGEST_SIZE]);
+
+#endif /* _CRYPTO_SM2_H */
extern int crypto_sm3_update(struct shash_desc *desc, const u8 *data,
unsigned int len);
+extern int crypto_sm3_final(struct shash_desc *desc, u8 *out);
+
extern int crypto_sm3_finup(struct shash_desc *desc, const u8 *data,
unsigned int len, u8 *hash);
#endif
* This structure represents the DSC PPS infoframe required to send the Picture
* Parameter Set metadata required before enabling VESA Display Stream
* Compression. This is based on the DP Secondary Data Packet structure and
- * comprises of SDP Header as defined &struct struct dp_sdp_header in drm_dp_helper.h
+ * comprises of SDP Header as defined &struct dp_sdp_header in drm_dp_helper.h
* and PPS payload defined in &struct drm_dsc_picture_parameter_set.
*
* @pps_header: Header for PPS as per DP SDP header format of type
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __DT_BINDINGS_MEDIATEK_MT6360_REGULATOR_H__
+#define __DT_BINDINGS_MEDIATEK_MT6360_REGULATOR_H__
+
+/*
+ * BUCK/LDO mode constants which may be used in devicetree properties
+ * (eg. regulator-allowed-modes).
+ * See the manufacturer's datasheet for more information on these modes.
+ */
+
+#define MT6360_OPMODE_LP 2
+#define MT6360_OPMODE_ULP 3
+#define MT6360_OPMODE_NORMAL 0
+
+#endif
bool ready;
bool created;
bool irq_level;
+ struct irq_work overflow_work;
};
#define kvm_arm_pmu_v3_ready(v) ((v)->arch.pmu.ready)
acpi_status acpi_os_prepare_extended_sleep(u8 sleep_state,
u32 val_a, u32 val_b);
-#ifdef CONFIG_X86
+#ifndef CONFIG_IA64
void arch_reserve_mem_area(acpi_physical_address addr, size_t size);
#else
static inline void arch_reserve_mem_area(acpi_physical_address addr,
--- /dev/null
+/*
+ * David A Rusling
+ *
+ * Copyright (C) 2001 ARM Limited
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of this archive
+ * for more details.
+ */
+
+#ifndef AMBA_CLCD_REGS_H
+#define AMBA_CLCD_REGS_H
+
+/*
+ * CLCD Controller Internal Register addresses
+ */
+#define CLCD_TIM0 0x00000000
+#define CLCD_TIM1 0x00000004
+#define CLCD_TIM2 0x00000008
+#define CLCD_TIM3 0x0000000c
+#define CLCD_UBAS 0x00000010
+#define CLCD_LBAS 0x00000014
+
+#define CLCD_PL110_IENB 0x00000018
+#define CLCD_PL110_CNTL 0x0000001c
+#define CLCD_PL110_STAT 0x00000020
+#define CLCD_PL110_INTR 0x00000024
+#define CLCD_PL110_UCUR 0x00000028
+#define CLCD_PL110_LCUR 0x0000002C
+
+#define CLCD_PL111_CNTL 0x00000018
+#define CLCD_PL111_IENB 0x0000001c
+#define CLCD_PL111_RIS 0x00000020
+#define CLCD_PL111_MIS 0x00000024
+#define CLCD_PL111_ICR 0x00000028
+#define CLCD_PL111_UCUR 0x0000002c
+#define CLCD_PL111_LCUR 0x00000030
+
+#define CLCD_PALL 0x00000200
+#define CLCD_PALETTE 0x00000200
+
+#define TIM2_PCD_LO_MASK GENMASK(4, 0)
+#define TIM2_PCD_LO_BITS 5
+#define TIM2_CLKSEL (1 << 5)
+#define TIM2_ACB_MASK GENMASK(10, 6)
+#define TIM2_IVS (1 << 11)
+#define TIM2_IHS (1 << 12)
+#define TIM2_IPC (1 << 13)
+#define TIM2_IOE (1 << 14)
+#define TIM2_BCD (1 << 26)
+#define TIM2_PCD_HI_MASK GENMASK(31, 27)
+#define TIM2_PCD_HI_BITS 5
+#define TIM2_PCD_HI_SHIFT 27
+
+#define CNTL_LCDEN (1 << 0)
+#define CNTL_LCDBPP1 (0 << 1)
+#define CNTL_LCDBPP2 (1 << 1)
+#define CNTL_LCDBPP4 (2 << 1)
+#define CNTL_LCDBPP8 (3 << 1)
+#define CNTL_LCDBPP16 (4 << 1)
+#define CNTL_LCDBPP16_565 (6 << 1)
+#define CNTL_LCDBPP16_444 (7 << 1)
+#define CNTL_LCDBPP24 (5 << 1)
+#define CNTL_LCDBW (1 << 4)
+#define CNTL_LCDTFT (1 << 5)
+#define CNTL_LCDMONO8 (1 << 6)
+#define CNTL_LCDDUAL (1 << 7)
+#define CNTL_BGR (1 << 8)
+#define CNTL_BEBO (1 << 9)
+#define CNTL_BEPO (1 << 10)
+#define CNTL_LCDPWR (1 << 11)
+#define CNTL_LCDVCOMP(x) ((x) << 12)
+#define CNTL_LDMAFIFOTIME (1 << 15)
+#define CNTL_WATERMARK (1 << 16)
+
+/* ST Microelectronics variant bits */
+#define CNTL_ST_1XBPP_444 0x0
+#define CNTL_ST_1XBPP_5551 (1 << 17)
+#define CNTL_ST_1XBPP_565 (1 << 18)
+#define CNTL_ST_CDWID_12 0x0
+#define CNTL_ST_CDWID_16 (1 << 19)
+#define CNTL_ST_CDWID_18 (1 << 20)
+#define CNTL_ST_CDWID_24 ((1 << 19)|(1 << 20))
+#define CNTL_ST_CEAEN (1 << 21)
+#define CNTL_ST_LCDBPP24_PACKED (6 << 1)
+
+#endif /* AMBA_CLCD_REGS_H */
--- /dev/null
+/*
+ * linux/include/asm-arm/hardware/amba_clcd.h -- Integrator LCD panel.
+ *
+ * David A Rusling
+ *
+ * Copyright (C) 2001 ARM Limited
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of this archive
+ * for more details.
+ */
+#include <linux/fb.h>
+#include <linux/amba/clcd-regs.h>
+
+enum {
+ /* individual formats */
+ CLCD_CAP_RGB444 = (1 << 0),
+ CLCD_CAP_RGB5551 = (1 << 1),
+ CLCD_CAP_RGB565 = (1 << 2),
+ CLCD_CAP_RGB888 = (1 << 3),
+ CLCD_CAP_BGR444 = (1 << 4),
+ CLCD_CAP_BGR5551 = (1 << 5),
+ CLCD_CAP_BGR565 = (1 << 6),
+ CLCD_CAP_BGR888 = (1 << 7),
+
+ /* connection layouts */
+ CLCD_CAP_444 = CLCD_CAP_RGB444 | CLCD_CAP_BGR444,
+ CLCD_CAP_5551 = CLCD_CAP_RGB5551 | CLCD_CAP_BGR5551,
+ CLCD_CAP_565 = CLCD_CAP_RGB565 | CLCD_CAP_BGR565,
+ CLCD_CAP_888 = CLCD_CAP_RGB888 | CLCD_CAP_BGR888,
+
+ /* red/blue ordering */
+ CLCD_CAP_RGB = CLCD_CAP_RGB444 | CLCD_CAP_RGB5551 |
+ CLCD_CAP_RGB565 | CLCD_CAP_RGB888,
+ CLCD_CAP_BGR = CLCD_CAP_BGR444 | CLCD_CAP_BGR5551 |
+ CLCD_CAP_BGR565 | CLCD_CAP_BGR888,
+
+ CLCD_CAP_ALL = CLCD_CAP_BGR | CLCD_CAP_RGB,
+};
+
+struct backlight_device;
+
+struct clcd_panel {
+ struct fb_videomode mode;
+ signed short width; /* width in mm */
+ signed short height; /* height in mm */
+ u32 tim2;
+ u32 tim3;
+ u32 cntl;
+ u32 caps;
+ unsigned int bpp:8,
+ fixedtimings:1,
+ grayscale:1;
+ unsigned int connector;
+ struct backlight_device *backlight;
+ /*
+ * If the B/R lines are switched between the CLCD
+ * and the panel we need to know this and not try to
+ * compensate with the BGR bit in the control register.
+ */
+ bool bgr_connection;
+};
+
+struct clcd_regs {
+ u32 tim0;
+ u32 tim1;
+ u32 tim2;
+ u32 tim3;
+ u32 cntl;
+ unsigned long pixclock;
+};
+
+struct clcd_fb;
+
+/*
+ * the board-type specific routines
+ */
+struct clcd_board {
+ const char *name;
+
+ /*
+ * Optional. Hardware capability flags.
+ */
+ u32 caps;
+
+ /*
+ * Optional. Check whether the var structure is acceptable
+ * for this display.
+ */
+ int (*check)(struct clcd_fb *fb, struct fb_var_screeninfo *var);
+
+ /*
+ * Compulsory. Decode fb->fb.var into regs->*. In the case of
+ * fixed timing, set regs->* to the register values required.
+ */
+ void (*decode)(struct clcd_fb *fb, struct clcd_regs *regs);
+
+ /*
+ * Optional. Disable any extra display hardware.
+ */
+ void (*disable)(struct clcd_fb *);
+
+ /*
+ * Optional. Enable any extra display hardware.
+ */
+ void (*enable)(struct clcd_fb *);
+
+ /*
+ * Setup platform specific parts of CLCD driver
+ */
+ int (*setup)(struct clcd_fb *);
+
+ /*
+ * mmap the framebuffer memory
+ */
+ int (*mmap)(struct clcd_fb *, struct vm_area_struct *);
+
+ /*
+ * Remove platform specific parts of CLCD driver
+ */
+ void (*remove)(struct clcd_fb *);
+};
+
+struct amba_device;
+struct clk;
+
+/* this data structure describes each frame buffer device we find */
+struct clcd_fb {
+ struct fb_info fb;
+ struct amba_device *dev;
+ struct clk *clk;
+ struct clcd_panel *panel;
+ struct clcd_board *board;
+ void *board_data;
+ void __iomem *regs;
+ u16 off_ienb;
+ u16 off_cntl;
+ u32 clcd_cntl;
+ u32 cmap[16];
+ bool clk_enabled;
+};
+
+static inline void clcdfb_decode(struct clcd_fb *fb, struct clcd_regs *regs)
+{
+ struct fb_var_screeninfo *var = &fb->fb.var;
+ u32 val, cpl;
+
+ /*
+ * Program the CLCD controller registers and start the CLCD
+ */
+ val = ((var->xres / 16) - 1) << 2;
+ val |= (var->hsync_len - 1) << 8;
+ val |= (var->right_margin - 1) << 16;
+ val |= (var->left_margin - 1) << 24;
+ regs->tim0 = val;
+
+ val = var->yres;
+ if (fb->panel->cntl & CNTL_LCDDUAL)
+ val /= 2;
+ val -= 1;
+ val |= (var->vsync_len - 1) << 10;
+ val |= var->lower_margin << 16;
+ val |= var->upper_margin << 24;
+ regs->tim1 = val;
+
+ val = fb->panel->tim2;
+ val |= var->sync & FB_SYNC_HOR_HIGH_ACT ? 0 : TIM2_IHS;
+ val |= var->sync & FB_SYNC_VERT_HIGH_ACT ? 0 : TIM2_IVS;
+
+ cpl = var->xres_virtual;
+ if (fb->panel->cntl & CNTL_LCDTFT) /* TFT */
+ /* / 1 */;
+ else if (!var->grayscale) /* STN color */
+ cpl = cpl * 8 / 3;
+ else if (fb->panel->cntl & CNTL_LCDMONO8) /* STN monochrome, 8bit */
+ cpl /= 8;
+ else /* STN monochrome, 4bit */
+ cpl /= 4;
+
+ regs->tim2 = val | ((cpl - 1) << 16);
+
+ regs->tim3 = fb->panel->tim3;
+
+ val = fb->panel->cntl;
+ if (var->grayscale)
+ val |= CNTL_LCDBW;
+
+ if (fb->panel->caps && fb->board->caps && var->bits_per_pixel >= 16) {
+ /*
+ * if board and panel supply capabilities, we can support
+ * changing BGR/RGB depending on supplied parameters. Here
+ * we switch to what the framebuffer is providing if need
+ * be, so if the framebuffer is BGR but the display connection
+ * is RGB (first case) we switch it around. Vice versa mutatis
+ * mutandis if the framebuffer is RGB but the display connection
+ * is BGR, we flip it around.
+ */
+ if (var->red.offset == 0)
+ val &= ~CNTL_BGR;
+ else
+ val |= CNTL_BGR;
+ if (fb->panel->bgr_connection)
+ val ^= CNTL_BGR;
+ }
+
+ switch (var->bits_per_pixel) {
+ case 1:
+ val |= CNTL_LCDBPP1;
+ break;
+ case 2:
+ val |= CNTL_LCDBPP2;
+ break;
+ case 4:
+ val |= CNTL_LCDBPP4;
+ break;
+ case 8:
+ val |= CNTL_LCDBPP8;
+ break;
+ case 16:
+ /*
+ * PL110 cannot choose between 5551 and 565 modes in its
+ * control register. It is possible to use 565 with
+ * custom external wiring.
+ */
+ if (amba_part(fb->dev) == 0x110 ||
+ var->green.length == 5)
+ val |= CNTL_LCDBPP16;
+ else if (var->green.length == 6)
+ val |= CNTL_LCDBPP16_565;
+ else
+ val |= CNTL_LCDBPP16_444;
+ break;
+ case 32:
+ val |= CNTL_LCDBPP24;
+ break;
+ }
+
+ regs->cntl = val;
+ regs->pixclock = var->pixclock;
+}
+
+static inline int clcdfb_check(struct clcd_fb *fb, struct fb_var_screeninfo *var)
+{
+ var->xres_virtual = var->xres = (var->xres + 15) & ~15;
+ var->yres_virtual = var->yres = (var->yres + 1) & ~1;
+
+#define CHECK(e,l,h) (var->e < l || var->e > h)
+ if (CHECK(right_margin, (5+1), 256) || /* back porch */
+ CHECK(left_margin, (5+1), 256) || /* front porch */
+ CHECK(hsync_len, (5+1), 256) ||
+ var->xres > 4096 ||
+ var->lower_margin > 255 || /* back porch */
+ var->upper_margin > 255 || /* front porch */
+ var->vsync_len > 32 ||
+ var->yres > 1024)
+ return -EINVAL;
+#undef CHECK
+
+ /* single panel mode: PCD = max(PCD, 1) */
+ /* dual panel mode: PCD = max(PCD, 5) */
+
+ /*
+ * You can't change the grayscale setting, and
+ * we can only do non-interlaced video.
+ */
+ if (var->grayscale != fb->fb.var.grayscale ||
+ (var->vmode & FB_VMODE_MASK) != FB_VMODE_NONINTERLACED)
+ return -EINVAL;
+
+#define CHECK(e) (var->e != fb->fb.var.e)
+ if (fb->panel->fixedtimings &&
+ (CHECK(xres) ||
+ CHECK(yres) ||
+ CHECK(bits_per_pixel) ||
+ CHECK(pixclock) ||
+ CHECK(left_margin) ||
+ CHECK(right_margin) ||
+ CHECK(upper_margin) ||
+ CHECK(lower_margin) ||
+ CHECK(hsync_len) ||
+ CHECK(vsync_len) ||
+ CHECK(sync)))
+ return -EINVAL;
+#undef CHECK
+
+ var->nonstd = 0;
+ var->accel_flags = 0;
+
+ return 0;
+}
*
* The function returns 0 on success or a negative value on error.
*/
-extern int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,
+extern int amd_iommu_bind_pasid(struct pci_dev *pdev, u32 pasid,
struct task_struct *task);
/**
* When this function returns the device is no longer using the PASID
* and the PASID is no longer bound to its task.
*/
-extern void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid);
+extern void amd_iommu_unbind_pasid(struct pci_dev *pdev, u32 pasid);
/**
* amd_iommu_set_invalid_ppr_cb() - Register a call-back for failed
#define AMD_IOMMU_INV_PRI_RSP_FAIL 2
typedef int (*amd_iommu_invalid_ppr_cb)(struct pci_dev *pdev,
- int pasid,
+ u32 pasid,
unsigned long address,
u16);
* @cb: The call-back function
*/
-typedef void (*amd_iommu_invalidate_ctx)(struct pci_dev *pdev, int pasid);
+typedef void (*amd_iommu_invalidate_ctx)(struct pci_dev *pdev, u32 pasid);
extern int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev,
amd_iommu_invalidate_ctx cb);
#endif /* CONFIG_BLK_INLINE_ENCRYPTION */
-void __bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask);
-static inline void bio_crypt_clone(struct bio *dst, struct bio *src,
- gfp_t gfp_mask)
+int __bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask);
+/**
+ * bio_crypt_clone - clone bio encryption context
+ * @dst: destination bio
+ * @src: source bio
+ * @gfp_mask: memory allocation flags
+ *
+ * If @src has an encryption context, clone it to @dst.
+ *
+ * Return: 0 on success, -ENOMEM if out of memory. -ENOMEM is only possible if
+ * @gfp_mask doesn't include %__GFP_DIRECT_RECLAIM.
+ */
+static inline int bio_crypt_clone(struct bio *dst, struct bio *src,
+ gfp_t gfp_mask)
{
if (bio_has_crypt_ctx(src))
- __bio_crypt_clone(dst, src, gfp_mask);
+ return __bio_crypt_clone(dst, src, gfp_mask);
+ return 0;
}
#endif /* __LINUX_BLK_CRYPTO_H */
typedef unsigned int blk_qc_t;
#define BLK_QC_T_NONE -1U
-#define BLK_QC_T_EAGAIN -2U
#define BLK_QC_T_SHIFT 16
#define BLK_QC_T_INTERNAL (1U << 31)
static inline bool blk_qc_t_valid(blk_qc_t cookie)
{
- return cookie != BLK_QC_T_NONE && cookie != BLK_QC_T_EAGAIN;
+ return cookie != BLK_QC_T_NONE;
}
static inline unsigned int blk_qc_t_to_queue_num(blk_qc_t cookie)
typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
void *data);
+void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model);
+
#ifdef CONFIG_BLK_DEV_ZONED
#define BLK_ALL_ZONES ((unsigned int)-1)
struct request *last_merge;
struct elevator_queue *elevator;
+ struct percpu_ref q_usage_counter;
+
struct blk_queue_stats *stats;
struct rq_qos *rq_qos;
* percpu_ref_kill() and percpu_ref_reinit().
*/
struct mutex mq_freeze_lock;
- struct percpu_ref q_usage_counter;
struct blk_mq_tag_set *tag_set;
struct list_head tag_set_list;
#define QUEUE_FLAG_PCI_P2PDMA 25 /* device supports PCI p2p requests */
#define QUEUE_FLAG_ZONE_RESETALL 26 /* supports Zone Reset All */
#define QUEUE_FLAG_RQ_ALLOC_TIME 27 /* record rq->alloc_time_ns */
-#define QUEUE_FLAG_HCTX_ACTIVE 28 /* at least one blk-mq hctx is active */
+#define QUEUE_FLAG_HCTX_ACTIVE 28 /* at least one blk-mq hctx is active */
+#define QUEUE_FLAG_NOWAIT 29 /* device supports NOWAIT */
#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
- (1 << QUEUE_FLAG_SAME_COMP))
+ (1 << QUEUE_FLAG_SAME_COMP) | \
+ (1 << QUEUE_FLAG_NOWAIT))
void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
#define blk_queue_pm_only(q) atomic_read(&(q)->pm_only)
#define blk_queue_fua(q) test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)
#define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
+#define blk_queue_nowait(q) test_bit(QUEUE_FLAG_NOWAIT, &(q)->queue_flags)
extern void blk_set_pm_only(struct request_queue *q);
extern void blk_clear_pm_only(struct request_queue *q);
extern int blk_verify_command(unsigned char *cmd, fmode_t mode);
+static inline bool bdev_is_partition(struct block_device *bdev)
+{
+ return bdev->bd_partno;
+}
+
enum blk_default_limits {
BLK_MAX_SEGMENTS = 128,
BLK_SAFE_MAX_SECTORS = 255,
static inline unsigned int queue_max_zone_append_sectors(const struct request_queue *q)
{
- return q->limits.max_zone_append_sectors;
+
+ const struct queue_limits *l = &q->limits;
+
+ return min(l->max_zone_append_sectors, l->max_sectors);
}
static inline unsigned queue_logical_block_size(const struct request_queue *q)
if (q->limits.misaligned)
return -1;
- if (bdev != bdev->bd_contains)
+ if (bdev_is_partition(bdev))
return queue_limit_alignment_offset(&q->limits,
bdev->bd_part->start_sect);
return q->limits.alignment_offset;
{
struct request_queue *q = bdev_get_queue(bdev);
- if (bdev != bdev->bd_contains)
+ if (bdev_is_partition(bdev))
return queue_limit_discard_alignment(&q->limits,
bdev->bd_part->start_sect);
return q->limits.discard_alignment;
extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
struct scatterlist *);
extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
-extern bool blk_integrity_merge_rq(struct request_queue *, struct request *,
- struct request *);
-extern bool blk_integrity_merge_bio(struct request_queue *, struct request *,
- struct bio *);
static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
{
{
return 0;
}
-static inline bool blk_integrity_merge_rq(struct request_queue *rq,
- struct request *r1,
- struct request *r2)
-{
- return true;
-}
-static inline bool blk_integrity_merge_bio(struct request_queue *rq,
- struct request *r,
- struct bio *b)
-{
- return true;
-}
static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
unsigned int sectors)
void blkdev_put(struct block_device *bdev, fmode_t mode);
struct block_device *I_BDEV(struct inode *inode);
-struct block_device *bdget(dev_t);
+struct block_device *bdget_part(struct hd_struct *part);
struct block_device *bdgrab(struct block_device *bdev);
void bdput(struct block_device *);
#define _LINUX_CACHEINFO_H
#include <linux/bitops.h>
+#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/smp.h>
const struct attribute_group *cache_get_priv_group(struct cacheinfo *this_leaf);
+/*
+ * Get the id of the cache associated with @cpu at level @level.
+ * cpuhp lock must be held.
+ */
+static inline int get_cpu_cacheinfo_id(int cpu, int level)
+{
+ struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu);
+ int i;
+
+ for (i = 0; i < ci->num_leaves; i++) {
+ if (ci->info_list[i].level == level) {
+ if (ci->info_list[i].attributes & CACHE_ID)
+ return ci->info_list[i].id;
+ return -1;
+ }
+ }
+
+ return -1;
+}
+
#endif /* _LINUX_CACHEINFO_H */
static inline long __do_compat_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))
#endif /* COMPAT_SYSCALL_DEFINEx */
+struct compat_iovec {
+ compat_uptr_t iov_base;
+ compat_size_t iov_len;
+};
+
#ifdef CONFIG_COMPAT
#ifndef compat_user_stack_pointer
} _sifields;
} compat_siginfo_t;
-struct compat_iovec {
- compat_uptr_t iov_base;
- compat_size_t iov_len;
-};
-
struct compat_rlimit {
compat_ulong_t rlim_cur;
compat_ulong_t rlim_max;
struct epoll_event; /* fortunately, this one is fixed-layout */
-extern ssize_t compat_rw_copy_check_uvector(int type,
- const struct compat_iovec __user *uvector,
- unsigned long nr_segs,
- unsigned long fast_segs, struct iovec *fast_pointer,
- struct iovec **ret_pointer);
-
extern void __user *compat_alloc_user_space(unsigned long len);
int compat_restore_altstack(const compat_stack_t __user *uss);
asmlinkage long compat_sys_ioctl(unsigned int fd, unsigned int cmd,
compat_ulong_t arg);
-/* fs/namespace.c */
-asmlinkage long compat_sys_mount(const char __user *dev_name,
- const char __user *dir_name,
- const char __user *type, compat_ulong_t flags,
- const void __user *data);
-
/* fs/open.c */
asmlinkage long compat_sys_statfs(const char __user *pathname,
struct compat_statfs __user *buf);
/* fs/read_write.c */
asmlinkage long compat_sys_lseek(unsigned int, compat_off_t, unsigned int);
-asmlinkage ssize_t compat_sys_readv(compat_ulong_t fd,
- const struct compat_iovec __user *vec, compat_ulong_t vlen);
-asmlinkage ssize_t compat_sys_writev(compat_ulong_t fd,
- const struct compat_iovec __user *vec, compat_ulong_t vlen);
/* No generic prototype for pread64 and pwrite64 */
asmlinkage ssize_t compat_sys_preadv(compat_ulong_t fd,
- const struct compat_iovec __user *vec,
+ const struct iovec __user *vec,
compat_ulong_t vlen, u32 pos_low, u32 pos_high);
asmlinkage ssize_t compat_sys_pwritev(compat_ulong_t fd,
- const struct compat_iovec __user *vec,
+ const struct iovec __user *vec,
compat_ulong_t vlen, u32 pos_low, u32 pos_high);
#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
asmlinkage long compat_sys_preadv64(unsigned long fd,
- const struct compat_iovec __user *vec,
+ const struct iovec __user *vec,
unsigned long vlen, loff_t pos);
#endif
#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
asmlinkage long compat_sys_pwritev64(unsigned long fd,
- const struct compat_iovec __user *vec,
+ const struct iovec __user *vec,
unsigned long vlen, loff_t pos);
#endif
const compat_sigset_t __user *sigmask,
compat_size_t sigsetsize, int flags);
-/* fs/splice.c */
-asmlinkage long compat_sys_vmsplice(int fd, const struct compat_iovec __user *,
- unsigned int nr_segs, unsigned int flags);
-
/* fs/stat.c */
asmlinkage long compat_sys_newfstatat(unsigned int dfd,
const char __user *filename,
int flags);
asmlinkage long compat_sys_sendmmsg(int fd, struct compat_mmsghdr __user *mmsg,
unsigned vlen, unsigned int flags);
-asmlinkage ssize_t compat_sys_process_vm_readv(compat_pid_t pid,
- const struct compat_iovec __user *lvec,
- compat_ulong_t liovcnt, const struct compat_iovec __user *rvec,
- compat_ulong_t riovcnt, compat_ulong_t flags);
-asmlinkage ssize_t compat_sys_process_vm_writev(compat_pid_t pid,
- const struct compat_iovec __user *lvec,
- compat_ulong_t liovcnt, const struct compat_iovec __user *rvec,
- compat_ulong_t riovcnt, compat_ulong_t flags);
asmlinkage long compat_sys_execveat(int dfd, const char __user *filename,
const compat_uptr_t __user *argv,
const compat_uptr_t __user *envp, int flags);
asmlinkage ssize_t compat_sys_preadv2(compat_ulong_t fd,
- const struct compat_iovec __user *vec,
+ const struct iovec __user *vec,
compat_ulong_t vlen, u32 pos_low, u32 pos_high, rwf_t flags);
asmlinkage ssize_t compat_sys_pwritev2(compat_ulong_t fd,
- const struct compat_iovec __user *vec,
+ const struct iovec __user *vec,
compat_ulong_t vlen, u32 pos_low, u32 pos_high, rwf_t flags);
#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
-asmlinkage long compat_sys_readv64v2(unsigned long fd,
- const struct compat_iovec __user *vec,
+asmlinkage long compat_sys_preadv64v2(unsigned long fd,
+ const struct iovec __user *vec,
unsigned long vlen, loff_t pos, rwf_t flags);
#endif
#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
asmlinkage long compat_sys_pwritev64v2(unsigned long fd,
- const struct compat_iovec __user *vec,
+ const struct iovec __user *vec,
unsigned long vlen, loff_t pos, rwf_t flags);
#endif
#endif /* CONFIG_COMPAT */
+/*
+ * Some legacy ABIs like the i386 one use less than natural alignment for 64-bit
+ * types, and will need special compat treatment for that. Most architectures
+ * don't need that special handling even for compat syscalls.
+ */
+#ifndef compat_need_64bit_alignment_fixup
+#define compat_need_64bit_alignment_fixup() false
+#endif
+
/*
* A pointer passed in from user mode. This should not
* be used for syscall parameters, just declare them
*/
#define __ADDRESSABLE(sym) \
static void * __section(.discard.addressable) __used \
- __PASTE(__addressable_##sym, __LINE__) = (void *)&sym;
+ __UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;
/**
* offset_to_ptr - convert a relative memory offset to an absolute pointer
#define CPER_MEM_VALID_RANK_NUMBER 0x8000
#define CPER_MEM_VALID_CARD_HANDLE 0x10000
#define CPER_MEM_VALID_MODULE_HANDLE 0x20000
+#define CPER_MEM_VALID_ROW_EXT 0x40000
+#define CPER_MEM_VALID_BANK_GROUP 0x80000
+#define CPER_MEM_VALID_BANK_ADDRESS 0x100000
+#define CPER_MEM_VALID_CHIP_ID 0x200000
+
+#define CPER_MEM_EXT_ROW_MASK 0x3
+#define CPER_MEM_EXT_ROW_SHIFT 16
+
+#define CPER_MEM_BANK_ADDRESS_MASK 0xff
+#define CPER_MEM_BANK_GROUP_SHIFT 8
+
+#define CPER_MEM_CHIP_ID_SHIFT 5
#define CPER_PCIE_VALID_PORT_TYPE 0x0001
#define CPER_PCIE_VALID_VERSION 0x0002
u8 error_type;
};
-/* Memory Error Section (UEFI >= v2.3), UEFI v2.7 sec N.2.5 */
+/* Memory Error Section (UEFI >= v2.3), UEFI v2.8 sec N.2.5 */
struct cper_sec_mem_err {
u64 validation_bits;
u64 error_status;
u64 responder_id;
u64 target_id;
u8 error_type;
- u8 reserved;
+ u8 extended;
u16 rank;
u16 mem_array_handle; /* "card handle" in UEFI 2.4 */
u16 mem_dev_handle; /* "module handle" in UEFI 2.4 */
u16 rank;
u16 mem_array_handle;
u16 mem_dev_handle;
+ u8 extended;
};
+static inline u32 cper_get_mem_extension(u64 mem_valid, u8 mem_extended)
+{
+ if (!(mem_valid & CPER_MEM_VALID_ROW_EXT))
+ return 0;
+ return (mem_extended & CPER_MEM_EXT_ROW_MASK) << CPER_MEM_EXT_ROW_SHIFT;
+}
+
/* PCI Express Error Section, UEFI v2.7 sec N.2.7 */
struct cper_sec_pcie {
u64 validation_bits;
CPUHP_X86_MCE_DEAD,
CPUHP_VIRT_NET_DEAD,
CPUHP_SLUB_DEAD,
+ CPUHP_DEBUG_OBJ_DEAD,
CPUHP_MM_WRITEBACK_DEAD,
CPUHP_MM_VMSTAT_DEAD,
CPUHP_SOFTIRQ_DEAD,
#define DCACHE_MAY_FREE 0x00800000
#define DCACHE_FALLTHRU 0x01000000 /* Fall through to lower layer */
-#define DCACHE_ENCRYPTED_NAME 0x02000000 /* Encrypted name (dir key was unavailable) */
+#define DCACHE_NOKEY_NAME 0x02000000 /* Encrypted name encoded without key */
#define DCACHE_OP_REAL 0x04000000
#define DCACHE_PAR_LOOKUP 0x10000000 /* being looked up (with parent locked shared) */
enum debug_obj_state state;
unsigned int astate;
void *object;
- struct debug_obj_descr *descr;
+ const struct debug_obj_descr *descr;
};
/**
};
#ifdef CONFIG_DEBUG_OBJECTS
-extern void debug_object_init (void *addr, struct debug_obj_descr *descr);
+extern void debug_object_init (void *addr, const struct debug_obj_descr *descr);
extern void
-debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr);
-extern int debug_object_activate (void *addr, struct debug_obj_descr *descr);
-extern void debug_object_deactivate(void *addr, struct debug_obj_descr *descr);
-extern void debug_object_destroy (void *addr, struct debug_obj_descr *descr);
-extern void debug_object_free (void *addr, struct debug_obj_descr *descr);
-extern void debug_object_assert_init(void *addr, struct debug_obj_descr *descr);
+debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr);
+extern int debug_object_activate (void *addr, const struct debug_obj_descr *descr);
+extern void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr);
+extern void debug_object_destroy (void *addr, const struct debug_obj_descr *descr);
+extern void debug_object_free (void *addr, const struct debug_obj_descr *descr);
+extern void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr);
/*
* Active state:
* - Must return to 0 before deactivation.
*/
extern void
-debug_object_active_state(void *addr, struct debug_obj_descr *descr,
+debug_object_active_state(void *addr, const struct debug_obj_descr *descr,
unsigned int expect, unsigned int next);
extern void debug_objects_early_init(void);
extern void debug_objects_mem_init(void);
#else
static inline void
-debug_object_init (void *addr, struct debug_obj_descr *descr) { }
+debug_object_init (void *addr, const struct debug_obj_descr *descr) { }
static inline void
-debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr) { }
+debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr) { }
static inline int
-debug_object_activate (void *addr, struct debug_obj_descr *descr) { return 0; }
+debug_object_activate (void *addr, const struct debug_obj_descr *descr) { return 0; }
static inline void
-debug_object_deactivate(void *addr, struct debug_obj_descr *descr) { }
+debug_object_deactivate(void *addr, const struct debug_obj_descr *descr) { }
static inline void
-debug_object_destroy (void *addr, struct debug_obj_descr *descr) { }
+debug_object_destroy (void *addr, const struct debug_obj_descr *descr) { }
static inline void
-debug_object_free (void *addr, struct debug_obj_descr *descr) { }
+debug_object_free (void *addr, const struct debug_obj_descr *descr) { }
static inline void
-debug_object_assert_init(void *addr, struct debug_obj_descr *descr) { }
+debug_object_assert_init(void *addr, const struct debug_obj_descr *descr) { }
static inline void debug_objects_early_init(void) { }
static inline void debug_objects_mem_init(void) { }
#define DM_TARGET_ZONED_HM 0x00000040
#define dm_target_supports_zoned_hm(type) ((type)->features & DM_TARGET_ZONED_HM)
+/*
+ * A target handles REQ_NOWAIT
+ */
+#define DM_TARGET_NOWAIT 0x00000080
+#define dm_target_supports_nowait(type) ((type)->features & DM_TARGET_NOWAIT)
+
struct dm_target {
struct dm_table *table;
struct target_type *type;
((u64)0x0000000000010000ULL) /* higher reliability */
#define EFI_MEMORY_RO ((u64)0x0000000000020000ULL) /* read-only */
#define EFI_MEMORY_SP ((u64)0x0000000000040000ULL) /* soft reserved */
+#define EFI_MEMORY_CPU_CRYPTO ((u64)0x0000000000080000ULL) /* supports encryption */
#define EFI_MEMORY_RUNTIME ((u64)0x8000000000000000ULL) /* range requires runtime mapping */
#define EFI_MEMORY_DESCRIPTOR_VERSION 1
#define LINUX_EFI_TPM_FINAL_LOG_GUID EFI_GUID(0x1e2ed096, 0x30e2, 0x4254, 0xbd, 0x89, 0x86, 0x3b, 0xbe, 0xf8, 0x23, 0x25)
#define LINUX_EFI_MEMRESERVE_TABLE_GUID EFI_GUID(0x888eb0c6, 0x8ede, 0x4ff5, 0xa8, 0xf0, 0x9a, 0xee, 0x5c, 0xb9, 0x77, 0xc2)
#define LINUX_EFI_INITRD_MEDIA_GUID EFI_GUID(0x5568e427, 0x68fc, 0x4f3d, 0xac, 0x74, 0xca, 0x55, 0x52, 0x31, 0xcc, 0x68)
+#define LINUX_EFI_MOK_VARIABLE_TABLE_GUID EFI_GUID(0xc451ed2b, 0x9694, 0x45d3, 0xba, 0xba, 0xed, 0x9f, 0x89, 0x88, 0xa3, 0x89)
/* OEM GUIDs */
#define DELLEMC_EFI_RCI2_TABLE_GUID EFI_GUID(0x2d9f28a2, 0xa886, 0x456a, 0x97, 0xa8, 0xf1, 0x1e, 0xf2, 0x4f, 0xf4, 0x55)
unsigned long esrt; /* ESRT table */
unsigned long tpm_log; /* TPM2 Event Log table */
unsigned long tpm_final_log; /* TPM2 Final Events Log table */
+ unsigned long mokvar_table; /* MOK variable config table */
efi_get_time_t *get_time;
efi_set_time_t *set_time;
bool deleting;
};
-extern struct list_head efivar_sysfs_list;
-
static inline void
efivar_unregister(struct efivar_entry *var)
{
bool efivar_variable_is_removable(efi_guid_t vendor, const char *name,
size_t len);
-extern struct work_struct efivar_work;
-void efivar_run_worker(void);
-
-#if defined(CONFIG_EFI_VARS) || defined(CONFIG_EFI_VARS_MODULE)
-int efivars_sysfs_init(void);
-
-#define EFIVARS_DATA_SIZE_MAX 1024
-
-#endif /* CONFIG_EFI_VARS */
extern bool efi_capsule_pending(int *reset_type);
extern int efi_capsule_supported(efi_guid_t guid, u32 flags,
char *efi_systab_show_arch(char *str);
+/*
+ * The LINUX_EFI_MOK_VARIABLE_TABLE_GUID config table can be provided
+ * to the kernel by an EFI boot loader. The table contains a packed
+ * sequence of these entries, one for each named MOK variable.
+ * The sequence is terminated by an entry with a completely NULL
+ * name and 0 data size.
+ */
+struct efi_mokvar_table_entry {
+ char name[256];
+ u64 data_size;
+ u8 data[];
+} __attribute((packed));
+
+#ifdef CONFIG_LOAD_UEFI_KEYS
+extern void __init efi_mokvar_table_init(void);
+extern struct efi_mokvar_table_entry *efi_mokvar_entry_next(
+ struct efi_mokvar_table_entry **mokvar_entry);
+extern struct efi_mokvar_table_entry *efi_mokvar_entry_find(const char *name);
+#else
+static inline void efi_mokvar_table_init(void) { }
+static inline struct efi_mokvar_table_entry *efi_mokvar_entry_next(
+ struct efi_mokvar_table_entry **mokvar_entry)
+{
+ return NULL;
+}
+static inline struct efi_mokvar_table_entry *efi_mokvar_entry_find(
+ const char *name)
+{
+ return NULL;
+}
+#endif
+
#endif /* _LINUX_EFI_H */
#endif
/*
- * TIF flags handled in syscall_enter_from_usermode()
+ * TIF flags handled in syscall_enter_from_user_mode()
*/
#ifndef ARCH_SYSCALL_ENTER_WORK
# define ARCH_SYSCALL_ENTER_WORK (0)
/* Max. length for the name of a predefined font */
#define MAX_FONT_NAME 32
+/* Extra word getters */
+#define REFCOUNT(fd) (((int *)(fd))[-1])
+#define FNTSIZE(fd) (((int *)(fd))[-2])
+#define FNTCHARCNT(fd) (((int *)(fd))[-3])
+#define FNTSUM(fd) (((int *)(fd))[-4])
+
+#define FONT_EXTRA_WORDS 4
+
+struct font_data {
+ unsigned int extra[FONT_EXTRA_WORDS];
+ const unsigned char data[];
+} __packed;
+
#endif /* _VIDEO_FONT_H */
/* File supports async buffered reads */
#define FMODE_BUF_RASYNC ((__force fmode_t)0x40000000)
-/*
- * Flag for rw_copy_check_uvector and compat_rw_copy_check_uvector
- * that indicates that they should check the contents of the iovec are
- * valid, but not check the memory that the iovec elements
- * points too.
- */
-#define CHECK_IOVEC_ONLY -1
-
/*
* Attribute flags. These should be or-ed together to figure out what
* has been changed!
WRITE_LIFE_EXTREME = RWH_WRITE_LIFE_EXTREME,
};
-#define IOCB_EVENTFD (1 << 0)
-#define IOCB_APPEND (1 << 1)
-#define IOCB_DIRECT (1 << 2)
-#define IOCB_HIPRI (1 << 3)
-#define IOCB_DSYNC (1 << 4)
-#define IOCB_SYNC (1 << 5)
-#define IOCB_WRITE (1 << 6)
-#define IOCB_NOWAIT (1 << 7)
+/* Match RWF_* bits to IOCB bits */
+#define IOCB_HIPRI (__force int) RWF_HIPRI
+#define IOCB_DSYNC (__force int) RWF_DSYNC
+#define IOCB_SYNC (__force int) RWF_SYNC
+#define IOCB_NOWAIT (__force int) RWF_NOWAIT
+#define IOCB_APPEND (__force int) RWF_APPEND
+
+/* non-RWF related bits - start at 16 */
+#define IOCB_EVENTFD (1 << 16)
+#define IOCB_DIRECT (1 << 17)
+#define IOCB_WRITE (1 << 18)
/* iocb->ki_waitq is valid */
-#define IOCB_WAITQ (1 << 8)
-#define IOCB_NOIO (1 << 9)
+#define IOCB_WAITQ (1 << 19)
+#define IOCB_NOIO (1 << 20)
struct kiocb {
struct file *ki_filp;
return file->f_op->mmap(file, vma);
}
-ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
- unsigned long nr_segs, unsigned long fast_segs,
- struct iovec *fast_pointer,
- struct iovec **ret_pointer);
-
extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *);
extern ssize_t vfs_write(struct file *, const char __user *, size_t, loff_t *);
extern ssize_t vfs_readv(struct file *, const struct iovec __user *,
DIO_SKIP_HOLES = 0x02,
};
-void dio_end_io(struct bio *bio);
-
ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
struct block_device *bdev, struct iov_iter *iter,
get_block_t get_block,
{
int kiocb_flags = 0;
+ /* make sure there's no overlap between RWF and private IOCB flags */
+ BUILD_BUG_ON((__force int) RWF_SUPPORTED & IOCB_EVENTFD);
+
if (!flags)
return 0;
if (unlikely(flags & ~RWF_SUPPORTED))
if (flags & RWF_NOWAIT) {
if (!(ki->ki_filp->f_mode & FMODE_NOWAIT))
return -EOPNOTSUPP;
- kiocb_flags |= IOCB_NOWAIT | IOCB_NOIO;
+ kiocb_flags |= IOCB_NOIO;
}
- if (flags & RWF_HIPRI)
- kiocb_flags |= IOCB_HIPRI;
- if (flags & RWF_DSYNC)
- kiocb_flags |= IOCB_DSYNC;
+ kiocb_flags |= (__force int) (flags & RWF_SUPPORTED);
if (flags & RWF_SYNC)
- kiocb_flags |= (IOCB_DSYNC | IOCB_SYNC);
- if (flags & RWF_APPEND)
- kiocb_flags |= IOCB_APPEND;
+ kiocb_flags |= IOCB_DSYNC;
ki->ki_flags |= kiocb_flags;
return 0;
extern int generic_fadvise(struct file *file, loff_t offset, loff_t len,
int advice);
-#if defined(CONFIG_IO_URING)
-extern struct sock *io_uring_get_socket(struct file *file);
-#else
-static inline struct sock *io_uring_get_socket(struct file *file)
-{
- return NULL;
-}
-#endif
-
int vfs_ioc_setflags_prepare(struct inode *inode, unsigned int oldflags,
unsigned int flags);
#include <linux/fs.h>
#include <linux/mm.h>
-#include <linux/parser.h>
#include <linux/slab.h>
#include <uapi/linux/fscrypt.h>
#define FS_CRYPTO_BLOCK_SIZE 16
-union fscrypt_context;
+union fscrypt_policy;
struct fscrypt_info;
struct seq_file;
u32 hash;
u32 minor_hash;
struct fscrypt_str crypto_buf;
- bool is_ciphertext_name;
+ bool is_nokey_name;
};
#define FSTR_INIT(n, l) { .name = n, .len = l }
int (*get_context)(struct inode *inode, void *ctx, size_t len);
int (*set_context)(struct inode *inode, const void *ctx, size_t len,
void *fs_data);
- const union fscrypt_context *(*get_dummy_context)(
- struct super_block *sb);
+ const union fscrypt_policy *(*get_dummy_policy)(struct super_block *sb);
bool (*empty_dir)(struct inode *inode);
unsigned int max_namelen;
bool (*has_stable_inodes)(struct super_block *sb);
return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode);
}
-static inline const union fscrypt_context *
-fscrypt_get_dummy_context(struct super_block *sb)
-{
- if (!sb->s_cop->get_dummy_context)
- return NULL;
- return sb->s_cop->get_dummy_context(sb);
-}
-
/*
- * When d_splice_alias() moves a directory's encrypted alias to its decrypted
- * alias as a result of the encryption key being added, DCACHE_ENCRYPTED_NAME
- * must be cleared. Note that we don't have to support arbitrary moves of this
- * flag because fscrypt doesn't allow encrypted aliases to be the source or
- * target of a rename().
+ * When d_splice_alias() moves a directory's no-key alias to its plaintext alias
+ * as a result of the encryption key being added, DCACHE_NOKEY_NAME must be
+ * cleared. Note that we don't have to support arbitrary moves of this flag
+ * because fscrypt doesn't allow no-key names to be the source or target of a
+ * rename().
*/
static inline void fscrypt_handle_d_move(struct dentry *dentry)
{
- dentry->d_flags &= ~DCACHE_ENCRYPTED_NAME;
+ dentry->d_flags &= ~DCACHE_NOKEY_NAME;
}
/* crypto.c */
int fscrypt_ioctl_get_policy_ex(struct file *filp, void __user *arg);
int fscrypt_ioctl_get_nonce(struct file *filp, void __user *arg);
int fscrypt_has_permitted_context(struct inode *parent, struct inode *child);
-int fscrypt_inherit_context(struct inode *parent, struct inode *child,
- void *fs_data, bool preload);
+int fscrypt_set_context(struct inode *inode, void *fs_data);
-struct fscrypt_dummy_context {
- const union fscrypt_context *ctx;
+struct fscrypt_dummy_policy {
+ const union fscrypt_policy *policy;
};
-int fscrypt_set_test_dummy_encryption(struct super_block *sb,
- const substring_t *arg,
- struct fscrypt_dummy_context *dummy_ctx);
+int fscrypt_set_test_dummy_encryption(struct super_block *sb, const char *arg,
+ struct fscrypt_dummy_policy *dummy_policy);
void fscrypt_show_test_dummy_encryption(struct seq_file *seq, char sep,
struct super_block *sb);
static inline void
-fscrypt_free_dummy_context(struct fscrypt_dummy_context *dummy_ctx)
+fscrypt_free_dummy_policy(struct fscrypt_dummy_policy *dummy_policy)
{
- kfree(dummy_ctx->ctx);
- dummy_ctx->ctx = NULL;
+ kfree(dummy_policy->policy);
+ dummy_policy->policy = NULL;
}
/* keyring.c */
/* keysetup.c */
int fscrypt_get_encryption_info(struct inode *inode);
+int fscrypt_prepare_new_inode(struct inode *dir, struct inode *inode,
+ bool *encrypt_ret);
void fscrypt_put_encryption_info(struct inode *inode);
void fscrypt_free_inode(struct inode *inode);
int fscrypt_drop_inode(struct inode *inode);
kfree(fname->crypto_buf.name);
}
-int fscrypt_fname_alloc_buffer(const struct inode *inode, u32 max_encrypted_len,
+int fscrypt_fname_alloc_buffer(u32 max_encrypted_len,
struct fscrypt_str *crypto_str);
void fscrypt_fname_free_buffer(struct fscrypt_str *crypto_str);
int fscrypt_fname_disk_to_usr(const struct inode *inode,
bool fscrypt_match_name(const struct fscrypt_name *fname,
const u8 *de_name, u32 de_name_len);
u64 fscrypt_fname_siphash(const struct inode *dir, const struct qstr *name);
+int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags);
/* bio.c */
void fscrypt_decrypt_bio(struct bio *bio);
struct fscrypt_name *fname);
int fscrypt_prepare_setflags(struct inode *inode,
unsigned int oldflags, unsigned int flags);
-int __fscrypt_prepare_symlink(struct inode *dir, unsigned int len,
- unsigned int max_len,
- struct fscrypt_str *disk_link);
+int fscrypt_prepare_symlink(struct inode *dir, const char *target,
+ unsigned int len, unsigned int max_len,
+ struct fscrypt_str *disk_link);
int __fscrypt_encrypt_symlink(struct inode *inode, const char *target,
unsigned int len, struct fscrypt_str *disk_link);
const char *fscrypt_get_symlink(struct inode *inode, const void *caddr,
return false;
}
-static inline const union fscrypt_context *
-fscrypt_get_dummy_context(struct super_block *sb)
-{
- return NULL;
-}
-
static inline void fscrypt_handle_d_move(struct dentry *dentry)
{
}
return 0;
}
-static inline int fscrypt_inherit_context(struct inode *parent,
- struct inode *child,
- void *fs_data, bool preload)
+static inline int fscrypt_set_context(struct inode *inode, void *fs_data)
{
return -EOPNOTSUPP;
}
-struct fscrypt_dummy_context {
+struct fscrypt_dummy_policy {
};
static inline void fscrypt_show_test_dummy_encryption(struct seq_file *seq,
}
static inline void
-fscrypt_free_dummy_context(struct fscrypt_dummy_context *dummy_ctx)
+fscrypt_free_dummy_policy(struct fscrypt_dummy_policy *dummy_policy)
{
}
return -EOPNOTSUPP;
}
+static inline int fscrypt_prepare_new_inode(struct inode *dir,
+ struct inode *inode,
+ bool *encrypt_ret)
+{
+ if (IS_ENCRYPTED(dir))
+ return -EOPNOTSUPP;
+ return 0;
+}
+
static inline void fscrypt_put_encryption_info(struct inode *inode)
{
return;
return;
}
-static inline int fscrypt_fname_alloc_buffer(const struct inode *inode,
- u32 max_encrypted_len,
+static inline int fscrypt_fname_alloc_buffer(u32 max_encrypted_len,
struct fscrypt_str *crypto_str)
{
return -EOPNOTSUPP;
return 0;
}
+static inline int fscrypt_d_revalidate(struct dentry *dentry,
+ unsigned int flags)
+{
+ return 1;
+}
+
/* bio.c */
static inline void fscrypt_decrypt_bio(struct bio *bio)
{
return 0;
}
-static inline int __fscrypt_prepare_symlink(struct inode *dir,
- unsigned int len,
- unsigned int max_len,
- struct fscrypt_str *disk_link)
+static inline int fscrypt_prepare_symlink(struct inode *dir,
+ const char *target,
+ unsigned int len,
+ unsigned int max_len,
+ struct fscrypt_str *disk_link)
{
- return -EOPNOTSUPP;
+ if (IS_ENCRYPTED(dir))
+ return -EOPNOTSUPP;
+ disk_link->name = (unsigned char *)target;
+ disk_link->len = len + 1;
+ if (disk_link->len > max_len)
+ return -ENAMETOOLONG;
+ return 0;
}
-
static inline int __fscrypt_encrypt_symlink(struct inode *inode,
const char *target,
unsigned int len,
* @fname: (output) the name to use to search the on-disk directory
*
* Prepare for ->lookup() in a directory which may be encrypted by determining
- * the name that will actually be used to search the directory on-disk. Lookups
- * can be done with or without the directory's encryption key; without the key,
- * filenames are presented in encrypted form. Therefore, we'll try to set up
- * the directory's encryption key, but even without it the lookup can continue.
+ * the name that will actually be used to search the directory on-disk. If the
+ * directory's encryption key is available, then the lookup is assumed to be by
+ * plaintext name; otherwise, it is assumed to be by no-key name.
*
* This also installs a custom ->d_revalidate() method which will invalidate the
* dentry if it was created without the key and the key is later added.
*
- * Return: 0 on success; -ENOENT if key is unavailable but the filename isn't a
- * correctly formed encoded ciphertext name, so a negative dentry should be
- * created; or another -errno code.
+ * Return: 0 on success; -ENOENT if the directory's key is unavailable but the
+ * filename isn't a valid no-key name, so a negative dentry should be created;
+ * or another -errno code.
*/
static inline int fscrypt_prepare_lookup(struct inode *dir,
struct dentry *dentry,
return 0;
}
-/**
- * fscrypt_prepare_symlink() - prepare to create a possibly-encrypted symlink
- * @dir: directory in which the symlink is being created
- * @target: plaintext symlink target
- * @len: length of @target excluding null terminator
- * @max_len: space the filesystem has available to store the symlink target
- * @disk_link: (out) the on-disk symlink target being prepared
- *
- * This function computes the size the symlink target will require on-disk,
- * stores it in @disk_link->len, and validates it against @max_len. An
- * encrypted symlink may be longer than the original.
- *
- * Additionally, @disk_link->name is set to @target if the symlink will be
- * unencrypted, but left NULL if the symlink will be encrypted. For encrypted
- * symlinks, the filesystem must call fscrypt_encrypt_symlink() to create the
- * on-disk target later. (The reason for the two-step process is that some
- * filesystems need to know the size of the symlink target before creating the
- * inode, e.g. to determine whether it will be a "fast" or "slow" symlink.)
- *
- * Return: 0 on success, -ENAMETOOLONG if the symlink target is too long,
- * -ENOKEY if the encryption key is missing, or another -errno code if a problem
- * occurred while setting up the encryption key.
- */
-static inline int fscrypt_prepare_symlink(struct inode *dir,
- const char *target,
- unsigned int len,
- unsigned int max_len,
- struct fscrypt_str *disk_link)
-{
- if (IS_ENCRYPTED(dir) || fscrypt_get_dummy_context(dir->i_sb) != NULL)
- return __fscrypt_prepare_symlink(dir, len, max_len, disk_link);
-
- disk_link->name = (unsigned char *)target;
- disk_link->len = len + 1;
- if (disk_link->len > max_len)
- return -ENAMETOOLONG;
- return 0;
-}
-
/**
* fscrypt_encrypt_symlink() - encrypt the symlink target if needed
* @inode: symlink inode
enum gpiod_flags dflags);
void gpiochip_free_own_desc(struct gpio_desc *desc);
-void devprop_gpiochip_set_names(struct gpio_chip *gc,
- const struct fwnode_handle *fwnode);
-
#ifdef CONFIG_GPIOLIB
/* lock/unlock as IRQ */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * When building position independent code with GCC using the -fPIC option,
+ * (or even the -fPIE one on older versions), it will assume that we are
+ * building a dynamic object (either a shared library or an executable) that
+ * may have symbol references that can only be resolved at load time. For a
+ * variety of reasons (ELF symbol preemption, the CoW footprint of the section
+ * that is modified by the loader), this results in all references to symbols
+ * with external linkage to go via entries in the Global Offset Table (GOT),
+ * which carries absolute addresses which need to be fixed up when the
+ * executable image is loaded at an offset which is different from its link
+ * time offset.
+ *
+ * Fortunately, there is a way to inform the compiler that such symbol
+ * references will be satisfied at link time rather than at load time, by
+ * giving them 'hidden' visibility.
+ */
+
+#pragma GCC visibility push(hidden)
void tegra_mipi_free(struct tegra_mipi_device *device);
int tegra_mipi_enable(struct tegra_mipi_device *device);
int tegra_mipi_disable(struct tegra_mipi_device *device);
-int tegra_mipi_calibrate(struct tegra_mipi_device *device);
-int tegra_mipi_wait(struct tegra_mipi_device *device);
+int tegra_mipi_start_calibration(struct tegra_mipi_device *device);
+int tegra_mipi_finish_calibration(struct tegra_mipi_device *device);
#endif
hwmon_temp_lowest,
hwmon_temp_highest,
hwmon_temp_reset_history,
+ hwmon_temp_rated_min,
+ hwmon_temp_rated_max,
};
#define HWMON_T_ENABLE BIT(hwmon_temp_enable)
#define HWMON_T_LOWEST BIT(hwmon_temp_lowest)
#define HWMON_T_HIGHEST BIT(hwmon_temp_highest)
#define HWMON_T_RESET_HISTORY BIT(hwmon_temp_reset_history)
+#define HWMON_T_RATED_MIN BIT(hwmon_temp_rated_min)
+#define HWMON_T_RATED_MAX BIT(hwmon_temp_rated_max)
enum hwmon_in_attributes {
hwmon_in_enable,
hwmon_in_max_alarm,
hwmon_in_lcrit_alarm,
hwmon_in_crit_alarm,
+ hwmon_in_rated_min,
+ hwmon_in_rated_max,
};
#define HWMON_I_ENABLE BIT(hwmon_in_enable)
#define HWMON_I_MAX_ALARM BIT(hwmon_in_max_alarm)
#define HWMON_I_LCRIT_ALARM BIT(hwmon_in_lcrit_alarm)
#define HWMON_I_CRIT_ALARM BIT(hwmon_in_crit_alarm)
+#define HWMON_I_RATED_MIN BIT(hwmon_in_rated_min)
+#define HWMON_I_RATED_MAX BIT(hwmon_in_rated_max)
enum hwmon_curr_attributes {
hwmon_curr_enable,
hwmon_curr_max_alarm,
hwmon_curr_lcrit_alarm,
hwmon_curr_crit_alarm,
+ hwmon_curr_rated_min,
+ hwmon_curr_rated_max,
};
#define HWMON_C_ENABLE BIT(hwmon_curr_enable)
#define HWMON_C_MAX_ALARM BIT(hwmon_curr_max_alarm)
#define HWMON_C_LCRIT_ALARM BIT(hwmon_curr_lcrit_alarm)
#define HWMON_C_CRIT_ALARM BIT(hwmon_curr_crit_alarm)
+#define HWMON_C_RATED_MIN BIT(hwmon_curr_rated_min)
+#define HWMON_C_RATED_MAX BIT(hwmon_curr_rated_max)
enum hwmon_power_attributes {
hwmon_power_enable,
hwmon_power_max_alarm,
hwmon_power_lcrit_alarm,
hwmon_power_crit_alarm,
+ hwmon_power_rated_min,
+ hwmon_power_rated_max,
};
#define HWMON_P_ENABLE BIT(hwmon_power_enable)
#define HWMON_P_MAX_ALARM BIT(hwmon_power_max_alarm)
#define HWMON_P_LCRIT_ALARM BIT(hwmon_power_lcrit_alarm)
#define HWMON_P_CRIT_ALARM BIT(hwmon_power_crit_alarm)
+#define HWMON_P_RATED_MIN BIT(hwmon_power_rated_min)
+#define HWMON_P_RATED_MAX BIT(hwmon_power_rated_max)
enum hwmon_energy_attributes {
hwmon_energy_enable,
hwmon_humidity_max_hyst,
hwmon_humidity_alarm,
hwmon_humidity_fault,
+ hwmon_humidity_rated_min,
+ hwmon_humidity_rated_max,
};
#define HWMON_H_ENABLE BIT(hwmon_humidity_enable)
#define HWMON_H_MAX_HYST BIT(hwmon_humidity_max_hyst)
#define HWMON_H_ALARM BIT(hwmon_humidity_alarm)
#define HWMON_H_FAULT BIT(hwmon_humidity_fault)
+#define HWMON_H_RATED_MIN BIT(hwmon_humidity_rated_min)
+#define HWMON_H_RATED_MAX BIT(hwmon_humidity_rated_max)
enum hwmon_fan_attributes {
hwmon_fan_enable,
*
* This utility must be called between IIO device allocation
* (via devm_iio_device_alloc()) & IIO device registration
- * (via {devm_}iio_device_register()).
+ * (via iio_device_register() and devm_iio_device_register())).
* By default, the device allocation will also assign a parent device to
* the IIO device object. In cases where devm_iio_device_alloc() is used,
* sometimes the parent device must be different than the device used to
kcsan_check_write(v, size);
}
+/**
+ * instrument_read_write - instrument regular read-write access
+ *
+ * Instrument a regular write access. The instrumentation should be inserted
+ * before the actual write happens.
+ *
+ * @ptr address of access
+ * @size size of access
+ */
+static __always_inline void instrument_read_write(const volatile void *v, size_t size)
+{
+ kasan_check_write(v, size);
+ kcsan_check_read_write(v, size);
+}
+
/**
* instrument_atomic_read - instrument atomic read access
*
kcsan_check_atomic_write(v, size);
}
+/**
+ * instrument_atomic_read_write - instrument atomic read-write access
+ *
+ * Instrument an atomic read-write access. The instrumentation should be
+ * inserted before the actual write happens.
+ *
+ * @ptr address of access
+ * @size size of access
+ */
+static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size)
+{
+ kasan_check_write(v, size);
+ kcsan_check_atomic_read_write(v, size);
+}
+
/**
* instrument_copy_to_user - instrument reads of copy_to_user
*
int free_cnt;
};
+struct dmar_pci_notify_info;
+
#ifdef CONFIG_IRQ_REMAP
/* 1MB - maximum possible interrupt remapping table size */
#define INTR_REMAP_PAGE_ORDER 8
struct irte *base;
unsigned long *bitmap;
};
+
+void intel_irq_remap_add_device(struct dmar_pci_notify_info *info);
+#else
+static inline void
+intel_irq_remap_add_device(struct dmar_pci_notify_info *info) { }
#endif
struct iommu_flush {
2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
u64 max_addr; /* maximum mapped address */
- int default_pasid; /*
+ u32 default_pasid; /*
* The default pasid used for non-SVM
* traffic on mediated devices.
*/
u32 pasid, u16 qdep, u64 addr,
unsigned int size_order);
void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did, u64 granu,
- int pasid);
+ u32 pasid);
int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
unsigned int count, unsigned long options);
extern int intel_svm_finish_prq(struct intel_iommu *iommu);
int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev,
struct iommu_gpasid_bind_data *data);
-int intel_svm_unbind_gpasid(struct device *dev, int pasid);
+int intel_svm_unbind_gpasid(struct device *dev, u32 pasid);
struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm,
void *drvdata);
void intel_svm_unbind(struct iommu_sva *handle);
-int intel_svm_get_pasid(struct iommu_sva *handle);
+u32 intel_svm_get_pasid(struct iommu_sva *handle);
int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt,
struct iommu_page_response *msg);
struct device *dev;
struct svm_dev_ops *ops;
struct iommu_sva sva;
- int pasid;
+ u32 pasid;
int users;
u16 did;
u16 dev_iotlb:1;
struct mm_struct *mm;
struct intel_iommu *iommu;
- int flags;
- int pasid;
+ unsigned int flags;
+ u32 pasid;
int gpasid; /* In case that guest PASID is different from host PASID */
struct list_head devs;
struct list_head list;
struct device;
struct svm_dev_ops {
- void (*fault_cb)(struct device *dev, int pasid, u64 address,
+ void (*fault_cb)(struct device *dev, u32 pasid, u64 address,
void *private, int rwxp, int response);
};
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _LINUX_IO_URING_H
+#define _LINUX_IO_URING_H
+
+#include <linux/sched.h>
+#include <linux/xarray.h>
+#include <linux/percpu-refcount.h>
+
+struct io_uring_task {
+ /* submission side */
+ struct xarray xa;
+ struct wait_queue_head wait;
+ struct file *last;
+ atomic_long_t req_issue;
+
+ /* completion side */
+ bool in_idle ____cacheline_aligned_in_smp;
+ atomic_long_t req_complete;
+};
+
+#if defined(CONFIG_IO_URING)
+struct sock *io_uring_get_socket(struct file *file);
+void __io_uring_task_cancel(void);
+void __io_uring_files_cancel(struct files_struct *files);
+void __io_uring_free(struct task_struct *tsk);
+
+static inline void io_uring_task_cancel(void)
+{
+ if (current->io_uring && !xa_empty(¤t->io_uring->xa))
+ __io_uring_task_cancel();
+}
+static inline void io_uring_files_cancel(struct files_struct *files)
+{
+ if (current->io_uring && !xa_empty(¤t->io_uring->xa))
+ __io_uring_files_cancel(files);
+}
+static inline void io_uring_free(struct task_struct *tsk)
+{
+ if (tsk->io_uring)
+ __io_uring_free(tsk);
+}
+#else
+static inline struct sock *io_uring_get_socket(struct file *file)
+{
+ return NULL;
+}
+static inline void io_uring_task_cancel(void)
+{
+}
+static inline void io_uring_files_cancel(struct files_struct *files)
+{
+}
+static inline void io_uring_free(struct task_struct *tsk)
+{
+}
+#endif
+
+#endif
struct iommu_sva *(*sva_bind)(struct device *dev, struct mm_struct *mm,
void *drvdata);
void (*sva_unbind)(struct iommu_sva *handle);
- int (*sva_get_pasid)(struct iommu_sva *handle);
+ u32 (*sva_get_pasid)(struct iommu_sva *handle);
int (*page_response)(struct device *dev,
struct iommu_fault_event *evt,
int (*sva_bind_gpasid)(struct iommu_domain *domain,
struct device *dev, struct iommu_gpasid_bind_data *data);
- int (*sva_unbind_gpasid)(struct device *dev, int pasid);
+ int (*sva_unbind_gpasid)(struct device *dev, u32 pasid);
int (*def_domain_type)(struct device *dev);
struct mm_struct *mm,
void *drvdata);
void iommu_sva_unbind_device(struct iommu_sva *handle);
-int iommu_sva_get_pasid(struct iommu_sva *handle);
+u32 iommu_sva_get_pasid(struct iommu_sva *handle);
#else /* CONFIG_IOMMU_API */
{
}
-static inline int iommu_sva_get_pasid(struct iommu_sva *handle)
+static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle)
{
return IOMMU_PASID_INVALID;
}
}
static inline int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
- struct device *dev, int pasid)
+ struct device *dev, u32 pasid)
{
return -ENODEV;
}
* it from the spurious interrupt detection
* mechanism and from core side polling.
* IRQ_DISABLE_UNLAZY - Disable lazy irq disable
+ * IRQ_HIDDEN - Don't show up in /proc/interrupts
*/
enum {
IRQ_TYPE_NONE = 0x00000000,
IRQ_PER_CPU_DEVID = (1 << 17),
IRQ_IS_POLLED = (1 << 18),
IRQ_DISABLE_UNLAZY = (1 << 19),
+ IRQ_HIDDEN = (1 << 20),
};
#define IRQF_MODIFY_MASK \
(IRQ_TYPE_SENSE_MASK | IRQ_NOPROBE | IRQ_NOREQUEST | \
IRQ_NOAUTOEN | IRQ_MOVE_PCNTXT | IRQ_LEVEL | IRQ_NO_BALANCING | \
IRQ_PER_CPU | IRQ_NESTED_THREAD | IRQ_NOTHREAD | IRQ_PER_CPU_DEVID | \
- IRQ_IS_POLLED | IRQ_DISABLE_UNLAZY)
+ IRQ_IS_POLLED | IRQ_DISABLE_UNLAZY | IRQ_HIDDEN)
#define IRQ_NO_BALANCING_MASK (IRQ_PER_CPU | IRQ_NO_BALANCING)
* from actual interrupt context.
* IRQD_AFFINITY_ON_ACTIVATE - Affinity is set on activation. Don't call
* irq_chip::irq_set_affinity() when deactivated.
+ * IRQD_IRQ_ENABLED_ON_SUSPEND - Interrupt is enabled on suspend by irq pm if
+ * irqchip have flag IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND set.
*/
enum {
IRQD_TRIGGER_MASK = 0xf,
IRQD_MSI_NOMASK_QUIRK = (1 << 27),
IRQD_HANDLE_ENFORCE_IRQCTX = (1 << 28),
IRQD_AFFINITY_ON_ACTIVATE = (1 << 29),
+ IRQD_IRQ_ENABLED_ON_SUSPEND = (1 << 30),
};
#define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors)
return __irqd_to_state(d) & IRQD_HANDLE_ENFORCE_IRQCTX;
}
+static inline bool irqd_is_enabled_on_suspend(struct irq_data *d)
+{
+ return __irqd_to_state(d) & IRQD_IRQ_ENABLED_ON_SUSPEND;
+}
+
static inline bool irqd_is_wakeup_set(struct irq_data *d)
{
return __irqd_to_state(d) & IRQD_WAKEUP_STATE;
/*
* irq_chip specific flags
*
- * IRQCHIP_SET_TYPE_MASKED: Mask before calling chip.irq_set_type()
- * IRQCHIP_EOI_IF_HANDLED: Only issue irq_eoi() when irq was handled
- * IRQCHIP_MASK_ON_SUSPEND: Mask non wake irqs in the suspend path
- * IRQCHIP_ONOFFLINE_ENABLED: Only call irq_on/off_line callbacks
- * when irq enabled
- * IRQCHIP_SKIP_SET_WAKE: Skip chip.irq_set_wake(), for this irq chip
- * IRQCHIP_ONESHOT_SAFE: One shot does not require mask/unmask
- * IRQCHIP_EOI_THREADED: Chip requires eoi() on unmask in threaded mode
- * IRQCHIP_SUPPORTS_LEVEL_MSI Chip can provide two doorbells for Level MSIs
- * IRQCHIP_SUPPORTS_NMI: Chip can deliver NMIs, only for root irqchips
+ * IRQCHIP_SET_TYPE_MASKED: Mask before calling chip.irq_set_type()
+ * IRQCHIP_EOI_IF_HANDLED: Only issue irq_eoi() when irq was handled
+ * IRQCHIP_MASK_ON_SUSPEND: Mask non wake irqs in the suspend path
+ * IRQCHIP_ONOFFLINE_ENABLED: Only call irq_on/off_line callbacks
+ * when irq enabled
+ * IRQCHIP_SKIP_SET_WAKE: Skip chip.irq_set_wake(), for this irq chip
+ * IRQCHIP_ONESHOT_SAFE: One shot does not require mask/unmask
+ * IRQCHIP_EOI_THREADED: Chip requires eoi() on unmask in threaded mode
+ * IRQCHIP_SUPPORTS_LEVEL_MSI: Chip can provide two doorbells for Level MSIs
+ * IRQCHIP_SUPPORTS_NMI: Chip can deliver NMIs, only for root irqchips
+ * IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND: Invokes __enable_irq()/__disable_irq() for wake irqs
+ * in the suspend path if they are in disabled state
*/
enum {
- IRQCHIP_SET_TYPE_MASKED = (1 << 0),
- IRQCHIP_EOI_IF_HANDLED = (1 << 1),
- IRQCHIP_MASK_ON_SUSPEND = (1 << 2),
- IRQCHIP_ONOFFLINE_ENABLED = (1 << 3),
- IRQCHIP_SKIP_SET_WAKE = (1 << 4),
- IRQCHIP_ONESHOT_SAFE = (1 << 5),
- IRQCHIP_EOI_THREADED = (1 << 6),
- IRQCHIP_SUPPORTS_LEVEL_MSI = (1 << 7),
- IRQCHIP_SUPPORTS_NMI = (1 << 8),
+ IRQCHIP_SET_TYPE_MASKED = (1 << 0),
+ IRQCHIP_EOI_IF_HANDLED = (1 << 1),
+ IRQCHIP_MASK_ON_SUSPEND = (1 << 2),
+ IRQCHIP_ONOFFLINE_ENABLED = (1 << 3),
+ IRQCHIP_SKIP_SET_WAKE = (1 << 4),
+ IRQCHIP_ONESHOT_SAFE = (1 << 5),
+ IRQCHIP_EOI_THREADED = (1 << 6),
+ IRQCHIP_SUPPORTS_LEVEL_MSI = (1 << 7),
+ IRQCHIP_SUPPORTS_NMI = (1 << 8),
+ IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND = (1 << 9),
};
#include <linux/irqdesc.h>
*/
extern void handle_level_irq(struct irq_desc *desc);
extern void handle_fasteoi_irq(struct irq_desc *desc);
+extern void handle_percpu_devid_fasteoi_ipi(struct irq_desc *desc);
extern void handle_edge_irq(struct irq_desc *desc);
extern void handle_edge_eoi_irq(struct irq_desc *desc);
extern void handle_simple_irq(struct irq_desc *desc);
* top-level IRQ handler.
*/
extern void (*handle_arch_irq)(struct pt_regs *) __ro_after_init;
+#else
+#define set_handle_irq(handle_irq) \
+ do { \
+ (void)handle_irq; \
+ WARN_ON(1); \
+ } while (0)
#endif
#endif /* _LINUX_IRQ_H */
DOMAIN_BUS_FSL_MC_MSI,
DOMAIN_BUS_TI_SCI_INTA_MSI,
DOMAIN_BUS_WAKEUP,
+ DOMAIN_BUS_VMD_MSI,
};
/**
unsigned int irq_base,
unsigned int nr_irqs);
+extern int irq_domain_disconnect_hierarchy(struct irq_domain *domain,
+ unsigned int virq);
+
static inline bool irq_domain_is_hierarchy(struct irq_domain *domain)
{
return domain->flags & IRQ_DOMAIN_FLAG_HIERARCHY;
#include <linux/compiler_attributes.h>
#include <linux/types.h>
-/*
- * ACCESS TYPE MODIFIERS
- *
- * <none>: normal read access;
- * WRITE : write access;
- * ATOMIC: access is atomic;
- * ASSERT: access is not a regular access, but an assertion;
- * SCOPED: access is a scoped access;
- */
-#define KCSAN_ACCESS_WRITE 0x1
-#define KCSAN_ACCESS_ATOMIC 0x2
-#define KCSAN_ACCESS_ASSERT 0x4
-#define KCSAN_ACCESS_SCOPED 0x8
+/* Access types -- if KCSAN_ACCESS_WRITE is not set, the access is a read. */
+#define KCSAN_ACCESS_WRITE (1 << 0) /* Access is a write. */
+#define KCSAN_ACCESS_COMPOUND (1 << 1) /* Compounded read-write instrumentation. */
+#define KCSAN_ACCESS_ATOMIC (1 << 2) /* Access is atomic. */
+/* The following are special, and never due to compiler instrumentation. */
+#define KCSAN_ACCESS_ASSERT (1 << 3) /* Access is an assertion. */
+#define KCSAN_ACCESS_SCOPED (1 << 4) /* Access is a scoped access. */
/*
* __kcsan_*: Always calls into the runtime when KCSAN is enabled. This may be used
#define __kcsan_check_write(ptr, size) \
__kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
+/**
+ * __kcsan_check_read_write - check regular read-write access for races
+ *
+ * @ptr: address of access
+ * @size: size of access
+ */
+#define __kcsan_check_read_write(ptr, size) \
+ __kcsan_check_access(ptr, size, KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
+
/**
* kcsan_check_read - check regular read access for races
*
#define kcsan_check_write(ptr, size) \
kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
+/**
+ * kcsan_check_read_write - check regular read-write access for races
+ *
+ * @ptr: address of access
+ * @size: size of access
+ */
+#define kcsan_check_read_write(ptr, size) \
+ kcsan_check_access(ptr, size, KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
+
/*
* Check for atomic accesses: if atomic accesses are not ignored, this simply
* aliases to kcsan_check_access(), otherwise becomes a no-op.
*/
#ifdef CONFIG_KCSAN_IGNORE_ATOMICS
-#define kcsan_check_atomic_read(...) do { } while (0)
-#define kcsan_check_atomic_write(...) do { } while (0)
+#define kcsan_check_atomic_read(...) do { } while (0)
+#define kcsan_check_atomic_write(...) do { } while (0)
+#define kcsan_check_atomic_read_write(...) do { } while (0)
#else
#define kcsan_check_atomic_read(ptr, size) \
kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC)
#define kcsan_check_atomic_write(ptr, size) \
kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE)
+#define kcsan_check_atomic_read_write(ptr, size) \
+ kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_COMPOUND)
#endif
/**
#define KPF_ARCH 38
#define KPF_UNCACHED 39
#define KPF_SOFTDIRTY 40
+#define KPF_ARCH_2 41
#endif /* LINUX_KERNEL_PAGE_FLAGS_H */
extern void __khugepaged_exit(struct mm_struct *mm);
extern int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
unsigned long vm_flags);
+extern void khugepaged_min_free_kbytes_update(void);
#ifdef CONFIG_SHMEM
extern void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr);
#else
unsigned long addr)
{
}
+
+static inline void khugepaged_min_free_kbytes_update(void)
+{
+}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif /* _LINUX_KHUGEPAGED_H */
};
struct kretprobe_instance {
- struct hlist_node hlist;
+ union {
+ struct hlist_node hlist;
+ struct rcu_head rcu;
+ };
struct kretprobe *rp;
kprobe_opcode_t *ret_addr;
struct task_struct *task;
return 1;
}
+extern void kprobe_busy_begin(void);
+extern void kprobe_busy_end(void);
+
#ifdef CONFIG_KRETPROBES
extern void arch_prepare_kretprobe(struct kretprobe_instance *ri,
struct pt_regs *regs);
extern int arch_trampoline_kprobe(struct kprobe *p);
+
+/* If the trampoline handler called from a kprobe, use this version */
+unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
+ void *trampoline_address,
+ void *frame_pointer);
+
+static nokprobe_inline
+unsigned long kretprobe_trampoline_handler(struct pt_regs *regs,
+ void *trampoline_address,
+ void *frame_pointer)
+{
+ unsigned long ret;
+ /*
+ * Set a dummy kprobe for avoiding kretprobe recursion.
+ * Since kretprobe never runs in kprobe handler, no kprobe must
+ * be running at this point.
+ */
+ kprobe_busy_begin();
+ ret = __kretprobe_trampoline_handler(regs, trampoline_address, frame_pointer);
+ kprobe_busy_end();
+
+ return ret;
+}
+
#else /* CONFIG_KRETPROBES */
static inline void arch_prepare_kretprobe(struct kretprobe *rp,
struct pt_regs *regs)
extern struct kretprobe_blackpoint kretprobe_blacklist[];
-static inline void kretprobe_assert(struct kretprobe_instance *ri,
- unsigned long orig_ret_address, unsigned long trampoline_address)
-{
- if (!orig_ret_address || (orig_ret_address == trampoline_address)) {
- printk("kretprobe BUG!: Processing kretprobe %p @ %p\n",
- ri->rp, ri->rp->kp.addr);
- BUG();
- }
-}
-
#ifdef CONFIG_KPROBES_SANITY_TEST
extern int init_test_probes(void);
#else
/* Get the kprobe at this addr (if any) - called with preemption disabled */
struct kprobe *get_kprobe(void *addr);
-void kretprobe_hash_lock(struct task_struct *tsk,
- struct hlist_head **head, unsigned long *flags);
-void kretprobe_hash_unlock(struct task_struct *tsk, unsigned long *flags);
-struct hlist_head * kretprobe_inst_table_head(struct task_struct *tsk);
/* kprobe_running() will just return the current_kprobe on this CPU */
static inline struct kprobe *kprobe_running(void)
return this_cpu_ptr(&kprobe_ctlblk);
}
-extern struct kprobe kprobe_busy;
-void kprobe_busy_begin(void);
-void kprobe_busy_end(void);
-
kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset);
int register_kprobe(struct kprobe *p);
void unregister_kprobe(struct kprobe *p);
void unregister_kretprobes(struct kretprobe **rps, int num);
void kprobe_flush_task(struct task_struct *tk);
-void recycle_rp_inst(struct kretprobe_instance *ri, struct hlist_head *head);
void kprobe_free_init_mem(void);
struct lock_class *class;
struct lock_class *links_to;
const struct lock_trace *trace;
- int distance;
+ u16 distance;
+ /* bitmap of different dependencies from head to this */
+ u8 dep;
+ /* used by BFS to record whether "prev -> this" only has -(*R)-> */
+ u8 only_xr;
/*
* The parent field is used to implement breadth-first search, and the
}
#endif
+/* Variable used to make lockdep treat read_lock() as recursive in selftests */
+#ifdef CONFIG_DEBUG_LOCKING_API_SELFTESTS
+extern unsigned int force_read_lock_recursive;
+#else /* CONFIG_DEBUG_LOCKING_API_SELFTESTS */
+#define force_read_lock_recursive 0
+#endif /* CONFIG_DEBUG_LOCKING_API_SELFTESTS */
+
+#ifdef CONFIG_LOCKDEP
+extern bool read_lock_is_recursive(void);
+#else /* CONFIG_LOCKDEP */
+/* If !LOCKDEP, the value is meaningless */
+#define read_lock_is_recursive() 0
+#endif
+
/*
* For trivial one-depth nesting of a lock-class, the following
* global define can be used. (Subsystems with multiple levels
#define spin_release(l, i) lock_release(l, i)
#define rwlock_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i)
-#define rwlock_acquire_read(l, s, t, i) lock_acquire_shared_recursive(l, s, t, NULL, i)
+#define rwlock_acquire_read(l, s, t, i) \
+do { \
+ if (read_lock_is_recursive()) \
+ lock_acquire_shared_recursive(l, s, t, NULL, i); \
+ else \
+ lock_acquire_shared(l, s, t, NULL, i); \
+} while (0)
+
#define rwlock_release(l, i) lock_release(l, i)
#define seqcount_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i)
#define lock_map_release(l) lock_release(l, _THIS_IP_)
#ifdef CONFIG_PROVE_LOCKING
-# define might_lock(lock) \
+# define might_lock(lock) \
do { \
typecheck(struct lockdep_map *, &(lock)->dep_map); \
lock_acquire(&(lock)->dep_map, 0, 0, 0, 1, NULL, _THIS_IP_); \
lock_release(&(lock)->dep_map, _THIS_IP_); \
} while (0)
-# define might_lock_read(lock) \
+# define might_lock_read(lock) \
do { \
typecheck(struct lockdep_map *, &(lock)->dep_map); \
lock_acquire(&(lock)->dep_map, 0, 0, 1, 1, NULL, _THIS_IP_); \
lock_release(&(lock)->dep_map, _THIS_IP_); \
} while (0)
-# define might_lock_nested(lock, subclass) \
+# define might_lock_nested(lock, subclass) \
do { \
typecheck(struct lockdep_map *, &(lock)->dep_map); \
lock_acquire(&(lock)->dep_map, subclass, 0, 1, 1, NULL, \
DECLARE_PER_CPU(int, hardirqs_enabled);
DECLARE_PER_CPU(int, hardirq_context);
+DECLARE_PER_CPU(unsigned int, lockdep_recursion);
-/*
- * The below lockdep_assert_*() macros use raw_cpu_read() to access the above
- * per-cpu variables. This is required because this_cpu_read() will potentially
- * call into preempt/irq-disable and that obviously isn't right. This is also
- * correct because when IRQs are enabled, it doesn't matter if we accidentally
- * read the value from our previous CPU.
- */
+#define __lockdep_enabled (debug_locks && !this_cpu_read(lockdep_recursion))
#define lockdep_assert_irqs_enabled() \
do { \
- WARN_ON_ONCE(debug_locks && !raw_cpu_read(hardirqs_enabled)); \
+ WARN_ON_ONCE(__lockdep_enabled && !this_cpu_read(hardirqs_enabled)); \
} while (0)
#define lockdep_assert_irqs_disabled() \
do { \
- WARN_ON_ONCE(debug_locks && raw_cpu_read(hardirqs_enabled)); \
+ WARN_ON_ONCE(__lockdep_enabled && this_cpu_read(hardirqs_enabled)); \
} while (0)
#define lockdep_assert_in_irq() \
do { \
- WARN_ON_ONCE(debug_locks && !raw_cpu_read(hardirq_context)); \
+ WARN_ON_ONCE(__lockdep_enabled && !this_cpu_read(hardirq_context)); \
} while (0)
#define lockdep_assert_preemption_enabled() \
do { \
WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT) && \
- debug_locks && \
+ __lockdep_enabled && \
(preempt_count() != 0 || \
- !raw_cpu_read(hardirqs_enabled))); \
+ !this_cpu_read(hardirqs_enabled))); \
} while (0)
#define lockdep_assert_preemption_disabled() \
do { \
WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT) && \
- debug_locks && \
+ __lockdep_enabled && \
(preempt_count() == 0 && \
- raw_cpu_read(hardirqs_enabled))); \
+ this_cpu_read(hardirqs_enabled))); \
} while (0)
#else
/*
* We'd rather not expose kernel/lockdep_states.h this wide, but we do need
* the total number of states... :-(
+ *
+ * XXX_LOCK_USAGE_STATES is the number of lines in lockdep_states.h, for each
+ * of those we generates 4 states, Additionally we report on USED and USED_READ.
*/
-#define XXX_LOCK_USAGE_STATES (1+2*4)
+#define XXX_LOCK_USAGE_STATES 2
+#define LOCK_TRACE_STATES (XXX_LOCK_USAGE_STATES*4 + 2)
/*
* NR_LOCKDEP_CACHING_CLASSES ... Number of classes
* IRQ/softirq usage tracking bits:
*/
unsigned long usage_mask;
- const struct lock_trace *usage_traces[XXX_LOCK_USAGE_STATES];
+ const struct lock_trace *usage_traces[LOCK_TRACE_STATES];
/*
* Generation counter, when doing certain classes of graph walking,
struct memstick_dev *card;
unsigned int retries;
+ bool removing;
/* Notify the host that some requests are pending. */
void (*request)(struct memstick_host *host);
#define TC3589x_RSTCTRL_KBDRST (1 << 1)
#define TC3589x_RSTCTRL_GPIRST (1 << 0)
+#define TC3589x_DKBDMSK_ELINT (1 << 1)
+#define TC3589x_DKBDMSK_EINT (1 << 0)
+
/* Keyboard Configuration Registers */
#define TC3589x_KBDSETTLE_REG 0x01
#define TC3589x_KBDBOUNCE 0x02
#define TC3589x_GPIOODM2 0xE4
#define TC3589x_GPIOODE2 0xE5
+#define TC3589x_DIRECT0 0xEC
+#define TC3589x_DKBDMSK 0xF3
+
#define TC3589x_INT_GPIIRQ 0
#define TC3589x_INT_TI0IRQ 1
#define TC3589x_INT_TI1IRQ 2
u64 ts2;
u16 op;
bool polling;
+ /* Track the max comp handlers */
+ refcount_t refcnt;
};
struct mlx5_pas {
int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,
void *out, int out_size);
void mlx5_cmd_mbox_status(void *out, u8 *status, u32 *syndrome);
+bool mlx5_cmd_is_down(struct mlx5_core_dev *dev);
int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type);
int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn);
# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
#endif
+#if defined(CONFIG_ARM64_MTE)
+# define VM_MTE VM_HIGH_ARCH_0 /* Use Tagged memory for access control */
+# define VM_MTE_ALLOWED VM_HIGH_ARCH_1 /* Tagged memory permitted */
+#else
+# define VM_MTE VM_NONE
+# define VM_MTE_ALLOWED VM_NONE
+#endif
+
#ifndef VM_GROWSUP
# define VM_GROWSUP VM_NONE
#endif
void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
unsigned long end, unsigned long floor, unsigned long ceiling);
int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
- struct vm_area_struct *vma);
+ struct vm_area_struct *vma, struct vm_area_struct *new);
int follow_pte_pmd(struct mm_struct *mm, unsigned long address,
struct mmu_notifier_range *range,
pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp);
extern void set_dma_reserve(unsigned long new_dma_reserve);
extern void memmap_init_zone(unsigned long, int, unsigned long, unsigned long,
- enum memmap_context, struct vmem_altmap *);
+ enum meminit_context, struct vmem_altmap *);
extern void setup_per_zone_wmarks(void);
extern int __meminit init_per_zone_wmark_min(void);
extern void mem_init(void);
*/
atomic_t mm_count;
+ /**
+ * @has_pinned: Whether this mm has pinned any pages. This can
+ * be either replaced in the future by @pinned_vm when it
+ * becomes stable, or grow into a counter on its own. We're
+ * aggresive on this bit now - even if the pinned pages were
+ * unpinned later on, we'll still keep this bit set for the
+ * lifecycle of this mm just for simplicity.
+ */
+ atomic_t has_pinned;
+
#ifdef CONFIG_MMU
atomic_long_t pgtables_bytes; /* PTE page table pages */
#endif
atomic_long_t hugetlb_usage;
#endif
struct work_struct async_put_work;
+
+#ifdef CONFIG_IOMMU_SUPPORT
+ u32 pasid;
+#endif
} __randomize_layout;
/*
}
/*
- * Allow architectures to handle additional protection bits
+ * Allow architectures to handle additional protection and flag bits. The
+ * overriding macros must be defined in the arch-specific asm/mman.h file.
*/
#ifndef arch_calc_vm_prot_bits
#define arch_calc_vm_prot_bits(prot, pkey) 0
#endif
+#ifndef arch_calc_vm_flag_bits
+#define arch_calc_vm_flag_bits(flags) 0
+#endif
+
#ifndef arch_vm_get_page_prot
#define arch_vm_get_page_prot(vm_flags) __pgprot(0)
#endif
#define arch_validate_prot arch_validate_prot
#endif
+#ifndef arch_validate_flags
+/*
+ * This is called from mmap() and mprotect() with the updated vma->vm_flags.
+ *
+ * Returns true if the VM_* flags are valid.
+ */
+static inline bool arch_validate_flags(unsigned long flags)
+{
+ return true;
+}
+#define arch_validate_flags arch_validate_flags
+#endif
+
/*
* Optimisation macro. It is equivalent to:
* (x & bit1) ? bit2 : 0
return _calc_vm_trans(flags, MAP_GROWSDOWN, VM_GROWSDOWN ) |
_calc_vm_trans(flags, MAP_DENYWRITE, VM_DENYWRITE ) |
_calc_vm_trans(flags, MAP_LOCKED, VM_LOCKED ) |
- _calc_vm_trans(flags, MAP_SYNC, VM_SYNC );
+ _calc_vm_trans(flags, MAP_SYNC, VM_SYNC ) |
+ arch_calc_vm_flag_bits(flags);
}
unsigned long vm_commit_limit(void);
struct sdio_cis cis; /* common tuple info */
struct sdio_func *sdio_func[SDIO_MAX_FUNCS]; /* SDIO functions (devices) */
struct sdio_func *sdio_single_irq; /* SDIO function when only one IRQ active */
+ u8 major_rev; /* major revision number */
+ u8 minor_rev; /* minor revision number */
unsigned num_info; /* number of info strings */
const char **info; /* info strings */
struct sdio_func_tuple *tuples; /* unknown common tuples */
int (*select_drive_strength)(struct mmc_card *card,
unsigned int max_dtr, int host_drv,
int card_drv, int *drv_type);
+ /* Reset the eMMC card via RST_n */
void (*hw_reset)(struct mmc_host *host);
void (*card_event)(struct mmc_host *host);
#define MMC_CAP_CD_WAKE (1 << 28) /* Enable card detect wake */
#define MMC_CAP_CMD_DURING_TFR (1 << 29) /* Commands during data transfer */
#define MMC_CAP_CMD23 (1 << 30) /* CMD23 supported. */
-#define MMC_CAP_HW_RESET (1 << 31) /* Hardware reset */
+#define MMC_CAP_HW_RESET (1 << 31) /* Reset the eMMC card via RST_n */
u32 caps2; /* More host capabilities */
unsigned int use_spi_crc:1;
unsigned int claimed:1; /* host exclusively claimed */
unsigned int bus_dead:1; /* bus has been released */
+ unsigned int doing_init_tune:1; /* initial tuning in progress */
unsigned int can_retune:1; /* re-tuning can be used */
unsigned int doing_retune:1; /* re-tuning in progress */
unsigned int retune_now:1; /* do re-tuning at next req */
return host->doing_retune == 1;
}
+static inline bool mmc_doing_tune(struct mmc_host *host)
+{
+ return host->doing_retune == 1 || host->doing_init_tune == 1;
+}
+
static inline enum dma_data_direction mmc_get_dma_dir(struct mmc_data *data)
{
return data->flags & MMC_DATA_WRITE ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
u8 *tmpbuf; /* DMA:able scratch buffer */
+ u8 major_rev; /* major revision number */
+ u8 minor_rev; /* minor revision number */
unsigned num_info; /* number of info strings */
const char **info; /* info strings */
unsigned int alloc_flags);
bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
unsigned long mark, int highest_zoneidx);
-enum memmap_context {
- MEMMAP_EARLY,
- MEMMAP_HOTPLUG,
+/*
+ * Memory initialization context, use to differentiate memory added by
+ * the platform statically or via memory hotplug interface.
+ */
+enum meminit_context {
+ MEMINIT_EARLY,
+ MEMINIT_HOTPLUG,
};
+
extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn,
unsigned long size);
#include <linux/error-injection.h>
#include <linux/tracepoint-defs.h>
#include <linux/srcu.h>
+#include <linux/static_call_types.h>
#include <linux/percpu.h>
#include <asm/module.h>
unsigned long *kprobe_blacklist;
unsigned int num_kprobe_blacklist;
#endif
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+ int num_static_call_sites;
+ struct static_call_site *static_call_sites;
+#endif
#ifdef CONFIG_LIVEPATCH
bool klp; /* Is this a livepatch module? */
typedef struct gcry_mpi *MPI;
#define mpi_get_nlimbs(a) ((a)->nlimbs)
+#define mpi_has_sign(a) ((a)->sign)
/*-- mpiutil.c --*/
MPI mpi_alloc(unsigned nlimbs);
+void mpi_clear(MPI a);
void mpi_free(MPI a);
int mpi_resize(MPI a, unsigned nlimbs);
+static inline MPI mpi_new(unsigned int nbits)
+{
+ return mpi_alloc((nbits + BITS_PER_MPI_LIMB - 1) / BITS_PER_MPI_LIMB);
+}
+
+MPI mpi_copy(MPI a);
+MPI mpi_alloc_like(MPI a);
+void mpi_snatch(MPI w, MPI u);
+MPI mpi_set(MPI w, MPI u);
+MPI mpi_set_ui(MPI w, unsigned long u);
+MPI mpi_alloc_set_ui(unsigned long u);
+void mpi_swap_cond(MPI a, MPI b, unsigned long swap);
+
+/* Constants used to return constant MPIs. See mpi_init if you
+ * want to add more constants.
+ */
+#define MPI_NUMBER_OF_CONSTANTS 6
+enum gcry_mpi_constants {
+ MPI_C_ZERO,
+ MPI_C_ONE,
+ MPI_C_TWO,
+ MPI_C_THREE,
+ MPI_C_FOUR,
+ MPI_C_EIGHT
+};
+
+MPI mpi_const(enum gcry_mpi_constants no);
+
/*-- mpicoder.c --*/
+
+/* Different formats of external big integer representation. */
+enum gcry_mpi_format {
+ GCRYMPI_FMT_NONE = 0,
+ GCRYMPI_FMT_STD = 1, /* Twos complement stored without length. */
+ GCRYMPI_FMT_PGP = 2, /* As used by OpenPGP (unsigned only). */
+ GCRYMPI_FMT_SSH = 3, /* As used by SSH (like STD but with length). */
+ GCRYMPI_FMT_HEX = 4, /* Hex format. */
+ GCRYMPI_FMT_USG = 5, /* Like STD but unsigned. */
+ GCRYMPI_FMT_OPAQUE = 8 /* Opaque format (some functions only). */
+};
+
MPI mpi_read_raw_data(const void *xbuffer, size_t nbytes);
MPI mpi_read_from_buffer(const void *buffer, unsigned *ret_nread);
+int mpi_fromstr(MPI val, const char *str);
+MPI mpi_scanval(const char *string);
MPI mpi_read_raw_from_sgl(struct scatterlist *sgl, unsigned int len);
void *mpi_get_buffer(MPI a, unsigned *nbytes, int *sign);
int mpi_read_buffer(MPI a, uint8_t *buf, unsigned buf_len, unsigned *nbytes,
int *sign);
int mpi_write_to_sgl(MPI a, struct scatterlist *sg, unsigned nbytes,
int *sign);
+int mpi_print(enum gcry_mpi_format format, unsigned char *buffer,
+ size_t buflen, size_t *nwritten, MPI a);
+
+/*-- mpi-mod.c --*/
+void mpi_mod(MPI rem, MPI dividend, MPI divisor);
+
+/* Context used with Barrett reduction. */
+struct barrett_ctx_s;
+typedef struct barrett_ctx_s *mpi_barrett_t;
+
+mpi_barrett_t mpi_barrett_init(MPI m, int copy);
+void mpi_barrett_free(mpi_barrett_t ctx);
+void mpi_mod_barrett(MPI r, MPI x, mpi_barrett_t ctx);
+void mpi_mul_barrett(MPI w, MPI u, MPI v, mpi_barrett_t ctx);
/*-- mpi-pow.c --*/
int mpi_powm(MPI res, MPI base, MPI exp, MPI mod);
/*-- mpi-cmp.c --*/
int mpi_cmp_ui(MPI u, ulong v);
int mpi_cmp(MPI u, MPI v);
+int mpi_cmpabs(MPI u, MPI v);
/*-- mpi-sub-ui.c --*/
int mpi_sub_ui(MPI w, MPI u, unsigned long vval);
/*-- mpi-bit.c --*/
void mpi_normalize(MPI a);
unsigned mpi_get_nbits(MPI a);
+int mpi_test_bit(MPI a, unsigned int n);
+void mpi_set_bit(MPI a, unsigned int n);
+void mpi_set_highbit(MPI a, unsigned int n);
+void mpi_clear_highbit(MPI a, unsigned int n);
+void mpi_clear_bit(MPI a, unsigned int n);
+void mpi_rshift_limbs(MPI a, unsigned int count);
+void mpi_rshift(MPI x, MPI a, unsigned int n);
+void mpi_lshift_limbs(MPI a, unsigned int count);
+void mpi_lshift(MPI x, MPI a, unsigned int n);
+
+/*-- mpi-add.c --*/
+void mpi_add_ui(MPI w, MPI u, unsigned long v);
+void mpi_add(MPI w, MPI u, MPI v);
+void mpi_sub(MPI w, MPI u, MPI v);
+void mpi_addm(MPI w, MPI u, MPI v, MPI m);
+void mpi_subm(MPI w, MPI u, MPI v, MPI m);
+
+/*-- mpi-mul.c --*/
+void mpi_mul(MPI w, MPI u, MPI v);
+void mpi_mulm(MPI w, MPI u, MPI v, MPI m);
+
+/*-- mpi-div.c --*/
+void mpi_tdiv_r(MPI rem, MPI num, MPI den);
+void mpi_fdiv_r(MPI rem, MPI dividend, MPI divisor);
+void mpi_fdiv_q(MPI quot, MPI dividend, MPI divisor);
+
+/*-- mpi-inv.c --*/
+int mpi_invm(MPI x, MPI a, MPI n);
+
+/*-- ec.c --*/
+
+/* Object to represent a point in projective coordinates */
+struct gcry_mpi_point {
+ MPI x;
+ MPI y;
+ MPI z;
+};
+
+typedef struct gcry_mpi_point *MPI_POINT;
+
+/* Models describing an elliptic curve */
+enum gcry_mpi_ec_models {
+ /* The Short Weierstrass equation is
+ * y^2 = x^3 + ax + b
+ */
+ MPI_EC_WEIERSTRASS = 0,
+ /* The Montgomery equation is
+ * by^2 = x^3 + ax^2 + x
+ */
+ MPI_EC_MONTGOMERY,
+ /* The Twisted Edwards equation is
+ * ax^2 + y^2 = 1 + bx^2y^2
+ * Note that we use 'b' instead of the commonly used 'd'.
+ */
+ MPI_EC_EDWARDS
+};
+
+/* Dialects used with elliptic curves */
+enum ecc_dialects {
+ ECC_DIALECT_STANDARD = 0,
+ ECC_DIALECT_ED25519,
+ ECC_DIALECT_SAFECURVE
+};
+
+/* This context is used with all our EC functions. */
+struct mpi_ec_ctx {
+ enum gcry_mpi_ec_models model; /* The model describing this curve. */
+ enum ecc_dialects dialect; /* The ECC dialect used with the curve. */
+ int flags; /* Public key flags (not always used). */
+ unsigned int nbits; /* Number of bits. */
+
+ /* Domain parameters. Note that they may not all be set and if set
+ * the MPIs may be flaged as constant.
+ */
+ MPI p; /* Prime specifying the field GF(p). */
+ MPI a; /* First coefficient of the Weierstrass equation. */
+ MPI b; /* Second coefficient of the Weierstrass equation. */
+ MPI_POINT G; /* Base point (generator). */
+ MPI n; /* Order of G. */
+ unsigned int h; /* Cofactor. */
+
+ /* The actual key. May not be set. */
+ MPI_POINT Q; /* Public key. */
+ MPI d; /* Private key. */
+
+ const char *name; /* Name of the curve. */
+
+ /* This structure is private to mpi/ec.c! */
+ struct {
+ struct {
+ unsigned int a_is_pminus3:1;
+ unsigned int two_inv_p:1;
+ } valid; /* Flags to help setting the helper vars below. */
+
+ int a_is_pminus3; /* True if A = P - 3. */
+
+ MPI two_inv_p;
+
+ mpi_barrett_t p_barrett;
+
+ /* Scratch variables. */
+ MPI scratch[11];
+
+ /* Helper for fast reduction. */
+ /* int nist_nbits; /\* If this is a NIST curve, the # of bits. *\/ */
+ /* MPI s[10]; */
+ /* MPI c; */
+ } t;
+
+ /* Curve specific computation routines for the field. */
+ void (*addm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx);
+ void (*subm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ec);
+ void (*mulm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx);
+ void (*pow2)(MPI w, const MPI b, struct mpi_ec_ctx *ctx);
+ void (*mul2)(MPI w, MPI u, struct mpi_ec_ctx *ctx);
+};
+
+void mpi_ec_init(struct mpi_ec_ctx *ctx, enum gcry_mpi_ec_models model,
+ enum ecc_dialects dialect,
+ int flags, MPI p, MPI a, MPI b);
+void mpi_ec_deinit(struct mpi_ec_ctx *ctx);
+MPI_POINT mpi_point_new(unsigned int nbits);
+void mpi_point_release(MPI_POINT p);
+void mpi_point_init(MPI_POINT p);
+void mpi_point_free_parts(MPI_POINT p);
+int mpi_ec_get_affine(MPI x, MPI y, MPI_POINT point, struct mpi_ec_ctx *ctx);
+void mpi_ec_add_points(MPI_POINT result,
+ MPI_POINT p1, MPI_POINT p2,
+ struct mpi_ec_ctx *ctx);
+void mpi_ec_mul_point(MPI_POINT result,
+ MPI scalar, MPI_POINT point,
+ struct mpi_ec_ctx *ctx);
+int mpi_ec_curve_point(MPI_POINT point, struct mpi_ec_ctx *ctx);
/* inline functions */
void pci_msi_unmask_irq(struct irq_data *data);
/*
- * The arch hooks to setup up msi irqs. Those functions are
- * implemented as weak symbols so that they /can/ be overriden by
- * architecture specific code if needed.
+ * The arch hooks to setup up msi irqs. Default functions are implemented
+ * as weak symbols so that they /can/ be overriden by architecture specific
+ * code if needed. These hooks must be enabled by the architecture or by
+ * drivers which depend on them via msi_controller based MSI handling.
+ *
+ * If CONFIG_PCI_MSI_ARCH_FALLBACKS is not selected they are replaced by
+ * stubs with warnings.
*/
+#ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS
int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc);
void arch_teardown_msi_irq(unsigned int irq);
int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type);
void arch_teardown_msi_irqs(struct pci_dev *dev);
-void arch_restore_msi_irqs(struct pci_dev *dev);
-
void default_teardown_msi_irqs(struct pci_dev *dev);
+#else
+static inline int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
+{
+ WARN_ON_ONCE(1);
+ return -ENODEV;
+}
+
+static inline void arch_teardown_msi_irqs(struct pci_dev *dev)
+{
+ WARN_ON_ONCE(1);
+}
+#endif
+
+/*
+ * The restore hooks are still available as they are useful even
+ * for fully irq domain based setups. Courtesy to XEN/X86.
+ */
+void arch_restore_msi_irqs(struct pci_dev *dev);
void default_restore_msi_irqs(struct pci_dev *dev);
struct msi_controller {
* @msi_finish: Optional callback to finalize the allocation
* @set_desc: Set the msi descriptor for an interrupt
* @handle_error: Optional error handler if the allocation fails
+ * @domain_alloc_irqs: Optional function to override the default allocation
+ * function.
+ * @domain_free_irqs: Optional function to override the default free
+ * function.
*
* @get_hwirq, @msi_init and @msi_free are callbacks used by
* msi_create_irq_domain() and related interfaces
* @msi_check, @msi_prepare, @msi_finish, @set_desc and @handle_error
* are callbacks used by msi_domain_alloc_irqs() and related
* interfaces which are based on msi_desc.
+ *
+ * @domain_alloc_irqs, @domain_free_irqs can be used to override the
+ * default allocation/free functions (__msi_domain_alloc/free_irqs). This
+ * is initially for a wrapper around XENs seperate MSI universe which can't
+ * be wrapped into the regular irq domains concepts by mere mortals. This
+ * allows to universally use msi_domain_alloc/free_irqs without having to
+ * special case XEN all over the place.
+ *
+ * Contrary to other operations @domain_alloc_irqs and @domain_free_irqs
+ * are set to the default implementation if NULL and even when
+ * MSI_FLAG_USE_DEF_DOM_OPS is not set to avoid breaking existing users and
+ * because these callbacks are obviously mandatory.
+ *
+ * This is NOT meant to be abused, but it can be useful to build wrappers
+ * for specialized MSI irq domains which need extra work before and after
+ * calling __msi_domain_alloc_irqs()/__msi_domain_free_irqs().
*/
struct msi_domain_ops {
irq_hw_number_t (*get_hwirq)(struct msi_domain_info *info,
struct msi_desc *desc);
int (*handle_error)(struct irq_domain *domain,
struct msi_desc *desc, int error);
+ int (*domain_alloc_irqs)(struct irq_domain *domain,
+ struct device *dev, int nvec);
+ void (*domain_free_irqs)(struct irq_domain *domain,
+ struct device *dev);
};
/**
struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode,
struct msi_domain_info *info,
struct irq_domain *parent);
+int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
+ int nvec);
int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
int nvec);
+void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev);
void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev);
struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain);
struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
struct msi_domain_info *info,
struct irq_domain *parent);
-irq_hw_number_t pci_msi_domain_calc_hwirq(struct pci_dev *dev,
- struct msi_desc *desc);
int pci_msi_domain_check_cap(struct irq_domain *domain,
struct msi_domain_info *info, struct device *dev);
u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev);
struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev);
+bool pci_dev_has_special_msi_domain(struct pci_dev *pdev);
#else
static inline struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)
{
#include <linux/rcupdate.h>
#include <linux/once.h>
#include <linux/fs.h>
+#include <linux/mm.h>
#include <linux/sockptr.h>
#include <uapi/linux/net.h>
#define net_get_random_once_wait(buf, nbytes) \
get_random_once_wait((buf), (nbytes))
+/*
+ * E.g. XFS meta- & log-data is in slab pages, or bcache meta
+ * data pages, or other high order pages allocated by
+ * __get_free_pages() without __GFP_COMP, which have a page_count
+ * of 0 and/or have PageSlab() set. We cannot use send_page for
+ * those, as that does get_page(); put_page(); and would cause
+ * either a VM_BUG directly, or __page_cache_release a page that
+ * would actually still be referenced by someone, leading to some
+ * obscure delayed Oops somewhere else.
+ */
+static inline bool sendpage_ok(struct page *page)
+{
+ return !PageSlab(page) && page_count(page) >= 1;
+}
+
int kernel_sendmsg(struct socket *sock, struct msghdr *msg, struct kvec *vec,
size_t num, size_t len);
int kernel_sendmsg_locked(struct sock *sk, struct msghdr *msg,
* @udp_tunnel_nic: UDP tunnel offload state
* @xdp_state: stores info on attached XDP BPF programs
*
+ * @nested_level: Used as as a parameter of spin_lock_nested() of
+ * dev->addr_list_lock.
+ * @unlink_list: As netif_addr_lock() can be called recursively,
+ * keep a list of interfaces to be deleted.
+ *
* FIXME: cleanup struct net_device such that network protocol info
* moves out.
*/
unsigned short type;
unsigned short hard_header_len;
unsigned char min_header_len;
+ unsigned char name_assign_type;
unsigned short needed_headroom;
unsigned short needed_tailroom;
unsigned char addr_len;
unsigned char upper_level;
unsigned char lower_level;
+
unsigned short neigh_priv_len;
unsigned short dev_id;
unsigned short dev_port;
spinlock_t addr_list_lock;
- unsigned char name_assign_type;
- bool uc_promisc;
+
struct netdev_hw_addr_list uc;
struct netdev_hw_addr_list mc;
struct netdev_hw_addr_list dev_addrs;
#ifdef CONFIG_SYSFS
struct kset *queues_kset;
+#endif
+#ifdef CONFIG_LOCKDEP
+ struct list_head unlink_list;
#endif
unsigned int promiscuity;
unsigned int allmulti;
+ bool uc_promisc;
+#ifdef CONFIG_LOCKDEP
+ unsigned char nested_level;
+#endif
/* Protocol-specific pointers */
static inline void netif_addr_lock(struct net_device *dev)
{
- spin_lock(&dev->addr_list_lock);
-}
+ unsigned char nest_level = 0;
-static inline void netif_addr_lock_nested(struct net_device *dev)
-{
- spin_lock_nested(&dev->addr_list_lock, dev->lower_level);
+#ifdef CONFIG_LOCKDEP
+ nest_level = dev->nested_level;
+#endif
+ spin_lock_nested(&dev->addr_list_lock, nest_level);
}
static inline void netif_addr_lock_bh(struct net_device *dev)
{
- spin_lock_bh(&dev->addr_list_lock);
+ unsigned char nest_level = 0;
+
+#ifdef CONFIG_LOCKDEP
+ nest_level = dev->nested_level;
+#endif
+ local_bh_disable();
+ spin_lock_nested(&dev->addr_list_lock, nest_level);
}
static inline void netif_addr_unlock(struct net_device *dev)
extern int dev_tx_weight;
extern int gro_normal_batch;
+enum {
+ NESTED_SYNC_IMM_BIT,
+ NESTED_SYNC_TODO_BIT,
+};
+
+#define __NESTED_SYNC_BIT(bit) ((u32)1 << (bit))
+#define __NESTED_SYNC(name) __NESTED_SYNC_BIT(NESTED_SYNC_ ## name ## _BIT)
+
+#define NESTED_SYNC_IMM __NESTED_SYNC(IMM)
+#define NESTED_SYNC_TODO __NESTED_SYNC(TODO)
+
+struct netdev_nested_priv {
+ unsigned char flags;
+ void *data;
+};
+
bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
struct list_head **iter);
struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
struct list_head **iter);
+#ifdef CONFIG_LOCKDEP
+static LIST_HEAD(net_unlink_list);
+
+static inline void net_unlink_todo(struct net_device *dev)
+{
+ if (list_empty(&dev->unlink_list))
+ list_add_tail(&dev->unlink_list, &net_unlink_list);
+}
+#endif
+
/* iterate through upper list, must be called under RCU read lock */
#define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
for (iter = &(dev)->adj_list.upper, \
int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
int (*fn)(struct net_device *upper_dev,
- void *data),
- void *data);
+ struct netdev_nested_priv *priv),
+ struct netdev_nested_priv *priv);
bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
struct net_device *upper_dev);
struct list_head **iter);
int netdev_walk_all_lower_dev(struct net_device *dev,
int (*fn)(struct net_device *lower_dev,
- void *data),
- void *data);
+ struct netdev_nested_priv *priv),
+ struct netdev_nested_priv *priv);
int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
int (*fn)(struct net_device *lower_dev,
- void *data),
- void *data);
+ struct netdev_nested_priv *priv),
+ struct netdev_nested_priv *priv);
void *netdev_adjacent_get_private(struct list_head *adj_list);
void *netdev_lower_get_first_private_rcu(struct net_device *dev);
__u64 mds_offset; /* Filelayout dense stripe */
struct nfs_page_array page_array;
struct nfs_client *ds_clp; /* pNFS data server */
- int ds_commit_idx; /* ds index if ds_clp is set */
- int pgio_mirror_idx;/* mirror index in pgio layer */
+ u32 ds_commit_idx; /* ds index if ds_clp is set */
+ u32 pgio_mirror_idx;/* mirror index in pgio layer */
};
struct nfs_mds_commit_info {
typedef void (*node_registration_func_t)(struct node *);
#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) && defined(CONFIG_NUMA)
-extern int link_mem_sections(int nid, unsigned long start_pfn,
- unsigned long end_pfn);
+int link_mem_sections(int nid, unsigned long start_pfn,
+ unsigned long end_pfn,
+ enum meminit_context context);
#else
static inline int link_mem_sections(int nid, unsigned long start_pfn,
- unsigned long end_pfn)
+ unsigned long end_pfn,
+ enum meminit_context context)
{
return 0;
}
if (error)
return error;
/* link memory sections under this node */
- error = link_mem_sections(nid, start_pfn, end_pfn);
+ error = link_mem_sections(nid, start_pfn, end_pfn,
+ MEMINIT_EARLY);
}
return error;
extern int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
unsigned long val, void *v);
-extern int __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
- unsigned long val, void *v, int nr_to_call, int *nr_calls);
extern int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
unsigned long val, void *v);
-extern int __blocking_notifier_call_chain(struct blocking_notifier_head *nh,
- unsigned long val, void *v, int nr_to_call, int *nr_calls);
extern int raw_notifier_call_chain(struct raw_notifier_head *nh,
unsigned long val, void *v);
-extern int __raw_notifier_call_chain(struct raw_notifier_head *nh,
- unsigned long val, void *v, int nr_to_call, int *nr_calls);
extern int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
unsigned long val, void *v);
-extern int __srcu_notifier_call_chain(struct srcu_notifier_head *nh,
- unsigned long val, void *v, int nr_to_call, int *nr_calls);
+
+extern int atomic_notifier_call_chain_robust(struct atomic_notifier_head *nh,
+ unsigned long val_up, unsigned long val_down, void *v);
+extern int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
+ unsigned long val_up, unsigned long val_down, void *v);
+extern int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
+ unsigned long val_up, unsigned long val_down, void *v);
#define NOTIFY_DONE 0x0000 /* Don't care */
#define NOTIFY_OK 0x0001 /* Suits me */
OID_gostTC26Sign512B, /* 1.2.643.7.1.2.1.2.2 */
OID_gostTC26Sign512C, /* 1.2.643.7.1.2.1.2.3 */
+ /* OSCCA */
+ OID_sm2, /* 1.2.156.10197.1.301 */
+ OID_sm3, /* 1.2.156.10197.1.401 */
+ OID_SM2_with_SM3, /* 1.2.156.10197.1.501 */
+ OID_sm3WithRSAEncryption, /* 1.2.156.10197.1.504 */
+
OID__NR
};
#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT)
PG_young,
PG_idle,
+#endif
+#ifdef CONFIG_64BIT
+ PG_arch_2,
#endif
__NR_PAGEFLAGS,
__filemap_set_wb_err(mapping, error);
/* Record it in superblock */
- errseq_set(&mapping->host->i_sb->s_wb_err, error);
+ if (mapping->host)
+ errseq_set(&mapping->host->i_sb->s_wb_err, error);
/* Record it in flags for now, for legacy callers */
if (error == -ENOSPC)
#define IMAGE_FILE_MACHINE_POWERPC 0x01f0
#define IMAGE_FILE_MACHINE_POWERPCFP 0x01f1
#define IMAGE_FILE_MACHINE_R4000 0x0166
+#define IMAGE_FILE_MACHINE_RISCV32 0x5032
+#define IMAGE_FILE_MACHINE_RISCV64 0x5064
+#define IMAGE_FILE_MACHINE_RISCV128 0x5128
#define IMAGE_FILE_MACHINE_SH3 0x01a2
#define IMAGE_FILE_MACHINE_SH3DSP 0x01a3
#define IMAGE_FILE_MACHINE_SH3E 0x01a4
PERCPU_REF_ALLOW_REINIT = 1 << 2,
};
-struct percpu_ref {
+struct percpu_ref_data {
atomic_long_t count;
- /*
- * The low bit of the pointer indicates whether the ref is in percpu
- * mode; if set, then get/put will manipulate the atomic_t.
- */
- unsigned long percpu_count_ptr;
percpu_ref_func_t *release;
percpu_ref_func_t *confirm_switch;
bool force_atomic:1;
bool allow_reinit:1;
struct rcu_head rcu;
+ struct percpu_ref *ref;
+};
+
+struct percpu_ref {
+ /*
+ * The low bit of the pointer indicates whether the ref is in percpu
+ * mode; if set, then get/put will manipulate the atomic_t.
+ */
+ unsigned long percpu_count_ptr;
+
+ /*
+ * 'percpu_ref' is often embedded into user structure, and only
+ * 'percpu_count_ptr' is required in fast path, move other fields
+ * into 'percpu_ref_data', so we can reduce memory footprint in
+ * fast path.
+ */
+ struct percpu_ref_data *data;
};
int __must_check percpu_ref_init(struct percpu_ref *ref,
percpu_ref_func_t *confirm_kill);
void percpu_ref_resurrect(struct percpu_ref *ref);
void percpu_ref_reinit(struct percpu_ref *ref);
+bool percpu_ref_is_zero(struct percpu_ref *ref);
/**
* percpu_ref_kill - drop the initial ref
if (__ref_is_percpu(ref, &percpu_count))
this_cpu_add(*percpu_count, nr);
else
- atomic_long_add(nr, &ref->count);
+ atomic_long_add(nr, &ref->data->count);
rcu_read_unlock();
}
this_cpu_add(*percpu_count, nr);
ret = true;
} else {
- ret = atomic_long_add_unless(&ref->count, nr, 0);
+ ret = atomic_long_add_unless(&ref->data->count, nr, 0);
}
rcu_read_unlock();
this_cpu_inc(*percpu_count);
ret = true;
} else if (!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)) {
- ret = atomic_long_inc_not_zero(&ref->count);
+ ret = atomic_long_inc_not_zero(&ref->data->count);
}
rcu_read_unlock();
if (__ref_is_percpu(ref, &percpu_count))
this_cpu_sub(*percpu_count, nr);
- else if (unlikely(atomic_long_sub_and_test(nr, &ref->count)))
- ref->release(ref);
+ else if (unlikely(atomic_long_sub_and_test(nr, &ref->data->count)))
+ ref->data->release(ref);
rcu_read_unlock();
}
return ref->percpu_count_ptr & __PERCPU_REF_DEAD;
}
-/**
- * percpu_ref_is_zero - test whether a percpu refcount reached zero
- * @ref: percpu_ref to test
- *
- * Returns %true if @ref reached zero.
- *
- * This function is safe to call as long as @ref is between init and exit.
- */
-static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
-{
- unsigned long __percpu *percpu_count;
-
- if (__ref_is_percpu(ref, &percpu_count))
- return false;
- return !atomic_long_read(&ref->count);
-}
-
#endif
ARMPMU_ATTR_GROUP_COMMON,
ARMPMU_ATTR_GROUP_EVENTS,
ARMPMU_ATTR_GROUP_FORMATS,
+ ARMPMU_ATTR_GROUP_CAPS,
ARMPMU_NR_ATTR_GROUPS
};
struct notifier_block cpu_pm_nb;
/* the attr_groups array must be NULL-terminated */
const struct attribute_group *attr_groups[ARMPMU_NR_ATTR_GROUPS + 1];
+ /* store the PMMIR_EL1 to expose slots */
+ u64 reg_pmmir;
/* Only to be used by ACPI probing code */
unsigned long acpi_cpuid;
*/
u64 sample_period;
- /*
- * The period we started this sample with.
- */
- u64 last_period;
+ union {
+ struct { /* Sampling */
+ /*
+ * The period we started this sample with.
+ */
+ u64 last_period;
- /*
- * However much is left of the current period; note that this is
- * a full 64bit value and allows for generation of periods longer
- * than hardware might allow.
- */
- local64_t period_left;
+ /*
+ * However much is left of the current period;
+ * note that this is a full 64bit value and
+ * allows for generation of periods longer
+ * than hardware might allow.
+ */
+ local64_t period_left;
+ };
+ struct { /* Topdown events counting for context switch */
+ u64 saved_metric;
+ u64 saved_slots;
+ };
+ };
/*
* State for throttling the event, see __perf_event_overflow() and
* PERF_EV_CAP_SOFTWARE: Is a software event.
* PERF_EV_CAP_READ_ACTIVE_PKG: A CPU event (or cgroup event) that can be read
* from any CPU in the package where it is active.
+ * PERF_EV_CAP_SIBLING: An event with this flag must be a group sibling and
+ * cannot be a group leader. If an event with this flag is detached from the
+ * group it is scheduled out and moved into an unrecoverable ERROR state.
*/
#define PERF_EV_CAP_SOFTWARE BIT(0)
#define PERF_EV_CAP_READ_ACTIVE_PKG BIT(1)
+#define PERF_EV_CAP_SIBLING BIT(2)
#define SWEVENT_HLIST_BITS 8
#define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
struct list_head cgrp_cpuctx_entry;
#endif
- struct list_head sched_cb_entry;
int sched_cb_usage;
int online;
}
#endif
+/*
+ * Allow architectures to preserve additional metadata associated with
+ * swapped-out pages. The corresponding __HAVE_ARCH_SWAP_* macros and function
+ * prototypes must be defined in the arch-specific asm/pgtable.h file.
+ */
+#ifndef __HAVE_ARCH_PREPARE_TO_SWAP
+static inline int arch_prepare_to_swap(struct page *page)
+{
+ return 0;
+}
+#endif
+
+#ifndef __HAVE_ARCH_SWAP_INVALIDATE
+static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
+{
+}
+
+static inline void arch_swap_invalidate_area(int type)
+{
+}
+#endif
+
+#ifndef __HAVE_ARCH_SWAP_RESTORE
+static inline void arch_swap_restore(swp_entry_t entry, struct page *page)
+{
+}
+#endif
+
#ifndef __HAVE_ARCH_PGD_OFFSET_GATE
#define pgd_offset_gate(mm, addr) pgd_offset(mm, addr)
#endif
#define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED)
#endif
+#ifndef p4d_offset_lockless
+#define p4d_offset_lockless(pgdp, pgd, address) p4d_offset(&(pgd), address)
+#endif
+#ifndef pud_offset_lockless
+#define pud_offset_lockless(p4dp, p4d, address) pud_offset(&(p4d), address)
+#endif
+#ifndef pmd_offset_lockless
+#define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address)
+#endif
+
/*
* p?d_leaf() - true if this entry is a final mapping to a physical address.
* This differs from p?d_huge() by the fact that they are always available (if
extern unsigned long pipe_user_pages_hard;
extern unsigned long pipe_user_pages_soft;
-/* Drop the inode semaphore and wait for a pipe event, atomically */
-void pipe_wait(struct pipe_inode_info *pipe);
+/* Wait for a pipe to be readable/writable while dropping the pipe lock */
+void pipe_wait_readable(struct pipe_inode_info *);
+void pipe_wait_writable(struct pipe_inode_info *);
struct pipe_inode_info *alloc_pipe_info(void);
void free_pipe_info(struct pipe_inode_info *);
#ifndef GPIO_DW_APB_H
#define GPIO_DW_APB_H
+#define DWAPB_MAX_GPIOS 32
+
struct dwapb_port_property {
struct fwnode_handle *fwnode;
unsigned int idx;
unsigned int ngpio;
unsigned int gpio_base;
- int irq[32];
+ int irq[DWAPB_MAX_GPIOS];
bool irq_shared;
};
#define AMD_FCH_GPIO_REG_GPIO49 0x40
#define AMD_FCH_GPIO_REG_GPIO50 0x41
#define AMD_FCH_GPIO_REG_GPIO51 0x42
-#define AMD_FCH_GPIO_REG_GPIO59_DEVSLP0 0x43
+#define AMD_FCH_GPIO_REG_GPIO55_DEVSLP0 0x43
#define AMD_FCH_GPIO_REG_GPIO57 0x44
#define AMD_FCH_GPIO_REG_GPIO58 0x45
#define AMD_FCH_GPIO_REG_GPIO59_DEVSLP1 0x46
mode_voltage_24bit,
mode_voltage_raw,
mode_voltage_16bit,
+ mode_fan,
mode_max,
};
return device_property_read_u64_array(dev, propname, NULL, 0);
}
+static inline int device_property_string_array_count(struct device *dev,
+ const char *propname)
+{
+ return device_property_read_string_array(dev, propname, NULL, 0);
+}
+
static inline bool fwnode_property_read_bool(const struct fwnode_handle *fwnode,
const char *propname)
{
return fwnode_property_read_u64_array(fwnode, propname, NULL, 0);
}
+static inline int
+fwnode_property_string_array_count(const struct fwnode_handle *fwnode,
+ const char *propname)
+{
+ return fwnode_property_read_string_array(fwnode, propname, NULL, 0);
+}
+
struct software_node;
/**
/* SPDX-License-Identifier: GPL-2.0-only */
/*
- * pxa2xx_ssp.h
- *
* Copyright (C) 2003 Russell King, All Rights Reserved.
*
* This driver supports the following PXA CPU/SSP ports:-
#ifndef __LINUX_SSP_H
#define __LINUX_SSP_H
-#include <linux/list.h>
+#include <linux/bits.h>
+#include <linux/compiler_types.h>
#include <linux/io.h>
-#include <linux/of.h>
+#include <linux/kconfig.h>
+#include <linux/list.h>
+#include <linux/types.h>
+struct clk;
+struct device;
+struct device_node;
/*
* SSP Serial Port Registers
#define SSACDD (0x40) /* SSP Audio Clock Dither Divider */
/* Common PXA2xx bits first */
-#define SSCR0_DSS (0x0000000f) /* Data Size Select (mask) */
+#define SSCR0_DSS GENMASK(3, 0) /* Data Size Select (mask) */
#define SSCR0_DataSize(x) ((x) - 1) /* Data Size Select [4..16] */
-#define SSCR0_FRF (0x00000030) /* FRame Format (mask) */
+#define SSCR0_FRF GENMASK(5, 4) /* FRame Format (mask) */
#define SSCR0_Motorola (0x0 << 4) /* Motorola's Serial Peripheral Interface (SPI) */
#define SSCR0_TI (0x1 << 4) /* Texas Instruments' Synchronous Serial Protocol (SSP) */
#define SSCR0_National (0x2 << 4) /* National Microwire */
-#define SSCR0_ECS (1 << 6) /* External clock select */
-#define SSCR0_SSE (1 << 7) /* Synchronous Serial Port Enable */
+#define SSCR0_ECS BIT(6) /* External clock select */
+#define SSCR0_SSE BIT(7) /* Synchronous Serial Port Enable */
#define SSCR0_SCR(x) ((x) << 8) /* Serial Clock Rate (mask) */
/* PXA27x, PXA3xx */
-#define SSCR0_EDSS (1 << 20) /* Extended data size select */
-#define SSCR0_NCS (1 << 21) /* Network clock select */
-#define SSCR0_RIM (1 << 22) /* Receive FIFO overrrun interrupt mask */
-#define SSCR0_TUM (1 << 23) /* Transmit FIFO underrun interrupt mask */
-#define SSCR0_FRDC (0x07000000) /* Frame rate divider control (mask) */
+#define SSCR0_EDSS BIT(20) /* Extended data size select */
+#define SSCR0_NCS BIT(21) /* Network clock select */
+#define SSCR0_RIM BIT(22) /* Receive FIFO overrrun interrupt mask */
+#define SSCR0_TUM BIT(23) /* Transmit FIFO underrun interrupt mask */
+#define SSCR0_FRDC GENMASK(26, 24) /* Frame rate divider control (mask) */
#define SSCR0_SlotsPerFrm(x) (((x) - 1) << 24) /* Time slots per frame [1..8] */
-#define SSCR0_FPCKE (1 << 29) /* FIFO packing enable */
-#define SSCR0_ACS (1 << 30) /* Audio clock select */
-#define SSCR0_MOD (1 << 31) /* Mode (normal or network) */
-
-
-#define SSCR1_RIE (1 << 0) /* Receive FIFO Interrupt Enable */
-#define SSCR1_TIE (1 << 1) /* Transmit FIFO Interrupt Enable */
-#define SSCR1_LBM (1 << 2) /* Loop-Back Mode */
-#define SSCR1_SPO (1 << 3) /* Motorola SPI SSPSCLK polarity setting */
-#define SSCR1_SPH (1 << 4) /* Motorola SPI SSPSCLK phase setting */
-#define SSCR1_MWDS (1 << 5) /* Microwire Transmit Data Size */
-
-#define SSSR_ALT_FRM_MASK 3 /* Masks the SFRM signal number */
-#define SSSR_TNF (1 << 2) /* Transmit FIFO Not Full */
-#define SSSR_RNE (1 << 3) /* Receive FIFO Not Empty */
-#define SSSR_BSY (1 << 4) /* SSP Busy */
-#define SSSR_TFS (1 << 5) /* Transmit FIFO Service Request */
-#define SSSR_RFS (1 << 6) /* Receive FIFO Service Request */
-#define SSSR_ROR (1 << 7) /* Receive FIFO Overrun */
+#define SSCR0_FPCKE BIT(29) /* FIFO packing enable */
+#define SSCR0_ACS BIT(30) /* Audio clock select */
+#define SSCR0_MOD BIT(31) /* Mode (normal or network) */
+
+#define SSCR1_RIE BIT(0) /* Receive FIFO Interrupt Enable */
+#define SSCR1_TIE BIT(1) /* Transmit FIFO Interrupt Enable */
+#define SSCR1_LBM BIT(2) /* Loop-Back Mode */
+#define SSCR1_SPO BIT(3) /* Motorola SPI SSPSCLK polarity setting */
+#define SSCR1_SPH BIT(4) /* Motorola SPI SSPSCLK phase setting */
+#define SSCR1_MWDS BIT(5) /* Microwire Transmit Data Size */
+
+#define SSSR_ALT_FRM_MASK GENMASK(1, 0) /* Masks the SFRM signal number */
+#define SSSR_TNF BIT(2) /* Transmit FIFO Not Full */
+#define SSSR_RNE BIT(3) /* Receive FIFO Not Empty */
+#define SSSR_BSY BIT(4) /* SSP Busy */
+#define SSSR_TFS BIT(5) /* Transmit FIFO Service Request */
+#define SSSR_RFS BIT(6) /* Receive FIFO Service Request */
+#define SSSR_ROR BIT(7) /* Receive FIFO Overrun */
#define RX_THRESH_DFLT 8
#define TX_THRESH_DFLT 8
-#define SSSR_TFL_MASK (0xf << 8) /* Transmit FIFO Level mask */
-#define SSSR_RFL_MASK (0xf << 12) /* Receive FIFO Level mask */
+#define SSSR_TFL_MASK GENMASK(11, 8) /* Transmit FIFO Level mask */
+#define SSSR_RFL_MASK GENMASK(15, 12) /* Receive FIFO Level mask */
-#define SSCR1_TFT (0x000003c0) /* Transmit FIFO Threshold (mask) */
+#define SSCR1_TFT GENMASK(9, 6) /* Transmit FIFO Threshold (mask) */
#define SSCR1_TxTresh(x) (((x) - 1) << 6) /* level [1..16] */
-#define SSCR1_RFT (0x00003c00) /* Receive FIFO Threshold (mask) */
+#define SSCR1_RFT GENMASK(13, 10) /* Receive FIFO Threshold (mask) */
#define SSCR1_RxTresh(x) (((x) - 1) << 10) /* level [1..16] */
#define RX_THRESH_CE4100_DFLT 2
#define TX_THRESH_CE4100_DFLT 2
-#define CE4100_SSSR_TFL_MASK (0x3 << 8) /* Transmit FIFO Level mask */
-#define CE4100_SSSR_RFL_MASK (0x3 << 12) /* Receive FIFO Level mask */
+#define CE4100_SSSR_TFL_MASK GENMASK(9, 8) /* Transmit FIFO Level mask */
+#define CE4100_SSSR_RFL_MASK GENMASK(13, 12) /* Receive FIFO Level mask */
-#define CE4100_SSCR1_TFT (0x000000c0) /* Transmit FIFO Threshold (mask) */
+#define CE4100_SSCR1_TFT GENMASK(7, 6) /* Transmit FIFO Threshold (mask) */
#define CE4100_SSCR1_TxTresh(x) (((x) - 1) << 6) /* level [1..4] */
-#define CE4100_SSCR1_RFT (0x00000c00) /* Receive FIFO Threshold (mask) */
+#define CE4100_SSCR1_RFT GENMASK(11, 10) /* Receive FIFO Threshold (mask) */
#define CE4100_SSCR1_RxTresh(x) (((x) - 1) << 10) /* level [1..4] */
/* QUARK_X1000 SSCR0 bit definition */
-#define QUARK_X1000_SSCR0_DSS (0x1F << 0) /* Data Size Select (mask) */
+#define QUARK_X1000_SSCR0_DSS GENMASK(4, 0) /* Data Size Select (mask) */
#define QUARK_X1000_SSCR0_DataSize(x) ((x) - 1) /* Data Size Select [4..32] */
-#define QUARK_X1000_SSCR0_FRF (0x3 << 5) /* FRame Format (mask) */
+#define QUARK_X1000_SSCR0_FRF GENMASK(6, 5) /* FRame Format (mask) */
#define QUARK_X1000_SSCR0_Motorola (0x0 << 5) /* Motorola's Serial Peripheral Interface (SPI) */
#define RX_THRESH_QUARK_X1000_DFLT 1
#define TX_THRESH_QUARK_X1000_DFLT 16
-#define QUARK_X1000_SSSR_TFL_MASK (0x1F << 8) /* Transmit FIFO Level mask */
-#define QUARK_X1000_SSSR_RFL_MASK (0x1F << 13) /* Receive FIFO Level mask */
+#define QUARK_X1000_SSSR_TFL_MASK GENMASK(12, 8) /* Transmit FIFO Level mask */
+#define QUARK_X1000_SSSR_RFL_MASK GENMASK(17, 13) /* Receive FIFO Level mask */
-#define QUARK_X1000_SSCR1_TFT (0x1F << 6) /* Transmit FIFO Threshold (mask) */
+#define QUARK_X1000_SSCR1_TFT GENMASK(10, 6) /* Transmit FIFO Threshold (mask) */
#define QUARK_X1000_SSCR1_TxTresh(x) (((x) - 1) << 6) /* level [1..32] */
-#define QUARK_X1000_SSCR1_RFT (0x1F << 11) /* Receive FIFO Threshold (mask) */
+#define QUARK_X1000_SSCR1_RFT GENMASK(15, 11) /* Receive FIFO Threshold (mask) */
#define QUARK_X1000_SSCR1_RxTresh(x) (((x) - 1) << 11) /* level [1..32] */
-#define QUARK_X1000_SSCR1_STRF (1 << 17) /* Select FIFO or EFWR */
-#define QUARK_X1000_SSCR1_EFWR (1 << 16) /* Enable FIFO Write/Read */
+#define QUARK_X1000_SSCR1_EFWR BIT(16) /* Enable FIFO Write/Read */
+#define QUARK_X1000_SSCR1_STRF BIT(17) /* Select FIFO or EFWR */
/* extra bits in PXA255, PXA26x and PXA27x SSP ports */
#define SSCR0_TISSP (1 << 4) /* TI Sync Serial Protocol */
#define SSCR0_PSP (3 << 4) /* PSP - Programmable Serial Protocol */
-#define SSCR1_TTELP (1 << 31) /* TXD Tristate Enable Last Phase */
-#define SSCR1_TTE (1 << 30) /* TXD Tristate Enable */
-#define SSCR1_EBCEI (1 << 29) /* Enable Bit Count Error interrupt */
-#define SSCR1_SCFR (1 << 28) /* Slave Clock free Running */
-#define SSCR1_ECRA (1 << 27) /* Enable Clock Request A */
-#define SSCR1_ECRB (1 << 26) /* Enable Clock request B */
-#define SSCR1_SCLKDIR (1 << 25) /* Serial Bit Rate Clock Direction */
-#define SSCR1_SFRMDIR (1 << 24) /* Frame Direction */
-#define SSCR1_RWOT (1 << 23) /* Receive Without Transmit */
-#define SSCR1_TRAIL (1 << 22) /* Trailing Byte */
-#define SSCR1_TSRE (1 << 21) /* Transmit Service Request Enable */
-#define SSCR1_RSRE (1 << 20) /* Receive Service Request Enable */
-#define SSCR1_TINTE (1 << 19) /* Receiver Time-out Interrupt enable */
-#define SSCR1_PINTE (1 << 18) /* Peripheral Trailing Byte Interrupt Enable */
-#define SSCR1_IFS (1 << 16) /* Invert Frame Signal */
-#define SSCR1_STRF (1 << 15) /* Select FIFO or EFWR */
-#define SSCR1_EFWR (1 << 14) /* Enable FIFO Write/Read */
-
-#define SSSR_BCE (1 << 23) /* Bit Count Error */
-#define SSSR_CSS (1 << 22) /* Clock Synchronisation Status */
-#define SSSR_TUR (1 << 21) /* Transmit FIFO Under Run */
-#define SSSR_EOC (1 << 20) /* End Of Chain */
-#define SSSR_TINT (1 << 19) /* Receiver Time-out Interrupt */
-#define SSSR_PINT (1 << 18) /* Peripheral Trailing Byte Interrupt */
+#define SSCR1_EFWR BIT(14) /* Enable FIFO Write/Read */
+#define SSCR1_STRF BIT(15) /* Select FIFO or EFWR */
+#define SSCR1_IFS BIT(16) /* Invert Frame Signal */
+#define SSCR1_PINTE BIT(18) /* Peripheral Trailing Byte Interrupt Enable */
+#define SSCR1_TINTE BIT(19) /* Receiver Time-out Interrupt enable */
+#define SSCR1_RSRE BIT(20) /* Receive Service Request Enable */
+#define SSCR1_TSRE BIT(21) /* Transmit Service Request Enable */
+#define SSCR1_TRAIL BIT(22) /* Trailing Byte */
+#define SSCR1_RWOT BIT(23) /* Receive Without Transmit */
+#define SSCR1_SFRMDIR BIT(24) /* Frame Direction */
+#define SSCR1_SCLKDIR BIT(25) /* Serial Bit Rate Clock Direction */
+#define SSCR1_ECRB BIT(26) /* Enable Clock request B */
+#define SSCR1_ECRA BIT(27) /* Enable Clock Request A */
+#define SSCR1_SCFR BIT(28) /* Slave Clock free Running */
+#define SSCR1_EBCEI BIT(29) /* Enable Bit Count Error interrupt */
+#define SSCR1_TTE BIT(30) /* TXD Tristate Enable */
+#define SSCR1_TTELP BIT(31) /* TXD Tristate Enable Last Phase */
+
+#define SSSR_PINT BIT(18) /* Peripheral Trailing Byte Interrupt */
+#define SSSR_TINT BIT(19) /* Receiver Time-out Interrupt */
+#define SSSR_EOC BIT(20) /* End Of Chain */
+#define SSSR_TUR BIT(21) /* Transmit FIFO Under Run */
+#define SSSR_CSS BIT(22) /* Clock Synchronisation Status */
+#define SSSR_BCE BIT(23) /* Bit Count Error */
#define SSPSP_SCMODE(x) ((x) << 0) /* Serial Bit Rate Clock Mode */
-#define SSPSP_SFRMP (1 << 2) /* Serial Frame Polarity */
-#define SSPSP_ETDS (1 << 3) /* End of Transfer data State */
+#define SSPSP_SFRMP BIT(2) /* Serial Frame Polarity */
+#define SSPSP_ETDS BIT(3) /* End of Transfer data State */
#define SSPSP_STRTDLY(x) ((x) << 4) /* Start Delay */
#define SSPSP_DMYSTRT(x) ((x) << 7) /* Dummy Start */
#define SSPSP_SFRMDLY(x) ((x) << 9) /* Serial Frame Delay */
#define SSPSP_SFRMWDTH(x) ((x) << 16) /* Serial Frame Width */
#define SSPSP_DMYSTOP(x) ((x) << 23) /* Dummy Stop */
-#define SSPSP_FSRT (1 << 25) /* Frame Sync Relative Timing */
+#define SSPSP_FSRT BIT(25) /* Frame Sync Relative Timing */
/* PXA3xx */
#define SSPSP_EDMYSTRT(x) ((x) << 26) /* Extended Dummy Start */
#define SSPSP_EDMYSTOP(x) ((x) << 28) /* Extended Dummy Stop */
#define SSPSP_TIMING_MASK (0x7f8001f0)
-#define SSACD_SCDB (1 << 3) /* SSPSYSCLK Divider Bypass */
-#define SSACD_ACPS(x) ((x) << 4) /* Audio clock PLL select */
#define SSACD_ACDS(x) ((x) << 0) /* Audio clock divider select */
#define SSACD_ACDS_1 (0)
#define SSACD_ACDS_2 (1)
#define SSACD_ACDS_8 (3)
#define SSACD_ACDS_16 (4)
#define SSACD_ACDS_32 (5)
+#define SSACD_SCDB BIT(3) /* SSPSYSCLK Divider Bypass */
#define SSACD_SCDB_4X (0)
#define SSACD_SCDB_1X (1)
-#define SSACD_SCDX8 (1 << 7) /* SYSCLK division ratio select */
+#define SSACD_ACPS(x) ((x) << 4) /* Audio clock PLL select */
+#define SSACD_SCDX8 BIT(7) /* SYSCLK division ratio select */
/* LPSS SSP */
#define SSITF 0x44 /* TX FIFO trigger level */
+#define SSITF_TxHiThresh(x) (((x) - 1) << 0)
#define SSITF_TxLoThresh(x) (((x) - 1) << 8)
-#define SSITF_TxHiThresh(x) ((x) - 1)
#define SSIRF 0x48 /* RX FIFO trigger level */
#define SSIRF_RxThresh(x) ((x) - 1)
+/* LPT/WPT SSP */
+#define SSCR2 (0x40) /* SSP Command / Status 2 */
+#define SSPSP2 (0x44) /* SSP Programmable Serial Protocol 2 */
+
enum pxa_ssp_type {
SSP_UNDEFINED = 0,
PXA25x_SSP, /* pxa 210, 250, 255, 26x */
extern int qcom_scm_restore_sec_cfg(u32 device_id, u32 spare);
extern int qcom_scm_iommu_secure_ptbl_size(u32 spare, size_t *size);
extern int qcom_scm_iommu_secure_ptbl_init(u64 addr, u32 size, u32 spare);
+extern int qcom_scm_mem_protect_video_var(u32 cp_start, u32 cp_size,
+ u32 cp_nonpixel_start,
+ u32 cp_nonpixel_size);
extern int qcom_scm_assign_mem(phys_addr_t mem_addr, size_t mem_sz,
unsigned int *src,
const struct qcom_scm_vmperm *newvm,
{ return -ENODEV; }
static inline int qcom_scm_iommu_secure_ptbl_init(u64 addr, u32 size, u32 spare)
{ return -ENODEV; }
+extern inline int qcom_scm_mem_protect_video_var(u32 cp_start, u32 cp_size,
+ u32 cp_nonpixel_start,
+ u32 cp_nonpixel_size)
+ { return -ENODEV; }
static inline int qcom_scm_assign_mem(phys_addr_t mem_addr, size_t mem_sz,
unsigned int *src, const struct qcom_scm_vmperm *newvm,
unsigned int dest_cnt) { return -ENODEV; }
(ia->ia_valid & ATTR_GID && !gid_eq(ia->ia_gid, inode->i_gid));
}
-int kernel_quotactl(unsigned int cmd, const char __user *special,
- qid_t id, void __user *addr);
-
#if defined(CONFIG_QUOTA)
#define quota_error(sb, fmt, args...) \
};
struct latch_tree_root {
- seqcount_t seq;
- struct rb_root tree[2];
+ seqcount_latch_t seq;
+ struct rb_root tree[2];
};
/**
do {
seq = raw_read_seqcount_latch(&root->seq);
node = __lt_find(key, root, seq & 1, ops->comp);
- } while (read_seqcount_retry(&root->seq, seq));
+ } while (read_seqcount_latch_retry(&root->seq, seq));
return node;
}
*
* Return: false if the passed refcount is 0, true otherwise
*/
-static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
+static inline __must_check bool __refcount_add_not_zero(int i, refcount_t *r, int *oldp)
{
int old = refcount_read(r);
break;
} while (!atomic_try_cmpxchg_relaxed(&r->refs, &old, old + i));
+ if (oldp)
+ *oldp = old;
+
if (unlikely(old < 0 || old + i < 0))
refcount_warn_saturate(r, REFCOUNT_ADD_NOT_ZERO_OVF);
return old;
}
+static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r)
+{
+ return __refcount_add_not_zero(i, r, NULL);
+}
+
/**
* refcount_add - add a value to a refcount
* @i: the value to add to the refcount
* cases, refcount_inc(), or one of its variants, should instead be used to
* increment a reference count.
*/
-static inline void refcount_add(int i, refcount_t *r)
+static inline void __refcount_add(int i, refcount_t *r, int *oldp)
{
int old = atomic_fetch_add_relaxed(i, &r->refs);
+ if (oldp)
+ *oldp = old;
+
if (unlikely(!old))
refcount_warn_saturate(r, REFCOUNT_ADD_UAF);
else if (unlikely(old < 0 || old + i < 0))
refcount_warn_saturate(r, REFCOUNT_ADD_OVF);
}
+static inline void refcount_add(int i, refcount_t *r)
+{
+ __refcount_add(i, r, NULL);
+}
+
/**
* refcount_inc_not_zero - increment a refcount unless it is 0
* @r: the refcount to increment
*
* Return: true if the increment was successful, false otherwise
*/
+static inline __must_check bool __refcount_inc_not_zero(refcount_t *r, int *oldp)
+{
+ return __refcount_add_not_zero(1, r, oldp);
+}
+
static inline __must_check bool refcount_inc_not_zero(refcount_t *r)
{
- return refcount_add_not_zero(1, r);
+ return __refcount_inc_not_zero(r, NULL);
}
/**
* Will WARN if the refcount is 0, as this represents a possible use-after-free
* condition.
*/
+static inline void __refcount_inc(refcount_t *r, int *oldp)
+{
+ __refcount_add(1, r, oldp);
+}
+
static inline void refcount_inc(refcount_t *r)
{
- refcount_add(1, r);
+ __refcount_inc(r, NULL);
}
/**
*
* Return: true if the resulting refcount is 0, false otherwise
*/
-static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r)
+static inline __must_check bool __refcount_sub_and_test(int i, refcount_t *r, int *oldp)
{
int old = atomic_fetch_sub_release(i, &r->refs);
+ if (oldp)
+ *oldp = old;
+
if (old == i) {
smp_acquire__after_ctrl_dep();
return true;
return false;
}
+static inline __must_check bool refcount_sub_and_test(int i, refcount_t *r)
+{
+ return __refcount_sub_and_test(i, r, NULL);
+}
+
/**
* refcount_dec_and_test - decrement a refcount and test if it is 0
* @r: the refcount
*
* Return: true if the resulting refcount is 0, false otherwise
*/
+static inline __must_check bool __refcount_dec_and_test(refcount_t *r, int *oldp)
+{
+ return __refcount_sub_and_test(1, r, oldp);
+}
+
static inline __must_check bool refcount_dec_and_test(refcount_t *r)
{
- return refcount_sub_and_test(1, r);
+ return __refcount_dec_and_test(r, NULL);
}
/**
* Provides release memory ordering, such that prior loads and stores are done
* before.
*/
-static inline void refcount_dec(refcount_t *r)
+static inline void __refcount_dec(refcount_t *r, int *oldp)
{
- if (unlikely(atomic_fetch_sub_release(1, &r->refs) <= 1))
+ int old = atomic_fetch_sub_release(1, &r->refs);
+
+ if (oldp)
+ *oldp = old;
+
+ if (unlikely(old <= 1))
refcount_warn_saturate(r, REFCOUNT_DEC_LEAK);
}
+static inline void refcount_dec(refcount_t *r)
+{
+ __refcount_dec(r, NULL);
+}
+
extern __must_check bool refcount_dec_if_one(refcount_t *r);
extern __must_check bool refcount_dec_not_one(refcount_t *r);
extern __must_check bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock);
* @hwlock_id: Specify the hardware spinlock id.
* @hwlock_mode: The hardware spinlock mode, should be HWLOCK_IRQSTATE,
* HWLOCK_IRQ or 0.
+ * @can_sleep: Optional, specifies whether regmap operations can sleep.
*/
struct regmap_config {
const char *name;
bool use_hwlock;
unsigned int hwlock_id;
unsigned int hwlock_mode;
+
+ bool can_sleep;
};
/**
const struct regmap_config *config,
struct lock_class_key *lock_key,
const char *lock_name);
+struct regmap *__regmap_init_spi_avmm(struct spi_device *spi,
+ const struct regmap_config *config,
+ struct lock_class_key *lock_key,
+ const char *lock_name);
struct regmap *__devm_regmap_init(struct device *dev,
const struct regmap_bus *bus,
const struct regmap_config *config,
struct lock_class_key *lock_key,
const char *lock_name);
+struct regmap *__devm_regmap_init_spi_avmm(struct spi_device *spi,
+ const struct regmap_config *config,
+ struct lock_class_key *lock_key,
+ const char *lock_name);
/*
* Wrapper for regmap_init macros to include a unique lockdep key and name
* for each call. No-op if CONFIG_LOCKDEP is not set.
__regmap_lockdep_wrapper(__regmap_init_sdw, #config, \
sdw, config)
+/**
+ * regmap_init_spi_avmm() - Initialize register map for Intel SPI Slave
+ * to AVMM Bus Bridge
+ *
+ * @spi: Device that will be interacted with
+ * @config: Configuration for register map
+ *
+ * The return value will be an ERR_PTR() on error or a valid pointer
+ * to a struct regmap.
+ */
+#define regmap_init_spi_avmm(spi, config) \
+ __regmap_lockdep_wrapper(__regmap_init_spi_avmm, #config, \
+ spi, config)
/**
* devm_regmap_init() - Initialise managed register map
__regmap_lockdep_wrapper(__devm_regmap_init_i3c, #config, \
i3c, config)
+/**
+ * devm_regmap_init_spi_avmm() - Initialize register map for Intel SPI Slave
+ * to AVMM Bus Bridge
+ *
+ * @spi: Device that will be interacted with
+ * @config: Configuration for register map
+ *
+ * The return value will be an ERR_PTR() on error or a valid pointer
+ * to a struct regmap. The map will be automatically freed by the
+ * device management code.
+ */
+#define devm_regmap_init_spi_avmm(spi, config) \
+ __regmap_lockdep_wrapper(__devm_regmap_init_spi_avmm, #config, \
+ spi, config)
+
int regmap_mmio_attach_clk(struct regmap *map, struct clk *clk);
void regmap_mmio_detach_clk(struct regmap *map);
void regmap_exit(struct regmap *map);
struct regmap *regmap, struct reg_field reg_field);
void devm_regmap_field_free(struct device *dev, struct regmap_field *field);
+int regmap_field_bulk_alloc(struct regmap *regmap,
+ struct regmap_field **rm_field,
+ struct reg_field *reg_field,
+ int num_fields);
+void regmap_field_bulk_free(struct regmap_field *field);
+int devm_regmap_field_bulk_alloc(struct device *dev, struct regmap *regmap,
+ struct regmap_field **field,
+ struct reg_field *reg_field, int num_fields);
+void devm_regmap_field_bulk_free(struct device *dev,
+ struct regmap_field *field);
+
int regmap_field_read(struct regmap_field *field, unsigned int *val);
int regmap_field_update_bits_base(struct regmap_field *field,
unsigned int mask, unsigned int val,
* @mask_invert: Inverted mask register: cleared bits are masked out.
* @use_ack: Use @ack register even if it is zero.
* @ack_invert: Inverted ack register: cleared bits for ack.
+ * @clear_ack: Use this to set 1 and 0 or vice-versa to clear interrupts.
* @wake_invert: Inverted wake register: cleared bits are wake enabled.
* @type_invert: Invert the type flags.
* @type_in_mask: Use the mask registers for controlling irq type. For
bool mask_invert:1;
bool use_ack:1;
bool ack_invert:1;
+ bool clear_ack:1;
bool wake_invert:1;
bool runtime_pm:1;
bool type_invert:1;
int regulator_get_current_limit_regmap(struct regulator_dev *rdev);
void *regulator_get_init_drvdata(struct regulator_init_data *reg_init_data);
-void regulator_lock(struct regulator_dev *rdev);
-void regulator_unlock(struct regulator_dev *rdev);
-
/*
* Helper functions intended to be used by regulator drivers prior registering
* their regulators.
#ifndef _RESCTRL_H
#define _RESCTRL_H
+#include <linux/pid.h>
+
#ifdef CONFIG_PROC_CPU_RESCTRL
int proc_resctrl_show(struct seq_file *m,
struct signal_struct;
struct task_delay_info;
struct task_group;
+struct io_uring_task;
/*
* Task state bitmask. NOTE! These bits are also
/* Open file information: */
struct files_struct *files;
+#ifdef CONFIG_IO_URING
+ struct io_uring_task *io_uring;
+#endif
+
/* Namespaces: */
struct nsproxy *nsproxy;
#endif
#ifdef CONFIG_X86_MCE
+ void __user *mce_vaddr;
+ __u64 mce_kflags;
u64 mce_addr;
__u64 mce_ripv : 1,
mce_whole_page : 1,
/*
* Per process flags
*/
+#define PF_VCPU 0x00000001 /* I'm a virtual CPU */
#define PF_IDLE 0x00000002 /* I am an IDLE thread */
#define PF_EXITING 0x00000004 /* Getting shut down */
-#define PF_VCPU 0x00000010 /* I'm a virtual CPU */
+#define PF_IO_WORKER 0x00000010 /* Task is an IO worker */
#define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */
#define PF_FORKNOEXEC 0x00000040 /* Forked but didn't exec */
#define PF_MCE_PROCESS 0x00000080 /* Process policy on mce errors */
#define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */
#define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */
#define PF_MEMALLOC_NOCMA 0x10000000 /* All allocation request will have _GFP_MOVABLE cleared */
-#define PF_IO_WORKER 0x20000000 /* Task is an IO worker */
#define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */
#define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */
const struct sched_avg *sched_trace_rq_avg_irq(struct rq *rq);
int sched_trace_rq_cpu(struct rq *rq);
+int sched_trace_rq_cpu_capacity(struct rq *rq);
int sched_trace_rq_nr_running(struct rq *rq);
const struct cpumask *sched_trace_rd_span(struct root_domain *rd);
MEMBARRIER_STATE_GLOBAL_EXPEDITED = (1U << 3),
MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY = (1U << 4),
MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE = (1U << 5),
+ MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY = (1U << 6),
+ MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ = (1U << 7),
};
enum {
MEMBARRIER_FLAG_SYNC_CORE = (1U << 0),
+ MEMBARRIER_FLAG_RSEQ = (1U << 1),
};
#ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * sched-domains (multiprocessor balancing) flag declarations.
+ */
+
+#ifndef SD_FLAG
+# error "Incorrect import of SD flags definitions"
+#endif
+
+/*
+ * Hierarchical metaflags
+ *
+ * SHARED_CHILD: These flags are meant to be set from the base domain upwards.
+ * If a domain has this flag set, all of its children should have it set. This
+ * is usually because the flag describes some shared resource (all CPUs in that
+ * domain share the same resource), or because they are tied to a scheduling
+ * behaviour that we want to disable at some point in the hierarchy for
+ * scalability reasons.
+ *
+ * In those cases it doesn't make sense to have the flag set for a domain but
+ * not have it in (some of) its children: sched domains ALWAYS span their child
+ * domains, so operations done with parent domains will cover CPUs in the lower
+ * child domains.
+ *
+ *
+ * SHARED_PARENT: These flags are meant to be set from the highest domain
+ * downwards. If a domain has this flag set, all of its parents should have it
+ * set. This is usually for topology properties that start to appear above a
+ * certain level (e.g. domain starts spanning CPUs outside of the base CPU's
+ * socket).
+ */
+#define SDF_SHARED_CHILD 0x1
+#define SDF_SHARED_PARENT 0x2
+
+/*
+ * Behavioural metaflags
+ *
+ * NEEDS_GROUPS: These flags are only relevant if the domain they are set on has
+ * more than one group. This is usually for balancing flags (load balancing
+ * involves equalizing a metric between groups), or for flags describing some
+ * shared resource (which would be shared between groups).
+ */
+#define SDF_NEEDS_GROUPS 0x4
+
+/*
+ * Balance when about to become idle
+ *
+ * SHARED_CHILD: Set from the base domain up to cpuset.sched_relax_domain_level.
+ * NEEDS_GROUPS: Load balancing flag.
+ */
+SD_FLAG(SD_BALANCE_NEWIDLE, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS)
+
+/*
+ * Balance on exec
+ *
+ * SHARED_CHILD: Set from the base domain up to the NUMA reclaim level.
+ * NEEDS_GROUPS: Load balancing flag.
+ */
+SD_FLAG(SD_BALANCE_EXEC, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS)
+
+/*
+ * Balance on fork, clone
+ *
+ * SHARED_CHILD: Set from the base domain up to the NUMA reclaim level.
+ * NEEDS_GROUPS: Load balancing flag.
+ */
+SD_FLAG(SD_BALANCE_FORK, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS)
+
+/*
+ * Balance on wakeup
+ *
+ * SHARED_CHILD: Set from the base domain up to cpuset.sched_relax_domain_level.
+ * NEEDS_GROUPS: Load balancing flag.
+ */
+SD_FLAG(SD_BALANCE_WAKE, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS)
+
+/*
+ * Consider waking task on waking CPU.
+ *
+ * SHARED_CHILD: Set from the base domain up to the NUMA reclaim level.
+ */
+SD_FLAG(SD_WAKE_AFFINE, SDF_SHARED_CHILD)
+
+/*
+ * Domain members have different CPU capacities
+ *
+ * SHARED_PARENT: Set from the topmost domain down to the first domain where
+ * asymmetry is detected.
+ * NEEDS_GROUPS: Per-CPU capacity is asymmetric between groups.
+ */
+SD_FLAG(SD_ASYM_CPUCAPACITY, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS)
+
+/*
+ * Domain members share CPU capacity (i.e. SMT)
+ *
+ * SHARED_CHILD: Set from the base domain up until spanned CPUs no longer share
+ * CPU capacity.
+ * NEEDS_GROUPS: Capacity is shared between groups.
+ */
+SD_FLAG(SD_SHARE_CPUCAPACITY, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS)
+
+/*
+ * Domain members share CPU package resources (i.e. caches)
+ *
+ * SHARED_CHILD: Set from the base domain up until spanned CPUs no longer share
+ * the same cache(s).
+ * NEEDS_GROUPS: Caches are shared between groups.
+ */
+SD_FLAG(SD_SHARE_PKG_RESOURCES, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS)
+
+/*
+ * Only a single load balancing instance
+ *
+ * SHARED_PARENT: Set for all NUMA levels above NODE. Could be set from a
+ * different level upwards, but it doesn't change that if a
+ * domain has this flag set, then all of its parents need to have
+ * it too (otherwise the serialization doesn't make sense).
+ * NEEDS_GROUPS: No point in preserving domain if it has a single group.
+ */
+SD_FLAG(SD_SERIALIZE, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS)
+
+/*
+ * Place busy tasks earlier in the domain
+ *
+ * SHARED_CHILD: Usually set on the SMT level. Technically could be set further
+ * up, but currently assumed to be set from the base domain
+ * upwards (see update_top_cache_domain()).
+ * NEEDS_GROUPS: Load balancing flag.
+ */
+SD_FLAG(SD_ASYM_PACKING, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS)
+
+/*
+ * Prefer to place tasks in a sibling domain
+ *
+ * Set up until domains start spanning NUMA nodes. Close to being a SHARED_CHILD
+ * flag, but cleared below domains with SD_ASYM_CPUCAPACITY.
+ *
+ * NEEDS_GROUPS: Load balancing flag.
+ */
+SD_FLAG(SD_PREFER_SIBLING, SDF_NEEDS_GROUPS)
+
+/*
+ * sched_groups of this level overlap
+ *
+ * SHARED_PARENT: Set for all NUMA levels above NODE.
+ * NEEDS_GROUPS: Overlaps can only exist with more than one group.
+ */
+SD_FLAG(SD_OVERLAP, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS)
+
+/*
+ * Cross-node balancing
+ *
+ * SHARED_PARENT: Set for all NUMA levels above NODE.
+ * NEEDS_GROUPS: No point in preserving domain if it has a single group.
+ */
+SD_FLAG(SD_NUMA, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS)
*/
#ifdef CONFIG_SMP
-#define SD_BALANCE_NEWIDLE 0x0001 /* Balance when about to become idle */
-#define SD_BALANCE_EXEC 0x0002 /* Balance on exec */
-#define SD_BALANCE_FORK 0x0004 /* Balance on fork, clone */
-#define SD_BALANCE_WAKE 0x0008 /* Balance on wakeup */
-#define SD_WAKE_AFFINE 0x0010 /* Wake task to waking CPU */
-#define SD_ASYM_CPUCAPACITY 0x0020 /* Domain members have different CPU capacities */
-#define SD_SHARE_CPUCAPACITY 0x0040 /* Domain members share CPU capacity */
-#define SD_SHARE_POWERDOMAIN 0x0080 /* Domain members share power domain */
-#define SD_SHARE_PKG_RESOURCES 0x0100 /* Domain members share CPU pkg resources */
-#define SD_SERIALIZE 0x0200 /* Only a single load balancing instance */
-#define SD_ASYM_PACKING 0x0400 /* Place busy groups earlier in the domain */
-#define SD_PREFER_SIBLING 0x0800 /* Prefer to place tasks in a sibling domain */
-#define SD_OVERLAP 0x1000 /* sched_domains of this level overlap */
-#define SD_NUMA 0x2000 /* cross-node balancing */
+/* Generate SD flag indexes */
+#define SD_FLAG(name, mflags) __##name,
+enum {
+ #include <linux/sched/sd_flags.h>
+ __SD_FLAG_CNT,
+};
+#undef SD_FLAG
+/* Generate SD flag bits */
+#define SD_FLAG(name, mflags) name = 1 << __##name,
+enum {
+ #include <linux/sched/sd_flags.h>
+};
+#undef SD_FLAG
+
+#ifdef CONFIG_SCHED_DEBUG
+
+struct sd_flag_debug {
+ unsigned int meta_flags;
+ char *name;
+};
+extern const struct sd_flag_debug sd_flag_debug[];
+
+#endif
#ifdef CONFIG_SCHED_SMT
static inline int cpu_smt_flags(void)
#include <linux/kcsan-checks.h>
#include <linux/lockdep.h>
#include <linux/mutex.h>
+#include <linux/ww_mutex.h>
#include <linux/preempt.h>
#include <linux/spinlock.h>
*
* If the write serialization mechanism is one of the common kernel
* locking primitives, use a sequence counter with associated lock
- * (seqcount_LOCKTYPE_t) instead.
+ * (seqcount_LOCKNAME_t) instead.
*
* If it's desired to automatically handle the sequence counter writer
* serialization and non-preemptibility requirements, use a sequential
#define SEQCNT_ZERO(name) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(name) }
/*
- * Sequence counters with associated locks (seqcount_LOCKTYPE_t)
+ * Sequence counters with associated locks (seqcount_LOCKNAME_t)
*
* A sequence counter which associates the lock used for writer
* serialization at initialization time. This enables lockdep to validate
* See Documentation/locking/seqlock.rst
*/
-#ifdef CONFIG_LOCKDEP
+/*
+ * For PREEMPT_RT, seqcount_LOCKNAME_t write side critical sections cannot
+ * disable preemption. It can lead to higher latencies, and the write side
+ * sections will not be able to acquire locks which become sleeping locks
+ * (e.g. spinlock_t).
+ *
+ * To remain preemptible while avoiding a possible livelock caused by the
+ * reader preempting the writer, use a different technique: let the reader
+ * detect if a seqcount_LOCKNAME_t writer is in progress. If that is the
+ * case, acquire then release the associated LOCKNAME writer serialization
+ * lock. This will allow any possibly-preempted writer to make progress
+ * until the end of its writer serialization lock critical section.
+ *
+ * This lock-unlock technique must be implemented for all of PREEMPT_RT
+ * sleeping locks. See Documentation/locking/locktypes.rst
+ */
+#if defined(CONFIG_LOCKDEP) || defined(CONFIG_PREEMPT_RT)
#define __SEQ_LOCK(expr) expr
#else
#define __SEQ_LOCK(expr)
#endif
/**
- * typedef seqcount_LOCKNAME_t - sequence counter with LOCKTYPR associated
+ * typedef seqcount_LOCKNAME_t - sequence counter with LOCKNAME associated
* @seqcount: The real sequence counter
- * @lock: Pointer to the associated spinlock
+ * @lock: Pointer to the associated lock
*
- * A plain sequence counter with external writer synchronization by a
- * spinlock. The spinlock is associated to the sequence count in the
+ * A plain sequence counter with external writer synchronization by
+ * LOCKNAME @lock. The lock is associated to the sequence counter in the
* static initializer or init function. This enables lockdep to validate
* that the write side critical section is properly serialized.
+ *
+ * LOCKNAME: raw_spinlock, spinlock, rwlock, mutex, or ww_mutex.
*/
-/**
+/*
* seqcount_LOCKNAME_init() - runtime initializer for seqcount_LOCKNAME_t
* @s: Pointer to the seqcount_LOCKNAME_t instance
- * @lock: Pointer to the associated LOCKTYPE
+ * @lock: Pointer to the associated lock
*/
+#define seqcount_LOCKNAME_init(s, _lock, lockname) \
+ do { \
+ seqcount_##lockname##_t *____s = (s); \
+ seqcount_init(&____s->seqcount); \
+ __SEQ_LOCK(____s->lock = (_lock)); \
+ } while (0)
+
+#define seqcount_raw_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, raw_spinlock)
+#define seqcount_spinlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, spinlock)
+#define seqcount_rwlock_init(s, lock) seqcount_LOCKNAME_init(s, lock, rwlock);
+#define seqcount_mutex_init(s, lock) seqcount_LOCKNAME_init(s, lock, mutex);
+#define seqcount_ww_mutex_init(s, lock) seqcount_LOCKNAME_init(s, lock, ww_mutex);
+
/*
- * SEQCOUNT_LOCKTYPE() - Instantiate seqcount_LOCKNAME_t and helpers
- * @locktype: actual typename
- * @lockname: name
+ * SEQCOUNT_LOCKNAME() - Instantiate seqcount_LOCKNAME_t and helpers
+ * seqprop_LOCKNAME_*() - Property accessors for seqcount_LOCKNAME_t
+ *
+ * @lockname: "LOCKNAME" part of seqcount_LOCKNAME_t
+ * @locktype: LOCKNAME canonical C data type
* @preemptible: preemptibility of above locktype
* @lockmember: argument for lockdep_assert_held()
+ * @lockbase: associated lock release function (prefix only)
+ * @lock_acquire: associated lock acquisition function (full call)
*/
-#define SEQCOUNT_LOCKTYPE(locktype, lockname, preemptible, lockmember) \
+#define SEQCOUNT_LOCKNAME(lockname, locktype, preemptible, lockmember, lockbase, lock_acquire) \
typedef struct seqcount_##lockname { \
seqcount_t seqcount; \
__SEQ_LOCK(locktype *lock); \
} seqcount_##lockname##_t; \
\
-static __always_inline void \
-seqcount_##lockname##_init(seqcount_##lockname##_t *s, locktype *lock) \
+static __always_inline seqcount_t * \
+__seqprop_##lockname##_ptr(seqcount_##lockname##_t *s) \
{ \
- seqcount_init(&s->seqcount); \
- __SEQ_LOCK(s->lock = lock); \
+ return &s->seqcount; \
} \
\
-static __always_inline seqcount_t * \
-__seqcount_##lockname##_ptr(seqcount_##lockname##_t *s) \
+static __always_inline unsigned \
+__seqprop_##lockname##_sequence(const seqcount_##lockname##_t *s) \
{ \
- return &s->seqcount; \
+ unsigned seq = READ_ONCE(s->seqcount.sequence); \
+ \
+ if (!IS_ENABLED(CONFIG_PREEMPT_RT)) \
+ return seq; \
+ \
+ if (preemptible && unlikely(seq & 1)) { \
+ __SEQ_LOCK(lock_acquire); \
+ __SEQ_LOCK(lockbase##_unlock(s->lock)); \
+ \
+ /* \
+ * Re-read the sequence counter since the (possibly \
+ * preempted) writer made progress. \
+ */ \
+ seq = READ_ONCE(s->seqcount.sequence); \
+ } \
+ \
+ return seq; \
} \
\
static __always_inline bool \
-__seqcount_##lockname##_preemptible(seqcount_##lockname##_t *s) \
+__seqprop_##lockname##_preemptible(const seqcount_##lockname##_t *s) \
{ \
- return preemptible; \
+ if (!IS_ENABLED(CONFIG_PREEMPT_RT)) \
+ return preemptible; \
+ \
+ /* PREEMPT_RT relies on the above LOCK+UNLOCK */ \
+ return false; \
} \
\
static __always_inline void \
-__seqcount_##lockname##_assert(seqcount_##lockname##_t *s) \
+__seqprop_##lockname##_assert(const seqcount_##lockname##_t *s) \
{ \
__SEQ_LOCK(lockdep_assert_held(lockmember)); \
}
* __seqprop() for seqcount_t
*/
-static inline seqcount_t *__seqcount_ptr(seqcount_t *s)
+static inline seqcount_t *__seqprop_ptr(seqcount_t *s)
{
return s;
}
-static inline bool __seqcount_preemptible(seqcount_t *s)
+static inline unsigned __seqprop_sequence(const seqcount_t *s)
+{
+ return READ_ONCE(s->sequence);
+}
+
+static inline bool __seqprop_preemptible(const seqcount_t *s)
{
return false;
}
-static inline void __seqcount_assert(seqcount_t *s)
+static inline void __seqprop_assert(const seqcount_t *s)
{
lockdep_assert_preemption_disabled();
}
-SEQCOUNT_LOCKTYPE(raw_spinlock_t, raw_spinlock, false, s->lock)
-SEQCOUNT_LOCKTYPE(spinlock_t, spinlock, false, s->lock)
-SEQCOUNT_LOCKTYPE(rwlock_t, rwlock, false, s->lock)
-SEQCOUNT_LOCKTYPE(struct mutex, mutex, true, s->lock)
-SEQCOUNT_LOCKTYPE(struct ww_mutex, ww_mutex, true, &s->lock->base)
+#define __SEQ_RT IS_ENABLED(CONFIG_PREEMPT_RT)
-/**
+SEQCOUNT_LOCKNAME(raw_spinlock, raw_spinlock_t, false, s->lock, raw_spin, raw_spin_lock(s->lock))
+SEQCOUNT_LOCKNAME(spinlock, spinlock_t, __SEQ_RT, s->lock, spin, spin_lock(s->lock))
+SEQCOUNT_LOCKNAME(rwlock, rwlock_t, __SEQ_RT, s->lock, read, read_lock(s->lock))
+SEQCOUNT_LOCKNAME(mutex, struct mutex, true, s->lock, mutex, mutex_lock(s->lock))
+SEQCOUNT_LOCKNAME(ww_mutex, struct ww_mutex, true, &s->lock->base, ww_mutex, ww_mutex_lock(s->lock, NULL))
+
+/*
* SEQCNT_LOCKNAME_ZERO - static initializer for seqcount_LOCKNAME_t
* @name: Name of the seqcount_LOCKNAME_t instance
- * @lock: Pointer to the associated LOCKTYPE
+ * @lock: Pointer to the associated LOCKNAME
*/
-#define SEQCOUNT_LOCKTYPE_ZERO(seq_name, assoc_lock) { \
+#define SEQCOUNT_LOCKNAME_ZERO(seq_name, assoc_lock) { \
.seqcount = SEQCNT_ZERO(seq_name.seqcount), \
__SEQ_LOCK(.lock = (assoc_lock)) \
}
-#define SEQCNT_SPINLOCK_ZERO(name, lock) SEQCOUNT_LOCKTYPE_ZERO(name, lock)
-#define SEQCNT_RAW_SPINLOCK_ZERO(name, lock) SEQCOUNT_LOCKTYPE_ZERO(name, lock)
-#define SEQCNT_RWLOCK_ZERO(name, lock) SEQCOUNT_LOCKTYPE_ZERO(name, lock)
-#define SEQCNT_MUTEX_ZERO(name, lock) SEQCOUNT_LOCKTYPE_ZERO(name, lock)
-#define SEQCNT_WW_MUTEX_ZERO(name, lock) SEQCOUNT_LOCKTYPE_ZERO(name, lock)
-
+#define SEQCNT_RAW_SPINLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock)
+#define SEQCNT_SPINLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock)
+#define SEQCNT_RWLOCK_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock)
+#define SEQCNT_MUTEX_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock)
+#define SEQCNT_WW_MUTEX_ZERO(name, lock) SEQCOUNT_LOCKNAME_ZERO(name, lock)
#define __seqprop_case(s, lockname, prop) \
- seqcount_##lockname##_t: __seqcount_##lockname##_##prop((void *)(s))
+ seqcount_##lockname##_t: __seqprop_##lockname##_##prop((void *)(s))
#define __seqprop(s, prop) _Generic(*(s), \
- seqcount_t: __seqcount_##prop((void *)(s)), \
+ seqcount_t: __seqprop_##prop((void *)(s)), \
__seqprop_case((s), raw_spinlock, prop), \
__seqprop_case((s), spinlock, prop), \
__seqprop_case((s), rwlock, prop), \
__seqprop_case((s), ww_mutex, prop))
#define __seqcount_ptr(s) __seqprop(s, ptr)
+#define __seqcount_sequence(s) __seqprop(s, sequence)
#define __seqcount_lock_preemptible(s) __seqprop(s, preemptible)
#define __seqcount_assert_lock_held(s) __seqprop(s, assert)
/**
* __read_seqcount_begin() - begin a seqcount_t read section w/o barrier
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb()
* barrier. Callers should ensure that smp_rmb() or equivalent ordering is
* Return: count to be passed to read_seqcount_retry()
*/
#define __read_seqcount_begin(s) \
- __read_seqcount_t_begin(__seqcount_ptr(s))
-
-static inline unsigned __read_seqcount_t_begin(const seqcount_t *s)
-{
- unsigned ret;
-
-repeat:
- ret = READ_ONCE(s->sequence);
- if (unlikely(ret & 1)) {
- cpu_relax();
- goto repeat;
- }
- kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX);
- return ret;
-}
+({ \
+ unsigned seq; \
+ \
+ while ((seq = __seqcount_sequence(s)) & 1) \
+ cpu_relax(); \
+ \
+ kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); \
+ seq; \
+})
/**
* raw_read_seqcount_begin() - begin a seqcount_t read section w/o lockdep
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* Return: count to be passed to read_seqcount_retry()
*/
#define raw_read_seqcount_begin(s) \
- raw_read_seqcount_t_begin(__seqcount_ptr(s))
-
-static inline unsigned raw_read_seqcount_t_begin(const seqcount_t *s)
-{
- unsigned ret = __read_seqcount_t_begin(s);
- smp_rmb();
- return ret;
-}
+({ \
+ unsigned seq = __read_seqcount_begin(s); \
+ \
+ smp_rmb(); \
+ seq; \
+})
/**
* read_seqcount_begin() - begin a seqcount_t read critical section
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* Return: count to be passed to read_seqcount_retry()
*/
#define read_seqcount_begin(s) \
- read_seqcount_t_begin(__seqcount_ptr(s))
-
-static inline unsigned read_seqcount_t_begin(const seqcount_t *s)
-{
- seqcount_lockdep_reader_access(s);
- return raw_read_seqcount_t_begin(s);
-}
+({ \
+ seqcount_lockdep_reader_access(__seqcount_ptr(s)); \
+ raw_read_seqcount_begin(s); \
+})
/**
* raw_read_seqcount() - read the raw seqcount_t counter value
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* raw_read_seqcount opens a read critical section of the given
* seqcount_t, without any lockdep checking, and without checking or
* Return: count to be passed to read_seqcount_retry()
*/
#define raw_read_seqcount(s) \
- raw_read_seqcount_t(__seqcount_ptr(s))
-
-static inline unsigned raw_read_seqcount_t(const seqcount_t *s)
-{
- unsigned ret = READ_ONCE(s->sequence);
- smp_rmb();
- kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX);
- return ret;
-}
+({ \
+ unsigned seq = __seqcount_sequence(s); \
+ \
+ smp_rmb(); \
+ kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); \
+ seq; \
+})
/**
* raw_seqcount_begin() - begin a seqcount_t read critical section w/o
* lockdep and w/o counter stabilization
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* raw_seqcount_begin opens a read critical section of the given
* seqcount_t. Unlike read_seqcount_begin(), this function will not wait
* Return: count to be passed to read_seqcount_retry()
*/
#define raw_seqcount_begin(s) \
- raw_seqcount_t_begin(__seqcount_ptr(s))
-
-static inline unsigned raw_seqcount_t_begin(const seqcount_t *s)
-{
- /*
- * If the counter is odd, let read_seqcount_retry() fail
- * by decrementing the counter.
- */
- return raw_read_seqcount_t(s) & ~1;
-}
+({ \
+ /* \
+ * If the counter is odd, let read_seqcount_retry() fail \
+ * by decrementing the counter. \
+ */ \
+ raw_read_seqcount(s) & ~1; \
+})
/**
* __read_seqcount_retry() - end a seqcount_t read section w/o barrier
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
* @start: count, from read_seqcount_begin()
*
* __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb()
/**
* read_seqcount_retry() - end a seqcount_t read critical section
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
* @start: count, from read_seqcount_begin()
*
* read_seqcount_retry closes the read critical section of given
/**
* raw_write_seqcount_begin() - start a seqcount_t write section w/o lockdep
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*/
#define raw_write_seqcount_begin(s) \
do { \
/**
* raw_write_seqcount_end() - end a seqcount_t write section w/o lockdep
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*/
#define raw_write_seqcount_end(s) \
do { \
/**
* write_seqcount_begin_nested() - start a seqcount_t write section with
* custom lockdep nesting level
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
* @subclass: lockdep nesting level
*
* See Documentation/locking/lockdep-design.rst
/**
* write_seqcount_begin() - start a seqcount_t write side critical section
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* write_seqcount_begin opens a write side critical section of the given
* seqcount_t.
/**
* write_seqcount_end() - end a seqcount_t write side critical section
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* The write section must've been opened with write_seqcount_begin().
*/
/**
* raw_write_seqcount_barrier() - do a seqcount_t write barrier
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* This can be used to provide an ordering guarantee instead of the usual
* consistency guarantee. It is one wmb cheaper, because it can collapse
/**
* write_seqcount_invalidate() - invalidate in-progress seqcount_t read
* side operations
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * @s: Pointer to seqcount_t or any of the seqcount_LOCKNAME_t variants
*
* After write_seqcount_invalidate, no seqcount_t read side operations
* will complete successfully and see data older than this.
kcsan_nestable_atomic_end();
}
-/**
- * raw_read_seqcount_latch() - pick even/odd seqcount_t latch data copy
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+/*
+ * Latch sequence counters (seqcount_latch_t)
*
- * Use seqcount_t latching to switch between two storage places protected
- * by a sequence counter. Doing so allows having interruptible, preemptible,
- * seqcount_t write side critical sections.
+ * A sequence counter variant where the counter even/odd value is used to
+ * switch between two copies of protected data. This allows the read path,
+ * typically NMIs, to safely interrupt the write side critical section.
*
- * Check raw_write_seqcount_latch() for more details and a full reader and
- * writer usage example.
+ * As the write sections are fully preemptible, no special handling for
+ * PREEMPT_RT is needed.
+ */
+typedef struct {
+ seqcount_t seqcount;
+} seqcount_latch_t;
+
+/**
+ * SEQCNT_LATCH_ZERO() - static initializer for seqcount_latch_t
+ * @seq_name: Name of the seqcount_latch_t instance
+ */
+#define SEQCNT_LATCH_ZERO(seq_name) { \
+ .seqcount = SEQCNT_ZERO(seq_name.seqcount), \
+}
+
+/**
+ * seqcount_latch_init() - runtime initializer for seqcount_latch_t
+ * @s: Pointer to the seqcount_latch_t instance
+ */
+static inline void seqcount_latch_init(seqcount_latch_t *s)
+{
+ seqcount_init(&s->seqcount);
+}
+
+/**
+ * raw_read_seqcount_latch() - pick even/odd latch data copy
+ * @s: Pointer to seqcount_latch_t
+ *
+ * See raw_write_seqcount_latch() for details and a full reader/writer
+ * usage example.
*
* Return: sequence counter raw value. Use the lowest bit as an index for
- * picking which data copy to read. The full counter value must then be
- * checked with read_seqcount_retry().
+ * picking which data copy to read. The full counter must then be checked
+ * with read_seqcount_latch_retry().
*/
-#define raw_read_seqcount_latch(s) \
- raw_read_seqcount_t_latch(__seqcount_ptr(s))
+static inline unsigned raw_read_seqcount_latch(const seqcount_latch_t *s)
+{
+ /*
+ * Pairs with the first smp_wmb() in raw_write_seqcount_latch().
+ * Due to the dependent load, a full smp_rmb() is not needed.
+ */
+ return READ_ONCE(s->seqcount.sequence);
+}
-static inline int raw_read_seqcount_t_latch(seqcount_t *s)
+/**
+ * read_seqcount_latch_retry() - end a seqcount_latch_t read section
+ * @s: Pointer to seqcount_latch_t
+ * @start: count, from raw_read_seqcount_latch()
+ *
+ * Return: true if a read section retry is required, else false
+ */
+static inline int
+read_seqcount_latch_retry(const seqcount_latch_t *s, unsigned start)
{
- /* Pairs with the first smp_wmb() in raw_write_seqcount_latch() */
- int seq = READ_ONCE(s->sequence); /* ^^^ */
- return seq;
+ return read_seqcount_retry(&s->seqcount, start);
}
/**
- * raw_write_seqcount_latch() - redirect readers to even/odd copy
- * @s: Pointer to seqcount_t or any of the seqcount_locktype_t variants
+ * raw_write_seqcount_latch() - redirect latch readers to even/odd copy
+ * @s: Pointer to seqcount_latch_t
*
* The latch technique is a multiversion concurrency control method that allows
* queries during non-atomic modifications. If you can guarantee queries never
* The basic form is a data structure like::
*
* struct latch_struct {
- * seqcount_t seq;
+ * seqcount_latch_t seq;
* struct data_struct data[2];
* };
*
* void latch_modify(struct latch_struct *latch, ...)
* {
* smp_wmb(); // Ensure that the last data[1] update is visible
- * latch->seq++;
+ * latch->seq.sequence++;
* smp_wmb(); // Ensure that the seqcount update is visible
*
* modify(latch->data[0], ...);
*
* smp_wmb(); // Ensure that the data[0] update is visible
- * latch->seq++;
+ * latch->seq.sequence++;
* smp_wmb(); // Ensure that the seqcount update is visible
*
* modify(latch->data[1], ...);
* idx = seq & 0x01;
* entry = data_query(latch->data[idx], ...);
*
- * // read_seqcount_retry() includes needed smp_rmb()
- * } while (read_seqcount_retry(&latch->seq, seq));
+ * // This includes needed smp_rmb()
+ * } while (read_seqcount_latch_retry(&latch->seq, seq));
*
* return entry;
* }
* to miss an entire modification sequence, once it resumes it might
* observe the new entry.
*
- * NOTE:
+ * NOTE2:
*
* When data is a dynamic data structure; one should use regular RCU
* patterns to manage the lifetimes of the objects within.
*/
-#define raw_write_seqcount_latch(s) \
- raw_write_seqcount_t_latch(__seqcount_ptr(s))
-
-static inline void raw_write_seqcount_t_latch(seqcount_t *s)
+static inline void raw_write_seqcount_latch(seqcount_latch_t *s)
{
- smp_wmb(); /* prior stores before incrementing "sequence" */
- s->sequence++;
- smp_wmb(); /* increment "sequence" before following stores */
+ smp_wmb(); /* prior stores before incrementing "sequence" */
+ s->seqcount.sequence++;
+ smp_wmb(); /* increment "sequence" before following stores */
}
/*
* - Documentation/locking/seqlock.rst
*/
typedef struct {
- struct seqcount seqcount;
+ /*
+ * Make sure that readers don't starve writers on PREEMPT_RT: use
+ * seqcount_spinlock_t instead of seqcount_t. Check __SEQ_LOCK().
+ */
+ seqcount_spinlock_t seqcount;
spinlock_t lock;
} seqlock_t;
#define __SEQLOCK_UNLOCKED(lockname) \
{ \
- .seqcount = SEQCNT_ZERO(lockname), \
+ .seqcount = SEQCNT_SPINLOCK_ZERO(lockname, &(lockname).lock), \
.lock = __SPIN_LOCK_UNLOCKED(lockname) \
}
*/
#define seqlock_init(sl) \
do { \
- seqcount_init(&(sl)->seqcount); \
spin_lock_init(&(sl)->lock); \
+ seqcount_spinlock_init(&(sl)->seqcount, &(sl)->lock); \
} while (0)
/**
- * DEFINE_SEQLOCK() - Define a statically allocated seqlock_t
+ * DEFINE_SEQLOCK(sl) - Define a statically allocated seqlock_t
* @sl: Name of the seqlock_t instance
*/
#define DEFINE_SEQLOCK(sl) \
return read_seqcount_retry(&sl->seqcount, start);
}
+/*
+ * For all seqlock_t write side functions, use write_seqcount_*t*_begin()
+ * instead of the generic write_seqcount_begin(). This way, no redundant
+ * lockdep_assert_held() checks are added.
+ */
+
/**
* write_seqlock() - start a seqlock_t write side critical section
* @sl: Pointer to seqlock_t
static inline void write_seqlock(seqlock_t *sl)
{
spin_lock(&sl->lock);
- write_seqcount_t_begin(&sl->seqcount);
+ write_seqcount_t_begin(&sl->seqcount.seqcount);
}
/**
*/
static inline void write_sequnlock(seqlock_t *sl)
{
- write_seqcount_t_end(&sl->seqcount);
+ write_seqcount_t_end(&sl->seqcount.seqcount);
spin_unlock(&sl->lock);
}
static inline void write_seqlock_bh(seqlock_t *sl)
{
spin_lock_bh(&sl->lock);
- write_seqcount_t_begin(&sl->seqcount);
+ write_seqcount_t_begin(&sl->seqcount.seqcount);
}
/**
*/
static inline void write_sequnlock_bh(seqlock_t *sl)
{
- write_seqcount_t_end(&sl->seqcount);
+ write_seqcount_t_end(&sl->seqcount.seqcount);
spin_unlock_bh(&sl->lock);
}
static inline void write_seqlock_irq(seqlock_t *sl)
{
spin_lock_irq(&sl->lock);
- write_seqcount_t_begin(&sl->seqcount);
+ write_seqcount_t_begin(&sl->seqcount.seqcount);
}
/**
*/
static inline void write_sequnlock_irq(seqlock_t *sl)
{
- write_seqcount_t_end(&sl->seqcount);
+ write_seqcount_t_end(&sl->seqcount.seqcount);
spin_unlock_irq(&sl->lock);
}
unsigned long flags;
spin_lock_irqsave(&sl->lock, flags);
- write_seqcount_t_begin(&sl->seqcount);
+ write_seqcount_t_begin(&sl->seqcount.seqcount);
return flags;
}
static inline void
write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags)
{
- write_seqcount_t_end(&sl->seqcount);
+ write_seqcount_t_end(&sl->seqcount.seqcount);
spin_unlock_irqrestore(&sl->lock, flags);
}
int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len);
int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len);
__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, u8 *to,
- int len, __wsum csum);
+ int len);
int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
struct pipe_inode_info *pipe, unsigned int len,
unsigned int flags);
#define QCOM_SMD_RPM_CLK_BUF_A 0x616B6C63
#define QCOM_SMD_RPM_LDOA 0x616f646c
#define QCOM_SMD_RPM_LDOB 0x626F646C
+#define QCOM_SMD_RPM_RWCX 0x78637772
+#define QCOM_SMD_RPM_RWMX 0x786d7772
+#define QCOM_SMD_RPM_RWLC 0x636c7772
+#define QCOM_SMD_RPM_RWLM 0x6d6c7772
#define QCOM_SMD_RPM_MEM_CLK 0x326b6c63
#define QCOM_SMD_RPM_MISC_CLK 0x306b6c63
#define QCOM_SMD_RPM_NCPA 0x6170636E
* stack_trace_consume_fn - Callback for arch_stack_walk()
* @cookie: Caller supplied pointer handed back by arch_stack_walk()
* @addr: The stack entry address to consume
- * @reliable: True when the stack entry is reliable. Required by
- * some printk based consumers.
*
* Return: True, if the entry was consumed or skipped
* False, if there is no space left to store
*/
-typedef bool (*stack_trace_consume_fn)(void *cookie, unsigned long addr,
- bool reliable);
+typedef bool (*stack_trace_consume_fn)(void *cookie, unsigned long addr);
/**
* arch_stack_walk - Architecture specific function to walk the stack
* @consume_entry: Callback which is invoked by the architecture code for
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_STATIC_CALL_H
+#define _LINUX_STATIC_CALL_H
+
+/*
+ * Static call support
+ *
+ * Static calls use code patching to hard-code function pointers into direct
+ * branch instructions. They give the flexibility of function pointers, but
+ * with improved performance. This is especially important for cases where
+ * retpolines would otherwise be used, as retpolines can significantly impact
+ * performance.
+ *
+ *
+ * API overview:
+ *
+ * DECLARE_STATIC_CALL(name, func);
+ * DEFINE_STATIC_CALL(name, func);
+ * DEFINE_STATIC_CALL_NULL(name, typename);
+ * static_call(name)(args...);
+ * static_call_cond(name)(args...);
+ * static_call_update(name, func);
+ *
+ * Usage example:
+ *
+ * # Start with the following functions (with identical prototypes):
+ * int func_a(int arg1, int arg2);
+ * int func_b(int arg1, int arg2);
+ *
+ * # Define a 'my_name' reference, associated with func_a() by default
+ * DEFINE_STATIC_CALL(my_name, func_a);
+ *
+ * # Call func_a()
+ * static_call(my_name)(arg1, arg2);
+ *
+ * # Update 'my_name' to point to func_b()
+ * static_call_update(my_name, &func_b);
+ *
+ * # Call func_b()
+ * static_call(my_name)(arg1, arg2);
+ *
+ *
+ * Implementation details:
+ *
+ * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL).
+ * Otherwise basic indirect calls are used (with function pointers).
+ *
+ * Each static_call() site calls into a trampoline associated with the name.
+ * The trampoline has a direct branch to the default function. Updates to a
+ * name will modify the trampoline's branch destination.
+ *
+ * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites
+ * themselves will be patched at runtime to call the functions directly,
+ * rather than calling through the trampoline. This requires objtool or a
+ * compiler plugin to detect all the static_call() sites and annotate them
+ * in the .static_call_sites section.
+ *
+ *
+ * Notes on NULL function pointers:
+ *
+ * Static_call()s support NULL functions, with many of the caveats that
+ * regular function pointers have.
+ *
+ * Clearly calling a NULL function pointer is 'BAD', so too for
+ * static_call()s (although when HAVE_STATIC_CALL it might not be immediately
+ * fatal). A NULL static_call can be the result of:
+ *
+ * DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int));
+ *
+ * which is equivalent to declaring a NULL function pointer with just a
+ * typename:
+ *
+ * void (*my_func_ptr)(int arg1) = NULL;
+ *
+ * or using static_call_update() with a NULL function. In both cases the
+ * HAVE_STATIC_CALL implementation will patch the trampoline with a RET
+ * instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE
+ * architectures can patch the trampoline call to a NOP.
+ *
+ * In all cases, any argument evaluation is unconditional. Unlike a regular
+ * conditional function pointer call:
+ *
+ * if (my_func_ptr)
+ * my_func_ptr(arg1)
+ *
+ * where the argument evaludation also depends on the pointer value.
+ *
+ * When calling a static_call that can be NULL, use:
+ *
+ * static_call_cond(name)(arg1);
+ *
+ * which will include the required value tests to avoid NULL-pointer
+ * dereferences.
+ */
+
+#include <linux/types.h>
+#include <linux/cpu.h>
+#include <linux/static_call_types.h>
+
+#ifdef CONFIG_HAVE_STATIC_CALL
+#include <asm/static_call.h>
+
+/*
+ * Either @site or @tramp can be NULL.
+ */
+extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);
+
+#define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
+
+/*
+ * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from
+ * the symbol table so that objtool can reference it when it generates the
+ * .static_call_sites section.
+ */
+#define __static_call(name) \
+({ \
+ __ADDRESSABLE(STATIC_CALL_KEY(name)); \
+ &STATIC_CALL_TRAMP(name); \
+})
+
+#else
+#define STATIC_CALL_TRAMP_ADDR(name) NULL
+#endif
+
+
+#define DECLARE_STATIC_CALL(name, func) \
+ extern struct static_call_key STATIC_CALL_KEY(name); \
+ extern typeof(func) STATIC_CALL_TRAMP(name);
+
+#define static_call_update(name, func) \
+({ \
+ BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name))); \
+ __static_call_update(&STATIC_CALL_KEY(name), \
+ STATIC_CALL_TRAMP_ADDR(name), func); \
+})
+
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+
+extern int __init static_call_init(void);
+
+struct static_call_mod {
+ struct static_call_mod *next;
+ struct module *mod; /* for vmlinux, mod == NULL */
+ struct static_call_site *sites;
+};
+
+struct static_call_key {
+ void *func;
+ union {
+ /* bit 0: 0 = mods, 1 = sites */
+ unsigned long type;
+ struct static_call_mod *mods;
+ struct static_call_site *sites;
+ };
+};
+
+extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
+extern int static_call_mod_init(struct module *mod);
+extern int static_call_text_reserved(void *start, void *end);
+
+#define DEFINE_STATIC_CALL(name, _func) \
+ DECLARE_STATIC_CALL(name, _func); \
+ struct static_call_key STATIC_CALL_KEY(name) = { \
+ .func = _func, \
+ .type = 1, \
+ }; \
+ ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
+
+#define DEFINE_STATIC_CALL_NULL(name, _func) \
+ DECLARE_STATIC_CALL(name, _func); \
+ struct static_call_key STATIC_CALL_KEY(name) = { \
+ .func = NULL, \
+ .type = 1, \
+ }; \
+ ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
+
+#define static_call(name) __static_call(name)
+#define static_call_cond(name) (void)__static_call(name)
+
+#define EXPORT_STATIC_CALL(name) \
+ EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
+ EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
+
+#define EXPORT_STATIC_CALL_GPL(name) \
+ EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
+ EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
+
+#elif defined(CONFIG_HAVE_STATIC_CALL)
+
+static inline int static_call_init(void) { return 0; }
+
+struct static_call_key {
+ void *func;
+};
+
+#define DEFINE_STATIC_CALL(name, _func) \
+ DECLARE_STATIC_CALL(name, _func); \
+ struct static_call_key STATIC_CALL_KEY(name) = { \
+ .func = _func, \
+ }; \
+ ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
+
+#define DEFINE_STATIC_CALL_NULL(name, _func) \
+ DECLARE_STATIC_CALL(name, _func); \
+ struct static_call_key STATIC_CALL_KEY(name) = { \
+ .func = NULL, \
+ }; \
+ ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
+
+#define static_call(name) __static_call(name)
+#define static_call_cond(name) (void)__static_call(name)
+
+static inline
+void __static_call_update(struct static_call_key *key, void *tramp, void *func)
+{
+ cpus_read_lock();
+ WRITE_ONCE(key->func, func);
+ arch_static_call_transform(NULL, tramp, func, false);
+ cpus_read_unlock();
+}
+
+static inline int static_call_text_reserved(void *start, void *end)
+{
+ return 0;
+}
+
+#define EXPORT_STATIC_CALL(name) \
+ EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
+ EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
+
+#define EXPORT_STATIC_CALL_GPL(name) \
+ EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
+ EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
+
+#else /* Generic implementation */
+
+static inline int static_call_init(void) { return 0; }
+
+struct static_call_key {
+ void *func;
+};
+
+#define DEFINE_STATIC_CALL(name, _func) \
+ DECLARE_STATIC_CALL(name, _func); \
+ struct static_call_key STATIC_CALL_KEY(name) = { \
+ .func = _func, \
+ }
+
+#define DEFINE_STATIC_CALL_NULL(name, _func) \
+ DECLARE_STATIC_CALL(name, _func); \
+ struct static_call_key STATIC_CALL_KEY(name) = { \
+ .func = NULL, \
+ }
+
+#define static_call(name) \
+ ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func))
+
+static inline void __static_call_nop(void) { }
+
+/*
+ * This horrific hack takes care of two things:
+ *
+ * - it ensures the compiler will only load the function pointer ONCE,
+ * which avoids a reload race.
+ *
+ * - it ensures the argument evaluation is unconditional, similar
+ * to the HAVE_STATIC_CALL variant.
+ *
+ * Sadly current GCC/Clang (10 for both) do not optimize this properly
+ * and will emit an indirect call for the NULL case :-(
+ */
+#define __static_call_cond(name) \
+({ \
+ void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \
+ if (!func) \
+ func = &__static_call_nop; \
+ (typeof(STATIC_CALL_TRAMP(name))*)func; \
+})
+
+#define static_call_cond(name) (void)__static_call_cond(name)
+
+static inline
+void __static_call_update(struct static_call_key *key, void *tramp, void *func)
+{
+ WRITE_ONCE(key->func, func);
+}
+
+static inline int static_call_text_reserved(void *start, void *end)
+{
+ return 0;
+}
+
+#define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name))
+#define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))
+
+#endif /* CONFIG_HAVE_STATIC_CALL */
+
+#endif /* _LINUX_STATIC_CALL_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _STATIC_CALL_TYPES_H
+#define _STATIC_CALL_TYPES_H
+
+#include <linux/types.h>
+#include <linux/stringify.h>
+
+#define STATIC_CALL_KEY_PREFIX __SCK__
+#define STATIC_CALL_KEY_PREFIX_STR __stringify(STATIC_CALL_KEY_PREFIX)
+#define STATIC_CALL_KEY_PREFIX_LEN (sizeof(STATIC_CALL_KEY_PREFIX_STR) - 1)
+#define STATIC_CALL_KEY(name) __PASTE(STATIC_CALL_KEY_PREFIX, name)
+
+#define STATIC_CALL_TRAMP_PREFIX __SCT__
+#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX)
+#define STATIC_CALL_TRAMP_PREFIX_LEN (sizeof(STATIC_CALL_TRAMP_PREFIX_STR) - 1)
+#define STATIC_CALL_TRAMP(name) __PASTE(STATIC_CALL_TRAMP_PREFIX, name)
+#define STATIC_CALL_TRAMP_STR(name) __stringify(STATIC_CALL_TRAMP(name))
+
+/*
+ * Flags in the low bits of static_call_site::key.
+ */
+#define STATIC_CALL_SITE_TAIL 1UL /* tail call */
+#define STATIC_CALL_SITE_INIT 2UL /* init section */
+#define STATIC_CALL_SITE_FLAGS 3UL
+
+/*
+ * The static call site table needs to be created by external tooling (objtool
+ * or a compiler plugin).
+ */
+struct static_call_site {
+ s32 addr;
+ s32 key;
+};
+
+#endif /* _STATIC_CALL_TYPES_H */
#ifndef __HAVE_ARCH_MEMCHR
extern void * memchr(const void *,int,__kernel_size_t);
#endif
-#ifndef __HAVE_ARCH_MEMCPY_MCSAFE
-static inline __must_check unsigned long memcpy_mcsafe(void *dst,
- const void *src, size_t cnt)
-{
- memcpy(dst, src, cnt);
- return 0;
-}
-#endif
#ifndef __HAVE_ARCH_MEMCPY_FLUSHCACHE
static inline void memcpy_flushcache(void *dst, const void *src, size_t cnt)
{
memcpy(dst, src, cnt);
}
#endif
+
void *memchr_inv(const void *s, int c, size_t n);
char *strreplace(char *s, char old, char new);
char *kstrdup_quotable_cmdline(struct task_struct *task, gfp_t gfp);
char *kstrdup_quotable_file(struct file *file, gfp_t gfp);
+void kfree_strarray(char **array, size_t n);
+
#endif
SGN_ALG_MD2_5 = 0x0001,
SGN_ALG_DES_MAC = 0x0002,
SGN_ALG_3 = 0x0003, /* not published */
- SGN_ALG_HMAC_MD5 = 0x0011, /* microsoft w2k; no support */
SGN_ALG_HMAC_SHA1_DES3_KD = 0x0004
};
enum seal_alg {
SEAL_ALG_NONE = 0xffff,
SEAL_ALG_DES = 0x0000,
SEAL_ALG_1 = 0x0001, /* not published */
- SEAL_ALG_MICROSOFT_RC4 = 0x0010,/* microsoft w2k; no support */
SEAL_ALG_DES3KD = 0x0002
};
struct xdr_buf *buf, u32 *plainoffset,
u32 *plainlen);
-int
-krb5_rc4_setup_seq_key(struct krb5_ctx *kctx,
- struct crypto_sync_skcipher *cipher,
- unsigned char *cksum);
-
-int
-krb5_rc4_setup_enc_key(struct krb5_ctx *kctx,
- struct crypto_sync_skcipher *cipher,
- s32 seqnum);
void
gss_krb5_make_confounder(char *p, u32 conflen);
#ifdef CONFIG_SUNRPC_DISABLE_INSECURE_ENCTYPES
/*
- * NB: This list includes encryption types that were deprecated
- * by RFC 8429 (DES3_CBC_SHA1 and ARCFOUR_HMAC).
+ * NB: This list includes DES3_CBC_SHA1, which was deprecated by RFC 8429.
*
* ENCTYPE_AES256_CTS_HMAC_SHA1_96
* ENCTYPE_AES128_CTS_HMAC_SHA1_96
* ENCTYPE_DES3_CBC_SHA1
- * ENCTYPE_ARCFOUR_HMAC
*/
-#define KRB5_SUPPORTED_ENCTYPES "18,17,16,23"
+#define KRB5_SUPPORTED_ENCTYPES "18,17,16"
#else /* CONFIG_SUNRPC_DISABLE_INSECURE_ENCTYPES */
* ENCTYPE_AES256_CTS_HMAC_SHA1_96
* ENCTYPE_AES128_CTS_HMAC_SHA1_96
* ENCTYPE_DES3_CBC_SHA1
- * ENCTYPE_ARCFOUR_HMAC
* ENCTYPE_DES_CBC_MD5
* ENCTYPE_DES_CBC_CRC
* ENCTYPE_DES_CBC_MD4
*/
-#define KRB5_SUPPORTED_ENCTYPES "18,17,16,23,3,1,2"
+#define KRB5_SUPPORTED_ENCTYPES "18,17,16,3,1,2"
#endif /* CONFIG_SUNRPC_DISABLE_INSECURE_ENCTYPES */
const char __user *const __user *argv,
const char __user *const __user *envp, int flags);
asmlinkage long sys_userfaultfd(int flags);
-asmlinkage long sys_membarrier(int cmd, int flags);
+asmlinkage long sys_membarrier(int cmd, unsigned int flags, int cpu_id);
asmlinkage long sys_mlock2(unsigned long start, size_t len, int flags);
asmlinkage long sys_copy_file_range(int fd_in, loff_t __user *off_in,
int fd_out, loff_t __user *off_out,
extern void timekeeping_inject_sleeptime64(const struct timespec64 *delta);
+/*
+ * struct ktime_timestanps - Simultaneous mono/boot/real timestamps
+ * @mono: Monotonic timestamp
+ * @boot: Boottime timestamp
+ * @real: Realtime timestamp
+ */
+struct ktime_timestamps {
+ u64 mono;
+ u64 boot;
+ u64 real;
+};
+
/**
* struct system_time_snapshot - simultaneous raw/real time capture with
* counter value
*/
extern void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot);
+/* NMI safe mono/boot/realtime timestamps */
+extern void ktime_get_fast_timestamps(struct ktime_timestamps *snap);
+
/*
* Persistent clock related interfaces
*/
#define TIMER_DEFERRABLE 0x00080000
#define TIMER_PINNED 0x00100000
#define TIMER_IRQSAFE 0x00200000
+#define TIMER_INIT_FLAGS (TIMER_DEFERRABLE | TIMER_PINNED | TIMER_IRQSAFE)
#define TIMER_ARRAYSHIFT 22
#define TIMER_ARRAYMASK 0xFFC00000
#include <linux/atomic.h>
#include <linux/static_key.h>
+struct static_call_key;
+
struct trace_print_flags {
unsigned long mask;
const char *name;
struct tracepoint {
const char *name; /* Tracepoint name */
struct static_key key;
+ struct static_call_key *static_call_key;
+ void *static_call_tramp;
+ void *iterator;
int (*regfunc)(void);
void (*unregfunc)(void);
struct tracepoint_func __rcu *funcs;
#include <linux/cpumask.h>
#include <linux/rcupdate.h>
#include <linux/tracepoint-defs.h>
+#include <linux/static_call.h>
struct module;
struct tracepoint;
extern void syscall_unregfunc(void);
#endif /* CONFIG_HAVE_SYSCALL_TRACEPOINTS */
+#ifndef PARAMS
#define PARAMS(args...) args
+#endif
#define TRACE_DEFINE_ENUM(x)
#define TRACE_DEFINE_SIZEOF(x)
#ifdef TRACEPOINTS_ENABLED
+#ifdef CONFIG_HAVE_STATIC_CALL
+#define __DO_TRACE_CALL(name) static_call(tp_func_##name)
+#else
+#define __DO_TRACE_CALL(name) __traceiter_##name
+#endif /* CONFIG_HAVE_STATIC_CALL */
+
/*
* it_func[0] is never NULL because there is at least one element in the array
* when the array itself is non NULL.
* has a "void" prototype, then it is invalid to declare a function
* as "(void *, void)".
*/
-#define __DO_TRACE(tp, proto, args, cond, rcuidle) \
+#define __DO_TRACE(name, proto, args, cond, rcuidle) \
do { \
struct tracepoint_func *it_func_ptr; \
- void *it_func; \
- void *__data; \
int __maybe_unused __idx = 0; \
+ void *__data; \
\
if (!(cond)) \
return; \
rcu_irq_enter_irqson(); \
} \
\
- it_func_ptr = rcu_dereference_raw((tp)->funcs); \
- \
+ it_func_ptr = \
+ rcu_dereference_raw((&__tracepoint_##name)->funcs); \
if (it_func_ptr) { \
- do { \
- it_func = (it_func_ptr)->func; \
- __data = (it_func_ptr)->data; \
- ((void(*)(proto))(it_func))(args); \
- } while ((++it_func_ptr)->func); \
+ __data = (it_func_ptr)->data; \
+ __DO_TRACE_CALL(name)(args); \
} \
\
if (rcuidle) { \
static inline void trace_##name##_rcuidle(proto) \
{ \
if (static_key_false(&__tracepoint_##name.key)) \
- __DO_TRACE(&__tracepoint_##name, \
+ __DO_TRACE(name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
TP_CONDITION(cond), 1); \
* poking RCU a bit.
*/
#define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
+ extern int __traceiter_##name(data_proto); \
+ DECLARE_STATIC_CALL(tp_func_##name, __traceiter_##name); \
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
if (static_key_false(&__tracepoint_##name.key)) \
- __DO_TRACE(&__tracepoint_##name, \
+ __DO_TRACE(name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
TP_CONDITION(cond), 0); \
* structures, so we create an array of pointers that will be used for iteration
* on the tracepoints.
*/
-#define DEFINE_TRACE_FN(name, reg, unreg) \
- static const char __tpstrtab_##name[] \
- __section(__tracepoints_strings) = #name; \
- struct tracepoint __tracepoint_##name __used \
- __section(__tracepoints) = \
- { __tpstrtab_##name, STATIC_KEY_INIT_FALSE, reg, unreg, NULL };\
- __TRACEPOINT_ENTRY(name);
+#define DEFINE_TRACE_FN(_name, _reg, _unreg, proto, args) \
+ static const char __tpstrtab_##_name[] \
+ __section(__tracepoints_strings) = #_name; \
+ extern struct static_call_key STATIC_CALL_KEY(tp_func_##_name); \
+ int __traceiter_##_name(void *__data, proto); \
+ struct tracepoint __tracepoint_##_name __used \
+ __section(__tracepoints) = { \
+ .name = __tpstrtab_##_name, \
+ .key = STATIC_KEY_INIT_FALSE, \
+ .static_call_key = &STATIC_CALL_KEY(tp_func_##_name), \
+ .static_call_tramp = STATIC_CALL_TRAMP_ADDR(tp_func_##_name), \
+ .iterator = &__traceiter_##_name, \
+ .regfunc = _reg, \
+ .unregfunc = _unreg, \
+ .funcs = NULL }; \
+ __TRACEPOINT_ENTRY(_name); \
+ int __traceiter_##_name(void *__data, proto) \
+ { \
+ struct tracepoint_func *it_func_ptr; \
+ void *it_func; \
+ \
+ it_func_ptr = \
+ rcu_dereference_raw((&__tracepoint_##_name)->funcs); \
+ do { \
+ it_func = (it_func_ptr)->func; \
+ __data = (it_func_ptr)->data; \
+ ((void(*)(void *, proto))(it_func))(__data, args); \
+ } while ((++it_func_ptr)->func); \
+ return 0; \
+ } \
+ DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name);
-#define DEFINE_TRACE(name) \
- DEFINE_TRACE_FN(name, NULL, NULL);
+#define DEFINE_TRACE(name, proto, args) \
+ DEFINE_TRACE_FN(name, NULL, NULL, PARAMS(proto), PARAMS(args));
#define EXPORT_TRACEPOINT_SYMBOL_GPL(name) \
- EXPORT_SYMBOL_GPL(__tracepoint_##name)
+ EXPORT_SYMBOL_GPL(__tracepoint_##name); \
+ EXPORT_SYMBOL_GPL(__traceiter_##name); \
+ EXPORT_STATIC_CALL_GPL(tp_func_##name)
#define EXPORT_TRACEPOINT_SYMBOL(name) \
- EXPORT_SYMBOL(__tracepoint_##name)
+ EXPORT_SYMBOL(__tracepoint_##name); \
+ EXPORT_SYMBOL(__traceiter_##name); \
+ EXPORT_STATIC_CALL(tp_func_##name)
+
#else /* !TRACEPOINTS_ENABLED */
#define __DECLARE_TRACE(name, proto, args, cond, data_proto, data_args) \
return false; \
}
-#define DEFINE_TRACE_FN(name, reg, unreg)
-#define DEFINE_TRACE(name)
+#define DEFINE_TRACE_FN(name, reg, unreg, proto, args)
+#define DEFINE_TRACE(name, proto, args)
#define EXPORT_TRACEPOINT_SYMBOL_GPL(name)
#define EXPORT_TRACEPOINT_SYMBOL(name)
struct list_head list;
struct uacce_qfile_region *qfrs[UACCE_MAX_REGION];
enum uacce_q_state state;
- int pasid;
+ u32 pasid;
struct iommu_sva *handle;
};
}
#endif
+#ifndef copy_mc_to_kernel
+/*
+ * Without arch opt-in this generic copy_mc_to_kernel() will not handle
+ * #MC (or arch equivalent) during source read.
+ */
+static inline unsigned long __must_check
+copy_mc_to_kernel(void *dst, const void *src, size_t cnt)
+{
+ memcpy(dst, src, cnt);
+ return 0;
+}
+#endif
+
static __always_inline void pagefault_disabled_inc(void)
{
current->pagefault_disabled++;
#define _copy_from_iter_flushcache _copy_from_iter_nocache
#endif
-#ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE
-size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i);
+#ifdef CONFIG_ARCH_HAS_COPY_MC
+size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
#else
-#define _copy_to_iter_mcsafe _copy_to_iter
+#define _copy_mc_to_iter _copy_to_iter
#endif
static __always_inline __must_check
}
static __always_inline __must_check
-size_t copy_to_iter_mcsafe(void *addr, size_t bytes, struct iov_iter *i)
+size_t copy_mc_to_iter(void *addr, size_t bytes, struct iov_iter *i)
{
if (unlikely(!check_copy_size(addr, bytes, true)))
return 0;
else
- return _copy_to_iter_mcsafe(addr, bytes, i);
+ return _copy_mc_to_iter(addr, bytes, i);
}
size_t iov_iter_zero(size_t bytes, struct iov_iter *);
size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
struct iov_iter *i);
-ssize_t import_iovec(int type, const struct iovec __user * uvector,
- unsigned nr_segs, unsigned fast_segs,
- struct iovec **iov, struct iov_iter *i);
-
-#ifdef CONFIG_COMPAT
-struct compat_iovec;
-ssize_t compat_import_iovec(int type, const struct compat_iovec __user * uvector,
- unsigned nr_segs, unsigned fast_segs,
- struct iovec **iov, struct iov_iter *i);
-#endif
-
+struct iovec *iovec_from_user(const struct iovec __user *uvector,
+ unsigned long nr_segs, unsigned long fast_segs,
+ struct iovec *fast_iov, bool compat);
+ssize_t import_iovec(int type, const struct iovec __user *uvec,
+ unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
+ struct iov_iter *i);
+ssize_t __import_iovec(int type, const struct iovec __user *uvec,
+ unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
+ struct iov_iter *i, bool compat);
int import_single_range(int type, void __user *buf, size_t len,
struct iovec *iov, struct iov_iter *i);
static inline void __mod_node_page_state(struct pglist_data *pgdat,
enum node_stat_item item, int delta)
{
+ if (vmstat_item_in_bytes(item)) {
+ VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
+ delta >>= PAGE_SHIFT;
+ }
+
node_page_state_add(delta, pgdat, item);
}
*/
#define watch_sizeof(STRUCT) (sizeof(STRUCT) << WATCH_INFO_LENGTH__SHIFT)
+#else
+static inline int watch_queue_init(struct pipe_inode_info *pipe)
+{
+ return -ENOPKG;
+}
+
#endif
#endif /* _LINUX_WATCH_QUEUE_H */
#endif
struct dentry *cec_dir;
- struct dentry *status_file;
- struct dentry *error_inj_file;
u32 sequence;
*/
#define V4L2_H264_NUM_DPB_ENTRIES 16
+#define V4L2_H264_REF_LIST_LEN (2 * V4L2_H264_NUM_DPB_ENTRIES)
+
/* Our pixel format isn't stable at the moment */
#define V4L2_PIX_FMT_H264_SLICE v4l2_fourcc('S', '2', '6', '4') /* H264 parsed slices */
#define V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS (V4L2_CID_MPEG_BASE+1004)
#define V4L2_CID_MPEG_VIDEO_H264_DECODE_MODE (V4L2_CID_MPEG_BASE+1005)
#define V4L2_CID_MPEG_VIDEO_H264_START_CODE (V4L2_CID_MPEG_BASE+1006)
+#define V4L2_CID_MPEG_VIDEO_H264_PRED_WEIGHTS (V4L2_CID_MPEG_BASE+1007)
/* enum v4l2_ctrl_type type values */
#define V4L2_CTRL_TYPE_H264_SPS 0x0110
#define V4L2_CTRL_TYPE_H264_SCALING_MATRIX 0x0112
#define V4L2_CTRL_TYPE_H264_SLICE_PARAMS 0x0113
#define V4L2_CTRL_TYPE_H264_DECODE_PARAMS 0x0114
+#define V4L2_CTRL_TYPE_H264_PRED_WEIGHTS 0x0115
enum v4l2_mpeg_video_h264_decode_mode {
V4L2_MPEG_VIDEO_H264_DECODE_MODE_SLICE_BASED,
#define V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED 0x0010
#define V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT 0x0020
#define V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE 0x0040
-#define V4L2_H264_PPS_FLAG_PIC_SCALING_MATRIX_PRESENT 0x0080
+#define V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT 0x0080
struct v4l2_ctrl_h264_pps {
__u8 pic_parameter_set_id;
__s16 chroma_offset[32][2];
};
-struct v4l2_h264_pred_weight_table {
+#define V4L2_H264_CTRL_PRED_WEIGHTS_REQUIRED(pps, slice) \
+ ((((pps)->flags & V4L2_H264_PPS_FLAG_WEIGHTED_PRED) && \
+ ((slice)->slice_type == V4L2_H264_SLICE_TYPE_P || \
+ (slice)->slice_type == V4L2_H264_SLICE_TYPE_SP)) || \
+ ((pps)->weighted_bipred_idc == 1 && \
+ (slice)->slice_type == V4L2_H264_SLICE_TYPE_B))
+
+struct v4l2_ctrl_h264_pred_weights {
__u16 luma_log2_weight_denom;
__u16 chroma_log2_weight_denom;
struct v4l2_h264_weight_factors weight_factors[2];
#define V4L2_H264_SLICE_TYPE_SP 3
#define V4L2_H264_SLICE_TYPE_SI 4
-#define V4L2_H264_SLICE_FLAG_FIELD_PIC 0x01
-#define V4L2_H264_SLICE_FLAG_BOTTOM_FIELD 0x02
-#define V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED 0x04
-#define V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH 0x08
+#define V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED 0x01
+#define V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH 0x02
-struct v4l2_ctrl_h264_slice_params {
- /* Size in bytes, including header */
- __u32 size;
+#define V4L2_H264_TOP_FIELD_REF 0x1
+#define V4L2_H264_BOTTOM_FIELD_REF 0x2
+#define V4L2_H264_FRAME_REF 0x3
- /* Offset in bytes to the start of slice in the OUTPUT buffer. */
- __u32 start_byte_offset;
+struct v4l2_h264_reference {
+ __u8 fields;
+
+ /* Index into v4l2_ctrl_h264_decode_params.dpb[] */
+ __u8 index;
+};
+struct v4l2_ctrl_h264_slice_params {
/* Offset in bits to slice_data() from the beginning of this slice. */
__u32 header_bit_size;
- __u16 first_mb_in_slice;
+ __u32 first_mb_in_slice;
+
__u8 slice_type;
- __u8 pic_parameter_set_id;
__u8 colour_plane_id;
__u8 redundant_pic_cnt;
- __u16 frame_num;
- __u16 idr_pic_id;
- __u16 pic_order_cnt_lsb;
- __s32 delta_pic_order_cnt_bottom;
- __s32 delta_pic_order_cnt0;
- __s32 delta_pic_order_cnt1;
-
- struct v4l2_h264_pred_weight_table pred_weight_table;
- /* Size in bits of dec_ref_pic_marking() syntax element. */
- __u32 dec_ref_pic_marking_bit_size;
- /* Size in bits of pic order count syntax. */
- __u32 pic_order_cnt_bit_size;
-
__u8 cabac_init_idc;
__s8 slice_qp_delta;
__s8 slice_qs_delta;
__s8 slice_beta_offset_div2;
__u8 num_ref_idx_l0_active_minus1;
__u8 num_ref_idx_l1_active_minus1;
- __u32 slice_group_change_cycle;
- /*
- * Entries on each list are indices into
- * v4l2_ctrl_h264_decode_params.dpb[].
- */
- __u8 ref_pic_list0[32];
- __u8 ref_pic_list1[32];
+ __u8 reserved;
+
+ struct v4l2_h264_reference ref_pic_list0[V4L2_H264_REF_LIST_LEN];
+ struct v4l2_h264_reference ref_pic_list1[V4L2_H264_REF_LIST_LEN];
__u32 flags;
};
#define V4L2_H264_DPB_ENTRY_FLAG_ACTIVE 0x02
#define V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM 0x04
#define V4L2_H264_DPB_ENTRY_FLAG_FIELD 0x08
-#define V4L2_H264_DPB_ENTRY_FLAG_BOTTOM_FIELD 0x10
struct v4l2_h264_dpb_entry {
__u64 reference_ts;
+ __u32 pic_num;
__u16 frame_num;
- __u16 pic_num;
+ __u8 fields;
+ __u8 reserved[5];
/* Note that field is indicated by v4l2_buffer.field */
__s32 top_field_order_cnt;
__s32 bottom_field_order_cnt;
__u32 flags; /* V4L2_H264_DPB_ENTRY_FLAG_* */
};
-#define V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC 0x01
+#define V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC 0x01
+#define V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC 0x02
+#define V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD 0x04
struct v4l2_ctrl_h264_decode_params {
struct v4l2_h264_dpb_entry dpb[V4L2_H264_NUM_DPB_ENTRIES];
- __u16 num_slices;
__u16 nal_ref_idc;
+ __u16 frame_num;
__s32 top_field_order_cnt;
__s32 bottom_field_order_cnt;
+ __u16 idr_pic_id;
+ __u16 pic_order_cnt_lsb;
+ __s32 delta_pic_order_cnt_bottom;
+ __s32 delta_pic_order_cnt0;
+ __s32 delta_pic_order_cnt1;
+ /* Size in bits of dec_ref_pic_marking() syntax element. */
+ __u32 dec_ref_pic_marking_bit_size;
+ /* Size in bits of pic order count syntax. */
+ __u32 pic_order_cnt_bit_size;
+ __u32 slice_group_change_cycle;
+
+ __u32 reserved;
__u32 flags; /* V4L2_H264_DECODE_PARAM_FLAG_* */
};
* @timeout: optional time after which device stops sending data
* @min_timeout: minimum timeout supported by device
* @max_timeout: maximum timeout supported by device
- * @rx_resolution : resolution (in ns) of input sampler
- * @tx_resolution: resolution (in ns) of output sampler
+ * @rx_resolution : resolution (in us) of input sampler
+ * @tx_resolution: resolution (in us) of output sampler
* @lirc_dev: lirc device
* @lirc_cdev: lirc char cdev
* @gap_start: time when gap starts
* @s_wakeup_filter: set the wakeup scancode filter. If the mask is zero
* then wakeup should be disabled. wakeup_protocol will be set to
* a valid protocol if mask is nonzero.
- * @s_timeout: set hardware timeout in ns
+ * @s_timeout: set hardware timeout in us
*/
struct rc_dev {
struct device dev;
unsigned carrier_report:1;
};
-#define IR_DEFAULT_TIMEOUT MS_TO_NS(125)
-#define IR_MAX_DURATION 500000000 /* 500 ms */
#define US_TO_NS(usec) ((usec) * 1000)
#define MS_TO_US(msec) ((msec) * 1000)
-#define MS_TO_NS(msec) ((msec) * 1000 * 1000)
+#define IR_MAX_DURATION MS_TO_US(500)
+#define IR_DEFAULT_TIMEOUT MS_TO_US(125)
void ir_raw_event_handle(struct rc_dev *dev);
int ir_raw_event_store(struct rc_dev *dev, struct ir_raw_event *ev);
static inline void tpg_s_hue(struct tpg_data *tpg,
s16 hue)
{
+ hue = clamp_t(s16, hue, -128, 128);
if (tpg->hue == hue)
return;
tpg->hue = hue;
* @notifier: pointer to &struct v4l2_async_notifier
* @asd: pointer to &struct v4l2_async_subdev
*
- * Call this function before registering a notifier to link the
- * provided asd to the notifiers master @asd_list.
+ * Call this function before registering a notifier to link the provided @asd to
+ * the notifiers master @asd_list. The @asd must be allocated with k*alloc() as
+ * it will be freed by the framework when the notifier is destroyed.
*/
int v4l2_async_notifier_add_subdev(struct v4l2_async_notifier *notifier,
struct v4l2_async_subdev *asd);
buf->timestamp.tv_usec = ts.tv_nsec / NSEC_PER_USEC;
}
+static inline bool v4l2_is_colorspace_valid(__u32 colorspace)
+{
+ return colorspace > V4L2_COLORSPACE_DEFAULT &&
+ colorspace <= V4L2_COLORSPACE_DCI_P3;
+}
+
+static inline bool v4l2_is_xfer_func_valid(__u32 xfer_func)
+{
+ return xfer_func > V4L2_XFER_FUNC_DEFAULT &&
+ xfer_func <= V4L2_XFER_FUNC_SMPTE2084;
+}
+
+static inline bool v4l2_is_ycbcr_enc_valid(__u8 ycbcr_enc)
+{
+ return ycbcr_enc > V4L2_YCBCR_ENC_DEFAULT &&
+ ycbcr_enc <= V4L2_YCBCR_ENC_SMPTE240M;
+}
+
+static inline bool v4l2_is_hsv_enc_valid(__u8 hsv_enc)
+{
+ return hsv_enc == V4L2_HSV_ENC_180 || hsv_enc == V4L2_HSV_ENC_256;
+}
+
+static inline bool v4l2_is_quant_valid(__u8 quantization)
+{
+ return quantization == V4L2_QUANTIZATION_FULL_RANGE ||
+ quantization == V4L2_QUANTIZATION_LIM_RANGE;
+}
+
#endif /* V4L2_COMMON_H_ */
* @p_h264_scaling_matrix: Pointer to a struct v4l2_ctrl_h264_scaling_matrix.
* @p_h264_slice_params: Pointer to a struct v4l2_ctrl_h264_slice_params.
* @p_h264_decode_params: Pointer to a struct v4l2_ctrl_h264_decode_params.
+ * @p_h264_pred_weights: Pointer to a struct v4l2_ctrl_h264_pred_weights.
* @p_vp8_frame_header: Pointer to a VP8 frame header structure.
* @p_hevc_sps: Pointer to an HEVC sequence parameter set structure.
* @p_hevc_pps: Pointer to an HEVC picture parameter set structure.
struct v4l2_ctrl_h264_scaling_matrix *p_h264_scaling_matrix;
struct v4l2_ctrl_h264_slice_params *p_h264_slice_params;
struct v4l2_ctrl_h264_decode_params *p_h264_decode_params;
+ struct v4l2_ctrl_h264_pred_weights *p_h264_pred_weights;
struct v4l2_ctrl_vp8_frame_header *p_vp8_frame_header;
struct v4l2_ctrl_hevc_sps *p_hevc_sps;
struct v4l2_ctrl_hevc_pps *p_hevc_pps;
unsigned int flags;
unsigned char data_lanes[V4L2_FWNODE_CSI2_MAX_DATA_LANES];
unsigned char clock_lane;
- unsigned short num_data_lanes;
+ unsigned char num_data_lanes;
bool lane_polarities[1 + V4L2_FWNODE_CSI2_MAX_DATA_LANES];
};
* struct v4l2_fwnode_endpoint - the endpoint data structure
* @base: fwnode endpoint of the v4l2_fwnode
* @bus_type: bus type
- * @bus: union with bus configuration data structure
+ * @bus: bus configuration data structure
* @bus.parallel: embedded &struct v4l2_fwnode_bus_parallel.
* Used if the bus is parallel.
* @bus.mipi_csi1: embedded &struct v4l2_fwnode_bus_mipi_csi1.
* v4l2_fwnode_endpoint_parse()
*/
enum v4l2_mbus_type bus_type;
- union {
+ struct {
struct v4l2_fwnode_bus_parallel parallel;
struct v4l2_fwnode_bus_mipi_csi1 mipi_csi1;
struct v4l2_fwnode_bus_mipi_csi2 mipi_csi2;
* call this function once the correct type is found --- with a default
* configuration valid for that type.
*
- * As a compatibility means guessing the bus type is also supported by setting
- * @vep.bus_type to V4L2_MBUS_UNKNOWN. The caller may not provide a default
- * configuration in this case as the defaults are specific to a given bus type.
- * This functionality is deprecated and should not be used in new drivers and it
- * is only supported for CSI-2 D-PHY, parallel and Bt.656 buses.
+ * It is also allowed to set @vep.bus_type to V4L2_MBUS_UNKNOWN. USING THIS
+ * FEATURE REQUIRES "bus-type" PROPERTY IN DT BINDINGS. For old drivers,
+ * guessing @vep.bus_type between CSI-2 D-PHY, parallel and BT.656 busses is
+ * supported. NEVER RELY ON GUESSING @vep.bus_type IN NEW DRIVERS!
*
* The function does not change the V4L2 fwnode endpoint state if it fails.
*
* call this function once the correct type is found --- with a default
* configuration valid for that type.
*
- * As a compatibility means guessing the bus type is also supported by setting
- * @vep.bus_type to V4L2_MBUS_UNKNOWN. The caller may not provide a default
- * configuration in this case as the defaults are specific to a given bus type.
- * This functionality is deprecated and should not be used in new drivers and it
- * is only supported for CSI-2 D-PHY, parallel and Bt.656 buses.
+ * It is also allowed to set @vep.bus_type to V4L2_MBUS_UNKNOWN. USING THIS
+ * FEATURE REQUIRES "bus-type" PROPERTY IN DT BINDINGS. For old drivers,
+ * guessing @vep.bus_type between CSI-2 D-PHY, parallel and BT.656 busses is
+ * supported. NEVER RELY ON GUESSING @vep.bus_type IN NEW DRIVERS!
*
* The function does not change the V4L2 fwnode endpoint state if it fails.
*
struct {
s32 pic_order_count;
int frame_num;
- u16 pic_num;
+ u32 pic_num;
u16 longterm : 1;
} refs[V4L2_H264_NUM_DPB_ENTRIES];
s32 cur_pic_order_count;
void
v4l2_h264_init_reflist_builder(struct v4l2_h264_reflist_builder *b,
const struct v4l2_ctrl_h264_decode_params *dec_params,
- const struct v4l2_ctrl_h264_slice_params *slice_params,
const struct v4l2_ctrl_h264_sps *sps,
const struct v4l2_h264_dpb_entry dpb[V4L2_H264_NUM_DPB_ENTRIES]);
#include <linux/v4l2-mediabus.h>
#include <linux/bitops.h>
+/*
+ * How to use the V4L2_MBUS_* flags:
+ * Flags are defined for each of the possible states and values of a media
+ * bus configuration parameter. One and only one bit of each group of flags
+ * shall be set by the users of the v4l2_subdev_pad_ops.get_mbus_config and
+ * v4l2_subdev_pad_ops.set_mbus_config operations to ensure that no
+ * conflicting settings are specified when reporting and setting the media bus
+ * configuration with the two operations respectively. For example, it is
+ * invalid to set or clear both the V4L2_MBUS_HSYNC_ACTIVE_HIGH and the
+ * V4L2_MBUS_HSYNC_ACTIVE_LOW flag at the same time. Instead either flag
+ * V4L2_MBUS_HSYNC_ACTIVE_HIGH or flag V4L2_MBUS_HSYNC_ACTIVE_LOW shall be
+ * set. The same is true for the V4L2_MBUS_CSI2_1/2/3/4_LANE flags group: only
+ * one of these four bits shall be set.
+ *
+ * TODO: replace the existing V4L2_MBUS_* flags with structures of fields
+ * to avoid conflicting settings.
+ *
+ * In example:
+ * #define V4L2_MBUS_HSYNC_ACTIVE_HIGH BIT(2)
+ * #define V4L2_MBUS_HSYNC_ACTIVE_LOW BIT(3)
+ * will be replaced by a field whose value reports the intended active state of
+ * the signal:
+ * unsigned int v4l2_mbus_hsync_active : 1;
+ */
+
/* Parallel flags */
/*
- * Can the client run in master or in slave mode. By "Master mode" an operation
+ * The client runs in master or in slave mode. By "Master mode" an operation
* mode is meant, when the client (e.g., a camera sensor) is producing
* horizontal and vertical synchronisation. In "Slave mode" the host is
* providing these signals to the slave.
#define V4L2_MBUS_DATA_ENABLE_LOW BIT(15)
/* Serial flags */
-/* How many lanes the client can use */
+/* CSI-2 D-PHY number of data lanes. */
#define V4L2_MBUS_CSI2_1_LANE BIT(0)
#define V4L2_MBUS_CSI2_2_LANE BIT(1)
#define V4L2_MBUS_CSI2_3_LANE BIT(2)
#define V4L2_MBUS_CSI2_4_LANE BIT(3)
-/* On which channels it can send video data */
+/* CSI-2 Virtual Channel identifiers. */
#define V4L2_MBUS_CSI2_CHANNEL_0 BIT(4)
#define V4L2_MBUS_CSI2_CHANNEL_1 BIT(5)
#define V4L2_MBUS_CSI2_CHANNEL_2 BIT(6)
#define V4L2_MBUS_CSI2_CHANNEL_3 BIT(7)
-/* Does it support only continuous or also non-continuous clock mode */
+/* Clock non-continuous mode support. */
#define V4L2_MBUS_CSI2_CONTINUOUS_CLOCK BIT(8)
#define V4L2_MBUS_CSI2_NONCONTINUOUS_CLOCK BIT(9)
void v4l2_m2m_last_buffer_done(struct v4l2_m2m_ctx *m2m_ctx,
struct vb2_v4l2_buffer *vbuf);
+/**
+ * v4l2_m2m_suspend() - stop new jobs from being run and wait for current job
+ * to finish
+ *
+ * @m2m_dev: opaque pointer to the internal data to handle M2M context
+ *
+ * Called by a driver in the suspend hook. Stop new jobs from being run, and
+ * wait for current running job to finish.
+ */
+void v4l2_m2m_suspend(struct v4l2_m2m_dev *m2m_dev);
+
+/**
+ * v4l2_m2m_resume() - resume job running and try to run a queued job
+ *
+ * @m2m_dev: opaque pointer to the internal data to handle M2M context
+ *
+ * Called by a driver in the resume hook. This reverts the operation of
+ * v4l2_m2m_suspend() and allows job to be run. Also try to run a queued job if
+ * there is any.
+ */
+void v4l2_m2m_resume(struct v4l2_m2m_dev *m2m_dev);
+
/**
* v4l2_m2m_reqbufs() - multi-queue-aware REQBUFS multiplexer
*
* OUTPUT device. This is ignored by video capture devices.
*
* @g_input_status: get input status. Same as the status field in the
- * &struct &v4l2_input
+ * &struct v4l2_input
*
* @s_stream: used to notify the driver that a video stream will start or has
* stopped.
*
* @query_dv_timings: callback for VIDIOC_QUERY_DV_TIMINGS() ioctl handler code.
*
- * @g_mbus_config: get supported mediabus configurations
- *
- * @s_mbus_config: set a certain mediabus configuration. This operation is added
- * for compatibility with soc-camera drivers and should not be used by new
- * software.
- *
* @s_rx_buffer: set a host allocated memory buffer for the subdev. The subdev
* can adjust @size to a lower value and must not write more data to the
* buffer starting at @data than the original value of @size.
struct v4l2_dv_timings *timings);
int (*query_dv_timings)(struct v4l2_subdev *sd,
struct v4l2_dv_timings *timings);
- int (*g_mbus_config)(struct v4l2_subdev *sd,
- struct v4l2_mbus_config *cfg);
- int (*s_mbus_config)(struct v4l2_subdev *sd,
- const struct v4l2_mbus_config *cfg);
int (*s_rx_buffer)(struct v4l2_subdev *sd, void *buf,
unsigned int *size);
};
*
* @set_frame_desc: set the low level media bus frame parameters, @fd array
* may be adjusted by the subdev driver to device capabilities.
+ *
+ * @get_mbus_config: get the media bus configuration of a remote sub-device.
+ * The media bus configuration is usually retrieved from the
+ * firmware interface at sub-device probe time, immediately
+ * applied to the hardware and eventually adjusted by the
+ * driver. Remote sub-devices (usually video receivers) shall
+ * use this operation to query the transmitting end bus
+ * configuration in order to adjust their own one accordingly.
+ * Callers should make sure they get the most up-to-date as
+ * possible configuration from the remote end, likely calling
+ * this operation as close as possible to stream on time. The
+ * operation shall fail if the pad index it has been called on
+ * is not valid or in case of unrecoverable failures.
+ *
+ * @set_mbus_config: set the media bus configuration of a remote sub-device.
+ * This operations is intended to allow, in combination with
+ * the get_mbus_config operation, the negotiation of media bus
+ * configuration parameters between media sub-devices. The
+ * operation shall not fail if the requested configuration is
+ * not supported, but the driver shall update the content of
+ * the %config argument to reflect what has been actually
+ * applied to the hardware. The operation shall fail if the
+ * pad index it has been called on is not valid or in case of
+ * unrecoverable failures.
*/
struct v4l2_subdev_pad_ops {
int (*init_cfg)(struct v4l2_subdev *sd,
struct v4l2_mbus_frame_desc *fd);
int (*set_frame_desc)(struct v4l2_subdev *sd, unsigned int pad,
struct v4l2_mbus_frame_desc *fd);
+ int (*get_mbus_config)(struct v4l2_subdev *sd, unsigned int pad,
+ struct v4l2_mbus_config *config);
+ int (*set_mbus_config)(struct v4l2_subdev *sd, unsigned int pad,
+ struct v4l2_mbus_config *config);
};
/**
/* common */
struct scatterlist *sglist;
int sglen;
- int nr_pages;
+ unsigned long nr_pages;
int direction;
};
* vb2_core_reqbufs() - Initiate streaming.
* @q: pointer to &struct vb2_queue with videobuf2 queue.
* @memory: memory type, as defined by &enum vb2_memory.
- * @flags: auxiliary queue/buffer management flags. Currently, the only
- * used flag is %V4L2_FLAG_MEMORY_NON_CONSISTENT.
* @count: requested buffer count.
*
* Videobuf2 core helper to implement VIDIOC_REQBUF() operation. It is called
* Return: returns zero on success; an error code otherwise.
*/
int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory,
- unsigned int flags, unsigned int *count);
+ unsigned int *count);
/**
* vb2_core_create_bufs() - Allocate buffers and any required auxiliary structs
* @q: pointer to &struct vb2_queue with videobuf2 queue.
* @memory: memory type, as defined by &enum vb2_memory.
- * @flags: auxiliary queue/buffer management flags.
* @count: requested buffer count.
* @requested_planes: number of planes requested.
* @requested_sizes: array with the size of the planes.
* Return: returns zero on success; an error code otherwise.
*/
int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory,
- unsigned int flags, unsigned int *count,
+ unsigned int *count,
unsigned int requested_planes,
const unsigned int requested_sizes[]);
#error VB2_MAX_PLANES != VIDEO_MAX_PLANES
#endif
+struct video_device;
+
/**
* struct vb2_v4l2_buffer - video buffer information for v4l2.
*
unsigned long len, unsigned long pgoff, unsigned long flags);
#endif
+/**
+ * vb2_video_unregister_device - unregister the video device and release queue
+ *
+ * @vdev: pointer to &struct video_device
+ *
+ * If the driver uses vb2_fop_release()/_vb2_fop_release(), then it should use
+ * vb2_video_unregister_device() instead of video_unregister_device().
+ *
+ * This function will call video_unregister_device() and then release the
+ * vb2_queue if streaming is in progress. This will stop streaming and
+ * this will simplify the unbind sequence since after this call all subdevs
+ * will have stopped streaming as well.
+ */
+void vb2_video_unregister_device(struct video_device *vdev);
+
/**
* vb2_ops_wait_prepare - helper function to lock a struct &vb2_queue
*
struct nlattr *est, struct tc_action **a,
const struct tc_action_ops *ops, int bind,
u32 flags);
-void tcf_idr_insert(struct tc_action_net *tn, struct tc_action *a);
-
void tcf_idr_cleanup(struct tc_action_net *tn, u32 index);
int tcf_idr_check_alloc(struct tc_action_net *tn, u32 *index,
struct tc_action **a, int bind);
#ifndef _HAVE_ARCH_COPY_AND_CSUM_FROM_USER
static inline
__wsum csum_and_copy_from_user (const void __user *src, void *dst,
- int len, __wsum sum, int *err_ptr)
+ int len)
{
if (copy_from_user(dst, src, len))
- *err_ptr = -EFAULT;
- return csum_partial(dst, len, sum);
+ return 0;
+ return csum_partial(dst, len, ~0U);
}
#endif
#ifndef HAVE_CSUM_COPY_USER
static __inline__ __wsum csum_and_copy_to_user
-(const void *src, void __user *dst, int len, __wsum sum, int *err_ptr)
+(const void *src, void __user *dst, int len)
{
- sum = csum_partial(src, len, sum);
+ __wsum sum = csum_partial(src, len, ~0U);
if (copy_to_user(dst, src, len) == 0)
return sum;
- if (len)
- *err_ptr = -EFAULT;
+ return 0;
+}
+#endif
- return (__force __wsum)-1; /* invalid checksum */
+#ifndef _HAVE_ARCH_CSUM_AND_COPY
+static inline __wsum
+csum_partial_copy_nocheck(const void *src, void *dst, int len)
+{
+ memcpy(dst, src, len);
+ return csum_partial(dst, len, 0);
}
#endif
* @cmd: command identifier
* @internal_flags: flags used by the family
* @flags: flags
+ * @validate: validation flags from enum genl_validate_flags
* @doit: standard command callback
* @start: start callback for dumps
* @dumpit: callback for dumpers
bool forwarding)
{
struct net *net = dev_net(dst->dev);
+ unsigned int mtu;
if (net->ipv4.sysctl_ip_fwd_use_pmtu ||
ip_mtu_locked(dst) ||
!forwarding)
return dst_mtu(dst);
+ /* 'forwarding = true' case should always honour route mtu */
+ mtu = dst_metric_raw(dst, RTAX_MTU);
+ if (mtu)
+ return mtu;
+
return min(READ_ONCE(dst->dev->mtu), IP_MAX_MTU);
}
int netlink_policy_dump_start(const struct nla_policy *policy,
unsigned int maxtype,
unsigned long *state);
-bool netlink_policy_dump_loop(unsigned long *state);
+bool netlink_policy_dump_loop(unsigned long state);
int netlink_policy_dump_write(struct sk_buff *skb, unsigned long state);
+void netlink_policy_dump_free(unsigned long state);
#endif
static inline int xfrm_replay_clone(struct xfrm_state *x,
struct xfrm_state *orig)
{
- x->replay_esn = kzalloc(xfrm_replay_state_esn_len(orig->replay_esn),
+
+ x->replay_esn = kmemdup(orig->replay_esn,
+ xfrm_replay_state_esn_len(orig->replay_esn),
GFP_KERNEL);
if (!x->replay_esn)
return -ENOMEM;
-
- x->replay_esn->bmp_len = orig->replay_esn->bmp_len;
- x->replay_esn->replay_window = orig->replay_esn->replay_window;
-
- x->preplay_esn = kmemdup(x->replay_esn,
- xfrm_replay_state_esn_len(x->replay_esn),
+ x->preplay_esn = kmemdup(orig->preplay_esn,
+ xfrm_replay_state_esn_len(orig->preplay_esn),
GFP_KERNEL);
- if (!x->preplay_esn) {
- kfree(x->replay_esn);
+ if (!x->preplay_esn)
return -ENOMEM;
- }
return 0;
}
#define ANA_SG_CONFIG_REG_3_LIST_LENGTH_M GENMASK(18, 16)
#define ANA_SG_CONFIG_REG_3_LIST_LENGTH_X(x) (((x) & GENMASK(18, 16)) >> 16)
#define ANA_SG_CONFIG_REG_3_GATE_ENABLE BIT(20)
-#define ANA_SG_CONFIG_REG_3_INIT_IPS(x) (((x) << 24) & GENMASK(27, 24))
-#define ANA_SG_CONFIG_REG_3_INIT_IPS_M GENMASK(27, 24)
-#define ANA_SG_CONFIG_REG_3_INIT_IPS_X(x) (((x) & GENMASK(27, 24)) >> 24)
-#define ANA_SG_CONFIG_REG_3_INIT_GATE_STATE BIT(28)
+#define ANA_SG_CONFIG_REG_3_INIT_IPS(x) (((x) << 21) & GENMASK(24, 21))
+#define ANA_SG_CONFIG_REG_3_INIT_IPS_M GENMASK(24, 21)
+#define ANA_SG_CONFIG_REG_3_INIT_IPS_X(x) (((x) & GENMASK(24, 21)) >> 21)
+#define ANA_SG_CONFIG_REG_3_INIT_GATE_STATE BIT(25)
#define ANA_SG_GCL_GS_CONFIG_RSZ 0x4
#undef TRACE_EVENT
#define TRACE_EVENT(name, proto, args, tstruct, assign, print) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
#undef TRACE_EVENT_CONDITION
#define TRACE_EVENT_CONDITION(name, proto, args, cond, tstruct, assign, print) \
#undef TRACE_EVENT_FN
#define TRACE_EVENT_FN(name, proto, args, tstruct, \
assign, print, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
#undef TRACE_EVENT_FN_COND
#define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \
assign, print, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
#undef TRACE_EVENT_NOP
#define TRACE_EVENT_NOP(name, proto, args, struct, assign, print)
#undef DEFINE_EVENT
#define DEFINE_EVENT(template, name, proto, args) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
#undef DEFINE_EVENT_FN
#define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \
- DEFINE_TRACE_FN(name, reg, unreg)
+ DEFINE_TRACE_FN(name, reg, unreg, PARAMS(proto), PARAMS(args))
#undef DEFINE_EVENT_PRINT
#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
#undef DEFINE_EVENT_CONDITION
#define DEFINE_EVENT_CONDITION(template, name, proto, args, cond) \
#undef DECLARE_TRACE
#define DECLARE_TRACE(name, proto, args) \
- DEFINE_TRACE(name)
+ DEFINE_TRACE(name, PARAMS(proto), PARAMS(args))
#undef TRACE_INCLUDE
#undef __TRACE_INCLUDE
#define IO_TREE_OWNER \
EM( IO_TREE_FS_PINNED_EXTENTS, "PINNED_EXTENTS") \
EM( IO_TREE_FS_EXCLUDED_EXTENTS, "EXCLUDED_EXTENTS") \
+ EM( IO_TREE_BTREE_INODE_IO, "BTREE_INODE_IO") \
EM( IO_TREE_INODE_IO, "INODE_IO") \
EM( IO_TREE_INODE_IO_FAILURE, "INODE_IO_FAILURE") \
EM( IO_TREE_RELOC_BLOCKS, "RELOC_BLOCKS") \
DECLARE_EVENT_CLASS(btrfs__ordered_extent,
- TP_PROTO(const struct inode *inode,
+ TP_PROTO(const struct btrfs_inode *inode,
const struct btrfs_ordered_extent *ordered),
TP_ARGS(inode, ordered),
__field( u64, truncated_len )
),
- TP_fast_assign_btrfs(btrfs_sb(inode->i_sb),
- __entry->ino = btrfs_ino(BTRFS_I(inode));
+ TP_fast_assign_btrfs(inode->root->fs_info,
+ __entry->ino = btrfs_ino(inode);
__entry->file_offset = ordered->file_offset;
__entry->start = ordered->disk_bytenr;
__entry->len = ordered->num_bytes;
__entry->flags = ordered->flags;
__entry->compress_type = ordered->compress_type;
__entry->refs = refcount_read(&ordered->refs);
- __entry->root_objectid =
- BTRFS_I(inode)->root->root_key.objectid;
+ __entry->root_objectid = inode->root->root_key.objectid;
__entry->truncated_len = ordered->truncated_len;
),
DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_add,
- TP_PROTO(const struct inode *inode,
+ TP_PROTO(const struct btrfs_inode *inode,
const struct btrfs_ordered_extent *ordered),
TP_ARGS(inode, ordered)
DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_remove,
- TP_PROTO(const struct inode *inode,
+ TP_PROTO(const struct btrfs_inode *inode,
const struct btrfs_ordered_extent *ordered),
TP_ARGS(inode, ordered)
DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_start,
- TP_PROTO(const struct inode *inode,
+ TP_PROTO(const struct btrfs_inode *inode,
const struct btrfs_ordered_extent *ordered),
TP_ARGS(inode, ordered)
DEFINE_EVENT(btrfs__ordered_extent, btrfs_ordered_extent_put,
- TP_PROTO(const struct inode *inode,
+ TP_PROTO(const struct btrfs_inode *inode,
const struct btrfs_ordered_extent *ordered),
TP_ARGS(inode, ordered)
TRACE_EVENT(find_free_extent,
- TP_PROTO(const struct btrfs_fs_info *fs_info, u64 num_bytes,
+ TP_PROTO(const struct btrfs_root *root, u64 num_bytes,
u64 empty_size, u64 data),
- TP_ARGS(fs_info, num_bytes, empty_size, data),
+ TP_ARGS(root, num_bytes, empty_size, data),
TP_STRUCT__entry_btrfs(
+ __field( u64, root_objectid )
__field( u64, num_bytes )
__field( u64, empty_size )
__field( u64, data )
),
- TP_fast_assign_btrfs(fs_info,
+ TP_fast_assign_btrfs(root->fs_info,
+ __entry->root_objectid = root->root_key.objectid;
__entry->num_bytes = num_bytes;
__entry->empty_size = empty_size;
__entry->data = data;
),
TP_printk_btrfs("root=%llu(%s) len=%llu empty_size=%llu flags=%llu(%s)",
- show_root_type(BTRFS_EXTENT_TREE_OBJECTID),
+ show_root_type(__entry->root_objectid),
__entry->num_bytes, __entry->empty_size, __entry->data,
__print_flags((unsigned long)__entry->data, "|",
BTRFS_GROUP_FLAGS))
__entry->ret = ret;
),
- TP_printk("fl=0x%p dev=0x%x:0x%x ino=0x%lx fl_blocker=0x%p fl_owner=0x%p fl_pid=%u fl_flags=%s fl_type=%s fl_start=%lld fl_end=%lld ret=%d",
+ TP_printk("fl=%p dev=0x%x:0x%x ino=0x%lx fl_blocker=%p fl_owner=%p fl_pid=%u fl_flags=%s fl_type=%s fl_start=%lld fl_end=%lld ret=%d",
__entry->fl, MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
__entry->i_ino, __entry->fl_blocker, __entry->fl_owner,
__entry->fl_pid, show_fl_flags(__entry->fl_flags),
__entry->fl_downgrade_time = fl ? fl->fl_downgrade_time : 0;
),
- TP_printk("fl=0x%p dev=0x%x:0x%x ino=0x%lx fl_blocker=0x%p fl_owner=0x%p fl_flags=%s fl_type=%s fl_break_time=%lu fl_downgrade_time=%lu",
+ TP_printk("fl=%p dev=0x%x:0x%x ino=0x%lx fl_blocker=%p fl_owner=%p fl_flags=%s fl_type=%s fl_break_time=%lu fl_downgrade_time=%lu",
__entry->fl, MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
__entry->i_ino, __entry->fl_blocker, __entry->fl_owner,
show_fl_flags(__entry->fl_flags),
__entry->fl_type = fl->fl_type;
),
- TP_printk("dev=0x%x:0x%x ino=0x%lx wcount=%d rcount=%d icount=%d fl_owner=0x%p fl_flags=%s fl_type=%s",
+ TP_printk("dev=0x%x:0x%x ino=0x%lx wcount=%d rcount=%d icount=%d fl_owner=%p fl_flags=%s fl_type=%s",
MAJOR(__entry->s_dev), MINOR(__entry->s_dev),
__entry->i_ino, __entry->wcount, __entry->rcount,
__entry->icount, __entry->fl_owner,
__entry->conflict = conflict;
),
- TP_printk("conflict %d: lease=0x%p fl_flags=%s fl_type=%s; breaker=0x%p fl_flags=%s fl_type=%s",
+ TP_printk("conflict %d: lease=%p fl_flags=%s fl_type=%s; breaker=%p fl_flags=%s fl_type=%s",
__entry->conflict,
__entry->lease,
show_fl_flags(__entry->l_fl_flags),
)
);
+TRACE_EVENT(iocost_iocg_forgive_debt,
+
+ TP_PROTO(struct ioc_gq *iocg, const char *path, struct ioc_now *now,
+ u32 usage_pct, u64 old_debt, u64 new_debt,
+ u64 old_delay, u64 new_delay),
+
+ TP_ARGS(iocg, path, now, usage_pct,
+ old_debt, new_debt, old_delay, new_delay),
+
+ TP_STRUCT__entry (
+ __string(devname, ioc_name(iocg->ioc))
+ __string(cgroup, path)
+ __field(u64, now)
+ __field(u64, vnow)
+ __field(u32, usage_pct)
+ __field(u64, old_debt)
+ __field(u64, new_debt)
+ __field(u64, old_delay)
+ __field(u64, new_delay)
+ ),
+
+ TP_fast_assign(
+ __assign_str(devname, ioc_name(iocg->ioc));
+ __assign_str(cgroup, path);
+ __entry->now = now->now;
+ __entry->vnow = now->vnow;
+ __entry->usage_pct = usage_pct;
+ __entry->old_debt = old_debt;
+ __entry->new_debt = new_debt;
+ __entry->old_delay = old_delay;
+ __entry->new_delay = new_delay;
+ ),
+
+ TP_printk("[%s:%s] now=%llu:%llu usage=%u debt=%llu->%llu delay=%llu->%llu",
+ __get_str(devname), __get_str(cgroup),
+ __entry->now, __entry->vnow, __entry->usage_pct,
+ __entry->old_debt, __entry->new_debt,
+ __entry->old_delay, __entry->new_delay
+ )
+);
+
#endif /* _TRACE_BLK_IOCOST_H */
/* This part must be outside protection */
#define IF_HAVE_PG_IDLE(flag,string)
#endif
+#ifdef CONFIG_64BIT
+#define IF_HAVE_PG_ARCH_2(flag,string) ,{1UL << flag, string}
+#else
+#define IF_HAVE_PG_ARCH_2(flag,string)
+#endif
+
#define __def_pageflag_names \
{1UL << PG_locked, "locked" }, \
{1UL << PG_waiters, "waiters" }, \
IF_HAVE_PG_UNCACHED(PG_uncached, "uncached" ) \
IF_HAVE_PG_HWPOISON(PG_hwpoison, "hwpoison" ) \
IF_HAVE_PG_IDLE(PG_young, "young" ) \
-IF_HAVE_PG_IDLE(PG_idle, "idle" )
+IF_HAVE_PG_IDLE(PG_idle, "idle" ) \
+IF_HAVE_PG_ARCH_2(PG_arch_2, "arch_2" )
#define show_page_flags(flags) \
(flags) ? __print_flags(flags, "|", \
TP_PROTO(struct sched_entity *se),
TP_ARGS(se));
+DECLARE_TRACE(sched_cpu_capacity_tp,
+ TP_PROTO(struct rq *rq),
+ TP_ARGS(rq));
+
DECLARE_TRACE(sched_overutilized_tp,
TP_PROTO(struct root_domain *rd, bool overutilized),
TP_ARGS(rd, overutilized));
DEFINE_XEN_MMU_SET_PTE(xen_mmu_set_pte);
-TRACE_EVENT(xen_mmu_set_pte_at,
- TP_PROTO(struct mm_struct *mm, unsigned long addr,
- pte_t *ptep, pte_t pteval),
- TP_ARGS(mm, addr, ptep, pteval),
- TP_STRUCT__entry(
- __field(struct mm_struct *, mm)
- __field(unsigned long, addr)
- __field(pte_t *, ptep)
- __field(pteval_t, pteval)
- ),
- TP_fast_assign(__entry->mm = mm;
- __entry->addr = addr;
- __entry->ptep = ptep;
- __entry->pteval = pteval.pte),
- TP_printk("mm %p addr %lx ptep %p pteval %0*llx (raw %0*llx)",
- __entry->mm, __entry->addr, __entry->ptep,
- (int)sizeof(pteval_t) * 2, (unsigned long long)pte_val(native_make_pte(__entry->pteval)),
- (int)sizeof(pteval_t) * 2, (unsigned long long)__entry->pteval)
- );
-
TRACE_DEFINE_SIZEOF(pmdval_t);
TRACE_EVENT(xen_mmu_set_pmd,
#define SEGV_ACCADI 5 /* ADI not enabled for mapped object */
#define SEGV_ADIDERR 6 /* Disrupting MCD error */
#define SEGV_ADIPERR 7 /* Precise MCD exception */
-#define NSIGSEGV 7
+#define SEGV_MTEAERR 8 /* Asynchronous ARM MTE error */
+#define SEGV_MTESERR 9 /* Synchronous ARM MTE exception */
+#define NSIGSEGV 9
/*
* SIGBUS si_codes
#define __NR_umount2 39
__SYSCALL(__NR_umount2, sys_umount)
#define __NR_mount 40
-__SC_COMP(__NR_mount, sys_mount, compat_sys_mount)
+__SYSCALL(__NR_mount, sys_mount)
#define __NR_pivot_root 41
__SYSCALL(__NR_pivot_root, sys_pivot_root)
#define __NR_write 64
__SYSCALL(__NR_write, sys_write)
#define __NR_readv 65
-__SC_COMP(__NR_readv, sys_readv, compat_sys_readv)
+__SC_COMP(__NR_readv, sys_readv, sys_readv)
#define __NR_writev 66
-__SC_COMP(__NR_writev, sys_writev, compat_sys_writev)
+__SC_COMP(__NR_writev, sys_writev, sys_writev)
#define __NR_pread64 67
__SC_COMP(__NR_pread64, sys_pread64, compat_sys_pread64)
#define __NR_pwrite64 68
/* fs/splice.c */
#define __NR_vmsplice 75
-__SC_COMP(__NR_vmsplice, sys_vmsplice, compat_sys_vmsplice)
+__SYSCALL(__NR_vmsplice, sys_vmsplice)
#define __NR_splice 76
__SYSCALL(__NR_splice, sys_splice)
#define __NR_tee 77
#define __NR_sendmmsg 269
__SC_COMP(__NR_sendmmsg, sys_sendmmsg, compat_sys_sendmmsg)
#define __NR_process_vm_readv 270
-__SC_COMP(__NR_process_vm_readv, sys_process_vm_readv, \
- compat_sys_process_vm_readv)
+__SYSCALL(__NR_process_vm_readv, sys_process_vm_readv)
#define __NR_process_vm_writev 271
-__SC_COMP(__NR_process_vm_writev, sys_process_vm_writev, \
- compat_sys_process_vm_writev)
+__SYSCALL(__NR_process_vm_writev, sys_process_vm_writev)
#define __NR_kcmp 272
__SYSCALL(__NR_kcmp, sys_kcmp)
#define __NR_finit_module 273
* @non_seq: Flag indicating that the zone is using non-sequential resources
* (for host-aware zoned block devices only).
* @reset: Flag indicating that a zone reset is recommended.
- * @reserved: Padding to 64 B to match the ZBC/ZAC defined zone descriptor size.
+ * @resv: Padding for 8B alignment.
+ * @capacity: Zone usable capacity in 512 B sector units
+ * @reserved: Padding to 64 B to match the ZBC, ZAC and ZNS defined zone
+ * descriptor size.
*
- * start, len and wp use the regular 512 B sector unit, regardless of the
- * device logical block size. The overall structure size is 64 B to match the
- * ZBC/ZAC defined zone descriptor and allow support for future additional
- * zone information.
+ * start, len, capacity and wp use the regular 512 B sector unit, regardless
+ * of the device logical block size. The overall structure size is 64 B to
+ * match the ZBC, ZAC and ZNS defined zone descriptor and allow support for
+ * future additional zone information.
*/
struct blk_zone {
__u64 start; /* Zone start sector */
*
* @sector: starting sector of report
* @nr_zones: IN maximum / OUT actual
- * @reserved: padding to 16 byte alignment
+ * @flags: one or more flags as defined by enum blk_zone_report_flags.
* @zones: Space to hold @nr_zones @zones entries on reply.
*
* The array of at most @nr_zones must follow this structure in memory.
#include <linux/btrfs.h>
#include <linux/types.h>
+#ifdef __KERNEL__
+#include <linux/stddef.h>
+#else
+#include <stddef.h>
+#endif
/*
* This header contains the structure definitions and constants used
__le64 reserved[8]; /* for future */
} __attribute__ ((__packed__));
+/*
+ * Btrfs root item used to be smaller than current size. The old format ends
+ * at where member generation_v2 is.
+ */
+static inline __u32 btrfs_legacy_root_item_size(void)
+{
+ return offsetof(struct btrfs_root_item, generation_v2);
+}
+
/*
* this is used for both forward and backward root refs
*/
#define NT_ARM_PAC_MASK 0x406 /* ARM pointer authentication code masks */
#define NT_ARM_PACA_KEYS 0x407 /* ARM pointer authentication address keys */
#define NT_ARM_PACG_KEYS 0x408 /* ARM pointer authentication generic key */
+#define NT_ARM_TAGGED_ADDR_CTRL 0x409 /* arm64 tagged address control (prctl()) */
#define NT_ARC_V2 0x600 /* ARCv2 accumulator/extra registers */
#define NT_VMCOREDD 0x700 /* Vmcore Device Dump Note */
#define NT_MIPS_DSP 0x800 /* MIPS DSP ASE registers */
__u8 flags;
__u8 master_key_descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE];
};
-#define fscrypt_policy fscrypt_policy_v1
/*
* Process-subscribed "logon" key description prefix and payload format.
__u32 __out_reserved[13];
};
-#define FS_IOC_SET_ENCRYPTION_POLICY _IOR('f', 19, struct fscrypt_policy)
+#define FS_IOC_SET_ENCRYPTION_POLICY _IOR('f', 19, struct fscrypt_policy_v1)
#define FS_IOC_GET_ENCRYPTION_PWSALT _IOW('f', 20, __u8[16])
-#define FS_IOC_GET_ENCRYPTION_POLICY _IOW('f', 21, struct fscrypt_policy)
+#define FS_IOC_GET_ENCRYPTION_POLICY _IOW('f', 21, struct fscrypt_policy_v1)
#define FS_IOC_GET_ENCRYPTION_POLICY_EX _IOWR('f', 22, __u8[9]) /* size + version */
#define FS_IOC_ADD_ENCRYPTION_KEY _IOWR('f', 23, struct fscrypt_add_key_arg)
#define FS_IOC_REMOVE_ENCRYPTION_KEY _IOWR('f', 24, struct fscrypt_remove_key_arg)
/* old names; don't add anything new here! */
#ifndef __KERNEL__
+#define fscrypt_policy fscrypt_policy_v1
#define FS_KEY_DESCRIPTOR_SIZE FSCRYPT_KEY_DESCRIPTOR_SIZE
#define FS_POLICY_FLAGS_PAD_4 FSCRYPT_POLICY_FLAGS_PAD_4
#define FS_POLICY_FLAGS_PAD_8 FSCRYPT_POLICY_FLAGS_PAD_8
#ifndef _UAPI_GPIO_H_
#define _UAPI_GPIO_H_
+#include <linux/const.h>
#include <linux/ioctl.h>
#include <linux/types.h>
+/*
+ * The maximum size of name and label arrays.
+ *
+ * Must be a multiple of 8 to ensure 32/64-bit alignment of structs.
+ */
+#define GPIO_MAX_NAME_SIZE 32
+
/**
* struct gpiochip_info - Information about a certain GPIO chip
* @name: the Linux kernel name of this GPIO chip
* @lines: number of GPIO lines on this chip
*/
struct gpiochip_info {
- char name[32];
- char label[32];
+ char name[GPIO_MAX_NAME_SIZE];
+ char label[GPIO_MAX_NAME_SIZE];
__u32 lines;
};
+/*
+ * Maximum number of requested lines.
+ *
+ * Must be no greater than 64, as bitmaps are restricted here to 64-bits
+ * for simplicity, and a multiple of 2 to ensure 32/64-bit alignment of
+ * structs.
+ */
+#define GPIO_V2_LINES_MAX 64
+
+/*
+ * The maximum number of configuration attributes associated with a line
+ * request.
+ */
+#define GPIO_V2_LINE_NUM_ATTRS_MAX 10
+
+/**
+ * enum gpio_v2_line_flag - &struct gpio_v2_line_attribute.flags values
+ * @GPIO_V2_LINE_FLAG_USED: line is not available for request
+ * @GPIO_V2_LINE_FLAG_ACTIVE_LOW: line active state is physical low
+ * @GPIO_V2_LINE_FLAG_INPUT: line is an input
+ * @GPIO_V2_LINE_FLAG_OUTPUT: line is an output
+ * @GPIO_V2_LINE_FLAG_EDGE_RISING: line detects rising (inactive to active)
+ * edges
+ * @GPIO_V2_LINE_FLAG_EDGE_FALLING: line detects falling (active to
+ * inactive) edges
+ * @GPIO_V2_LINE_FLAG_OPEN_DRAIN: line is an open drain output
+ * @GPIO_V2_LINE_FLAG_OPEN_SOURCE: line is an open source output
+ * @GPIO_V2_LINE_FLAG_BIAS_PULL_UP: line has pull-up bias enabled
+ * @GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN: line has pull-down bias enabled
+ * @GPIO_V2_LINE_FLAG_BIAS_DISABLED: line has bias disabled
+ */
+enum gpio_v2_line_flag {
+ GPIO_V2_LINE_FLAG_USED = _BITULL(0),
+ GPIO_V2_LINE_FLAG_ACTIVE_LOW = _BITULL(1),
+ GPIO_V2_LINE_FLAG_INPUT = _BITULL(2),
+ GPIO_V2_LINE_FLAG_OUTPUT = _BITULL(3),
+ GPIO_V2_LINE_FLAG_EDGE_RISING = _BITULL(4),
+ GPIO_V2_LINE_FLAG_EDGE_FALLING = _BITULL(5),
+ GPIO_V2_LINE_FLAG_OPEN_DRAIN = _BITULL(6),
+ GPIO_V2_LINE_FLAG_OPEN_SOURCE = _BITULL(7),
+ GPIO_V2_LINE_FLAG_BIAS_PULL_UP = _BITULL(8),
+ GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN = _BITULL(9),
+ GPIO_V2_LINE_FLAG_BIAS_DISABLED = _BITULL(10),
+};
+
+/**
+ * struct gpio_v2_line_values - Values of GPIO lines
+ * @bits: a bitmap containing the value of the lines, set to 1 for active
+ * and 0 for inactive.
+ * @mask: a bitmap identifying the lines to get or set, with each bit
+ * number corresponding to the index into &struct
+ * gpio_v2_line_request.offsets.
+ */
+struct gpio_v2_line_values {
+ __aligned_u64 bits;
+ __aligned_u64 mask;
+};
+
+/**
+ * enum gpio_v2_line_attr_id - &struct gpio_v2_line_attribute.id values
+ * identifying which field of the attribute union is in use.
+ * @GPIO_V2_LINE_ATTR_ID_FLAGS: flags field is in use
+ * @GPIO_V2_LINE_ATTR_ID_OUTPUT_VALUES: values field is in use
+ * @GPIO_V2_LINE_ATTR_ID_DEBOUNCE: debounce_period_us is in use
+ */
+enum gpio_v2_line_attr_id {
+ GPIO_V2_LINE_ATTR_ID_FLAGS = 1,
+ GPIO_V2_LINE_ATTR_ID_OUTPUT_VALUES = 2,
+ GPIO_V2_LINE_ATTR_ID_DEBOUNCE = 3,
+};
+
+/**
+ * struct gpio_v2_line_attribute - a configurable attribute of a line
+ * @id: attribute identifier with value from &enum gpio_v2_line_attr_id
+ * @padding: reserved for future use and must be zero filled
+ * @flags: if id is GPIO_V2_LINE_ATTR_ID_FLAGS, the flags for the GPIO
+ * line, with values from enum gpio_v2_line_flag, such as
+ * GPIO_V2_LINE_FLAG_ACTIVE_LOW, GPIO_V2_LINE_FLAG_OUTPUT etc, OR:ed
+ * together. This overrides the default flags contained in the &struct
+ * gpio_v2_line_config for the associated line.
+ * @values: if id is GPIO_V2_LINE_ATTR_ID_OUTPUT_VALUES, a bitmap
+ * containing the values to which the lines will be set, with each bit
+ * number corresponding to the index into &struct
+ * gpio_v2_line_request.offsets.
+ * @debounce_period_us: if id is GPIO_V2_LINE_ATTR_ID_DEBOUNCE, the desired
+ * debounce period, in microseconds
+ */
+struct gpio_v2_line_attribute {
+ __u32 id;
+ __u32 padding;
+ union {
+ __aligned_u64 flags;
+ __aligned_u64 values;
+ __u32 debounce_period_us;
+ };
+};
+
+/**
+ * struct gpio_v2_line_config_attribute - a configuration attribute
+ * associated with one or more of the requested lines.
+ * @attr: the configurable attribute
+ * @mask: a bitmap identifying the lines to which the attribute applies,
+ * with each bit number corresponding to the index into &struct
+ * gpio_v2_line_request.offsets.
+ */
+struct gpio_v2_line_config_attribute {
+ struct gpio_v2_line_attribute attr;
+ __aligned_u64 mask;
+};
+
+/**
+ * struct gpio_v2_line_config - Configuration for GPIO lines
+ * @flags: flags for the GPIO lines, with values from enum
+ * gpio_v2_line_flag, such as GPIO_V2_LINE_FLAG_ACTIVE_LOW,
+ * GPIO_V2_LINE_FLAG_OUTPUT etc, OR:ed together. This is the default for
+ * all requested lines but may be overridden for particular lines using
+ * attrs.
+ * @num_attrs: the number of attributes in attrs
+ * @padding: reserved for future use and must be zero filled
+ * @attrs: the configuration attributes associated with the requested
+ * lines. Any attribute should only be associated with a particular line
+ * once. If an attribute is associated with a line multiple times then the
+ * first occurrence (i.e. lowest index) has precedence.
+ */
+struct gpio_v2_line_config {
+ __aligned_u64 flags;
+ __u32 num_attrs;
+ /* Pad to fill implicit padding and reserve space for future use. */
+ __u32 padding[5];
+ struct gpio_v2_line_config_attribute attrs[GPIO_V2_LINE_NUM_ATTRS_MAX];
+};
+
+/**
+ * struct gpio_v2_line_request - Information about a request for GPIO lines
+ * @offsets: an array of desired lines, specified by offset index for the
+ * associated GPIO chip
+ * @consumer: a desired consumer label for the selected GPIO lines such as
+ * "my-bitbanged-relay"
+ * @config: requested configuration for the lines.
+ * @num_lines: number of lines requested in this request, i.e. the number
+ * of valid fields in the GPIO_V2_LINES_MAX sized arrays, set to 1 to
+ * request a single line
+ * @event_buffer_size: a suggested minimum number of line events that the
+ * kernel should buffer. This is only relevant if edge detection is
+ * enabled in the configuration. Note that this is only a suggested value
+ * and the kernel may allocate a larger buffer or cap the size of the
+ * buffer. If this field is zero then the buffer size defaults to a minimum
+ * of num_lines*16.
+ * @padding: reserved for future use and must be zero filled
+ * @fd: if successful this field will contain a valid anonymous file handle
+ * after a GPIO_GET_LINE_IOCTL operation, zero or negative value means
+ * error
+ */
+struct gpio_v2_line_request {
+ __u32 offsets[GPIO_V2_LINES_MAX];
+ char consumer[GPIO_MAX_NAME_SIZE];
+ struct gpio_v2_line_config config;
+ __u32 num_lines;
+ __u32 event_buffer_size;
+ /* Pad to fill implicit padding and reserve space for future use. */
+ __u32 padding[5];
+ __s32 fd;
+};
+
+/**
+ * struct gpio_v2_line_info - Information about a certain GPIO line
+ * @name: the name of this GPIO line, such as the output pin of the line on
+ * the chip, a rail or a pin header name on a board, as specified by the
+ * GPIO chip, may be empty
+ * @consumer: a functional name for the consumer of this GPIO line as set
+ * by whatever is using it, will be empty if there is no current user but
+ * may also be empty if the consumer doesn't set this up
+ * @flags: flags for the GPIO line, such as GPIO_V2_LINE_FLAG_ACTIVE_LOW,
+ * GPIO_V2_LINE_FLAG_OUTPUT etc, OR:ed together
+ * @offset: the local offset on this GPIO chip, fill this in when
+ * requesting the line information from the kernel
+ * @num_attrs: the number of attributes in attrs
+ * @attrs: the configuration attributes associated with the line
+ * @padding: reserved for future use
+ */
+struct gpio_v2_line_info {
+ char name[GPIO_MAX_NAME_SIZE];
+ char consumer[GPIO_MAX_NAME_SIZE];
+ __u32 offset;
+ __u32 num_attrs;
+ __aligned_u64 flags;
+ struct gpio_v2_line_attribute attrs[GPIO_V2_LINE_NUM_ATTRS_MAX];
+ /* Space reserved for future use. */
+ __u32 padding[4];
+};
+
+/**
+ * enum gpio_v2_line_changed_type - &struct gpio_v2_line_changed.event_type
+ * values
+ * @GPIO_V2_LINE_CHANGED_REQUESTED: line has been requested
+ * @GPIO_V2_LINE_CHANGED_RELEASED: line has been released
+ * @GPIO_V2_LINE_CHANGED_CONFIG: line has been reconfigured
+ */
+enum gpio_v2_line_changed_type {
+ GPIO_V2_LINE_CHANGED_REQUESTED = 1,
+ GPIO_V2_LINE_CHANGED_RELEASED = 2,
+ GPIO_V2_LINE_CHANGED_CONFIG = 3,
+};
+
+/**
+ * struct gpio_v2_line_info_changed - Information about a change in status
+ * of a GPIO line
+ * @info: updated line information
+ * @timestamp_ns: estimate of time of status change occurrence, in nanoseconds
+ * @event_type: the type of change with a value from enum
+ * gpio_v2_line_changed_type
+ * @padding: reserved for future use
+ */
+struct gpio_v2_line_info_changed {
+ struct gpio_v2_line_info info;
+ __aligned_u64 timestamp_ns;
+ __u32 event_type;
+ /* Pad struct to 64-bit boundary and reserve space for future use. */
+ __u32 padding[5];
+};
+
+/**
+ * enum gpio_v2_line_event_id - &struct gpio_v2_line_event.id values
+ * @GPIO_V2_LINE_EVENT_RISING_EDGE: event triggered by a rising edge
+ * @GPIO_V2_LINE_EVENT_FALLING_EDGE: event triggered by a falling edge
+ */
+enum gpio_v2_line_event_id {
+ GPIO_V2_LINE_EVENT_RISING_EDGE = 1,
+ GPIO_V2_LINE_EVENT_FALLING_EDGE = 2,
+};
+
+/**
+ * struct gpio_v2_line_event - The actual event being pushed to userspace
+ * @timestamp_ns: best estimate of time of event occurrence, in nanoseconds.
+ * The timestamp_ns is read from CLOCK_MONOTONIC and is intended to allow the
+ * accurate measurement of the time between events. It does not provide
+ * the wall-clock time.
+ * @id: event identifier with value from enum gpio_v2_line_event_id
+ * @offset: the offset of the line that triggered the event
+ * @seqno: the sequence number for this event in the sequence of events for
+ * all the lines in this line request
+ * @line_seqno: the sequence number for this event in the sequence of
+ * events on this particular line
+ * @padding: reserved for future use
+ */
+struct gpio_v2_line_event {
+ __aligned_u64 timestamp_ns;
+ __u32 id;
+ __u32 offset;
+ __u32 seqno;
+ __u32 line_seqno;
+ /* Space reserved for future use. */
+ __u32 padding[6];
+};
+
+/*
+ * ABI v1
+ *
+ * This version of the ABI is deprecated.
+ * Use the latest version of the ABI, defined above, instead.
+ */
+
/* Informational flags */
#define GPIOLINE_FLAG_KERNEL (1UL << 0) /* Line used by the kernel */
#define GPIOLINE_FLAG_IS_OUT (1UL << 1)
* @consumer: a functional name for the consumer of this GPIO line as set by
* whatever is using it, will be empty if there is no current user but may
* also be empty if the consumer doesn't set this up
+ *
+ * This struct is part of ABI v1 and is deprecated.
+ * Use struct gpio_v2_line_info instead.
*/
struct gpioline_info {
__u32 line_offset;
__u32 flags;
- char name[32];
- char consumer[32];
+ char name[GPIO_MAX_NAME_SIZE];
+ char consumer[GPIO_MAX_NAME_SIZE];
};
/* Maximum number of requested handles */
* guarantee there are no implicit holes between it and subsequent members.
* The 20-byte padding at the end makes sure we don't add any implicit padding
* at the end of the structure on 64-bit architectures.
+ *
+ * This struct is part of ABI v1 and is deprecated.
+ * Use struct gpio_v2_line_info_changed instead.
*/
struct gpioline_info_changed {
struct gpioline_info info;
* @fd: if successful this field will contain a valid anonymous file handle
* after a GPIO_GET_LINEHANDLE_IOCTL operation, zero or negative value
* means error
+ *
+ * This struct is part of ABI v1 and is deprecated.
+ * Use struct gpio_v2_line_request instead.
*/
struct gpiohandle_request {
__u32 lineoffsets[GPIOHANDLES_MAX];
__u32 flags;
__u8 default_values[GPIOHANDLES_MAX];
- char consumer_label[32];
+ char consumer_label[GPIO_MAX_NAME_SIZE];
__u32 lines;
int fd;
};
* this specifies the default output value, should be 0 (low) or
* 1 (high), anything else than 0 or 1 will be interpreted as 1 (high)
* @padding: reserved for future use and should be zero filled
+ *
+ * This struct is part of ABI v1 and is deprecated.
+ * Use struct gpio_v2_line_config instead.
*/
struct gpiohandle_config {
__u32 flags;
__u32 padding[4]; /* padding for future use */
};
-#define GPIOHANDLE_SET_CONFIG_IOCTL _IOWR(0xB4, 0x0a, struct gpiohandle_config)
-
/**
* struct gpiohandle_data - Information of values on a GPIO handle
* @values: when getting the state of lines this contains the current
* state of a line, when setting the state of lines these should contain
* the desired target state
+ *
+ * This struct is part of ABI v1 and is deprecated.
+ * Use struct gpio_v2_line_values instead.
*/
struct gpiohandle_data {
__u8 values[GPIOHANDLES_MAX];
};
-#define GPIOHANDLE_GET_LINE_VALUES_IOCTL _IOWR(0xB4, 0x08, struct gpiohandle_data)
-#define GPIOHANDLE_SET_LINE_VALUES_IOCTL _IOWR(0xB4, 0x09, struct gpiohandle_data)
-
/* Eventrequest flags */
#define GPIOEVENT_REQUEST_RISING_EDGE (1UL << 0)
#define GPIOEVENT_REQUEST_FALLING_EDGE (1UL << 1)
* @fd: if successful this field will contain a valid anonymous file handle
* after a GPIO_GET_LINEEVENT_IOCTL operation, zero or negative value
* means error
+ *
+ * This struct is part of ABI v1 and is deprecated.
+ * Use struct gpio_v2_line_request instead.
*/
struct gpioevent_request {
__u32 lineoffset;
__u32 handleflags;
__u32 eventflags;
- char consumer_label[32];
+ char consumer_label[GPIO_MAX_NAME_SIZE];
int fd;
};
* struct gpioevent_data - The actual event being pushed to userspace
* @timestamp: best estimate of time of event occurrence, in nanoseconds
* @id: event identifier
+ *
+ * This struct is part of ABI v1 and is deprecated.
+ * Use struct gpio_v2_line_event instead.
*/
struct gpioevent_data {
__u64 timestamp;
__u32 id;
};
+/*
+ * v1 and v2 ioctl()s
+ */
#define GPIO_GET_CHIPINFO_IOCTL _IOR(0xB4, 0x01, struct gpiochip_info)
+#define GPIO_GET_LINEINFO_UNWATCH_IOCTL _IOWR(0xB4, 0x0C, __u32)
+
+/*
+ * v2 ioctl()s
+ */
+#define GPIO_V2_GET_LINEINFO_IOCTL _IOWR(0xB4, 0x05, struct gpio_v2_line_info)
+#define GPIO_V2_GET_LINEINFO_WATCH_IOCTL _IOWR(0xB4, 0x06, struct gpio_v2_line_info)
+#define GPIO_V2_GET_LINE_IOCTL _IOWR(0xB4, 0x07, struct gpio_v2_line_request)
+#define GPIO_V2_LINE_SET_CONFIG_IOCTL _IOWR(0xB4, 0x0D, struct gpio_v2_line_config)
+#define GPIO_V2_LINE_GET_VALUES_IOCTL _IOWR(0xB4, 0x0E, struct gpio_v2_line_values)
+#define GPIO_V2_LINE_SET_VALUES_IOCTL _IOWR(0xB4, 0x0F, struct gpio_v2_line_values)
+
+/*
+ * v1 ioctl()s
+ *
+ * These ioctl()s are deprecated. Use the v2 equivalent instead.
+ */
#define GPIO_GET_LINEINFO_IOCTL _IOWR(0xB4, 0x02, struct gpioline_info)
-#define GPIO_GET_LINEINFO_WATCH_IOCTL _IOWR(0xB4, 0x0b, struct gpioline_info)
-#define GPIO_GET_LINEINFO_UNWATCH_IOCTL _IOWR(0xB4, 0x0c, __u32)
#define GPIO_GET_LINEHANDLE_IOCTL _IOWR(0xB4, 0x03, struct gpiohandle_request)
#define GPIO_GET_LINEEVENT_IOCTL _IOWR(0xB4, 0x04, struct gpioevent_request)
+#define GPIOHANDLE_GET_LINE_VALUES_IOCTL _IOWR(0xB4, 0x08, struct gpiohandle_data)
+#define GPIOHANDLE_SET_LINE_VALUES_IOCTL _IOWR(0xB4, 0x09, struct gpiohandle_data)
+#define GPIOHANDLE_SET_CONFIG_IOCTL _IOWR(0xB4, 0x0A, struct gpiohandle_config)
+#define GPIO_GET_LINEINFO_WATCH_IOCTL _IOWR(0xB4, 0x0B, struct gpioline_info)
#endif /* _UAPI_GPIO_H_ */
#define ALG_SET_OP 3
#define ALG_SET_AEAD_ASSOCLEN 4
#define ALG_SET_AEAD_AUTHSIZE 5
+#define ALG_SET_DRBG_ENTROPY 6
/* Operations */
#define ALG_OP_DECRYPT 0
#define IORING_SETUP_CQSIZE (1U << 3) /* app defines CQ size */
#define IORING_SETUP_CLAMP (1U << 4) /* clamp SQ/CQ ring sizes */
#define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */
+#define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */
enum {
IORING_OP_NOP,
*/
#define IORING_ENTER_GETEVENTS (1U << 0)
#define IORING_ENTER_SQ_WAKEUP (1U << 1)
+#define IORING_ENTER_SQ_WAIT (1U << 2)
/*
* Passed in for io_uring_setup(2). Copied back with updated info on success
/*
* io_uring_register(2) opcodes and arguments
*/
-#define IORING_REGISTER_BUFFERS 0
-#define IORING_UNREGISTER_BUFFERS 1
-#define IORING_REGISTER_FILES 2
-#define IORING_UNREGISTER_FILES 3
-#define IORING_REGISTER_EVENTFD 4
-#define IORING_UNREGISTER_EVENTFD 5
-#define IORING_REGISTER_FILES_UPDATE 6
-#define IORING_REGISTER_EVENTFD_ASYNC 7
-#define IORING_REGISTER_PROBE 8
-#define IORING_REGISTER_PERSONALITY 9
-#define IORING_UNREGISTER_PERSONALITY 10
+enum {
+ IORING_REGISTER_BUFFERS = 0,
+ IORING_UNREGISTER_BUFFERS = 1,
+ IORING_REGISTER_FILES = 2,
+ IORING_UNREGISTER_FILES = 3,
+ IORING_REGISTER_EVENTFD = 4,
+ IORING_UNREGISTER_EVENTFD = 5,
+ IORING_REGISTER_FILES_UPDATE = 6,
+ IORING_REGISTER_EVENTFD_ASYNC = 7,
+ IORING_REGISTER_PROBE = 8,
+ IORING_REGISTER_PERSONALITY = 9,
+ IORING_UNREGISTER_PERSONALITY = 10,
+ IORING_REGISTER_RESTRICTIONS = 11,
+ IORING_REGISTER_ENABLE_RINGS = 12,
+
+ /* this goes last */
+ IORING_REGISTER_LAST
+};
struct io_uring_files_update {
__u32 offset;
struct io_uring_probe_op ops[0];
};
+struct io_uring_restriction {
+ __u16 opcode;
+ union {
+ __u8 register_op; /* IORING_RESTRICTION_REGISTER_OP */
+ __u8 sqe_op; /* IORING_RESTRICTION_SQE_OP */
+ __u8 sqe_flags; /* IORING_RESTRICTION_SQE_FLAGS_* */
+ };
+ __u8 resv;
+ __u32 resv2[3];
+};
+
+/*
+ * io_uring_restriction->opcode values
+ */
+enum {
+ /* Allow an io_uring_register(2) opcode */
+ IORING_RESTRICTION_REGISTER_OP = 0,
+
+ /* Allow an sqe opcode */
+ IORING_RESTRICTION_SQE_OP = 1,
+
+ /* Allow sqe flags */
+ IORING_RESTRICTION_SQE_FLAGS_ALLOWED = 2,
+
+ /* Require sqe flags (these flags must be set on each submission) */
+ IORING_RESTRICTION_SQE_FLAGS_REQUIRED = 3,
+
+ IORING_RESTRICTION_LAST
+};
+
#endif
* If this command is not implemented by an
* architecture, -EINVAL is returned.
* Returns 0 on success.
+ * @MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ:
+ * Ensure the caller thread, upon return from
+ * system call, that all its running thread
+ * siblings have any currently running rseq
+ * critical sections restarted if @flags
+ * parameter is 0; if @flags parameter is
+ * MEMBARRIER_CMD_FLAG_CPU,
+ * then this operation is performed only
+ * on CPU indicated by @cpu_id. If this command is
+ * not implemented by an architecture, -EINVAL
+ * is returned. A process needs to register its
+ * intent to use the private expedited rseq
+ * command prior to using it, otherwise
+ * this command returns -EPERM.
+ * @MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ:
+ * Register the process intent to use
+ * MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ.
+ * If this command is not implemented by an
+ * architecture, -EINVAL is returned.
+ * Returns 0 on success.
* @MEMBARRIER_CMD_SHARED:
* Alias to MEMBARRIER_CMD_GLOBAL. Provided for
* header backward compatibility.
MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4),
MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE = (1 << 5),
MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE = (1 << 6),
+ MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ = (1 << 7),
+ MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ = (1 << 8),
/* Alias for header backward compatibility. */
MEMBARRIER_CMD_SHARED = MEMBARRIER_CMD_GLOBAL,
};
+enum membarrier_cmd_flag {
+ MEMBARRIER_CMD_FLAG_CPU = (1 << 0),
+};
+
#endif /* _UAPI_LINUX_MEMBARRIER_H */
#define PR_SET_TAGGED_ADDR_CTRL 55
#define PR_GET_TAGGED_ADDR_CTRL 56
# define PR_TAGGED_ADDR_ENABLE (1UL << 0)
+/* MTE tag check fault modes */
+# define PR_MTE_TCF_SHIFT 1
+# define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
+# define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
+# define PR_MTE_TCF_ASYNC (2UL << PR_MTE_TCF_SHIFT)
+# define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
+/* MTE tag inclusion mask */
+# define PR_MTE_TAG_SHIFT 3
+# define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT)
/* Control reclaim behavior when allocating memory */
#define PR_SET_IO_FLUSHER 57
RXRPC_BUSY = 6, /* -r: server busy received [terminal] */
RXRPC_LOCAL_ERROR = 7, /* -r: local error generated [terminal] */
RXRPC_NEW_CALL = 8, /* -r: [Service] new incoming call notification */
- RXRPC_ACCEPT = 9, /* s-: [Service] accept request */
RXRPC_EXCLUSIVE_CALL = 10, /* s-: Call should be on exclusive connection */
RXRPC_UPGRADE_SERVICE = 11, /* s-: Request service upgrade for client call */
RXRPC_TX_LENGTH = 12, /* s-: Total length of Tx data */
RXRPC_SET_CALL_TIMEOUT = 13, /* s-: Set one or more call timeouts */
+ RXRPC_CHARGE_ACCEPT = 14, /* s-: Charge the accept pool with a user call ID */
RXRPC__SUPPORTED
};
LINUX_MIB_TCPTIMEOUTREHASH, /* TCPTimeoutRehash */
LINUX_MIB_TCPDUPLICATEDATAREHASH, /* TCPDuplicateDataRehash */
LINUX_MIB_TCPDSACKRECVSEGS, /* TCPDSACKRecvSegs */
+ LINUX_MIB_TCPDSACKIGNOREDDUBIOUS, /* TCPDSACKIgnoredDubious */
__LINUX_MIB_MAX
};
enum v4l2_mpeg_video_bitrate_mode {
V4L2_MPEG_VIDEO_BITRATE_MODE_VBR = 0,
V4L2_MPEG_VIDEO_BITRATE_MODE_CBR = 1,
+ V4L2_MPEG_VIDEO_BITRATE_MODE_CQ = 2,
};
#define V4L2_CID_MPEG_VIDEO_BITRATE (V4L2_CID_MPEG_BASE+207)
#define V4L2_CID_MPEG_VIDEO_BITRATE_PEAK (V4L2_CID_MPEG_BASE+208)
V4L2_MPEG_VIDEO_VP9_PROFILE_2 = 2,
V4L2_MPEG_VIDEO_VP9_PROFILE_3 = 3,
};
+#define V4L2_CID_MPEG_VIDEO_VP9_LEVEL (V4L2_CID_MPEG_BASE+513)
+enum v4l2_mpeg_video_vp9_level {
+ V4L2_MPEG_VIDEO_VP9_LEVEL_1_0 = 0,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_1_1 = 1,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_2_0 = 2,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_2_1 = 3,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_3_0 = 4,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_3_1 = 5,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_4_0 = 6,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_4_1 = 7,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_5_0 = 8,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_5_1 = 9,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_5_2 = 10,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_6_0 = 11,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_6_1 = 12,
+ V4L2_MPEG_VIDEO_VP9_LEVEL_6_2 = 13,
+};
/* CIDs for HEVC encoding. */
#define V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L6_BR (V4L2_CID_MPEG_BASE + 642)
#define V4L2_CID_MPEG_VIDEO_REF_NUMBER_FOR_PFRAMES (V4L2_CID_MPEG_BASE + 643)
#define V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR (V4L2_CID_MPEG_BASE + 644)
+#define V4L2_CID_MPEG_VIDEO_CONSTANT_QUALITY (V4L2_CID_MPEG_BASE + 645)
+#define V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE (V4L2_CID_MPEG_BASE + 646)
+enum v4l2_mpeg_video_frame_skip_mode {
+ V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED = 0,
+ V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_LEVEL_LIMIT = 1,
+ V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT = 2,
+};
/* MPEG-class control IDs specific to the CX2341x driver as defined by V4L2 */
#define V4L2_CID_MPEG_CX2341X_BASE (V4L2_CTRL_CLASS_MPEG | 0x1000)
#include <linux/types.h>
#include <linux/videodev2.h>
+#define V4L2_MBUS_FRAMEFMT_SET_CSC 0x0001
+
/**
* struct v4l2_mbus_framefmt - frame format on the media bus
* @width: image width
* @field: used interlacing type (from enum v4l2_field)
* @colorspace: colorspace of the data (from enum v4l2_colorspace)
* @ycbcr_enc: YCbCr encoding of the data (from enum v4l2_ycbcr_encoding)
+ * @hsv_enc: HSV encoding of the data (from enum v4l2_hsv_encoding)
* @quantization: quantization of the data (from enum v4l2_quantization)
* @xfer_func: transfer function of the data (from enum v4l2_xfer_func)
+ * @flags: flags (V4L2_MBUS_FRAMEFMT_*)
+ * @reserved: reserved bytes that can be later used
*/
struct v4l2_mbus_framefmt {
__u32 width;
__u32 code;
__u32 field;
__u32 colorspace;
- __u16 ycbcr_enc;
+ union {
+ /* enum v4l2_ycbcr_encoding */
+ __u16 ycbcr_enc;
+ /* enum v4l2_hsv_encoding */
+ __u16 hsv_enc;
+ };
__u16 quantization;
__u16 xfer_func;
- __u16 reserved[11];
+ __u16 flags;
+ __u16 reserved[10];
};
#ifndef __KERNEL__
__u32 reserved[8];
};
+#define V4L2_SUBDEV_MBUS_CODE_CSC_COLORSPACE 0x00000001
+#define V4L2_SUBDEV_MBUS_CODE_CSC_XFER_FUNC 0x00000002
+#define V4L2_SUBDEV_MBUS_CODE_CSC_YCBCR_ENC 0x00000004
+#define V4L2_SUBDEV_MBUS_CODE_CSC_HSV_ENC V4L2_SUBDEV_MBUS_CODE_CSC_YCBCR_ENC
+#define V4L2_SUBDEV_MBUS_CODE_CSC_QUANTIZATION 0x00000008
+
/**
* struct v4l2_subdev_mbus_code_enum - Media bus format enumeration
* @pad: pad number, as reported by the media API
* @index: format index during enumeration
* @code: format code (MEDIA_BUS_FMT_ definitions)
* @which: format type (from enum v4l2_subdev_format_whence)
+ * @flags: flags set by the driver, (V4L2_SUBDEV_MBUS_CODE_*)
*/
struct v4l2_subdev_mbus_code_enum {
__u32 pad;
__u32 index;
__u32 code;
__u32 which;
- __u32 reserved[8];
+ __u32 flags;
+ __u32 reserved[7];
};
/**
V4L2_MEMORY_DMABUF = 4,
};
-#define V4L2_FLAG_MEMORY_NON_CONSISTENT (1 << 0)
-
/* see also http://vektor.theorem.ca/graphics/ycbcr/ */
enum v4l2_colorspace {
/*
enum v4l2_quantization {
/*
- * The default for R'G'B' quantization is always full range, except
- * for the BT2020 colorspace. For Y'CbCr the quantization is always
- * limited range, except for COLORSPACE_JPEG: this is full range.
+ * The default for R'G'B' quantization is always full range.
+ * For Y'CbCr the quantization is always limited range, except
+ * for COLORSPACE_JPEG: this is full range.
*/
V4L2_QUANTIZATION_DEFAULT = 0,
V4L2_QUANTIZATION_FULL_RANGE = 1,
/*
* Determine how QUANTIZATION_DEFAULT should map to a proper quantization.
- * This depends on whether the image is RGB or not, the colorspace and the
- * Y'CbCr encoding.
+ * This depends on whether the image is RGB or not, the colorspace.
+ * The Y'CbCr encoding is not used anymore, but is still there for backwards
+ * compatibility.
*/
#define V4L2_MAP_QUANTIZATION_DEFAULT(is_rgb_or_hsv, colsp, ycbcr_enc) \
- (((is_rgb_or_hsv) && (colsp) == V4L2_COLORSPACE_BT2020) ? \
- V4L2_QUANTIZATION_LIM_RANGE : \
- (((is_rgb_or_hsv) || (colsp) == V4L2_COLORSPACE_JPEG) ? \
- V4L2_QUANTIZATION_FULL_RANGE : V4L2_QUANTIZATION_LIM_RANGE))
+ (((is_rgb_or_hsv) || (colsp) == V4L2_COLORSPACE_JPEG) ? \
+ V4L2_QUANTIZATION_FULL_RANGE : V4L2_QUANTIZATION_LIM_RANGE)
/*
* Deprecated names for opRGB colorspace (IEC 61966-2-5)
/* Flags */
#define V4L2_PIX_FMT_FLAG_PREMUL_ALPHA 0x00000001
+#define V4L2_PIX_FMT_FLAG_SET_CSC 0x00000002
/*
* F O R M A T E N U M E R A T I O N
#define V4L2_FMT_FLAG_CONTINUOUS_BYTESTREAM 0x0004
#define V4L2_FMT_FLAG_DYN_RESOLUTION 0x0008
#define V4L2_FMT_FLAG_ENC_CAP_FRAME_INTERVAL 0x0010
+#define V4L2_FMT_FLAG_CSC_COLORSPACE 0x0020
+#define V4L2_FMT_FLAG_CSC_XFER_FUNC 0x0040
+#define V4L2_FMT_FLAG_CSC_YCBCR_ENC 0x0080
+#define V4L2_FMT_FLAG_CSC_HSV_ENC V4L2_FMT_FLAG_CSC_YCBCR_ENC
+#define V4L2_FMT_FLAG_CSC_QUANTIZATION 0x0100
/* Frame Size and frame rate enumeration */
/*
__u32 type; /* enum v4l2_buf_type */
__u32 memory; /* enum v4l2_memory */
__u32 capabilities;
- union {
- __u32 flags;
- __u32 reserved[1];
- };
+ __u32 reserved[1];
};
/* capabilities for struct v4l2_requestbuffers and v4l2_create_buffers */
* @memory: enum v4l2_memory; buffer memory type
* @format: frame format, for which buffers are requested
* @capabilities: capabilities of this buffer type.
- * @flags: additional buffer management attributes (ignored unless the
- * queue has V4L2_BUF_CAP_SUPPORTS_MMAP_CACHE_HINTS capability
- * and configured for MMAP streaming I/O).
* @reserved: future extensions
*/
struct v4l2_create_buffers {
__u32 memory;
struct v4l2_format format;
__u32 capabilities;
- __u32 flags;
- __u32 reserved[6];
+ __u32 reserved[7];
};
/*
})
#define gfn_to_virt(m) (__va(gfn_to_pfn(m) << XEN_PAGE_SHIFT))
+#define percpu_to_gfn(v) \
+ (pfn_to_gfn(per_cpu_ptr_to_phys(v) >> XEN_PAGE_SHIFT))
+
/* Only used in PV code. But ARM guests are always HVM. */
static inline xmaddr_t arbitrary_virt_to_machine(void *vaddr)
{
userspace. Since that isn't generally a problem on no-MMU systems,
it is normally safe to say Y here.
- See Documentation/mm/nommu-mmap.rst for more information.
+ See Documentation/admin-guide/mm/nommu-mmap.rst for more information.
config SYSTEM_DATA_VERIFICATION
def_bool n
.thread = INIT_THREAD,
.fs = &init_fs,
.files = &init_files,
+#ifdef CONFIG_IO_URING
+ .io_uring = NULL,
+#endif
.signal = &init_signals,
.sighand = &init_sighand,
.nsproxy = &init_nsproxy,
obj-$(CONFIG_BPF) += bpf/
obj-$(CONFIG_KCSAN) += kcsan/
obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
+obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call.o
obj-$(CONFIG_PERF_EVENTS) += events/
static int __init btf_vmlinux_init(void)
{
- if (!__start_BTF)
+ bin_attr_btf_vmlinux.size = __stop_BTF - __start_BTF;
+
+ if (!__start_BTF || bin_attr_btf_vmlinux.size == 0)
return 0;
btf_kobj = kobject_create_and_add("btf", kernel_kobj);
if (!btf_kobj)
return -ENOMEM;
- bin_attr_btf_vmlinux.size = __stop_BTF - __start_BTF;
-
return sysfs_create_bin_file(btf_kobj, &bin_attr_btf_vmlinux);
}
bool src_known = tnum_subreg_is_const(src_reg->var_off);
bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
struct tnum var32_off = tnum_subreg(dst_reg->var_off);
- s32 smin_val = src_reg->smin_value;
- u32 umin_val = src_reg->umin_value;
+ s32 smin_val = src_reg->s32_min_value;
+ u32 umin_val = src_reg->u32_min_value;
/* Assuming scalar64_min_max_or will be called so it is safe
* to skip updating register for known case.
/* ORing two positives gives a positive, so safe to
* cast result into s64.
*/
- dst_reg->s32_min_value = dst_reg->umin_value;
- dst_reg->s32_max_value = dst_reg->umax_value;
+ dst_reg->s32_min_value = dst_reg->u32_min_value;
+ dst_reg->s32_max_value = dst_reg->u32_max_value;
}
}
static ATOMIC_NOTIFIER_HEAD(cpu_pm_notifier_chain);
-static int cpu_pm_notify(enum cpu_pm_event event, int nr_to_call, int *nr_calls)
+static int cpu_pm_notify(enum cpu_pm_event event)
{
int ret;
/*
- * __atomic_notifier_call_chain has a RCU read critical section, which
+ * atomic_notifier_call_chain has a RCU read critical section, which
* could be disfunctional in cpu idle. Copy RCU_NONIDLE code to let
* RCU know this.
*/
rcu_irq_enter_irqson();
- ret = __atomic_notifier_call_chain(&cpu_pm_notifier_chain, event, NULL,
- nr_to_call, nr_calls);
+ ret = atomic_notifier_call_chain(&cpu_pm_notifier_chain, event, NULL);
+ rcu_irq_exit_irqson();
+
+ return notifier_to_errno(ret);
+}
+
+static int cpu_pm_notify_robust(enum cpu_pm_event event_up, enum cpu_pm_event event_down)
+{
+ int ret;
+
+ rcu_irq_enter_irqson();
+ ret = atomic_notifier_call_chain_robust(&cpu_pm_notifier_chain, event_up, event_down, NULL);
rcu_irq_exit_irqson();
return notifier_to_errno(ret);
*/
int cpu_pm_enter(void)
{
- int nr_calls = 0;
- int ret = 0;
-
- ret = cpu_pm_notify(CPU_PM_ENTER, -1, &nr_calls);
- if (ret)
- /*
- * Inform listeners (nr_calls - 1) about failure of CPU PM
- * PM entry who are notified earlier to prepare for it.
- */
- cpu_pm_notify(CPU_PM_ENTER_FAILED, nr_calls - 1, NULL);
-
- return ret;
+ return cpu_pm_notify_robust(CPU_PM_ENTER, CPU_PM_ENTER_FAILED);
}
EXPORT_SYMBOL_GPL(cpu_pm_enter);
*/
int cpu_pm_exit(void)
{
- return cpu_pm_notify(CPU_PM_EXIT, -1, NULL);
+ return cpu_pm_notify(CPU_PM_EXIT);
}
EXPORT_SYMBOL_GPL(cpu_pm_exit);
*/
int cpu_cluster_pm_enter(void)
{
- int nr_calls = 0;
- int ret = 0;
-
- ret = cpu_pm_notify(CPU_CLUSTER_PM_ENTER, -1, &nr_calls);
- if (ret)
- /*
- * Inform listeners (nr_calls - 1) about failure of CPU cluster
- * PM entry who are notified earlier to prepare for it.
- */
- cpu_pm_notify(CPU_CLUSTER_PM_ENTER_FAILED, nr_calls - 1, NULL);
-
- return ret;
+ return cpu_pm_notify_robust(CPU_CLUSTER_PM_ENTER, CPU_CLUSTER_PM_ENTER_FAILED);
}
EXPORT_SYMBOL_GPL(cpu_cluster_pm_enter);
*/
int cpu_cluster_pm_exit(void)
{
- return cpu_pm_notify(CPU_CLUSTER_PM_EXIT, -1, NULL);
+ return cpu_pm_notify(CPU_CLUSTER_PM_EXIT);
}
EXPORT_SYMBOL_GPL(cpu_cluster_pm_exit);
/*
* If TIF_SYSCALL_EMU is set, then the only reason to report is when
* TIF_SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall
- * instruction has been already reported in syscall_enter_from_usermode().
+ * instruction has been already reported in syscall_enter_from_user_mode().
*/
#define SYSEMU_STEP (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU)
* retry due to any failures in smp_call_function_single(), such as if the
* task_cpu() goes offline concurrently.
*
- * returns @func return value or -ESRCH when the process isn't running
+ * returns @func return value or -ESRCH or -ENXIO when the process isn't running
*/
static int
task_function_call(struct task_struct *p, remote_function_f func, void *info)
for (;;) {
ret = smp_call_function_single(task_cpu(p), remote_function,
&data, 1);
- ret = !ret ? data.ret : -EAGAIN;
+ if (!ret)
+ ret = data.ret;
if (ret != -EAGAIN)
break;
static atomic_t perf_sched_count;
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
-static DEFINE_PER_CPU(int, perf_sched_cb_usages);
static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
static atomic_t nr_mmap_events __read_mostly;
return event->attr.pinned ? &ctx->pinned_active : &ctx->flexible_active;
}
+/*
+ * Events that have PERF_EV_CAP_SIBLING require being part of a group and
+ * cannot exist on their own, schedule them out and move them into the ERROR
+ * state. Also see _perf_event_enable(), it will not be able to recover
+ * this ERROR state.
+ */
+static inline void perf_remove_sibling_event(struct perf_event *event)
+{
+ struct perf_event_context *ctx = event->ctx;
+ struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
+
+ event_sched_out(event, cpuctx, ctx);
+ perf_event_set_state(event, PERF_EVENT_STATE_ERROR);
+}
+
static void perf_group_detach(struct perf_event *event)
{
+ struct perf_event *leader = event->group_leader;
struct perf_event *sibling, *tmp;
struct perf_event_context *ctx = event->ctx;
/*
* If this is a sibling, remove it from its group.
*/
- if (event->group_leader != event) {
+ if (leader != event) {
list_del_init(&event->sibling_list);
event->group_leader->nr_siblings--;
goto out;
*/
list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {
+ if (sibling->event_caps & PERF_EV_CAP_SIBLING)
+ perf_remove_sibling_event(sibling);
+
sibling->group_leader = sibling;
list_del_init(&sibling->sibling_list);
}
out:
- perf_event__header_size(event->group_leader);
-
- for_each_sibling_event(tmp, event->group_leader)
+ for_each_sibling_event(tmp, leader)
perf_event__header_size(tmp);
+
+ perf_event__header_size(leader);
}
static bool is_orphaned_event(struct perf_event *event)
raw_spin_lock_irq(&ctx->lock);
if (event->state >= PERF_EVENT_STATE_INACTIVE ||
event->state < PERF_EVENT_STATE_ERROR) {
+out:
raw_spin_unlock_irq(&ctx->lock);
return;
}
* has gone back into error state, as distinct from the task having
* been scheduled away before the cross-call arrived.
*/
- if (event->state == PERF_EVENT_STATE_ERROR)
+ if (event->state == PERF_EVENT_STATE_ERROR) {
+ /*
+ * Detached SIBLING events cannot leave ERROR state.
+ */
+ if (event->event_caps & PERF_EV_CAP_SIBLING &&
+ event->group_leader == event)
+ goto out;
+
event->state = PERF_EVENT_STATE_OFF;
+ }
raw_spin_unlock_irq(&ctx->lock);
event_function_call(event, __perf_event_enable, NULL);
struct perf_event_context *parent, *next_parent;
struct perf_cpu_context *cpuctx;
int do_switch = 1;
+ struct pmu *pmu;
if (likely(!ctx))
return;
+ pmu = ctx->pmu;
cpuctx = __get_cpu_context(ctx);
if (!cpuctx->task_ctx)
return;
raw_spin_lock(&ctx->lock);
raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
if (context_equiv(ctx, next_ctx)) {
- struct pmu *pmu = ctx->pmu;
WRITE_ONCE(ctx->task, next);
WRITE_ONCE(next_ctx->task, task);
+ perf_pmu_disable(pmu);
+
+ if (cpuctx->sched_cb_usage && pmu->sched_task)
+ pmu->sched_task(ctx, false);
+
/*
* PMU specific parts of task perf context can require
* additional synchronization. As an example of such
else
swap(ctx->task_ctx_data, next_ctx->task_ctx_data);
+ perf_pmu_enable(pmu);
+
/*
* RCU_INIT_POINTER here is safe because we've not
* modified the ctx and the above modification of
if (do_switch) {
raw_spin_lock(&ctx->lock);
+ perf_pmu_disable(pmu);
+
+ if (cpuctx->sched_cb_usage && pmu->sched_task)
+ pmu->sched_task(ctx, false);
task_ctx_sched_out(cpuctx, ctx, EVENT_ALL);
+
+ perf_pmu_enable(pmu);
raw_spin_unlock(&ctx->lock);
}
}
-static DEFINE_PER_CPU(struct list_head, sched_cb_list);
-
void perf_sched_cb_dec(struct pmu *pmu)
{
struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
- this_cpu_dec(perf_sched_cb_usages);
-
- if (!--cpuctx->sched_cb_usage)
- list_del(&cpuctx->sched_cb_entry);
+ --cpuctx->sched_cb_usage;
}
{
struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
- if (!cpuctx->sched_cb_usage++)
- list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list));
-
- this_cpu_inc(perf_sched_cb_usages);
+ cpuctx->sched_cb_usage++;
}
/*
* PEBS requires this to provide PID/TID information. This requires we flush
* all queued PEBS records before we context switch to a new task.
*/
-static void perf_pmu_sched_task(struct task_struct *prev,
- struct task_struct *next,
- bool sched_in)
+static void __perf_pmu_sched_task(struct perf_cpu_context *cpuctx, bool sched_in)
{
- struct perf_cpu_context *cpuctx;
struct pmu *pmu;
- if (prev == next)
- return;
-
- list_for_each_entry(cpuctx, this_cpu_ptr(&sched_cb_list), sched_cb_entry) {
- pmu = cpuctx->ctx.pmu; /* software PMUs will not have sched_task */
+ pmu = cpuctx->ctx.pmu; /* software PMUs will not have sched_task */
- if (WARN_ON_ONCE(!pmu->sched_task))
- continue;
+ if (WARN_ON_ONCE(!pmu->sched_task))
+ return;
- perf_ctx_lock(cpuctx, cpuctx->task_ctx);
- perf_pmu_disable(pmu);
+ perf_ctx_lock(cpuctx, cpuctx->task_ctx);
+ perf_pmu_disable(pmu);
- pmu->sched_task(cpuctx->task_ctx, sched_in);
+ pmu->sched_task(cpuctx->task_ctx, sched_in);
- perf_pmu_enable(pmu);
- perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
- }
+ perf_pmu_enable(pmu);
+ perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
}
static void perf_event_switch(struct task_struct *task,
{
int ctxn;
- if (__this_cpu_read(perf_sched_cb_usages))
- perf_pmu_sched_task(task, next, false);
-
if (atomic_read(&nr_switch_events))
perf_event_switch(task, next, false);
struct task_struct *task)
{
struct perf_cpu_context *cpuctx;
+ struct pmu *pmu = ctx->pmu;
cpuctx = __get_cpu_context(ctx);
- if (cpuctx->task_ctx == ctx)
+ if (cpuctx->task_ctx == ctx) {
+ if (cpuctx->sched_cb_usage)
+ __perf_pmu_sched_task(cpuctx, true);
return;
+ }
perf_ctx_lock(cpuctx, ctx);
/*
if (!ctx->nr_events)
goto unlock;
- perf_pmu_disable(ctx->pmu);
+ perf_pmu_disable(pmu);
/*
* We want to keep the following priority order:
* cpu pinned (that don't need to move), task pinned,
if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
perf_event_sched_in(cpuctx, ctx, task);
- perf_pmu_enable(ctx->pmu);
+
+ if (cpuctx->sched_cb_usage && pmu->sched_task)
+ pmu->sched_task(cpuctx->task_ctx, true);
+
+ perf_pmu_enable(pmu);
unlock:
perf_ctx_unlock(cpuctx, ctx);
if (atomic_read(&nr_switch_events))
perf_event_switch(task, prev, true);
-
- if (__this_cpu_read(perf_sched_cb_usages))
- perf_pmu_sched_task(prev, task, true);
}
static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
static void perf_mmap_close(struct vm_area_struct *vma)
{
struct perf_event *event = vma->vm_file->private_data;
-
struct perf_buffer *rb = ring_buffer_get(event);
struct user_struct *mmap_user = rb->mmap_user;
int mmap_locked = rb->mmap_locked;
unsigned long size = perf_data_size(rb);
+ bool detach_rest = false;
if (event->pmu->event_unmapped)
event->pmu->event_unmapped(event, vma->vm_mm);
mutex_unlock(&event->mmap_mutex);
}
- atomic_dec(&rb->mmap_count);
+ if (atomic_dec_and_test(&rb->mmap_count))
+ detach_rest = true;
if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
goto out_put;
mutex_unlock(&event->mmap_mutex);
/* If there's still other mmap()s of this buffer, we're done. */
- if (atomic_read(&rb->mmap_count))
+ if (!detach_rest)
goto out_put;
/*
#ifdef CONFIG_CGROUP_PERF
INIT_LIST_HEAD(&per_cpu(cgrp_cpuctx_list, cpu));
#endif
- INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu));
}
}
#include <linux/stackleak.h>
#include <linux/kasan.h>
#include <linux/scs.h>
+#include <linux/io_uring.h>
#include <asm/pgalloc.h>
#include <linux/uaccess.h>
mm->map_count++;
if (!(tmp->vm_flags & VM_WIPEONFORK))
- retval = copy_page_range(mm, oldmm, mpnt);
+ retval = copy_page_range(mm, oldmm, mpnt, tmp);
if (tmp->vm_ops && tmp->vm_ops->open)
tmp->vm_ops->open(tmp);
WARN_ON(refcount_read(&tsk->usage));
WARN_ON(tsk == current);
+ io_uring_free(tsk);
cgroup_free(tsk);
task_numa_free(tsk, true);
security_task_free(tsk);
mm_pgtables_bytes_init(mm);
mm->map_count = 0;
mm->locked_vm = 0;
+ atomic_set(&mm->has_pinned, 0);
atomic64_set(&mm->pinned_vm, 0);
memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
spin_lock_init(&mm->page_table_lock);
p->vtime.state = VTIME_INACTIVE;
#endif
+#ifdef CONFIG_IO_URING
+ p->io_uring = NULL;
+#endif
+
#if defined(SPLIT_RSS_COUNTING)
memset(&p->rss_stat, 0, sizeof(p->rss_stat));
#endif
chip->irq_eoi(&desc->irq_data);
}
+/**
+ * handle_percpu_devid_fasteoi_ipi - Per CPU local IPI handler with per cpu
+ * dev ids
+ * @desc: the interrupt description structure for this irq
+ *
+ * The biggest difference with the IRQ version is that the interrupt is
+ * EOIed early, as the IPI could result in a context switch, and we need to
+ * make sure the IPI can fire again. We also assume that the arch code has
+ * registered an action. If not, we are positively doomed.
+ */
+void handle_percpu_devid_fasteoi_ipi(struct irq_desc *desc)
+{
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct irqaction *action = desc->action;
+ unsigned int irq = irq_desc_get_irq(desc);
+ irqreturn_t res;
+
+ __kstat_incr_irqs_this_cpu(desc);
+
+ if (chip->irq_eoi)
+ chip->irq_eoi(&desc->irq_data);
+
+ trace_irq_handler_entry(irq, action);
+ res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
+ trace_irq_handler_exit(irq, action, res);
+}
+
/**
* handle_percpu_devid_fasteoi_nmi - Per CPU local NMI handler with per cpu
* dev ids
*/
int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
{
- struct irq_data *pos = NULL;
+ struct irq_data *pos;
-#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
- for (; data; data = data->parent_data)
-#endif
+ for (pos = NULL; !pos && data; data = irqd_get_parent_data(data)) {
if (data->chip && data->chip->irq_compose_msi_msg)
pos = data;
+ }
+
if (!pos)
return -ENOSYS;
pos->chip->irq_compose_msi_msg(pos, msg);
-
return 0;
}
BIT_MASK_DESCR(IRQCHIP_EOI_THREADED),
BIT_MASK_DESCR(IRQCHIP_SUPPORTS_LEVEL_MSI),
BIT_MASK_DESCR(IRQCHIP_SUPPORTS_NMI),
+ BIT_MASK_DESCR(IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND),
};
static void
BIT_MASK_DESCR(IRQD_DEFAULT_TRIGGER_SET),
BIT_MASK_DESCR(IRQD_HANDLE_ENFORCE_IRQCTX),
+
+ BIT_MASK_DESCR(IRQD_IRQ_ENABLED_ON_SUSPEND),
};
static const struct irq_bit_descr irqdesc_states[] = {
BIT_MASK_DESCR(_IRQ_PER_CPU_DEVID),
BIT_MASK_DESCR(_IRQ_IS_POLLED),
BIT_MASK_DESCR(_IRQ_DISABLE_UNLAZY),
+ BIT_MASK_DESCR(_IRQ_HIDDEN),
};
static const struct irq_bit_descr irqdesc_istates[] = {
}
#endif
+static inline struct irq_data *irqd_get_parent_data(struct irq_data *irqd)
+{
+#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
+ return irqd->parent_data;
+#else
+ return NULL;
+#endif
+}
+
#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
#include <linux/debugfs.h>
return irq_data;
}
+static void __irq_domain_free_hierarchy(struct irq_data *irq_data)
+{
+ struct irq_data *tmp;
+
+ while (irq_data) {
+ tmp = irq_data;
+ irq_data = irq_data->parent_data;
+ kfree(tmp);
+ }
+}
+
static void irq_domain_free_irq_data(unsigned int virq, unsigned int nr_irqs)
{
struct irq_data *irq_data, *tmp;
irq_data->parent_data = NULL;
irq_data->domain = NULL;
- while (tmp) {
- irq_data = tmp;
- tmp = tmp->parent_data;
- kfree(irq_data);
+ __irq_domain_free_hierarchy(tmp);
+ }
+}
+
+/**
+ * irq_domain_disconnect_hierarchy - Mark the first unused level of a hierarchy
+ * @domain: IRQ domain from which the hierarchy is to be disconnected
+ * @virq: IRQ number where the hierarchy is to be trimmed
+ *
+ * Marks the @virq level belonging to @domain as disconnected.
+ * Returns -EINVAL if @virq doesn't have a valid irq_data pointing
+ * to @domain.
+ *
+ * Its only use is to be able to trim levels of hierarchy that do not
+ * have any real meaning for this interrupt, and that the driver marks
+ * as such from its .alloc() callback.
+ */
+int irq_domain_disconnect_hierarchy(struct irq_domain *domain,
+ unsigned int virq)
+{
+ struct irq_data *irqd;
+
+ irqd = irq_domain_get_irq_data(domain, virq);
+ if (!irqd)
+ return -EINVAL;
+
+ irqd->chip = ERR_PTR(-ENOTCONN);
+ return 0;
+}
+
+static int irq_domain_trim_hierarchy(unsigned int virq)
+{
+ struct irq_data *tail, *irqd, *irq_data;
+
+ irq_data = irq_get_irq_data(virq);
+ tail = NULL;
+
+ /* The first entry must have a valid irqchip */
+ if (!irq_data->chip || IS_ERR(irq_data->chip))
+ return -EINVAL;
+
+ /*
+ * Validate that the irq_data chain is sane in the presence of
+ * a hierarchy trimming marker.
+ */
+ for (irqd = irq_data->parent_data; irqd; irq_data = irqd, irqd = irqd->parent_data) {
+ /* Can't have a valid irqchip after a trim marker */
+ if (irqd->chip && tail)
+ return -EINVAL;
+
+ /* Can't have an empty irqchip before a trim marker */
+ if (!irqd->chip && !tail)
+ return -EINVAL;
+
+ if (IS_ERR(irqd->chip)) {
+ /* Only -ENOTCONN is a valid trim marker */
+ if (PTR_ERR(irqd->chip) != -ENOTCONN)
+ return -EINVAL;
+
+ tail = irq_data;
}
}
+
+ /* No trim marker, nothing to do */
+ if (!tail)
+ return 0;
+
+ pr_info("IRQ%d: trimming hierarchy from %s\n",
+ virq, tail->parent_data->domain->name);
+
+ /* Sever the inner part of the hierarchy... */
+ irqd = tail;
+ tail = tail->parent_data;
+ irqd->parent_data = NULL;
+ __irq_domain_free_hierarchy(tail);
+
+ return 0;
}
static int irq_domain_alloc_irq_data(struct irq_domain *domain,
mutex_unlock(&irq_domain_mutex);
goto out_free_irq_data;
}
+
+ for (i = 0; i < nr_irqs; i++) {
+ ret = irq_domain_trim_hierarchy(virq + i);
+ if (ret) {
+ mutex_unlock(&irq_domain_mutex);
+ goto out_free_irq_data;
+ }
+ }
+
for (i = 0; i < nr_irqs; i++)
irq_domain_insert_irq(virq + i);
mutex_unlock(&irq_domain_mutex);
.deactivate = msi_domain_deactivate,
};
-#ifdef GENERIC_MSI_DOMAIN_OPS
static irq_hw_number_t msi_domain_ops_get_hwirq(struct msi_domain_info *info,
msi_alloc_info_t *arg)
{
{
arg->desc = desc;
}
-#else
-#define msi_domain_ops_get_hwirq NULL
-#define msi_domain_ops_prepare NULL
-#define msi_domain_ops_set_desc NULL
-#endif /* !GENERIC_MSI_DOMAIN_OPS */
static int msi_domain_ops_init(struct irq_domain *domain,
struct msi_domain_info *info,
}
static struct msi_domain_ops msi_domain_ops_default = {
- .get_hwirq = msi_domain_ops_get_hwirq,
- .msi_init = msi_domain_ops_init,
- .msi_check = msi_domain_ops_check,
- .msi_prepare = msi_domain_ops_prepare,
- .set_desc = msi_domain_ops_set_desc,
+ .get_hwirq = msi_domain_ops_get_hwirq,
+ .msi_init = msi_domain_ops_init,
+ .msi_check = msi_domain_ops_check,
+ .msi_prepare = msi_domain_ops_prepare,
+ .set_desc = msi_domain_ops_set_desc,
+ .domain_alloc_irqs = __msi_domain_alloc_irqs,
+ .domain_free_irqs = __msi_domain_free_irqs,
};
static void msi_domain_update_dom_ops(struct msi_domain_info *info)
return;
}
+ if (ops->domain_alloc_irqs == NULL)
+ ops->domain_alloc_irqs = msi_domain_ops_default.domain_alloc_irqs;
+ if (ops->domain_free_irqs == NULL)
+ ops->domain_free_irqs = msi_domain_ops_default.domain_free_irqs;
+
+ if (!(info->flags & MSI_FLAG_USE_DEF_DOM_OPS))
+ return;
+
if (ops->get_hwirq == NULL)
ops->get_hwirq = msi_domain_ops_default.get_hwirq;
if (ops->msi_init == NULL)
{
struct irq_domain *domain;
- if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
- msi_domain_update_dom_ops(info);
+ msi_domain_update_dom_ops(info);
if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
msi_domain_update_chip_ops(info);
{
struct msi_desc *desc;
- if (domain->bus_token != DOMAIN_BUS_PCI_MSI)
+ switch(domain->bus_token) {
+ case DOMAIN_BUS_PCI_MSI:
+ case DOMAIN_BUS_VMD_MSI:
+ break;
+ default:
return false;
+ }
if (!(info->flags & MSI_FLAG_MUST_REACTIVATE))
return false;
return desc->msi_attrib.is_msix || desc->msi_attrib.maskbit;
}
-/**
- * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain
- * @domain: The domain to allocate from
- * @dev: Pointer to device struct of the device for which the interrupts
- * are allocated
- * @nvec: The number of interrupts to allocate
- *
- * Returns 0 on success or an error code.
- */
-int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
- int nvec)
+int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
+ int nvec)
{
struct msi_domain_info *info = domain->host_data;
struct msi_domain_ops *ops = info->ops;
}
/**
- * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated tp @dev
- * @domain: The domain to managing the interrupts
+ * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain
+ * @domain: The domain to allocate from
* @dev: Pointer to device struct of the device for which the interrupts
- * are free
+ * are allocated
+ * @nvec: The number of interrupts to allocate
+ *
+ * Returns 0 on success or an error code.
*/
-void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)
+int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
+ int nvec)
+{
+ struct msi_domain_info *info = domain->host_data;
+ struct msi_domain_ops *ops = info->ops;
+
+ return ops->domain_alloc_irqs(domain, dev, nvec);
+}
+
+void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)
{
struct msi_desc *desc;
}
}
+/**
+ * __msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated tp @dev
+ * @domain: The domain to managing the interrupts
+ * @dev: Pointer to device struct of the device for which the interrupts
+ * are free
+ */
+void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)
+{
+ struct msi_domain_info *info = domain->host_data;
+ struct msi_domain_ops *ops = info->ops;
+
+ return ops->domain_free_irqs(domain, dev);
+}
+
/**
* msi_get_domain_info - Get the MSI interrupt domain info for @domain
* @domain: The interrupt domain to retrieve data from
static bool suspend_device_irq(struct irq_desc *desc)
{
+ unsigned long chipflags = irq_desc_get_chip(desc)->flags;
+ struct irq_data *irqd = &desc->irq_data;
+
if (!desc->action || irq_desc_is_chained(desc) ||
desc->no_suspend_depth)
return false;
- if (irqd_is_wakeup_set(&desc->irq_data)) {
- irqd_set(&desc->irq_data, IRQD_WAKEUP_ARMED);
+ if (irqd_is_wakeup_set(irqd)) {
+ irqd_set(irqd, IRQD_WAKEUP_ARMED);
+
+ if ((chipflags & IRQCHIP_ENABLE_WAKEUP_ON_SUSPEND) &&
+ irqd_irq_disabled(irqd)) {
+ /*
+ * Interrupt marked for wakeup is in disabled state.
+ * Enable interrupt here to unmask/enable in irqchip
+ * to be able to resume with such interrupts.
+ */
+ __enable_irq(desc);
+ irqd_set(irqd, IRQD_IRQ_ENABLED_ON_SUSPEND);
+ }
/*
* We return true here to force the caller to issue
* synchronize_irq(). We need to make sure that the
* chip level. The chip implementation indicates that with
* IRQCHIP_MASK_ON_SUSPEND.
*/
- if (irq_desc_get_chip(desc)->flags & IRQCHIP_MASK_ON_SUSPEND)
+ if (chipflags & IRQCHIP_MASK_ON_SUSPEND)
mask_irq(desc);
return true;
}
static void resume_irq(struct irq_desc *desc)
{
- irqd_clear(&desc->irq_data, IRQD_WAKEUP_ARMED);
+ struct irq_data *irqd = &desc->irq_data;
+
+ irqd_clear(irqd, IRQD_WAKEUP_ARMED);
+
+ if (irqd_is_enabled_on_suspend(irqd)) {
+ /*
+ * Interrupt marked for wakeup was enabled during suspend
+ * entry. Disable such interrupts to restore them back to
+ * original state.
+ */
+ __disable_irq(desc);
+ irqd_clear(irqd, IRQD_IRQ_ENABLED_ON_SUSPEND);
+ }
if (desc->istate & IRQS_SUSPENDED)
goto resume;
rcu_read_lock();
desc = irq_to_desc(i);
- if (!desc)
+ if (!desc || irq_settings_is_hidden(desc))
goto outsparse;
if (desc->kstat_irqs)
}
#endif
+static int try_retrigger(struct irq_desc *desc)
+{
+ if (desc->irq_data.chip->irq_retrigger)
+ return desc->irq_data.chip->irq_retrigger(&desc->irq_data);
+
+#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
+ return irq_chip_retrigger_hierarchy(&desc->irq_data);
+#else
+ return 0;
+#endif
+}
+
/*
* IRQ resend
*
desc->istate &= ~IRQS_PENDING;
- if (!desc->irq_data.chip->irq_retrigger ||
- !desc->irq_data.chip->irq_retrigger(&desc->irq_data))
+ if (!try_retrigger(desc))
err = irq_sw_resend(desc);
/* If the retrigger was successfull, mark it with the REPLAY bit */
_IRQ_PER_CPU_DEVID = IRQ_PER_CPU_DEVID,
_IRQ_IS_POLLED = IRQ_IS_POLLED,
_IRQ_DISABLE_UNLAZY = IRQ_DISABLE_UNLAZY,
+ _IRQ_HIDDEN = IRQ_HIDDEN,
_IRQF_MODIFY_MASK = IRQF_MODIFY_MASK,
};
#define IRQ_PER_CPU_DEVID GOT_YOU_MORON
#define IRQ_IS_POLLED GOT_YOU_MORON
#define IRQ_DISABLE_UNLAZY GOT_YOU_MORON
+#define IRQ_HIDDEN GOT_YOU_MORON
#undef IRQF_MODIFY_MASK
#define IRQF_MODIFY_MASK GOT_YOU_MORON
{
desc->status_use_accessors &= ~_IRQ_DISABLE_UNLAZY;
}
+
+static inline bool irq_settings_is_hidden(struct irq_desc *desc)
+{
+ return desc->status_use_accessors & _IRQ_HIDDEN;
+}
static int __jump_label_mod_text_reserved(void *start, void *end)
{
struct module *mod;
+ int ret;
preempt_disable();
mod = __module_text_address((unsigned long)start);
WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
+ if (!try_module_get(mod))
+ mod = NULL;
preempt_enable();
if (!mod)
return 0;
-
- return __jump_label_text_reserved(mod->jump_entries,
+ ret = __jump_label_text_reserved(mod->jump_entries,
mod->jump_entries + mod->num_jump_entries,
start, end);
+
+ module_put(mod);
+
+ return ret;
}
static void __jump_label_mod_update(struct static_key *key)
// SPDX-License-Identifier: GPL-2.0
+#define pr_fmt(fmt) "kcsan: " fmt
+
#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/delay.h>
*/
static DEFINE_PER_CPU(long, kcsan_skip);
+/* For kcsan_prandom_u32_max(). */
+static DEFINE_PER_CPU(struct rnd_state, kcsan_rand_state);
+
static __always_inline atomic_long_t *find_watchpoint(unsigned long addr,
size_t size,
bool expect_write,
if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) &&
(type & KCSAN_ACCESS_WRITE) && size <= sizeof(long) &&
- IS_ALIGNED((unsigned long)ptr, size))
+ !(type & KCSAN_ACCESS_COMPOUND) && IS_ALIGNED((unsigned long)ptr, size))
return true; /* Assume aligned writes up to word size are atomic. */
if (ctx->atomic_next > 0) {
return true;
}
+/*
+ * Returns a pseudo-random number in interval [0, ep_ro). See prandom_u32_max()
+ * for more details.
+ *
+ * The open-coded version here is using only safe primitives for all contexts
+ * where we can have KCSAN instrumentation. In particular, we cannot use
+ * prandom_u32() directly, as its tracepoint could cause recursion.
+ */
+static u32 kcsan_prandom_u32_max(u32 ep_ro)
+{
+ struct rnd_state *state = &get_cpu_var(kcsan_rand_state);
+ const u32 res = prandom_u32_state(state);
+
+ put_cpu_var(kcsan_rand_state);
+ return (u32)(((u64) res * ep_ro) >> 32);
+}
+
static inline void reset_kcsan_skip(void)
{
long skip_count = kcsan_skip_watch -
(IS_ENABLED(CONFIG_KCSAN_SKIP_WATCH_RANDOMIZE) ?
- prandom_u32_max(kcsan_skip_watch) :
+ kcsan_prandom_u32_max(kcsan_skip_watch) :
0);
this_cpu_write(kcsan_skip, skip_count);
}
return READ_ONCE(kcsan_enabled) && get_ctx()->disable_count == 0;
}
-static inline unsigned int get_delay(void)
+/* Introduce delay depending on context and configuration. */
+static void delay_access(int type)
{
unsigned int delay = in_task() ? kcsan_udelay_task : kcsan_udelay_interrupt;
- return delay - (IS_ENABLED(CONFIG_KCSAN_DELAY_RANDOMIZE) ?
- prandom_u32_max(delay) :
- 0);
+ /* For certain access types, skew the random delay to be longer. */
+ unsigned int skew_delay_order =
+ (type & (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_ASSERT)) ? 1 : 0;
+
+ delay -= IS_ENABLED(CONFIG_KCSAN_DELAY_RANDOMIZE) ?
+ kcsan_prandom_u32_max(delay >> skew_delay_order) :
+ 0;
+ udelay(delay);
}
void kcsan_save_irqtrace(struct task_struct *task)
* already removed the watchpoint, or another thread consumed
* the watchpoint before this thread.
*/
- kcsan_counter_inc(KCSAN_COUNTER_REPORT_RACES);
+ atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_REPORT_RACES]);
}
if ((type & KCSAN_ACCESS_ASSERT) != 0)
- kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES);
+ atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_ASSERT_FAILURES]);
else
- kcsan_counter_inc(KCSAN_COUNTER_DATA_RACES);
+ atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_DATA_RACES]);
user_access_restore(flags);
}
goto out;
if (!check_encodable((unsigned long)ptr, size)) {
- kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES);
+ atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_UNENCODABLE_ACCESSES]);
goto out;
}
* with which should_watch() returns true should be tweaked so
* that this case happens very rarely.
*/
- kcsan_counter_inc(KCSAN_COUNTER_NO_CAPACITY);
+ atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_NO_CAPACITY]);
goto out_unlock;
}
- kcsan_counter_inc(KCSAN_COUNTER_SETUP_WATCHPOINTS);
- kcsan_counter_inc(KCSAN_COUNTER_USED_WATCHPOINTS);
+ atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_SETUP_WATCHPOINTS]);
+ atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]);
/*
* Read the current value, to later check and infer a race if the data
if (IS_ENABLED(CONFIG_KCSAN_DEBUG)) {
kcsan_disable_current();
- pr_err("KCSAN: watching %s, size: %zu, addr: %px [slot: %d, encoded: %lx]\n",
+ pr_err("watching %s, size: %zu, addr: %px [slot: %d, encoded: %lx]\n",
is_write ? "write" : "read", size, ptr,
watchpoint_slot((unsigned long)ptr),
encode_watchpoint((unsigned long)ptr, size, is_write));
* Delay this thread, to increase probability of observing a racy
* conflicting access.
*/
- udelay(get_delay());
+ delay_access(type);
/*
* Re-read value, and check if it is as expected; if not, we infer a
* increment this counter.
*/
if (is_assert && value_change == KCSAN_VALUE_CHANGE_TRUE)
- kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES);
+ atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_ASSERT_FAILURES]);
kcsan_report(ptr, size, type, value_change, KCSAN_REPORT_RACE_SIGNAL,
watchpoint - watchpoints);
} else if (value_change == KCSAN_VALUE_CHANGE_TRUE) {
/* Inferring a race, since the value should not have changed. */
- kcsan_counter_inc(KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN);
+ atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN]);
if (is_assert)
- kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES);
+ atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_ASSERT_FAILURES]);
if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN) || is_assert)
kcsan_report(ptr, size, type, KCSAN_VALUE_CHANGE_TRUE,
* reused after this point.
*/
remove_watchpoint(watchpoint);
- kcsan_counter_dec(KCSAN_COUNTER_USED_WATCHPOINTS);
+ atomic_long_dec(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]);
out_unlock:
if (!kcsan_interrupt_watcher)
local_irq_restore(irq_flags);
BUG_ON(!in_task());
kcsan_debugfs_init();
+ prandom_seed_full_state(&kcsan_rand_state);
/*
* We are in the init task, and no other tasks should be running;
* WRITE_ONCE without memory barrier is sufficient.
*/
- if (kcsan_early_enable)
+ if (kcsan_early_enable) {
+ pr_info("enabled early\n");
WRITE_ONCE(kcsan_enabled, true);
+ }
}
/* === Exported interface =================================================== */
EXPORT_SYMBOL(__tsan_write##size); \
void __tsan_unaligned_write##size(void *ptr) \
__alias(__tsan_write##size); \
- EXPORT_SYMBOL(__tsan_unaligned_write##size)
+ EXPORT_SYMBOL(__tsan_unaligned_write##size); \
+ void __tsan_read_write##size(void *ptr); \
+ void __tsan_read_write##size(void *ptr) \
+ { \
+ check_access(ptr, size, \
+ KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE); \
+ } \
+ EXPORT_SYMBOL(__tsan_read_write##size); \
+ void __tsan_unaligned_read_write##size(void *ptr) \
+ __alias(__tsan_read_write##size); \
+ EXPORT_SYMBOL(__tsan_unaligned_read_write##size)
DEFINE_TSAN_READ_WRITE(1);
DEFINE_TSAN_READ_WRITE(2);
{
}
EXPORT_SYMBOL(__tsan_init);
+
+/*
+ * Instrumentation for atomic builtins (__atomic_*, __sync_*).
+ *
+ * Normal kernel code _should not_ be using them directly, but some
+ * architectures may implement some or all atomics using the compilers'
+ * builtins.
+ *
+ * Note: If an architecture decides to fully implement atomics using the
+ * builtins, because they are implicitly instrumented by KCSAN (and KASAN,
+ * etc.), implementing the ARCH_ATOMIC interface (to get instrumentation via
+ * atomic-instrumented) is no longer necessary.
+ *
+ * TSAN instrumentation replaces atomic accesses with calls to any of the below
+ * functions, whose job is to also execute the operation itself.
+ */
+
+#define DEFINE_TSAN_ATOMIC_LOAD_STORE(bits) \
+ u##bits __tsan_atomic##bits##_load(const u##bits *ptr, int memorder); \
+ u##bits __tsan_atomic##bits##_load(const u##bits *ptr, int memorder) \
+ { \
+ if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
+ check_access(ptr, bits / BITS_PER_BYTE, KCSAN_ACCESS_ATOMIC); \
+ } \
+ return __atomic_load_n(ptr, memorder); \
+ } \
+ EXPORT_SYMBOL(__tsan_atomic##bits##_load); \
+ void __tsan_atomic##bits##_store(u##bits *ptr, u##bits v, int memorder); \
+ void __tsan_atomic##bits##_store(u##bits *ptr, u##bits v, int memorder) \
+ { \
+ if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
+ check_access(ptr, bits / BITS_PER_BYTE, \
+ KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC); \
+ } \
+ __atomic_store_n(ptr, v, memorder); \
+ } \
+ EXPORT_SYMBOL(__tsan_atomic##bits##_store)
+
+#define DEFINE_TSAN_ATOMIC_RMW(op, bits, suffix) \
+ u##bits __tsan_atomic##bits##_##op(u##bits *ptr, u##bits v, int memorder); \
+ u##bits __tsan_atomic##bits##_##op(u##bits *ptr, u##bits v, int memorder) \
+ { \
+ if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
+ check_access(ptr, bits / BITS_PER_BYTE, \
+ KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \
+ KCSAN_ACCESS_ATOMIC); \
+ } \
+ return __atomic_##op##suffix(ptr, v, memorder); \
+ } \
+ EXPORT_SYMBOL(__tsan_atomic##bits##_##op)
+
+/*
+ * Note: CAS operations are always classified as write, even in case they
+ * fail. We cannot perform check_access() after a write, as it might lead to
+ * false positives, in cases such as:
+ *
+ * T0: __atomic_compare_exchange_n(&p->flag, &old, 1, ...)
+ *
+ * T1: if (__atomic_load_n(&p->flag, ...)) {
+ * modify *p;
+ * p->flag = 0;
+ * }
+ *
+ * The only downside is that, if there are 3 threads, with one CAS that
+ * succeeds, another CAS that fails, and an unmarked racing operation, we may
+ * point at the wrong CAS as the source of the race. However, if we assume that
+ * all CAS can succeed in some other execution, the data race is still valid.
+ */
+#define DEFINE_TSAN_ATOMIC_CMPXCHG(bits, strength, weak) \
+ int __tsan_atomic##bits##_compare_exchange_##strength(u##bits *ptr, u##bits *exp, \
+ u##bits val, int mo, int fail_mo); \
+ int __tsan_atomic##bits##_compare_exchange_##strength(u##bits *ptr, u##bits *exp, \
+ u##bits val, int mo, int fail_mo) \
+ { \
+ if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
+ check_access(ptr, bits / BITS_PER_BYTE, \
+ KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \
+ KCSAN_ACCESS_ATOMIC); \
+ } \
+ return __atomic_compare_exchange_n(ptr, exp, val, weak, mo, fail_mo); \
+ } \
+ EXPORT_SYMBOL(__tsan_atomic##bits##_compare_exchange_##strength)
+
+#define DEFINE_TSAN_ATOMIC_CMPXCHG_VAL(bits) \
+ u##bits __tsan_atomic##bits##_compare_exchange_val(u##bits *ptr, u##bits exp, u##bits val, \
+ int mo, int fail_mo); \
+ u##bits __tsan_atomic##bits##_compare_exchange_val(u##bits *ptr, u##bits exp, u##bits val, \
+ int mo, int fail_mo) \
+ { \
+ if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
+ check_access(ptr, bits / BITS_PER_BYTE, \
+ KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \
+ KCSAN_ACCESS_ATOMIC); \
+ } \
+ __atomic_compare_exchange_n(ptr, &exp, val, 0, mo, fail_mo); \
+ return exp; \
+ } \
+ EXPORT_SYMBOL(__tsan_atomic##bits##_compare_exchange_val)
+
+#define DEFINE_TSAN_ATOMIC_OPS(bits) \
+ DEFINE_TSAN_ATOMIC_LOAD_STORE(bits); \
+ DEFINE_TSAN_ATOMIC_RMW(exchange, bits, _n); \
+ DEFINE_TSAN_ATOMIC_RMW(fetch_add, bits, ); \
+ DEFINE_TSAN_ATOMIC_RMW(fetch_sub, bits, ); \
+ DEFINE_TSAN_ATOMIC_RMW(fetch_and, bits, ); \
+ DEFINE_TSAN_ATOMIC_RMW(fetch_or, bits, ); \
+ DEFINE_TSAN_ATOMIC_RMW(fetch_xor, bits, ); \
+ DEFINE_TSAN_ATOMIC_RMW(fetch_nand, bits, ); \
+ DEFINE_TSAN_ATOMIC_CMPXCHG(bits, strong, 0); \
+ DEFINE_TSAN_ATOMIC_CMPXCHG(bits, weak, 1); \
+ DEFINE_TSAN_ATOMIC_CMPXCHG_VAL(bits)
+
+DEFINE_TSAN_ATOMIC_OPS(8);
+DEFINE_TSAN_ATOMIC_OPS(16);
+DEFINE_TSAN_ATOMIC_OPS(32);
+DEFINE_TSAN_ATOMIC_OPS(64);
+
+void __tsan_atomic_thread_fence(int memorder);
+void __tsan_atomic_thread_fence(int memorder)
+{
+ __atomic_thread_fence(memorder);
+}
+EXPORT_SYMBOL(__tsan_atomic_thread_fence);
+
+void __tsan_atomic_signal_fence(int memorder);
+void __tsan_atomic_signal_fence(int memorder) { }
+EXPORT_SYMBOL(__tsan_atomic_signal_fence);
// SPDX-License-Identifier: GPL-2.0
+#define pr_fmt(fmt) "kcsan: " fmt
+
#include <linux/atomic.h>
#include <linux/bsearch.h>
#include <linux/bug.h>
#include "kcsan.h"
-/*
- * Statistics counters.
- */
-static atomic_long_t counters[KCSAN_COUNTER_COUNT];
+atomic_long_t kcsan_counters[KCSAN_COUNTER_COUNT];
+static const char *const counter_names[] = {
+ [KCSAN_COUNTER_USED_WATCHPOINTS] = "used_watchpoints",
+ [KCSAN_COUNTER_SETUP_WATCHPOINTS] = "setup_watchpoints",
+ [KCSAN_COUNTER_DATA_RACES] = "data_races",
+ [KCSAN_COUNTER_ASSERT_FAILURES] = "assert_failures",
+ [KCSAN_COUNTER_NO_CAPACITY] = "no_capacity",
+ [KCSAN_COUNTER_REPORT_RACES] = "report_races",
+ [KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN] = "races_unknown_origin",
+ [KCSAN_COUNTER_UNENCODABLE_ACCESSES] = "unencodable_accesses",
+ [KCSAN_COUNTER_ENCODING_FALSE_POSITIVES] = "encoding_false_positives",
+};
+static_assert(ARRAY_SIZE(counter_names) == KCSAN_COUNTER_COUNT);
/*
* Addresses for filtering functions from reporting. This list can be used as a
};
static DEFINE_SPINLOCK(report_filterlist_lock);
-static const char *counter_to_name(enum kcsan_counter_id id)
-{
- switch (id) {
- case KCSAN_COUNTER_USED_WATCHPOINTS: return "used_watchpoints";
- case KCSAN_COUNTER_SETUP_WATCHPOINTS: return "setup_watchpoints";
- case KCSAN_COUNTER_DATA_RACES: return "data_races";
- case KCSAN_COUNTER_ASSERT_FAILURES: return "assert_failures";
- case KCSAN_COUNTER_NO_CAPACITY: return "no_capacity";
- case KCSAN_COUNTER_REPORT_RACES: return "report_races";
- case KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN: return "races_unknown_origin";
- case KCSAN_COUNTER_UNENCODABLE_ACCESSES: return "unencodable_accesses";
- case KCSAN_COUNTER_ENCODING_FALSE_POSITIVES: return "encoding_false_positives";
- case KCSAN_COUNTER_COUNT:
- BUG();
- }
- return NULL;
-}
-
-void kcsan_counter_inc(enum kcsan_counter_id id)
-{
- atomic_long_inc(&counters[id]);
-}
-
-void kcsan_counter_dec(enum kcsan_counter_id id)
-{
- atomic_long_dec(&counters[id]);
-}
-
/*
* The microbenchmark allows benchmarking KCSAN core runtime only. To run
* multiple threads, pipe 'microbench=<iters>' from multiple tasks into the
*/
WRITE_ONCE(kcsan_enabled, false);
- pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters);
+ pr_info("%s begin | iters: %lu\n", __func__, iters);
cycles = get_cycles();
while (iters--) {
}
cycles = get_cycles() - cycles;
- pr_info("KCSAN: %s end | cycles: %llu\n", __func__, cycles);
+ pr_info("%s end | cycles: %llu\n", __func__, cycles);
WRITE_ONCE(kcsan_enabled, was_enabled);
/* restore context */
current->kcsan_ctx = ctx_save;
}
-/*
- * Simple test to create conflicting accesses. Write 'test=<iters>' to KCSAN's
- * debugfs file from multiple tasks to generate real conflicts and show reports.
- */
-static long test_dummy;
-static long test_flags;
-static long test_scoped;
-static noinline void test_thread(unsigned long iters)
-{
- const long CHANGE_BITS = 0xff00ff00ff00ff00L;
- const struct kcsan_ctx ctx_save = current->kcsan_ctx;
- cycles_t cycles;
-
- /* We may have been called from an atomic region; reset context. */
- memset(¤t->kcsan_ctx, 0, sizeof(current->kcsan_ctx));
-
- pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters);
- pr_info("test_dummy@%px, test_flags@%px, test_scoped@%px,\n",
- &test_dummy, &test_flags, &test_scoped);
-
- cycles = get_cycles();
- while (iters--) {
- /* These all should generate reports. */
- __kcsan_check_read(&test_dummy, sizeof(test_dummy));
- ASSERT_EXCLUSIVE_WRITER(test_dummy);
- ASSERT_EXCLUSIVE_ACCESS(test_dummy);
-
- ASSERT_EXCLUSIVE_BITS(test_flags, ~CHANGE_BITS); /* no report */
- __kcsan_check_read(&test_flags, sizeof(test_flags)); /* no report */
-
- ASSERT_EXCLUSIVE_BITS(test_flags, CHANGE_BITS); /* report */
- __kcsan_check_read(&test_flags, sizeof(test_flags)); /* no report */
-
- /* not actually instrumented */
- WRITE_ONCE(test_dummy, iters); /* to observe value-change */
- __kcsan_check_write(&test_dummy, sizeof(test_dummy));
-
- test_flags ^= CHANGE_BITS; /* generate value-change */
- __kcsan_check_write(&test_flags, sizeof(test_flags));
-
- BUG_ON(current->kcsan_ctx.scoped_accesses.prev);
- {
- /* Should generate reports anywhere in this block. */
- ASSERT_EXCLUSIVE_WRITER_SCOPED(test_scoped);
- ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_scoped);
- BUG_ON(!current->kcsan_ctx.scoped_accesses.prev);
- /* Unrelated accesses. */
- __kcsan_check_access(&cycles, sizeof(cycles), 0);
- __kcsan_check_access(&cycles, sizeof(cycles), KCSAN_ACCESS_ATOMIC);
- }
- BUG_ON(current->kcsan_ctx.scoped_accesses.prev);
- }
- cycles = get_cycles() - cycles;
-
- pr_info("KCSAN: %s end | cycles: %llu\n", __func__, cycles);
-
- /* restore context */
- current->kcsan_ctx = ctx_save;
-}
-
static int cmp_filterlist_addrs(const void *rhs, const void *lhs)
{
const unsigned long a = *(const unsigned long *)rhs;
ssize_t ret = 0;
if (!addr) {
- pr_err("KCSAN: could not find function: '%s'\n", func);
+ pr_err("could not find function: '%s'\n", func);
return -ENOENT;
}
/* show stats */
seq_printf(file, "enabled: %i\n", READ_ONCE(kcsan_enabled));
- for (i = 0; i < KCSAN_COUNTER_COUNT; ++i)
- seq_printf(file, "%s: %ld\n", counter_to_name(i),
- atomic_long_read(&counters[i]));
+ for (i = 0; i < KCSAN_COUNTER_COUNT; ++i) {
+ seq_printf(file, "%s: %ld\n", counter_names[i],
+ atomic_long_read(&kcsan_counters[i]));
+ }
/* show filter functions, and filter type */
spin_lock_irqsave(&report_filterlist_lock, flags);
WRITE_ONCE(kcsan_enabled, true);
} else if (!strcmp(arg, "off")) {
WRITE_ONCE(kcsan_enabled, false);
- } else if (!strncmp(arg, "microbench=", sizeof("microbench=") - 1)) {
+ } else if (str_has_prefix(arg, "microbench=")) {
unsigned long iters;
- if (kstrtoul(&arg[sizeof("microbench=") - 1], 0, &iters))
+ if (kstrtoul(&arg[strlen("microbench=")], 0, &iters))
return -EINVAL;
microbenchmark(iters);
- } else if (!strncmp(arg, "test=", sizeof("test=") - 1)) {
- unsigned long iters;
-
- if (kstrtoul(&arg[sizeof("test=") - 1], 0, &iters))
- return -EINVAL;
- test_thread(iters);
} else if (!strcmp(arg, "whitelist")) {
set_report_filterlist_whitelist(true);
} else if (!strcmp(arg, "blacklist")) {
#include <linux/types.h>
#include <trace/events/printk.h>
+#ifdef CONFIG_CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE
+#define __KCSAN_ACCESS_RW(alt) (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
+#else
+#define __KCSAN_ACCESS_RW(alt) (alt)
+#endif
+
/* Points to current test-case memory access "kernels". */
static void (*access_kernels[2])(void);
/* Access 1 & 2 */
for (i = 0; i < 2; ++i) {
+ const int ty = r->access[i].type;
const char *const access_type =
- (r->access[i].type & KCSAN_ACCESS_ASSERT) ?
- ((r->access[i].type & KCSAN_ACCESS_WRITE) ?
- "assert no accesses" :
- "assert no writes") :
- ((r->access[i].type & KCSAN_ACCESS_WRITE) ?
- "write" :
- "read");
+ (ty & KCSAN_ACCESS_ASSERT) ?
+ ((ty & KCSAN_ACCESS_WRITE) ?
+ "assert no accesses" :
+ "assert no writes") :
+ ((ty & KCSAN_ACCESS_WRITE) ?
+ ((ty & KCSAN_ACCESS_COMPOUND) ?
+ "read-write" :
+ "write") :
+ "read");
const char *const access_type_aux =
- (r->access[i].type & KCSAN_ACCESS_ATOMIC) ?
- " (marked)" :
- ((r->access[i].type & KCSAN_ACCESS_SCOPED) ?
- " (scoped)" :
- "");
+ (ty & KCSAN_ACCESS_ATOMIC) ?
+ " (marked)" :
+ ((ty & KCSAN_ACCESS_SCOPED) ? " (scoped)" : "");
if (i == 1) {
/* Access 2 */
WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1);
}
+static noinline void test_kernel_atomic_rmw(void)
+{
+ /* Use builtin, so we can set up the "bad" atomic/non-atomic scenario. */
+ __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED);
+}
+
__no_kcsan
static noinline void test_kernel_write_uninstrumented(void) { test_var++; }
write_sequnlock_irqrestore(&test_seqlock, flags);
}
+static noinline void test_kernel_atomic_builtins(void)
+{
+ /*
+ * Generate concurrent accesses, expecting no reports, ensuring KCSAN
+ * treats builtin atomics as actually atomic.
+ */
+ __atomic_load_n(&test_var, __ATOMIC_RELAXED);
+}
+
/* ===== Test cases ===== */
/* Simple test with normal data race. */
const struct expect_report expect = {
.access = {
/* NULL will match any address. */
- { test_kernel_rmw_array, NULL, 0, KCSAN_ACCESS_WRITE },
- { test_kernel_rmw_array, NULL, 0, 0 },
+ { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
+ { test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(0) },
},
};
static const struct expect_report never = {
KUNIT_EXPECT_TRUE(test, match_expect);
}
+/* Test that atomic RMWs generate correct report. */
+__no_kcsan
+static void test_read_plain_atomic_rmw(struct kunit *test)
+{
+ const struct expect_report expect = {
+ .access = {
+ { test_kernel_read, &test_var, sizeof(test_var), 0 },
+ { test_kernel_atomic_rmw, &test_var, sizeof(test_var),
+ KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
+ },
+ };
+ bool match_expect = false;
+
+ if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS))
+ return;
+
+ begin_test_checks(test_kernel_read, test_kernel_atomic_rmw);
+ do {
+ match_expect = report_matches(&expect);
+ } while (!end_test_checks(match_expect));
+ KUNIT_EXPECT_TRUE(test, match_expect);
+}
+
/* Zero-sized accesses should never cause data race reports. */
__no_kcsan
static void test_zero_size_access(struct kunit *test)
KUNIT_EXPECT_FALSE(test, match_never);
}
+/*
+ * Test atomic builtins work and required instrumentation functions exist. We
+ * also test that KCSAN understands they're atomic by racing with them via
+ * test_kernel_atomic_builtins(), and expect no reports.
+ *
+ * The atomic builtins _SHOULD NOT_ be used in normal kernel code!
+ */
+static void test_atomic_builtins(struct kunit *test)
+{
+ bool match_never = false;
+
+ begin_test_checks(test_kernel_atomic_builtins, test_kernel_atomic_builtins);
+ do {
+ long tmp;
+
+ kcsan_enable_current();
+
+ __atomic_store_n(&test_var, 42L, __ATOMIC_RELAXED);
+ KUNIT_EXPECT_EQ(test, 42L, __atomic_load_n(&test_var, __ATOMIC_RELAXED));
+
+ KUNIT_EXPECT_EQ(test, 42L, __atomic_exchange_n(&test_var, 20, __ATOMIC_RELAXED));
+ KUNIT_EXPECT_EQ(test, 20L, test_var);
+
+ tmp = 20L;
+ KUNIT_EXPECT_TRUE(test, __atomic_compare_exchange_n(&test_var, &tmp, 30L,
+ 0, __ATOMIC_RELAXED,
+ __ATOMIC_RELAXED));
+ KUNIT_EXPECT_EQ(test, tmp, 20L);
+ KUNIT_EXPECT_EQ(test, test_var, 30L);
+ KUNIT_EXPECT_FALSE(test, __atomic_compare_exchange_n(&test_var, &tmp, 40L,
+ 1, __ATOMIC_RELAXED,
+ __ATOMIC_RELAXED));
+ KUNIT_EXPECT_EQ(test, tmp, 30L);
+ KUNIT_EXPECT_EQ(test, test_var, 30L);
+
+ KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED));
+ KUNIT_EXPECT_EQ(test, 31L, __atomic_fetch_sub(&test_var, 1, __ATOMIC_RELAXED));
+ KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_and(&test_var, 0xf, __ATOMIC_RELAXED));
+ KUNIT_EXPECT_EQ(test, 14L, __atomic_fetch_xor(&test_var, 0xf, __ATOMIC_RELAXED));
+ KUNIT_EXPECT_EQ(test, 1L, __atomic_fetch_or(&test_var, 0xf0, __ATOMIC_RELAXED));
+ KUNIT_EXPECT_EQ(test, 241L, __atomic_fetch_nand(&test_var, 0xf, __ATOMIC_RELAXED));
+ KUNIT_EXPECT_EQ(test, -2L, test_var);
+
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+ __atomic_signal_fence(__ATOMIC_SEQ_CST);
+
+ kcsan_disable_current();
+
+ match_never = report_available();
+ } while (!end_test_checks(match_never));
+ KUNIT_EXPECT_FALSE(test, match_never);
+}
+
/*
* Each test case is run with different numbers of threads. Until KUnit supports
* passing arguments for each test case, we encode #threads in the test case
KCSAN_KUNIT_CASE(test_write_write_struct_part),
KCSAN_KUNIT_CASE(test_read_atomic_write_atomic),
KCSAN_KUNIT_CASE(test_read_plain_atomic_write),
+ KCSAN_KUNIT_CASE(test_read_plain_atomic_rmw),
KCSAN_KUNIT_CASE(test_zero_size_access),
KCSAN_KUNIT_CASE(test_data_race),
KCSAN_KUNIT_CASE(test_assert_exclusive_writer),
KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped),
KCSAN_KUNIT_CASE(test_jiffies_noreport),
KCSAN_KUNIT_CASE(test_seqlock_noreport),
+ KCSAN_KUNIT_CASE(test_atomic_builtins),
{},
};
#ifndef _KERNEL_KCSAN_KCSAN_H
#define _KERNEL_KCSAN_KCSAN_H
+#include <linux/atomic.h>
#include <linux/kcsan.h>
#include <linux/sched.h>
*/
void kcsan_debugfs_init(void);
+/*
+ * Statistics counters displayed via debugfs; should only be modified in
+ * slow-paths.
+ */
enum kcsan_counter_id {
/*
* Number of watchpoints currently in use.
KCSAN_COUNTER_COUNT, /* number of counters */
};
-
-/*
- * Increment/decrement counter with given id; avoid calling these in fast-path.
- */
-extern void kcsan_counter_inc(enum kcsan_counter_id id);
-extern void kcsan_counter_dec(enum kcsan_counter_id id);
+extern atomic_long_t kcsan_counters[KCSAN_COUNTER_COUNT];
/*
* Returns true if data races in the function symbol that maps to func_addr
return "write";
case KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC:
return "write (marked)";
+ case KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE:
+ return "read-write";
+ case KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC:
+ return "read-write (marked)";
case KCSAN_ACCESS_SCOPED:
return "read (scoped)";
case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_ATOMIC:
cur = strnstr(buf, "kcsan_", len);
if (cur) {
- cur += sizeof("kcsan_") - 1;
- if (strncmp(cur, "test", sizeof("test") - 1))
+ cur += strlen("kcsan_");
+ if (!str_has_prefix(cur, "test"))
continue; /* KCSAN runtime function. */
/* KCSAN related test. */
}
* If the actual accesses to not match, this was a false
* positive due to watchpoint encoding.
*/
- kcsan_counter_inc(KCSAN_COUNTER_ENCODING_FALSE_POSITIVES);
+ atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_ENCODING_FALSE_POSITIVES]);
goto discard;
}
// SPDX-License-Identifier: GPL-2.0
+#define pr_fmt(fmt) "kcsan: " fmt
+
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/printk.h>
if (do_test()) \
++passed; \
else \
- pr_err("KCSAN selftest: " #do_test " failed"); \
+ pr_err("selftest: " #do_test " failed"); \
} while (0)
RUN_TEST(test_requires);
RUN_TEST(test_encode_decode);
RUN_TEST(test_matching_access);
- pr_info("KCSAN selftest: %d/%d tests passed\n", passed, total);
+ pr_info("selftest: %d/%d tests passed\n", passed, total);
if (passed != total)
- panic("KCSAN selftests failed");
+ panic("selftests failed");
return 0;
}
postcore_initcall(kcsan_selftest);
#include <linux/cpu.h>
#include <linux/jump_label.h>
#include <linux/perf_event.h>
+#include <linux/static_call.h>
#include <asm/sections.h>
#include <asm/cacheflush.h>
}
NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
-void recycle_rp_inst(struct kretprobe_instance *ri,
- struct hlist_head *head)
+static void recycle_rp_inst(struct kretprobe_instance *ri)
{
struct kretprobe *rp = ri->rp;
hlist_add_head(&ri->hlist, &rp->free_instances);
raw_spin_unlock(&rp->lock);
} else
- /* Unregistering */
- hlist_add_head(&ri->hlist, head);
+ kfree_rcu(ri, rcu);
}
NOKPROBE_SYMBOL(recycle_rp_inst);
-void kretprobe_hash_lock(struct task_struct *tsk,
+static void kretprobe_hash_lock(struct task_struct *tsk,
struct hlist_head **head, unsigned long *flags)
__acquires(hlist_lock)
{
}
NOKPROBE_SYMBOL(kretprobe_table_lock);
-void kretprobe_hash_unlock(struct task_struct *tsk,
+static void kretprobe_hash_unlock(struct task_struct *tsk,
unsigned long *flags)
__releases(hlist_lock)
{
}
NOKPROBE_SYMBOL(kretprobe_table_unlock);
-struct kprobe kprobe_busy = {
+static struct kprobe kprobe_busy = {
.addr = (void *) get_kprobe,
};
void kprobe_flush_task(struct task_struct *tk)
{
struct kretprobe_instance *ri;
- struct hlist_head *head, empty_rp;
+ struct hlist_head *head;
struct hlist_node *tmp;
unsigned long hash, flags = 0;
kprobe_busy_begin();
- INIT_HLIST_HEAD(&empty_rp);
hash = hash_ptr(tk, KPROBE_HASH_BITS);
head = &kretprobe_inst_table[hash];
kretprobe_table_lock(hash, &flags);
hlist_for_each_entry_safe(ri, tmp, head, hlist) {
if (ri->task == tk)
- recycle_rp_inst(ri, &empty_rp);
+ recycle_rp_inst(ri);
}
kretprobe_table_unlock(hash, &flags);
- hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
- hlist_del(&ri->hlist);
- kfree(ri);
- }
kprobe_busy_end();
}
struct hlist_node *next;
struct hlist_head *head;
- /* No race here */
+ /* To avoid recursive kretprobe by NMI, set kprobe busy here */
+ kprobe_busy_begin();
for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
kretprobe_table_lock(hash, &flags);
head = &kretprobe_inst_table[hash];
}
kretprobe_table_unlock(hash, &flags);
}
+ kprobe_busy_end();
+
free_rp_inst(rp);
}
NOKPROBE_SYMBOL(cleanup_rp_inst);
if (!kernel_text_address((unsigned long) p->addr) ||
within_kprobe_blacklist((unsigned long) p->addr) ||
jump_label_text_reserved(p->addr, p->addr) ||
+ static_call_text_reserved(p->addr, p->addr) ||
find_bug((unsigned long)p->addr)) {
ret = -EINVAL;
goto out;
}
#ifdef CONFIG_KRETPROBES
+
+unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
+ void *trampoline_address,
+ void *frame_pointer)
+{
+ struct kretprobe_instance *ri = NULL, *last = NULL;
+ struct hlist_head *head;
+ struct hlist_node *tmp;
+ unsigned long flags;
+ kprobe_opcode_t *correct_ret_addr = NULL;
+ bool skipped = false;
+
+ kretprobe_hash_lock(current, &head, &flags);
+
+ /*
+ * It is possible to have multiple instances associated with a given
+ * task either because multiple functions in the call path have
+ * return probes installed on them, and/or more than one
+ * return probe was registered for a target function.
+ *
+ * We can handle this because:
+ * - instances are always pushed into the head of the list
+ * - when multiple return probes are registered for the same
+ * function, the (chronologically) first instance's ret_addr
+ * will be the real return address, and all the rest will
+ * point to kretprobe_trampoline.
+ */
+ hlist_for_each_entry(ri, head, hlist) {
+ if (ri->task != current)
+ /* another task is sharing our hash bucket */
+ continue;
+ /*
+ * Return probes must be pushed on this hash list correct
+ * order (same as return order) so that it can be popped
+ * correctly. However, if we find it is pushed it incorrect
+ * order, this means we find a function which should not be
+ * probed, because the wrong order entry is pushed on the
+ * path of processing other kretprobe itself.
+ */
+ if (ri->fp != frame_pointer) {
+ if (!skipped)
+ pr_warn("kretprobe is stacked incorrectly. Trying to fixup.\n");
+ skipped = true;
+ continue;
+ }
+
+ correct_ret_addr = ri->ret_addr;
+ if (skipped)
+ pr_warn("%ps must be blacklisted because of incorrect kretprobe order\n",
+ ri->rp->kp.addr);
+
+ if (correct_ret_addr != trampoline_address)
+ /*
+ * This is the real return address. Any other
+ * instances associated with this task are for
+ * other calls deeper on the call stack
+ */
+ break;
+ }
+
+ BUG_ON(!correct_ret_addr || (correct_ret_addr == trampoline_address));
+ last = ri;
+
+ hlist_for_each_entry_safe(ri, tmp, head, hlist) {
+ if (ri->task != current)
+ /* another task is sharing our hash bucket */
+ continue;
+ if (ri->fp != frame_pointer)
+ continue;
+
+ if (ri->rp && ri->rp->handler) {
+ struct kprobe *prev = kprobe_running();
+
+ __this_cpu_write(current_kprobe, &ri->rp->kp);
+ ri->ret_addr = correct_ret_addr;
+ ri->rp->handler(ri, regs);
+ __this_cpu_write(current_kprobe, prev);
+ }
+
+ recycle_rp_inst(ri);
+
+ if (ri == last)
+ break;
+ }
+
+ kretprobe_hash_unlock(current, &flags);
+
+ return (unsigned long)correct_ret_addr;
+}
+NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
+
/*
* This kprobe pre_handler is registered with every kretprobe. When probe
* hits it will set up the return probe.
unsigned long hash, flags = 0;
struct kretprobe_instance *ri;
- /*
- * To avoid deadlocks, prohibit return probing in NMI contexts,
- * just skip the probe and increase the (inexact) 'nmissed'
- * statistical counter, so that the user is informed that
- * something happened:
- */
- if (unlikely(in_nmi())) {
- rp->nmissed++;
- return 0;
- }
-
/* TODO: consider to only swap the RA after the last pre_handler fired */
hash = hash_ptr(current, KPROBE_HASH_BITS);
raw_spin_lock_irqsave(&rp->lock, flags);
#define lock_stat 0
#endif
+DEFINE_PER_CPU(unsigned int, lockdep_recursion);
+EXPORT_PER_CPU_SYMBOL_GPL(lockdep_recursion);
+
+static inline bool lockdep_enabled(void)
+{
+ if (!debug_locks)
+ return false;
+
+ if (raw_cpu_read(lockdep_recursion))
+ return false;
+
+ if (current->lockdep_recursion)
+ return false;
+
+ return true;
+}
+
/*
* lockdep_lock: protects the lockdep graph, the hashes and the
* class/list/hash allocators.
arch_spin_lock(&__lock);
__owner = current;
- current->lockdep_recursion++;
+ __this_cpu_inc(lockdep_recursion);
}
static inline void lockdep_unlock(void)
if (debug_locks && DEBUG_LOCKS_WARN_ON(__owner != current))
return;
- current->lockdep_recursion--;
+ __this_cpu_dec(lockdep_recursion);
__owner = NULL;
arch_spin_unlock(&__lock);
}
static struct hlist_head chainhash_table[CHAINHASH_SIZE];
+/*
+ * the id of held_lock
+ */
+static inline u16 hlock_id(struct held_lock *hlock)
+{
+ BUILD_BUG_ON(MAX_LOCKDEP_KEYS_BITS + 2 > 16);
+
+ return (hlock->class_idx | (hlock->read << MAX_LOCKDEP_KEYS_BITS));
+}
+
+static inline unsigned int chain_hlock_class_idx(u16 hlock_id)
+{
+ return hlock_id & (MAX_LOCKDEP_KEYS - 1);
+}
+
/*
* The hash key of the lock dependency chains is a hash itself too:
* it's a hash of all locks taken up to that lock, including that lock.
task->lockdep_recursion = 0;
}
+static __always_inline void lockdep_recursion_inc(void)
+{
+ __this_cpu_inc(lockdep_recursion);
+}
+
static __always_inline void lockdep_recursion_finish(void)
{
- if (WARN_ON_ONCE((--current->lockdep_recursion) & LOCKDEP_RECURSION_MASK))
- current->lockdep_recursion = 0;
+ if (WARN_ON_ONCE(__this_cpu_dec_return(lockdep_recursion)))
+ __this_cpu_write(lockdep_recursion, 0);
}
void lockdep_set_selftest_task(struct task_struct *task)
#include "lockdep_states.h"
#undef LOCKDEP_STATE
[LOCK_USED] = "INITIAL USE",
+ [LOCK_USED_READ] = "INITIAL READ USE",
+ /* abused as string storage for verify_lock_unused() */
[LOCK_USAGE_STATES] = "IN-NMI",
};
#endif
*/
static int add_lock_to_list(struct lock_class *this,
struct lock_class *links_to, struct list_head *head,
- unsigned long ip, int distance,
+ unsigned long ip, u16 distance, u8 dep,
const struct lock_trace *trace)
{
struct lock_list *entry;
entry->class = this;
entry->links_to = links_to;
+ entry->dep = dep;
entry->distance = distance;
entry->trace = trace;
/*
return (cq->rear - cq->front) & CQ_MASK;
}
-static inline void mark_lock_accessed(struct lock_list *lock,
- struct lock_list *parent)
+static inline void mark_lock_accessed(struct lock_list *lock)
{
- unsigned long nr;
+ lock->class->dep_gen_id = lockdep_dependency_gen_id;
+}
- nr = lock - list_entries;
- WARN_ON(nr >= ARRAY_SIZE(list_entries)); /* Out-of-bounds, input fail */
+static inline void visit_lock_entry(struct lock_list *lock,
+ struct lock_list *parent)
+{
lock->parent = parent;
- lock->class->dep_gen_id = lockdep_dependency_gen_id;
}
static inline unsigned long lock_accessed(struct lock_list *lock)
{
- unsigned long nr;
-
- nr = lock - list_entries;
- WARN_ON(nr >= ARRAY_SIZE(list_entries)); /* Out-of-bounds, input fail */
return lock->class->dep_gen_id == lockdep_dependency_gen_id;
}
return lock_class + offset;
}
+/*
+ * Return values of a bfs search:
+ *
+ * BFS_E* indicates an error
+ * BFS_R* indicates a result (match or not)
+ *
+ * BFS_EINVALIDNODE: Find a invalid node in the graph.
+ *
+ * BFS_EQUEUEFULL: The queue is full while doing the bfs.
+ *
+ * BFS_RMATCH: Find the matched node in the graph, and put that node into
+ * *@target_entry.
+ *
+ * BFS_RNOMATCH: Haven't found the matched node and keep *@target_entry
+ * _unchanged_.
+ */
+enum bfs_result {
+ BFS_EINVALIDNODE = -2,
+ BFS_EQUEUEFULL = -1,
+ BFS_RMATCH = 0,
+ BFS_RNOMATCH = 1,
+};
+
+/*
+ * bfs_result < 0 means error
+ */
+static inline bool bfs_error(enum bfs_result res)
+{
+ return res < 0;
+}
+
+/*
+ * DEP_*_BIT in lock_list::dep
+ *
+ * For dependency @prev -> @next:
+ *
+ * SR: @prev is shared reader (->read != 0) and @next is recursive reader
+ * (->read == 2)
+ * ER: @prev is exclusive locker (->read == 0) and @next is recursive reader
+ * SN: @prev is shared reader and @next is non-recursive locker (->read != 2)
+ * EN: @prev is exclusive locker and @next is non-recursive locker
+ *
+ * Note that we define the value of DEP_*_BITs so that:
+ * bit0 is prev->read == 0
+ * bit1 is next->read != 2
+ */
+#define DEP_SR_BIT (0 + (0 << 1)) /* 0 */
+#define DEP_ER_BIT (1 + (0 << 1)) /* 1 */
+#define DEP_SN_BIT (0 + (1 << 1)) /* 2 */
+#define DEP_EN_BIT (1 + (1 << 1)) /* 3 */
+
+#define DEP_SR_MASK (1U << (DEP_SR_BIT))
+#define DEP_ER_MASK (1U << (DEP_ER_BIT))
+#define DEP_SN_MASK (1U << (DEP_SN_BIT))
+#define DEP_EN_MASK (1U << (DEP_EN_BIT))
+
+static inline unsigned int
+__calc_dep_bit(struct held_lock *prev, struct held_lock *next)
+{
+ return (prev->read == 0) + ((next->read != 2) << 1);
+}
+
+static inline u8 calc_dep(struct held_lock *prev, struct held_lock *next)
+{
+ return 1U << __calc_dep_bit(prev, next);
+}
+
+/*
+ * calculate the dep_bit for backwards edges. We care about whether @prev is
+ * shared and whether @next is recursive.
+ */
+static inline unsigned int
+__calc_dep_bitb(struct held_lock *prev, struct held_lock *next)
+{
+ return (next->read != 2) + ((prev->read == 0) << 1);
+}
+
+static inline u8 calc_depb(struct held_lock *prev, struct held_lock *next)
+{
+ return 1U << __calc_dep_bitb(prev, next);
+}
+
+/*
+ * Initialize a lock_list entry @lock belonging to @class as the root for a BFS
+ * search.
+ */
+static inline void __bfs_init_root(struct lock_list *lock,
+ struct lock_class *class)
+{
+ lock->class = class;
+ lock->parent = NULL;
+ lock->only_xr = 0;
+}
+
+/*
+ * Initialize a lock_list entry @lock based on a lock acquisition @hlock as the
+ * root for a BFS search.
+ *
+ * ->only_xr of the initial lock node is set to @hlock->read == 2, to make sure
+ * that <prev> -> @hlock and @hlock -> <whatever __bfs() found> is not -(*R)->
+ * and -(S*)->.
+ */
+static inline void bfs_init_root(struct lock_list *lock,
+ struct held_lock *hlock)
+{
+ __bfs_init_root(lock, hlock_class(hlock));
+ lock->only_xr = (hlock->read == 2);
+}
/*
- * Forward- or backward-dependency search, used for both circular dependency
- * checking and hardirq-unsafe/softirq-unsafe checking.
+ * Similar to bfs_init_root() but initialize the root for backwards BFS.
+ *
+ * ->only_xr of the initial lock node is set to @hlock->read != 0, to make sure
+ * that <next> -> @hlock and @hlock -> <whatever backwards BFS found> is not
+ * -(*S)-> and -(R*)-> (reverse order of -(*R)-> and -(S*)->).
*/
-static int __bfs(struct lock_list *source_entry,
- void *data,
- int (*match)(struct lock_list *entry, void *data),
- struct lock_list **target_entry,
- int offset)
+static inline void bfs_init_rootb(struct lock_list *lock,
+ struct held_lock *hlock)
+{
+ __bfs_init_root(lock, hlock_class(hlock));
+ lock->only_xr = (hlock->read != 0);
+}
+
+static inline struct lock_list *__bfs_next(struct lock_list *lock, int offset)
{
+ if (!lock || !lock->parent)
+ return NULL;
+
+ return list_next_or_null_rcu(get_dep_list(lock->parent, offset),
+ &lock->entry, struct lock_list, entry);
+}
+
+/*
+ * Breadth-First Search to find a strong path in the dependency graph.
+ *
+ * @source_entry: the source of the path we are searching for.
+ * @data: data used for the second parameter of @match function
+ * @match: match function for the search
+ * @target_entry: pointer to the target of a matched path
+ * @offset: the offset to struct lock_class to determine whether it is
+ * locks_after or locks_before
+ *
+ * We may have multiple edges (considering different kinds of dependencies,
+ * e.g. ER and SN) between two nodes in the dependency graph. But
+ * only the strong dependency path in the graph is relevant to deadlocks. A
+ * strong dependency path is a dependency path that doesn't have two adjacent
+ * dependencies as -(*R)-> -(S*)->, please see:
+ *
+ * Documentation/locking/lockdep-design.rst
+ *
+ * for more explanation of the definition of strong dependency paths
+ *
+ * In __bfs(), we only traverse in the strong dependency path:
+ *
+ * In lock_list::only_xr, we record whether the previous dependency only
+ * has -(*R)-> in the search, and if it does (prev only has -(*R)->), we
+ * filter out any -(S*)-> in the current dependency and after that, the
+ * ->only_xr is set according to whether we only have -(*R)-> left.
+ */
+static enum bfs_result __bfs(struct lock_list *source_entry,
+ void *data,
+ bool (*match)(struct lock_list *entry, void *data),
+ struct lock_list **target_entry,
+ int offset)
+{
+ struct circular_queue *cq = &lock_cq;
+ struct lock_list *lock = NULL;
struct lock_list *entry;
- struct lock_list *lock;
struct list_head *head;
- struct circular_queue *cq = &lock_cq;
- int ret = 1;
+ unsigned int cq_depth;
+ bool first;
lockdep_assert_locked();
- if (match(source_entry, data)) {
- *target_entry = source_entry;
- ret = 0;
- goto exit;
- }
-
- head = get_dep_list(source_entry, offset);
- if (list_empty(head))
- goto exit;
-
__cq_init(cq);
__cq_enqueue(cq, source_entry);
- while ((lock = __cq_dequeue(cq))) {
+ while ((lock = __bfs_next(lock, offset)) || (lock = __cq_dequeue(cq))) {
+ if (!lock->class)
+ return BFS_EINVALIDNODE;
+
+ /*
+ * Step 1: check whether we already finish on this one.
+ *
+ * If we have visited all the dependencies from this @lock to
+ * others (iow, if we have visited all lock_list entries in
+ * @lock->class->locks_{after,before}) we skip, otherwise go
+ * and visit all the dependencies in the list and mark this
+ * list accessed.
+ */
+ if (lock_accessed(lock))
+ continue;
+ else
+ mark_lock_accessed(lock);
- if (!lock->class) {
- ret = -2;
- goto exit;
+ /*
+ * Step 2: check whether prev dependency and this form a strong
+ * dependency path.
+ */
+ if (lock->parent) { /* Parent exists, check prev dependency */
+ u8 dep = lock->dep;
+ bool prev_only_xr = lock->parent->only_xr;
+
+ /*
+ * Mask out all -(S*)-> if we only have *R in previous
+ * step, because -(*R)-> -(S*)-> don't make up a strong
+ * dependency.
+ */
+ if (prev_only_xr)
+ dep &= ~(DEP_SR_MASK | DEP_SN_MASK);
+
+ /* If nothing left, we skip */
+ if (!dep)
+ continue;
+
+ /* If there are only -(*R)-> left, set that for the next step */
+ lock->only_xr = !(dep & (DEP_SN_MASK | DEP_EN_MASK));
}
- head = get_dep_list(lock, offset);
+ /*
+ * Step 3: we haven't visited this and there is a strong
+ * dependency path to this, so check with @match.
+ */
+ if (match(lock, data)) {
+ *target_entry = lock;
+ return BFS_RMATCH;
+ }
+ /*
+ * Step 4: if not match, expand the path by adding the
+ * forward or backwards dependencis in the search
+ *
+ */
+ first = true;
+ head = get_dep_list(lock, offset);
list_for_each_entry_rcu(entry, head, entry) {
- if (!lock_accessed(entry)) {
- unsigned int cq_depth;
- mark_lock_accessed(entry, lock);
- if (match(entry, data)) {
- *target_entry = entry;
- ret = 0;
- goto exit;
- }
+ visit_lock_entry(entry, lock);
- if (__cq_enqueue(cq, entry)) {
- ret = -1;
- goto exit;
- }
- cq_depth = __cq_get_elem_count(cq);
- if (max_bfs_queue_depth < cq_depth)
- max_bfs_queue_depth = cq_depth;
- }
+ /*
+ * Note we only enqueue the first of the list into the
+ * queue, because we can always find a sibling
+ * dependency from one (see __bfs_next()), as a result
+ * the space of queue is saved.
+ */
+ if (!first)
+ continue;
+
+ first = false;
+
+ if (__cq_enqueue(cq, entry))
+ return BFS_EQUEUEFULL;
+
+ cq_depth = __cq_get_elem_count(cq);
+ if (max_bfs_queue_depth < cq_depth)
+ max_bfs_queue_depth = cq_depth;
}
}
-exit:
- return ret;
+
+ return BFS_RNOMATCH;
}
-static inline int __bfs_forwards(struct lock_list *src_entry,
- void *data,
- int (*match)(struct lock_list *entry, void *data),
- struct lock_list **target_entry)
+static inline enum bfs_result
+__bfs_forwards(struct lock_list *src_entry,
+ void *data,
+ bool (*match)(struct lock_list *entry, void *data),
+ struct lock_list **target_entry)
{
return __bfs(src_entry, data, match, target_entry,
offsetof(struct lock_class, locks_after));
}
-static inline int __bfs_backwards(struct lock_list *src_entry,
- void *data,
- int (*match)(struct lock_list *entry, void *data),
- struct lock_list **target_entry)
+static inline enum bfs_result
+__bfs_backwards(struct lock_list *src_entry,
+ void *data,
+ bool (*match)(struct lock_list *entry, void *data),
+ struct lock_list **target_entry)
{
return __bfs(src_entry, data, match, target_entry,
offsetof(struct lock_class, locks_before));
print_circular_bug_entry(entry, depth);
}
-static inline int class_equal(struct lock_list *entry, void *data)
+/*
+ * We are about to add A -> B into the dependency graph, and in __bfs() a
+ * strong dependency path A -> .. -> B is found: hlock_class equals
+ * entry->class.
+ *
+ * If A -> .. -> B can replace A -> B in any __bfs() search (means the former
+ * is _stronger_ than or equal to the latter), we consider A -> B as redundant.
+ * For example if A -> .. -> B is -(EN)-> (i.e. A -(E*)-> .. -(*N)-> B), and A
+ * -> B is -(ER)-> or -(EN)->, then we don't need to add A -> B into the
+ * dependency graph, as any strong path ..-> A -> B ->.. we can get with
+ * having dependency A -> B, we could already get a equivalent path ..-> A ->
+ * .. -> B -> .. with A -> .. -> B. Therefore A -> B is reduntant.
+ *
+ * We need to make sure both the start and the end of A -> .. -> B is not
+ * weaker than A -> B. For the start part, please see the comment in
+ * check_redundant(). For the end part, we need:
+ *
+ * Either
+ *
+ * a) A -> B is -(*R)-> (everything is not weaker than that)
+ *
+ * or
+ *
+ * b) A -> .. -> B is -(*N)-> (nothing is stronger than this)
+ *
+ */
+static inline bool hlock_equal(struct lock_list *entry, void *data)
+{
+ struct held_lock *hlock = (struct held_lock *)data;
+
+ return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */
+ (hlock->read == 2 || /* A -> B is -(*R)-> */
+ !entry->only_xr); /* A -> .. -> B is -(*N)-> */
+}
+
+/*
+ * We are about to add B -> A into the dependency graph, and in __bfs() a
+ * strong dependency path A -> .. -> B is found: hlock_class equals
+ * entry->class.
+ *
+ * We will have a deadlock case (conflict) if A -> .. -> B -> A is a strong
+ * dependency cycle, that means:
+ *
+ * Either
+ *
+ * a) B -> A is -(E*)->
+ *
+ * or
+ *
+ * b) A -> .. -> B is -(*N)-> (i.e. A -> .. -(*N)-> B)
+ *
+ * as then we don't have -(*R)-> -(S*)-> in the cycle.
+ */
+static inline bool hlock_conflict(struct lock_list *entry, void *data)
{
- return entry->class == data;
+ struct held_lock *hlock = (struct held_lock *)data;
+
+ return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */
+ (hlock->read == 0 || /* B -> A is -(E*)-> */
+ !entry->only_xr); /* A -> .. -> B is -(*N)-> */
}
static noinline void print_circular_bug(struct lock_list *this,
- struct lock_list *target,
- struct held_lock *check_src,
- struct held_lock *check_tgt)
+ struct lock_list *target,
+ struct held_lock *check_src,
+ struct held_lock *check_tgt)
{
struct task_struct *curr = current;
struct lock_list *parent;
WARN(1, "lockdep bfs error:%d\n", ret);
}
-static int noop_count(struct lock_list *entry, void *data)
+static bool noop_count(struct lock_list *entry, void *data)
{
(*(unsigned long *)data)++;
- return 0;
+ return false;
}
static unsigned long __lockdep_count_forward_deps(struct lock_list *this)
unsigned long ret, flags;
struct lock_list this;
- this.parent = NULL;
- this.class = class;
+ __bfs_init_root(&this, class);
raw_local_irq_save(flags);
lockdep_lock();
unsigned long ret, flags;
struct lock_list this;
- this.parent = NULL;
- this.class = class;
+ __bfs_init_root(&this, class);
raw_local_irq_save(flags);
lockdep_lock();
/*
* Check that the dependency graph starting at <src> can lead to
- * <target> or not. Print an error and return 0 if it does.
+ * <target> or not.
*/
-static noinline int
-check_path(struct lock_class *target, struct lock_list *src_entry,
+static noinline enum bfs_result
+check_path(struct held_lock *target, struct lock_list *src_entry,
+ bool (*match)(struct lock_list *entry, void *data),
struct lock_list **target_entry)
{
- int ret;
+ enum bfs_result ret;
- ret = __bfs_forwards(src_entry, (void *)target, class_equal,
- target_entry);
+ ret = __bfs_forwards(src_entry, target, match, target_entry);
- if (unlikely(ret < 0))
+ if (unlikely(bfs_error(ret)))
print_bfs_bug(ret);
return ret;
* lead to <target>. If it can, there is a circle when adding
* <target> -> <src> dependency.
*
- * Print an error and return 0 if it does.
+ * Print an error and return BFS_RMATCH if it does.
*/
-static noinline int
+static noinline enum bfs_result
check_noncircular(struct held_lock *src, struct held_lock *target,
struct lock_trace **const trace)
{
- int ret;
+ enum bfs_result ret;
struct lock_list *target_entry;
- struct lock_list src_entry = {
- .class = hlock_class(src),
- .parent = NULL,
- };
+ struct lock_list src_entry;
+
+ bfs_init_root(&src_entry, src);
debug_atomic_inc(nr_cyclic_checks);
- ret = check_path(hlock_class(target), &src_entry, &target_entry);
+ ret = check_path(target, &src_entry, hlock_conflict, &target_entry);
- if (unlikely(!ret)) {
+ if (unlikely(ret == BFS_RMATCH)) {
if (!*trace) {
/*
* If save_trace fails here, the printing might
* <target> or not. If it can, <src> -> <target> dependency is already
* in the graph.
*
- * Print an error and return 2 if it does or 1 if it does not.
+ * Return BFS_RMATCH if it does, or BFS_RMATCH if it does not, return BFS_E* if
+ * any error appears in the bfs search.
*/
-static noinline int
+static noinline enum bfs_result
check_redundant(struct held_lock *src, struct held_lock *target)
{
- int ret;
+ enum bfs_result ret;
struct lock_list *target_entry;
- struct lock_list src_entry = {
- .class = hlock_class(src),
- .parent = NULL,
- };
+ struct lock_list src_entry;
+
+ bfs_init_root(&src_entry, src);
+ /*
+ * Special setup for check_redundant().
+ *
+ * To report redundant, we need to find a strong dependency path that
+ * is equal to or stronger than <src> -> <target>. So if <src> is E,
+ * we need to let __bfs() only search for a path starting at a -(E*)->,
+ * we achieve this by setting the initial node's ->only_xr to true in
+ * that case. And if <prev> is S, we set initial ->only_xr to false
+ * because both -(S*)-> (equal) and -(E*)-> (stronger) are redundant.
+ */
+ src_entry.only_xr = src->read == 0;
debug_atomic_inc(nr_redundant_checks);
- ret = check_path(hlock_class(target), &src_entry, &target_entry);
+ ret = check_path(target, &src_entry, hlock_equal, &target_entry);
- if (!ret) {
+ if (ret == BFS_RMATCH)
debug_atomic_inc(nr_redundant);
- ret = 2;
- } else if (ret < 0)
- ret = 0;
return ret;
}
#ifdef CONFIG_TRACE_IRQFLAGS
-static inline int usage_accumulate(struct lock_list *entry, void *mask)
-{
- *(unsigned long *)mask |= entry->class->usage_mask;
-
- return 0;
-}
-
/*
* Forwards and backwards subgraph searching, for the purposes of
* proving that two subgraphs can be connected by a new dependency
* without creating any illegal irq-safe -> irq-unsafe lock dependency.
+ *
+ * A irq safe->unsafe deadlock happens with the following conditions:
+ *
+ * 1) We have a strong dependency path A -> ... -> B
+ *
+ * 2) and we have ENABLED_IRQ usage of B and USED_IN_IRQ usage of A, therefore
+ * irq can create a new dependency B -> A (consider the case that a holder
+ * of B gets interrupted by an irq whose handler will try to acquire A).
+ *
+ * 3) the dependency circle A -> ... -> B -> A we get from 1) and 2) is a
+ * strong circle:
+ *
+ * For the usage bits of B:
+ * a) if A -> B is -(*N)->, then B -> A could be any type, so any
+ * ENABLED_IRQ usage suffices.
+ * b) if A -> B is -(*R)->, then B -> A must be -(E*)->, so only
+ * ENABLED_IRQ_*_READ usage suffices.
+ *
+ * For the usage bits of A:
+ * c) if A -> B is -(E*)->, then B -> A could be any type, so any
+ * USED_IN_IRQ usage suffices.
+ * d) if A -> B is -(S*)->, then B -> A must be -(*N)->, so only
+ * USED_IN_IRQ_*_READ usage suffices.
*/
-static inline int usage_match(struct lock_list *entry, void *mask)
+/*
+ * There is a strong dependency path in the dependency graph: A -> B, and now
+ * we need to decide which usage bit of A should be accumulated to detect
+ * safe->unsafe bugs.
+ *
+ * Note that usage_accumulate() is used in backwards search, so ->only_xr
+ * stands for whether A -> B only has -(S*)-> (in this case ->only_xr is true).
+ *
+ * As above, if only_xr is false, which means A -> B has -(E*)-> dependency
+ * path, any usage of A should be considered. Otherwise, we should only
+ * consider _READ usage.
+ */
+static inline bool usage_accumulate(struct lock_list *entry, void *mask)
{
- return entry->class->usage_mask & *(unsigned long *)mask;
+ if (!entry->only_xr)
+ *(unsigned long *)mask |= entry->class->usage_mask;
+ else /* Mask out _READ usage bits */
+ *(unsigned long *)mask |= (entry->class->usage_mask & LOCKF_IRQ);
+
+ return false;
+}
+
+/*
+ * There is a strong dependency path in the dependency graph: A -> B, and now
+ * we need to decide which usage bit of B conflicts with the usage bits of A,
+ * i.e. which usage bit of B may introduce safe->unsafe deadlocks.
+ *
+ * As above, if only_xr is false, which means A -> B has -(*N)-> dependency
+ * path, any usage of B should be considered. Otherwise, we should only
+ * consider _READ usage.
+ */
+static inline bool usage_match(struct lock_list *entry, void *mask)
+{
+ if (!entry->only_xr)
+ return !!(entry->class->usage_mask & *(unsigned long *)mask);
+ else /* Mask out _READ usage bits */
+ return !!((entry->class->usage_mask & LOCKF_IRQ) & *(unsigned long *)mask);
}
/*
* Find a node in the forwards-direction dependency sub-graph starting
* at @root->class that matches @bit.
*
- * Return 0 if such a node exists in the subgraph, and put that node
+ * Return BFS_MATCH if such a node exists in the subgraph, and put that node
* into *@target_entry.
- *
- * Return 1 otherwise and keep *@target_entry unchanged.
- * Return <0 on error.
*/
-static int
+static enum bfs_result
find_usage_forwards(struct lock_list *root, unsigned long usage_mask,
struct lock_list **target_entry)
{
- int result;
+ enum bfs_result result;
debug_atomic_inc(nr_find_usage_forwards_checks);
/*
* Find a node in the backwards-direction dependency sub-graph starting
* at @root->class that matches @bit.
- *
- * Return 0 if such a node exists in the subgraph, and put that node
- * into *@target_entry.
- *
- * Return 1 otherwise and keep *@target_entry unchanged.
- * Return <0 on error.
*/
-static int
+static enum bfs_result
find_usage_backwards(struct lock_list *root, unsigned long usage_mask,
struct lock_list **target_entry)
{
- int result;
+ enum bfs_result result;
debug_atomic_inc(nr_find_usage_backwards_checks);
#endif
printk(KERN_CONT " {\n");
- for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
+ for (bit = 0; bit < LOCK_TRACE_STATES; bit++) {
if (class->usage_mask & (1 << bit)) {
int len = depth;
}
/*
- * As above, we clear bitnr0 (LOCK_*_READ off) with bitmask ops. First, for all
- * bits with bitnr0 set (LOCK_*_READ), add those with bitnr0 cleared (LOCK_*).
- * And then mask out all bitnr0.
+ * Note that a LOCK_ENABLED_IRQ_*_READ usage and a LOCK_USED_IN_IRQ_*_READ
+ * usage may cause deadlock too, for example:
+ *
+ * P1 P2
+ * <irq disabled>
+ * write_lock(l1); <irq enabled>
+ * read_lock(l2);
+ * write_lock(l2);
+ * <in irq>
+ * read_lock(l1);
+ *
+ * , in above case, l1 will be marked as LOCK_USED_IN_IRQ_HARDIRQ_READ and l2
+ * will marked as LOCK_ENABLE_IRQ_HARDIRQ_READ, and this is a possible
+ * deadlock.
+ *
+ * In fact, all of the following cases may cause deadlocks:
+ *
+ * LOCK_USED_IN_IRQ_* -> LOCK_ENABLED_IRQ_*
+ * LOCK_USED_IN_IRQ_*_READ -> LOCK_ENABLED_IRQ_*
+ * LOCK_USED_IN_IRQ_* -> LOCK_ENABLED_IRQ_*_READ
+ * LOCK_USED_IN_IRQ_*_READ -> LOCK_ENABLED_IRQ_*_READ
+ *
+ * As a result, to calculate the "exclusive mask", first we invert the
+ * direction (USED_IN/ENABLED) of the original mask, and 1) for all bits with
+ * bitnr0 set (LOCK_*_READ), add those with bitnr0 cleared (LOCK_*). 2) for all
+ * bits with bitnr0 cleared (LOCK_*_READ), add those with bitnr0 set (LOCK_*).
*/
static unsigned long exclusive_mask(unsigned long mask)
{
unsigned long excl = invert_dir_mask(mask);
- /* Strip read */
excl |= (excl & LOCKF_IRQ_READ) >> LOCK_USAGE_READ_MASK;
- excl &= ~LOCKF_IRQ_READ;
+ excl |= (excl & LOCKF_IRQ) << LOCK_USAGE_READ_MASK;
return excl;
}
unsigned long excl = invert_dir_mask(mask);
/* Include read in existing usages */
+ excl |= (excl & LOCKF_IRQ_READ) >> LOCK_USAGE_READ_MASK;
excl |= (excl & LOCKF_IRQ) << LOCK_USAGE_READ_MASK;
return excl;
enum lock_usage_bit *bitp,
enum lock_usage_bit *excl_bitp)
{
- int bit, excl;
+ int bit, excl, excl_read;
for_each_set_bit(bit, &mask, LOCK_USED) {
+ /*
+ * exclusive_bit() strips the read bit, however,
+ * LOCK_ENABLED_IRQ_*_READ may cause deadlocks too, so we need
+ * to search excl | LOCK_USAGE_READ_MASK as well.
+ */
excl = exclusive_bit(bit);
+ excl_read = excl | LOCK_USAGE_READ_MASK;
if (excl_mask & lock_flag(excl)) {
*bitp = bit;
*excl_bitp = excl;
return 0;
+ } else if (excl_mask & lock_flag(excl_read)) {
+ *bitp = bit;
+ *excl_bitp = excl_read;
+ return 0;
}
}
return -1;
struct lock_list *target_entry1;
struct lock_list *target_entry;
struct lock_list this, that;
- int ret;
+ enum bfs_result ret;
/*
* Step 1: gather all hard/soft IRQs usages backward in an
* accumulated usage mask.
*/
- this.parent = NULL;
- this.class = hlock_class(prev);
+ bfs_init_rootb(&this, prev);
ret = __bfs_backwards(&this, &usage_mask, usage_accumulate, NULL);
- if (ret < 0) {
+ if (bfs_error(ret)) {
print_bfs_bug(ret);
return 0;
}
*/
forward_mask = exclusive_mask(usage_mask);
- that.parent = NULL;
- that.class = hlock_class(next);
+ bfs_init_root(&that, next);
ret = find_usage_forwards(&that, forward_mask, &target_entry1);
- if (ret < 0) {
+ if (bfs_error(ret)) {
print_bfs_bug(ret);
return 0;
}
- if (ret == 1)
- return ret;
+ if (ret == BFS_RNOMATCH)
+ return 1;
/*
* Step 3: we found a bad match! Now retrieve a lock from the backward
backward_mask = original_mask(target_entry1->class->usage_mask);
ret = find_usage_backwards(&this, backward_mask, &target_entry);
- if (ret < 0) {
+ if (bfs_error(ret)) {
print_bfs_bug(ret);
return 0;
}
- if (DEBUG_LOCKS_WARN_ON(ret == 1))
+ if (DEBUG_LOCKS_WARN_ON(ret == BFS_RNOMATCH))
return 1;
/*
*/
static int
check_prev_add(struct task_struct *curr, struct held_lock *prev,
- struct held_lock *next, int distance,
+ struct held_lock *next, u16 distance,
struct lock_trace **const trace)
{
struct lock_list *entry;
- int ret;
+ enum bfs_result ret;
if (!hlock_class(prev)->key || !hlock_class(next)->key) {
/*
* in the graph whose neighbours are to be checked.
*/
ret = check_noncircular(next, prev, trace);
- if (unlikely(ret <= 0))
+ if (unlikely(bfs_error(ret) || ret == BFS_RMATCH))
return 0;
if (!check_irq_usage(curr, prev, next))
return 0;
- /*
- * For recursive read-locks we do all the dependency checks,
- * but we dont store read-triggered dependencies (only
- * write-triggered dependencies). This ensures that only the
- * write-side dependencies matter, and that if for example a
- * write-lock never takes any other locks, then the reads are
- * equivalent to a NOP.
- */
- if (next->read == 2 || prev->read == 2)
- return 1;
/*
* Is the <prev> -> <next> dependency already present?
*
if (entry->class == hlock_class(next)) {
if (distance == 1)
entry->distance = 1;
- return 1;
+ entry->dep |= calc_dep(prev, next);
+
+ /*
+ * Also, update the reverse dependency in @next's
+ * ->locks_before list.
+ *
+ * Here we reuse @entry as the cursor, which is fine
+ * because we won't go to the next iteration of the
+ * outer loop:
+ *
+ * For normal cases, we return in the inner loop.
+ *
+ * If we fail to return, we have inconsistency, i.e.
+ * <prev>::locks_after contains <next> while
+ * <next>::locks_before doesn't contain <prev>. In
+ * that case, we return after the inner and indicate
+ * something is wrong.
+ */
+ list_for_each_entry(entry, &hlock_class(next)->locks_before, entry) {
+ if (entry->class == hlock_class(prev)) {
+ if (distance == 1)
+ entry->distance = 1;
+ entry->dep |= calc_depb(prev, next);
+ return 1;
+ }
+ }
+
+ /* <prev> is not found in <next>::locks_before */
+ return 0;
}
}
* Is the <prev> -> <next> link redundant?
*/
ret = check_redundant(prev, next);
- if (ret != 1)
- return ret;
+ if (bfs_error(ret))
+ return 0;
+ else if (ret == BFS_RMATCH)
+ return 2;
#endif
if (!*trace) {
*/
ret = add_lock_to_list(hlock_class(next), hlock_class(prev),
&hlock_class(prev)->locks_after,
- next->acquire_ip, distance, *trace);
+ next->acquire_ip, distance,
+ calc_dep(prev, next),
+ *trace);
if (!ret)
return 0;
ret = add_lock_to_list(hlock_class(prev), hlock_class(next),
&hlock_class(next)->locks_before,
- next->acquire_ip, distance, *trace);
+ next->acquire_ip, distance,
+ calc_depb(prev, next),
+ *trace);
if (!ret)
return 0;
goto out_bug;
for (;;) {
- int distance = curr->lockdep_depth - depth + 1;
+ u16 distance = curr->lockdep_depth - depth + 1;
hlock = curr->held_locks + depth - 1;
- /*
- * Only non-recursive-read entries get new dependencies
- * added:
- */
- if (hlock->read != 2 && hlock->check) {
- int ret = check_prev_add(curr, hlock, next, distance,
- &trace);
+ if (hlock->check) {
+ int ret = check_prev_add(curr, hlock, next, distance, &trace);
if (!ret)
return 0;
struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i)
{
- return lock_classes + chain_hlocks[chain->base + i];
+ u16 chain_hlock = chain_hlocks[chain->base + i];
+ unsigned int class_idx = chain_hlock_class_idx(chain_hlock);
+
+ return lock_classes + class_idx - 1;
}
/*
/*
* Returns the next chain_key iteration
*/
-static u64 print_chain_key_iteration(int class_idx, u64 chain_key)
+static u64 print_chain_key_iteration(u16 hlock_id, u64 chain_key)
{
- u64 new_chain_key = iterate_chain_key(chain_key, class_idx);
+ u64 new_chain_key = iterate_chain_key(chain_key, hlock_id);
- printk(" class_idx:%d -> chain_key:%016Lx",
- class_idx,
+ printk(" hlock_id:%d -> chain_key:%016Lx",
+ (unsigned int)hlock_id,
(unsigned long long)new_chain_key);
return new_chain_key;
}
hlock_next->irq_context);
for (; i < depth; i++) {
hlock = curr->held_locks + i;
- chain_key = print_chain_key_iteration(hlock->class_idx, chain_key);
+ chain_key = print_chain_key_iteration(hlock_id(hlock), chain_key);
print_lock(hlock);
}
- print_chain_key_iteration(hlock_next->class_idx, chain_key);
+ print_chain_key_iteration(hlock_id(hlock_next), chain_key);
print_lock(hlock_next);
}
{
int i;
u64 chain_key = INITIAL_CHAIN_KEY;
- int class_id;
+ u16 hlock_id;
printk("depth: %u\n", chain->depth);
for (i = 0; i < chain->depth; i++) {
- class_id = chain_hlocks[chain->base + i];
- chain_key = print_chain_key_iteration(class_id, chain_key);
+ hlock_id = chain_hlocks[chain->base + i];
+ chain_key = print_chain_key_iteration(hlock_id, chain_key);
- print_lock_name(lock_classes + class_id);
+ print_lock_name(lock_classes + chain_hlock_class_idx(hlock_id) - 1);
printk("\n");
}
}
}
for (j = 0; j < chain->depth - 1; j++, i++) {
- id = curr->held_locks[i].class_idx;
+ id = hlock_id(&curr->held_locks[i]);
if (DEBUG_LOCKS_WARN_ON(chain_hlocks[chain->base + j] != id)) {
print_collision(curr, hlock, chain);
struct held_lock *hlock,
u64 chain_key)
{
- struct lock_class *class = hlock_class(hlock);
struct hlist_head *hash_head = chainhashentry(chain_key);
struct lock_chain *chain;
int i, j;
chain->base = j;
for (j = 0; j < chain->depth - 1; j++, i++) {
- int lock_id = curr->held_locks[i].class_idx;
+ int lock_id = hlock_id(curr->held_locks + i);
chain_hlocks[chain->base + j] = lock_id;
}
- chain_hlocks[chain->base + j] = class - lock_classes;
+ chain_hlocks[chain->base + j] = hlock_id(hlock);
hlist_add_head_rcu(&chain->entry, hash_head);
debug_atomic_inc(chain_lookup_misses);
inc_chains(chain->irq_context);
if (prev_hlock && (prev_hlock->irq_context !=
hlock->irq_context))
chain_key = INITIAL_CHAIN_KEY;
- chain_key = iterate_chain_key(chain_key, hlock->class_idx);
+ chain_key = iterate_chain_key(chain_key, hlock_id(hlock));
prev_hlock = hlock;
}
if (chain_key != curr->curr_chain_key) {
*/
static int
check_usage_forwards(struct task_struct *curr, struct held_lock *this,
- enum lock_usage_bit bit, const char *irqclass)
+ enum lock_usage_bit bit)
{
- int ret;
+ enum bfs_result ret;
struct lock_list root;
struct lock_list *target_entry;
+ enum lock_usage_bit read_bit = bit + LOCK_USAGE_READ_MASK;
+ unsigned usage_mask = lock_flag(bit) | lock_flag(read_bit);
- root.parent = NULL;
- root.class = hlock_class(this);
- ret = find_usage_forwards(&root, lock_flag(bit), &target_entry);
- if (ret < 0) {
+ bfs_init_root(&root, this);
+ ret = find_usage_forwards(&root, usage_mask, &target_entry);
+ if (bfs_error(ret)) {
print_bfs_bug(ret);
return 0;
}
- if (ret == 1)
- return ret;
+ if (ret == BFS_RNOMATCH)
+ return 1;
+
+ /* Check whether write or read usage is the match */
+ if (target_entry->class->usage_mask & lock_flag(bit)) {
+ print_irq_inversion_bug(curr, &root, target_entry,
+ this, 1, state_name(bit));
+ } else {
+ print_irq_inversion_bug(curr, &root, target_entry,
+ this, 1, state_name(read_bit));
+ }
- print_irq_inversion_bug(curr, &root, target_entry,
- this, 1, irqclass);
return 0;
}
*/
static int
check_usage_backwards(struct task_struct *curr, struct held_lock *this,
- enum lock_usage_bit bit, const char *irqclass)
+ enum lock_usage_bit bit)
{
- int ret;
+ enum bfs_result ret;
struct lock_list root;
struct lock_list *target_entry;
+ enum lock_usage_bit read_bit = bit + LOCK_USAGE_READ_MASK;
+ unsigned usage_mask = lock_flag(bit) | lock_flag(read_bit);
- root.parent = NULL;
- root.class = hlock_class(this);
- ret = find_usage_backwards(&root, lock_flag(bit), &target_entry);
- if (ret < 0) {
+ bfs_init_rootb(&root, this);
+ ret = find_usage_backwards(&root, usage_mask, &target_entry);
+ if (bfs_error(ret)) {
print_bfs_bug(ret);
return 0;
}
- if (ret == 1)
- return ret;
+ if (ret == BFS_RNOMATCH)
+ return 1;
+
+ /* Check whether write or read usage is the match */
+ if (target_entry->class->usage_mask & lock_flag(bit)) {
+ print_irq_inversion_bug(curr, &root, target_entry,
+ this, 0, state_name(bit));
+ } else {
+ print_irq_inversion_bug(curr, &root, target_entry,
+ this, 0, state_name(read_bit));
+ }
- print_irq_inversion_bug(curr, &root, target_entry,
- this, 0, irqclass);
return 0;
}
return 0;
}
-#define STRICT_READ_CHECKS 1
-
static int (*state_verbose_f[])(struct lock_class *class) = {
#define LOCKDEP_STATE(__STATE) \
__STATE##_verbose,
int read = new_bit & LOCK_USAGE_READ_MASK;
int dir = new_bit & LOCK_USAGE_DIR_MASK;
- /*
- * mark USED_IN has to look forwards -- to ensure no dependency
- * has ENABLED state, which would allow recursion deadlocks.
- *
- * mark ENABLED has to look backwards -- to ensure no dependee
- * has USED_IN state, which, again, would allow recursion deadlocks.
- */
- check_usage_f usage = dir ?
- check_usage_backwards : check_usage_forwards;
-
/*
* Validate that this particular lock does not have conflicting
* usage states.
return 0;
/*
- * Validate that the lock dependencies don't have conflicting usage
- * states.
+ * Check for read in write conflicts
*/
- if ((!read || STRICT_READ_CHECKS) &&
- !usage(curr, this, excl_bit, state_name(new_bit & ~LOCK_USAGE_READ_MASK)))
+ if (!read && !valid_state(curr, this, new_bit,
+ excl_bit + LOCK_USAGE_READ_MASK))
return 0;
+
/*
- * Check for read in write conflicts
+ * Validate that the lock dependencies don't have conflicting usage
+ * states.
*/
- if (!read) {
- if (!valid_state(curr, this, new_bit, excl_bit + LOCK_USAGE_READ_MASK))
+ if (dir) {
+ /*
+ * mark ENABLED has to look backwards -- to ensure no dependee
+ * has USED_IN state, which, again, would allow recursion deadlocks.
+ */
+ if (!check_usage_backwards(curr, this, excl_bit))
return 0;
-
- if (STRICT_READ_CHECKS &&
- !usage(curr, this, excl_bit + LOCK_USAGE_READ_MASK,
- state_name(new_bit + LOCK_USAGE_READ_MASK)))
+ } else {
+ /*
+ * mark USED_IN has to look forwards -- to ensure no dependency
+ * has ENABLED state, which would allow recursion deadlocks.
+ */
+ if (!check_usage_forwards(curr, this, excl_bit))
return 0;
}
if (unlikely(in_nmi()))
return;
- if (unlikely(current->lockdep_recursion & LOCKDEP_RECURSION_MASK))
+ if (unlikely(__this_cpu_read(lockdep_recursion)))
return;
if (unlikely(lockdep_hardirqs_enabled())) {
current->hardirq_chain_key = current->curr_chain_key;
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
__trace_hardirqs_on_caller();
lockdep_recursion_finish();
}
goto skip_checks;
}
- if (unlikely(current->lockdep_recursion & LOCKDEP_RECURSION_MASK))
+ if (unlikely(__this_cpu_read(lockdep_recursion)))
return;
if (lockdep_hardirqs_enabled()) {
if (in_nmi()) {
if (!IS_ENABLED(CONFIG_TRACE_IRQFLAGS_NMI))
return;
- } else if (current->lockdep_recursion & LOCKDEP_RECURSION_MASK)
+ } else if (__this_cpu_read(lockdep_recursion))
return;
/*
{
struct irqtrace_events *trace = ¤t->irqtrace;
- if (unlikely(!debug_locks || current->lockdep_recursion))
+ if (unlikely(!lockdep_enabled()))
return;
/*
return;
}
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
/*
* We'll do an OFF -> ON transition:
*/
*/
void lockdep_softirqs_off(unsigned long ip)
{
- if (unlikely(!debug_locks || current->lockdep_recursion))
+ if (unlikely(!lockdep_enabled()))
return;
/*
static int mark_lock(struct task_struct *curr, struct held_lock *this,
enum lock_usage_bit new_bit)
{
- unsigned int old_mask, new_mask, ret = 1;
+ unsigned int new_mask, ret = 1;
if (new_bit >= LOCK_USAGE_STATES) {
DEBUG_LOCKS_WARN_ON(1);
if (unlikely(hlock_class(this)->usage_mask & new_mask))
goto unlock;
- old_mask = hlock_class(this)->usage_mask;
hlock_class(this)->usage_mask |= new_mask;
- /*
- * Save one usage_traces[] entry and map both LOCK_USED and
- * LOCK_USED_READ onto the same entry.
- */
- if (new_bit == LOCK_USED || new_bit == LOCK_USED_READ) {
- if (old_mask & (LOCKF_USED | LOCKF_USED_READ))
- goto unlock;
- new_bit = LOCK_USED;
+ if (new_bit < LOCK_TRACE_STATES) {
+ if (!(hlock_class(this)->usage_traces[new_bit] = save_trace()))
+ return 0;
}
- if (!(hlock_class(this)->usage_traces[new_bit] = save_trace()))
- return 0;
-
switch (new_bit) {
+ case 0 ... LOCK_USED-1:
+ ret = mark_lock_irq(curr, this, new_bit);
+ if (!ret)
+ return 0;
+ break;
+
case LOCK_USED:
debug_atomic_dec(nr_unused_locks);
break;
+
default:
- ret = mark_lock_irq(curr, this, new_bit);
- if (!ret)
- return 0;
+ break;
}
unlock:
if (subclass) {
unsigned long flags;
- if (DEBUG_LOCKS_WARN_ON(current->lockdep_recursion))
+ if (DEBUG_LOCKS_WARN_ON(!lockdep_enabled()))
return;
raw_local_irq_save(flags);
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
register_lock_class(lock, subclass, 1);
lockdep_recursion_finish();
raw_local_irq_restore(flags);
chain_key = INITIAL_CHAIN_KEY;
chain_head = 1;
}
- chain_key = iterate_chain_key(chain_key, class_idx);
+ chain_key = iterate_chain_key(chain_key, hlock_id(hlock));
if (nest_lock && !__lock_is_held(nest_lock, -1)) {
print_lock_nested_lock_not_held(curr, hlock, ip);
{
unsigned long flags;
- if (unlikely(current->lockdep_recursion))
+ if (unlikely(!lockdep_enabled()))
return;
raw_local_irq_save(flags);
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
check_flags(flags);
if (__lock_set_class(lock, name, key, subclass, ip))
check_chain_key(current);
{
unsigned long flags;
- if (unlikely(current->lockdep_recursion))
+ if (unlikely(!lockdep_enabled()))
return;
raw_local_irq_save(flags);
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
check_flags(flags);
if (__lock_downgrade(lock, ip))
check_chain_key(current);
static bool lockdep_nmi(void)
{
- if (current->lockdep_recursion & LOCKDEP_RECURSION_MASK)
+ if (raw_cpu_read(lockdep_recursion))
return false;
if (!in_nmi())
return true;
}
+/*
+ * read_lock() is recursive if:
+ * 1. We force lockdep think this way in selftests or
+ * 2. The implementation is not queued read/write lock or
+ * 3. The locker is at an in_interrupt() context.
+ */
+bool read_lock_is_recursive(void)
+{
+ return force_read_lock_recursive ||
+ !IS_ENABLED(CONFIG_QUEUED_RWLOCKS) ||
+ in_interrupt();
+}
+EXPORT_SYMBOL_GPL(read_lock_is_recursive);
+
/*
* We are not always called with irqs disabled - do that here,
* and also avoid lockdep recursion:
trace_lock_acquire(lock, subclass, trylock, read, check, nest_lock, ip);
- if (unlikely(current->lockdep_recursion)) {
+ if (!debug_locks)
+ return;
+
+ if (unlikely(!lockdep_enabled())) {
/* XXX allow trylock from NMI ?!? */
if (lockdep_nmi() && !trylock) {
struct held_lock hlock;
raw_local_irq_save(flags);
check_flags(flags);
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
__lock_acquire(lock, subclass, trylock, read, check,
irqs_disabled_flags(flags), nest_lock, ip, 0, 0);
lockdep_recursion_finish();
trace_lock_release(lock, ip);
- if (unlikely(current->lockdep_recursion))
+ if (unlikely(!lockdep_enabled()))
return;
raw_local_irq_save(flags);
check_flags(flags);
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
if (__lock_release(lock, ip))
check_chain_key(current);
lockdep_recursion_finish();
unsigned long flags;
int ret = 0;
- if (unlikely(current->lockdep_recursion))
+ if (unlikely(!lockdep_enabled()))
return 1; /* avoid false negative lockdep_assert_held() */
raw_local_irq_save(flags);
check_flags(flags);
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
ret = __lock_is_held(lock, read);
lockdep_recursion_finish();
raw_local_irq_restore(flags);
struct pin_cookie cookie = NIL_COOKIE;
unsigned long flags;
- if (unlikely(current->lockdep_recursion))
+ if (unlikely(!lockdep_enabled()))
return cookie;
raw_local_irq_save(flags);
check_flags(flags);
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
cookie = __lock_pin_lock(lock);
lockdep_recursion_finish();
raw_local_irq_restore(flags);
{
unsigned long flags;
- if (unlikely(current->lockdep_recursion))
+ if (unlikely(!lockdep_enabled()))
return;
raw_local_irq_save(flags);
check_flags(flags);
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
__lock_repin_lock(lock, cookie);
lockdep_recursion_finish();
raw_local_irq_restore(flags);
{
unsigned long flags;
- if (unlikely(current->lockdep_recursion))
+ if (unlikely(!lockdep_enabled()))
return;
raw_local_irq_save(flags);
check_flags(flags);
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
__lock_unpin_lock(lock, cookie);
lockdep_recursion_finish();
raw_local_irq_restore(flags);
trace_lock_acquired(lock, ip);
- if (unlikely(!lock_stat || !debug_locks))
- return;
-
- if (unlikely(current->lockdep_recursion))
+ if (unlikely(!lock_stat || !lockdep_enabled()))
return;
raw_local_irq_save(flags);
check_flags(flags);
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
__lock_contended(lock, ip);
lockdep_recursion_finish();
raw_local_irq_restore(flags);
trace_lock_contended(lock, ip);
- if (unlikely(!lock_stat || !debug_locks))
- return;
-
- if (unlikely(current->lockdep_recursion))
+ if (unlikely(!lock_stat || !lockdep_enabled()))
return;
raw_local_irq_save(flags);
check_flags(flags);
- current->lockdep_recursion++;
+ lockdep_recursion_inc();
__lock_acquired(lock, ip);
lockdep_recursion_finish();
raw_local_irq_restore(flags);
int i;
for (i = chain->base; i < chain->base + chain->depth; i++) {
- if (chain_hlocks[i] != class - lock_classes)
+ if (chain_hlock_class_idx(chain_hlocks[i]) != class - lock_classes)
continue;
/*
* Each lock class occurs at most once in a lock chain so once
#undef LOCKDEP_STATE
LOCK_USED,
LOCK_USED_READ,
- LOCK_USAGE_STATES
+ LOCK_USAGE_STATES,
};
+/* states after LOCK_USED_READ are not traced and printed */
+static_assert(LOCK_TRACE_STATES == LOCK_USAGE_STATES);
+
#define LOCK_USAGE_READ_MASK 1
#define LOCK_USAGE_DIR_MASK 2
#define LOCK_USAGE_STATE_MASK (~(LOCK_USAGE_READ_MASK | LOCK_USAGE_DIR_MASK))
extern struct list_head all_lock_classes;
extern struct lock_chain lock_chains[];
-#define LOCK_USAGE_CHARS (1+LOCK_USAGE_STATES/2)
+#define LOCK_USAGE_CHARS (2*XXX_LOCK_USAGE_STATES + 1)
extern void get_usage_chars(struct lock_class *class,
char usage[LOCK_USAGE_CHARS]);
mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist",
sizeof(unsigned long),
&mod->num_kprobe_blacklist);
+#endif
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+ mod->static_call_sites = section_objs(info, ".static_call_sites",
+ sizeof(*mod->static_call_sites),
+ &mod->num_static_call_sites);
#endif
mod->extable = section_objs(info, "__ex_table",
sizeof(*mod->extable), &mod->num_exentries);
if (err)
return err;
- blocking_notifier_call_chain(&module_notify_list,
- MODULE_STATE_COMING, mod);
- return 0;
+ err = blocking_notifier_call_chain_robust(&module_notify_list,
+ MODULE_STATE_COMING, MODULE_STATE_GOING, mod);
+ err = notifier_to_errno(err);
+ if (err)
+ klp_module_going(mod);
+
+ return err;
}
static int unknown_module_param_cb(char *param, char *val, const char *modname,
}
NOKPROBE_SYMBOL(notifier_call_chain);
+/**
+ * notifier_call_chain_robust - Inform the registered notifiers about an event
+ * and rollback on error.
+ * @nl: Pointer to head of the blocking notifier chain
+ * @val_up: Value passed unmodified to the notifier function
+ * @val_down: Value passed unmodified to the notifier function when recovering
+ * from an error on @val_up
+ * @v Pointer passed unmodified to the notifier function
+ *
+ * NOTE: It is important the @nl chain doesn't change between the two
+ * invocations of notifier_call_chain() such that we visit the
+ * exact same notifier callbacks; this rules out any RCU usage.
+ *
+ * Returns: the return value of the @val_up call.
+ */
+static int notifier_call_chain_robust(struct notifier_block **nl,
+ unsigned long val_up, unsigned long val_down,
+ void *v)
+{
+ int ret, nr = 0;
+
+ ret = notifier_call_chain(nl, val_up, v, -1, &nr);
+ if (ret & NOTIFY_STOP_MASK)
+ notifier_call_chain(nl, val_down, v, nr-1, NULL);
+
+ return ret;
+}
+
/*
* Atomic notifier chain routines. Registration and unregistration
* use a spinlock, and call_chain is synchronized by RCU (no locks).
}
EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
+int atomic_notifier_call_chain_robust(struct atomic_notifier_head *nh,
+ unsigned long val_up, unsigned long val_down, void *v)
+{
+ unsigned long flags;
+ int ret;
+
+ /*
+ * Musn't use RCU; because then the notifier list can
+ * change between the up and down traversal.
+ */
+ spin_lock_irqsave(&nh->lock, flags);
+ ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
+ spin_unlock_irqrestore(&nh->lock, flags);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(atomic_notifier_call_chain_robust);
+NOKPROBE_SYMBOL(atomic_notifier_call_chain_robust);
+
/**
- * __atomic_notifier_call_chain - Call functions in an atomic notifier chain
+ * atomic_notifier_call_chain - Call functions in an atomic notifier chain
* @nh: Pointer to head of the atomic notifier chain
* @val: Value passed unmodified to notifier function
* @v: Pointer passed unmodified to notifier function
- * @nr_to_call: See the comment for notifier_call_chain.
- * @nr_calls: See the comment for notifier_call_chain.
*
* Calls each function in a notifier chain in turn. The functions
* run in an atomic context, so they must not block.
* Otherwise the return value is the return value
* of the last notifier function called.
*/
-int __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
- unsigned long val, void *v,
- int nr_to_call, int *nr_calls)
+int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
+ unsigned long val, void *v)
{
int ret;
rcu_read_lock();
- ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
+ ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
rcu_read_unlock();
- return ret;
-}
-EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain);
-NOKPROBE_SYMBOL(__atomic_notifier_call_chain);
-int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
- unsigned long val, void *v)
-{
- return __atomic_notifier_call_chain(nh, val, v, -1, NULL);
+ return ret;
}
EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
NOKPROBE_SYMBOL(atomic_notifier_call_chain);
}
EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
+int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
+ unsigned long val_up, unsigned long val_down, void *v)
+{
+ int ret = NOTIFY_DONE;
+
+ /*
+ * We check the head outside the lock, but if this access is
+ * racy then it does not matter what the result of the test
+ * is, we re-check the list after having taken the lock anyway:
+ */
+ if (rcu_access_pointer(nh->head)) {
+ down_read(&nh->rwsem);
+ ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
+ up_read(&nh->rwsem);
+ }
+ return ret;
+}
+EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
+
/**
- * __blocking_notifier_call_chain - Call functions in a blocking notifier chain
+ * blocking_notifier_call_chain - Call functions in a blocking notifier chain
* @nh: Pointer to head of the blocking notifier chain
* @val: Value passed unmodified to notifier function
* @v: Pointer passed unmodified to notifier function
- * @nr_to_call: See comment for notifier_call_chain.
- * @nr_calls: See comment for notifier_call_chain.
*
* Calls each function in a notifier chain in turn. The functions
* run in a process context, so they are allowed to block.
* Otherwise the return value is the return value
* of the last notifier function called.
*/
-int __blocking_notifier_call_chain(struct blocking_notifier_head *nh,
- unsigned long val, void *v,
- int nr_to_call, int *nr_calls)
+int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
+ unsigned long val, void *v)
{
int ret = NOTIFY_DONE;
*/
if (rcu_access_pointer(nh->head)) {
down_read(&nh->rwsem);
- ret = notifier_call_chain(&nh->head, val, v, nr_to_call,
- nr_calls);
+ ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
up_read(&nh->rwsem);
}
return ret;
}
-EXPORT_SYMBOL_GPL(__blocking_notifier_call_chain);
-
-int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
- unsigned long val, void *v)
-{
- return __blocking_notifier_call_chain(nh, val, v, -1, NULL);
-}
EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
/*
}
EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
+int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
+ unsigned long val_up, unsigned long val_down, void *v)
+{
+ return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
+}
+EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
+
/**
- * __raw_notifier_call_chain - Call functions in a raw notifier chain
+ * raw_notifier_call_chain - Call functions in a raw notifier chain
* @nh: Pointer to head of the raw notifier chain
* @val: Value passed unmodified to notifier function
* @v: Pointer passed unmodified to notifier function
- * @nr_to_call: See comment for notifier_call_chain.
- * @nr_calls: See comment for notifier_call_chain
*
* Calls each function in a notifier chain in turn. The functions
* run in an undefined context.
* Otherwise the return value is the return value
* of the last notifier function called.
*/
-int __raw_notifier_call_chain(struct raw_notifier_head *nh,
- unsigned long val, void *v,
- int nr_to_call, int *nr_calls)
-{
- return notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
-}
-EXPORT_SYMBOL_GPL(__raw_notifier_call_chain);
-
int raw_notifier_call_chain(struct raw_notifier_head *nh,
unsigned long val, void *v)
{
- return __raw_notifier_call_chain(nh, val, v, -1, NULL);
+ return notifier_call_chain(&nh->head, val, v, -1, NULL);
}
EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
/**
- * __srcu_notifier_call_chain - Call functions in an SRCU notifier chain
+ * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
* @nh: Pointer to head of the SRCU notifier chain
* @val: Value passed unmodified to notifier function
* @v: Pointer passed unmodified to notifier function
- * @nr_to_call: See comment for notifier_call_chain.
- * @nr_calls: See comment for notifier_call_chain
*
* Calls each function in a notifier chain in turn. The functions
* run in a process context, so they are allowed to block.
* Otherwise the return value is the return value
* of the last notifier function called.
*/
-int __srcu_notifier_call_chain(struct srcu_notifier_head *nh,
- unsigned long val, void *v,
- int nr_to_call, int *nr_calls)
+int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
+ unsigned long val, void *v)
{
int ret;
int idx;
idx = srcu_read_lock(&nh->srcu);
- ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
+ ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
srcu_read_unlock(&nh->srcu, idx);
return ret;
}
-EXPORT_SYMBOL_GPL(__srcu_notifier_call_chain);
-
-int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
- unsigned long val, void *v)
-{
- return __srcu_notifier_call_chain(nh, val, v, -1, NULL);
-}
EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
/**
*/
int hibernate(void)
{
- int error, nr_calls = 0;
bool snapshot_test = false;
+ int error;
if (!hibernation_available()) {
pm_pr_dbg("Hibernation not available.\n");
pr_info("hibernation entry\n");
pm_prepare_console();
- error = __pm_notifier_call_chain(PM_HIBERNATION_PREPARE, -1, &nr_calls);
- if (error) {
- nr_calls--;
- goto Exit;
- }
+ error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION);
+ if (error)
+ goto Restore;
ksys_sync_helper();
/* Don't bother checking whether freezer_test_done is true */
freezer_test_done = false;
Exit:
- __pm_notifier_call_chain(PM_POST_HIBERNATION, nr_calls, NULL);
+ pm_notifier_call_chain(PM_POST_HIBERNATION);
+ Restore:
pm_restore_console();
hibernate_release();
Unlock:
*/
int hibernate_quiet_exec(int (*func)(void *data), void *data)
{
- int error, nr_calls = 0;
+ int error;
lock_system_sleep();
pm_prepare_console();
- error = __pm_notifier_call_chain(PM_HIBERNATION_PREPARE, -1, &nr_calls);
- if (error) {
- nr_calls--;
- goto exit;
- }
+ error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION);
+ if (error)
+ goto restore;
error = freeze_processes();
if (error)
thaw_processes();
exit:
- __pm_notifier_call_chain(PM_POST_HIBERNATION, nr_calls, NULL);
+ pm_notifier_call_chain(PM_POST_HIBERNATION);
+restore:
pm_restore_console();
hibernate_release();
*/
static int software_resume(void)
{
- int error, nr_calls = 0;
+ int error;
/*
* If the user said "noresume".. bail out early.
pr_info("resume from hibernation\n");
pm_prepare_console();
- error = __pm_notifier_call_chain(PM_RESTORE_PREPARE, -1, &nr_calls);
- if (error) {
- nr_calls--;
- goto Close_Finish;
- }
+ error = pm_notifier_call_chain_robust(PM_RESTORE_PREPARE, PM_POST_RESTORE);
+ if (error)
+ goto Restore;
pm_pr_dbg("Preparing processes for hibernation restore.\n");
error = freeze_processes();
error = load_image_and_restore();
thaw_processes();
Finish:
- __pm_notifier_call_chain(PM_POST_RESTORE, nr_calls, NULL);
+ pm_notifier_call_chain(PM_POST_RESTORE);
+ Restore:
pm_restore_console();
pr_info("resume failed (%d)\n", error);
hibernate_release();
}
EXPORT_SYMBOL_GPL(unregister_pm_notifier);
-int __pm_notifier_call_chain(unsigned long val, int nr_to_call, int *nr_calls)
+int pm_notifier_call_chain_robust(unsigned long val_up, unsigned long val_down)
{
int ret;
- ret = __blocking_notifier_call_chain(&pm_chain_head, val, NULL,
- nr_to_call, nr_calls);
+ ret = blocking_notifier_call_chain_robust(&pm_chain_head, val_up, val_down, NULL);
return notifier_to_errno(ret);
}
+
int pm_notifier_call_chain(unsigned long val)
{
- return __pm_notifier_call_chain(val, -1, NULL);
+ return blocking_notifier_call_chain(&pm_chain_head, val, NULL);
}
/* If set, devices may be suspended and resumed asynchronously. */
#ifdef CONFIG_PM_SLEEP
/* kernel/power/main.c */
-extern int __pm_notifier_call_chain(unsigned long val, int nr_to_call,
- int *nr_calls);
+extern int pm_notifier_call_chain_robust(unsigned long val_up, unsigned long val_down);
extern int pm_notifier_call_chain(unsigned long val);
#endif
*/
static int suspend_prepare(suspend_state_t state)
{
- int error, nr_calls = 0;
+ int error;
if (!sleep_state_supported(state))
return -EPERM;
pm_prepare_console();
- error = __pm_notifier_call_chain(PM_SUSPEND_PREPARE, -1, &nr_calls);
- if (error) {
- nr_calls--;
- goto Finish;
- }
+ error = pm_notifier_call_chain_robust(PM_SUSPEND_PREPARE, PM_POST_SUSPEND);
+ if (error)
+ goto Restore;
trace_suspend_resume(TPS("freeze_processes"), 0, true);
error = suspend_freeze_processes();
suspend_stats.failed_freeze++;
dpm_save_failed_step(SUSPEND_FREEZE);
- Finish:
- __pm_notifier_call_chain(PM_POST_SUSPEND, nr_calls, NULL);
+ pm_notifier_call_chain(PM_POST_SUSPEND);
+ Restore:
pm_restore_console();
return error;
}
static int snapshot_open(struct inode *inode, struct file *filp)
{
struct snapshot_data *data;
- int error, nr_calls = 0;
+ int error;
if (!hibernation_available())
return -EPERM;
data->swap = swap_type_of(swsusp_resume_device, 0);
data->mode = O_RDONLY;
data->free_bitmaps = false;
- error = __pm_notifier_call_chain(PM_HIBERNATION_PREPARE, -1, &nr_calls);
- if (error)
- __pm_notifier_call_chain(PM_POST_HIBERNATION, --nr_calls, NULL);
+ error = pm_notifier_call_chain_robust(PM_HIBERNATION_PREPARE, PM_POST_HIBERNATION);
} else {
/*
* Resuming. We may need to wait for the image device to
data->swap = -1;
data->mode = O_WRONLY;
- error = __pm_notifier_call_chain(PM_RESTORE_PREPARE, -1, &nr_calls);
+ error = pm_notifier_call_chain_robust(PM_RESTORE_PREPARE, PM_POST_RESTORE);
if (!error) {
error = create_basic_memory_bitmaps();
data->free_bitmaps = !error;
- } else
- nr_calls--;
-
- if (error)
- __pm_notifier_call_chain(PM_POST_RESTORE, nr_calls, NULL);
+ }
}
if (error)
hibernate_release();
# define STATE_RCU_HEAD_READY 0
# define STATE_RCU_HEAD_QUEUED 1
-extern struct debug_obj_descr rcuhead_debug_descr;
+extern const struct debug_obj_descr rcuhead_debug_descr;
static inline int debug_rcu_head_queue(struct rcu_head *head)
{
lockdep_assert_irqs_disabled();
rcu_eqs_enter(false);
}
+EXPORT_SYMBOL_GPL(rcu_idle_enter);
#ifdef CONFIG_NO_HZ_FULL
/**
rcu_eqs_exit(false);
local_irq_restore(flags);
}
+EXPORT_SYMBOL_GPL(rcu_idle_exit);
#ifdef CONFIG_NO_HZ_FULL
/**
}
EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
-struct debug_obj_descr rcuhead_debug_descr = {
+const struct debug_obj_descr rcuhead_debug_descr = {
.name = "rcu_head",
.is_static_object = rcuhead_is_static_object,
};
EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp);
EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp);
EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp);
+EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp);
EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp);
EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp);
EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp);
return clamp_value / UCLAMP_BUCKET_DELTA;
}
-static inline unsigned int uclamp_bucket_base_value(unsigned int clamp_value)
-{
- return UCLAMP_BUCKET_DELTA * uclamp_bucket_id(clamp_value);
-}
-
static inline unsigned int uclamp_none(enum uclamp_id clamp_id)
{
if (clamp_id == UCLAMP_MIN)
static inline void sched_submit_work(struct task_struct *tsk)
{
+ unsigned int task_flags;
+
if (!tsk->state)
return;
+ task_flags = tsk->flags;
/*
* If a worker went to sleep, notify and ask workqueue whether
* it wants to wake up a task to maintain concurrency.
* in the possible wakeup of a kworker and because wq_worker_sleeping()
* requires it.
*/
- if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
+ if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
preempt_disable();
- if (tsk->flags & PF_WQ_WORKER)
+ if (task_flags & PF_WQ_WORKER)
wq_worker_sleeping(tsk);
else
io_wq_worker_sleeping(tsk);
*/
if (pi_task && dl_prio(pi_task->normal_prio) && p->dl.dl_boosted) {
pi_se = &pi_task->dl;
+ /*
+ * Because of delays in the detection of the overrun of a
+ * thread's runtime, it might be the case that a thread
+ * goes to sleep in a rt mutex with negative runtime. As
+ * a consequence, the thread will be throttled.
+ *
+ * While waiting for the mutex, this thread can also be
+ * boosted via PI, resulting in a thread that is throttled
+ * and boosted at the same time.
+ *
+ * In this case, the boost overrides the throttle.
+ */
+ if (p->dl.dl_throttled) {
+ /*
+ * The replenish timer needs to be canceled. No
+ * problem if it fires concurrently: boosted threads
+ * are ignored in dl_task_timer().
+ */
+ hrtimer_try_to_cancel(&p->dl.dl_timer);
+ p->dl.dl_throttled = 0;
+ }
} else if (!dl_prio(p->normal_prio)) {
/*
- * Special case in which we have a !SCHED_DEADLINE task
- * that is going to be deboosted, but exceeds its
- * runtime while doing so. No point in replenishing
- * it, as it's going to return back to its original
- * scheduling class after this.
+ * Special case in which we have a !SCHED_DEADLINE task that is going
+ * to be deboosted, but exceeds its runtime while doing so. No point in
+ * replenishing it, as it's going to return back to its original
+ * scheduling class after this. If it has been throttled, we need to
+ * clear the flag, otherwise the task may wake up as throttled after
+ * being boosted again with no means to replenish the runtime and clear
+ * the throttle.
*/
+ p->dl.dl_throttled = 0;
BUG_ON(!p->dl.dl_boosted || flags != ENQUEUE_REPLENISH);
return;
}
entry->proc_handler = proc_handler;
}
+static int sd_ctl_doflags(struct ctl_table *table, int write,
+ void *buffer, size_t *lenp, loff_t *ppos)
+{
+ unsigned long flags = *(unsigned long *)table->data;
+ size_t data_size = 0;
+ size_t len = 0;
+ char *tmp;
+ int idx;
+
+ if (write)
+ return 0;
+
+ for_each_set_bit(idx, &flags, __SD_FLAG_CNT) {
+ char *name = sd_flag_debug[idx].name;
+
+ /* Name plus whitespace */
+ data_size += strlen(name) + 1;
+ }
+
+ if (*ppos > data_size) {
+ *lenp = 0;
+ return 0;
+ }
+
+ tmp = kcalloc(data_size + 1, sizeof(*tmp), GFP_KERNEL);
+ if (!tmp)
+ return -ENOMEM;
+
+ for_each_set_bit(idx, &flags, __SD_FLAG_CNT) {
+ char *name = sd_flag_debug[idx].name;
+
+ len += snprintf(tmp + len, strlen(name) + 2, "%s ", name);
+ }
+
+ tmp += *ppos;
+ len -= *ppos;
+
+ if (len > *lenp)
+ len = *lenp;
+ if (len)
+ memcpy(buffer, tmp, len);
+ if (len < *lenp) {
+ ((char *)buffer)[len] = '\n';
+ len++;
+ }
+
+ *lenp = len;
+ *ppos += len;
+
+ kfree(tmp);
+
+ return 0;
+}
+
static struct ctl_table *
sd_alloc_ctl_domain_table(struct sched_domain *sd)
{
set_table_entry(&table[2], "busy_factor", &sd->busy_factor, sizeof(int), 0644, proc_dointvec_minmax);
set_table_entry(&table[3], "imbalance_pct", &sd->imbalance_pct, sizeof(int), 0644, proc_dointvec_minmax);
set_table_entry(&table[4], "cache_nice_tries", &sd->cache_nice_tries, sizeof(int), 0644, proc_dointvec_minmax);
- set_table_entry(&table[5], "flags", &sd->flags, sizeof(int), 0444, proc_dointvec_minmax);
+ set_table_entry(&table[5], "flags", &sd->flags, sizeof(int), 0444, sd_ctl_doflags);
set_table_entry(&table[6], "max_newidle_lb_cost", &sd->max_newidle_lb_cost, sizeof(long), 0644, proc_doulongvec_minmax);
set_table_entry(&table[7], "name", sd->name, CORENAME_MAX_SIZE, 0444, proc_dostring);
/* &table[8] is terminator */
void post_init_entity_util_avg(struct task_struct *p)
{
}
-static void update_tg_load_avg(struct cfs_rq *cfs_rq, int force)
+static void update_tg_load_avg(struct cfs_rq *cfs_rq)
{
}
#endif /* CONFIG_SMP */
/* Cached statistics for all CPUs within a node */
struct numa_stats {
unsigned long load;
+ unsigned long runnable;
unsigned long util;
/* Total compute capacity of CPUs on a node */
unsigned long compute_capacity;
};
static unsigned long cpu_load(struct rq *rq);
+static unsigned long cpu_runnable(struct rq *rq);
static unsigned long cpu_util(int cpu);
-static inline long adjust_numa_imbalance(int imbalance, int src_nr_running);
+static inline long adjust_numa_imbalance(int imbalance, int nr_running);
static inline enum
numa_type numa_classify(unsigned int imbalance_pct,
struct numa_stats *ns)
{
if ((ns->nr_running > ns->weight) &&
- ((ns->compute_capacity * 100) < (ns->util * imbalance_pct)))
+ (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) ||
+ ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100))))
return node_overloaded;
if ((ns->nr_running < ns->weight) ||
- ((ns->compute_capacity * 100) > (ns->util * imbalance_pct)))
+ (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) &&
+ ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100))))
return node_has_spare;
return node_fully_busy;
struct rq *rq = cpu_rq(cpu);
ns->load += cpu_load(rq);
+ ns->runnable += cpu_runnable(rq);
ns->util += cpu_util(cpu);
ns->nr_running += rq->cfs.h_nr_running;
ns->compute_capacity += capacity_of(cpu);
src_running = env->src_stats.nr_running - 1;
dst_running = env->dst_stats.nr_running + 1;
imbalance = max(0, dst_running - src_running);
- imbalance = adjust_numa_imbalance(imbalance, src_running);
+ imbalance = adjust_numa_imbalance(imbalance, dst_running);
/* Use idle CPU if there is no imbalance */
if (!imbalance) {
/* commit outstanding execution time */
if (cfs_rq->curr == se)
update_curr(cfs_rq);
- account_entity_dequeue(cfs_rq, se);
+ update_load_sub(&cfs_rq->load, se->load.weight);
}
dequeue_load_avg(cfs_rq, se);
enqueue_load_avg(cfs_rq, se);
if (se->on_rq)
- account_entity_enqueue(cfs_rq, se);
+ update_load_add(&cfs_rq->load, se->load.weight);
}
/**
* update_tg_load_avg - update the tg's load avg
* @cfs_rq: the cfs_rq whose avg changed
- * @force: update regardless of how small the difference
*
* This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load.
* However, because tg->load_avg is a global value there are performance
*
* Updating tg's load_avg is necessary before update_cfs_share().
*/
-static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force)
+static inline void update_tg_load_avg(struct cfs_rq *cfs_rq)
{
long delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
if (cfs_rq->tg == &root_task_group)
return;
- if (force || abs(delta) > cfs_rq->tg_load_avg_contrib / 64) {
+ if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) {
atomic_long_add(delta, &cfs_rq->tg->load_avg);
cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;
}
#else /* CONFIG_FAIR_GROUP_SCHED */
-static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {}
+static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {}
static inline int propagate_entity_load_avg(struct sched_entity *se)
{
* IOW we're enqueueing a task on a new CPU.
*/
attach_entity_load_avg(cfs_rq, se);
- update_tg_load_avg(cfs_rq, 0);
+ update_tg_load_avg(cfs_rq);
} else if (decayed) {
cfs_rq_util_change(cfs_rq, 0);
if (flags & UPDATE_TG)
- update_tg_load_avg(cfs_rq, 0);
+ update_tg_load_avg(cfs_rq);
}
}
se = second;
}
- /*
- * Prefer last buddy, try to return the CPU to a preempted task.
- */
- if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1)
- se = cfs_rq->last;
-
- /*
- * Someone really wants this to run. If it's not unfair, run it.
- */
- if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1)
+ if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) {
+ /*
+ * Someone really wants this to run. If it's not unfair, run it.
+ */
se = cfs_rq->next;
+ } else if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) {
+ /*
+ * Prefer last buddy, try to return the CPU to a preempted task.
+ */
+ se = cfs_rq->last;
+ }
clear_buddies(cfs_rq, se);
/*
* Scan the local SMT mask for idle CPUs.
*/
-static int select_idle_smt(struct task_struct *p, int target)
+static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
{
int cpu;
return -1;
for_each_cpu(cpu, cpu_smt_mask(target)) {
- if (!cpumask_test_cpu(cpu, p->cpus_ptr))
+ if (!cpumask_test_cpu(cpu, p->cpus_ptr) ||
+ !cpumask_test_cpu(cpu, sched_domain_span(sd)))
continue;
if (available_idle_cpu(cpu) || sched_idle_cpu(cpu))
return cpu;
return -1;
}
-static inline int select_idle_smt(struct task_struct *p, int target)
+static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
{
return -1;
}
if ((unsigned)i < nr_cpumask_bits)
return i;
- i = select_idle_smt(p, target);
+ i = select_idle_smt(p, sd, target);
if ((unsigned)i < nr_cpumask_bits)
return i;
util = cpu_util_next(cpu, p, cpu);
cpu_cap = capacity_of(cpu);
- spare_cap = cpu_cap - util;
+ spare_cap = cpu_cap;
+ lsub_positive(&spare_cap, util);
/*
* Skip CPUs that cannot satisfy the capacity request.
if (unlikely(task_has_idle_policy(p)))
return 0;
+ /* SMT siblings share cache */
+ if (env->sd->flags & SD_SHARE_CPUCAPACITY)
+ return 0;
+
/*
* Buddy candidates are cache hot:
*/
* scheduler fails to find a good waiting task to
* migrate.
*/
- if (load/2 > env->imbalance &&
- env->sd->nr_balance_failed <= env->sd->cache_nice_tries)
+
+ if ((load >> env->sd->nr_balance_failed) > env->imbalance)
goto next;
env->imbalance -= load;
struct sched_entity *se;
if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) {
- update_tg_load_avg(cfs_rq, 0);
+ update_tg_load_avg(cfs_rq);
if (cfs_rq == &rq->cfs)
decayed = true;
capacity = 1;
cpu_rq(cpu)->cpu_capacity = capacity;
+ trace_sched_cpu_capacity_tp(cpu_rq(cpu));
+
sdg->sgc->capacity = capacity;
sdg->sgc->min_capacity = capacity;
sdg->sgc->max_capacity = capacity;
}
}
-static inline long adjust_numa_imbalance(int imbalance, int src_nr_running)
+static inline long adjust_numa_imbalance(int imbalance, int nr_running)
{
unsigned int imbalance_min;
* tasks that remain local when the source domain is almost idle.
*/
imbalance_min = 2;
- if (src_nr_running <= imbalance_min)
+ if (nr_running <= imbalance_min)
return 0;
return imbalance;
/* scale ms to jiffies */
interval = msecs_to_jiffies(interval);
+
+ /*
+ * Reduce likelihood of busy balancing at higher domains racing with
+ * balancing at lower domains by preventing their balancing periods
+ * from being multiples of each other.
+ */
+ if (cpu_busy)
+ interval -= 1;
+
interval = clamp(interval, 1UL, max_load_balance_interval);
return interval;
/* Catch up with the cfs_rq and remove our load when we leave */
update_load_avg(cfs_rq, se, 0);
detach_entity_load_avg(cfs_rq, se);
- update_tg_load_avg(cfs_rq, false);
+ update_tg_load_avg(cfs_rq);
propagate_entity_cfs_rq(se);
}
/* Synchronize entity with its cfs_rq */
update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD);
attach_entity_load_avg(cfs_rq, se);
- update_tg_load_avg(cfs_rq, false);
+ update_tg_load_avg(cfs_rq);
propagate_entity_cfs_rq(se);
}
}
EXPORT_SYMBOL_GPL(sched_trace_rq_cpu);
+int sched_trace_rq_cpu_capacity(struct rq *rq)
+{
+ return rq ?
+#ifdef CONFIG_SMP
+ rq->cpu_capacity
+#else
+ SCHED_CAPACITY_SCALE
+#endif
+ : -1;
+}
+EXPORT_SYMBOL_GPL(sched_trace_rq_cpu_capacity);
+
const struct cpumask *sched_trace_rd_span(struct root_domain *rd)
{
#ifdef CONFIG_SMP
SCHED_FEAT(RT_PUSH_IPI, true)
#endif
-SCHED_FEAT(RT_RUNTIME_SHARE, true)
+SCHED_FEAT(RT_RUNTIME_SHARE, false)
SCHED_FEAT(LB_MIN, false)
SCHED_FEAT(ATTACH_AGE_LOAD, true)
#define MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK 0
#endif
+#ifdef CONFIG_RSEQ
+#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK \
+ (MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ \
+ | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ_BITMASK)
+#else
+#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK 0
+#endif
+
#define MEMBARRIER_CMD_BITMASK \
(MEMBARRIER_CMD_GLOBAL | MEMBARRIER_CMD_GLOBAL_EXPEDITED \
| MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED \
smp_mb(); /* IPIs should be serializing but paranoid. */
}
+static void ipi_rseq(void *info)
+{
+ rseq_preempt(current);
+}
+
static void ipi_sync_rq_state(void *info)
{
struct mm_struct *mm = (struct mm_struct *) info;
return 0;
}
-static int membarrier_private_expedited(int flags)
+static int membarrier_private_expedited(int flags, int cpu_id)
{
- int cpu;
cpumask_var_t tmpmask;
struct mm_struct *mm = current->mm;
+ smp_call_func_t ipi_func = ipi_mb;
- if (flags & MEMBARRIER_FLAG_SYNC_CORE) {
+ if (flags == MEMBARRIER_FLAG_SYNC_CORE) {
if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE))
return -EINVAL;
if (!(atomic_read(&mm->membarrier_state) &
MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY))
return -EPERM;
+ } else if (flags == MEMBARRIER_FLAG_RSEQ) {
+ if (!IS_ENABLED(CONFIG_RSEQ))
+ return -EINVAL;
+ if (!(atomic_read(&mm->membarrier_state) &
+ MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY))
+ return -EPERM;
+ ipi_func = ipi_rseq;
} else {
+ WARN_ON_ONCE(flags);
if (!(atomic_read(&mm->membarrier_state) &
MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY))
return -EPERM;
*/
smp_mb(); /* system call entry is not a mb. */
- if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
+ if (cpu_id < 0 && !zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
return -ENOMEM;
cpus_read_lock();
- rcu_read_lock();
- for_each_online_cpu(cpu) {
+
+ if (cpu_id >= 0) {
struct task_struct *p;
- /*
- * Skipping the current CPU is OK even through we can be
- * migrated at any point. The current CPU, at the point
- * where we read raw_smp_processor_id(), is ensured to
- * be in program order with respect to the caller
- * thread. Therefore, we can skip this CPU from the
- * iteration.
- */
- if (cpu == raw_smp_processor_id())
- continue;
- p = rcu_dereference(cpu_rq(cpu)->curr);
- if (p && p->mm == mm)
- __cpumask_set_cpu(cpu, tmpmask);
+ if (cpu_id >= nr_cpu_ids || !cpu_online(cpu_id))
+ goto out;
+ if (cpu_id == raw_smp_processor_id())
+ goto out;
+ rcu_read_lock();
+ p = rcu_dereference(cpu_rq(cpu_id)->curr);
+ if (!p || p->mm != mm) {
+ rcu_read_unlock();
+ goto out;
+ }
+ rcu_read_unlock();
+ } else {
+ int cpu;
+
+ rcu_read_lock();
+ for_each_online_cpu(cpu) {
+ struct task_struct *p;
+
+ /*
+ * Skipping the current CPU is OK even through we can be
+ * migrated at any point. The current CPU, at the point
+ * where we read raw_smp_processor_id(), is ensured to
+ * be in program order with respect to the caller
+ * thread. Therefore, we can skip this CPU from the
+ * iteration.
+ */
+ if (cpu == raw_smp_processor_id())
+ continue;
+ p = rcu_dereference(cpu_rq(cpu)->curr);
+ if (p && p->mm == mm)
+ __cpumask_set_cpu(cpu, tmpmask);
+ }
+ rcu_read_unlock();
}
- rcu_read_unlock();
preempt_disable();
- smp_call_function_many(tmpmask, ipi_mb, NULL, 1);
+ if (cpu_id >= 0)
+ smp_call_function_single(cpu_id, ipi_func, NULL, 1);
+ else
+ smp_call_function_many(tmpmask, ipi_func, NULL, 1);
preempt_enable();
- free_cpumask_var(tmpmask);
+out:
+ if (cpu_id < 0)
+ free_cpumask_var(tmpmask);
cpus_read_unlock();
/*
set_state = MEMBARRIER_STATE_PRIVATE_EXPEDITED,
ret;
- if (flags & MEMBARRIER_FLAG_SYNC_CORE) {
+ if (flags == MEMBARRIER_FLAG_SYNC_CORE) {
if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE))
return -EINVAL;
ready_state =
MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY;
+ } else if (flags == MEMBARRIER_FLAG_RSEQ) {
+ if (!IS_ENABLED(CONFIG_RSEQ))
+ return -EINVAL;
+ ready_state =
+ MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY;
+ } else {
+ WARN_ON_ONCE(flags);
}
/*
return 0;
if (flags & MEMBARRIER_FLAG_SYNC_CORE)
set_state |= MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE;
+ if (flags & MEMBARRIER_FLAG_RSEQ)
+ set_state |= MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ;
atomic_or(set_state, &mm->membarrier_state);
ret = sync_runqueues_membarrier_state(mm);
if (ret)
/**
* sys_membarrier - issue memory barriers on a set of threads
- * @cmd: Takes command values defined in enum membarrier_cmd.
- * @flags: Currently needs to be 0. For future extensions.
+ * @cmd: Takes command values defined in enum membarrier_cmd.
+ * @flags: Currently needs to be 0 for all commands other than
+ * MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ: in the latter
+ * case it can be MEMBARRIER_CMD_FLAG_CPU, indicating that @cpu_id
+ * contains the CPU on which to interrupt (= restart)
+ * the RSEQ critical section.
+ * @cpu_id: if @flags == MEMBARRIER_CMD_FLAG_CPU, indicates the cpu on which
+ * RSEQ CS should be interrupted (@cmd must be
+ * MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ).
*
* If this system call is not implemented, -ENOSYS is returned. If the
* command specified does not exist, not available on the running
* smp_mb() X O O
* sys_membarrier() O O O
*/
-SYSCALL_DEFINE2(membarrier, int, cmd, int, flags)
+SYSCALL_DEFINE3(membarrier, int, cmd, unsigned int, flags, int, cpu_id)
{
- if (unlikely(flags))
- return -EINVAL;
+ switch (cmd) {
+ case MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ:
+ if (unlikely(flags && flags != MEMBARRIER_CMD_FLAG_CPU))
+ return -EINVAL;
+ break;
+ default:
+ if (unlikely(flags))
+ return -EINVAL;
+ }
+
+ if (!(flags & MEMBARRIER_CMD_FLAG_CPU))
+ cpu_id = -1;
+
switch (cmd) {
case MEMBARRIER_CMD_QUERY:
{
case MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED:
return membarrier_register_global_expedited();
case MEMBARRIER_CMD_PRIVATE_EXPEDITED:
- return membarrier_private_expedited(0);
+ return membarrier_private_expedited(0, cpu_id);
case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED:
return membarrier_register_private_expedited(0);
case MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE:
- return membarrier_private_expedited(MEMBARRIER_FLAG_SYNC_CORE);
+ return membarrier_private_expedited(MEMBARRIER_FLAG_SYNC_CORE, cpu_id);
case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE:
return membarrier_register_private_expedited(MEMBARRIER_FLAG_SYNC_CORE);
+ case MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ:
+ return membarrier_private_expedited(MEMBARRIER_FLAG_RSEQ, cpu_id);
+ case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ:
+ return membarrier_register_private_expedited(MEMBARRIER_FLAG_RSEQ);
default:
return -EINVAL;
}
return sched_debug_enabled;
}
+#define SD_FLAG(_name, mflags) [__##_name] = { .meta_flags = mflags, .name = #_name },
+const struct sd_flag_debug sd_flag_debug[] = {
+#include <linux/sched/sd_flags.h>
+};
+#undef SD_FLAG
+
static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
struct cpumask *groupmask)
{
struct sched_group *group = sd->groups;
+ unsigned long flags = sd->flags;
+ unsigned int idx;
cpumask_clear(groupmask);
printk(KERN_ERR "ERROR: domain->groups does not contain CPU%d\n", cpu);
}
+ for_each_set_bit(idx, &flags, __SD_FLAG_CNT) {
+ unsigned int flag = BIT(idx);
+ unsigned int meta_flags = sd_flag_debug[idx].meta_flags;
+
+ if ((meta_flags & SDF_SHARED_CHILD) && sd->child &&
+ !(sd->child->flags & flag))
+ printk(KERN_ERR "ERROR: flag %s set here but not in child\n",
+ sd_flag_debug[idx].name);
+
+ if ((meta_flags & SDF_SHARED_PARENT) && sd->parent &&
+ !(sd->parent->flags & flag))
+ printk(KERN_ERR "ERROR: flag %s set here but not in parent\n",
+ sd_flag_debug[idx].name);
+ }
+
printk(KERN_DEBUG "%*s groups:", level + 1, "");
do {
if (!group) {
}
#endif /* CONFIG_SCHED_DEBUG */
+/* Generate a mask of SD flags with the SDF_NEEDS_GROUPS metaflag */
+#define SD_FLAG(name, mflags) (name * !!((mflags) & SDF_NEEDS_GROUPS)) |
+static const unsigned int SD_DEGENERATE_GROUPS_MASK =
+#include <linux/sched/sd_flags.h>
+0;
+#undef SD_FLAG
+
static int sd_degenerate(struct sched_domain *sd)
{
if (cpumask_weight(sched_domain_span(sd)) == 1)
return 1;
/* Following flags need at least 2 groups */
- if (sd->flags & (SD_BALANCE_NEWIDLE |
- SD_BALANCE_FORK |
- SD_BALANCE_EXEC |
- SD_SHARE_CPUCAPACITY |
- SD_ASYM_CPUCAPACITY |
- SD_SHARE_PKG_RESOURCES |
- SD_SHARE_POWERDOMAIN)) {
- if (sd->groups != sd->groups->next)
- return 0;
- }
+ if ((sd->flags & SD_DEGENERATE_GROUPS_MASK) &&
+ (sd->groups != sd->groups->next))
+ return 0;
/* Following flags don't use groups */
if (sd->flags & (SD_WAKE_AFFINE))
return 0;
/* Flags needing groups don't count if only 1 group in parent */
- if (parent->groups == parent->groups->next) {
- pflags &= ~(SD_BALANCE_NEWIDLE |
- SD_BALANCE_FORK |
- SD_BALANCE_EXEC |
- SD_ASYM_CPUCAPACITY |
- SD_SHARE_CPUCAPACITY |
- SD_SHARE_PKG_RESOURCES |
- SD_PREFER_SIBLING |
- SD_SHARE_POWERDOMAIN);
- if (nr_node_ids == 1)
- pflags &= ~SD_SERIALIZE;
- }
+ if (parent->groups == parent->groups->next)
+ pflags &= ~SD_DEGENERATE_GROUPS_MASK;
+
if (~cflags & pflags)
return 0;
* SD_SHARE_CPUCAPACITY - describes SMT topologies
* SD_SHARE_PKG_RESOURCES - describes shared caches
* SD_NUMA - describes NUMA topologies
- * SD_SHARE_POWERDOMAIN - describes shared power domain
*
* Odd one out, which beside describing the topology has a quirk also
* prescribes the desired behaviour that goes along with it:
(SD_SHARE_CPUCAPACITY | \
SD_SHARE_PKG_RESOURCES | \
SD_NUMA | \
- SD_ASYM_PACKING | \
- SD_SHARE_POWERDOMAIN)
+ SD_ASYM_PACKING)
static struct sched_domain *
sd_init(struct sched_domain_topology_level *tl,
*sd = (struct sched_domain){
.min_interval = sd_weight,
.max_interval = 2*sd_weight,
- .busy_factor = 32,
- .imbalance_pct = 125,
+ .busy_factor = 16,
+ .imbalance_pct = 117,
.cache_nice_tries = 0,
/* Set up domains for CPUs specified by the cpu_map: */
for_each_cpu(i, cpu_map) {
struct sched_domain_topology_level *tl;
+ int dflags = 0;
sd = NULL;
for_each_sd_topology(tl) {
- int dflags = 0;
-
if (tl == tl_asym) {
dflags |= SD_ASYM_CPUCAPACITY;
has_asym = true;
void __raise_softirq_irqoff(unsigned int nr)
{
+ lockdep_assert_irqs_disabled();
trace_softirq_raise(nr);
or_softirq_pending(1UL << nr);
}
unsigned int len;
};
-static bool stack_trace_consume_entry(void *cookie, unsigned long addr,
- bool reliable)
+static bool stack_trace_consume_entry(void *cookie, unsigned long addr)
{
struct stacktrace_cookie *c = cookie;
return c->len < c->size;
}
-static bool stack_trace_consume_entry_nosched(void *cookie, unsigned long addr,
- bool reliable)
+static bool stack_trace_consume_entry_nosched(void *cookie, unsigned long addr)
{
if (in_sched_functions(addr))
return true;
- return stack_trace_consume_entry(cookie, addr, reliable);
+ return stack_trace_consume_entry(cookie, addr);
}
/**
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/init.h>
+#include <linux/static_call.h>
+#include <linux/bug.h>
+#include <linux/smp.h>
+#include <linux/sort.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/cpu.h>
+#include <linux/processor.h>
+#include <asm/sections.h>
+
+extern struct static_call_site __start_static_call_sites[],
+ __stop_static_call_sites[];
+
+static bool static_call_initialized;
+
+/* mutex to protect key modules/sites */
+static DEFINE_MUTEX(static_call_mutex);
+
+static void static_call_lock(void)
+{
+ mutex_lock(&static_call_mutex);
+}
+
+static void static_call_unlock(void)
+{
+ mutex_unlock(&static_call_mutex);
+}
+
+static inline void *static_call_addr(struct static_call_site *site)
+{
+ return (void *)((long)site->addr + (long)&site->addr);
+}
+
+
+static inline struct static_call_key *static_call_key(const struct static_call_site *site)
+{
+ return (struct static_call_key *)
+ (((long)site->key + (long)&site->key) & ~STATIC_CALL_SITE_FLAGS);
+}
+
+/* These assume the key is word-aligned. */
+static inline bool static_call_is_init(struct static_call_site *site)
+{
+ return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_INIT;
+}
+
+static inline bool static_call_is_tail(struct static_call_site *site)
+{
+ return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_TAIL;
+}
+
+static inline void static_call_set_init(struct static_call_site *site)
+{
+ site->key = ((long)static_call_key(site) | STATIC_CALL_SITE_INIT) -
+ (long)&site->key;
+}
+
+static int static_call_site_cmp(const void *_a, const void *_b)
+{
+ const struct static_call_site *a = _a;
+ const struct static_call_site *b = _b;
+ const struct static_call_key *key_a = static_call_key(a);
+ const struct static_call_key *key_b = static_call_key(b);
+
+ if (key_a < key_b)
+ return -1;
+
+ if (key_a > key_b)
+ return 1;
+
+ return 0;
+}
+
+static void static_call_site_swap(void *_a, void *_b, int size)
+{
+ long delta = (unsigned long)_a - (unsigned long)_b;
+ struct static_call_site *a = _a;
+ struct static_call_site *b = _b;
+ struct static_call_site tmp = *a;
+
+ a->addr = b->addr - delta;
+ a->key = b->key - delta;
+
+ b->addr = tmp.addr + delta;
+ b->key = tmp.key + delta;
+}
+
+static inline void static_call_sort_entries(struct static_call_site *start,
+ struct static_call_site *stop)
+{
+ sort(start, stop - start, sizeof(struct static_call_site),
+ static_call_site_cmp, static_call_site_swap);
+}
+
+static inline bool static_call_key_has_mods(struct static_call_key *key)
+{
+ return !(key->type & 1);
+}
+
+static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)
+{
+ if (!static_call_key_has_mods(key))
+ return NULL;
+
+ return key->mods;
+}
+
+static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)
+{
+ if (static_call_key_has_mods(key))
+ return NULL;
+
+ return (struct static_call_site *)(key->type & ~1);
+}
+
+void __static_call_update(struct static_call_key *key, void *tramp, void *func)
+{
+ struct static_call_site *site, *stop;
+ struct static_call_mod *site_mod, first;
+
+ cpus_read_lock();
+ static_call_lock();
+
+ if (key->func == func)
+ goto done;
+
+ key->func = func;
+
+ arch_static_call_transform(NULL, tramp, func, false);
+
+ /*
+ * If uninitialized, we'll not update the callsites, but they still
+ * point to the trampoline and we just patched that.
+ */
+ if (WARN_ON_ONCE(!static_call_initialized))
+ goto done;
+
+ first = (struct static_call_mod){
+ .next = static_call_key_next(key),
+ .mod = NULL,
+ .sites = static_call_key_sites(key),
+ };
+
+ for (site_mod = &first; site_mod; site_mod = site_mod->next) {
+ struct module *mod = site_mod->mod;
+
+ if (!site_mod->sites) {
+ /*
+ * This can happen if the static call key is defined in
+ * a module which doesn't use it.
+ *
+ * It also happens in the has_mods case, where the
+ * 'first' entry has no sites associated with it.
+ */
+ continue;
+ }
+
+ stop = __stop_static_call_sites;
+
+#ifdef CONFIG_MODULES
+ if (mod) {
+ stop = mod->static_call_sites +
+ mod->num_static_call_sites;
+ }
+#endif
+
+ for (site = site_mod->sites;
+ site < stop && static_call_key(site) == key; site++) {
+ void *site_addr = static_call_addr(site);
+
+ if (static_call_is_init(site)) {
+ /*
+ * Don't write to call sites which were in
+ * initmem and have since been freed.
+ */
+ if (!mod && system_state >= SYSTEM_RUNNING)
+ continue;
+ if (mod && !within_module_init((unsigned long)site_addr, mod))
+ continue;
+ }
+
+ if (!kernel_text_address((unsigned long)site_addr)) {
+ WARN_ONCE(1, "can't patch static call site at %pS",
+ site_addr);
+ continue;
+ }
+
+ arch_static_call_transform(site_addr, NULL, func,
+ static_call_is_tail(site));
+ }
+ }
+
+done:
+ static_call_unlock();
+ cpus_read_unlock();
+}
+EXPORT_SYMBOL_GPL(__static_call_update);
+
+static int __static_call_init(struct module *mod,
+ struct static_call_site *start,
+ struct static_call_site *stop)
+{
+ struct static_call_site *site;
+ struct static_call_key *key, *prev_key = NULL;
+ struct static_call_mod *site_mod;
+
+ if (start == stop)
+ return 0;
+
+ static_call_sort_entries(start, stop);
+
+ for (site = start; site < stop; site++) {
+ void *site_addr = static_call_addr(site);
+
+ if ((mod && within_module_init((unsigned long)site_addr, mod)) ||
+ (!mod && init_section_contains(site_addr, 1)))
+ static_call_set_init(site);
+
+ key = static_call_key(site);
+ if (key != prev_key) {
+ prev_key = key;
+
+ /*
+ * For vmlinux (!mod) avoid the allocation by storing
+ * the sites pointer in the key itself. Also see
+ * __static_call_update()'s @first.
+ *
+ * This allows architectures (eg. x86) to call
+ * static_call_init() before memory allocation works.
+ */
+ if (!mod) {
+ key->sites = site;
+ key->type |= 1;
+ goto do_transform;
+ }
+
+ site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
+ if (!site_mod)
+ return -ENOMEM;
+
+ /*
+ * When the key has a direct sites pointer, extract
+ * that into an explicit struct static_call_mod, so we
+ * can have a list of modules.
+ */
+ if (static_call_key_sites(key)) {
+ site_mod->mod = NULL;
+ site_mod->next = NULL;
+ site_mod->sites = static_call_key_sites(key);
+
+ key->mods = site_mod;
+
+ site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
+ if (!site_mod)
+ return -ENOMEM;
+ }
+
+ site_mod->mod = mod;
+ site_mod->sites = site;
+ site_mod->next = static_call_key_next(key);
+ key->mods = site_mod;
+ }
+
+do_transform:
+ arch_static_call_transform(site_addr, NULL, key->func,
+ static_call_is_tail(site));
+ }
+
+ return 0;
+}
+
+static int addr_conflict(struct static_call_site *site, void *start, void *end)
+{
+ unsigned long addr = (unsigned long)static_call_addr(site);
+
+ if (addr <= (unsigned long)end &&
+ addr + CALL_INSN_SIZE > (unsigned long)start)
+ return 1;
+
+ return 0;
+}
+
+static int __static_call_text_reserved(struct static_call_site *iter_start,
+ struct static_call_site *iter_stop,
+ void *start, void *end)
+{
+ struct static_call_site *iter = iter_start;
+
+ while (iter < iter_stop) {
+ if (addr_conflict(iter, start, end))
+ return 1;
+ iter++;
+ }
+
+ return 0;
+}
+
+#ifdef CONFIG_MODULES
+
+static int __static_call_mod_text_reserved(void *start, void *end)
+{
+ struct module *mod;
+ int ret;
+
+ preempt_disable();
+ mod = __module_text_address((unsigned long)start);
+ WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
+ if (!try_module_get(mod))
+ mod = NULL;
+ preempt_enable();
+
+ if (!mod)
+ return 0;
+
+ ret = __static_call_text_reserved(mod->static_call_sites,
+ mod->static_call_sites + mod->num_static_call_sites,
+ start, end);
+
+ module_put(mod);
+
+ return ret;
+}
+
+static int static_call_add_module(struct module *mod)
+{
+ return __static_call_init(mod, mod->static_call_sites,
+ mod->static_call_sites + mod->num_static_call_sites);
+}
+
+static void static_call_del_module(struct module *mod)
+{
+ struct static_call_site *start = mod->static_call_sites;
+ struct static_call_site *stop = mod->static_call_sites +
+ mod->num_static_call_sites;
+ struct static_call_key *key, *prev_key = NULL;
+ struct static_call_mod *site_mod, **prev;
+ struct static_call_site *site;
+
+ for (site = start; site < stop; site++) {
+ key = static_call_key(site);
+ if (key == prev_key)
+ continue;
+
+ prev_key = key;
+
+ for (prev = &key->mods, site_mod = key->mods;
+ site_mod && site_mod->mod != mod;
+ prev = &site_mod->next, site_mod = site_mod->next)
+ ;
+
+ if (!site_mod)
+ continue;
+
+ *prev = site_mod->next;
+ kfree(site_mod);
+ }
+}
+
+static int static_call_module_notify(struct notifier_block *nb,
+ unsigned long val, void *data)
+{
+ struct module *mod = data;
+ int ret = 0;
+
+ cpus_read_lock();
+ static_call_lock();
+
+ switch (val) {
+ case MODULE_STATE_COMING:
+ ret = static_call_add_module(mod);
+ if (ret) {
+ WARN(1, "Failed to allocate memory for static calls");
+ static_call_del_module(mod);
+ }
+ break;
+ case MODULE_STATE_GOING:
+ static_call_del_module(mod);
+ break;
+ }
+
+ static_call_unlock();
+ cpus_read_unlock();
+
+ return notifier_from_errno(ret);
+}
+
+static struct notifier_block static_call_module_nb = {
+ .notifier_call = static_call_module_notify,
+};
+
+#else
+
+static inline int __static_call_mod_text_reserved(void *start, void *end)
+{
+ return 0;
+}
+
+#endif /* CONFIG_MODULES */
+
+int static_call_text_reserved(void *start, void *end)
+{
+ int ret = __static_call_text_reserved(__start_static_call_sites,
+ __stop_static_call_sites, start, end);
+
+ if (ret)
+ return ret;
+
+ return __static_call_mod_text_reserved(start, end);
+}
+
+int __init static_call_init(void)
+{
+ int ret;
+
+ if (static_call_initialized)
+ return 0;
+
+ cpus_read_lock();
+ static_call_lock();
+ ret = __static_call_init(NULL, __start_static_call_sites,
+ __stop_static_call_sites);
+ static_call_unlock();
+ cpus_read_unlock();
+
+ if (ret) {
+ pr_err("Failed to allocate memory for static_call!\n");
+ BUG();
+ }
+
+ static_call_initialized = true;
+
+#ifdef CONFIG_MODULES
+ register_module_notifier(&static_call_module_nb);
+#endif
+ return 0;
+}
+early_initcall(static_call_init);
+
+#ifdef CONFIG_STATIC_CALL_SELFTEST
+
+static int func_a(int x)
+{
+ return x+1;
+}
+
+static int func_b(int x)
+{
+ return x+2;
+}
+
+DEFINE_STATIC_CALL(sc_selftest, func_a);
+
+static struct static_call_data {
+ int (*func)(int);
+ int val;
+ int expect;
+} static_call_data [] __initdata = {
+ { NULL, 2, 3 },
+ { func_b, 2, 4 },
+ { func_a, 2, 3 }
+};
+
+static int __init test_static_call_init(void)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(static_call_data); i++ ) {
+ struct static_call_data *scd = &static_call_data[i];
+
+ if (scd->func)
+ static_call_update(sc_selftest, scd->func);
+
+ WARN_ON(static_call(sc_selftest)(scd->val) != scd->expect);
+ }
+
+ return 0;
+}
+early_initcall(test_static_call_init);
+
+#endif /* CONFIG_STATIC_CALL_SELFTEST */
/* x86 */
COND_SYSCALL(vm86old);
COND_SYSCALL(modify_ldt);
-COND_SYSCALL_COMPAT(quotactl32);
COND_SYSCALL(vm86);
COND_SYSCALL(kexec_file_load);
/* Initialize alarm bases */
alarm_bases[ALARM_REALTIME].base_clockid = CLOCK_REALTIME;
alarm_bases[ALARM_REALTIME].get_ktime = &ktime_get_real;
- alarm_bases[ALARM_REALTIME].get_timespec = ktime_get_real_ts64,
+ alarm_bases[ALARM_REALTIME].get_timespec = ktime_get_real_ts64;
alarm_bases[ALARM_BOOTTIME].base_clockid = CLOCK_BOOTTIME;
alarm_bases[ALARM_BOOTTIME].get_ktime = &ktime_get_boottime;
alarm_bases[ALARM_BOOTTIME].get_timespec = get_boottime_timespec;
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
-static struct debug_obj_descr hrtimer_debug_descr;
+static const struct debug_obj_descr hrtimer_debug_descr;
static void *hrtimer_debug_hint(void *addr)
{
}
}
-static struct debug_obj_descr hrtimer_debug_descr = {
+static const struct debug_obj_descr hrtimer_debug_descr = {
.name = "hrtimer",
.debug_hint = hrtimer_debug_hint,
.fixup_init = hrtimer_fixup_init,
* into a single 64-byte cache line.
*/
struct clock_data {
- seqcount_t seq;
+ seqcount_latch_t seq;
struct clock_read_data read_data[2];
ktime_t wrap_kt;
unsigned long rate;
int sched_clock_read_retry(unsigned int seq)
{
- return read_seqcount_retry(&cd.seq, seq);
+ return read_seqcount_latch_retry(&cd.seq, seq);
}
unsigned long long notrace sched_clock(void)
*/
static u64 notrace suspended_sched_clock_read(void)
{
- unsigned int seq = raw_read_seqcount(&cd.seq);
+ unsigned int seq = raw_read_seqcount_latch(&cd.seq);
return cd.read_data[seq & 1].epoch_cyc;
}
static struct timekeeper shadow_timekeeper;
+/* flag for if timekeeping is suspended */
+int __read_mostly timekeeping_suspended;
+
/**
* struct tk_fast - NMI safe timekeeper
* @seq: Sequence counter for protecting updates. The lowest bit
* See @update_fast_timekeeper() below.
*/
struct tk_fast {
- seqcount_raw_spinlock_t seq;
+ seqcount_latch_t seq;
struct tk_read_base base[2];
};
static u64 dummy_clock_read(struct clocksource *cs)
{
- return cycles_at_suspend;
+ if (timekeeping_suspended)
+ return cycles_at_suspend;
+ return local_clock();
}
static struct clocksource dummy_clock = {
.read = dummy_clock_read,
};
+/*
+ * Boot time initialization which allows local_clock() to be utilized
+ * during early boot when clocksources are not available. local_clock()
+ * returns nanoseconds already so no conversion is required, hence mult=1
+ * and shift=0. When the first proper clocksource is installed then
+ * the fast time keepers are updated with the correct values.
+ */
+#define FAST_TK_INIT \
+ { \
+ .clock = &dummy_clock, \
+ .mask = CLOCKSOURCE_MASK(64), \
+ .mult = 1, \
+ .shift = 0, \
+ }
+
static struct tk_fast tk_fast_mono ____cacheline_aligned = {
- .seq = SEQCNT_RAW_SPINLOCK_ZERO(tk_fast_mono.seq, &timekeeper_lock),
- .base[0] = { .clock = &dummy_clock, },
- .base[1] = { .clock = &dummy_clock, },
+ .seq = SEQCNT_LATCH_ZERO(tk_fast_mono.seq),
+ .base[0] = FAST_TK_INIT,
+ .base[1] = FAST_TK_INIT,
};
static struct tk_fast tk_fast_raw ____cacheline_aligned = {
- .seq = SEQCNT_RAW_SPINLOCK_ZERO(tk_fast_raw.seq, &timekeeper_lock),
- .base[0] = { .clock = &dummy_clock, },
- .base[1] = { .clock = &dummy_clock, },
+ .seq = SEQCNT_LATCH_ZERO(tk_fast_raw.seq),
+ .base[0] = FAST_TK_INIT,
+ .base[1] = FAST_TK_INIT,
};
-/* flag for if timekeeping is suspended */
-int __read_mostly timekeeping_suspended;
-
static inline void tk_normalize_xtime(struct timekeeper *tk)
{
while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
tk_clock_read(tkr),
tkr->cycle_last,
tkr->mask));
- } while (read_seqcount_retry(&tkf->seq, seq));
+ } while (read_seqcount_latch_retry(&tkf->seq, seq));
return now;
}
}
EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
-
/*
* See comment for __ktime_get_fast_ns() vs. timestamp ordering
*/
-static __always_inline u64 __ktime_get_real_fast_ns(struct tk_fast *tkf)
+static __always_inline u64 __ktime_get_real_fast(struct tk_fast *tkf, u64 *mono)
{
struct tk_read_base *tkr;
+ u64 basem, baser, delta;
unsigned int seq;
- u64 now;
do {
seq = raw_read_seqcount_latch(&tkf->seq);
tkr = tkf->base + (seq & 0x01);
- now = ktime_to_ns(tkr->base_real);
+ basem = ktime_to_ns(tkr->base);
+ baser = ktime_to_ns(tkr->base_real);
- now += timekeeping_delta_to_ns(tkr,
- clocksource_delta(
- tk_clock_read(tkr),
- tkr->cycle_last,
- tkr->mask));
- } while (read_seqcount_retry(&tkf->seq, seq));
+ delta = timekeeping_delta_to_ns(tkr,
+ clocksource_delta(tk_clock_read(tkr),
+ tkr->cycle_last, tkr->mask));
+ } while (read_seqcount_latch_retry(&tkf->seq, seq));
- return now;
+ if (mono)
+ *mono = basem + delta;
+ return baser + delta;
}
/**
*/
u64 ktime_get_real_fast_ns(void)
{
- return __ktime_get_real_fast_ns(&tk_fast_mono);
+ return __ktime_get_real_fast(&tk_fast_mono, NULL);
}
EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns);
+/**
+ * ktime_get_fast_timestamps: - NMI safe timestamps
+ * @snapshot: Pointer to timestamp storage
+ *
+ * Stores clock monotonic, boottime and realtime timestamps.
+ *
+ * Boot time is a racy access on 32bit systems if the sleep time injection
+ * happens late during resume and not in timekeeping_resume(). That could
+ * be avoided by expanding struct tk_read_base with boot offset for 32bit
+ * and adding more overhead to the update. As this is a hard to observe
+ * once per resume event which can be filtered with reasonable effort using
+ * the accurate mono/real timestamps, it's probably not worth the trouble.
+ *
+ * Aside of that it might be possible on 32 and 64 bit to observe the
+ * following when the sleep time injection happens late:
+ *
+ * CPU 0 CPU 1
+ * timekeeping_resume()
+ * ktime_get_fast_timestamps()
+ * mono, real = __ktime_get_real_fast()
+ * inject_sleep_time()
+ * update boot offset
+ * boot = mono + bootoffset;
+ *
+ * That means that boot time already has the sleep time adjustment, but
+ * real time does not. On the next readout both are in sync again.
+ *
+ * Preventing this for 64bit is not really feasible without destroying the
+ * careful cache layout of the timekeeper because the sequence count and
+ * struct tk_read_base would then need two cache lines instead of one.
+ *
+ * Access to the time keeper clock source is disabled accross the innermost
+ * steps of suspend/resume. The accessors still work, but the timestamps
+ * are frozen until time keeping is resumed which happens very early.
+ *
+ * For regular suspend/resume there is no observable difference vs. sched
+ * clock, but it might affect some of the nasty low level debug printks.
+ *
+ * OTOH, access to sched clock is not guaranteed accross suspend/resume on
+ * all systems either so it depends on the hardware in use.
+ *
+ * If that turns out to be a real problem then this could be mitigated by
+ * using sched clock in a similar way as during early boot. But it's not as
+ * trivial as on early boot because it needs some careful protection
+ * against the clock monotonic timestamp jumping backwards on resume.
+ */
+void ktime_get_fast_timestamps(struct ktime_timestamps *snapshot)
+{
+ struct timekeeper *tk = &tk_core.timekeeper;
+
+ snapshot->real = __ktime_get_real_fast(&tk_fast_mono, &snapshot->mono);
+ snapshot->boot = snapshot->mono + ktime_to_ns(data_race(tk->offs_boot));
+}
+
/**
* halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
* @tk: Timekeeper to snapshot.
#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
-static struct debug_obj_descr timer_debug_descr;
+static const struct debug_obj_descr timer_debug_descr;
static void *timer_debug_hint(void *addr)
{
}
}
-static struct debug_obj_descr timer_debug_descr = {
+static const struct debug_obj_descr timer_debug_descr = {
.name = "timer_list",
.debug_hint = timer_debug_hint,
.is_static_object = timer_is_static_object,
{
timer->entry.pprev = NULL;
timer->function = func;
+ if (WARN_ON_ONCE(flags & ~TIMER_INIT_FLAGS))
+ flags &= TIMER_INIT_FLAGS;
timer->flags = flags | raw_smp_processor_id();
lockdep_init_map(&timer->lockdep_map, name, key, 0);
}
* and scsi-generic block devices we create a temporary new debugfs
* directory that will be removed once the trace ends.
*/
- if (bdev && bdev == bdev->bd_contains)
+ if (bdev && !bdev_is_partition(bdev))
dir = q->debugfs_dir;
else
bt->dir = dir = debugfs_create_dir(buts->name, blk_debugfs_root);
struct device_attribute *attr,
char *buf)
{
- struct hd_struct *p = dev_to_part(dev);
+ struct block_device *bdev = bdget_part(dev_to_part(dev));
struct request_queue *q;
- struct block_device *bdev;
struct blk_trace *bt;
ssize_t ret = -ENXIO;
- bdev = bdget(part_devt(p));
if (bdev == NULL)
goto out;
{
struct block_device *bdev;
struct request_queue *q;
- struct hd_struct *p;
struct blk_trace *bt;
u64 value;
ssize_t ret = -EINVAL;
goto out;
ret = -ENXIO;
-
- p = dev_to_part(dev);
- bdev = bdget(part_devt(p));
+ bdev = bdget_part(dev_to_part(dev));
if (bdev == NULL)
goto out;
{
struct bpf_trace_module *btm, *tmp;
struct module *mod = module;
+ int ret = 0;
if (mod->num_bpf_raw_events == 0 ||
(op != MODULE_STATE_COMING && op != MODULE_STATE_GOING))
- return 0;
+ goto out;
mutex_lock(&bpf_module_mutex);
if (btm) {
btm->module = module;
list_add(&btm->list, &bpf_trace_modules);
+ } else {
+ ret = -ENOMEM;
}
break;
case MODULE_STATE_GOING:
mutex_unlock(&bpf_module_mutex);
- return 0;
+out:
+ return notifier_from_errno(ret);
}
static struct notifier_block bpf_module_nb = {
{
int bit;
- if ((op->flags & FTRACE_OPS_FL_RCU) && !rcu_is_watching())
- return;
-
bit = trace_test_and_set_recursion(TRACE_LIST_START, TRACE_LIST_MAX);
if (bit < 0)
return;
preempt_disable_notrace();
- op->func(ip, parent_ip, op, regs);
+ if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching())
+ op->func(ip, parent_ip, op, regs);
preempt_enable_notrace();
trace_clear_recursion(bit);
if (iter->ent && iter->ent != iter->temp) {
if ((!iter->temp || iter->temp_size < iter->ent_size) &&
!WARN_ON_ONCE(iter->temp == static_temp_buf)) {
- kfree(iter->temp);
- iter->temp = kmalloc(iter->ent_size, GFP_KERNEL);
- if (!iter->temp)
+ void *temp;
+ temp = kmalloc(iter->ent_size, GFP_KERNEL);
+ if (!temp)
return NULL;
+ kfree(iter->temp);
+ iter->temp = temp;
+ iter->temp_size = iter->ent_size;
}
memcpy(iter->temp, iter->ent, iter->ent_size);
- iter->temp_size = iter->ent_size;
iter->ent = iter->temp;
}
entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
break;
}
- return 0;
+ return NOTIFY_OK;
}
static struct notifier_block trace_module_nb = {
mutex_unlock(&trace_types_lock);
mutex_unlock(&event_mutex);
- return 0;
+ return NOTIFY_OK;
}
static struct notifier_block trace_module_nb = {
static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
struct module *mod)
{
- int len = strlen(mod->name);
+ int len = strlen(module_name(mod));
const char *name = trace_kprobe_symbol(tk);
- return strncmp(mod->name, name, len) == 0 && name[len] == ':';
+
+ return strncmp(module_name(mod), name, len) == 0 && name[len] == ':';
}
static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
if (ret)
pr_warn("Failed to re-register probe %s on %s: %d\n",
trace_probe_name(&tk->tp),
- mod->name, ret);
+ module_name(mod), ret);
}
}
mutex_unlock(&event_mutex);
if (val == MODULE_STATE_COMING)
hold_module_trace_bprintk_format(start, end);
}
- return 0;
+ return NOTIFY_OK;
}
/*
module_trace_bprintk_format_notify(struct notifier_block *self,
unsigned long val, void *data)
{
- return 0;
+ return NOTIFY_OK;
}
static inline const char **
find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos)
return old;
}
+static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs, bool sync)
+{
+ void *func = tp->iterator;
+
+ /* Synthetic events do not have static call sites */
+ if (!tp->static_call_key)
+ return;
+
+ if (!tp_funcs[1].func) {
+ func = tp_funcs[0].func;
+ /*
+ * If going from the iterator back to a single caller,
+ * we need to synchronize with __DO_TRACE to make sure
+ * that the data passed to the callback is the one that
+ * belongs to that callback.
+ */
+ if (sync)
+ tracepoint_synchronize_unregister();
+ }
+
+ __static_call_update(tp->static_call_key, tp->static_call_tramp, func);
+}
+
/*
* Add the probe function to a tracepoint.
*/
* include/linux/tracepoint.h using rcu_dereference_sched().
*/
rcu_assign_pointer(tp->funcs, tp_funcs);
- if (!static_key_enabled(&tp->key))
- static_key_slow_inc(&tp->key);
+ tracepoint_update_call(tp, tp_funcs, false);
+ static_key_enable(&tp->key);
+
release_probes(old);
return 0;
}
if (tp->unregfunc && static_key_enabled(&tp->key))
tp->unregfunc();
- if (static_key_enabled(&tp->key))
- static_key_slow_dec(&tp->key);
+ static_key_disable(&tp->key);
+ rcu_assign_pointer(tp->funcs, tp_funcs);
+ } else {
+ rcu_assign_pointer(tp->funcs, tp_funcs);
+ tracepoint_update_call(tp, tp_funcs,
+ tp_funcs[0].func != old[0].func);
}
- rcu_assign_pointer(tp->funcs, tp_funcs);
release_probes(old);
return 0;
}
case MODULE_STATE_UNFORMED:
break;
}
- return ret;
+ return notifier_from_errno(ret);
}
static struct notifier_block tracepoint_module_nb = {
#include <linux/cred.h>
#include <linux/file.h>
#include <linux/fdtable.h>
+#include <linux/fs_struct.h>
#include <linux/workqueue.h>
#include <linux/security.h>
#include <linux/mount.h>
flush_signal_handlers(current, 1);
spin_unlock_irq(¤t->sighand->siglock);
+ /*
+ * Initial kernel threads share ther FS with init, in order to
+ * get the init root directory. But we've now created a new
+ * thread that is going to execve a user process and has its own
+ * 'struct fs_struct'. Reset umask to the default.
+ */
+ current->fs->umask = 0022;
+
/*
* Our parent (unbound workqueue) runs with elevated scheduling
* priority. Avoid propagating that into the userspace child.
#ifdef CONFIG_DEBUG_OBJECTS_WORK
-static struct debug_obj_descr work_debug_descr;
+static const struct debug_obj_descr work_debug_descr;
static void *work_debug_hint(void *addr)
{
}
}
-static struct debug_obj_descr work_debug_descr = {
+static const struct debug_obj_descr work_debug_descr = {
.name = "work_struct",
.debug_hint = work_debug_hint,
.is_static_object = work_is_static_object,
config ARCH_HAS_UACCESS_FLUSHCACHE
bool
-config ARCH_HAS_UACCESS_MCSAFE
+# arch has a concept of a recoverable synchronous exception due to a
+# memory-read error like x86 machine-check or ARM data-abort, and
+# implements copy_mc_to_{user,kernel} to abort and report
+# 'bytes-transferred' if that exception fires when accessing the source
+# buffer.
+config ARCH_HAS_COPY_MC
bool
# Temporary. Goes away when all archs are cleaned up
if KCSAN
+# Compiler capabilities that should not fail the test if they are unavailable.
+config CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE
+ def_bool (CC_IS_CLANG && $(cc-option,-fsanitize=thread -mllvm -tsan-compound-read-before-write=1)) || \
+ (CC_IS_GCC && $(cc-option,-fsanitize=thread --param tsan-compound-read-before-write=1))
+
config KCSAN_VERBOSE
bool "Show verbose reports with more information about system state"
depends on PROVE_LOCKING
static struct xbc_node *last_parent __initdata;
static const char *xbc_err_msg __initdata;
static int xbc_err_pos __initdata;
+static int open_brace[XBC_DEPTH_MAX] __initdata;
+static int brace_index __initdata;
static int __init xbc_parse_error(const char *msg, const char *p)
{
return p;
}
-static int __init __xbc_open_brace(void)
+static int __init __xbc_open_brace(char *p)
{
- /* Mark the last key as open brace */
- last_parent->next = XBC_NODE_MAX;
+ /* Push the last key as open brace */
+ open_brace[brace_index++] = xbc_node_index(last_parent);
+ if (brace_index >= XBC_DEPTH_MAX)
+ return xbc_parse_error("Exceed max depth of braces", p);
return 0;
}
static int __init __xbc_close_brace(char *p)
{
- struct xbc_node *node;
-
- if (!last_parent || last_parent->next != XBC_NODE_MAX)
+ brace_index--;
+ if (!last_parent || brace_index < 0 ||
+ (open_brace[brace_index] != xbc_node_index(last_parent)))
return xbc_parse_error("Unexpected closing brace", p);
- node = last_parent;
- node->next = 0;
- do {
- node = xbc_node_get_parent(node);
- } while (node && node->next != XBC_NODE_MAX);
- last_parent = node;
+ if (brace_index == 0)
+ last_parent = NULL;
+ else
+ last_parent = &xbc_nodes[open_brace[brace_index - 1]];
return 0;
}
break;
}
if (strchr(",;\n#}", c)) {
- v = strim(v);
*p++ = '\0';
+ v = strim(v);
break;
}
}
return ret;
*k = n;
- return __xbc_open_brace();
+ return __xbc_open_brace(n - 1);
}
static int __init xbc_close_brace(char **k, char *n)
int i, depth, len, wlen;
struct xbc_node *n, *m;
+ /* Brace closing */
+ if (brace_index) {
+ n = &xbc_nodes[open_brace[brace_index]];
+ return xbc_parse_error("Brace is not closed",
+ xbc_node_get_data(n));
+ }
+
/* Empty tree */
if (xbc_node_num == 0) {
xbc_parse_error("Empty config", xbc_data);
xbc_node_num = 0;
memblock_free(__pa(xbc_nodes), sizeof(struct xbc_node) * XBC_NODE_MAX);
xbc_nodes = NULL;
+ brace_index = 0;
}
/**
}
EXPORT_SYMBOL(ip_compute_csum);
-/*
- * copy from ds while checksumming, otherwise like csum_partial
- */
-__wsum
-csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
-{
- memcpy(dst, src, len);
- return csum_partial(dst, len, sum);
-}
-EXPORT_SYMBOL(csum_partial_copy_nocheck);
-
#ifndef csum_tcpudp_nofold
static inline u32 from64to32(u64 x)
{
poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
}
- flags = SG_MITER_TO_SG;
- if (!preemptible())
- flags |= SG_MITER_ATOMIC;
+ flags = SG_MITER_TO_SG | SG_MITER_ATOMIC;
sg_miter_start(&miter, src, sg_nents(src), flags);
#include <linux/slab.h>
#include <linux/hash.h>
#include <linux/kmemleak.h>
+#include <linux/cpu.h>
#define ODEBUG_HASH_BITS 14
#define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS)
= ODEBUG_POOL_SIZE;
static int debug_objects_pool_min_level __read_mostly
= ODEBUG_POOL_MIN_LEVEL;
-static struct debug_obj_descr *descr_test __read_mostly;
+static const struct debug_obj_descr *descr_test __read_mostly;
static struct kmem_cache *obj_cache __read_mostly;
/*
* Must be called with interrupts disabled.
*/
static struct debug_obj *
-alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
+alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr)
{
struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool);
struct debug_obj *obj;
}
}
+#ifdef CONFIG_HOTPLUG_CPU
+static int object_cpu_offline(unsigned int cpu)
+{
+ struct debug_percpu_free *percpu_pool;
+ struct hlist_node *tmp;
+ struct debug_obj *obj;
+
+ /* Remote access is safe as the CPU is dead already */
+ percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu);
+ hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) {
+ hlist_del(&obj->node);
+ kmem_cache_free(obj_cache, obj);
+ }
+ percpu_pool->obj_free = 0;
+
+ return 0;
+}
+#endif
+
/*
* We run out of memory. That means we probably have tons of objects
* allocated.
static void debug_print_object(struct debug_obj *obj, char *msg)
{
- struct debug_obj_descr *descr = obj->descr;
+ const struct debug_obj_descr *descr = obj->descr;
static int limit;
if (limit < 5 && descr != descr_test) {
}
static void
-__debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
+__debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack)
{
enum debug_obj_state state;
bool check_stack = false;
* @addr: address of the object
* @descr: pointer to an object specific debug description structure
*/
-void debug_object_init(void *addr, struct debug_obj_descr *descr)
+void debug_object_init(void *addr, const struct debug_obj_descr *descr)
{
if (!debug_objects_enabled)
return;
* @addr: address of the object
* @descr: pointer to an object specific debug description structure
*/
-void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
+void debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr)
{
if (!debug_objects_enabled)
return;
* @descr: pointer to an object specific debug description structure
* Returns 0 for success, -EINVAL for check failed.
*/
-int debug_object_activate(void *addr, struct debug_obj_descr *descr)
+int debug_object_activate(void *addr, const struct debug_obj_descr *descr)
{
enum debug_obj_state state;
struct debug_bucket *db;
* @addr: address of the object
* @descr: pointer to an object specific debug description structure
*/
-void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
+void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr)
{
struct debug_bucket *db;
struct debug_obj *obj;
* @addr: address of the object
* @descr: pointer to an object specific debug description structure
*/
-void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
+void debug_object_destroy(void *addr, const struct debug_obj_descr *descr)
{
enum debug_obj_state state;
struct debug_bucket *db;
* @addr: address of the object
* @descr: pointer to an object specific debug description structure
*/
-void debug_object_free(void *addr, struct debug_obj_descr *descr)
+void debug_object_free(void *addr, const struct debug_obj_descr *descr)
{
enum debug_obj_state state;
struct debug_bucket *db;
* @addr: address of the object
* @descr: pointer to an object specific debug description structure
*/
-void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
+void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr)
{
struct debug_bucket *db;
struct debug_obj *obj;
* @next: state to move to if expected state is found
*/
void
-debug_object_active_state(void *addr, struct debug_obj_descr *descr,
+debug_object_active_state(void *addr, const struct debug_obj_descr *descr,
unsigned int expect, unsigned int next)
{
struct debug_bucket *db;
static void __debug_check_no_obj_freed(const void *address, unsigned long size)
{
unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
- struct debug_obj_descr *descr;
+ const struct debug_obj_descr *descr;
enum debug_obj_state state;
struct debug_bucket *db;
struct hlist_node *tmp;
unsigned long dummy2[3];
};
-static __initdata struct debug_obj_descr descr_type_test;
+static __initconst const struct debug_obj_descr descr_type_test;
static bool __init is_static_object(void *addr)
{
return res;
}
-static __initdata struct debug_obj_descr descr_type_test = {
+static __initconst const struct debug_obj_descr descr_type_test = {
.name = "selftest",
.is_static_object = is_static_object,
.fixup_init = fixup_init,
} else
debug_objects_selftest();
+#ifdef CONFIG_HOTPLUG_CPU
+ cpuhp_setup_state_nocalls(CPUHP_DEBUG_OBJ_DEAD, "object:offline", NULL,
+ object_cpu_offline);
+#endif
+
/*
* Increase the thresholds for allocating and freeing objects
* according to the number of possible CPUs available in the system.
#define FONTDATAMAX 9216
-static const unsigned char fontdata_10x18[FONTDATAMAX] = {
-
+static struct font_data fontdata_10x18 = {
+ { 0, 0, FONTDATAMAX, 0 }, {
/* 0 0x00 '^@' */
0x00, 0x00, /* 0000000000 */
0x00, 0x00, /* 0000000000 */
0x00, 0x00, /* 0000000000 */
0x00, 0x00, /* 0000000000 */
0x00, 0x00, /* 0000000000 */
-
-};
+} };
const struct font_desc font_10x18 = {
.name = "10x18",
.width = 10,
.height = 18,
- .data = fontdata_10x18,
+ .data = fontdata_10x18.data,
#ifdef __sparc__
.pref = 5,
#else
// SPDX-License-Identifier: GPL-2.0
#include <linux/font.h>
-static const unsigned char fontdata_6x10[] = {
+#define FONTDATAMAX 2560
+static struct font_data fontdata_6x10 = {
+ { 0, 0, FONTDATAMAX, 0 }, {
/* 0 0x00 '^@' */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
-
-};
+} };
const struct font_desc font_6x10 = {
.idx = FONT6x10_IDX,
.name = "6x10",
.width = 6,
.height = 10,
- .data = fontdata_6x10,
+ .data = fontdata_6x10.data,
.pref = 0,
};
#define FONTDATAMAX (11*256)
-static const unsigned char fontdata_6x11[FONTDATAMAX] = {
-
+static struct font_data fontdata_6x11 = {
+ { 0, 0, FONTDATAMAX, 0 }, {
/* 0 0x00 '^@' */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
-
-};
+} };
const struct font_desc font_vga_6x11 = {
.name = "ProFont6x11",
.width = 6,
.height = 11,
- .data = fontdata_6x11,
+ .data = fontdata_6x11.data,
/* Try avoiding this font if possible unless on MAC */
.pref = -2000,
};
#define FONTDATAMAX 3584
-static const unsigned char fontdata_7x14[FONTDATAMAX] = {
-
+static struct font_data fontdata_7x14 = {
+ { 0, 0, FONTDATAMAX, 0 }, {
/* 0 0x00 '^@' */
0x00, /* 0000000 */
0x00, /* 0000000 */
0x00, /* 0000000 */
0x00, /* 0000000 */
0x00, /* 0000000 */
-
-};
+} };
const struct font_desc font_7x14 = {
.name = "7x14",
.width = 7,
.height = 14,
- .data = fontdata_7x14,
+ .data = fontdata_7x14.data,
.pref = 0,
};
#define FONTDATAMAX 4096
-static const unsigned char fontdata_8x16[FONTDATAMAX] = {
-
+static struct font_data fontdata_8x16 = {
+ { 0, 0, FONTDATAMAX, 0 }, {
/* 0 0x00 '^@' */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
-
-};
+} };
const struct font_desc font_vga_8x16 = {
.name = "VGA8x16",
.width = 8,
.height = 16,
- .data = fontdata_8x16,
+ .data = fontdata_8x16.data,
.pref = 0,
};
EXPORT_SYMBOL(font_vga_8x16);
#define FONTDATAMAX 2048
-static const unsigned char fontdata_8x8[FONTDATAMAX] = {
-
+static struct font_data fontdata_8x8 = {
+ { 0, 0, FONTDATAMAX, 0 }, {
/* 0 0x00 '^@' */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
-
-};
+} };
const struct font_desc font_vga_8x8 = {
.name = "VGA8x8",
.width = 8,
.height = 8,
- .data = fontdata_8x8,
+ .data = fontdata_8x8.data,
.pref = 0,
};
#include <linux/font.h>
-static const unsigned char acorndata_8x8[] = {
+#define FONTDATAMAX 2048
+
+static struct font_data acorndata_8x8 = {
+{ 0, 0, FONTDATAMAX, 0 }, {
/* 00 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* ^@ */
/* 01 */ 0x7e, 0x81, 0xa5, 0x81, 0xbd, 0x99, 0x81, 0x7e, /* ^A */
/* 02 */ 0x7e, 0xff, 0xbd, 0xff, 0xc3, 0xe7, 0xff, 0x7e, /* ^B */
/* FD */ 0x38, 0x04, 0x18, 0x20, 0x3c, 0x00, 0x00, 0x00,
/* FE */ 0x00, 0x00, 0x3c, 0x3c, 0x3c, 0x3c, 0x00, 0x00,
/* FF */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
-};
+} };
const struct font_desc font_acorn_8x8 = {
.idx = ACORN8x8_IDX,
.name = "Acorn8x8",
.width = 8,
.height = 8,
- .data = acorndata_8x8,
+ .data = acorndata_8x8.data,
#ifdef CONFIG_ARCH_ACORN
.pref = 20,
#else
#define FONTDATAMAX 1536
-static const unsigned char fontdata_mini_4x6[FONTDATAMAX] = {
-
+static struct font_data fontdata_mini_4x6 = {
+ { 0, 0, FONTDATAMAX, 0 }, {
/*{*/
/* Char 0: ' ' */
0xee, /*= [*** ] */
0xee, /*= [*** ] */
0x00, /*= [ ] */
/*}*/
-};
+} };
const struct font_desc font_mini_4x6 = {
.idx = MINI4x6_IDX,
.name = "MINI4x6",
.width = 4,
.height = 6,
- .data = fontdata_mini_4x6,
+ .data = fontdata_mini_4x6.data,
.pref = 3,
};
#define FONTDATAMAX 2048
-static const unsigned char fontdata_pearl8x8[FONTDATAMAX] = {
-
+static struct font_data fontdata_pearl8x8 = {
+ { 0, 0, FONTDATAMAX, 0 }, {
/* 0 0x00 '^@' */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
0x00, /* 00000000 */
-
-};
+} };
const struct font_desc font_pearl_8x8 = {
.idx = PEARL8x8_IDX,
.name = "PEARL8x8",
.width = 8,
.height = 8,
- .data = fontdata_pearl8x8,
+ .data = fontdata_pearl8x8.data,
.pref = 2,
};
#define FONTDATAMAX 11264
-static const unsigned char fontdata_sun12x22[FONTDATAMAX] = {
-
+static struct font_data fontdata_sun12x22 = {
+ { 0, 0, FONTDATAMAX, 0 }, {
/* 0 0x00 '^@' */
0x00, 0x00, /* 000000000000 */
0x00, 0x00, /* 000000000000 */
0x00, 0x00, /* 000000000000 */
0x00, 0x00, /* 000000000000 */
0x00, 0x00, /* 000000000000 */
-
-};
+} };
const struct font_desc font_sun_12x22 = {
.name = "SUN12x22",
.width = 12,
.height = 22,
- .data = fontdata_sun12x22,
+ .data = fontdata_sun12x22.data,
#ifdef __sparc__
.pref = 5,
#else
#define FONTDATAMAX 4096
-static const unsigned char fontdata_sun8x16[FONTDATAMAX] = {
+static struct font_data fontdata_sun8x16 = {
+{ 0, 0, FONTDATAMAX, 0 }, {
/* */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
/* */ 0x00,0x00,0x7e,0x81,0xa5,0x81,0x81,0xbd,0x99,0x81,0x81,0x7e,0x00,0x00,0x00,0x00,
/* */ 0x00,0x00,0x7e,0xff,0xdb,0xff,0xff,0xc3,0xe7,0xff,0xff,0x7e,0x00,0x00,0x00,0x00,
/* */ 0x00,0x70,0xd8,0x30,0x60,0xc8,0xf8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
/* */ 0x00,0x00,0x00,0x00,0x7c,0x7c,0x7c,0x7c,0x7c,0x7c,0x7c,0x00,0x00,0x00,0x00,0x00,
/* */ 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-};
+} };
const struct font_desc font_sun_8x16 = {
.idx = SUN8x16_IDX,
.name = "SUN8x16",
.width = 8,
.height = 16,
- .data = fontdata_sun8x16,
+ .data = fontdata_sun8x16.data,
#ifdef __sparc__
.pref = 10,
#else
#define FONTDATAMAX 16384
-static const unsigned char fontdata_ter16x32[FONTDATAMAX] = {
-
+static struct font_data fontdata_ter16x32 = {
+ { 0, 0, FONTDATAMAX, 0 }, {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x7f, 0xfc, 0x7f, 0xfc,
0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c, 0x70, 0x1c,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 255 */
-
-};
+} };
const struct font_desc font_ter_16x32 = {
.name = "TER16x32",
.width = 16,
.height = 32,
- .data = fontdata_ter16x32,
+ .data = fontdata_ter16x32.data,
#ifdef __sparc__
.pref = 5,
#else
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/splice.h>
+#include <linux/compat.h>
#include <net/checksum.h>
#include <linux/scatterlist.h>
#include <linux/instrumented.h>
static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
__wsum sum, size_t off)
{
- __wsum next = csum_partial_copy_nocheck(from, to, len, 0);
+ __wsum next = csum_partial_copy_nocheck(from, to, len);
return csum_block_add(sum, next, off);
}
}
EXPORT_SYMBOL(_copy_to_iter);
-#ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE
-static int copyout_mcsafe(void __user *to, const void *from, size_t n)
+#ifdef CONFIG_ARCH_HAS_COPY_MC
+static int copyout_mc(void __user *to, const void *from, size_t n)
{
if (access_ok(to, n)) {
instrument_copy_to_user(to, from, n);
- n = copy_to_user_mcsafe((__force void *) to, from, n);
+ n = copy_mc_to_user((__force void *) to, from, n);
}
return n;
}
-static unsigned long memcpy_mcsafe_to_page(struct page *page, size_t offset,
+static unsigned long copy_mc_to_page(struct page *page, size_t offset,
const char *from, size_t len)
{
unsigned long ret;
char *to;
to = kmap_atomic(page);
- ret = memcpy_mcsafe(to + offset, from, len);
+ ret = copy_mc_to_kernel(to + offset, from, len);
kunmap_atomic(to);
return ret;
}
-static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes,
+static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,
struct iov_iter *i)
{
struct pipe_inode_info *pipe = i->pipe;
size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
unsigned long rem;
- rem = memcpy_mcsafe_to_page(pipe->bufs[i_head & p_mask].page,
+ rem = copy_mc_to_page(pipe->bufs[i_head & p_mask].page,
off, addr, chunk);
i->head = i_head;
i->iov_offset = off + chunk - rem;
}
/**
- * _copy_to_iter_mcsafe - copy to user with source-read error exception handling
+ * _copy_mc_to_iter - copy to iter with source memory error exception handling
* @addr: source kernel address
* @bytes: total transfer length
* @iter: destination iterator
*
- * The pmem driver arranges for filesystem-dax to use this facility via
- * dax_copy_to_iter() for protecting read/write to persistent memory.
- * Unless / until an architecture can guarantee identical performance
- * between _copy_to_iter_mcsafe() and _copy_to_iter() it would be a
- * performance regression to switch more users to the mcsafe version.
+ * The pmem driver deploys this for the dax operation
+ * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
+ * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
+ * successfully copied.
*
- * Otherwise, the main differences between this and typical _copy_to_iter().
+ * The main differences between this and typical _copy_to_iter().
*
* * Typical tail/residue handling after a fault retries the copy
* byte-by-byte until the fault happens again. Re-triggering machine
* * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
* Compare to copy_to_iter() where only ITER_IOVEC attempts might return
* a short copy.
- *
- * See MCSAFE_TEST for self-test.
*/
-size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i)
+size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
{
const char *from = addr;
unsigned long rem, curr_addr, s_addr = (unsigned long) addr;
if (unlikely(iov_iter_is_pipe(i)))
- return copy_pipe_to_iter_mcsafe(addr, bytes, i);
+ return copy_mc_pipe_to_iter(addr, bytes, i);
if (iter_is_iovec(i))
might_fault();
iterate_and_advance(i, bytes, v,
- copyout_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
+ copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len,
+ v.iov_len),
({
- rem = memcpy_mcsafe_to_page(v.bv_page, v.bv_offset,
- (from += v.bv_len) - v.bv_len, v.bv_len);
+ rem = copy_mc_to_page(v.bv_page, v.bv_offset,
+ (from += v.bv_len) - v.bv_len, v.bv_len);
if (rem) {
curr_addr = (unsigned long) from;
bytes = curr_addr - s_addr - rem;
}
}),
({
- rem = memcpy_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len,
- v.iov_len);
+ rem = copy_mc_to_kernel(v.iov_base, (from += v.iov_len)
+ - v.iov_len, v.iov_len);
if (rem) {
curr_addr = (unsigned long) from;
bytes = curr_addr - s_addr - rem;
return bytes;
}
-EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe);
-#endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */
+EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
+#endif /* CONFIG_ARCH_HAS_COPY_MC */
size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
{
return 0;
}
iterate_and_advance(i, bytes, v, ({
- int err = 0;
next = csum_and_copy_from_user(v.iov_base,
(to += v.iov_len) - v.iov_len,
- v.iov_len, 0, &err);
- if (!err) {
+ v.iov_len);
+ if (next) {
sum = csum_block_add(sum, next, off);
off += v.iov_len;
}
- err ? v.iov_len : 0;
+ next ? 0 : v.iov_len;
}), ({
char *p = kmap_atomic(v.bv_page);
sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
if (unlikely(i->count < bytes))
return false;
iterate_all_kinds(i, bytes, v, ({
- int err = 0;
next = csum_and_copy_from_user(v.iov_base,
(to += v.iov_len) - v.iov_len,
- v.iov_len, 0, &err);
- if (err)
+ v.iov_len);
+ if (!next)
return false;
sum = csum_block_add(sum, next, off);
off += v.iov_len;
return 0;
}
iterate_and_advance(i, bytes, v, ({
- int err = 0;
next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
v.iov_base,
- v.iov_len, 0, &err);
- if (!err) {
+ v.iov_len);
+ if (next) {
sum = csum_block_add(sum, next, off);
off += v.iov_len;
}
- err ? v.iov_len : 0;
+ next ? 0 : v.iov_len;
}), ({
char *p = kmap_atomic(v.bv_page);
sum = csum_and_memcpy(p + v.bv_offset,
}
EXPORT_SYMBOL(dup_iter);
+static int copy_compat_iovec_from_user(struct iovec *iov,
+ const struct iovec __user *uvec, unsigned long nr_segs)
+{
+ const struct compat_iovec __user *uiov =
+ (const struct compat_iovec __user *)uvec;
+ int ret = -EFAULT, i;
+
+ if (!user_access_begin(uvec, nr_segs * sizeof(*uvec)))
+ return -EFAULT;
+
+ for (i = 0; i < nr_segs; i++) {
+ compat_uptr_t buf;
+ compat_ssize_t len;
+
+ unsafe_get_user(len, &uiov[i].iov_len, uaccess_end);
+ unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end);
+
+ /* check for compat_size_t not fitting in compat_ssize_t .. */
+ if (len < 0) {
+ ret = -EINVAL;
+ goto uaccess_end;
+ }
+ iov[i].iov_base = compat_ptr(buf);
+ iov[i].iov_len = len;
+ }
+
+ ret = 0;
+uaccess_end:
+ user_access_end();
+ return ret;
+}
+
+static int copy_iovec_from_user(struct iovec *iov,
+ const struct iovec __user *uvec, unsigned long nr_segs)
+{
+ unsigned long seg;
+
+ if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec)))
+ return -EFAULT;
+ for (seg = 0; seg < nr_segs; seg++) {
+ if ((ssize_t)iov[seg].iov_len < 0)
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+struct iovec *iovec_from_user(const struct iovec __user *uvec,
+ unsigned long nr_segs, unsigned long fast_segs,
+ struct iovec *fast_iov, bool compat)
+{
+ struct iovec *iov = fast_iov;
+ int ret;
+
+ /*
+ * SuS says "The readv() function *may* fail if the iovcnt argument was
+ * less than or equal to 0, or greater than {IOV_MAX}. Linux has
+ * traditionally returned zero for zero segments, so...
+ */
+ if (nr_segs == 0)
+ return iov;
+ if (nr_segs > UIO_MAXIOV)
+ return ERR_PTR(-EINVAL);
+ if (nr_segs > fast_segs) {
+ iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
+ if (!iov)
+ return ERR_PTR(-ENOMEM);
+ }
+
+ if (compat)
+ ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);
+ else
+ ret = copy_iovec_from_user(iov, uvec, nr_segs);
+ if (ret) {
+ if (iov != fast_iov)
+ kfree(iov);
+ return ERR_PTR(ret);
+ }
+
+ return iov;
+}
+
+ssize_t __import_iovec(int type, const struct iovec __user *uvec,
+ unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
+ struct iov_iter *i, bool compat)
+{
+ ssize_t total_len = 0;
+ unsigned long seg;
+ struct iovec *iov;
+
+ iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat);
+ if (IS_ERR(iov)) {
+ *iovp = NULL;
+ return PTR_ERR(iov);
+ }
+
+ /*
+ * According to the Single Unix Specification we should return EINVAL if
+ * an element length is < 0 when cast to ssize_t or if the total length
+ * would overflow the ssize_t return value of the system call.
+ *
+ * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
+ * overflow case.
+ */
+ for (seg = 0; seg < nr_segs; seg++) {
+ ssize_t len = (ssize_t)iov[seg].iov_len;
+
+ if (!access_ok(iov[seg].iov_base, len)) {
+ if (iov != *iovp)
+ kfree(iov);
+ *iovp = NULL;
+ return -EFAULT;
+ }
+
+ if (len > MAX_RW_COUNT - total_len) {
+ len = MAX_RW_COUNT - total_len;
+ iov[seg].iov_len = len;
+ }
+ total_len += len;
+ }
+
+ iov_iter_init(i, type, iov, nr_segs, total_len);
+ if (iov == *iovp)
+ *iovp = NULL;
+ else
+ *iovp = iov;
+ return total_len;
+}
+
/**
* import_iovec() - Copy an array of &struct iovec from userspace
* into the kernel, check that it is valid, and initialize a new
* &struct iov_iter iterator to access it.
*
* @type: One of %READ or %WRITE.
- * @uvector: Pointer to the userspace array.
+ * @uvec: Pointer to the userspace array.
* @nr_segs: Number of elements in userspace array.
* @fast_segs: Number of elements in @iov.
- * @iov: (input and output parameter) Pointer to pointer to (usually small
+ * @iovp: (input and output parameter) Pointer to pointer to (usually small
* on-stack) kernel array.
* @i: Pointer to iterator that will be initialized on success.
*
*
* Return: Negative error code on error, bytes imported on success
*/
-ssize_t import_iovec(int type, const struct iovec __user * uvector,
+ssize_t import_iovec(int type, const struct iovec __user *uvec,
unsigned nr_segs, unsigned fast_segs,
- struct iovec **iov, struct iov_iter *i)
+ struct iovec **iovp, struct iov_iter *i)
{
- ssize_t n;
- struct iovec *p;
- n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
- *iov, &p);
- if (n < 0) {
- if (p != *iov)
- kfree(p);
- *iov = NULL;
- return n;
- }
- iov_iter_init(i, type, p, nr_segs, n);
- *iov = p == *iov ? NULL : p;
- return n;
+ return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i,
+ in_compat_syscall());
}
EXPORT_SYMBOL(import_iovec);
-#ifdef CONFIG_COMPAT
-#include <linux/compat.h>
-
-ssize_t compat_import_iovec(int type,
- const struct compat_iovec __user * uvector,
- unsigned nr_segs, unsigned fast_segs,
- struct iovec **iov, struct iov_iter *i)
-{
- ssize_t n;
- struct iovec *p;
- n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
- *iov, &p);
- if (n < 0) {
- if (p != *iov)
- kfree(p);
- *iov = NULL;
- return n;
- }
- iov_iter_init(i, type, p, nr_segs, n);
- *iov = p == *iov ? NULL : p;
- return n;
-}
-EXPORT_SYMBOL(compat_import_iovec);
-#endif
-
int import_single_range(int rw, void __user *buf, size_t len,
struct iovec *iov, struct iov_iter *i)
{
* Change this to 1 if you want to see the failure printouts:
*/
static unsigned int debug_locks_verbose;
+unsigned int force_read_lock_recursive;
static DEFINE_WD_CLASS(ww_lockdep);
MU(Y1); // should fail
}
+/*
+ * read_lock(A)
+ * spin_lock(B)
+ * spin_lock(B)
+ * write_lock(A)
+ *
+ * This test case is aimed at poking whether the chain cache prevents us from
+ * detecting a read-lock/lock-write deadlock: if the chain cache doesn't differ
+ * read/write locks, the following case may happen
+ *
+ * { read_lock(A)->lock(B) dependency exists }
+ *
+ * P0:
+ * lock(B);
+ * read_lock(A);
+ *
+ * { Not a deadlock, B -> A is added in the chain cache }
+ *
+ * P1:
+ * lock(B);
+ * write_lock(A);
+ *
+ * { B->A found in chain cache, not reported as a deadlock }
+ *
+ */
+static void rlock_chaincache_ABBA1(void)
+{
+ RL(X1);
+ L(Y1);
+ U(Y1);
+ RU(X1);
+
+ L(Y1);
+ RL(X1);
+ RU(X1);
+ U(Y1);
+
+ L(Y1);
+ WL(X1);
+ WU(X1);
+ U(Y1); // should fail
+}
+
/*
* read_lock(A)
* spin_lock(B)
#undef E2
#undef E3
+/*
+ * write-read / write-read / write-read deadlock even if read is recursive
+ */
+
+#define E1() \
+ \
+ WL(X1); \
+ RL(Y1); \
+ RU(Y1); \
+ WU(X1);
+
+#define E2() \
+ \
+ WL(Y1); \
+ RL(Z1); \
+ RU(Z1); \
+ WU(Y1);
+
+#define E3() \
+ \
+ WL(Z1); \
+ RL(X1); \
+ RU(X1); \
+ WU(Z1);
+
+#include "locking-selftest-rlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(W1R2_W2R3_W3R1)
+
+#undef E1
+#undef E2
+#undef E3
+
+/*
+ * write-write / read-read / write-read deadlock even if read is recursive
+ */
+
+#define E1() \
+ \
+ WL(X1); \
+ WL(Y1); \
+ WU(Y1); \
+ WU(X1);
+
+#define E2() \
+ \
+ RL(Y1); \
+ RL(Z1); \
+ RU(Z1); \
+ RU(Y1);
+
+#define E3() \
+ \
+ WL(Z1); \
+ RL(X1); \
+ RU(X1); \
+ WU(Z1);
+
+#include "locking-selftest-rlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_W3R1)
+
+#undef E1
+#undef E2
+#undef E3
+
+/*
+ * write-write / read-read / read-write is not deadlock when read is recursive
+ */
+
+#define E1() \
+ \
+ WL(X1); \
+ WL(Y1); \
+ WU(Y1); \
+ WU(X1);
+
+#define E2() \
+ \
+ RL(Y1); \
+ RL(Z1); \
+ RU(Z1); \
+ RU(Y1);
+
+#define E3() \
+ \
+ RL(Z1); \
+ WL(X1); \
+ WU(X1); \
+ RU(Z1);
+
+#include "locking-selftest-rlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(W1R2_R2R3_W3W1)
+
+#undef E1
+#undef E2
+#undef E3
+
+/*
+ * write-read / read-read / write-write is not deadlock when read is recursive
+ */
+
+#define E1() \
+ \
+ WL(X1); \
+ RL(Y1); \
+ RU(Y1); \
+ WU(X1);
+
+#define E2() \
+ \
+ RL(Y1); \
+ RL(Z1); \
+ RU(Z1); \
+ RU(Y1);
+
+#define E3() \
+ \
+ WL(Z1); \
+ WL(X1); \
+ WU(X1); \
+ WU(Z1);
+
+#include "locking-selftest-rlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(W1W2_R2R3_R3W1)
+
+#undef E1
+#undef E2
+#undef E3
/*
* read-lock / write-lock recursion that is actually safe.
*/
#define E3() \
\
IRQ_ENTER(); \
- RL(A); \
+ LOCK(A); \
L(B); \
U(B); \
- RU(A); \
+ UNLOCK(A); \
IRQ_EXIT();
/*
- * Generate 12 testcases:
+ * Generate 24 testcases:
*/
#include "locking-selftest-hardirq.h"
-GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard)
+#include "locking-selftest-rlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_rlock)
+
+#include "locking-selftest-wlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_hard_wlock)
#include "locking-selftest-softirq.h"
-GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft)
+#include "locking-selftest-rlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_rlock)
+
+#include "locking-selftest-wlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft_wlock)
#undef E1
#undef E2
\
IRQ_DISABLE(); \
L(B); \
- WL(A); \
- WU(A); \
+ LOCK(A); \
+ UNLOCK(A); \
U(B); \
IRQ_ENABLE();
IRQ_EXIT();
/*
- * Generate 12 testcases:
+ * Generate 24 testcases:
*/
#include "locking-selftest-hardirq.h"
-// GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard)
+#include "locking-selftest-rlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_rlock)
+
+#include "locking-selftest-wlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_hard_wlock)
#include "locking-selftest-softirq.h"
-// GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft)
+#include "locking-selftest-rlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_rlock)
+
+#include "locking-selftest-wlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion2_soft_wlock)
+
+#undef E1
+#undef E2
+#undef E3
+/*
+ * read-lock / write-lock recursion that is unsafe.
+ *
+ * A is a ENABLED_*_READ lock
+ * B is a USED_IN_*_READ lock
+ *
+ * read_lock(A);
+ * write_lock(B);
+ * <interrupt>
+ * read_lock(B);
+ * write_lock(A); // if this one is read_lock(), no deadlock
+ */
+
+#define E1() \
+ \
+ IRQ_DISABLE(); \
+ WL(B); \
+ LOCK(A); \
+ UNLOCK(A); \
+ WU(B); \
+ IRQ_ENABLE();
+
+#define E2() \
+ \
+ RL(A); \
+ RU(A); \
+
+#define E3() \
+ \
+ IRQ_ENTER(); \
+ RL(B); \
+ RU(B); \
+ IRQ_EXIT();
+
+/*
+ * Generate 24 testcases:
+ */
+#include "locking-selftest-hardirq.h"
+#include "locking-selftest-rlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_rlock)
+
+#include "locking-selftest-wlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_hard_wlock)
+
+#include "locking-selftest-softirq.h"
+#include "locking-selftest-rlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_rlock)
+
+#include "locking-selftest-wlock.h"
+GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion3_soft_wlock)
#ifdef CONFIG_DEBUG_LOCK_ALLOC
# define I_SPINLOCK(x) lockdep_reset_lock(&lock_##x.dep_map)
dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK); \
pr_cont("\n");
+#define DO_TESTCASE_1RR(desc, name, nr) \
+ print_testname(desc"/"#nr); \
+ pr_cont(" |"); \
+ dotest(name##_##nr, SUCCESS, LOCKTYPE_RWLOCK); \
+ pr_cont("\n");
+
+#define DO_TESTCASE_1RRB(desc, name, nr) \
+ print_testname(desc"/"#nr); \
+ pr_cont(" |"); \
+ dotest(name##_##nr, FAILURE, LOCKTYPE_RWLOCK); \
+ pr_cont("\n");
+
+
#define DO_TESTCASE_3(desc, name, nr) \
print_testname(desc"/"#nr); \
dotest(name##_spin_##nr, FAILURE, LOCKTYPE_SPIN); \
dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK); \
pr_cont("\n");
+#define DO_TESTCASE_2RW(desc, name, nr) \
+ print_testname(desc"/"#nr); \
+ pr_cont(" |"); \
+ dotest(name##_wlock_##nr, FAILURE, LOCKTYPE_RWLOCK); \
+ dotest(name##_rlock_##nr, SUCCESS, LOCKTYPE_RWLOCK); \
+ pr_cont("\n");
+
+#define DO_TESTCASE_2x2RW(desc, name, nr) \
+ DO_TESTCASE_2RW("hard-"desc, name##_hard, nr) \
+ DO_TESTCASE_2RW("soft-"desc, name##_soft, nr) \
+
+#define DO_TESTCASE_6x2x2RW(desc, name) \
+ DO_TESTCASE_2x2RW(desc, name, 123); \
+ DO_TESTCASE_2x2RW(desc, name, 132); \
+ DO_TESTCASE_2x2RW(desc, name, 213); \
+ DO_TESTCASE_2x2RW(desc, name, 231); \
+ DO_TESTCASE_2x2RW(desc, name, 312); \
+ DO_TESTCASE_2x2RW(desc, name, 321);
+
#define DO_TESTCASE_6(desc, name) \
print_testname(desc); \
dotest(name##_spin, FAILURE, LOCKTYPE_SPIN); \
DO_TESTCASE_2IB(desc, name, 312); \
DO_TESTCASE_2IB(desc, name, 321);
+#define DO_TESTCASE_6x1RR(desc, name) \
+ DO_TESTCASE_1RR(desc, name, 123); \
+ DO_TESTCASE_1RR(desc, name, 132); \
+ DO_TESTCASE_1RR(desc, name, 213); \
+ DO_TESTCASE_1RR(desc, name, 231); \
+ DO_TESTCASE_1RR(desc, name, 312); \
+ DO_TESTCASE_1RR(desc, name, 321);
+
+#define DO_TESTCASE_6x1RRB(desc, name) \
+ DO_TESTCASE_1RRB(desc, name, 123); \
+ DO_TESTCASE_1RRB(desc, name, 132); \
+ DO_TESTCASE_1RRB(desc, name, 213); \
+ DO_TESTCASE_1RRB(desc, name, 231); \
+ DO_TESTCASE_1RRB(desc, name, 312); \
+ DO_TESTCASE_1RRB(desc, name, 321);
+
#define DO_TESTCASE_6x6(desc, name) \
DO_TESTCASE_6I(desc, name, 123); \
DO_TESTCASE_6I(desc, name, 132); \
pr_cont("\n");
}
+
+/*
+ * <in hardirq handler>
+ * read_lock(&A);
+ * <hardirq disable>
+ * spin_lock(&B);
+ * spin_lock(&B);
+ * read_lock(&A);
+ *
+ * is a deadlock.
+ */
+static void queued_read_lock_hardirq_RE_Er(void)
+{
+ HARDIRQ_ENTER();
+ read_lock(&rwlock_A);
+ LOCK(B);
+ UNLOCK(B);
+ read_unlock(&rwlock_A);
+ HARDIRQ_EXIT();
+
+ HARDIRQ_DISABLE();
+ LOCK(B);
+ read_lock(&rwlock_A);
+ read_unlock(&rwlock_A);
+ UNLOCK(B);
+ HARDIRQ_ENABLE();
+}
+
+/*
+ * <in hardirq handler>
+ * spin_lock(&B);
+ * <hardirq disable>
+ * read_lock(&A);
+ * read_lock(&A);
+ * spin_lock(&B);
+ *
+ * is not a deadlock.
+ */
+static void queued_read_lock_hardirq_ER_rE(void)
+{
+ HARDIRQ_ENTER();
+ LOCK(B);
+ read_lock(&rwlock_A);
+ read_unlock(&rwlock_A);
+ UNLOCK(B);
+ HARDIRQ_EXIT();
+
+ HARDIRQ_DISABLE();
+ read_lock(&rwlock_A);
+ LOCK(B);
+ UNLOCK(B);
+ read_unlock(&rwlock_A);
+ HARDIRQ_ENABLE();
+}
+
+/*
+ * <hardirq disable>
+ * spin_lock(&B);
+ * read_lock(&A);
+ * <in hardirq handler>
+ * spin_lock(&B);
+ * read_lock(&A);
+ *
+ * is a deadlock. Because the two read_lock()s are both non-recursive readers.
+ */
+static void queued_read_lock_hardirq_inversion(void)
+{
+
+ HARDIRQ_ENTER();
+ LOCK(B);
+ UNLOCK(B);
+ HARDIRQ_EXIT();
+
+ HARDIRQ_DISABLE();
+ LOCK(B);
+ read_lock(&rwlock_A);
+ read_unlock(&rwlock_A);
+ UNLOCK(B);
+ HARDIRQ_ENABLE();
+
+ read_lock(&rwlock_A);
+ read_unlock(&rwlock_A);
+}
+
+static void queued_read_lock_tests(void)
+{
+ printk(" --------------------------------------------------------------------------\n");
+ printk(" | queued read lock tests |\n");
+ printk(" ---------------------------\n");
+ print_testname("hardirq read-lock/lock-read");
+ dotest(queued_read_lock_hardirq_RE_Er, FAILURE, LOCKTYPE_RWLOCK);
+ pr_cont("\n");
+
+ print_testname("hardirq lock-read/read-lock");
+ dotest(queued_read_lock_hardirq_ER_rE, SUCCESS, LOCKTYPE_RWLOCK);
+ pr_cont("\n");
+
+ print_testname("hardirq inversion");
+ dotest(queued_read_lock_hardirq_inversion, FAILURE, LOCKTYPE_RWLOCK);
+ pr_cont("\n");
+}
+
void locking_selftest(void)
{
/*
return;
}
+ /*
+ * treats read_lock() as recursive read locks for testing purpose
+ */
+ force_read_lock_recursive = 1;
+
/*
* Run the testsuite:
*/
print_testname("mixed read-lock/lock-write ABBA");
pr_cont(" |");
dotest(rlock_ABBA1, FAILURE, LOCKTYPE_RWLOCK);
-#ifdef CONFIG_PROVE_LOCKING
- /*
- * Lockdep does indeed fail here, but there's nothing we can do about
- * that now. Don't kill lockdep for it.
- */
- unexpected_testcase_failures--;
-#endif
-
pr_cont(" |");
dotest(rwsem_ABBA1, FAILURE, LOCKTYPE_RWSEM);
pr_cont(" |");
dotest(rwsem_ABBA3, FAILURE, LOCKTYPE_RWSEM);
+ print_testname("chain cached mixed R-L/L-W ABBA");
+ pr_cont(" |");
+ dotest(rlock_chaincache_ABBA1, FAILURE, LOCKTYPE_RWLOCK);
+
+ DO_TESTCASE_6x1RRB("rlock W1R2/W2R3/W3R1", W1R2_W2R3_W3R1);
+ DO_TESTCASE_6x1RRB("rlock W1W2/R2R3/W3R1", W1W2_R2R3_W3R1);
+ DO_TESTCASE_6x1RR("rlock W1W2/R2R3/R3W1", W1W2_R2R3_R3W1);
+ DO_TESTCASE_6x1RR("rlock W1R2/R2R3/W3W1", W1R2_R2R3_W3W1);
+
printk(" --------------------------------------------------------------------------\n");
/*
DO_TESTCASE_6x6("safe-A + unsafe-B #2", irqsafe4);
DO_TESTCASE_6x6RW("irq lock-inversion", irq_inversion);
- DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion);
-// DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2);
+ DO_TESTCASE_6x2x2RW("irq read-recursion", irq_read_recursion);
+ DO_TESTCASE_6x2x2RW("irq read-recursion #2", irq_read_recursion2);
+ DO_TESTCASE_6x2x2RW("irq read-recursion #3", irq_read_recursion3);
ww_tests();
+ force_read_lock_recursive = 0;
+ /*
+ * queued_read_lock() specific test cases can be put here
+ */
+ if (IS_ENABLED(CONFIG_QUEUED_RWLOCKS))
+ queued_read_lock_tests();
+
if (unexpected_testcase_failures) {
printk("-----------------------------------------------------------------\n");
debug_locks = 0;
/* identifiers for device / performance-differentiated memory regions */
#include <linux/idr.h>
#include <linux/types.h>
+#include <linux/memregion.h>
static DEFINE_IDA(memregion_ids);
generic_mpih-rshift.o \
generic_mpih-sub1.o \
generic_mpih-add1.o \
+ ec.o \
mpicoder.o \
+ mpi-add.o \
mpi-bit.o \
mpi-cmp.o \
mpi-sub-ui.o \
+ mpi-div.o \
+ mpi-inv.o \
+ mpi-mod.o \
+ mpi-mul.o \
mpih-cmp.o \
mpih-div.o \
mpih-mul.o \
--- /dev/null
+/* ec.c - Elliptic Curve functions
+ * Copyright (C) 2007 Free Software Foundation, Inc.
+ * Copyright (C) 2013 g10 Code GmbH
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mpi-internal.h"
+#include "longlong.h"
+
+#define point_init(a) mpi_point_init((a))
+#define point_free(a) mpi_point_free_parts((a))
+
+#define log_error(fmt, ...) pr_err(fmt, ##__VA_ARGS__)
+#define log_fatal(fmt, ...) pr_err(fmt, ##__VA_ARGS__)
+
+#define DIM(v) (sizeof(v)/sizeof((v)[0]))
+
+
+/* Create a new point option. NBITS gives the size in bits of one
+ * coordinate; it is only used to pre-allocate some resources and
+ * might also be passed as 0 to use a default value.
+ */
+MPI_POINT mpi_point_new(unsigned int nbits)
+{
+ MPI_POINT p;
+
+ (void)nbits; /* Currently not used. */
+
+ p = kmalloc(sizeof(*p), GFP_KERNEL);
+ if (p)
+ mpi_point_init(p);
+ return p;
+}
+EXPORT_SYMBOL_GPL(mpi_point_new);
+
+/* Release the point object P. P may be NULL. */
+void mpi_point_release(MPI_POINT p)
+{
+ if (p) {
+ mpi_point_free_parts(p);
+ kfree(p);
+ }
+}
+EXPORT_SYMBOL_GPL(mpi_point_release);
+
+/* Initialize the fields of a point object. gcry_mpi_point_free_parts
+ * may be used to release the fields.
+ */
+void mpi_point_init(MPI_POINT p)
+{
+ p->x = mpi_new(0);
+ p->y = mpi_new(0);
+ p->z = mpi_new(0);
+}
+EXPORT_SYMBOL_GPL(mpi_point_init);
+
+/* Release the parts of a point object. */
+void mpi_point_free_parts(MPI_POINT p)
+{
+ mpi_free(p->x); p->x = NULL;
+ mpi_free(p->y); p->y = NULL;
+ mpi_free(p->z); p->z = NULL;
+}
+EXPORT_SYMBOL_GPL(mpi_point_free_parts);
+
+/* Set the value from S into D. */
+static void point_set(MPI_POINT d, MPI_POINT s)
+{
+ mpi_set(d->x, s->x);
+ mpi_set(d->y, s->y);
+ mpi_set(d->z, s->z);
+}
+
+static void point_resize(MPI_POINT p, struct mpi_ec_ctx *ctx)
+{
+ size_t nlimbs = ctx->p->nlimbs;
+
+ mpi_resize(p->x, nlimbs);
+ p->x->nlimbs = nlimbs;
+ mpi_resize(p->z, nlimbs);
+ p->z->nlimbs = nlimbs;
+
+ if (ctx->model != MPI_EC_MONTGOMERY) {
+ mpi_resize(p->y, nlimbs);
+ p->y->nlimbs = nlimbs;
+ }
+}
+
+static void point_swap_cond(MPI_POINT d, MPI_POINT s, unsigned long swap,
+ struct mpi_ec_ctx *ctx)
+{
+ mpi_swap_cond(d->x, s->x, swap);
+ if (ctx->model != MPI_EC_MONTGOMERY)
+ mpi_swap_cond(d->y, s->y, swap);
+ mpi_swap_cond(d->z, s->z, swap);
+}
+
+
+/* W = W mod P. */
+static void ec_mod(MPI w, struct mpi_ec_ctx *ec)
+{
+ if (ec->t.p_barrett)
+ mpi_mod_barrett(w, w, ec->t.p_barrett);
+ else
+ mpi_mod(w, w, ec->p);
+}
+
+static void ec_addm(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx)
+{
+ mpi_add(w, u, v);
+ ec_mod(w, ctx);
+}
+
+static void ec_subm(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ec)
+{
+ mpi_sub(w, u, v);
+ while (w->sign)
+ mpi_add(w, w, ec->p);
+ /*ec_mod(w, ec);*/
+}
+
+static void ec_mulm(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx)
+{
+ mpi_mul(w, u, v);
+ ec_mod(w, ctx);
+}
+
+/* W = 2 * U mod P. */
+static void ec_mul2(MPI w, MPI u, struct mpi_ec_ctx *ctx)
+{
+ mpi_lshift(w, u, 1);
+ ec_mod(w, ctx);
+}
+
+static void ec_powm(MPI w, const MPI b, const MPI e,
+ struct mpi_ec_ctx *ctx)
+{
+ mpi_powm(w, b, e, ctx->p);
+ /* mpi_abs(w); */
+}
+
+/* Shortcut for
+ * ec_powm(B, B, mpi_const(MPI_C_TWO), ctx);
+ * for easier optimization.
+ */
+static void ec_pow2(MPI w, const MPI b, struct mpi_ec_ctx *ctx)
+{
+ /* Using mpi_mul is slightly faster (at least on amd64). */
+ /* mpi_powm(w, b, mpi_const(MPI_C_TWO), ctx->p); */
+ ec_mulm(w, b, b, ctx);
+}
+
+/* Shortcut for
+ * ec_powm(B, B, mpi_const(MPI_C_THREE), ctx);
+ * for easier optimization.
+ */
+static void ec_pow3(MPI w, const MPI b, struct mpi_ec_ctx *ctx)
+{
+ mpi_powm(w, b, mpi_const(MPI_C_THREE), ctx->p);
+}
+
+static void ec_invm(MPI x, MPI a, struct mpi_ec_ctx *ctx)
+{
+ if (!mpi_invm(x, a, ctx->p))
+ log_error("ec_invm: inverse does not exist:\n");
+}
+
+static void mpih_set_cond(mpi_ptr_t wp, mpi_ptr_t up,
+ mpi_size_t usize, unsigned long set)
+{
+ mpi_size_t i;
+ mpi_limb_t mask = ((mpi_limb_t)0) - set;
+ mpi_limb_t x;
+
+ for (i = 0; i < usize; i++) {
+ x = mask & (wp[i] ^ up[i]);
+ wp[i] = wp[i] ^ x;
+ }
+}
+
+/* Routines for 2^255 - 19. */
+
+#define LIMB_SIZE_25519 ((256+BITS_PER_MPI_LIMB-1)/BITS_PER_MPI_LIMB)
+
+static void ec_addm_25519(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx)
+{
+ mpi_ptr_t wp, up, vp;
+ mpi_size_t wsize = LIMB_SIZE_25519;
+ mpi_limb_t n[LIMB_SIZE_25519];
+ mpi_limb_t borrow;
+
+ if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize)
+ log_bug("addm_25519: different sizes\n");
+
+ memset(n, 0, sizeof(n));
+ up = u->d;
+ vp = v->d;
+ wp = w->d;
+
+ mpihelp_add_n(wp, up, vp, wsize);
+ borrow = mpihelp_sub_n(wp, wp, ctx->p->d, wsize);
+ mpih_set_cond(n, ctx->p->d, wsize, (borrow != 0UL));
+ mpihelp_add_n(wp, wp, n, wsize);
+ wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB));
+}
+
+static void ec_subm_25519(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx)
+{
+ mpi_ptr_t wp, up, vp;
+ mpi_size_t wsize = LIMB_SIZE_25519;
+ mpi_limb_t n[LIMB_SIZE_25519];
+ mpi_limb_t borrow;
+
+ if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize)
+ log_bug("subm_25519: different sizes\n");
+
+ memset(n, 0, sizeof(n));
+ up = u->d;
+ vp = v->d;
+ wp = w->d;
+
+ borrow = mpihelp_sub_n(wp, up, vp, wsize);
+ mpih_set_cond(n, ctx->p->d, wsize, (borrow != 0UL));
+ mpihelp_add_n(wp, wp, n, wsize);
+ wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB));
+}
+
+static void ec_mulm_25519(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx)
+{
+ mpi_ptr_t wp, up, vp;
+ mpi_size_t wsize = LIMB_SIZE_25519;
+ mpi_limb_t n[LIMB_SIZE_25519*2];
+ mpi_limb_t m[LIMB_SIZE_25519+1];
+ mpi_limb_t cy;
+ int msb;
+
+ (void)ctx;
+ if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize)
+ log_bug("mulm_25519: different sizes\n");
+
+ up = u->d;
+ vp = v->d;
+ wp = w->d;
+
+ mpihelp_mul_n(n, up, vp, wsize);
+ memcpy(wp, n, wsize * BYTES_PER_MPI_LIMB);
+ wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB));
+
+ memcpy(m, n+LIMB_SIZE_25519-1, (wsize+1) * BYTES_PER_MPI_LIMB);
+ mpihelp_rshift(m, m, LIMB_SIZE_25519+1, (255 % BITS_PER_MPI_LIMB));
+
+ memcpy(n, m, wsize * BYTES_PER_MPI_LIMB);
+ cy = mpihelp_lshift(m, m, LIMB_SIZE_25519, 4);
+ m[LIMB_SIZE_25519] = cy;
+ cy = mpihelp_add_n(m, m, n, wsize);
+ m[LIMB_SIZE_25519] += cy;
+ cy = mpihelp_add_n(m, m, n, wsize);
+ m[LIMB_SIZE_25519] += cy;
+ cy = mpihelp_add_n(m, m, n, wsize);
+ m[LIMB_SIZE_25519] += cy;
+
+ cy = mpihelp_add_n(wp, wp, m, wsize);
+ m[LIMB_SIZE_25519] += cy;
+
+ memset(m, 0, wsize * BYTES_PER_MPI_LIMB);
+ msb = (wp[LIMB_SIZE_25519-1] >> (255 % BITS_PER_MPI_LIMB));
+ m[0] = (m[LIMB_SIZE_25519] * 2 + msb) * 19;
+ wp[LIMB_SIZE_25519-1] &= ~((mpi_limb_t)1 << (255 % BITS_PER_MPI_LIMB));
+ mpihelp_add_n(wp, wp, m, wsize);
+
+ m[0] = 0;
+ cy = mpihelp_sub_n(wp, wp, ctx->p->d, wsize);
+ mpih_set_cond(m, ctx->p->d, wsize, (cy != 0UL));
+ mpihelp_add_n(wp, wp, m, wsize);
+}
+
+static void ec_mul2_25519(MPI w, MPI u, struct mpi_ec_ctx *ctx)
+{
+ ec_addm_25519(w, u, u, ctx);
+}
+
+static void ec_pow2_25519(MPI w, const MPI b, struct mpi_ec_ctx *ctx)
+{
+ ec_mulm_25519(w, b, b, ctx);
+}
+
+/* Routines for 2^448 - 2^224 - 1. */
+
+#define LIMB_SIZE_448 ((448+BITS_PER_MPI_LIMB-1)/BITS_PER_MPI_LIMB)
+#define LIMB_SIZE_HALF_448 ((LIMB_SIZE_448+1)/2)
+
+static void ec_addm_448(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx)
+{
+ mpi_ptr_t wp, up, vp;
+ mpi_size_t wsize = LIMB_SIZE_448;
+ mpi_limb_t n[LIMB_SIZE_448];
+ mpi_limb_t cy;
+
+ if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize)
+ log_bug("addm_448: different sizes\n");
+
+ memset(n, 0, sizeof(n));
+ up = u->d;
+ vp = v->d;
+ wp = w->d;
+
+ cy = mpihelp_add_n(wp, up, vp, wsize);
+ mpih_set_cond(n, ctx->p->d, wsize, (cy != 0UL));
+ mpihelp_sub_n(wp, wp, n, wsize);
+}
+
+static void ec_subm_448(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx)
+{
+ mpi_ptr_t wp, up, vp;
+ mpi_size_t wsize = LIMB_SIZE_448;
+ mpi_limb_t n[LIMB_SIZE_448];
+ mpi_limb_t borrow;
+
+ if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize)
+ log_bug("subm_448: different sizes\n");
+
+ memset(n, 0, sizeof(n));
+ up = u->d;
+ vp = v->d;
+ wp = w->d;
+
+ borrow = mpihelp_sub_n(wp, up, vp, wsize);
+ mpih_set_cond(n, ctx->p->d, wsize, (borrow != 0UL));
+ mpihelp_add_n(wp, wp, n, wsize);
+}
+
+static void ec_mulm_448(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx)
+{
+ mpi_ptr_t wp, up, vp;
+ mpi_size_t wsize = LIMB_SIZE_448;
+ mpi_limb_t n[LIMB_SIZE_448*2];
+ mpi_limb_t a2[LIMB_SIZE_HALF_448];
+ mpi_limb_t a3[LIMB_SIZE_HALF_448];
+ mpi_limb_t b0[LIMB_SIZE_HALF_448];
+ mpi_limb_t b1[LIMB_SIZE_HALF_448];
+ mpi_limb_t cy;
+ int i;
+#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2)
+ mpi_limb_t b1_rest, a3_rest;
+#endif
+
+ if (w->nlimbs != wsize || u->nlimbs != wsize || v->nlimbs != wsize)
+ log_bug("mulm_448: different sizes\n");
+
+ up = u->d;
+ vp = v->d;
+ wp = w->d;
+
+ mpihelp_mul_n(n, up, vp, wsize);
+
+ for (i = 0; i < (wsize + 1) / 2; i++) {
+ b0[i] = n[i];
+ b1[i] = n[i+wsize/2];
+ a2[i] = n[i+wsize];
+ a3[i] = n[i+wsize+wsize/2];
+ }
+
+#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2)
+ b0[LIMB_SIZE_HALF_448-1] &= ((mpi_limb_t)1UL << 32)-1;
+ a2[LIMB_SIZE_HALF_448-1] &= ((mpi_limb_t)1UL << 32)-1;
+
+ b1_rest = 0;
+ a3_rest = 0;
+
+ for (i = (wsize + 1) / 2 - 1; i >= 0; i--) {
+ mpi_limb_t b1v, a3v;
+ b1v = b1[i];
+ a3v = a3[i];
+ b1[i] = (b1_rest << 32) | (b1v >> 32);
+ a3[i] = (a3_rest << 32) | (a3v >> 32);
+ b1_rest = b1v & (((mpi_limb_t)1UL << 32)-1);
+ a3_rest = a3v & (((mpi_limb_t)1UL << 32)-1);
+ }
+#endif
+
+ cy = mpihelp_add_n(b0, b0, a2, LIMB_SIZE_HALF_448);
+ cy += mpihelp_add_n(b0, b0, a3, LIMB_SIZE_HALF_448);
+ for (i = 0; i < (wsize + 1) / 2; i++)
+ wp[i] = b0[i];
+#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2)
+ wp[LIMB_SIZE_HALF_448-1] &= (((mpi_limb_t)1UL << 32)-1);
+#endif
+
+#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2)
+ cy = b0[LIMB_SIZE_HALF_448-1] >> 32;
+#endif
+
+ cy = mpihelp_add_1(b1, b1, LIMB_SIZE_HALF_448, cy);
+ cy += mpihelp_add_n(b1, b1, a2, LIMB_SIZE_HALF_448);
+ cy += mpihelp_add_n(b1, b1, a3, LIMB_SIZE_HALF_448);
+ cy += mpihelp_add_n(b1, b1, a3, LIMB_SIZE_HALF_448);
+#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2)
+ b1_rest = 0;
+ for (i = (wsize + 1) / 2 - 1; i >= 0; i--) {
+ mpi_limb_t b1v = b1[i];
+ b1[i] = (b1_rest << 32) | (b1v >> 32);
+ b1_rest = b1v & (((mpi_limb_t)1UL << 32)-1);
+ }
+ wp[LIMB_SIZE_HALF_448-1] |= (b1_rest << 32);
+#endif
+ for (i = 0; i < wsize / 2; i++)
+ wp[i+(wsize + 1) / 2] = b1[i];
+
+#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2)
+ cy = b1[LIMB_SIZE_HALF_448-1];
+#endif
+
+ memset(n, 0, wsize * BYTES_PER_MPI_LIMB);
+
+#if (LIMB_SIZE_HALF_448 > LIMB_SIZE_448/2)
+ n[LIMB_SIZE_HALF_448-1] = cy << 32;
+#else
+ n[LIMB_SIZE_HALF_448] = cy;
+#endif
+ n[0] = cy;
+ mpihelp_add_n(wp, wp, n, wsize);
+
+ memset(n, 0, wsize * BYTES_PER_MPI_LIMB);
+ cy = mpihelp_sub_n(wp, wp, ctx->p->d, wsize);
+ mpih_set_cond(n, ctx->p->d, wsize, (cy != 0UL));
+ mpihelp_add_n(wp, wp, n, wsize);
+}
+
+static void ec_mul2_448(MPI w, MPI u, struct mpi_ec_ctx *ctx)
+{
+ ec_addm_448(w, u, u, ctx);
+}
+
+static void ec_pow2_448(MPI w, const MPI b, struct mpi_ec_ctx *ctx)
+{
+ ec_mulm_448(w, b, b, ctx);
+}
+
+struct field_table {
+ const char *p;
+
+ /* computation routines for the field. */
+ void (*addm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx);
+ void (*subm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx);
+ void (*mulm)(MPI w, MPI u, MPI v, struct mpi_ec_ctx *ctx);
+ void (*mul2)(MPI w, MPI u, struct mpi_ec_ctx *ctx);
+ void (*pow2)(MPI w, const MPI b, struct mpi_ec_ctx *ctx);
+};
+
+static const struct field_table field_table[] = {
+ {
+ "0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFED",
+ ec_addm_25519,
+ ec_subm_25519,
+ ec_mulm_25519,
+ ec_mul2_25519,
+ ec_pow2_25519
+ },
+ {
+ "0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"
+ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+ ec_addm_448,
+ ec_subm_448,
+ ec_mulm_448,
+ ec_mul2_448,
+ ec_pow2_448
+ },
+ { NULL, NULL, NULL, NULL, NULL, NULL },
+};
+
+/* Force recomputation of all helper variables. */
+static void mpi_ec_get_reset(struct mpi_ec_ctx *ec)
+{
+ ec->t.valid.a_is_pminus3 = 0;
+ ec->t.valid.two_inv_p = 0;
+}
+
+/* Accessor for helper variable. */
+static int ec_get_a_is_pminus3(struct mpi_ec_ctx *ec)
+{
+ MPI tmp;
+
+ if (!ec->t.valid.a_is_pminus3) {
+ ec->t.valid.a_is_pminus3 = 1;
+ tmp = mpi_alloc_like(ec->p);
+ mpi_sub_ui(tmp, ec->p, 3);
+ ec->t.a_is_pminus3 = !mpi_cmp(ec->a, tmp);
+ mpi_free(tmp);
+ }
+
+ return ec->t.a_is_pminus3;
+}
+
+/* Accessor for helper variable. */
+static MPI ec_get_two_inv_p(struct mpi_ec_ctx *ec)
+{
+ if (!ec->t.valid.two_inv_p) {
+ ec->t.valid.two_inv_p = 1;
+ if (!ec->t.two_inv_p)
+ ec->t.two_inv_p = mpi_alloc(0);
+ ec_invm(ec->t.two_inv_p, mpi_const(MPI_C_TWO), ec);
+ }
+ return ec->t.two_inv_p;
+}
+
+static const char *const curve25519_bad_points[] = {
+ "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed",
+ "0x0000000000000000000000000000000000000000000000000000000000000000",
+ "0x0000000000000000000000000000000000000000000000000000000000000001",
+ "0x00b8495f16056286fdb1329ceb8d09da6ac49ff1fae35616aeb8413b7c7aebe0",
+ "0x57119fd0dd4e22d8868e1c58c45c44045bef839c55b1d0b1248c50a3bc959c5f",
+ "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffec",
+ "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffee",
+ NULL
+};
+
+static const char *const curve448_bad_points[] = {
+ "0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffe"
+ "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+ "0x00000000000000000000000000000000000000000000000000000000"
+ "00000000000000000000000000000000000000000000000000000000",
+ "0x00000000000000000000000000000000000000000000000000000000"
+ "00000000000000000000000000000000000000000000000000000001",
+ "0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffe"
+ "fffffffffffffffffffffffffffffffffffffffffffffffffffffffe",
+ "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
+ "00000000000000000000000000000000000000000000000000000000",
+ NULL
+};
+
+static const char *const *bad_points_table[] = {
+ curve25519_bad_points,
+ curve448_bad_points,
+};
+
+static void mpi_ec_coefficient_normalize(MPI a, MPI p)
+{
+ if (a->sign) {
+ mpi_resize(a, p->nlimbs);
+ mpihelp_sub_n(a->d, p->d, a->d, p->nlimbs);
+ a->nlimbs = p->nlimbs;
+ a->sign = 0;
+ }
+}
+
+/* This function initialized a context for elliptic curve based on the
+ * field GF(p). P is the prime specifying this field, A is the first
+ * coefficient. CTX is expected to be zeroized.
+ */
+void mpi_ec_init(struct mpi_ec_ctx *ctx, enum gcry_mpi_ec_models model,
+ enum ecc_dialects dialect,
+ int flags, MPI p, MPI a, MPI b)
+{
+ int i;
+ static int use_barrett = -1 /* TODO: 1 or -1 */;
+
+ mpi_ec_coefficient_normalize(a, p);
+ mpi_ec_coefficient_normalize(b, p);
+
+ /* Fixme: Do we want to check some constraints? e.g. a < p */
+
+ ctx->model = model;
+ ctx->dialect = dialect;
+ ctx->flags = flags;
+ if (dialect == ECC_DIALECT_ED25519)
+ ctx->nbits = 256;
+ else
+ ctx->nbits = mpi_get_nbits(p);
+ ctx->p = mpi_copy(p);
+ ctx->a = mpi_copy(a);
+ ctx->b = mpi_copy(b);
+
+ ctx->t.p_barrett = use_barrett > 0 ? mpi_barrett_init(ctx->p, 0) : NULL;
+
+ mpi_ec_get_reset(ctx);
+
+ if (model == MPI_EC_MONTGOMERY) {
+ for (i = 0; i < DIM(bad_points_table); i++) {
+ MPI p_candidate = mpi_scanval(bad_points_table[i][0]);
+ int match_p = !mpi_cmp(ctx->p, p_candidate);
+ int j;
+
+ mpi_free(p_candidate);
+ if (!match_p)
+ continue;
+
+ for (j = 0; i < DIM(ctx->t.scratch) && bad_points_table[i][j]; j++)
+ ctx->t.scratch[j] = mpi_scanval(bad_points_table[i][j]);
+ }
+ } else {
+ /* Allocate scratch variables. */
+ for (i = 0; i < DIM(ctx->t.scratch); i++)
+ ctx->t.scratch[i] = mpi_alloc_like(ctx->p);
+ }
+
+ ctx->addm = ec_addm;
+ ctx->subm = ec_subm;
+ ctx->mulm = ec_mulm;
+ ctx->mul2 = ec_mul2;
+ ctx->pow2 = ec_pow2;
+
+ for (i = 0; field_table[i].p; i++) {
+ MPI f_p;
+
+ f_p = mpi_scanval(field_table[i].p);
+ if (!f_p)
+ break;
+
+ if (!mpi_cmp(p, f_p)) {
+ ctx->addm = field_table[i].addm;
+ ctx->subm = field_table[i].subm;
+ ctx->mulm = field_table[i].mulm;
+ ctx->mul2 = field_table[i].mul2;
+ ctx->pow2 = field_table[i].pow2;
+ mpi_free(f_p);
+
+ mpi_resize(ctx->a, ctx->p->nlimbs);
+ ctx->a->nlimbs = ctx->p->nlimbs;
+
+ mpi_resize(ctx->b, ctx->p->nlimbs);
+ ctx->b->nlimbs = ctx->p->nlimbs;
+
+ for (i = 0; i < DIM(ctx->t.scratch) && ctx->t.scratch[i]; i++)
+ ctx->t.scratch[i]->nlimbs = ctx->p->nlimbs;
+
+ break;
+ }
+
+ mpi_free(f_p);
+ }
+}
+EXPORT_SYMBOL_GPL(mpi_ec_init);
+
+void mpi_ec_deinit(struct mpi_ec_ctx *ctx)
+{
+ int i;
+
+ mpi_barrett_free(ctx->t.p_barrett);
+
+ /* Domain parameter. */
+ mpi_free(ctx->p);
+ mpi_free(ctx->a);
+ mpi_free(ctx->b);
+ mpi_point_release(ctx->G);
+ mpi_free(ctx->n);
+
+ /* The key. */
+ mpi_point_release(ctx->Q);
+ mpi_free(ctx->d);
+
+ /* Private data of ec.c. */
+ mpi_free(ctx->t.two_inv_p);
+
+ for (i = 0; i < DIM(ctx->t.scratch); i++)
+ mpi_free(ctx->t.scratch[i]);
+}
+EXPORT_SYMBOL_GPL(mpi_ec_deinit);
+
+/* Compute the affine coordinates from the projective coordinates in
+ * POINT. Set them into X and Y. If one coordinate is not required,
+ * X or Y may be passed as NULL. CTX is the usual context. Returns: 0
+ * on success or !0 if POINT is at infinity.
+ */
+int mpi_ec_get_affine(MPI x, MPI y, MPI_POINT point, struct mpi_ec_ctx *ctx)
+{
+ if (!mpi_cmp_ui(point->z, 0))
+ return -1;
+
+ switch (ctx->model) {
+ case MPI_EC_WEIERSTRASS: /* Using Jacobian coordinates. */
+ {
+ MPI z1, z2, z3;
+
+ z1 = mpi_new(0);
+ z2 = mpi_new(0);
+ ec_invm(z1, point->z, ctx); /* z1 = z^(-1) mod p */
+ ec_mulm(z2, z1, z1, ctx); /* z2 = z^(-2) mod p */
+
+ if (x)
+ ec_mulm(x, point->x, z2, ctx);
+
+ if (y) {
+ z3 = mpi_new(0);
+ ec_mulm(z3, z2, z1, ctx); /* z3 = z^(-3) mod p */
+ ec_mulm(y, point->y, z3, ctx);
+ mpi_free(z3);
+ }
+
+ mpi_free(z2);
+ mpi_free(z1);
+ }
+ return 0;
+
+ case MPI_EC_MONTGOMERY:
+ {
+ if (x)
+ mpi_set(x, point->x);
+
+ if (y) {
+ log_fatal("%s: Getting Y-coordinate on %s is not supported\n",
+ "mpi_ec_get_affine", "Montgomery");
+ return -1;
+ }
+ }
+ return 0;
+
+ case MPI_EC_EDWARDS:
+ {
+ MPI z;
+
+ z = mpi_new(0);
+ ec_invm(z, point->z, ctx);
+
+ mpi_resize(z, ctx->p->nlimbs);
+ z->nlimbs = ctx->p->nlimbs;
+
+ if (x) {
+ mpi_resize(x, ctx->p->nlimbs);
+ x->nlimbs = ctx->p->nlimbs;
+ ctx->mulm(x, point->x, z, ctx);
+ }
+ if (y) {
+ mpi_resize(y, ctx->p->nlimbs);
+ y->nlimbs = ctx->p->nlimbs;
+ ctx->mulm(y, point->y, z, ctx);
+ }
+
+ mpi_free(z);
+ }
+ return 0;
+
+ default:
+ return -1;
+ }
+}
+EXPORT_SYMBOL_GPL(mpi_ec_get_affine);
+
+/* RESULT = 2 * POINT (Weierstrass version). */
+static void dup_point_weierstrass(MPI_POINT result,
+ MPI_POINT point, struct mpi_ec_ctx *ctx)
+{
+#define x3 (result->x)
+#define y3 (result->y)
+#define z3 (result->z)
+#define t1 (ctx->t.scratch[0])
+#define t2 (ctx->t.scratch[1])
+#define t3 (ctx->t.scratch[2])
+#define l1 (ctx->t.scratch[3])
+#define l2 (ctx->t.scratch[4])
+#define l3 (ctx->t.scratch[5])
+
+ if (!mpi_cmp_ui(point->y, 0) || !mpi_cmp_ui(point->z, 0)) {
+ /* P_y == 0 || P_z == 0 => [1:1:0] */
+ mpi_set_ui(x3, 1);
+ mpi_set_ui(y3, 1);
+ mpi_set_ui(z3, 0);
+ } else {
+ if (ec_get_a_is_pminus3(ctx)) {
+ /* Use the faster case. */
+ /* L1 = 3(X - Z^2)(X + Z^2) */
+ /* T1: used for Z^2. */
+ /* T2: used for the right term. */
+ ec_pow2(t1, point->z, ctx);
+ ec_subm(l1, point->x, t1, ctx);
+ ec_mulm(l1, l1, mpi_const(MPI_C_THREE), ctx);
+ ec_addm(t2, point->x, t1, ctx);
+ ec_mulm(l1, l1, t2, ctx);
+ } else {
+ /* Standard case. */
+ /* L1 = 3X^2 + aZ^4 */
+ /* T1: used for aZ^4. */
+ ec_pow2(l1, point->x, ctx);
+ ec_mulm(l1, l1, mpi_const(MPI_C_THREE), ctx);
+ ec_powm(t1, point->z, mpi_const(MPI_C_FOUR), ctx);
+ ec_mulm(t1, t1, ctx->a, ctx);
+ ec_addm(l1, l1, t1, ctx);
+ }
+ /* Z3 = 2YZ */
+ ec_mulm(z3, point->y, point->z, ctx);
+ ec_mul2(z3, z3, ctx);
+
+ /* L2 = 4XY^2 */
+ /* T2: used for Y2; required later. */
+ ec_pow2(t2, point->y, ctx);
+ ec_mulm(l2, t2, point->x, ctx);
+ ec_mulm(l2, l2, mpi_const(MPI_C_FOUR), ctx);
+
+ /* X3 = L1^2 - 2L2 */
+ /* T1: used for L2^2. */
+ ec_pow2(x3, l1, ctx);
+ ec_mul2(t1, l2, ctx);
+ ec_subm(x3, x3, t1, ctx);
+
+ /* L3 = 8Y^4 */
+ /* T2: taken from above. */
+ ec_pow2(t2, t2, ctx);
+ ec_mulm(l3, t2, mpi_const(MPI_C_EIGHT), ctx);
+
+ /* Y3 = L1(L2 - X3) - L3 */
+ ec_subm(y3, l2, x3, ctx);
+ ec_mulm(y3, y3, l1, ctx);
+ ec_subm(y3, y3, l3, ctx);
+ }
+
+#undef x3
+#undef y3
+#undef z3
+#undef t1
+#undef t2
+#undef t3
+#undef l1
+#undef l2
+#undef l3
+}
+
+/* RESULT = 2 * POINT (Montgomery version). */
+static void dup_point_montgomery(MPI_POINT result,
+ MPI_POINT point, struct mpi_ec_ctx *ctx)
+{
+ (void)result;
+ (void)point;
+ (void)ctx;
+ log_fatal("%s: %s not yet supported\n",
+ "mpi_ec_dup_point", "Montgomery");
+}
+
+/* RESULT = 2 * POINT (Twisted Edwards version). */
+static void dup_point_edwards(MPI_POINT result,
+ MPI_POINT point, struct mpi_ec_ctx *ctx)
+{
+#define X1 (point->x)
+#define Y1 (point->y)
+#define Z1 (point->z)
+#define X3 (result->x)
+#define Y3 (result->y)
+#define Z3 (result->z)
+#define B (ctx->t.scratch[0])
+#define C (ctx->t.scratch[1])
+#define D (ctx->t.scratch[2])
+#define E (ctx->t.scratch[3])
+#define F (ctx->t.scratch[4])
+#define H (ctx->t.scratch[5])
+#define J (ctx->t.scratch[6])
+
+ /* Compute: (X_3 : Y_3 : Z_3) = 2( X_1 : Y_1 : Z_1 ) */
+
+ /* B = (X_1 + Y_1)^2 */
+ ctx->addm(B, X1, Y1, ctx);
+ ctx->pow2(B, B, ctx);
+
+ /* C = X_1^2 */
+ /* D = Y_1^2 */
+ ctx->pow2(C, X1, ctx);
+ ctx->pow2(D, Y1, ctx);
+
+ /* E = aC */
+ if (ctx->dialect == ECC_DIALECT_ED25519)
+ ctx->subm(E, ctx->p, C, ctx);
+ else
+ ctx->mulm(E, ctx->a, C, ctx);
+
+ /* F = E + D */
+ ctx->addm(F, E, D, ctx);
+
+ /* H = Z_1^2 */
+ ctx->pow2(H, Z1, ctx);
+
+ /* J = F - 2H */
+ ctx->mul2(J, H, ctx);
+ ctx->subm(J, F, J, ctx);
+
+ /* X_3 = (B - C - D) · J */
+ ctx->subm(X3, B, C, ctx);
+ ctx->subm(X3, X3, D, ctx);
+ ctx->mulm(X3, X3, J, ctx);
+
+ /* Y_3 = F · (E - D) */
+ ctx->subm(Y3, E, D, ctx);
+ ctx->mulm(Y3, Y3, F, ctx);
+
+ /* Z_3 = F · J */
+ ctx->mulm(Z3, F, J, ctx);
+
+#undef X1
+#undef Y1
+#undef Z1
+#undef X3
+#undef Y3
+#undef Z3
+#undef B
+#undef C
+#undef D
+#undef E
+#undef F
+#undef H
+#undef J
+}
+
+/* RESULT = 2 * POINT */
+static void
+mpi_ec_dup_point(MPI_POINT result, MPI_POINT point, struct mpi_ec_ctx *ctx)
+{
+ switch (ctx->model) {
+ case MPI_EC_WEIERSTRASS:
+ dup_point_weierstrass(result, point, ctx);
+ break;
+ case MPI_EC_MONTGOMERY:
+ dup_point_montgomery(result, point, ctx);
+ break;
+ case MPI_EC_EDWARDS:
+ dup_point_edwards(result, point, ctx);
+ break;
+ }
+}
+
+/* RESULT = P1 + P2 (Weierstrass version).*/
+static void add_points_weierstrass(MPI_POINT result,
+ MPI_POINT p1, MPI_POINT p2,
+ struct mpi_ec_ctx *ctx)
+{
+#define x1 (p1->x)
+#define y1 (p1->y)
+#define z1 (p1->z)
+#define x2 (p2->x)
+#define y2 (p2->y)
+#define z2 (p2->z)
+#define x3 (result->x)
+#define y3 (result->y)
+#define z3 (result->z)
+#define l1 (ctx->t.scratch[0])
+#define l2 (ctx->t.scratch[1])
+#define l3 (ctx->t.scratch[2])
+#define l4 (ctx->t.scratch[3])
+#define l5 (ctx->t.scratch[4])
+#define l6 (ctx->t.scratch[5])
+#define l7 (ctx->t.scratch[6])
+#define l8 (ctx->t.scratch[7])
+#define l9 (ctx->t.scratch[8])
+#define t1 (ctx->t.scratch[9])
+#define t2 (ctx->t.scratch[10])
+
+ if ((!mpi_cmp(x1, x2)) && (!mpi_cmp(y1, y2)) && (!mpi_cmp(z1, z2))) {
+ /* Same point; need to call the duplicate function. */
+ mpi_ec_dup_point(result, p1, ctx);
+ } else if (!mpi_cmp_ui(z1, 0)) {
+ /* P1 is at infinity. */
+ mpi_set(x3, p2->x);
+ mpi_set(y3, p2->y);
+ mpi_set(z3, p2->z);
+ } else if (!mpi_cmp_ui(z2, 0)) {
+ /* P2 is at infinity. */
+ mpi_set(x3, p1->x);
+ mpi_set(y3, p1->y);
+ mpi_set(z3, p1->z);
+ } else {
+ int z1_is_one = !mpi_cmp_ui(z1, 1);
+ int z2_is_one = !mpi_cmp_ui(z2, 1);
+
+ /* l1 = x1 z2^2 */
+ /* l2 = x2 z1^2 */
+ if (z2_is_one)
+ mpi_set(l1, x1);
+ else {
+ ec_pow2(l1, z2, ctx);
+ ec_mulm(l1, l1, x1, ctx);
+ }
+ if (z1_is_one)
+ mpi_set(l2, x2);
+ else {
+ ec_pow2(l2, z1, ctx);
+ ec_mulm(l2, l2, x2, ctx);
+ }
+ /* l3 = l1 - l2 */
+ ec_subm(l3, l1, l2, ctx);
+ /* l4 = y1 z2^3 */
+ ec_powm(l4, z2, mpi_const(MPI_C_THREE), ctx);
+ ec_mulm(l4, l4, y1, ctx);
+ /* l5 = y2 z1^3 */
+ ec_powm(l5, z1, mpi_const(MPI_C_THREE), ctx);
+ ec_mulm(l5, l5, y2, ctx);
+ /* l6 = l4 - l5 */
+ ec_subm(l6, l4, l5, ctx);
+
+ if (!mpi_cmp_ui(l3, 0)) {
+ if (!mpi_cmp_ui(l6, 0)) {
+ /* P1 and P2 are the same - use duplicate function. */
+ mpi_ec_dup_point(result, p1, ctx);
+ } else {
+ /* P1 is the inverse of P2. */
+ mpi_set_ui(x3, 1);
+ mpi_set_ui(y3, 1);
+ mpi_set_ui(z3, 0);
+ }
+ } else {
+ /* l7 = l1 + l2 */
+ ec_addm(l7, l1, l2, ctx);
+ /* l8 = l4 + l5 */
+ ec_addm(l8, l4, l5, ctx);
+ /* z3 = z1 z2 l3 */
+ ec_mulm(z3, z1, z2, ctx);
+ ec_mulm(z3, z3, l3, ctx);
+ /* x3 = l6^2 - l7 l3^2 */
+ ec_pow2(t1, l6, ctx);
+ ec_pow2(t2, l3, ctx);
+ ec_mulm(t2, t2, l7, ctx);
+ ec_subm(x3, t1, t2, ctx);
+ /* l9 = l7 l3^2 - 2 x3 */
+ ec_mul2(t1, x3, ctx);
+ ec_subm(l9, t2, t1, ctx);
+ /* y3 = (l9 l6 - l8 l3^3)/2 */
+ ec_mulm(l9, l9, l6, ctx);
+ ec_powm(t1, l3, mpi_const(MPI_C_THREE), ctx); /* fixme: Use saved value*/
+ ec_mulm(t1, t1, l8, ctx);
+ ec_subm(y3, l9, t1, ctx);
+ ec_mulm(y3, y3, ec_get_two_inv_p(ctx), ctx);
+ }
+ }
+
+#undef x1
+#undef y1
+#undef z1
+#undef x2
+#undef y2
+#undef z2
+#undef x3
+#undef y3
+#undef z3
+#undef l1
+#undef l2
+#undef l3
+#undef l4
+#undef l5
+#undef l6
+#undef l7
+#undef l8
+#undef l9
+#undef t1
+#undef t2
+}
+
+/* RESULT = P1 + P2 (Montgomery version).*/
+static void add_points_montgomery(MPI_POINT result,
+ MPI_POINT p1, MPI_POINT p2,
+ struct mpi_ec_ctx *ctx)
+{
+ (void)result;
+ (void)p1;
+ (void)p2;
+ (void)ctx;
+ log_fatal("%s: %s not yet supported\n",
+ "mpi_ec_add_points", "Montgomery");
+}
+
+/* RESULT = P1 + P2 (Twisted Edwards version).*/
+static void add_points_edwards(MPI_POINT result,
+ MPI_POINT p1, MPI_POINT p2,
+ struct mpi_ec_ctx *ctx)
+{
+#define X1 (p1->x)
+#define Y1 (p1->y)
+#define Z1 (p1->z)
+#define X2 (p2->x)
+#define Y2 (p2->y)
+#define Z2 (p2->z)
+#define X3 (result->x)
+#define Y3 (result->y)
+#define Z3 (result->z)
+#define A (ctx->t.scratch[0])
+#define B (ctx->t.scratch[1])
+#define C (ctx->t.scratch[2])
+#define D (ctx->t.scratch[3])
+#define E (ctx->t.scratch[4])
+#define F (ctx->t.scratch[5])
+#define G (ctx->t.scratch[6])
+#define tmp (ctx->t.scratch[7])
+
+ point_resize(result, ctx);
+
+ /* Compute: (X_3 : Y_3 : Z_3) = (X_1 : Y_1 : Z_1) + (X_2 : Y_2 : Z_3) */
+
+ /* A = Z1 · Z2 */
+ ctx->mulm(A, Z1, Z2, ctx);
+
+ /* B = A^2 */
+ ctx->pow2(B, A, ctx);
+
+ /* C = X1 · X2 */
+ ctx->mulm(C, X1, X2, ctx);
+
+ /* D = Y1 · Y2 */
+ ctx->mulm(D, Y1, Y2, ctx);
+
+ /* E = d · C · D */
+ ctx->mulm(E, ctx->b, C, ctx);
+ ctx->mulm(E, E, D, ctx);
+
+ /* F = B - E */
+ ctx->subm(F, B, E, ctx);
+
+ /* G = B + E */
+ ctx->addm(G, B, E, ctx);
+
+ /* X_3 = A · F · ((X_1 + Y_1) · (X_2 + Y_2) - C - D) */
+ ctx->addm(tmp, X1, Y1, ctx);
+ ctx->addm(X3, X2, Y2, ctx);
+ ctx->mulm(X3, X3, tmp, ctx);
+ ctx->subm(X3, X3, C, ctx);
+ ctx->subm(X3, X3, D, ctx);
+ ctx->mulm(X3, X3, F, ctx);
+ ctx->mulm(X3, X3, A, ctx);
+
+ /* Y_3 = A · G · (D - aC) */
+ if (ctx->dialect == ECC_DIALECT_ED25519) {
+ ctx->addm(Y3, D, C, ctx);
+ } else {
+ ctx->mulm(Y3, ctx->a, C, ctx);
+ ctx->subm(Y3, D, Y3, ctx);
+ }
+ ctx->mulm(Y3, Y3, G, ctx);
+ ctx->mulm(Y3, Y3, A, ctx);
+
+ /* Z_3 = F · G */
+ ctx->mulm(Z3, F, G, ctx);
+
+
+#undef X1
+#undef Y1
+#undef Z1
+#undef X2
+#undef Y2
+#undef Z2
+#undef X3
+#undef Y3
+#undef Z3
+#undef A
+#undef B
+#undef C
+#undef D
+#undef E
+#undef F
+#undef G
+#undef tmp
+}
+
+/* Compute a step of Montgomery Ladder (only use X and Z in the point).
+ * Inputs: P1, P2, and x-coordinate of DIF = P1 - P1.
+ * Outputs: PRD = 2 * P1 and SUM = P1 + P2.
+ */
+static void montgomery_ladder(MPI_POINT prd, MPI_POINT sum,
+ MPI_POINT p1, MPI_POINT p2, MPI dif_x,
+ struct mpi_ec_ctx *ctx)
+{
+ ctx->addm(sum->x, p2->x, p2->z, ctx);
+ ctx->subm(p2->z, p2->x, p2->z, ctx);
+ ctx->addm(prd->x, p1->x, p1->z, ctx);
+ ctx->subm(p1->z, p1->x, p1->z, ctx);
+ ctx->mulm(p2->x, p1->z, sum->x, ctx);
+ ctx->mulm(p2->z, prd->x, p2->z, ctx);
+ ctx->pow2(p1->x, prd->x, ctx);
+ ctx->pow2(p1->z, p1->z, ctx);
+ ctx->addm(sum->x, p2->x, p2->z, ctx);
+ ctx->subm(p2->z, p2->x, p2->z, ctx);
+ ctx->mulm(prd->x, p1->x, p1->z, ctx);
+ ctx->subm(p1->z, p1->x, p1->z, ctx);
+ ctx->pow2(sum->x, sum->x, ctx);
+ ctx->pow2(sum->z, p2->z, ctx);
+ ctx->mulm(prd->z, p1->z, ctx->a, ctx); /* CTX->A: (a-2)/4 */
+ ctx->mulm(sum->z, sum->z, dif_x, ctx);
+ ctx->addm(prd->z, p1->x, prd->z, ctx);
+ ctx->mulm(prd->z, prd->z, p1->z, ctx);
+}
+
+/* RESULT = P1 + P2 */
+void mpi_ec_add_points(MPI_POINT result,
+ MPI_POINT p1, MPI_POINT p2,
+ struct mpi_ec_ctx *ctx)
+{
+ switch (ctx->model) {
+ case MPI_EC_WEIERSTRASS:
+ add_points_weierstrass(result, p1, p2, ctx);
+ break;
+ case MPI_EC_MONTGOMERY:
+ add_points_montgomery(result, p1, p2, ctx);
+ break;
+ case MPI_EC_EDWARDS:
+ add_points_edwards(result, p1, p2, ctx);
+ break;
+ }
+}
+EXPORT_SYMBOL_GPL(mpi_ec_add_points);
+
+/* Scalar point multiplication - the main function for ECC. If takes
+ * an integer SCALAR and a POINT as well as the usual context CTX.
+ * RESULT will be set to the resulting point.
+ */
+void mpi_ec_mul_point(MPI_POINT result,
+ MPI scalar, MPI_POINT point,
+ struct mpi_ec_ctx *ctx)
+{
+ MPI x1, y1, z1, k, h, yy;
+ unsigned int i, loops;
+ struct gcry_mpi_point p1, p2, p1inv;
+
+ if (ctx->model == MPI_EC_EDWARDS) {
+ /* Simple left to right binary method. Algorithm 3.27 from
+ * {author={Hankerson, Darrel and Menezes, Alfred J. and Vanstone, Scott},
+ * title = {Guide to Elliptic Curve Cryptography},
+ * year = {2003}, isbn = {038795273X},
+ * url = {http://www.cacr.math.uwaterloo.ca/ecc/},
+ * publisher = {Springer-Verlag New York, Inc.}}
+ */
+ unsigned int nbits;
+ int j;
+
+ if (mpi_cmp(scalar, ctx->p) >= 0)
+ nbits = mpi_get_nbits(scalar);
+ else
+ nbits = mpi_get_nbits(ctx->p);
+
+ mpi_set_ui(result->x, 0);
+ mpi_set_ui(result->y, 1);
+ mpi_set_ui(result->z, 1);
+ point_resize(point, ctx);
+
+ point_resize(result, ctx);
+ point_resize(point, ctx);
+
+ for (j = nbits-1; j >= 0; j--) {
+ mpi_ec_dup_point(result, result, ctx);
+ if (mpi_test_bit(scalar, j))
+ mpi_ec_add_points(result, result, point, ctx);
+ }
+ return;
+ } else if (ctx->model == MPI_EC_MONTGOMERY) {
+ unsigned int nbits;
+ int j;
+ struct gcry_mpi_point p1_, p2_;
+ MPI_POINT q1, q2, prd, sum;
+ unsigned long sw;
+ mpi_size_t rsize;
+ int scalar_copied = 0;
+
+ /* Compute scalar point multiplication with Montgomery Ladder.
+ * Note that we don't use Y-coordinate in the points at all.
+ * RESULT->Y will be filled by zero.
+ */
+
+ nbits = mpi_get_nbits(scalar);
+ point_init(&p1);
+ point_init(&p2);
+ point_init(&p1_);
+ point_init(&p2_);
+ mpi_set_ui(p1.x, 1);
+ mpi_free(p2.x);
+ p2.x = mpi_copy(point->x);
+ mpi_set_ui(p2.z, 1);
+
+ point_resize(&p1, ctx);
+ point_resize(&p2, ctx);
+ point_resize(&p1_, ctx);
+ point_resize(&p2_, ctx);
+
+ mpi_resize(point->x, ctx->p->nlimbs);
+ point->x->nlimbs = ctx->p->nlimbs;
+
+ q1 = &p1;
+ q2 = &p2;
+ prd = &p1_;
+ sum = &p2_;
+
+ for (j = nbits-1; j >= 0; j--) {
+ MPI_POINT t;
+
+ sw = mpi_test_bit(scalar, j);
+ point_swap_cond(q1, q2, sw, ctx);
+ montgomery_ladder(prd, sum, q1, q2, point->x, ctx);
+ point_swap_cond(prd, sum, sw, ctx);
+ t = q1; q1 = prd; prd = t;
+ t = q2; q2 = sum; sum = t;
+ }
+
+ mpi_clear(result->y);
+ sw = (nbits & 1);
+ point_swap_cond(&p1, &p1_, sw, ctx);
+
+ rsize = p1.z->nlimbs;
+ MPN_NORMALIZE(p1.z->d, rsize);
+ if (rsize == 0) {
+ mpi_set_ui(result->x, 1);
+ mpi_set_ui(result->z, 0);
+ } else {
+ z1 = mpi_new(0);
+ ec_invm(z1, p1.z, ctx);
+ ec_mulm(result->x, p1.x, z1, ctx);
+ mpi_set_ui(result->z, 1);
+ mpi_free(z1);
+ }
+
+ point_free(&p1);
+ point_free(&p2);
+ point_free(&p1_);
+ point_free(&p2_);
+ if (scalar_copied)
+ mpi_free(scalar);
+ return;
+ }
+
+ x1 = mpi_alloc_like(ctx->p);
+ y1 = mpi_alloc_like(ctx->p);
+ h = mpi_alloc_like(ctx->p);
+ k = mpi_copy(scalar);
+ yy = mpi_copy(point->y);
+
+ if (mpi_has_sign(k)) {
+ k->sign = 0;
+ ec_invm(yy, yy, ctx);
+ }
+
+ if (!mpi_cmp_ui(point->z, 1)) {
+ mpi_set(x1, point->x);
+ mpi_set(y1, yy);
+ } else {
+ MPI z2, z3;
+
+ z2 = mpi_alloc_like(ctx->p);
+ z3 = mpi_alloc_like(ctx->p);
+ ec_mulm(z2, point->z, point->z, ctx);
+ ec_mulm(z3, point->z, z2, ctx);
+ ec_invm(z2, z2, ctx);
+ ec_mulm(x1, point->x, z2, ctx);
+ ec_invm(z3, z3, ctx);
+ ec_mulm(y1, yy, z3, ctx);
+ mpi_free(z2);
+ mpi_free(z3);
+ }
+ z1 = mpi_copy(mpi_const(MPI_C_ONE));
+
+ mpi_mul(h, k, mpi_const(MPI_C_THREE)); /* h = 3k */
+ loops = mpi_get_nbits(h);
+ if (loops < 2) {
+ /* If SCALAR is zero, the above mpi_mul sets H to zero and thus
+ * LOOPs will be zero. To avoid an underflow of I in the main
+ * loop we set LOOP to 2 and the result to (0,0,0).
+ */
+ loops = 2;
+ mpi_clear(result->x);
+ mpi_clear(result->y);
+ mpi_clear(result->z);
+ } else {
+ mpi_set(result->x, point->x);
+ mpi_set(result->y, yy);
+ mpi_set(result->z, point->z);
+ }
+ mpi_free(yy); yy = NULL;
+
+ p1.x = x1; x1 = NULL;
+ p1.y = y1; y1 = NULL;
+ p1.z = z1; z1 = NULL;
+ point_init(&p2);
+ point_init(&p1inv);
+
+ /* Invert point: y = p - y mod p */
+ point_set(&p1inv, &p1);
+ ec_subm(p1inv.y, ctx->p, p1inv.y, ctx);
+
+ for (i = loops-2; i > 0; i--) {
+ mpi_ec_dup_point(result, result, ctx);
+ if (mpi_test_bit(h, i) == 1 && mpi_test_bit(k, i) == 0) {
+ point_set(&p2, result);
+ mpi_ec_add_points(result, &p2, &p1, ctx);
+ }
+ if (mpi_test_bit(h, i) == 0 && mpi_test_bit(k, i) == 1) {
+ point_set(&p2, result);
+ mpi_ec_add_points(result, &p2, &p1inv, ctx);
+ }
+ }
+
+ point_free(&p1);
+ point_free(&p2);
+ point_free(&p1inv);
+ mpi_free(h);
+ mpi_free(k);
+}
+EXPORT_SYMBOL_GPL(mpi_ec_mul_point);
+
+/* Return true if POINT is on the curve described by CTX. */
+int mpi_ec_curve_point(MPI_POINT point, struct mpi_ec_ctx *ctx)
+{
+ int res = 0;
+ MPI x, y, w;
+
+ x = mpi_new(0);
+ y = mpi_new(0);
+ w = mpi_new(0);
+
+ /* Check that the point is in range. This needs to be done here and
+ * not after conversion to affine coordinates.
+ */
+ if (mpi_cmpabs(point->x, ctx->p) >= 0)
+ goto leave;
+ if (mpi_cmpabs(point->y, ctx->p) >= 0)
+ goto leave;
+ if (mpi_cmpabs(point->z, ctx->p) >= 0)
+ goto leave;
+
+ switch (ctx->model) {
+ case MPI_EC_WEIERSTRASS:
+ {
+ MPI xxx;
+
+ if (mpi_ec_get_affine(x, y, point, ctx))
+ goto leave;
+
+ xxx = mpi_new(0);
+
+ /* y^2 == x^3 + a·x + b */
+ ec_pow2(y, y, ctx);
+
+ ec_pow3(xxx, x, ctx);
+ ec_mulm(w, ctx->a, x, ctx);
+ ec_addm(w, w, ctx->b, ctx);
+ ec_addm(w, w, xxx, ctx);
+
+ if (!mpi_cmp(y, w))
+ res = 1;
+
+ mpi_free(xxx);
+ }
+ break;
+
+ case MPI_EC_MONTGOMERY:
+ {
+#define xx y
+ /* With Montgomery curve, only X-coordinate is valid. */
+ if (mpi_ec_get_affine(x, NULL, point, ctx))
+ goto leave;
+
+ /* The equation is: b * y^2 == x^3 + a · x^2 + x */
+ /* We check if right hand is quadratic residue or not by
+ * Euler's criterion.
+ */
+ /* CTX->A has (a-2)/4 and CTX->B has b^-1 */
+ ec_mulm(w, ctx->a, mpi_const(MPI_C_FOUR), ctx);
+ ec_addm(w, w, mpi_const(MPI_C_TWO), ctx);
+ ec_mulm(w, w, x, ctx);
+ ec_pow2(xx, x, ctx);
+ ec_addm(w, w, xx, ctx);
+ ec_addm(w, w, mpi_const(MPI_C_ONE), ctx);
+ ec_mulm(w, w, x, ctx);
+ ec_mulm(w, w, ctx->b, ctx);
+#undef xx
+ /* Compute Euler's criterion: w^(p-1)/2 */
+#define p_minus1 y
+ ec_subm(p_minus1, ctx->p, mpi_const(MPI_C_ONE), ctx);
+ mpi_rshift(p_minus1, p_minus1, 1);
+ ec_powm(w, w, p_minus1, ctx);
+
+ res = !mpi_cmp_ui(w, 1);
+#undef p_minus1
+ }
+ break;
+
+ case MPI_EC_EDWARDS:
+ {
+ if (mpi_ec_get_affine(x, y, point, ctx))
+ goto leave;
+
+ mpi_resize(w, ctx->p->nlimbs);
+ w->nlimbs = ctx->p->nlimbs;
+
+ /* a · x^2 + y^2 - 1 - b · x^2 · y^2 == 0 */
+ ctx->pow2(x, x, ctx);
+ ctx->pow2(y, y, ctx);
+ if (ctx->dialect == ECC_DIALECT_ED25519)
+ ctx->subm(w, ctx->p, x, ctx);
+ else
+ ctx->mulm(w, ctx->a, x, ctx);
+ ctx->addm(w, w, y, ctx);
+ ctx->mulm(x, x, y, ctx);
+ ctx->mulm(x, x, ctx->b, ctx);
+ ctx->subm(w, w, x, ctx);
+ if (!mpi_cmp_ui(w, 1))
+ res = 1;
+ }
+ break;
+ }
+
+leave:
+ mpi_free(w);
+ mpi_free(x);
+ mpi_free(y);
+
+ return res;
+}
+EXPORT_SYMBOL_GPL(mpi_ec_curve_point);
--- /dev/null
+/* mpi-add.c - MPI functions
+ * Copyright (C) 1994, 1996, 1998, 2001, 2002,
+ * 2003 Free Software Foundation, Inc.
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Note: This code is heavily based on the GNU MP Library.
+ * Actually it's the same code with only minor changes in the
+ * way the data is stored; this is to support the abstraction
+ * of an optional secure memory allocation which may be used
+ * to avoid revealing of sensitive data due to paging etc.
+ */
+
+#include "mpi-internal.h"
+
+/****************
+ * Add the unsigned integer V to the mpi-integer U and store the
+ * result in W. U and V may be the same.
+ */
+void mpi_add_ui(MPI w, MPI u, unsigned long v)
+{
+ mpi_ptr_t wp, up;
+ mpi_size_t usize, wsize;
+ int usign, wsign;
+
+ usize = u->nlimbs;
+ usign = u->sign;
+ wsign = 0;
+
+ /* If not space for W (and possible carry), increase space. */
+ wsize = usize + 1;
+ if (w->alloced < wsize)
+ mpi_resize(w, wsize);
+
+ /* These must be after realloc (U may be the same as W). */
+ up = u->d;
+ wp = w->d;
+
+ if (!usize) { /* simple */
+ wp[0] = v;
+ wsize = v ? 1:0;
+ } else if (!usign) { /* mpi is not negative */
+ mpi_limb_t cy;
+ cy = mpihelp_add_1(wp, up, usize, v);
+ wp[usize] = cy;
+ wsize = usize + cy;
+ } else {
+ /* The signs are different. Need exact comparison to determine
+ * which operand to subtract from which.
+ */
+ if (usize == 1 && up[0] < v) {
+ wp[0] = v - up[0];
+ wsize = 1;
+ } else {
+ mpihelp_sub_1(wp, up, usize, v);
+ /* Size can decrease with at most one limb. */
+ wsize = usize - (wp[usize-1] == 0);
+ wsign = 1;
+ }
+ }
+
+ w->nlimbs = wsize;
+ w->sign = wsign;
+}
+
+
+void mpi_add(MPI w, MPI u, MPI v)
+{
+ mpi_ptr_t wp, up, vp;
+ mpi_size_t usize, vsize, wsize;
+ int usign, vsign, wsign;
+
+ if (u->nlimbs < v->nlimbs) { /* Swap U and V. */
+ usize = v->nlimbs;
+ usign = v->sign;
+ vsize = u->nlimbs;
+ vsign = u->sign;
+ wsize = usize + 1;
+ RESIZE_IF_NEEDED(w, wsize);
+ /* These must be after realloc (u or v may be the same as w). */
+ up = v->d;
+ vp = u->d;
+ } else {
+ usize = u->nlimbs;
+ usign = u->sign;
+ vsize = v->nlimbs;
+ vsign = v->sign;
+ wsize = usize + 1;
+ RESIZE_IF_NEEDED(w, wsize);
+ /* These must be after realloc (u or v may be the same as w). */
+ up = u->d;
+ vp = v->d;
+ }
+ wp = w->d;
+ wsign = 0;
+
+ if (!vsize) { /* simple */
+ MPN_COPY(wp, up, usize);
+ wsize = usize;
+ wsign = usign;
+ } else if (usign != vsign) { /* different sign */
+ /* This test is right since USIZE >= VSIZE */
+ if (usize != vsize) {
+ mpihelp_sub(wp, up, usize, vp, vsize);
+ wsize = usize;
+ MPN_NORMALIZE(wp, wsize);
+ wsign = usign;
+ } else if (mpihelp_cmp(up, vp, usize) < 0) {
+ mpihelp_sub_n(wp, vp, up, usize);
+ wsize = usize;
+ MPN_NORMALIZE(wp, wsize);
+ if (!usign)
+ wsign = 1;
+ } else {
+ mpihelp_sub_n(wp, up, vp, usize);
+ wsize = usize;
+ MPN_NORMALIZE(wp, wsize);
+ if (usign)
+ wsign = 1;
+ }
+ } else { /* U and V have same sign. Add them. */
+ mpi_limb_t cy = mpihelp_add(wp, up, usize, vp, vsize);
+ wp[usize] = cy;
+ wsize = usize + cy;
+ if (usign)
+ wsign = 1;
+ }
+
+ w->nlimbs = wsize;
+ w->sign = wsign;
+}
+EXPORT_SYMBOL_GPL(mpi_add);
+
+void mpi_sub(MPI w, MPI u, MPI v)
+{
+ MPI vv = mpi_copy(v);
+ vv->sign = !vv->sign;
+ mpi_add(w, u, vv);
+ mpi_free(vv);
+}
+
+
+void mpi_addm(MPI w, MPI u, MPI v, MPI m)
+{
+ mpi_add(w, u, v);
+ mpi_mod(w, w, m);
+}
+EXPORT_SYMBOL_GPL(mpi_addm);
+
+void mpi_subm(MPI w, MPI u, MPI v, MPI m)
+{
+ mpi_sub(w, u, v);
+ mpi_mod(w, w, m);
+}
+EXPORT_SYMBOL_GPL(mpi_subm);
for (; a->nlimbs && !a->d[a->nlimbs - 1]; a->nlimbs--)
;
}
+EXPORT_SYMBOL_GPL(mpi_normalize);
/****************
* Return the number of bits in A.
return n;
}
EXPORT_SYMBOL_GPL(mpi_get_nbits);
+
+/****************
+ * Test whether bit N is set.
+ */
+int mpi_test_bit(MPI a, unsigned int n)
+{
+ unsigned int limbno, bitno;
+ mpi_limb_t limb;
+
+ limbno = n / BITS_PER_MPI_LIMB;
+ bitno = n % BITS_PER_MPI_LIMB;
+
+ if (limbno >= a->nlimbs)
+ return 0; /* too far left: this is a 0 */
+ limb = a->d[limbno];
+ return (limb & (A_LIMB_1 << bitno)) ? 1 : 0;
+}
+EXPORT_SYMBOL_GPL(mpi_test_bit);
+
+/****************
+ * Set bit N of A.
+ */
+void mpi_set_bit(MPI a, unsigned int n)
+{
+ unsigned int i, limbno, bitno;
+
+ limbno = n / BITS_PER_MPI_LIMB;
+ bitno = n % BITS_PER_MPI_LIMB;
+
+ if (limbno >= a->nlimbs) {
+ for (i = a->nlimbs; i < a->alloced; i++)
+ a->d[i] = 0;
+ mpi_resize(a, limbno+1);
+ a->nlimbs = limbno+1;
+ }
+ a->d[limbno] |= (A_LIMB_1<<bitno);
+}
+
+/****************
+ * Set bit N of A. and clear all bits above
+ */
+void mpi_set_highbit(MPI a, unsigned int n)
+{
+ unsigned int i, limbno, bitno;
+
+ limbno = n / BITS_PER_MPI_LIMB;
+ bitno = n % BITS_PER_MPI_LIMB;
+
+ if (limbno >= a->nlimbs) {
+ for (i = a->nlimbs; i < a->alloced; i++)
+ a->d[i] = 0;
+ mpi_resize(a, limbno+1);
+ a->nlimbs = limbno+1;
+ }
+ a->d[limbno] |= (A_LIMB_1<<bitno);
+ for (bitno++; bitno < BITS_PER_MPI_LIMB; bitno++)
+ a->d[limbno] &= ~(A_LIMB_1 << bitno);
+ a->nlimbs = limbno+1;
+}
+EXPORT_SYMBOL_GPL(mpi_set_highbit);
+
+/****************
+ * clear bit N of A and all bits above
+ */
+void mpi_clear_highbit(MPI a, unsigned int n)
+{
+ unsigned int limbno, bitno;
+
+ limbno = n / BITS_PER_MPI_LIMB;
+ bitno = n % BITS_PER_MPI_LIMB;
+
+ if (limbno >= a->nlimbs)
+ return; /* not allocated, therefore no need to clear bits :-) */
+
+ for ( ; bitno < BITS_PER_MPI_LIMB; bitno++)
+ a->d[limbno] &= ~(A_LIMB_1 << bitno);
+ a->nlimbs = limbno+1;
+}
+
+/****************
+ * Clear bit N of A.
+ */
+void mpi_clear_bit(MPI a, unsigned int n)
+{
+ unsigned int limbno, bitno;
+
+ limbno = n / BITS_PER_MPI_LIMB;
+ bitno = n % BITS_PER_MPI_LIMB;
+
+ if (limbno >= a->nlimbs)
+ return; /* Don't need to clear this bit, it's far too left. */
+ a->d[limbno] &= ~(A_LIMB_1 << bitno);
+}
+EXPORT_SYMBOL_GPL(mpi_clear_bit);
+
+
+/****************
+ * Shift A by COUNT limbs to the right
+ * This is used only within the MPI library
+ */
+void mpi_rshift_limbs(MPI a, unsigned int count)
+{
+ mpi_ptr_t ap = a->d;
+ mpi_size_t n = a->nlimbs;
+ unsigned int i;
+
+ if (count >= n) {
+ a->nlimbs = 0;
+ return;
+ }
+
+ for (i = 0; i < n - count; i++)
+ ap[i] = ap[i+count];
+ ap[i] = 0;
+ a->nlimbs -= count;
+}
+
+/*
+ * Shift A by N bits to the right.
+ */
+void mpi_rshift(MPI x, MPI a, unsigned int n)
+{
+ mpi_size_t xsize;
+ unsigned int i;
+ unsigned int nlimbs = (n/BITS_PER_MPI_LIMB);
+ unsigned int nbits = (n%BITS_PER_MPI_LIMB);
+
+ if (x == a) {
+ /* In-place operation. */
+ if (nlimbs >= x->nlimbs) {
+ x->nlimbs = 0;
+ return;
+ }
+
+ if (nlimbs) {
+ for (i = 0; i < x->nlimbs - nlimbs; i++)
+ x->d[i] = x->d[i+nlimbs];
+ x->d[i] = 0;
+ x->nlimbs -= nlimbs;
+ }
+ if (x->nlimbs && nbits)
+ mpihelp_rshift(x->d, x->d, x->nlimbs, nbits);
+ } else if (nlimbs) {
+ /* Copy and shift by more or equal bits than in a limb. */
+ xsize = a->nlimbs;
+ x->sign = a->sign;
+ RESIZE_IF_NEEDED(x, xsize);
+ x->nlimbs = xsize;
+ for (i = 0; i < a->nlimbs; i++)
+ x->d[i] = a->d[i];
+ x->nlimbs = i;
+
+ if (nlimbs >= x->nlimbs) {
+ x->nlimbs = 0;
+ return;
+ }
+
+ if (nlimbs) {
+ for (i = 0; i < x->nlimbs - nlimbs; i++)
+ x->d[i] = x->d[i+nlimbs];
+ x->d[i] = 0;
+ x->nlimbs -= nlimbs;
+ }
+
+ if (x->nlimbs && nbits)
+ mpihelp_rshift(x->d, x->d, x->nlimbs, nbits);
+ } else {
+ /* Copy and shift by less than bits in a limb. */
+ xsize = a->nlimbs;
+ x->sign = a->sign;
+ RESIZE_IF_NEEDED(x, xsize);
+ x->nlimbs = xsize;
+
+ if (xsize) {
+ if (nbits)
+ mpihelp_rshift(x->d, a->d, x->nlimbs, nbits);
+ else {
+ /* The rshift helper function is not specified for
+ * NBITS==0, thus we do a plain copy here.
+ */
+ for (i = 0; i < x->nlimbs; i++)
+ x->d[i] = a->d[i];
+ }
+ }
+ }
+ MPN_NORMALIZE(x->d, x->nlimbs);
+}
+
+/****************
+ * Shift A by COUNT limbs to the left
+ * This is used only within the MPI library
+ */
+void mpi_lshift_limbs(MPI a, unsigned int count)
+{
+ mpi_ptr_t ap;
+ int n = a->nlimbs;
+ int i;
+
+ if (!count || !n)
+ return;
+
+ RESIZE_IF_NEEDED(a, n+count);
+
+ ap = a->d;
+ for (i = n-1; i >= 0; i--)
+ ap[i+count] = ap[i];
+ for (i = 0; i < count; i++)
+ ap[i] = 0;
+ a->nlimbs += count;
+}
+
+/*
+ * Shift A by N bits to the left.
+ */
+void mpi_lshift(MPI x, MPI a, unsigned int n)
+{
+ unsigned int nlimbs = (n/BITS_PER_MPI_LIMB);
+ unsigned int nbits = (n%BITS_PER_MPI_LIMB);
+
+ if (x == a && !n)
+ return; /* In-place shift with an amount of zero. */
+
+ if (x != a) {
+ /* Copy A to X. */
+ unsigned int alimbs = a->nlimbs;
+ int asign = a->sign;
+ mpi_ptr_t xp, ap;
+
+ RESIZE_IF_NEEDED(x, alimbs+nlimbs+1);
+ xp = x->d;
+ ap = a->d;
+ MPN_COPY(xp, ap, alimbs);
+ x->nlimbs = alimbs;
+ x->flags = a->flags;
+ x->sign = asign;
+ }
+
+ if (nlimbs && !nbits) {
+ /* Shift a full number of limbs. */
+ mpi_lshift_limbs(x, nlimbs);
+ } else if (n) {
+ /* We use a very dump approach: Shift left by the number of
+ * limbs plus one and than fix it up by an rshift.
+ */
+ mpi_lshift_limbs(x, nlimbs+1);
+ mpi_rshift(x, x, BITS_PER_MPI_LIMB - nbits);
+ }
+
+ MPN_NORMALIZE(x->d, x->nlimbs);
+}
}
EXPORT_SYMBOL_GPL(mpi_cmp_ui);
-int mpi_cmp(MPI u, MPI v)
+static int do_mpi_cmp(MPI u, MPI v, int absmode)
{
- mpi_size_t usize, vsize;
+ mpi_size_t usize;
+ mpi_size_t vsize;
+ int usign;
+ int vsign;
int cmp;
mpi_normalize(u);
mpi_normalize(v);
+
usize = u->nlimbs;
vsize = v->nlimbs;
- if (!u->sign && v->sign)
+ usign = absmode ? 0 : u->sign;
+ vsign = absmode ? 0 : v->sign;
+
+ /* Compare sign bits. */
+
+ if (!usign && vsign)
return 1;
- if (u->sign && !v->sign)
+ if (usign && !vsign)
return -1;
- if (usize != vsize && !u->sign && !v->sign)
+
+ /* U and V are either both positive or both negative. */
+
+ if (usize != vsize && !usign && !vsign)
return usize - vsize;
- if (usize != vsize && u->sign && v->sign)
- return vsize - usize;
+ if (usize != vsize && usign && vsign)
+ return vsize + usize;
if (!usize)
return 0;
cmp = mpihelp_cmp(u->d, v->d, usize);
- if (u->sign)
- return -cmp;
- return cmp;
+ if (!cmp)
+ return 0;
+ if ((cmp < 0?1:0) == (usign?1:0))
+ return 1;
+
+ return -1;
+}
+
+int mpi_cmp(MPI u, MPI v)
+{
+ return do_mpi_cmp(u, v, 0);
}
EXPORT_SYMBOL_GPL(mpi_cmp);
+
+int mpi_cmpabs(MPI u, MPI v)
+{
+ return do_mpi_cmp(u, v, 1);
+}
+EXPORT_SYMBOL_GPL(mpi_cmpabs);
--- /dev/null
+/* mpi-div.c - MPI functions
+ * Copyright (C) 1994, 1996, 1998, 2001, 2002,
+ * 2003 Free Software Foundation, Inc.
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Note: This code is heavily based on the GNU MP Library.
+ * Actually it's the same code with only minor changes in the
+ * way the data is stored; this is to support the abstraction
+ * of an optional secure memory allocation which may be used
+ * to avoid revealing of sensitive data due to paging etc.
+ */
+
+#include "mpi-internal.h"
+#include "longlong.h"
+
+void mpi_tdiv_qr(MPI quot, MPI rem, MPI num, MPI den);
+void mpi_fdiv_qr(MPI quot, MPI rem, MPI dividend, MPI divisor);
+
+void mpi_fdiv_r(MPI rem, MPI dividend, MPI divisor)
+{
+ int divisor_sign = divisor->sign;
+ MPI temp_divisor = NULL;
+
+ /* We need the original value of the divisor after the remainder has been
+ * preliminary calculated. We have to copy it to temporary space if it's
+ * the same variable as REM.
+ */
+ if (rem == divisor) {
+ temp_divisor = mpi_copy(divisor);
+ divisor = temp_divisor;
+ }
+
+ mpi_tdiv_r(rem, dividend, divisor);
+
+ if (((divisor_sign?1:0) ^ (dividend->sign?1:0)) && rem->nlimbs)
+ mpi_add(rem, rem, divisor);
+
+ if (temp_divisor)
+ mpi_free(temp_divisor);
+}
+
+void mpi_fdiv_q(MPI quot, MPI dividend, MPI divisor)
+{
+ MPI tmp = mpi_alloc(mpi_get_nlimbs(quot));
+ mpi_fdiv_qr(quot, tmp, dividend, divisor);
+ mpi_free(tmp);
+}
+
+void mpi_fdiv_qr(MPI quot, MPI rem, MPI dividend, MPI divisor)
+{
+ int divisor_sign = divisor->sign;
+ MPI temp_divisor = NULL;
+
+ if (quot == divisor || rem == divisor) {
+ temp_divisor = mpi_copy(divisor);
+ divisor = temp_divisor;
+ }
+
+ mpi_tdiv_qr(quot, rem, dividend, divisor);
+
+ if ((divisor_sign ^ dividend->sign) && rem->nlimbs) {
+ mpi_sub_ui(quot, quot, 1);
+ mpi_add(rem, rem, divisor);
+ }
+
+ if (temp_divisor)
+ mpi_free(temp_divisor);
+}
+
+/* If den == quot, den needs temporary storage.
+ * If den == rem, den needs temporary storage.
+ * If num == quot, num needs temporary storage.
+ * If den has temporary storage, it can be normalized while being copied,
+ * i.e no extra storage should be allocated.
+ */
+
+void mpi_tdiv_r(MPI rem, MPI num, MPI den)
+{
+ mpi_tdiv_qr(NULL, rem, num, den);
+}
+
+void mpi_tdiv_qr(MPI quot, MPI rem, MPI num, MPI den)
+{
+ mpi_ptr_t np, dp;
+ mpi_ptr_t qp, rp;
+ mpi_size_t nsize = num->nlimbs;
+ mpi_size_t dsize = den->nlimbs;
+ mpi_size_t qsize, rsize;
+ mpi_size_t sign_remainder = num->sign;
+ mpi_size_t sign_quotient = num->sign ^ den->sign;
+ unsigned int normalization_steps;
+ mpi_limb_t q_limb;
+ mpi_ptr_t marker[5];
+ int markidx = 0;
+
+ /* Ensure space is enough for quotient and remainder.
+ * We need space for an extra limb in the remainder, because it's
+ * up-shifted (normalized) below.
+ */
+ rsize = nsize + 1;
+ mpi_resize(rem, rsize);
+
+ qsize = rsize - dsize; /* qsize cannot be bigger than this. */
+ if (qsize <= 0) {
+ if (num != rem) {
+ rem->nlimbs = num->nlimbs;
+ rem->sign = num->sign;
+ MPN_COPY(rem->d, num->d, nsize);
+ }
+ if (quot) {
+ /* This needs to follow the assignment to rem, in case the
+ * numerator and quotient are the same.
+ */
+ quot->nlimbs = 0;
+ quot->sign = 0;
+ }
+ return;
+ }
+
+ if (quot)
+ mpi_resize(quot, qsize);
+
+ /* Read pointers here, when reallocation is finished. */
+ np = num->d;
+ dp = den->d;
+ rp = rem->d;
+
+ /* Optimize division by a single-limb divisor. */
+ if (dsize == 1) {
+ mpi_limb_t rlimb;
+ if (quot) {
+ qp = quot->d;
+ rlimb = mpihelp_divmod_1(qp, np, nsize, dp[0]);
+ qsize -= qp[qsize - 1] == 0;
+ quot->nlimbs = qsize;
+ quot->sign = sign_quotient;
+ } else
+ rlimb = mpihelp_mod_1(np, nsize, dp[0]);
+ rp[0] = rlimb;
+ rsize = rlimb != 0?1:0;
+ rem->nlimbs = rsize;
+ rem->sign = sign_remainder;
+ return;
+ }
+
+
+ if (quot) {
+ qp = quot->d;
+ /* Make sure QP and NP point to different objects. Otherwise the
+ * numerator would be gradually overwritten by the quotient limbs.
+ */
+ if (qp == np) { /* Copy NP object to temporary space. */
+ np = marker[markidx++] = mpi_alloc_limb_space(nsize);
+ MPN_COPY(np, qp, nsize);
+ }
+ } else /* Put quotient at top of remainder. */
+ qp = rp + dsize;
+
+ normalization_steps = count_leading_zeros(dp[dsize - 1]);
+
+ /* Normalize the denominator, i.e. make its most significant bit set by
+ * shifting it NORMALIZATION_STEPS bits to the left. Also shift the
+ * numerator the same number of steps (to keep the quotient the same!).
+ */
+ if (normalization_steps) {
+ mpi_ptr_t tp;
+ mpi_limb_t nlimb;
+
+ /* Shift up the denominator setting the most significant bit of
+ * the most significant word. Use temporary storage not to clobber
+ * the original contents of the denominator.
+ */
+ tp = marker[markidx++] = mpi_alloc_limb_space(dsize);
+ mpihelp_lshift(tp, dp, dsize, normalization_steps);
+ dp = tp;
+
+ /* Shift up the numerator, possibly introducing a new most
+ * significant word. Move the shifted numerator in the remainder
+ * meanwhile.
+ */
+ nlimb = mpihelp_lshift(rp, np, nsize, normalization_steps);
+ if (nlimb) {
+ rp[nsize] = nlimb;
+ rsize = nsize + 1;
+ } else
+ rsize = nsize;
+ } else {
+ /* The denominator is already normalized, as required. Copy it to
+ * temporary space if it overlaps with the quotient or remainder.
+ */
+ if (dp == rp || (quot && (dp == qp))) {
+ mpi_ptr_t tp;
+
+ tp = marker[markidx++] = mpi_alloc_limb_space(dsize);
+ MPN_COPY(tp, dp, dsize);
+ dp = tp;
+ }
+
+ /* Move the numerator to the remainder. */
+ if (rp != np)
+ MPN_COPY(rp, np, nsize);
+
+ rsize = nsize;
+ }
+
+ q_limb = mpihelp_divrem(qp, 0, rp, rsize, dp, dsize);
+
+ if (quot) {
+ qsize = rsize - dsize;
+ if (q_limb) {
+ qp[qsize] = q_limb;
+ qsize += 1;
+ }
+
+ quot->nlimbs = qsize;
+ quot->sign = sign_quotient;
+ }
+
+ rsize = dsize;
+ MPN_NORMALIZE(rp, rsize);
+
+ if (normalization_steps && rsize) {
+ mpihelp_rshift(rp, rp, rsize, normalization_steps);
+ rsize -= rp[rsize - 1] == 0?1:0;
+ }
+
+ rem->nlimbs = rsize;
+ rem->sign = sign_remainder;
+ while (markidx) {
+ markidx--;
+ mpi_free_limb_space(marker[markidx]);
+ }
+}
typedef mpi_limb_t *mpi_ptr_t; /* pointer to a limb */
typedef int mpi_size_t; /* (must be a signed type) */
+#define RESIZE_IF_NEEDED(a, b) \
+ do { \
+ if ((a)->alloced < (b)) \
+ mpi_resize((a), (b)); \
+ } while (0)
+
/* Copy N limbs from S to D. */
#define MPN_COPY(d, s, n) \
do { \
(d)[_i] = (s)[_i]; \
} while (0)
+#define MPN_COPY_INCR(d, s, n) \
+ do { \
+ mpi_size_t _i; \
+ for (_i = 0; _i < (n); _i++) \
+ (d)[_i] = (s)[_i]; \
+ } while (0)
+
+
#define MPN_COPY_DECR(d, s, n) \
do { \
mpi_size_t _i; \
mul_n(prodp, up, vp, size, tspace); \
} while (0);
+/* Divide the two-limb number in (NH,,NL) by D, with DI being the largest
+ * limb not larger than (2**(2*BITS_PER_MP_LIMB))/D - (2**BITS_PER_MP_LIMB).
+ * If this would yield overflow, DI should be the largest possible number
+ * (i.e., only ones). For correct operation, the most significant bit of D
+ * has to be set. Put the quotient in Q and the remainder in R.
+ */
+#define UDIV_QRNND_PREINV(q, r, nh, nl, d, di) \
+ do { \
+ mpi_limb_t _ql __maybe_unused; \
+ mpi_limb_t _q, _r; \
+ mpi_limb_t _xh, _xl; \
+ umul_ppmm(_q, _ql, (nh), (di)); \
+ _q += (nh); /* DI is 2**BITS_PER_MPI_LIMB too small */ \
+ umul_ppmm(_xh, _xl, _q, (d)); \
+ sub_ddmmss(_xh, _r, (nh), (nl), _xh, _xl); \
+ if (_xh) { \
+ sub_ddmmss(_xh, _r, _xh, _r, 0, (d)); \
+ _q++; \
+ if (_xh) { \
+ sub_ddmmss(_xh, _r, _xh, _r, 0, (d)); \
+ _q++; \
+ } \
+ } \
+ if (_r >= (d)) { \
+ _r -= (d); \
+ _q++; \
+ } \
+ (r) = _r; \
+ (q) = _q; \
+ } while (0)
+
+
/*-- mpiutil.c --*/
mpi_ptr_t mpi_alloc_limb_space(unsigned nlimbs);
void mpi_free_limb_space(mpi_ptr_t a);
void mpih_sqr_n_basecase(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t size);
void mpih_sqr_n(mpi_ptr_t prodp, mpi_ptr_t up, mpi_size_t size,
mpi_ptr_t tspace);
+void mpihelp_mul_n(mpi_ptr_t prodp,
+ mpi_ptr_t up, mpi_ptr_t vp, mpi_size_t size);
int mpihelp_mul_karatsuba_case(mpi_ptr_t prodp,
mpi_ptr_t up, mpi_size_t usize,
mpi_size_t s1_size, mpi_limb_t s2_limb);
/*-- mpih-div.c --*/
+mpi_limb_t mpihelp_mod_1(mpi_ptr_t dividend_ptr, mpi_size_t dividend_size,
+ mpi_limb_t divisor_limb);
mpi_limb_t mpihelp_divrem(mpi_ptr_t qp, mpi_size_t qextra_limbs,
mpi_ptr_t np, mpi_size_t nsize,
mpi_ptr_t dp, mpi_size_t dsize);
+mpi_limb_t mpihelp_divmod_1(mpi_ptr_t quot_ptr,
+ mpi_ptr_t dividend_ptr, mpi_size_t dividend_size,
+ mpi_limb_t divisor_limb);
/*-- generic_mpih-[lr]shift.c --*/
mpi_limb_t mpihelp_lshift(mpi_ptr_t wp, mpi_ptr_t up, mpi_size_t usize,
--- /dev/null
+/* mpi-inv.c - MPI functions
+ * Copyright (C) 1998, 2001, 2002, 2003 Free Software Foundation, Inc.
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "mpi-internal.h"
+
+/****************
+ * Calculate the multiplicative inverse X of A mod N
+ * That is: Find the solution x for
+ * 1 = (a*x) mod n
+ */
+int mpi_invm(MPI x, MPI a, MPI n)
+{
+ /* Extended Euclid's algorithm (See TAOCP Vol II, 4.5.2, Alg X)
+ * modified according to Michael Penk's solution for Exercise 35
+ * with further enhancement
+ */
+ MPI u, v, u1, u2 = NULL, u3, v1, v2 = NULL, v3, t1, t2 = NULL, t3;
+ unsigned int k;
+ int sign;
+ int odd;
+
+ if (!mpi_cmp_ui(a, 0))
+ return 0; /* Inverse does not exists. */
+ if (!mpi_cmp_ui(n, 1))
+ return 0; /* Inverse does not exists. */
+
+ u = mpi_copy(a);
+ v = mpi_copy(n);
+
+ for (k = 0; !mpi_test_bit(u, 0) && !mpi_test_bit(v, 0); k++) {
+ mpi_rshift(u, u, 1);
+ mpi_rshift(v, v, 1);
+ }
+ odd = mpi_test_bit(v, 0);
+
+ u1 = mpi_alloc_set_ui(1);
+ if (!odd)
+ u2 = mpi_alloc_set_ui(0);
+ u3 = mpi_copy(u);
+ v1 = mpi_copy(v);
+ if (!odd) {
+ v2 = mpi_alloc(mpi_get_nlimbs(u));
+ mpi_sub(v2, u1, u); /* U is used as const 1 */
+ }
+ v3 = mpi_copy(v);
+ if (mpi_test_bit(u, 0)) { /* u is odd */
+ t1 = mpi_alloc_set_ui(0);
+ if (!odd) {
+ t2 = mpi_alloc_set_ui(1);
+ t2->sign = 1;
+ }
+ t3 = mpi_copy(v);
+ t3->sign = !t3->sign;
+ goto Y4;
+ } else {
+ t1 = mpi_alloc_set_ui(1);
+ if (!odd)
+ t2 = mpi_alloc_set_ui(0);
+ t3 = mpi_copy(u);
+ }
+
+ do {
+ do {
+ if (!odd) {
+ if (mpi_test_bit(t1, 0) || mpi_test_bit(t2, 0)) {
+ /* one is odd */
+ mpi_add(t1, t1, v);
+ mpi_sub(t2, t2, u);
+ }
+ mpi_rshift(t1, t1, 1);
+ mpi_rshift(t2, t2, 1);
+ mpi_rshift(t3, t3, 1);
+ } else {
+ if (mpi_test_bit(t1, 0))
+ mpi_add(t1, t1, v);
+ mpi_rshift(t1, t1, 1);
+ mpi_rshift(t3, t3, 1);
+ }
+Y4:
+ ;
+ } while (!mpi_test_bit(t3, 0)); /* while t3 is even */
+
+ if (!t3->sign) {
+ mpi_set(u1, t1);
+ if (!odd)
+ mpi_set(u2, t2);
+ mpi_set(u3, t3);
+ } else {
+ mpi_sub(v1, v, t1);
+ sign = u->sign; u->sign = !u->sign;
+ if (!odd)
+ mpi_sub(v2, u, t2);
+ u->sign = sign;
+ sign = t3->sign; t3->sign = !t3->sign;
+ mpi_set(v3, t3);
+ t3->sign = sign;
+ }
+ mpi_sub(t1, u1, v1);
+ if (!odd)
+ mpi_sub(t2, u2, v2);
+ mpi_sub(t3, u3, v3);
+ if (t1->sign) {
+ mpi_add(t1, t1, v);
+ if (!odd)
+ mpi_sub(t2, t2, u);
+ }
+ } while (mpi_cmp_ui(t3, 0)); /* while t3 != 0 */
+ /* mpi_lshift( u3, k ); */
+ mpi_set(x, u1);
+
+ mpi_free(u1);
+ mpi_free(v1);
+ mpi_free(t1);
+ if (!odd) {
+ mpi_free(u2);
+ mpi_free(v2);
+ mpi_free(t2);
+ }
+ mpi_free(u3);
+ mpi_free(v3);
+ mpi_free(t3);
+
+ mpi_free(u);
+ mpi_free(v);
+ return 1;
+}
+EXPORT_SYMBOL_GPL(mpi_invm);
--- /dev/null
+/* mpi-mod.c - Modular reduction
+ * Copyright (C) 1998, 1999, 2001, 2002, 2003,
+ * 2007 Free Software Foundation, Inc.
+ *
+ * This file is part of Libgcrypt.
+ */
+
+
+#include "mpi-internal.h"
+#include "longlong.h"
+
+/* Context used with Barrett reduction. */
+struct barrett_ctx_s {
+ MPI m; /* The modulus - may not be modified. */
+ int m_copied; /* If true, M needs to be released. */
+ int k;
+ MPI y;
+ MPI r1; /* Helper MPI. */
+ MPI r2; /* Helper MPI. */
+ MPI r3; /* Helper MPI allocated on demand. */
+};
+
+
+
+void mpi_mod(MPI rem, MPI dividend, MPI divisor)
+{
+ mpi_fdiv_r(rem, dividend, divisor);
+}
+
+/* This function returns a new context for Barrett based operations on
+ * the modulus M. This context needs to be released using
+ * _gcry_mpi_barrett_free. If COPY is true M will be transferred to
+ * the context and the user may change M. If COPY is false, M may not
+ * be changed until gcry_mpi_barrett_free has been called.
+ */
+mpi_barrett_t mpi_barrett_init(MPI m, int copy)
+{
+ mpi_barrett_t ctx;
+ MPI tmp;
+
+ mpi_normalize(m);
+ ctx = kcalloc(1, sizeof(*ctx), GFP_KERNEL);
+
+ if (copy) {
+ ctx->m = mpi_copy(m);
+ ctx->m_copied = 1;
+ } else
+ ctx->m = m;
+
+ ctx->k = mpi_get_nlimbs(m);
+ tmp = mpi_alloc(ctx->k + 1);
+
+ /* Barrett precalculation: y = floor(b^(2k) / m). */
+ mpi_set_ui(tmp, 1);
+ mpi_lshift_limbs(tmp, 2 * ctx->k);
+ mpi_fdiv_q(tmp, tmp, m);
+
+ ctx->y = tmp;
+ ctx->r1 = mpi_alloc(2 * ctx->k + 1);
+ ctx->r2 = mpi_alloc(2 * ctx->k + 1);
+
+ return ctx;
+}
+
+void mpi_barrett_free(mpi_barrett_t ctx)
+{
+ if (ctx) {
+ mpi_free(ctx->y);
+ mpi_free(ctx->r1);
+ mpi_free(ctx->r2);
+ if (ctx->r3)
+ mpi_free(ctx->r3);
+ if (ctx->m_copied)
+ mpi_free(ctx->m);
+ kfree(ctx);
+ }
+}
+
+
+/* R = X mod M
+ *
+ * Using Barrett reduction. Before using this function
+ * _gcry_mpi_barrett_init must have been called to do the
+ * precalculations. CTX is the context created by this precalculation
+ * and also conveys M. If the Barret reduction could no be done a
+ * straightforward reduction method is used.
+ *
+ * We assume that these conditions are met:
+ * Input: x =(x_2k-1 ...x_0)_b
+ * m =(m_k-1 ....m_0)_b with m_k-1 != 0
+ * Output: r = x mod m
+ */
+void mpi_mod_barrett(MPI r, MPI x, mpi_barrett_t ctx)
+{
+ MPI m = ctx->m;
+ int k = ctx->k;
+ MPI y = ctx->y;
+ MPI r1 = ctx->r1;
+ MPI r2 = ctx->r2;
+ int sign;
+
+ mpi_normalize(x);
+ if (mpi_get_nlimbs(x) > 2*k) {
+ mpi_mod(r, x, m);
+ return;
+ }
+
+ sign = x->sign;
+ x->sign = 0;
+
+ /* 1. q1 = floor( x / b^k-1)
+ * q2 = q1 * y
+ * q3 = floor( q2 / b^k+1 )
+ * Actually, we don't need qx, we can work direct on r2
+ */
+ mpi_set(r2, x);
+ mpi_rshift_limbs(r2, k-1);
+ mpi_mul(r2, r2, y);
+ mpi_rshift_limbs(r2, k+1);
+
+ /* 2. r1 = x mod b^k+1
+ * r2 = q3 * m mod b^k+1
+ * r = r1 - r2
+ * 3. if r < 0 then r = r + b^k+1
+ */
+ mpi_set(r1, x);
+ if (r1->nlimbs > k+1) /* Quick modulo operation. */
+ r1->nlimbs = k+1;
+ mpi_mul(r2, r2, m);
+ if (r2->nlimbs > k+1) /* Quick modulo operation. */
+ r2->nlimbs = k+1;
+ mpi_sub(r, r1, r2);
+
+ if (mpi_has_sign(r)) {
+ if (!ctx->r3) {
+ ctx->r3 = mpi_alloc(k + 2);
+ mpi_set_ui(ctx->r3, 1);
+ mpi_lshift_limbs(ctx->r3, k + 1);
+ }
+ mpi_add(r, r, ctx->r3);
+ }
+
+ /* 4. while r >= m do r = r - m */
+ while (mpi_cmp(r, m) >= 0)
+ mpi_sub(r, r, m);
+
+ x->sign = sign;
+}
+
+
+void mpi_mul_barrett(MPI w, MPI u, MPI v, mpi_barrett_t ctx)
+{
+ mpi_mul(w, u, v);
+ mpi_mod_barrett(w, w, ctx);
+}
--- /dev/null
+/* mpi-mul.c - MPI functions
+ * Copyright (C) 1994, 1996, 1998, 2001, 2002,
+ * 2003 Free Software Foundation, Inc.
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Note: This code is heavily based on the GNU MP Library.
+ * Actually it's the same code with only minor changes in the
+ * way the data is stored; this is to support the abstraction
+ * of an optional secure memory allocation which may be used
+ * to avoid revealing of sensitive data due to paging etc.
+ */
+
+#include "mpi-internal.h"
+
+void mpi_mul(MPI w, MPI u, MPI v)
+{
+ mpi_size_t usize, vsize, wsize;
+ mpi_ptr_t up, vp, wp;
+ mpi_limb_t cy;
+ int usign, vsign, sign_product;
+ int assign_wp = 0;
+ mpi_ptr_t tmp_limb = NULL;
+
+ if (u->nlimbs < v->nlimbs) {
+ /* Swap U and V. */
+ usize = v->nlimbs;
+ usign = v->sign;
+ up = v->d;
+ vsize = u->nlimbs;
+ vsign = u->sign;
+ vp = u->d;
+ } else {
+ usize = u->nlimbs;
+ usign = u->sign;
+ up = u->d;
+ vsize = v->nlimbs;
+ vsign = v->sign;
+ vp = v->d;
+ }
+ sign_product = usign ^ vsign;
+ wp = w->d;
+
+ /* Ensure W has space enough to store the result. */
+ wsize = usize + vsize;
+ if (w->alloced < wsize) {
+ if (wp == up || wp == vp) {
+ wp = mpi_alloc_limb_space(wsize);
+ assign_wp = 1;
+ } else {
+ mpi_resize(w, wsize);
+ wp = w->d;
+ }
+ } else { /* Make U and V not overlap with W. */
+ if (wp == up) {
+ /* W and U are identical. Allocate temporary space for U. */
+ up = tmp_limb = mpi_alloc_limb_space(usize);
+ /* Is V identical too? Keep it identical with U. */
+ if (wp == vp)
+ vp = up;
+ /* Copy to the temporary space. */
+ MPN_COPY(up, wp, usize);
+ } else if (wp == vp) {
+ /* W and V are identical. Allocate temporary space for V. */
+ vp = tmp_limb = mpi_alloc_limb_space(vsize);
+ /* Copy to the temporary space. */
+ MPN_COPY(vp, wp, vsize);
+ }
+ }
+
+ if (!vsize)
+ wsize = 0;
+ else {
+ mpihelp_mul(wp, up, usize, vp, vsize, &cy);
+ wsize -= cy ? 0:1;
+ }
+
+ if (assign_wp)
+ mpi_assign_limb_space(w, wp, wsize);
+ w->nlimbs = wsize;
+ w->sign = sign_product;
+ if (tmp_limb)
+ mpi_free_limb_space(tmp_limb);
+}
+
+void mpi_mulm(MPI w, MPI u, MPI v, MPI m)
+{
+ mpi_mul(w, u, v);
+ mpi_tdiv_r(w, w, m);
+}
+EXPORT_SYMBOL_GPL(mpi_mulm);
#include <linux/string.h>
#include "mpi-internal.h"
+#define MAX_EXTERN_SCAN_BYTES (16*1024*1024)
#define MAX_EXTERN_MPI_BITS 16384
/**
}
EXPORT_SYMBOL_GPL(mpi_read_from_buffer);
+/****************
+ * Fill the mpi VAL from the hex string in STR.
+ */
+int mpi_fromstr(MPI val, const char *str)
+{
+ int sign = 0;
+ int prepend_zero = 0;
+ int i, j, c, c1, c2;
+ unsigned int nbits, nbytes, nlimbs;
+ mpi_limb_t a;
+
+ if (*str == '-') {
+ sign = 1;
+ str++;
+ }
+
+ /* Skip optional hex prefix. */
+ if (*str == '0' && str[1] == 'x')
+ str += 2;
+
+ nbits = strlen(str);
+ if (nbits > MAX_EXTERN_SCAN_BYTES) {
+ mpi_clear(val);
+ return -EINVAL;
+ }
+ nbits *= 4;
+ if ((nbits % 8))
+ prepend_zero = 1;
+
+ nbytes = (nbits+7) / 8;
+ nlimbs = (nbytes+BYTES_PER_MPI_LIMB-1) / BYTES_PER_MPI_LIMB;
+
+ if (val->alloced < nlimbs)
+ mpi_resize(val, nlimbs);
+
+ i = BYTES_PER_MPI_LIMB - (nbytes % BYTES_PER_MPI_LIMB);
+ i %= BYTES_PER_MPI_LIMB;
+ j = val->nlimbs = nlimbs;
+ val->sign = sign;
+ for (; j > 0; j--) {
+ a = 0;
+ for (; i < BYTES_PER_MPI_LIMB; i++) {
+ if (prepend_zero) {
+ c1 = '0';
+ prepend_zero = 0;
+ } else
+ c1 = *str++;
+
+ if (!c1) {
+ mpi_clear(val);
+ return -EINVAL;
+ }
+ c2 = *str++;
+ if (!c2) {
+ mpi_clear(val);
+ return -EINVAL;
+ }
+ if (c1 >= '0' && c1 <= '9')
+ c = c1 - '0';
+ else if (c1 >= 'a' && c1 <= 'f')
+ c = c1 - 'a' + 10;
+ else if (c1 >= 'A' && c1 <= 'F')
+ c = c1 - 'A' + 10;
+ else {
+ mpi_clear(val);
+ return -EINVAL;
+ }
+ c <<= 4;
+ if (c2 >= '0' && c2 <= '9')
+ c |= c2 - '0';
+ else if (c2 >= 'a' && c2 <= 'f')
+ c |= c2 - 'a' + 10;
+ else if (c2 >= 'A' && c2 <= 'F')
+ c |= c2 - 'A' + 10;
+ else {
+ mpi_clear(val);
+ return -EINVAL;
+ }
+ a <<= 8;
+ a |= c;
+ }
+ i = 0;
+ val->d[j-1] = a;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(mpi_fromstr);
+
+MPI mpi_scanval(const char *string)
+{
+ MPI a;
+
+ a = mpi_alloc(0);
+ if (!a)
+ return NULL;
+
+ if (mpi_fromstr(a, string)) {
+ mpi_free(a);
+ return NULL;
+ }
+ mpi_normalize(a);
+ return a;
+}
+EXPORT_SYMBOL_GPL(mpi_scanval);
+
static int count_lzeros(MPI a)
{
mpi_limb_t alimb;
return val;
}
EXPORT_SYMBOL_GPL(mpi_read_raw_from_sgl);
+
+/* Perform a two's complement operation on buffer P of size N bytes. */
+static void twocompl(unsigned char *p, unsigned int n)
+{
+ int i;
+
+ for (i = n-1; i >= 0 && !p[i]; i--)
+ ;
+ if (i >= 0) {
+ if ((p[i] & 0x01))
+ p[i] = (((p[i] ^ 0xfe) | 0x01) & 0xff);
+ else if ((p[i] & 0x02))
+ p[i] = (((p[i] ^ 0xfc) | 0x02) & 0xfe);
+ else if ((p[i] & 0x04))
+ p[i] = (((p[i] ^ 0xf8) | 0x04) & 0xfc);
+ else if ((p[i] & 0x08))
+ p[i] = (((p[i] ^ 0xf0) | 0x08) & 0xf8);
+ else if ((p[i] & 0x10))
+ p[i] = (((p[i] ^ 0xe0) | 0x10) & 0xf0);
+ else if ((p[i] & 0x20))
+ p[i] = (((p[i] ^ 0xc0) | 0x20) & 0xe0);
+ else if ((p[i] & 0x40))
+ p[i] = (((p[i] ^ 0x80) | 0x40) & 0xc0);
+ else
+ p[i] = 0x80;
+
+ for (i--; i >= 0; i--)
+ p[i] ^= 0xff;
+ }
+}
+
+int mpi_print(enum gcry_mpi_format format, unsigned char *buffer,
+ size_t buflen, size_t *nwritten, MPI a)
+{
+ unsigned int nbits = mpi_get_nbits(a);
+ size_t len;
+ size_t dummy_nwritten;
+ int negative;
+
+ if (!nwritten)
+ nwritten = &dummy_nwritten;
+
+ /* Libgcrypt does no always care to set clear the sign if the value
+ * is 0. For printing this is a bit of a surprise, in particular
+ * because if some of the formats don't support negative numbers but
+ * should be able to print a zero. Thus we need this extra test
+ * for a negative number.
+ */
+ if (a->sign && mpi_cmp_ui(a, 0))
+ negative = 1;
+ else
+ negative = 0;
+
+ len = buflen;
+ *nwritten = 0;
+ if (format == GCRYMPI_FMT_STD) {
+ unsigned char *tmp;
+ int extra = 0;
+ unsigned int n;
+
+ tmp = mpi_get_buffer(a, &n, NULL);
+ if (!tmp)
+ return -EINVAL;
+
+ if (negative) {
+ twocompl(tmp, n);
+ if (!(*tmp & 0x80)) {
+ /* Need to extend the sign. */
+ n++;
+ extra = 2;
+ }
+ } else if (n && (*tmp & 0x80)) {
+ /* Positive but the high bit of the returned buffer is set.
+ * Thus we need to print an extra leading 0x00 so that the
+ * output is interpreted as a positive number.
+ */
+ n++;
+ extra = 1;
+ }
+
+ if (buffer && n > len) {
+ /* The provided buffer is too short. */
+ kfree(tmp);
+ return -E2BIG;
+ }
+ if (buffer) {
+ unsigned char *s = buffer;
+
+ if (extra == 1)
+ *s++ = 0;
+ else if (extra)
+ *s++ = 0xff;
+ memcpy(s, tmp, n-!!extra);
+ }
+ kfree(tmp);
+ *nwritten = n;
+ return 0;
+ } else if (format == GCRYMPI_FMT_USG) {
+ unsigned int n = (nbits + 7)/8;
+
+ /* Note: We ignore the sign for this format. */
+ /* FIXME: for performance reasons we should put this into
+ * mpi_aprint because we can then use the buffer directly.
+ */
+
+ if (buffer && n > len)
+ return -E2BIG;
+ if (buffer) {
+ unsigned char *tmp;
+
+ tmp = mpi_get_buffer(a, &n, NULL);
+ if (!tmp)
+ return -EINVAL;
+ memcpy(buffer, tmp, n);
+ kfree(tmp);
+ }
+ *nwritten = n;
+ return 0;
+ } else if (format == GCRYMPI_FMT_PGP) {
+ unsigned int n = (nbits + 7)/8;
+
+ /* The PGP format can only handle unsigned integers. */
+ if (negative)
+ return -EINVAL;
+
+ if (buffer && n+2 > len)
+ return -E2BIG;
+
+ if (buffer) {
+ unsigned char *tmp;
+ unsigned char *s = buffer;
+
+ s[0] = nbits >> 8;
+ s[1] = nbits;
+
+ tmp = mpi_get_buffer(a, &n, NULL);
+ if (!tmp)
+ return -EINVAL;
+ memcpy(s+2, tmp, n);
+ kfree(tmp);
+ }
+ *nwritten = n+2;
+ return 0;
+ } else if (format == GCRYMPI_FMT_SSH) {
+ unsigned char *tmp;
+ int extra = 0;
+ unsigned int n;
+
+ tmp = mpi_get_buffer(a, &n, NULL);
+ if (!tmp)
+ return -EINVAL;
+
+ if (negative) {
+ twocompl(tmp, n);
+ if (!(*tmp & 0x80)) {
+ /* Need to extend the sign. */
+ n++;
+ extra = 2;
+ }
+ } else if (n && (*tmp & 0x80)) {
+ n++;
+ extra = 1;
+ }
+
+ if (buffer && n+4 > len) {
+ kfree(tmp);
+ return -E2BIG;
+ }
+
+ if (buffer) {
+ unsigned char *s = buffer;
+
+ *s++ = n >> 24;
+ *s++ = n >> 16;
+ *s++ = n >> 8;
+ *s++ = n;
+ if (extra == 1)
+ *s++ = 0;
+ else if (extra)
+ *s++ = 0xff;
+ memcpy(s, tmp, n-!!extra);
+ }
+ kfree(tmp);
+ *nwritten = 4+n;
+ return 0;
+ } else if (format == GCRYMPI_FMT_HEX) {
+ unsigned char *tmp;
+ int i;
+ int extra = 0;
+ unsigned int n = 0;
+
+ tmp = mpi_get_buffer(a, &n, NULL);
+ if (!tmp)
+ return -EINVAL;
+ if (!n || (*tmp & 0x80))
+ extra = 2;
+
+ if (buffer && 2*n + extra + negative + 1 > len) {
+ kfree(tmp);
+ return -E2BIG;
+ }
+ if (buffer) {
+ unsigned char *s = buffer;
+
+ if (negative)
+ *s++ = '-';
+ if (extra) {
+ *s++ = '0';
+ *s++ = '0';
+ }
+
+ for (i = 0; i < n; i++) {
+ unsigned int c = tmp[i];
+
+ *s++ = (c >> 4) < 10 ? '0'+(c>>4) : 'A'+(c>>4)-10;
+ c &= 15;
+ *s++ = c < 10 ? '0'+c : 'A'+c-10;
+ }
+ *s++ = 0;
+ *nwritten = s - buffer;
+ } else {
+ *nwritten = 2*n + extra + negative + 1;
+ }
+ kfree(tmp);
+ return 0;
+ } else
+ return -EINVAL;
+}
+EXPORT_SYMBOL_GPL(mpi_print);
#define UDIV_TIME UMUL_TIME
#endif
+
+mpi_limb_t
+mpihelp_mod_1(mpi_ptr_t dividend_ptr, mpi_size_t dividend_size,
+ mpi_limb_t divisor_limb)
+{
+ mpi_size_t i;
+ mpi_limb_t n1, n0, r;
+ mpi_limb_t dummy __maybe_unused;
+
+ /* Botch: Should this be handled at all? Rely on callers? */
+ if (!dividend_size)
+ return 0;
+
+ /* If multiplication is much faster than division, and the
+ * dividend is large, pre-invert the divisor, and use
+ * only multiplications in the inner loop.
+ *
+ * This test should be read:
+ * Does it ever help to use udiv_qrnnd_preinv?
+ * && Does what we save compensate for the inversion overhead?
+ */
+ if (UDIV_TIME > (2 * UMUL_TIME + 6)
+ && (UDIV_TIME - (2 * UMUL_TIME + 6)) * dividend_size > UDIV_TIME) {
+ int normalization_steps;
+
+ normalization_steps = count_leading_zeros(divisor_limb);
+ if (normalization_steps) {
+ mpi_limb_t divisor_limb_inverted;
+
+ divisor_limb <<= normalization_steps;
+
+ /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The
+ * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the
+ * most significant bit (with weight 2**N) implicit.
+ *
+ * Special case for DIVISOR_LIMB == 100...000.
+ */
+ if (!(divisor_limb << 1))
+ divisor_limb_inverted = ~(mpi_limb_t)0;
+ else
+ udiv_qrnnd(divisor_limb_inverted, dummy,
+ -divisor_limb, 0, divisor_limb);
+
+ n1 = dividend_ptr[dividend_size - 1];
+ r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps);
+
+ /* Possible optimization:
+ * if (r == 0
+ * && divisor_limb > ((n1 << normalization_steps)
+ * | (dividend_ptr[dividend_size - 2] >> ...)))
+ * ...one division less...
+ */
+ for (i = dividend_size - 2; i >= 0; i--) {
+ n0 = dividend_ptr[i];
+ UDIV_QRNND_PREINV(dummy, r, r,
+ ((n1 << normalization_steps)
+ | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))),
+ divisor_limb, divisor_limb_inverted);
+ n1 = n0;
+ }
+ UDIV_QRNND_PREINV(dummy, r, r,
+ n1 << normalization_steps,
+ divisor_limb, divisor_limb_inverted);
+ return r >> normalization_steps;
+ } else {
+ mpi_limb_t divisor_limb_inverted;
+
+ /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The
+ * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the
+ * most significant bit (with weight 2**N) implicit.
+ *
+ * Special case for DIVISOR_LIMB == 100...000.
+ */
+ if (!(divisor_limb << 1))
+ divisor_limb_inverted = ~(mpi_limb_t)0;
+ else
+ udiv_qrnnd(divisor_limb_inverted, dummy,
+ -divisor_limb, 0, divisor_limb);
+
+ i = dividend_size - 1;
+ r = dividend_ptr[i];
+
+ if (r >= divisor_limb)
+ r = 0;
+ else
+ i--;
+
+ for ( ; i >= 0; i--) {
+ n0 = dividend_ptr[i];
+ UDIV_QRNND_PREINV(dummy, r, r,
+ n0, divisor_limb, divisor_limb_inverted);
+ }
+ return r;
+ }
+ } else {
+ if (UDIV_NEEDS_NORMALIZATION) {
+ int normalization_steps;
+
+ normalization_steps = count_leading_zeros(divisor_limb);
+ if (normalization_steps) {
+ divisor_limb <<= normalization_steps;
+
+ n1 = dividend_ptr[dividend_size - 1];
+ r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps);
+
+ /* Possible optimization:
+ * if (r == 0
+ * && divisor_limb > ((n1 << normalization_steps)
+ * | (dividend_ptr[dividend_size - 2] >> ...)))
+ * ...one division less...
+ */
+ for (i = dividend_size - 2; i >= 0; i--) {
+ n0 = dividend_ptr[i];
+ udiv_qrnnd(dummy, r, r,
+ ((n1 << normalization_steps)
+ | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))),
+ divisor_limb);
+ n1 = n0;
+ }
+ udiv_qrnnd(dummy, r, r,
+ n1 << normalization_steps,
+ divisor_limb);
+ return r >> normalization_steps;
+ }
+ }
+ /* No normalization needed, either because udiv_qrnnd doesn't require
+ * it, or because DIVISOR_LIMB is already normalized.
+ */
+ i = dividend_size - 1;
+ r = dividend_ptr[i];
+
+ if (r >= divisor_limb)
+ r = 0;
+ else
+ i--;
+
+ for (; i >= 0; i--) {
+ n0 = dividend_ptr[i];
+ udiv_qrnnd(dummy, r, r, n0, divisor_limb);
+ }
+ return r;
+ }
+}
+
/* Divide num (NP/NSIZE) by den (DP/DSIZE) and write
* the NSIZE-DSIZE least significant quotient limbs at QP
* and the DSIZE long remainder at NP. If QEXTRA_LIMBS is
return most_significant_q_limb;
}
+
+/****************
+ * Divide (DIVIDEND_PTR,,DIVIDEND_SIZE) by DIVISOR_LIMB.
+ * Write DIVIDEND_SIZE limbs of quotient at QUOT_PTR.
+ * Return the single-limb remainder.
+ * There are no constraints on the value of the divisor.
+ *
+ * QUOT_PTR and DIVIDEND_PTR might point to the same limb.
+ */
+
+mpi_limb_t
+mpihelp_divmod_1(mpi_ptr_t quot_ptr,
+ mpi_ptr_t dividend_ptr, mpi_size_t dividend_size,
+ mpi_limb_t divisor_limb)
+{
+ mpi_size_t i;
+ mpi_limb_t n1, n0, r;
+ mpi_limb_t dummy __maybe_unused;
+
+ if (!dividend_size)
+ return 0;
+
+ /* If multiplication is much faster than division, and the
+ * dividend is large, pre-invert the divisor, and use
+ * only multiplications in the inner loop.
+ *
+ * This test should be read:
+ * Does it ever help to use udiv_qrnnd_preinv?
+ * && Does what we save compensate for the inversion overhead?
+ */
+ if (UDIV_TIME > (2 * UMUL_TIME + 6)
+ && (UDIV_TIME - (2 * UMUL_TIME + 6)) * dividend_size > UDIV_TIME) {
+ int normalization_steps;
+
+ normalization_steps = count_leading_zeros(divisor_limb);
+ if (normalization_steps) {
+ mpi_limb_t divisor_limb_inverted;
+
+ divisor_limb <<= normalization_steps;
+
+ /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The
+ * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the
+ * most significant bit (with weight 2**N) implicit.
+ */
+ /* Special case for DIVISOR_LIMB == 100...000. */
+ if (!(divisor_limb << 1))
+ divisor_limb_inverted = ~(mpi_limb_t)0;
+ else
+ udiv_qrnnd(divisor_limb_inverted, dummy,
+ -divisor_limb, 0, divisor_limb);
+
+ n1 = dividend_ptr[dividend_size - 1];
+ r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps);
+
+ /* Possible optimization:
+ * if (r == 0
+ * && divisor_limb > ((n1 << normalization_steps)
+ * | (dividend_ptr[dividend_size - 2] >> ...)))
+ * ...one division less...
+ */
+ for (i = dividend_size - 2; i >= 0; i--) {
+ n0 = dividend_ptr[i];
+ UDIV_QRNND_PREINV(quot_ptr[i + 1], r, r,
+ ((n1 << normalization_steps)
+ | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))),
+ divisor_limb, divisor_limb_inverted);
+ n1 = n0;
+ }
+ UDIV_QRNND_PREINV(quot_ptr[0], r, r,
+ n1 << normalization_steps,
+ divisor_limb, divisor_limb_inverted);
+ return r >> normalization_steps;
+ } else {
+ mpi_limb_t divisor_limb_inverted;
+
+ /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB. The
+ * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the
+ * most significant bit (with weight 2**N) implicit.
+ */
+ /* Special case for DIVISOR_LIMB == 100...000. */
+ if (!(divisor_limb << 1))
+ divisor_limb_inverted = ~(mpi_limb_t) 0;
+ else
+ udiv_qrnnd(divisor_limb_inverted, dummy,
+ -divisor_limb, 0, divisor_limb);
+
+ i = dividend_size - 1;
+ r = dividend_ptr[i];
+
+ if (r >= divisor_limb)
+ r = 0;
+ else
+ quot_ptr[i--] = 0;
+
+ for ( ; i >= 0; i--) {
+ n0 = dividend_ptr[i];
+ UDIV_QRNND_PREINV(quot_ptr[i], r, r,
+ n0, divisor_limb, divisor_limb_inverted);
+ }
+ return r;
+ }
+ } else {
+ if (UDIV_NEEDS_NORMALIZATION) {
+ int normalization_steps;
+
+ normalization_steps = count_leading_zeros(divisor_limb);
+ if (normalization_steps) {
+ divisor_limb <<= normalization_steps;
+
+ n1 = dividend_ptr[dividend_size - 1];
+ r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps);
+
+ /* Possible optimization:
+ * if (r == 0
+ * && divisor_limb > ((n1 << normalization_steps)
+ * | (dividend_ptr[dividend_size - 2] >> ...)))
+ * ...one division less...
+ */
+ for (i = dividend_size - 2; i >= 0; i--) {
+ n0 = dividend_ptr[i];
+ udiv_qrnnd(quot_ptr[i + 1], r, r,
+ ((n1 << normalization_steps)
+ | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))),
+ divisor_limb);
+ n1 = n0;
+ }
+ udiv_qrnnd(quot_ptr[0], r, r,
+ n1 << normalization_steps,
+ divisor_limb);
+ return r >> normalization_steps;
+ }
+ }
+ /* No normalization needed, either because udiv_qrnnd doesn't require
+ * it, or because DIVISOR_LIMB is already normalized.
+ */
+ i = dividend_size - 1;
+ r = dividend_ptr[i];
+
+ if (r >= divisor_limb)
+ r = 0;
+ else
+ quot_ptr[i--] = 0;
+
+ for (; i >= 0; i--) {
+ n0 = dividend_ptr[i];
+ udiv_qrnnd(quot_ptr[i], r, r, n0, divisor_limb);
+ }
+ return r;
+ }
+}
}
}
+
+void mpihelp_mul_n(mpi_ptr_t prodp,
+ mpi_ptr_t up, mpi_ptr_t vp, mpi_size_t size)
+{
+ if (up == vp) {
+ if (size < KARATSUBA_THRESHOLD)
+ mpih_sqr_n_basecase(prodp, up, size);
+ else {
+ mpi_ptr_t tspace;
+ tspace = mpi_alloc_limb_space(2 * size);
+ mpih_sqr_n(prodp, up, size, tspace);
+ mpi_free_limb_space(tspace);
+ }
+ } else {
+ if (size < KARATSUBA_THRESHOLD)
+ mul_n_basecase(prodp, up, vp, size);
+ else {
+ mpi_ptr_t tspace;
+ tspace = mpi_alloc_limb_space(2 * size);
+ mul_n(prodp, up, vp, size, tspace);
+ mpi_free_limb_space(tspace);
+ }
+ }
+}
+
int
mpihelp_mul_karatsuba_case(mpi_ptr_t prodp,
mpi_ptr_t up, mpi_size_t usize,
#include "mpi-internal.h"
+/* Constants allocated right away at startup. */
+static MPI constants[MPI_NUMBER_OF_CONSTANTS];
+
+/* Initialize the MPI subsystem. This is called early and allows to
+ * do some initialization without taking care of threading issues.
+ */
+static int __init mpi_init(void)
+{
+ int idx;
+ unsigned long value;
+
+ for (idx = 0; idx < MPI_NUMBER_OF_CONSTANTS; idx++) {
+ switch (idx) {
+ case MPI_C_ZERO:
+ value = 0;
+ break;
+ case MPI_C_ONE:
+ value = 1;
+ break;
+ case MPI_C_TWO:
+ value = 2;
+ break;
+ case MPI_C_THREE:
+ value = 3;
+ break;
+ case MPI_C_FOUR:
+ value = 4;
+ break;
+ case MPI_C_EIGHT:
+ value = 8;
+ break;
+ default:
+ pr_err("MPI: invalid mpi_const selector %d\n", idx);
+ return -EFAULT;
+ }
+ constants[idx] = mpi_alloc_set_ui(value);
+ constants[idx]->flags = (16|32);
+ }
+
+ return 0;
+}
+postcore_initcall(mpi_init);
+
+/* Return a constant MPI descripbed by NO which is one of the
+ * MPI_C_xxx macros. There is no need to copy this returned value; it
+ * may be used directly.
+ */
+MPI mpi_const(enum gcry_mpi_constants no)
+{
+ if ((int)no < 0 || no > MPI_NUMBER_OF_CONSTANTS)
+ pr_err("MPI: invalid mpi_const selector %d\n", no);
+ if (!constants[no])
+ pr_err("MPI: MPI subsystem not initialized\n");
+ return constants[no];
+}
+EXPORT_SYMBOL_GPL(mpi_const);
+
/****************
* Note: It was a bad idea to use the number of limbs to allocate
* because on a alpha the limbs are large but we normally need
return 0;
}
+void mpi_clear(MPI a)
+{
+ if (!a)
+ return;
+ a->nlimbs = 0;
+ a->flags = 0;
+}
+EXPORT_SYMBOL_GPL(mpi_clear);
+
void mpi_free(MPI a)
{
if (!a)
}
EXPORT_SYMBOL_GPL(mpi_free);
+/****************
+ * Note: This copy function should not interpret the MPI
+ * but copy it transparently.
+ */
+MPI mpi_copy(MPI a)
+{
+ int i;
+ MPI b;
+
+ if (a) {
+ b = mpi_alloc(a->nlimbs);
+ b->nlimbs = a->nlimbs;
+ b->sign = a->sign;
+ b->flags = a->flags;
+ b->flags &= ~(16|32); /* Reset the immutable and constant flags. */
+ for (i = 0; i < b->nlimbs; i++)
+ b->d[i] = a->d[i];
+ } else
+ b = NULL;
+ return b;
+}
+
+/****************
+ * This function allocates an MPI which is optimized to hold
+ * a value as large as the one given in the argument and allocates it
+ * with the same flags as A.
+ */
+MPI mpi_alloc_like(MPI a)
+{
+ MPI b;
+
+ if (a) {
+ b = mpi_alloc(a->nlimbs);
+ b->nlimbs = 0;
+ b->sign = 0;
+ b->flags = a->flags;
+ } else
+ b = NULL;
+
+ return b;
+}
+
+
+/* Set U into W and release U. If W is NULL only U will be released. */
+void mpi_snatch(MPI w, MPI u)
+{
+ if (w) {
+ mpi_assign_limb_space(w, u->d, u->alloced);
+ w->nlimbs = u->nlimbs;
+ w->sign = u->sign;
+ w->flags = u->flags;
+ u->alloced = 0;
+ u->nlimbs = 0;
+ u->d = NULL;
+ }
+ mpi_free(u);
+}
+
+
+MPI mpi_set(MPI w, MPI u)
+{
+ mpi_ptr_t wp, up;
+ mpi_size_t usize = u->nlimbs;
+ int usign = u->sign;
+
+ if (!w)
+ w = mpi_alloc(mpi_get_nlimbs(u));
+ RESIZE_IF_NEEDED(w, usize);
+ wp = w->d;
+ up = u->d;
+ MPN_COPY(wp, up, usize);
+ w->nlimbs = usize;
+ w->flags = u->flags;
+ w->flags &= ~(16|32); /* Reset the immutable and constant flags. */
+ w->sign = usign;
+ return w;
+}
+EXPORT_SYMBOL_GPL(mpi_set);
+
+MPI mpi_set_ui(MPI w, unsigned long u)
+{
+ if (!w)
+ w = mpi_alloc(1);
+ /* FIXME: If U is 0 we have no need to resize and thus possible
+ * allocating the the limbs.
+ */
+ RESIZE_IF_NEEDED(w, 1);
+ w->d[0] = u;
+ w->nlimbs = u ? 1 : 0;
+ w->sign = 0;
+ w->flags = 0;
+ return w;
+}
+EXPORT_SYMBOL_GPL(mpi_set_ui);
+
+MPI mpi_alloc_set_ui(unsigned long u)
+{
+ MPI w = mpi_alloc(1);
+ w->d[0] = u;
+ w->nlimbs = u ? 1 : 0;
+ w->sign = 0;
+ return w;
+}
+
+/****************
+ * Swap the value of A and B, when SWAP is 1.
+ * Leave the value when SWAP is 0.
+ * This implementation should be constant-time regardless of SWAP.
+ */
+void mpi_swap_cond(MPI a, MPI b, unsigned long swap)
+{
+ mpi_size_t i;
+ mpi_size_t nlimbs;
+ mpi_limb_t mask = ((mpi_limb_t)0) - swap;
+ mpi_limb_t x;
+
+ if (a->alloced > b->alloced)
+ nlimbs = b->alloced;
+ else
+ nlimbs = a->alloced;
+ if (a->nlimbs > nlimbs || b->nlimbs > nlimbs)
+ return;
+
+ for (i = 0; i < nlimbs; i++) {
+ x = mask & (a->d[i] ^ b->d[i]);
+ a->d[i] = a->d[i] ^ x;
+ b->d[i] = b->d[i] ^ x;
+ }
+
+ x = mask & (a->nlimbs ^ b->nlimbs);
+ a->nlimbs = a->nlimbs ^ x;
+ b->nlimbs = b->nlimbs ^ x;
+
+ x = mask & (a->sign ^ b->sign);
+ a->sign = a->sign ^ x;
+ b->sign = b->sign ^ x;
+}
+
MODULE_DESCRIPTION("Multiprecision maths library");
MODULE_LICENSE("GPL");
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/wait.h>
+#include <linux/slab.h>
#include <linux/percpu-refcount.h>
/*
size_t align = max_t(size_t, 1 << __PERCPU_REF_FLAG_BITS,
__alignof__(unsigned long));
unsigned long start_count = 0;
+ struct percpu_ref_data *data;
ref->percpu_count_ptr = (unsigned long)
__alloc_percpu_gfp(sizeof(unsigned long), align, gfp);
if (!ref->percpu_count_ptr)
return -ENOMEM;
- ref->force_atomic = flags & PERCPU_REF_INIT_ATOMIC;
- ref->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT;
+ data = kzalloc(sizeof(*ref->data), gfp);
+ if (!data) {
+ free_percpu((void __percpu *)ref->percpu_count_ptr);
+ return -ENOMEM;
+ }
+
+ data->force_atomic = flags & PERCPU_REF_INIT_ATOMIC;
+ data->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT;
if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD)) {
ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC;
- ref->allow_reinit = true;
+ data->allow_reinit = true;
} else {
start_count += PERCPU_COUNT_BIAS;
}
else
start_count++;
- atomic_long_set(&ref->count, start_count);
+ atomic_long_set(&data->count, start_count);
- ref->release = release;
- ref->confirm_switch = NULL;
+ data->release = release;
+ data->confirm_switch = NULL;
+ data->ref = ref;
+ ref->data = data;
return 0;
}
EXPORT_SYMBOL_GPL(percpu_ref_init);
+static void __percpu_ref_exit(struct percpu_ref *ref)
+{
+ unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
+
+ if (percpu_count) {
+ /* non-NULL confirm_switch indicates switching in progress */
+ WARN_ON_ONCE(ref->data && ref->data->confirm_switch);
+ free_percpu(percpu_count);
+ ref->percpu_count_ptr = __PERCPU_REF_ATOMIC_DEAD;
+ }
+}
+
/**
* percpu_ref_exit - undo percpu_ref_init()
* @ref: percpu_ref to exit
*/
void percpu_ref_exit(struct percpu_ref *ref)
{
- unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
+ struct percpu_ref_data *data = ref->data;
+ unsigned long flags;
- if (percpu_count) {
- /* non-NULL confirm_switch indicates switching in progress */
- WARN_ON_ONCE(ref->confirm_switch);
- free_percpu(percpu_count);
- ref->percpu_count_ptr = __PERCPU_REF_ATOMIC_DEAD;
- }
+ __percpu_ref_exit(ref);
+
+ if (!data)
+ return;
+
+ spin_lock_irqsave(&percpu_ref_switch_lock, flags);
+ ref->percpu_count_ptr |= atomic_long_read(&ref->data->count) <<
+ __PERCPU_REF_FLAG_BITS;
+ ref->data = NULL;
+ spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
+
+ kfree(data);
}
EXPORT_SYMBOL_GPL(percpu_ref_exit);
static void percpu_ref_call_confirm_rcu(struct rcu_head *rcu)
{
- struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);
+ struct percpu_ref_data *data = container_of(rcu,
+ struct percpu_ref_data, rcu);
+ struct percpu_ref *ref = data->ref;
- ref->confirm_switch(ref);
- ref->confirm_switch = NULL;
+ data->confirm_switch(ref);
+ data->confirm_switch = NULL;
wake_up_all(&percpu_ref_switch_waitq);
- if (!ref->allow_reinit)
- percpu_ref_exit(ref);
+ if (!data->allow_reinit)
+ __percpu_ref_exit(ref);
/* drop ref from percpu_ref_switch_to_atomic() */
percpu_ref_put(ref);
static void percpu_ref_switch_to_atomic_rcu(struct rcu_head *rcu)
{
- struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);
+ struct percpu_ref_data *data = container_of(rcu,
+ struct percpu_ref_data, rcu);
+ struct percpu_ref *ref = data->ref;
unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
unsigned long count = 0;
int cpu;
count += *per_cpu_ptr(percpu_count, cpu);
pr_debug("global %lu percpu %lu\n",
- atomic_long_read(&ref->count), count);
+ atomic_long_read(&data->count), count);
/*
* It's crucial that we sum the percpu counters _before_ adding the sum
* reaching 0 before we add the percpu counts. But doing it at the same
* time is equivalent and saves us atomic operations:
*/
- atomic_long_add((long)count - PERCPU_COUNT_BIAS, &ref->count);
+ atomic_long_add((long)count - PERCPU_COUNT_BIAS, &data->count);
- WARN_ONCE(atomic_long_read(&ref->count) <= 0,
+ WARN_ONCE(atomic_long_read(&data->count) <= 0,
"percpu ref (%ps) <= 0 (%ld) after switching to atomic",
- ref->release, atomic_long_read(&ref->count));
+ data->release, atomic_long_read(&data->count));
/* @ref is viewed as dead on all CPUs, send out switch confirmation */
percpu_ref_call_confirm_rcu(rcu);
* Non-NULL ->confirm_switch is used to indicate that switching is
* in progress. Use noop one if unspecified.
*/
- ref->confirm_switch = confirm_switch ?: percpu_ref_noop_confirm_switch;
+ ref->data->confirm_switch = confirm_switch ?:
+ percpu_ref_noop_confirm_switch;
percpu_ref_get(ref); /* put after confirmation */
- call_rcu(&ref->rcu, percpu_ref_switch_to_atomic_rcu);
+ call_rcu(&ref->data->rcu, percpu_ref_switch_to_atomic_rcu);
}
static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref)
if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC))
return;
- if (WARN_ON_ONCE(!ref->allow_reinit))
+ if (WARN_ON_ONCE(!ref->data->allow_reinit))
return;
- atomic_long_add(PERCPU_COUNT_BIAS, &ref->count);
+ atomic_long_add(PERCPU_COUNT_BIAS, &ref->data->count);
/*
* Restore per-cpu operation. smp_store_release() is paired
static void __percpu_ref_switch_mode(struct percpu_ref *ref,
percpu_ref_func_t *confirm_switch)
{
+ struct percpu_ref_data *data = ref->data;
+
lockdep_assert_held(&percpu_ref_switch_lock);
/*
* its completion. If the caller ensures that ATOMIC switching
* isn't in progress, this function can be called from any context.
*/
- wait_event_lock_irq(percpu_ref_switch_waitq, !ref->confirm_switch,
+ wait_event_lock_irq(percpu_ref_switch_waitq, !data->confirm_switch,
percpu_ref_switch_lock);
- if (ref->force_atomic || (ref->percpu_count_ptr & __PERCPU_REF_DEAD))
+ if (data->force_atomic || (ref->percpu_count_ptr & __PERCPU_REF_DEAD))
__percpu_ref_switch_to_atomic(ref, confirm_switch);
else
__percpu_ref_switch_to_percpu(ref);
spin_lock_irqsave(&percpu_ref_switch_lock, flags);
- ref->force_atomic = true;
+ ref->data->force_atomic = true;
__percpu_ref_switch_mode(ref, confirm_switch);
spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref)
{
percpu_ref_switch_to_atomic(ref, NULL);
- wait_event(percpu_ref_switch_waitq, !ref->confirm_switch);
+ wait_event(percpu_ref_switch_waitq, !ref->data->confirm_switch);
}
EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic_sync);
spin_lock_irqsave(&percpu_ref_switch_lock, flags);
- ref->force_atomic = false;
+ ref->data->force_atomic = false;
__percpu_ref_switch_mode(ref, NULL);
spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
spin_lock_irqsave(&percpu_ref_switch_lock, flags);
WARN_ONCE(ref->percpu_count_ptr & __PERCPU_REF_DEAD,
- "%s called more than once on %ps!", __func__, ref->release);
+ "%s called more than once on %ps!", __func__,
+ ref->data->release);
ref->percpu_count_ptr |= __PERCPU_REF_DEAD;
__percpu_ref_switch_mode(ref, confirm_kill);
}
EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm);
+/**
+ * percpu_ref_is_zero - test whether a percpu refcount reached zero
+ * @ref: percpu_ref to test
+ *
+ * Returns %true if @ref reached zero.
+ *
+ * This function is safe to call as long as @ref is between init and exit.
+ */
+bool percpu_ref_is_zero(struct percpu_ref *ref)
+{
+ unsigned long __percpu *percpu_count;
+ unsigned long count, flags;
+
+ if (__ref_is_percpu(ref, &percpu_count))
+ return false;
+
+ /* protect us from being destroyed */
+ spin_lock_irqsave(&percpu_ref_switch_lock, flags);
+ if (ref->data)
+ count = atomic_long_read(&ref->data->count);
+ else
+ count = ref->percpu_count_ptr >> __PERCPU_REF_FLAG_BITS;
+ spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
+
+ return count == 0;
+}
+EXPORT_SYMBOL_GPL(percpu_ref_is_zero);
+
/**
* percpu_ref_reinit - re-initialize a percpu refcount
* @ref: perpcu_ref to re-initialize
#ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
-static struct debug_obj_descr percpu_counter_debug_descr;
+static const struct debug_obj_descr percpu_counter_debug_descr;
static bool percpu_counter_fixup_free(void *addr, enum debug_obj_state state)
{
}
}
-static struct debug_obj_descr percpu_counter_debug_descr = {
+static const struct debug_obj_descr percpu_counter_debug_descr = {
.name = "percpu_counter",
.fixup_free = percpu_counter_fixup_free,
};
}
#endif
-DEFINE_PER_CPU(struct rnd_state, net_rand_state);
+DEFINE_PER_CPU(struct rnd_state, net_rand_state) __latent_entropy;
/**
* prandom_u32_state - seeded pseudo-random number generator.
}
EXPORT_SYMBOL(strscpy_pad);
+/**
+ * stpcpy - copy a string from src to dest returning a pointer to the new end
+ * of dest, including src's %NUL-terminator. May overrun dest.
+ * @dest: pointer to end of string being copied into. Must be large enough
+ * to receive copy.
+ * @src: pointer to the beginning of string being copied from. Must not overlap
+ * dest.
+ *
+ * stpcpy differs from strcpy in a key way: the return value is a pointer
+ * to the new %NUL-terminating character in @dest. (For strcpy, the return
+ * value is a pointer to the start of @dest). This interface is considered
+ * unsafe as it doesn't perform bounds checking of the inputs. As such it's
+ * not recommended for usage. Instead, its definition is provided in case
+ * the compiler lowers other libcalls to stpcpy.
+ */
+char *stpcpy(char *__restrict__ dest, const char *__restrict__ src);
+char *stpcpy(char *__restrict__ dest, const char *__restrict__ src)
+{
+ while ((*dest++ = *src++) != '\0')
+ /* nothing */;
+ return --dest;
+}
+EXPORT_SYMBOL(stpcpy);
+
#ifndef __HAVE_ARCH_STRCAT
/**
* strcat - Append one %NUL-terminated string to another
return pathname;
}
EXPORT_SYMBOL_GPL(kstrdup_quotable_file);
+
+/**
+ * kfree_strarray - free a number of dynamically allocated strings contained
+ * in an array and the array itself
+ *
+ * @array: Dynamically allocated array of strings to free.
+ * @n: Number of strings (starting from the beginning of the array) to free.
+ *
+ * Passing a non-NULL @array and @n == 0 as well as NULL @array are valid
+ * use-cases. If @array is NULL, the function does nothing.
+ */
+void kfree_strarray(char **array, size_t n)
+{
+ unsigned int i;
+
+ if (!array)
+ return;
+
+ for (i = 0; i < n; i++)
+ kfree(array[i]);
+ kfree(array);
+}
+EXPORT_SYMBOL_GPL(kfree_strarray);
hd = bdev->bd_disk;
buf = string(buf, end, hd->disk_name, spec);
- if (bdev->bd_part->partno) {
+ if (bdev->bd_partno) {
if (isdigit(hd->disk_name[strlen(hd->disk_name)-1])) {
if (buf < end)
*buf = 'p';
buf++;
}
- buf = number(buf, end, bdev->bd_part->partno, spec);
+ buf = number(buf, end, bdev->bd_partno, spec);
}
return buf;
}
This option specifies the initial value of this option. The default
of 1 says that all excess pages should be trimmed.
- See Documentation/mm/nommu-mmap.rst for more information.
+ See Documentation/admin-guide/mm/nommu-mmap.rst for more information.
config TRANSPARENT_HUGEPAGE
bool "Transparent Hugepage Support"
}
if (!PageUptodate(page)) {
- error = lock_page_killable(page);
+ if (iocb->ki_flags & IOCB_WAITQ)
+ error = lock_page_async(page, iocb->ki_waitq);
+ else
+ error = lock_page_killable(page);
+
if (unlikely(error))
goto readpage_error;
if (!PageUptodate(page)) {
BUG_ON(*locked != 1);
}
+ if (flags & FOLL_PIN)
+ atomic_set(&mm->has_pinned, 1);
+
/*
* FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior
* is to set FOLL_GET if the caller wants pages[] filled in (but has
return 1;
}
-static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
+static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end,
unsigned int flags, struct page **pages, int *nr)
{
unsigned long next;
pmd_t *pmdp;
- pmdp = pmd_offset(&pud, addr);
+ pmdp = pmd_offset_lockless(pudp, pud, addr);
do {
pmd_t pmd = READ_ONCE(*pmdp);
return 1;
}
-static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,
+static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end,
unsigned int flags, struct page **pages, int *nr)
{
unsigned long next;
pud_t *pudp;
- pudp = pud_offset(&p4d, addr);
+ pudp = pud_offset_lockless(p4dp, p4d, addr);
do {
pud_t pud = READ_ONCE(*pudp);
if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
PUD_SHIFT, next, flags, pages, nr))
return 0;
- } else if (!gup_pmd_range(pud, addr, next, flags, pages, nr))
+ } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
return 0;
} while (pudp++, addr = next, addr != end);
return 1;
}
-static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
+static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end,
unsigned int flags, struct page **pages, int *nr)
{
unsigned long next;
p4d_t *p4dp;
- p4dp = p4d_offset(&pgd, addr);
+ p4dp = p4d_offset_lockless(pgdp, pgd, addr);
do {
p4d_t p4d = READ_ONCE(*p4dp);
if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
P4D_SHIFT, next, flags, pages, nr))
return 0;
- } else if (!gup_pud_range(p4d, addr, next, flags, pages, nr))
+ } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
return 0;
} while (p4dp++, addr = next, addr != end);
if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
PGDIR_SHIFT, next, flags, pages, nr))
return;
- } else if (!gup_p4d_range(pgd, addr, next, flags, pages, nr))
+ } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
return;
} while (pgdp++, addr = next, addr != end);
}
FOLL_FAST_ONLY)))
return -EINVAL;
+ if (gup_flags & FOLL_PIN)
+ atomic_set(¤t->mm->has_pinned, 1);
+
if (!(gup_flags & FOLL_FAST_ONLY))
might_lock_read(¤t->mm->mmap_lock);
src_page = pmd_page(pmd);
VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
+
+ /*
+ * If this page is a potentially pinned page, split and retry the fault
+ * with smaller page size. Normally this should not happen because the
+ * userspace should use MADV_DONTFORK upon pinned regions. This is a
+ * best effort that the pinned pages won't be replaced by another
+ * random page during the coming copy-on-write.
+ */
+ if (unlikely(is_cow_mapping(vma->vm_flags) &&
+ atomic_read(&src_mm->has_pinned) &&
+ page_maybe_dma_pinned(src_page))) {
+ pte_free(dst_mm, pgtable);
+ spin_unlock(src_ptl);
+ spin_unlock(dst_ptl);
+ __split_huge_pmd(vma, src_pmd, addr, false, NULL);
+ return -EAGAIN;
+ }
+
get_page(src_page);
page_dup_rmap(src_page, true);
add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
/* No huge zero pud yet */
}
+ /* Please refer to comments in copy_huge_pmd() */
+ if (unlikely(is_cow_mapping(vma->vm_flags) &&
+ atomic_read(&src_mm->has_pinned) &&
+ page_maybe_dma_pinned(pud_page(pud)))) {
+ spin_unlock(src_ptl);
+ spin_unlock(dst_ptl);
+ __split_huge_pud(vma, src_pud, addr);
+ return -EAGAIN;
+ }
+
pudp_set_wrprotect(src_mm, addr, src_pud);
pud = pud_mkold(pud_wrprotect(pud));
set_pud_at(dst_mm, addr, dst_pud, pud);
(1L << PG_workingset) |
(1L << PG_locked) |
(1L << PG_unevictable) |
+#ifdef CONFIG_64BIT
+ (1L << PG_arch_2) |
+#endif
(1L << PG_dirty)));
/* ->mapping in first tail page is compound_mapcount */
#define CREATE_TRACE_POINTS
#include <trace/events/huge_memory.h>
+static struct task_struct *khugepaged_thread __read_mostly;
+static DEFINE_MUTEX(khugepaged_mutex);
+
/* default scan 8*512 pte (or vmas) every 30 second */
static unsigned int khugepaged_pages_to_scan __read_mostly;
static unsigned int khugepaged_pages_collapsed;
static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
{
+ /*
+ * If the hpage allocated earlier was briefly exposed in page cache
+ * before collapse_file() failed, it is possible that racing lookups
+ * have not yet completed, and would then be unpleasantly surprised by
+ * finding the hpage reused for the same mapping at a different offset.
+ * Just release the previous allocation if there is any danger of that.
+ */
+ if (*hpage && page_count(*hpage) > 1) {
+ put_page(*hpage);
+ *hpage = NULL;
+ }
+
if (!*hpage)
*hpage = khugepaged_alloc_hugepage(wait);
int start_stop_khugepaged(void)
{
- static struct task_struct *khugepaged_thread __read_mostly;
- static DEFINE_MUTEX(khugepaged_mutex);
int err = 0;
mutex_lock(&khugepaged_mutex);
mutex_unlock(&khugepaged_mutex);
return err;
}
+
+void khugepaged_min_free_kbytes_update(void)
+{
+ mutex_lock(&khugepaged_mutex);
+ if (khugepaged_enabled() && khugepaged_thread)
+ set_recommended_min_free_kbytes();
+ mutex_unlock(&khugepaged_mutex);
+}
return 0;
}
+regular_page:
if (pmd_trans_unstable(pmd))
return 0;
-regular_page:
#endif
tlb_change_page_size(tlb, PAGE_SIZE);
orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
memcg_page_state(memcg, WORKINGSET_ACTIVATE_ANON));
seq_buf_printf(&s, "workingset_activate_file %lu\n",
memcg_page_state(memcg, WORKINGSET_ACTIVATE_FILE));
- seq_buf_printf(&s, "workingset_restore %lu\n",
+ seq_buf_printf(&s, "workingset_restore_anon %lu\n",
memcg_page_state(memcg, WORKINGSET_RESTORE_ANON));
- seq_buf_printf(&s, "workingset_restore %lu\n",
+ seq_buf_printf(&s, "workingset_restore_file %lu\n",
memcg_page_state(memcg, WORKINGSET_RESTORE_FILE));
seq_buf_printf(&s, "workingset_nodereclaim %lu\n",
memcg_page_state(memcg, WORKINGSET_NODERECLAIM));
* covered by this vma.
*/
-static inline unsigned long
-copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
+static unsigned long
+copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
unsigned long addr, int *rss)
{
unsigned long vm_flags = vma->vm_flags;
pte_t pte = *src_pte;
struct page *page;
+ swp_entry_t entry = pte_to_swp_entry(pte);
+
+ if (likely(!non_swap_entry(entry))) {
+ if (swap_duplicate(entry) < 0)
+ return entry.val;
+
+ /* make sure dst_mm is on swapoff's mmlist. */
+ if (unlikely(list_empty(&dst_mm->mmlist))) {
+ spin_lock(&mmlist_lock);
+ if (list_empty(&dst_mm->mmlist))
+ list_add(&dst_mm->mmlist,
+ &src_mm->mmlist);
+ spin_unlock(&mmlist_lock);
+ }
+ rss[MM_SWAPENTS]++;
+ } else if (is_migration_entry(entry)) {
+ page = migration_entry_to_page(entry);
- /* pte contains position in swap or file, so copy. */
- if (unlikely(!pte_present(pte))) {
- swp_entry_t entry = pte_to_swp_entry(pte);
-
- if (likely(!non_swap_entry(entry))) {
- if (swap_duplicate(entry) < 0)
- return entry.val;
-
- /* make sure dst_mm is on swapoff's mmlist. */
- if (unlikely(list_empty(&dst_mm->mmlist))) {
- spin_lock(&mmlist_lock);
- if (list_empty(&dst_mm->mmlist))
- list_add(&dst_mm->mmlist,
- &src_mm->mmlist);
- spin_unlock(&mmlist_lock);
- }
- rss[MM_SWAPENTS]++;
- } else if (is_migration_entry(entry)) {
- page = migration_entry_to_page(entry);
-
- rss[mm_counter(page)]++;
-
- if (is_write_migration_entry(entry) &&
- is_cow_mapping(vm_flags)) {
- /*
- * COW mappings require pages in both
- * parent and child to be set to read.
- */
- make_migration_entry_read(&entry);
- pte = swp_entry_to_pte(entry);
- if (pte_swp_soft_dirty(*src_pte))
- pte = pte_swp_mksoft_dirty(pte);
- if (pte_swp_uffd_wp(*src_pte))
- pte = pte_swp_mkuffd_wp(pte);
- set_pte_at(src_mm, addr, src_pte, pte);
- }
- } else if (is_device_private_entry(entry)) {
- page = device_private_entry_to_page(entry);
+ rss[mm_counter(page)]++;
+ if (is_write_migration_entry(entry) &&
+ is_cow_mapping(vm_flags)) {
/*
- * Update rss count even for unaddressable pages, as
- * they should treated just like normal pages in this
- * respect.
- *
- * We will likely want to have some new rss counters
- * for unaddressable pages, at some point. But for now
- * keep things as they are.
+ * COW mappings require pages in both
+ * parent and child to be set to read.
*/
- get_page(page);
- rss[mm_counter(page)]++;
- page_dup_rmap(page, false);
+ make_migration_entry_read(&entry);
+ pte = swp_entry_to_pte(entry);
+ if (pte_swp_soft_dirty(*src_pte))
+ pte = pte_swp_mksoft_dirty(pte);
+ if (pte_swp_uffd_wp(*src_pte))
+ pte = pte_swp_mkuffd_wp(pte);
+ set_pte_at(src_mm, addr, src_pte, pte);
+ }
+ } else if (is_device_private_entry(entry)) {
+ page = device_private_entry_to_page(entry);
- /*
- * We do not preserve soft-dirty information, because so
- * far, checkpoint/restore is the only feature that
- * requires that. And checkpoint/restore does not work
- * when a device driver is involved (you cannot easily
- * save and restore device driver state).
- */
- if (is_write_device_private_entry(entry) &&
- is_cow_mapping(vm_flags)) {
- make_device_private_entry_read(&entry);
- pte = swp_entry_to_pte(entry);
- if (pte_swp_uffd_wp(*src_pte))
- pte = pte_swp_mkuffd_wp(pte);
- set_pte_at(src_mm, addr, src_pte, pte);
- }
+ /*
+ * Update rss count even for unaddressable pages, as
+ * they should treated just like normal pages in this
+ * respect.
+ *
+ * We will likely want to have some new rss counters
+ * for unaddressable pages, at some point. But for now
+ * keep things as they are.
+ */
+ get_page(page);
+ rss[mm_counter(page)]++;
+ page_dup_rmap(page, false);
+
+ /*
+ * We do not preserve soft-dirty information, because so
+ * far, checkpoint/restore is the only feature that
+ * requires that. And checkpoint/restore does not work
+ * when a device driver is involved (you cannot easily
+ * save and restore device driver state).
+ */
+ if (is_write_device_private_entry(entry) &&
+ is_cow_mapping(vm_flags)) {
+ make_device_private_entry_read(&entry);
+ pte = swp_entry_to_pte(entry);
+ if (pte_swp_uffd_wp(*src_pte))
+ pte = pte_swp_mkuffd_wp(pte);
+ set_pte_at(src_mm, addr, src_pte, pte);
}
- goto out_set_pte;
+ }
+ set_pte_at(dst_mm, addr, dst_pte, pte);
+ return 0;
+}
+
+/*
+ * Copy a present and normal page if necessary.
+ *
+ * NOTE! The usual case is that this doesn't need to do
+ * anything, and can just return a positive value. That
+ * will let the caller know that it can just increase
+ * the page refcount and re-use the pte the traditional
+ * way.
+ *
+ * But _if_ we need to copy it because it needs to be
+ * pinned in the parent (and the child should get its own
+ * copy rather than just a reference to the same page),
+ * we'll do that here and return zero to let the caller
+ * know we're done.
+ *
+ * And if we need a pre-allocated page but don't yet have
+ * one, return a negative error to let the preallocation
+ * code know so that it can do so outside the page table
+ * lock.
+ */
+static inline int
+copy_present_page(struct mm_struct *dst_mm, struct mm_struct *src_mm,
+ pte_t *dst_pte, pte_t *src_pte,
+ struct vm_area_struct *vma, struct vm_area_struct *new,
+ unsigned long addr, int *rss, struct page **prealloc,
+ pte_t pte, struct page *page)
+{
+ struct page *new_page;
+
+ if (!is_cow_mapping(vma->vm_flags))
+ return 1;
+
+ /*
+ * What we want to do is to check whether this page may
+ * have been pinned by the parent process. If so,
+ * instead of wrprotect the pte on both sides, we copy
+ * the page immediately so that we'll always guarantee
+ * the pinned page won't be randomly replaced in the
+ * future.
+ *
+ * The page pinning checks are just "has this mm ever
+ * seen pinning", along with the (inexact) check of
+ * the page count. That might give false positives for
+ * for pinning, but it will work correctly.
+ */
+ if (likely(!atomic_read(&src_mm->has_pinned)))
+ return 1;
+ if (likely(!page_maybe_dma_pinned(page)))
+ return 1;
+
+ new_page = *prealloc;
+ if (!new_page)
+ return -EAGAIN;
+
+ /*
+ * We have a prealloc page, all good! Take it
+ * over and copy the page & arm it.
+ */
+ *prealloc = NULL;
+ copy_user_highpage(new_page, page, addr, vma);
+ __SetPageUptodate(new_page);
+ page_add_new_anon_rmap(new_page, new, addr, false);
+ lru_cache_add_inactive_or_unevictable(new_page, new);
+ rss[mm_counter(new_page)]++;
+
+ /* All done, just insert the new page copy in the child */
+ pte = mk_pte(new_page, new->vm_page_prot);
+ pte = maybe_mkwrite(pte_mkdirty(pte), new);
+ set_pte_at(dst_mm, addr, dst_pte, pte);
+ return 0;
+}
+
+/*
+ * Copy one pte. Returns 0 if succeeded, or -EAGAIN if one preallocated page
+ * is required to copy this pte.
+ */
+static inline int
+copy_present_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
+ pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
+ struct vm_area_struct *new,
+ unsigned long addr, int *rss, struct page **prealloc)
+{
+ unsigned long vm_flags = vma->vm_flags;
+ pte_t pte = *src_pte;
+ struct page *page;
+
+ page = vm_normal_page(vma, addr, pte);
+ if (page) {
+ int retval;
+
+ retval = copy_present_page(dst_mm, src_mm,
+ dst_pte, src_pte,
+ vma, new,
+ addr, rss, prealloc,
+ pte, page);
+ if (retval <= 0)
+ return retval;
+
+ get_page(page);
+ page_dup_rmap(page, false);
+ rss[mm_counter(page)]++;
}
/*
if (!(vm_flags & VM_UFFD_WP))
pte = pte_clear_uffd_wp(pte);
- page = vm_normal_page(vma, addr, pte);
- if (page) {
- get_page(page);
- page_dup_rmap(page, false);
- rss[mm_counter(page)]++;
- }
-
-out_set_pte:
set_pte_at(dst_mm, addr, dst_pte, pte);
return 0;
}
+static inline struct page *
+page_copy_prealloc(struct mm_struct *src_mm, struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ struct page *new_page;
+
+ new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, addr);
+ if (!new_page)
+ return NULL;
+
+ if (mem_cgroup_charge(new_page, src_mm, GFP_KERNEL)) {
+ put_page(new_page);
+ return NULL;
+ }
+ cgroup_throttle_swaprate(new_page, GFP_KERNEL);
+
+ return new_page;
+}
+
static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
+ struct vm_area_struct *new,
unsigned long addr, unsigned long end)
{
pte_t *orig_src_pte, *orig_dst_pte;
pte_t *src_pte, *dst_pte;
spinlock_t *src_ptl, *dst_ptl;
- int progress = 0;
+ int progress, ret = 0;
int rss[NR_MM_COUNTERS];
swp_entry_t entry = (swp_entry_t){0};
+ struct page *prealloc = NULL;
again:
+ progress = 0;
init_rss_vec(rss);
dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
- if (!dst_pte)
- return -ENOMEM;
+ if (!dst_pte) {
+ ret = -ENOMEM;
+ goto out;
+ }
src_pte = pte_offset_map(src_pmd, addr);
src_ptl = pte_lockptr(src_mm, src_pmd);
spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
progress++;
continue;
}
- entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte,
+ if (unlikely(!pte_present(*src_pte))) {
+ entry.val = copy_nonpresent_pte(dst_mm, src_mm,
+ dst_pte, src_pte,
vma, addr, rss);
- if (entry.val)
+ if (entry.val)
+ break;
+ progress += 8;
+ continue;
+ }
+ /* copy_present_pte() will clear `*prealloc' if consumed */
+ ret = copy_present_pte(dst_mm, src_mm, dst_pte, src_pte,
+ vma, new, addr, rss, &prealloc);
+ /*
+ * If we need a pre-allocated page for this pte, drop the
+ * locks, allocate, and try again.
+ */
+ if (unlikely(ret == -EAGAIN))
break;
+ if (unlikely(prealloc)) {
+ /*
+ * pre-alloc page cannot be reused by next time so as
+ * to strictly follow mempolicy (e.g., alloc_page_vma()
+ * will allocate page according to address). This
+ * could only happen if one pinned pte changed.
+ */
+ put_page(prealloc);
+ prealloc = NULL;
+ }
progress += 8;
} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
cond_resched();
if (entry.val) {
- if (add_swap_count_continuation(entry, GFP_KERNEL) < 0)
+ if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ entry.val = 0;
+ } else if (ret) {
+ WARN_ON_ONCE(ret != -EAGAIN);
+ prealloc = page_copy_prealloc(src_mm, vma, addr);
+ if (!prealloc)
return -ENOMEM;
- progress = 0;
+ /* We've captured and resolved the error. Reset, try again. */
+ ret = 0;
}
if (addr != end)
goto again;
- return 0;
+out:
+ if (unlikely(prealloc))
+ put_page(prealloc);
+ return ret;
}
static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma,
+ struct vm_area_struct *new,
unsigned long addr, unsigned long end)
{
pmd_t *src_pmd, *dst_pmd;
if (pmd_none_or_clear_bad(src_pmd))
continue;
if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,
- vma, addr, next))
+ vma, new, addr, next))
return -ENOMEM;
} while (dst_pmd++, src_pmd++, addr = next, addr != end);
return 0;
static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
p4d_t *dst_p4d, p4d_t *src_p4d, struct vm_area_struct *vma,
+ struct vm_area_struct *new,
unsigned long addr, unsigned long end)
{
pud_t *src_pud, *dst_pud;
if (pud_none_or_clear_bad(src_pud))
continue;
if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud,
- vma, addr, next))
+ vma, new, addr, next))
return -ENOMEM;
} while (dst_pud++, src_pud++, addr = next, addr != end);
return 0;
static inline int copy_p4d_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
+ struct vm_area_struct *new,
unsigned long addr, unsigned long end)
{
p4d_t *src_p4d, *dst_p4d;
if (p4d_none_or_clear_bad(src_p4d))
continue;
if (copy_pud_range(dst_mm, src_mm, dst_p4d, src_p4d,
- vma, addr, next))
+ vma, new, addr, next))
return -ENOMEM;
} while (dst_p4d++, src_p4d++, addr = next, addr != end);
return 0;
}
int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
- struct vm_area_struct *vma)
+ struct vm_area_struct *vma, struct vm_area_struct *new)
{
pgd_t *src_pgd, *dst_pgd;
unsigned long next;
if (pgd_none_or_clear_bad(src_pgd))
continue;
if (unlikely(copy_p4d_range(dst_mm, src_mm, dst_pgd, src_pgd,
- vma, addr, next))) {
+ vma, new, addr, next))) {
ret = -ENOMEM;
break;
}
* page count reference, and the page is locked,
* it's dark out, and we're wearing sunglasses. Hit it.
*/
- wp_page_reuse(vmf);
unlock_page(page);
+ wp_page_reuse(vmf);
return VM_FAULT_WRITE;
} else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
(VM_WRITE|VM_SHARED))) {
* are reserved so nobody should be touching them so we should be safe
*/
memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn,
- MEMMAP_HOTPLUG, altmap);
+ MEMINIT_HOTPLUG, altmap);
set_zone_contiguous(zone);
}
}
/* link memory sections under this node.*/
- ret = link_mem_sections(nid, PFN_DOWN(start), PFN_UP(start + size - 1));
+ ret = link_mem_sections(nid, PFN_DOWN(start), PFN_UP(start + size - 1),
+ MEMINIT_HOTPLUG);
BUG_ON(ret);
/* create new memmap entry */
* Capture required information that might get lost
* during migration.
*/
- is_thp = PageTransHuge(page);
+ is_thp = PageTransHuge(page) && !PageHuge(page);
nr_subpages = thp_nr_pages(page);
cond_resched();
* we encounter them after the rest of the list
* is processed.
*/
- if (PageTransHuge(page) && !PageHuge(page)) {
+ if (is_thp) {
lock_page(page);
rc = split_huge_page_to_list(page, from);
unlock_page(page);
nr_thp_split++;
goto retry;
}
- }
- if (is_thp) {
+
nr_thp_failed++;
nr_failed += nr_subpages;
goto out;
merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags,
NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX);
if (merge) {
- fput(file);
+ /* ->mmap() can change vma->vm_file and fput the original file. So
+ * fput the vma->vm_file here or we would add an extra fput for file
+ * and cause general protection fault ultimately.
+ */
+ fput(vma->vm_file);
vm_area_free(vma);
vma = merge;
/* Update vm_flags and possible addr to pick up the change. We don't
vma_set_anonymous(vma);
}
+ /* Allow architectures to sanity-check the vm_flags */
+ if (!arch_validate_flags(vma->vm_flags)) {
+ error = -EINVAL;
+ if (file)
+ goto unmap_and_free_vma;
+ else
+ goto free_vma;
+ }
+
vma_link(mm, vma, prev, rb_link, rb_parent);
/* Once vma denies write, undo our temporary denial count */
if (file) {
goto out;
}
+ /* Allow architectures to sanity-check the new flags */
+ if (!arch_validate_flags(newflags)) {
+ error = -EINVAL;
+ goto out;
+ }
+
error = security_file_mprotect(vma, reqprot, prot);
if (error)
goto out;
* Replacement code for mm functions to support CPU's that don't
* have any form of memory management unit (thus no virtual memory).
*
- * See Documentation/mm/nommu-mmap.rst
+ * See Documentation/admin-guide/mm/nommu-mmap.rst
*
* Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
* Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
#include <linux/nmi.h>
#include <linux/psi.h>
#include <linux/padata.h>
+#include <linux/khugepaged.h>
#include <asm/sections.h>
#include <asm/tlbflush.h>
struct page *page;
if (likely(order == 0)) {
- page = rmqueue_pcplist(preferred_zone, zone, gfp_flags,
+ /*
+ * MIGRATE_MOVABLE pcplist could have the pages on CMA area and
+ * we need to skip it when CMA area isn't allowed.
+ */
+ if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA ||
+ migratetype != MIGRATE_MOVABLE) {
+ page = rmqueue_pcplist(preferred_zone, zone, gfp_flags,
migratetype, alloc_flags);
- goto out;
+ goto out;
+ }
}
/*
do {
page = NULL;
- if (alloc_flags & ALLOC_HARDER) {
+ /*
+ * order-0 request can reach here when the pcplist is skipped
+ * due to non-CMA allocation context. HIGHATOMIC area is
+ * reserved for high-order atomic allocation, so order-0
+ * request should skip it.
+ */
+ if (order > 0 && alloc_flags & ALLOC_HARDER) {
page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
if (page)
trace_mm_page_alloc_zone_locked(page, order, migratetype);
* done. Non-atomic initialization, single-pass.
*/
void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
- unsigned long start_pfn, enum memmap_context context,
+ unsigned long start_pfn, enum meminit_context context,
struct vmem_altmap *altmap)
{
unsigned long pfn, end_pfn = start_pfn + size;
* There can be holes in boot-time mem_map[]s handed to this
* function. They do not exist on hotplugged memory.
*/
- if (context == MEMMAP_EARLY) {
+ if (context == MEMINIT_EARLY) {
if (overlap_memmap_init(zone, &pfn))
continue;
if (defer_init(nid, pfn, end_pfn))
page = pfn_to_page(pfn);
__init_single_page(page, pfn, zone, nid);
- if (context == MEMMAP_HOTPLUG)
+ if (context == MEMINIT_HOTPLUG)
__SetPageReserved(page);
/*
* check here not to call set_pageblock_migratetype() against
* pfn out of zone.
*
- * Please note that MEMMAP_HOTPLUG path doesn't clear memmap
+ * Please note that MEMINIT_HOTPLUG path doesn't clear memmap
* because this is done early in section_activate()
*/
if (!(pfn & (pageblock_nr_pages - 1))) {
if (end_pfn > start_pfn) {
size = end_pfn - start_pfn;
memmap_init_zone(size, nid, zone, start_pfn,
- MEMMAP_EARLY, NULL);
+ MEMINIT_EARLY, NULL);
}
}
}
setup_min_slab_ratio();
#endif
+ khugepaged_min_free_kbytes_update();
+
return 0;
}
postcore_initcall(init_per_zone_wmark_min)
unlock_page(page);
goto out;
}
+ /*
+ * Arch code may have to preserve more data than just the page
+ * contents, e.g. memory tags.
+ */
+ ret = arch_prepare_to_swap(page);
+ if (ret) {
+ set_page_dirty(page);
+ unlock_page(page);
+ goto out;
+ }
if (frontswap_store(page) == 0) {
set_page_writeback(page);
unlock_page(page);
#include <linux/slab.h>
#include <linux/syscalls.h>
-#ifdef CONFIG_COMPAT
-#include <linux/compat.h>
-#endif
-
/**
* process_vm_rw_pages - read/write pages from task specified
* @pages: array of pointers to pages we want to copy
if (rc < 0)
return rc;
if (!iov_iter_count(&iter))
- goto free_iovecs;
-
- rc = rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt, UIO_FASTIOV,
- iovstack_r, &iov_r);
- if (rc <= 0)
- goto free_iovecs;
-
+ goto free_iov_l;
+ iov_r = iovec_from_user(rvec, riovcnt, UIO_FASTIOV, iovstack_r, false);
+ if (IS_ERR(iov_r)) {
+ rc = PTR_ERR(iov_r);
+ goto free_iov_l;
+ }
rc = process_vm_rw_core(pid, &iter, iov_r, riovcnt, flags, vm_write);
-
-free_iovecs:
if (iov_r != iovstack_r)
kfree(iov_r);
+free_iov_l:
kfree(iov_l);
-
return rc;
}
{
return process_vm_rw(pid, lvec, liovcnt, rvec, riovcnt, flags, 1);
}
-
-#ifdef CONFIG_COMPAT
-
-static ssize_t
-compat_process_vm_rw(compat_pid_t pid,
- const struct compat_iovec __user *lvec,
- unsigned long liovcnt,
- const struct compat_iovec __user *rvec,
- unsigned long riovcnt,
- unsigned long flags, int vm_write)
-{
- struct iovec iovstack_l[UIO_FASTIOV];
- struct iovec iovstack_r[UIO_FASTIOV];
- struct iovec *iov_l = iovstack_l;
- struct iovec *iov_r = iovstack_r;
- struct iov_iter iter;
- ssize_t rc = -EFAULT;
- int dir = vm_write ? WRITE : READ;
-
- if (flags != 0)
- return -EINVAL;
-
- rc = compat_import_iovec(dir, lvec, liovcnt, UIO_FASTIOV, &iov_l, &iter);
- if (rc < 0)
- return rc;
- if (!iov_iter_count(&iter))
- goto free_iovecs;
- rc = compat_rw_copy_check_uvector(CHECK_IOVEC_ONLY, rvec, riovcnt,
- UIO_FASTIOV, iovstack_r,
- &iov_r);
- if (rc <= 0)
- goto free_iovecs;
-
- rc = process_vm_rw_core(pid, &iter, iov_r, riovcnt, flags, vm_write);
-
-free_iovecs:
- if (iov_r != iovstack_r)
- kfree(iov_r);
- kfree(iov_l);
- return rc;
-}
-
-COMPAT_SYSCALL_DEFINE6(process_vm_readv, compat_pid_t, pid,
- const struct compat_iovec __user *, lvec,
- compat_ulong_t, liovcnt,
- const struct compat_iovec __user *, rvec,
- compat_ulong_t, riovcnt,
- compat_ulong_t, flags)
-{
- return compat_process_vm_rw(pid, lvec, liovcnt, rvec,
- riovcnt, flags, 0);
-}
-
-COMPAT_SYSCALL_DEFINE6(process_vm_writev, compat_pid_t, pid,
- const struct compat_iovec __user *, lvec,
- compat_ulong_t, liovcnt,
- const struct compat_iovec __user *, rvec,
- compat_ulong_t, riovcnt,
- compat_ulong_t, flags)
-{
- return compat_process_vm_rw(pid, lvec, liovcnt, rvec,
- riovcnt, flags, 1);
-}
-
-#endif
}
wait_on_page_writeback(page);
+ /*
+ * Some architectures may have to restore extra metadata to the
+ * physical page after reading from swap.
+ */
+ arch_swap_restore(swap, page);
+
if (shmem_should_replace_page(page, gfp)) {
error = shmem_replace_page(&page, gfp, info, index);
if (error)
vma->vm_flags &= ~(VM_MAYWRITE);
}
+ /* arm64 - allow memory tagging on RAM-based files */
+ vma->vm_flags |= VM_MTE_ALLOWED;
+
file_accessed(file);
vma->vm_ops = &shmem_vm_ops;
if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
kmem_cache_free(cachep->freelist_cache, freelist);
}
+/*
+ * Update the size of the caches before calling slabs_destroy as it may
+ * recursively call kfree.
+ */
static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list)
{
struct page *page, *n;
spin_lock(&n->list_lock);
free_block(cachep, ac->entry, ac->avail, node, &list);
spin_unlock(&n->list_lock);
- slabs_destroy(cachep, &list);
ac->avail = 0;
+ slabs_destroy(cachep, &list);
}
static void drain_cpu_caches(struct kmem_cache *cachep)
}
#endif
spin_unlock(&n->list_lock);
- slabs_destroy(cachep, &list);
ac->avail -= batchcount;
memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
+ slabs_destroy(cachep, &list);
}
/*
char *next_block;
slab_flags_t block_flags;
- /* If slub_debug = 0, it folds into the if conditional. */
- if (!slub_debug_string)
- return flags | slub_debug;
-
len = strlen(name);
next_block = slub_debug_string;
/* Go through all blocks of debug options, see if any matches our slab's name */
}
}
- return slub_debug;
+ return flags | slub_debug;
}
#else /* !CONFIG_SLUB_DEBUG */
static inline void setup_object_debug(struct kmem_cache *s,
*/
void lru_add_drain_all(void)
{
- static seqcount_t seqcount = SEQCNT_ZERO(seqcount);
- static DEFINE_MUTEX(lock);
+ /*
+ * lru_drain_gen - Global pages generation number
+ *
+ * (A) Definition: global lru_drain_gen = x implies that all generations
+ * 0 < n <= x are already *scheduled* for draining.
+ *
+ * This is an optimization for the highly-contended use case where a
+ * user space workload keeps constantly generating a flow of pages for
+ * each CPU.
+ */
+ static unsigned int lru_drain_gen;
static struct cpumask has_work;
- int cpu, seq;
+ static DEFINE_MUTEX(lock);
+ unsigned cpu, this_gen;
/*
* Make sure nobody triggers this path before mm_percpu_wq is fully
if (WARN_ON(!mm_percpu_wq))
return;
- seq = raw_read_seqcount_latch(&seqcount);
+ /*
+ * Guarantee pagevec counter stores visible by this CPU are visible to
+ * other CPUs before loading the current drain generation.
+ */
+ smp_mb();
+
+ /*
+ * (B) Locally cache global LRU draining generation number
+ *
+ * The read barrier ensures that the counter is loaded before the mutex
+ * is taken. It pairs with smp_mb() inside the mutex critical section
+ * at (D).
+ */
+ this_gen = smp_load_acquire(&lru_drain_gen);
mutex_lock(&lock);
/*
- * Piggyback on drain started and finished while we waited for lock:
- * all pages pended at the time of our enter were drained from vectors.
+ * (C) Exit the draining operation if a newer generation, from another
+ * lru_add_drain_all(), was already scheduled for draining. Check (A).
*/
- if (__read_seqcount_retry(&seqcount, seq))
+ if (unlikely(this_gen != lru_drain_gen))
goto done;
- raw_write_seqcount_latch(&seqcount);
+ /*
+ * (D) Increment global generation number
+ *
+ * Pairs with smp_load_acquire() at (B), outside of the critical
+ * section. Use a full memory barrier to guarantee that the new global
+ * drain generation number is stored before loading pagevec counters.
+ *
+ * This pairing must be done here, before the for_each_online_cpu loop
+ * below which drains the page vectors.
+ *
+ * Let x, y, and z represent some system CPU numbers, where x < y < z.
+ * Assume CPU #z is is in the middle of the for_each_online_cpu loop
+ * below and has already reached CPU #y's per-cpu data. CPU #x comes
+ * along, adds some pages to its per-cpu vectors, then calls
+ * lru_add_drain_all().
+ *
+ * If the paired barrier is done at any later step, e.g. after the
+ * loop, CPU #x will just exit at (C) and miss flushing out all of its
+ * added pages.
+ */
+ WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1);
+ smp_mb();
cpumask_clear(&has_work);
-
for_each_online_cpu(cpu) {
struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
need_activate_page_drain(cpu)) {
INIT_WORK(work, lru_add_drain_per_cpu);
queue_work_on(cpu, mm_percpu_wq, work);
- cpumask_set_cpu(cpu, &has_work);
+ __cpumask_set_cpu(cpu, &has_work);
}
}
{
lru_add_drain();
}
-#endif
+#endif /* CONFIG_SMP */
/**
* release_pages - batched put_page()
else
swap_slot_free_notify = NULL;
while (offset <= end) {
+ arch_swap_invalidate_page(si->type, offset);
frontswap_invalidate_page(si->type, offset);
if (swap_slot_free_notify)
swap_slot_free_notify(si->bdev, offset);
goto nextsi;
}
if (size == SWAPFILE_CLUSTER) {
- if (!(si->flags & SWP_FS))
+ if (si->flags & SWP_BLKDEV)
n_ret = swap_alloc_cluster(si, swp_entries);
} else
n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE,
frontswap_map = frontswap_map_get(p);
spin_unlock(&p->lock);
spin_unlock(&swap_lock);
+ arch_swap_invalidate_area(p->type);
frontswap_invalidate_area(p->type);
frontswap_map_set(p, NULL);
mutex_unlock(&swapon_mutex);
return res;
}
-int memcmp_pages(struct page *page1, struct page *page2)
+int __weak memcmp_pages(struct page *page1, struct page *page2)
{
char *addr1, *addr2;
int ret;
}
}
-static int br_chk_addr_ip(struct net_device *dev, void *data)
+static int br_chk_addr_ip(struct net_device *dev,
+ struct netdev_nested_priv *priv)
{
- __be32 ip = *(__be32 *)data;
+ __be32 ip = *(__be32 *)priv->data;
struct in_device *in_dev;
__be32 addr = 0;
static bool br_is_local_ip(struct net_device *dev, __be32 ip)
{
- if (br_chk_addr_ip(dev, &ip))
+ struct netdev_nested_priv priv = {
+ .data = (void *)&ip,
+ };
+
+ if (br_chk_addr_ip(dev, &priv))
return true;
/* check if ip is configured on upper dev */
- if (netdev_walk_all_upper_dev_rcu(dev, br_chk_addr_ip, &ip))
+ if (netdev_walk_all_upper_dev_rcu(dev, br_chk_addr_ip, &priv))
return true;
return false;
}
}
-static int br_chk_addr_ip6(struct net_device *dev, void *data)
+static int br_chk_addr_ip6(struct net_device *dev,
+ struct netdev_nested_priv *priv)
{
- struct in6_addr *addr = (struct in6_addr *)data;
+ struct in6_addr *addr = (struct in6_addr *)priv->data;
if (ipv6_chk_addr(dev_net(dev), addr, dev, 0))
return 1;
static bool br_is_local_ip6(struct net_device *dev, struct in6_addr *addr)
{
- if (br_chk_addr_ip6(dev, addr))
+ struct netdev_nested_priv priv = {
+ .data = (void *)addr,
+ };
+
+ if (br_chk_addr_ip6(dev, &priv))
return true;
/* check if ip is configured on upper dev */
- if (netdev_walk_all_upper_dev_rcu(dev, br_chk_addr_ip6, addr))
+ if (netdev_walk_all_upper_dev_rcu(dev, br_chk_addr_ip6, &priv))
return true;
return false;
if (!do_all)
if (test_bit(BR_FDB_STATIC, &f->flags) ||
+ (test_bit(BR_FDB_ADDED_BY_EXT_LEARN, &f->flags) &&
+ !test_bit(BR_FDB_OFFLOADED, &f->flags)) ||
(vid && f->key.vlan_id != vid))
continue;
u32 filter_mask, const struct net_device *dev)
{
u8 operstate = netif_running(dev) ? dev->operstate : IF_OPER_DOWN;
+ struct nlattr *af = NULL;
struct net_bridge *br;
struct ifinfomsg *hdr;
struct nlmsghdr *nlh;
nla_nest_end(skb, nest);
}
+ if (filter_mask & (RTEXT_FILTER_BRVLAN |
+ RTEXT_FILTER_BRVLAN_COMPRESSED |
+ RTEXT_FILTER_MRP)) {
+ af = nla_nest_start_noflag(skb, IFLA_AF_SPEC);
+ if (!af)
+ goto nla_put_failure;
+ }
+
/* Check if the VID information is requested */
if ((filter_mask & RTEXT_FILTER_BRVLAN) ||
(filter_mask & RTEXT_FILTER_BRVLAN_COMPRESSED)) {
struct net_bridge_vlan_group *vg;
- struct nlattr *af;
int err;
/* RCU needed because of the VLAN locking rules (rcu || rtnl) */
rcu_read_unlock();
goto done;
}
- af = nla_nest_start_noflag(skb, IFLA_AF_SPEC);
- if (!af) {
- rcu_read_unlock();
- goto nla_put_failure;
- }
if (filter_mask & RTEXT_FILTER_BRVLAN_COMPRESSED)
err = br_fill_ifvlaninfo_compressed(skb, vg);
else
rcu_read_unlock();
if (err)
goto nla_put_failure;
-
- nla_nest_end(skb, af);
}
if (filter_mask & RTEXT_FILTER_MRP) {
- struct nlattr *af;
int err;
if (!br_mrp_enabled(br) || port)
goto done;
- af = nla_nest_start_noflag(skb, IFLA_AF_SPEC);
- if (!af)
- goto nla_put_failure;
-
rcu_read_lock();
err = br_mrp_fill_info(skb, br);
rcu_read_unlock();
if (err)
goto nla_put_failure;
-
- nla_nest_end(skb, af);
}
done:
+ if (af)
+ nla_nest_end(skb, af);
nlmsg_end(skb, nlh);
return 0;
}
static int br_vlan_is_bind_vlan_dev_fn(struct net_device *dev,
- __always_unused void *data)
+ __always_unused struct netdev_nested_priv *priv)
{
return br_vlan_is_bind_vlan_dev(dev);
}
};
static int br_vlan_match_bind_vlan_dev_fn(struct net_device *dev,
- void *data_in)
+ struct netdev_nested_priv *priv)
{
- struct br_vlan_bind_walk_data *data = data_in;
+ struct br_vlan_bind_walk_data *data = priv->data;
int found = 0;
if (br_vlan_is_bind_vlan_dev(dev) &&
struct br_vlan_bind_walk_data data = {
.vid = vid,
};
+ struct netdev_nested_priv priv = {
+ .data = (void *)&data,
+ };
rcu_read_lock();
netdev_walk_all_upper_dev_rcu(dev, br_vlan_match_bind_vlan_dev_fn,
- &data);
+ &priv);
rcu_read_unlock();
return data.result;
};
static int br_vlan_link_state_change_fn(struct net_device *vlan_dev,
- void *data_in)
+ struct netdev_nested_priv *priv)
{
- struct br_vlan_link_state_walk_data *data = data_in;
+ struct br_vlan_link_state_walk_data *data = priv->data;
if (br_vlan_is_bind_vlan_dev(vlan_dev))
br_vlan_set_vlan_dev_state(data->br, vlan_dev);
struct br_vlan_link_state_walk_data data = {
.br = br
};
+ struct netdev_nested_priv priv = {
+ .data = (void *)&data,
+ };
rcu_read_lock();
netdev_walk_all_upper_dev_rcu(dev, br_vlan_link_state_change_fn,
- &data);
+ &priv);
rcu_read_unlock();
}
* coalescing neighboring slab objects into a single frag which
* triggers one of hardened usercopy checks.
*/
- if (page_count(page) >= 1 && !PageSlab(page))
+ if (sendpage_ok(page))
sendpage = sock->ops->sendpage;
else
sendpage = sock_no_sendpage;
if (err)
return err;
- err = compat_import_iovec(save_addr ? READ : WRITE, compat_ptr(ptr),
- len, UIO_FASTIOV, iov, &kmsg->msg_iter);
+ err = import_iovec(save_addr ? READ : WRITE, compat_ptr(ptr), len,
+ UIO_FASTIOV, iov, &kmsg->msg_iter);
return err < 0 ? err : 0;
}
return NULL;
}
-static int ____netdev_has_upper_dev(struct net_device *upper_dev, void *data)
+static int ____netdev_has_upper_dev(struct net_device *upper_dev,
+ struct netdev_nested_priv *priv)
{
- struct net_device *dev = data;
+ struct net_device *dev = (struct net_device *)priv->data;
return upper_dev == dev;
}
bool netdev_has_upper_dev(struct net_device *dev,
struct net_device *upper_dev)
{
+ struct netdev_nested_priv priv = {
+ .data = (void *)upper_dev,
+ };
+
ASSERT_RTNL();
return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
- upper_dev);
+ &priv);
}
EXPORT_SYMBOL(netdev_has_upper_dev);
bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
struct net_device *upper_dev)
{
+ struct netdev_nested_priv priv = {
+ .data = (void *)upper_dev,
+ };
+
return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
- upper_dev);
+ &priv);
}
EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
static int __netdev_walk_all_upper_dev(struct net_device *dev,
int (*fn)(struct net_device *dev,
- void *data),
- void *data)
+ struct netdev_nested_priv *priv),
+ struct netdev_nested_priv *priv)
{
struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
while (1) {
if (now != dev) {
- ret = fn(now, data);
+ ret = fn(now, priv);
if (ret)
return ret;
}
int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
int (*fn)(struct net_device *dev,
- void *data),
- void *data)
+ struct netdev_nested_priv *priv),
+ struct netdev_nested_priv *priv)
{
struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
while (1) {
if (now != dev) {
- ret = fn(now, data);
+ ret = fn(now, priv);
if (ret)
return ret;
}
static bool __netdev_has_upper_dev(struct net_device *dev,
struct net_device *upper_dev)
{
+ struct netdev_nested_priv priv = {
+ .flags = 0,
+ .data = (void *)upper_dev,
+ };
+
ASSERT_RTNL();
return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
- upper_dev);
+ &priv);
}
/**
int netdev_walk_all_lower_dev(struct net_device *dev,
int (*fn)(struct net_device *dev,
- void *data),
- void *data)
+ struct netdev_nested_priv *priv),
+ struct netdev_nested_priv *priv)
{
struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
while (1) {
if (now != dev) {
- ret = fn(now, data);
+ ret = fn(now, priv);
if (ret)
return ret;
}
static int __netdev_walk_all_lower_dev(struct net_device *dev,
int (*fn)(struct net_device *dev,
- void *data),
- void *data)
+ struct netdev_nested_priv *priv),
+ struct netdev_nested_priv *priv)
{
struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
while (1) {
if (now != dev) {
- ret = fn(now, data);
+ ret = fn(now, priv);
if (ret)
return ret;
}
return max_depth;
}
-static int __netdev_update_upper_level(struct net_device *dev, void *data)
+static int __netdev_update_upper_level(struct net_device *dev,
+ struct netdev_nested_priv *__unused)
{
dev->upper_level = __netdev_upper_depth(dev) + 1;
return 0;
}
-static int __netdev_update_lower_level(struct net_device *dev, void *data)
+static int __netdev_update_lower_level(struct net_device *dev,
+ struct netdev_nested_priv *priv)
{
dev->lower_level = __netdev_lower_depth(dev) + 1;
+
+#ifdef CONFIG_LOCKDEP
+ if (!priv)
+ return 0;
+
+ if (priv->flags & NESTED_SYNC_IMM)
+ dev->nested_level = dev->lower_level - 1;
+ if (priv->flags & NESTED_SYNC_TODO)
+ net_unlink_todo(dev);
+#endif
return 0;
}
int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
int (*fn)(struct net_device *dev,
- void *data),
- void *data)
+ struct netdev_nested_priv *priv),
+ struct netdev_nested_priv *priv)
{
struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
while (1) {
if (now != dev) {
- ret = fn(now, data);
+ ret = fn(now, priv);
if (ret)
return ret;
}
static int __netdev_upper_dev_link(struct net_device *dev,
struct net_device *upper_dev, bool master,
void *upper_priv, void *upper_info,
+ struct netdev_nested_priv *priv,
struct netlink_ext_ack *extack)
{
struct netdev_notifier_changeupper_info changeupper_info = {
__netdev_update_upper_level(dev, NULL);
__netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
- __netdev_update_lower_level(upper_dev, NULL);
+ __netdev_update_lower_level(upper_dev, priv);
__netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
- NULL);
+ priv);
return 0;
struct net_device *upper_dev,
struct netlink_ext_ack *extack)
{
+ struct netdev_nested_priv priv = {
+ .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
+ .data = NULL,
+ };
+
return __netdev_upper_dev_link(dev, upper_dev, false,
- NULL, NULL, extack);
+ NULL, NULL, &priv, extack);
}
EXPORT_SYMBOL(netdev_upper_dev_link);
void *upper_priv, void *upper_info,
struct netlink_ext_ack *extack)
{
+ struct netdev_nested_priv priv = {
+ .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
+ .data = NULL,
+ };
+
return __netdev_upper_dev_link(dev, upper_dev, true,
- upper_priv, upper_info, extack);
+ upper_priv, upper_info, &priv, extack);
}
EXPORT_SYMBOL(netdev_master_upper_dev_link);
-/**
- * netdev_upper_dev_unlink - Removes a link to upper device
- * @dev: device
- * @upper_dev: new upper device
- *
- * Removes a link to device which is upper to this one. The caller must hold
- * the RTNL lock.
- */
-void netdev_upper_dev_unlink(struct net_device *dev,
- struct net_device *upper_dev)
+static void __netdev_upper_dev_unlink(struct net_device *dev,
+ struct net_device *upper_dev,
+ struct netdev_nested_priv *priv)
{
struct netdev_notifier_changeupper_info changeupper_info = {
.info = {
__netdev_update_upper_level(dev, NULL);
__netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
- __netdev_update_lower_level(upper_dev, NULL);
+ __netdev_update_lower_level(upper_dev, priv);
__netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
- NULL);
+ priv);
+}
+
+/**
+ * netdev_upper_dev_unlink - Removes a link to upper device
+ * @dev: device
+ * @upper_dev: new upper device
+ *
+ * Removes a link to device which is upper to this one. The caller must hold
+ * the RTNL lock.
+ */
+void netdev_upper_dev_unlink(struct net_device *dev,
+ struct net_device *upper_dev)
+{
+ struct netdev_nested_priv priv = {
+ .flags = NESTED_SYNC_TODO,
+ .data = NULL,
+ };
+
+ __netdev_upper_dev_unlink(dev, upper_dev, &priv);
}
EXPORT_SYMBOL(netdev_upper_dev_unlink);
struct net_device *dev,
struct netlink_ext_ack *extack)
{
+ struct netdev_nested_priv priv = {
+ .flags = 0,
+ .data = NULL,
+ };
int err;
if (!new_dev)
if (old_dev && new_dev != old_dev)
netdev_adjacent_dev_disable(dev, old_dev);
-
- err = netdev_upper_dev_link(new_dev, dev, extack);
+ err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
+ extack);
if (err) {
if (old_dev && new_dev != old_dev)
netdev_adjacent_dev_enable(dev, old_dev);
struct net_device *new_dev,
struct net_device *dev)
{
+ struct netdev_nested_priv priv = {
+ .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
+ .data = NULL,
+ };
+
if (!new_dev || !old_dev)
return;
return;
netdev_adjacent_dev_enable(dev, old_dev);
- netdev_upper_dev_unlink(old_dev, dev);
+ __netdev_upper_dev_unlink(old_dev, dev, &priv);
}
EXPORT_SYMBOL(netdev_adjacent_change_commit);
struct net_device *new_dev,
struct net_device *dev)
{
+ struct netdev_nested_priv priv = {
+ .flags = 0,
+ .data = NULL,
+ };
+
if (!new_dev)
return;
if (old_dev && new_dev != old_dev)
netdev_adjacent_dev_enable(dev, old_dev);
- netdev_upper_dev_unlink(new_dev, dev);
+ __netdev_upper_dev_unlink(new_dev, dev, &priv);
}
EXPORT_SYMBOL(netdev_adjacent_change_abort);
void netdev_run_todo(void)
{
struct list_head list;
+#ifdef CONFIG_LOCKDEP
+ struct list_head unlink_list;
+
+ list_replace_init(&net_unlink_list, &unlink_list);
+
+ while (!list_empty(&unlink_list)) {
+ struct net_device *dev = list_first_entry(&unlink_list,
+ struct net_device,
+ unlink_list);
+ list_del(&dev->unlink_list);
+ dev->nested_level = dev->lower_level - 1;
+ }
+#endif
/* Snapshot list, allow later requests */
list_replace_init(&net_todo_list, &list);
dev->gso_max_segs = GSO_MAX_SEGS;
dev->upper_level = 1;
dev->lower_level = 1;
+#ifdef CONFIG_LOCKDEP
+ dev->nested_level = 0;
+ INIT_LIST_HEAD(&dev->unlink_list);
+#endif
INIT_LIST_HEAD(&dev->napi_list);
INIT_LIST_HEAD(&dev->unreg_list);
if (to->addr_len != from->addr_len)
return -EINVAL;
- netif_addr_lock_nested(to);
+ netif_addr_lock(to);
err = __hw_addr_sync(&to->uc, &from->uc, to->addr_len);
if (!err)
__dev_set_rx_mode(to);
if (to->addr_len != from->addr_len)
return -EINVAL;
- netif_addr_lock_nested(to);
+ netif_addr_lock(to);
err = __hw_addr_sync_multiple(&to->uc, &from->uc, to->addr_len);
if (!err)
__dev_set_rx_mode(to);
* larger.
*/
netif_addr_lock_bh(from);
- netif_addr_lock_nested(to);
+ netif_addr_lock(to);
__hw_addr_unsync(&to->uc, &from->uc, to->addr_len);
__dev_set_rx_mode(to);
netif_addr_unlock(to);
if (to->addr_len != from->addr_len)
return -EINVAL;
- netif_addr_lock_nested(to);
+ netif_addr_lock(to);
err = __hw_addr_sync(&to->mc, &from->mc, to->addr_len);
if (!err)
__dev_set_rx_mode(to);
if (to->addr_len != from->addr_len)
return -EINVAL;
- netif_addr_lock_nested(to);
+ netif_addr_lock(to);
err = __hw_addr_sync_multiple(&to->mc, &from->mc, to->addr_len);
if (!err)
__dev_set_rx_mode(to);
/* See the above comments inside dev_uc_unsync(). */
netif_addr_lock_bh(from);
- netif_addr_lock_nested(to);
+ netif_addr_lock(to);
__hw_addr_unsync(&to->mc, &from->mc, to->addr_len);
__dev_set_rx_mode(to);
netif_addr_unlock(to);
BPF_CALL_1(bpf_skc_to_tcp_timewait_sock, struct sock *, sk)
{
+ /* BTF types for tcp_timewait_sock and inet_timewait_sock are not
+ * generated if CONFIG_INET=n. Trigger an explicit generation here.
+ */
+ BTF_TYPE_EMIT(struct inet_timewait_sock);
+ BTF_TYPE_EMIT(struct tcp_timewait_sock);
+
#ifdef CONFIG_INET
if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_TIME_WAIT)
return (unsigned long)sk;
/* Both of above in one bottle. */
__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
- u8 *to, int len, __wsum csum)
+ u8 *to, int len)
{
int start = skb_headlen(skb);
int i, copy = start - offset;
struct sk_buff *frag_iter;
int pos = 0;
+ __wsum csum = 0;
/* Copy header. */
if (copy > 0) {
if (copy > len)
copy = len;
csum = csum_partial_copy_nocheck(skb->data + offset, to,
- copy, csum);
+ copy);
if ((len -= copy) == 0)
return csum;
offset += copy;
vaddr = kmap_atomic(p);
csum2 = csum_partial_copy_nocheck(vaddr + p_off,
to + copied,
- p_len, 0);
+ p_len);
kunmap_atomic(vaddr);
csum = csum_block_add(csum, csum2, pos);
pos += p_len;
copy = len;
csum2 = skb_copy_and_csum_bits(frag_iter,
offset - start,
- to, copy, 0);
+ to, copy);
csum = csum_block_add(csum, csum2, pos);
if ((len -= copy) == 0)
return csum;
csum = 0;
if (csstart != skb->len)
csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
- skb->len - csstart, 0);
+ skb->len - csstart);
if (skb->ip_summed == CHECKSUM_PARTIAL) {
long csstuff = csstart + skb->csum_offset;
skb_copy_and_csum_bits(head_skb, offset,
skb_put(nskb,
len),
- len, 0);
+ len);
SKB_GSO_CB(nskb)->csum_start =
skb_headroom(nskb) + doffset;
} else {
lse->label_stack_entry = mpls_lse;
skb_postpush_rcsum(skb, lse, MPLS_HLEN);
- if (ethernet)
+ if (ethernet && mac_len >= ETH_HLEN)
skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
skb->protocol = mpls_proto;
skb_reset_mac_header(skb);
skb_set_network_header(skb, mac_len);
- if (ethernet) {
+ if (ethernet && mac_len >= ETH_HLEN) {
struct ethhdr *hdr;
/* use mpls_hdr() to get ethertype to account for VLANs. */
[ETHNL_MCGRP_MONITOR] = { .name = ETHTOOL_MCGRP_MONITOR_NAME },
};
-static struct genl_family ethtool_genl_family = {
+static struct genl_family ethtool_genl_family __ro_after_init = {
.name = ETHTOOL_GENL_NAME,
.version = ETHTOOL_GENL_VERSION,
.netnsok = true,
csum = skb_copy_and_csum_bits(icmp_param->skb,
icmp_param->offset + offset,
- to, len, 0);
+ to, len);
skb->csum = csum_block_add(skb->csum, csum, odd);
if (icmp_pointers[icmp_param->data.icmph.type].error)
ip_flush_pending_frames(sk);
} else if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) {
struct icmphdr *icmph = icmp_hdr(skb);
- __wsum csum = 0;
+ __wsum csum;
struct sk_buff *skb1;
+ csum = csum_partial_copy_nocheck((void *)&icmp_param->data,
+ (char *)icmph,
+ icmp_param->head_len);
skb_queue_walk(&sk->sk_write_queue, skb1) {
csum = csum_add(csum, skb1->csum);
}
- csum = csum_partial_copy_nocheck((void *)&icmp_param->data,
- (char *)icmph,
- icmp_param->head_len, csum);
icmph->checksum = csum_fold(csum);
skb->ip_summed = CHECKSUM_NONE;
ip_push_pending_frames(sk, fl4);
if (fraggap) {
skb->csum = skb_copy_and_csum_bits(
skb_prev, maxfraglen,
- data + transhdrlen, fraggap, 0);
+ data + transhdrlen, fraggap);
skb_prev->csum = csum_sub(skb_prev->csum,
skb->csum);
data += fraggap;
skb->csum = skb_copy_and_csum_bits(skb_prev,
maxfraglen,
skb_transport_header(skb),
- fraggap, 0);
+ fraggap);
skb_prev->csum = csum_sub(skb_prev->csum,
skb->csum);
pskb_trim_unique(skb_prev, maxfraglen);
{
__wsum csum;
- csum = csum_partial_copy_nocheck(dptr+offset, to, len, 0);
+ csum = csum_partial_copy_nocheck(dptr+offset, to, len);
skb->csum = csum_block_add(skb->csum, csum, odd);
return 0;
}
.priority = 0,
};
+#if IS_ENABLED(CONFIG_IPV6)
static struct xfrm_tunnel vti_ipip6_handler __read_mostly = {
.handler = vti_rcv_tunnel,
.cb_handler = vti_rcv_cb,
.priority = 0,
};
#endif
+#endif
static int __net_init vti_init_net(struct net *net)
{
SNMP_MIB_ITEM("TcpTimeoutRehash", LINUX_MIB_TCPTIMEOUTREHASH),
SNMP_MIB_ITEM("TcpDuplicateDataRehash", LINUX_MIB_TCPDUPLICATEDATAREHASH),
SNMP_MIB_ITEM("TCPDSACKRecvSegs", LINUX_MIB_TCPDSACKRECVSEGS),
+ SNMP_MIB_ITEM("TCPDSACKIgnoredDubious", LINUX_MIB_TCPDSACKIGNOREDDUBIOUS),
SNMP_MIB_SENTINEL
};
skb->csum = csum_block_add(
skb->csum,
csum_partial_copy_nocheck(rfv->hdr.c + offset,
- to, copy, 0),
+ to, copy),
odd);
odd = 0;
sock_rps_save_rxhash(child, skb);
if (rsk_drop_req(req)) {
- refcount_set(&req->rsk_refcnt, 2);
+ reqsk_put(req);
return child;
}
long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
if (IS_ENABLED(CONFIG_DEBUG_VM) &&
- WARN_ONCE(PageSlab(page), "page must not be a Slab one"))
+ WARN_ONCE(!sendpage_ok(page),
+ "page must not be a Slab one and have page_count > 0"))
return -EINVAL;
/* Wait for a connection to finish. One exception is TCP Fast Open
struct rate_sample *rate;
};
-/* Take a notice that peer is sending D-SACKs */
+/* Take a notice that peer is sending D-SACKs. Skip update of data delivery
+ * and spurious retransmission information if this DSACK is unlikely caused by
+ * sender's action:
+ * - DSACKed sequence range is larger than maximum receiver's window.
+ * - Total no. of DSACKed segments exceed the total no. of retransmitted segs.
+ */
static u32 tcp_dsack_seen(struct tcp_sock *tp, u32 start_seq,
u32 end_seq, struct tcp_sacktag_state *state)
{
u32 seq_len, dup_segs = 1;
- if (before(start_seq, end_seq)) {
- seq_len = end_seq - start_seq;
- if (seq_len > tp->mss_cache)
- dup_segs = DIV_ROUND_UP(seq_len, tp->mss_cache);
- }
+ if (!before(start_seq, end_seq))
+ return 0;
+
+ seq_len = end_seq - start_seq;
+ /* Dubious DSACK: DSACKed range greater than maximum advertised rwnd */
+ if (seq_len > tp->max_window)
+ return 0;
+ if (seq_len > tp->mss_cache)
+ dup_segs = DIV_ROUND_UP(seq_len, tp->mss_cache);
+
+ tp->dsack_dups += dup_segs;
+ /* Skip the DSACK if dup segs weren't retransmitted by sender */
+ if (tp->dsack_dups > tp->total_retrans)
+ return 0;
tp->rx_opt.sack_ok |= TCP_DSACK_SEEN;
tp->rack.dsack_seen = 1;
- tp->dsack_dups += dup_segs;
state->flag |= FLAG_DSACKING_ACK;
/* A spurious retransmission is delivered */
}
dup_segs = tcp_dsack_seen(tp, start_seq_0, end_seq_0, state);
+ if (!dup_segs) { /* Skip dubious DSACK */
+ NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDSACKIGNOREDDUBIOUS);
+ return false;
+ }
+
NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPDSACKRECVSEGS, dup_segs);
/* D-SACK for already forgotten data... Do dumb counting. */
__skb_pull(skb, hdrlen);
if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
- thtail->window = th->window;
-
TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
- if (after(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))
+ if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
+ thtail->window = th->window;
+ }
/* We have to update both TCP_SKB_CB(tail)->tcp_flags and
* thtail->fin, so that the fast path in tcp_rcv_established()
{
struct icmpv6_msg *msg = (struct icmpv6_msg *) from;
struct sk_buff *org_skb = msg->skb;
- __wsum csum = 0;
+ __wsum csum;
csum = skb_copy_and_csum_bits(org_skb, msg->offset + offset,
- to, len, csum);
+ to, len);
skb->csum = csum_block_add(skb->csum, csum, odd);
if (!(msg->type & ICMPV6_INFOMSG_MASK))
nf_ct_attach(skb, org_skb);
if (fraggap) {
skb->csum = skb_copy_and_csum_bits(
skb_prev, maxfraglen,
- data + transhdrlen, fraggap, 0);
+ data + transhdrlen, fraggap);
skb_prev->csum = csum_sub(skb_prev->csum,
skb->csum);
data += fraggap;
skb->csum = csum_block_add(
skb->csum,
csum_partial_copy_nocheck(rfv->c + offset,
- to, copy, 0),
+ to, copy),
odd);
odd = 0;
static void mptcp_write_data_fin(struct mptcp_subflow_context *subflow,
struct sk_buff *skb, struct mptcp_ext *ext)
{
- u64 data_fin_tx_seq = READ_ONCE(mptcp_sk(subflow->conn)->write_seq);
+ /* The write_seq value has already been incremented, so the actual
+ * sequence number for the DATA_FIN is one less.
+ */
+ u64 data_fin_tx_seq = READ_ONCE(mptcp_sk(subflow->conn)->write_seq) - 1;
if (!ext->use_map || !skb->len) {
/* RFC6824 requires a DSS mapping with specific values
ext->data_fin = 1;
ext->use_map = 1;
ext->dsn64 = 1;
- /* The write_seq value has already been incremented, so
- * the actual sequence number for the DATA_FIN is one less.
- */
- ext->data_seq = data_fin_tx_seq - 1;
+ ext->data_seq = data_fin_tx_seq;
ext->subflow_seq = 0;
ext->data_len = 1;
} else if (ext->data_seq + ext->data_len == data_fin_tx_seq) {
if (subflow->use_64bit_ack) {
ack_size = TCPOLEN_MPTCP_DSS_ACK64;
- opts->ext_copy.data_ack = msk->ack_seq;
+ opts->ext_copy.data_ack = READ_ONCE(msk->ack_seq);
opts->ext_copy.ack64 = 1;
} else {
ack_size = TCPOLEN_MPTCP_DSS_ACK32;
- opts->ext_copy.data_ack32 = (uint32_t)(msk->ack_seq);
+ opts->ext_copy.data_ack32 = (uint32_t)READ_ONCE(msk->ack_seq);
opts->ext_copy.ack64 = 0;
}
opts->ext_copy.use_ack = 1;
}
}
-bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq)
+bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq, bool use_64bit)
{
/* Skip if DATA_FIN was already received.
* If updating simultaneously with the recvmsg loop, values
if (READ_ONCE(msk->rcv_data_fin) || !READ_ONCE(msk->first))
return false;
- WRITE_ONCE(msk->rcv_data_fin_seq, data_fin_seq);
+ WRITE_ONCE(msk->rcv_data_fin_seq,
+ expand_ack(READ_ONCE(msk->ack_seq), data_fin_seq, use_64bit));
WRITE_ONCE(msk->rcv_data_fin, 1);
return true;
*/
if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
if (mp_opt.data_fin && mp_opt.data_len == 1 &&
- mptcp_update_rcv_data_fin(msk, mp_opt.data_seq) &&
+ mptcp_update_rcv_data_fin(msk, mp_opt.data_seq, mp_opt.dsn64) &&
schedule_work(&msk->work))
sock_hold(subflow->conn);
skb_ext_reset(skb);
skb_orphan(skb);
- msk->ack_seq += copy_len;
+ WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len);
tail = skb_peek_tail(&sk->sk_receive_queue);
if (offset == 0 && tail) {
if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) {
struct mptcp_subflow_context *subflow;
- msk->ack_seq++;
+ WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1);
WRITE_ONCE(msk->rcv_data_fin, 0);
sk->sk_shutdown |= RCV_SHUTDOWN;
msk->remote_key = mp_opt->sndr_key;
mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq);
ack_seq++;
- msk->ack_seq = ack_seq;
+ WRITE_ONCE(msk->ack_seq, ack_seq);
}
sock_reset_flag(nsk, SOCK_RCU_FREE);
parent_sock = READ_ONCE(parent->sk_socket);
if (parent_sock && !sk->sk_socket)
mptcp_sock_graft(sk, parent_sock);
- subflow->map_seq = msk->ack_seq;
+ subflow->map_seq = READ_ONCE(msk->ack_seq);
return true;
}
bool mptcp_finish_join(struct sock *sk);
void mptcp_data_acked(struct sock *sk);
void mptcp_subflow_eof(struct sock *sk);
-bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq);
+bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq, bool use_64bit);
void __init mptcp_token_init(void);
static inline void mptcp_token_init_request(struct request_sock *req)
if (mpext->data_fin == 1) {
if (data_len == 1) {
- mptcp_update_rcv_data_fin(msk, mpext->data_seq);
+ bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq,
+ mpext->dsn64);
pr_debug("DATA_FIN with no payload seq=%llu", mpext->data_seq);
if (subflow->map_valid) {
/* A DATA_FIN might arrive in a DSS
skb_ext_del(skb, SKB_EXT_MPTCP);
return MAPPING_OK;
} else {
+ if (updated && schedule_work(&msk->work))
+ sock_hold((struct sock *)msk);
+
return MAPPING_DATA_FIN;
}
} else {
- mptcp_update_rcv_data_fin(msk, mpext->data_seq + data_len);
- pr_debug("DATA_FIN with mapping seq=%llu", mpext->data_seq + data_len);
+ u64 data_fin_seq = mpext->data_seq + data_len - 1;
+
+ /* If mpext->data_seq is a 32-bit value, data_fin_seq
+ * must also be limited to 32 bits.
+ */
+ if (!mpext->dsn64)
+ data_fin_seq &= GENMASK_ULL(31, 0);
+
+ mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64);
+ pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d",
+ data_fin_seq, mpext->dsn64);
}
/* Adjust for DATA_FIN using 1 byte of sequence space */
if (err)
return err;
- while (netlink_policy_dump_loop(&cb->args[1])) {
+ while (netlink_policy_dump_loop(cb->args[1])) {
void *hdr;
struct nlattr *nest;
return skb->len;
}
+static int ctrl_dumppolicy_done(struct netlink_callback *cb)
+{
+ netlink_policy_dump_free(cb->args[1]);
+ return 0;
+}
+
static const struct genl_ops genl_ctrl_ops[] = {
{
.cmd = CTRL_CMD_GETFAMILY,
{
.cmd = CTRL_CMD_GETPOLICY,
.dumpit = ctrl_dumppolicy,
+ .done = ctrl_dumppolicy_done,
},
};
unsigned int policy_idx;
int err;
- /* also returns 0 if "*_state" is our ERR_PTR() end marker */
if (*_state)
return 0;
!state->policies[state->policy_idx].policy;
}
-bool netlink_policy_dump_loop(unsigned long *_state)
+bool netlink_policy_dump_loop(unsigned long _state)
{
- struct nl_policy_dump *state = (void *)*_state;
-
- if (IS_ERR(state))
- return false;
-
- if (netlink_policy_dump_finished(state)) {
- kfree(state);
- /* store end marker instead of freed state */
- *_state = (unsigned long)ERR_PTR(-ENOENT);
- return false;
- }
+ struct nl_policy_dump *state = (void *)_state;
- return true;
+ return !netlink_policy_dump_finished(state);
}
int netlink_policy_dump_write(struct sk_buff *skb, unsigned long _state)
nla_nest_cancel(skb, policy);
return -ENOBUFS;
}
+
+void netlink_policy_dump_free(unsigned long _state)
+{
+ struct nl_policy_dump *state = (void *)_state;
+
+ kfree(state);
+}
}
err = ovs_ct_nat_execute(skb, ct, ctinfo, &info->range, maniptype);
- if (err == NF_ACCEPT &&
- ct->status & IPS_SRC_NAT && ct->status & IPS_DST_NAT) {
- if (maniptype == NF_NAT_MANIP_SRC)
- maniptype = NF_NAT_MANIP_DST;
- else
- maniptype = NF_NAT_MANIP_SRC;
-
- err = ovs_ct_nat_execute(skb, ct, ctinfo, &info->range,
- maniptype);
+ if (err == NF_ACCEPT && ct->status & IPS_DST_NAT) {
+ if (ct->status & IPS_SRC_NAT) {
+ if (maniptype == NF_NAT_MANIP_SRC)
+ maniptype = NF_NAT_MANIP_DST;
+ else
+ maniptype = NF_NAT_MANIP_SRC;
+
+ err = ovs_ct_nat_execute(skb, ct, ctinfo, &info->range,
+ maniptype);
+ } else if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL) {
+ err = ovs_ct_nat_execute(skb, ct, ctinfo, NULL,
+ NF_NAT_MANIP_SRC);
+ }
}
/* Mark NAT done if successful and update the flow key. */
if (!node)
return 0;
+ rcu_read_lock();
/* Announce the list of servers registered in this node */
radix_tree_for_each_slot(slot, &node->servers, &iter, 0) {
srv = radix_tree_deref_slot(slot);
+ if (!srv)
+ continue;
+ if (radix_tree_deref_retry(srv)) {
+ slot = radix_tree_iter_retry(&iter);
+ continue;
+ }
+ slot = radix_tree_iter_resume(slot, &iter);
+ rcu_read_unlock();
ret = service_announce_new(sq, srv);
if (ret < 0) {
pr_err("failed to announce new service\n");
return ret;
}
+
+ rcu_read_lock();
}
+ rcu_read_unlock();
+
return 0;
}
if (!node)
return 0;
+ rcu_read_lock();
/* Advertise removal of this client to all servers of remote node */
radix_tree_for_each_slot(slot, &node->servers, &iter, 0) {
srv = radix_tree_deref_slot(slot);
+ if (!srv)
+ continue;
+ if (radix_tree_deref_retry(srv)) {
+ slot = radix_tree_iter_retry(&iter);
+ continue;
+ }
+ slot = radix_tree_iter_resume(slot, &iter);
+ rcu_read_unlock();
server_del(node, srv->port);
+ rcu_read_lock();
}
+ rcu_read_unlock();
/* Advertise the removal of this client to all local servers */
local_node = node_get(qrtr_ns.local_node);
pkt.cmd = cpu_to_le32(QRTR_TYPE_BYE);
pkt.client.node = cpu_to_le32(from->sq_node);
+ rcu_read_lock();
radix_tree_for_each_slot(slot, &local_node->servers, &iter, 0) {
srv = radix_tree_deref_slot(slot);
+ if (!srv)
+ continue;
+ if (radix_tree_deref_retry(srv)) {
+ slot = radix_tree_iter_retry(&iter);
+ continue;
+ }
+ slot = radix_tree_iter_resume(slot, &iter);
+ rcu_read_unlock();
sq.sq_family = AF_QIPCRTR;
sq.sq_node = srv->node;
pr_err("failed to send bye cmd\n");
return ret;
}
+ rcu_read_lock();
}
+ rcu_read_unlock();
+
return 0;
}
pkt.client.node = cpu_to_le32(node_id);
pkt.client.port = cpu_to_le32(port);
+ rcu_read_lock();
radix_tree_for_each_slot(slot, &local_node->servers, &iter, 0) {
srv = radix_tree_deref_slot(slot);
+ if (!srv)
+ continue;
+ if (radix_tree_deref_retry(srv)) {
+ slot = radix_tree_iter_retry(&iter);
+ continue;
+ }
+ slot = radix_tree_iter_resume(slot, &iter);
+ rcu_read_unlock();
sq.sq_family = AF_QIPCRTR;
sq.sq_node = srv->node;
pr_err("failed to send del client cmd\n");
return ret;
}
+ rcu_read_lock();
}
+ rcu_read_unlock();
+
return 0;
}
filter.service = service;
filter.instance = instance;
+ rcu_read_lock();
radix_tree_for_each_slot(node_slot, &nodes, &node_iter, 0) {
node = radix_tree_deref_slot(node_slot);
+ if (!node)
+ continue;
+ if (radix_tree_deref_retry(node)) {
+ node_slot = radix_tree_iter_retry(&node_iter);
+ continue;
+ }
+ node_slot = radix_tree_iter_resume(node_slot, &node_iter);
radix_tree_for_each_slot(srv_slot, &node->servers,
&srv_iter, 0) {
struct qrtr_server *srv;
srv = radix_tree_deref_slot(srv_slot);
+ if (!srv)
+ continue;
+ if (radix_tree_deref_retry(srv)) {
+ srv_slot = radix_tree_iter_retry(&srv_iter);
+ continue;
+ }
+
if (!server_match(srv, &filter))
continue;
+ srv_slot = radix_tree_iter_resume(srv_slot, &srv_iter);
+
+ rcu_read_unlock();
lookup_notify(from, srv, true);
+ rcu_read_lock();
}
}
+ rcu_read_unlock();
/* Empty notification, to indicate end of listing */
lookup_notify(from, NULL, true);
RXRPC_CALL_CLIENT_RECV_REPLY, /* - client receiving reply phase */
RXRPC_CALL_SERVER_PREALLOC, /* - service preallocation */
RXRPC_CALL_SERVER_SECURING, /* - server securing request connection */
- RXRPC_CALL_SERVER_ACCEPTING, /* - server accepting request */
RXRPC_CALL_SERVER_RECV_REQUEST, /* - server receiving request */
RXRPC_CALL_SERVER_ACK_REQUEST, /* - server pending ACK of request */
RXRPC_CALL_SERVER_SEND_REPLY, /* - server sending reply */
enum rxrpc_command {
RXRPC_CMD_SEND_DATA, /* send data message */
RXRPC_CMD_SEND_ABORT, /* request abort generation */
- RXRPC_CMD_ACCEPT, /* [server] accept incoming call */
RXRPC_CMD_REJECT_BUSY, /* [server] reject a call as busy */
+ RXRPC_CMD_CHARGE_ACCEPT, /* [server] charge accept preallocation */
};
struct rxrpc_call_params {
struct rxrpc_sock *,
struct sk_buff *);
void rxrpc_accept_incoming_calls(struct rxrpc_local *);
-struct rxrpc_call *rxrpc_accept_call(struct rxrpc_sock *, unsigned long,
- rxrpc_notify_rx_t);
-int rxrpc_reject_call(struct rxrpc_sock *);
+int rxrpc_user_charge_accept(struct rxrpc_sock *, unsigned long);
/*
* call_event.c
unsigned int debug_id)
{
const void *here = __builtin_return_address(0);
- struct rxrpc_call *call;
+ struct rxrpc_call *call, *xcall;
struct rxrpc_net *rxnet = rxrpc_net(sock_net(&rx->sk));
+ struct rb_node *parent, **pp;
int max, tmp;
unsigned int size = RXRPC_BACKLOG_MAX;
unsigned int head, tail, call_head, call_tail;
}
/* Now it gets complicated, because calls get registered with the
- * socket here, particularly if a user ID is preassigned by the user.
+ * socket here, with a user ID preassigned by the user.
*/
call = rxrpc_alloc_call(rx, gfp, debug_id);
if (!call)
here, (const void *)user_call_ID);
write_lock(&rx->call_lock);
- if (user_attach_call) {
- struct rxrpc_call *xcall;
- struct rb_node *parent, **pp;
-
- /* Check the user ID isn't already in use */
- pp = &rx->calls.rb_node;
- parent = NULL;
- while (*pp) {
- parent = *pp;
- xcall = rb_entry(parent, struct rxrpc_call, sock_node);
- if (user_call_ID < xcall->user_call_ID)
- pp = &(*pp)->rb_left;
- else if (user_call_ID > xcall->user_call_ID)
- pp = &(*pp)->rb_right;
- else
- goto id_in_use;
- }
- call->user_call_ID = user_call_ID;
- call->notify_rx = notify_rx;
+ /* Check the user ID isn't already in use */
+ pp = &rx->calls.rb_node;
+ parent = NULL;
+ while (*pp) {
+ parent = *pp;
+ xcall = rb_entry(parent, struct rxrpc_call, sock_node);
+ if (user_call_ID < xcall->user_call_ID)
+ pp = &(*pp)->rb_left;
+ else if (user_call_ID > xcall->user_call_ID)
+ pp = &(*pp)->rb_right;
+ else
+ goto id_in_use;
+ }
+
+ call->user_call_ID = user_call_ID;
+ call->notify_rx = notify_rx;
+ if (user_attach_call) {
rxrpc_get_call(call, rxrpc_call_got_kernel);
user_attach_call(call, user_call_ID);
- rxrpc_get_call(call, rxrpc_call_got_userid);
- rb_link_node(&call->sock_node, parent, pp);
- rb_insert_color(&call->sock_node, &rx->calls);
- set_bit(RXRPC_CALL_HAS_USERID, &call->flags);
}
+ rxrpc_get_call(call, rxrpc_call_got_userid);
+ rb_link_node(&call->sock_node, parent, pp);
+ rb_insert_color(&call->sock_node, &rx->calls);
+ set_bit(RXRPC_CALL_HAS_USERID, &call->flags);
+
list_add(&call->sock_link, &rx->sock_calls);
write_unlock(&rx->call_lock);
}
/*
- * Preallocate sufficient service connections, calls and peers to cover the
- * entire backlog of a socket. When a new call comes in, if we don't have
- * sufficient of each available, the call gets rejected as busy or ignored.
- *
- * The backlog is replenished when a connection is accepted or rejected.
+ * Allocate the preallocation buffers for incoming service calls. These must
+ * be charged manually.
*/
int rxrpc_service_prealloc(struct rxrpc_sock *rx, gfp_t gfp)
{
rx->backlog = b;
}
- if (rx->discard_new_call)
- return 0;
-
- while (rxrpc_service_prealloc_one(rx, b, NULL, NULL, 0, gfp,
- atomic_inc_return(&rxrpc_debug_id)) == 0)
- ;
-
return 0;
}
rxrpc_see_call(call);
call->conn = conn;
call->security = conn->security;
+ call->security_ix = conn->security_ix;
call->peer = rxrpc_get_peer(conn->params.peer);
call->cong_cwnd = call->peer->cong_cwnd;
return call;
if (rx->notify_new_call)
rx->notify_new_call(&rx->sk, call, call->user_call_ID);
- else
- sk_acceptq_added(&rx->sk);
spin_lock(&conn->state_lock);
switch (conn->state) {
case RXRPC_CONN_SERVICE:
write_lock(&call->state_lock);
- if (call->state < RXRPC_CALL_COMPLETE) {
- if (rx->discard_new_call)
- call->state = RXRPC_CALL_SERVER_RECV_REQUEST;
- else
- call->state = RXRPC_CALL_SERVER_ACCEPTING;
- }
+ if (call->state < RXRPC_CALL_COMPLETE)
+ call->state = RXRPC_CALL_SERVER_RECV_REQUEST;
write_unlock(&call->state_lock);
break;
rxrpc_send_ping(call, skb);
- if (call->state == RXRPC_CALL_SERVER_ACCEPTING)
- rxrpc_notify_socket(call);
-
/* We have to discard the prealloc queue's ref here and rely on a
* combination of the RCU read lock and refs held either by the socket
* (recvmsg queue, to-be-accepted queue or user ID tree) or the kernel
}
/*
- * handle acceptance of a call by userspace
- * - assign the user call ID to the call at the front of the queue
- * - called with the socket locked.
+ * Charge up socket with preallocated calls, attaching user call IDs.
*/
-struct rxrpc_call *rxrpc_accept_call(struct rxrpc_sock *rx,
- unsigned long user_call_ID,
- rxrpc_notify_rx_t notify_rx)
- __releases(&rx->sk.sk_lock.slock)
- __acquires(call->user_mutex)
+int rxrpc_user_charge_accept(struct rxrpc_sock *rx, unsigned long user_call_ID)
{
- struct rxrpc_call *call;
- struct rb_node *parent, **pp;
- int ret;
-
- _enter(",%lx", user_call_ID);
-
- ASSERT(!irqs_disabled());
-
- write_lock(&rx->call_lock);
-
- if (list_empty(&rx->to_be_accepted)) {
- write_unlock(&rx->call_lock);
- release_sock(&rx->sk);
- kleave(" = -ENODATA [empty]");
- return ERR_PTR(-ENODATA);
- }
-
- /* check the user ID isn't already in use */
- pp = &rx->calls.rb_node;
- parent = NULL;
- while (*pp) {
- parent = *pp;
- call = rb_entry(parent, struct rxrpc_call, sock_node);
-
- if (user_call_ID < call->user_call_ID)
- pp = &(*pp)->rb_left;
- else if (user_call_ID > call->user_call_ID)
- pp = &(*pp)->rb_right;
- else
- goto id_in_use;
- }
-
- /* Dequeue the first call and check it's still valid. We gain
- * responsibility for the queue's reference.
- */
- call = list_entry(rx->to_be_accepted.next,
- struct rxrpc_call, accept_link);
- write_unlock(&rx->call_lock);
-
- /* We need to gain the mutex from the interrupt handler without
- * upsetting lockdep, so we have to release it there and take it here.
- * We are, however, still holding the socket lock, so other accepts
- * must wait for us and no one can add the user ID behind our backs.
- */
- if (mutex_lock_interruptible(&call->user_mutex) < 0) {
- release_sock(&rx->sk);
- kleave(" = -ERESTARTSYS");
- return ERR_PTR(-ERESTARTSYS);
- }
-
- write_lock(&rx->call_lock);
- list_del_init(&call->accept_link);
- sk_acceptq_removed(&rx->sk);
- rxrpc_see_call(call);
-
- /* Find the user ID insertion point. */
- pp = &rx->calls.rb_node;
- parent = NULL;
- while (*pp) {
- parent = *pp;
- call = rb_entry(parent, struct rxrpc_call, sock_node);
-
- if (user_call_ID < call->user_call_ID)
- pp = &(*pp)->rb_left;
- else if (user_call_ID > call->user_call_ID)
- pp = &(*pp)->rb_right;
- else
- BUG();
- }
-
- write_lock_bh(&call->state_lock);
- switch (call->state) {
- case RXRPC_CALL_SERVER_ACCEPTING:
- call->state = RXRPC_CALL_SERVER_RECV_REQUEST;
- break;
- case RXRPC_CALL_COMPLETE:
- ret = call->error;
- goto out_release;
- default:
- BUG();
- }
-
- /* formalise the acceptance */
- call->notify_rx = notify_rx;
- call->user_call_ID = user_call_ID;
- rxrpc_get_call(call, rxrpc_call_got_userid);
- rb_link_node(&call->sock_node, parent, pp);
- rb_insert_color(&call->sock_node, &rx->calls);
- if (test_and_set_bit(RXRPC_CALL_HAS_USERID, &call->flags))
- BUG();
-
- write_unlock_bh(&call->state_lock);
- write_unlock(&rx->call_lock);
- rxrpc_notify_socket(call);
- rxrpc_service_prealloc(rx, GFP_KERNEL);
- release_sock(&rx->sk);
- _leave(" = %p{%d}", call, call->debug_id);
- return call;
-
-out_release:
- _debug("release %p", call);
- write_unlock_bh(&call->state_lock);
- write_unlock(&rx->call_lock);
- rxrpc_release_call(rx, call);
- rxrpc_put_call(call, rxrpc_call_put);
- goto out;
-
-id_in_use:
- ret = -EBADSLT;
- write_unlock(&rx->call_lock);
-out:
- rxrpc_service_prealloc(rx, GFP_KERNEL);
- release_sock(&rx->sk);
- _leave(" = %d", ret);
- return ERR_PTR(ret);
-}
-
-/*
- * Handle rejection of a call by userspace
- * - reject the call at the front of the queue
- */
-int rxrpc_reject_call(struct rxrpc_sock *rx)
-{
- struct rxrpc_call *call;
- bool abort = false;
- int ret;
-
- _enter("");
-
- ASSERT(!irqs_disabled());
-
- write_lock(&rx->call_lock);
-
- if (list_empty(&rx->to_be_accepted)) {
- write_unlock(&rx->call_lock);
- return -ENODATA;
- }
-
- /* Dequeue the first call and check it's still valid. We gain
- * responsibility for the queue's reference.
- */
- call = list_entry(rx->to_be_accepted.next,
- struct rxrpc_call, accept_link);
- list_del_init(&call->accept_link);
- sk_acceptq_removed(&rx->sk);
- rxrpc_see_call(call);
+ struct rxrpc_backlog *b = rx->backlog;
- write_lock_bh(&call->state_lock);
- switch (call->state) {
- case RXRPC_CALL_SERVER_ACCEPTING:
- __rxrpc_abort_call("REJ", call, 1, RX_USER_ABORT, -ECONNABORTED);
- abort = true;
- fallthrough;
- case RXRPC_CALL_COMPLETE:
- ret = call->error;
- goto out_discard;
- default:
- BUG();
- }
+ if (rx->sk.sk_state == RXRPC_CLOSE)
+ return -ESHUTDOWN;
-out_discard:
- write_unlock_bh(&call->state_lock);
- write_unlock(&rx->call_lock);
- if (abort) {
- rxrpc_send_abort_packet(call);
- rxrpc_release_call(rx, call);
- rxrpc_put_call(call, rxrpc_call_put);
- }
- rxrpc_service_prealloc(rx, GFP_KERNEL);
- _leave(" = %d", ret);
- return ret;
+ return rxrpc_service_prealloc_one(rx, b, NULL, NULL, user_call_ID,
+ GFP_KERNEL,
+ atomic_inc_return(&rxrpc_debug_id));
}
/*
[RXRPC_CALL_CLIENT_RECV_REPLY] = "ClRcvRpl",
[RXRPC_CALL_SERVER_PREALLOC] = "SvPrealc",
[RXRPC_CALL_SERVER_SECURING] = "SvSecure",
- [RXRPC_CALL_SERVER_ACCEPTING] = "SvAccept",
[RXRPC_CALL_SERVER_RECV_REQUEST] = "SvRcvReq",
[RXRPC_CALL_SERVER_ACK_REQUEST] = "SvAckReq",
[RXRPC_CALL_SERVER_SEND_REPLY] = "SvSndRpl",
call->call_id = sp->hdr.callNumber;
call->service_id = sp->hdr.serviceId;
call->cid = sp->hdr.cid;
- call->state = RXRPC_CALL_SERVER_ACCEPTING;
- if (sp->hdr.securityIndex > 0)
- call->state = RXRPC_CALL_SERVER_SECURING;
+ call->state = RXRPC_CALL_SERVER_SECURING;
call->cong_tstamp = skb->tstamp;
/* Set the channel for this call. We don't get channel_lock as we're
if (call) {
write_lock_bh(&call->state_lock);
if (call->state == RXRPC_CALL_SERVER_SECURING) {
- call->state = RXRPC_CALL_SERVER_ACCEPTING;
+ call->state = RXRPC_CALL_SERVER_RECV_REQUEST;
rxrpc_notify_socket(call);
}
write_unlock_bh(&call->state_lock);
return ret;
spin_lock(&conn->channel_lock);
- spin_lock(&conn->state_lock);
+ spin_lock_bh(&conn->state_lock);
if (conn->state == RXRPC_CONN_SERVICE_CHALLENGING) {
conn->state = RXRPC_CONN_SERVICE;
- spin_unlock(&conn->state_lock);
+ spin_unlock_bh(&conn->state_lock);
for (loop = 0; loop < RXRPC_MAXCALLS; loop++)
rxrpc_call_is_secure(
rcu_dereference_protected(
conn->channels[loop].call,
lockdep_is_held(&conn->channel_lock)));
} else {
- spin_unlock(&conn->state_lock);
+ spin_unlock_bh(&conn->state_lock);
}
spin_unlock(&conn->channel_lock);
_enter("");
- if (optlen <= 0 || optlen > PAGE_SIZE - 1)
+ if (optlen <= 0 || optlen > PAGE_SIZE - 1 || rx->securities)
return -EINVAL;
description = memdup_sockptr_nul(optval, optlen);
if (IS_ERR(description))
return PTR_ERR(description);
- key = request_key_net(&key_type_keyring, description, sock_net(&rx->sk), NULL);
+ key = request_key(&key_type_keyring, description, NULL);
if (IS_ERR(key)) {
kfree(description);
_leave(" = %ld", PTR_ERR(key));
switch (token->security_index) {
case RXRPC_SECURITY_RXKAD:
- toksize += 9 * 4; /* viceid, kvno, key*2 + len, begin,
+ toksize += 8 * 4; /* viceid, kvno, key*2, begin,
* end, primary, tktlen */
toksize += RND(token->kad->ticket_len);
break;
break;
default: /* we have a ticket we can't encode */
- BUG();
+ pr_err("Unsupported key token type (%u)\n",
+ token->security_index);
continue;
}
memcpy((u8 *)xdr + _l, &zero, 4 - (_l & 3)); \
xdr += (_l + 3) >> 2; \
} while(0)
+#define ENCODE_BYTES(l, s) \
+ do { \
+ u32 _l = (l); \
+ memcpy(xdr, (s), _l); \
+ if (_l & 3) \
+ memcpy((u8 *)xdr + _l, &zero, 4 - (_l & 3)); \
+ xdr += (_l + 3) >> 2; \
+ } while(0)
#define ENCODE64(x) \
do { \
__be64 y = cpu_to_be64(x); \
case RXRPC_SECURITY_RXKAD:
ENCODE(token->kad->vice_id);
ENCODE(token->kad->kvno);
- ENCODE_DATA(8, token->kad->session_key);
+ ENCODE_BYTES(8, token->kad->session_key);
ENCODE(token->kad->start);
ENCODE(token->kad->expiry);
ENCODE(token->kad->primary_flag);
break;
default:
- BUG();
break;
}
return ret;
}
-/*
- * Pass back notification of a new call. The call is added to the
- * to-be-accepted list. This means that the next call to be accepted might not
- * be the last call seen awaiting acceptance, but unless we leave this on the
- * front of the queue and block all other messages until someone gives us a
- * user_ID for it, there's not a lot we can do.
- */
-static int rxrpc_recvmsg_new_call(struct rxrpc_sock *rx,
- struct rxrpc_call *call,
- struct msghdr *msg, int flags)
-{
- int tmp = 0, ret;
-
- ret = put_cmsg(msg, SOL_RXRPC, RXRPC_NEW_CALL, 0, &tmp);
-
- if (ret == 0 && !(flags & MSG_PEEK)) {
- _debug("to be accepted");
- write_lock_bh(&rx->recvmsg_lock);
- list_del_init(&call->recvmsg_link);
- write_unlock_bh(&rx->recvmsg_lock);
-
- rxrpc_get_call(call, rxrpc_call_got);
- write_lock(&rx->call_lock);
- list_add_tail(&call->accept_link, &rx->to_be_accepted);
- write_unlock(&rx->call_lock);
- }
-
- trace_rxrpc_recvmsg(call, rxrpc_recvmsg_to_be_accepted, 1, 0, 0, ret);
- return ret;
-}
-
/*
* End the packet reception phase.
*/
}
switch (READ_ONCE(call->state)) {
- case RXRPC_CALL_SERVER_ACCEPTING:
- ret = rxrpc_recvmsg_new_call(rx, call, msg, flags);
- break;
case RXRPC_CALL_CLIENT_RECV_REPLY:
case RXRPC_CALL_SERVER_RECV_REQUEST:
case RXRPC_CALL_SERVER_ACK_REQUEST:
call->debug_id, rxrpc_call_states[call->state],
iov_iter_count(iter), want_more);
- ASSERTCMP(call->state, !=, RXRPC_CALL_SERVER_ACCEPTING);
+ ASSERTCMP(call->state, !=, RXRPC_CALL_SERVER_SECURING);
mutex_lock(&call->user_mutex);
return -EINVAL;
break;
- case RXRPC_ACCEPT:
+ case RXRPC_CHARGE_ACCEPT:
if (p->command != RXRPC_CMD_SEND_DATA)
return -EINVAL;
- p->command = RXRPC_CMD_ACCEPT;
+ p->command = RXRPC_CMD_CHARGE_ACCEPT;
if (len != 0)
return -EINVAL;
break;
if (ret < 0)
goto error_release_sock;
- if (p.command == RXRPC_CMD_ACCEPT) {
+ if (p.command == RXRPC_CMD_CHARGE_ACCEPT) {
ret = -EINVAL;
if (rx->sk.sk_state != RXRPC_SERVER_LISTENING)
goto error_release_sock;
- call = rxrpc_accept_call(rx, p.call.user_call_ID, NULL);
- /* The socket is now unlocked. */
- if (IS_ERR(call))
- return PTR_ERR(call);
- ret = 0;
- goto out_put_unlock;
+ ret = rxrpc_user_charge_accept(rx, p.call.user_call_ID);
+ goto error_release_sock;
}
call = rxrpc_find_call_by_user_ID(rx, p.call.user_call_ID);
case RXRPC_CALL_CLIENT_AWAIT_CONN:
case RXRPC_CALL_SERVER_PREALLOC:
case RXRPC_CALL_SERVER_SECURING:
- case RXRPC_CALL_SERVER_ACCEPTING:
rxrpc_put_call(call, rxrpc_call_put);
ret = -EBUSY;
goto error_release_sock;
index++;
if (index < s_i)
continue;
+ if (IS_ERR(p))
+ continue;
if (jiffy_since &&
time_after(jiffy_since,
mutex_lock(&idrinfo->lock);
idr_for_each_entry_ul(idr, p, tmp, id) {
+ if (IS_ERR(p))
+ continue;
ret = tcf_idr_release_unsafe(p);
if (ret == ACT_P_DELETED) {
module_put(ops->owner);
}
EXPORT_SYMBOL(tcf_idr_create_from_flags);
-void tcf_idr_insert(struct tc_action_net *tn, struct tc_action *a)
-{
- struct tcf_idrinfo *idrinfo = tn->idrinfo;
-
- mutex_lock(&idrinfo->lock);
- /* Replace ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */
- WARN_ON(!IS_ERR(idr_replace(&idrinfo->action_idr, a, a->tcfa_index)));
- mutex_unlock(&idrinfo->lock);
-}
-EXPORT_SYMBOL(tcf_idr_insert);
-
/* Cleanup idr index that was allocated but not initialized. */
void tcf_idr_cleanup(struct tc_action_net *tn, u32 index)
return ret;
}
-static int tcf_action_destroy_1(struct tc_action *a, int bind)
-{
- struct tc_action *actions[] = { a, NULL };
-
- return tcf_action_destroy(actions, bind);
-}
-
static int tcf_action_put(struct tc_action *p)
{
return __tcf_action_put(p, false);
[TCA_ACT_HW_STATS] = NLA_POLICY_BITFIELD32(TCA_ACT_HW_STATS_ANY),
};
+static void tcf_idr_insert_many(struct tc_action *actions[])
+{
+ int i;
+
+ for (i = 0; i < TCA_ACT_MAX_PRIO; i++) {
+ struct tc_action *a = actions[i];
+ struct tcf_idrinfo *idrinfo;
+
+ if (!a)
+ continue;
+ idrinfo = a->idrinfo;
+ mutex_lock(&idrinfo->lock);
+ /* Replace ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc if
+ * it is just created, otherwise this is just a nop.
+ */
+ idr_replace(&idrinfo->action_idr, a, a->tcfa_index);
+ mutex_unlock(&idrinfo->lock);
+ }
+}
+
struct tc_action *tcf_action_init_1(struct net *net, struct tcf_proto *tp,
struct nlattr *nla, struct nlattr *est,
char *name, int ovr, int bind,
if (err != ACT_P_CREATED)
module_put(a_o->owner);
- if (TC_ACT_EXT_CMP(a->tcfa_action, TC_ACT_GOTO_CHAIN) &&
- !rcu_access_pointer(a->goto_chain)) {
- tcf_action_destroy_1(a, bind);
- NL_SET_ERR_MSG(extack, "can't use goto chain with NULL chain");
- return ERR_PTR(-EINVAL);
- }
-
return a;
err_mod:
actions[i - 1] = act;
}
+ /* We have to commit them all together, because if any error happened in
+ * between, we could not handle the failure gracefully.
+ */
+ tcf_idr_insert_many(actions);
+
*attr_size = tcf_action_full_attrs_size(sz);
return i - 1;
if (goto_ch)
tcf_chain_put_by_act(goto_ch);
- if (res == ACT_P_CREATED) {
- tcf_idr_insert(tn, *act);
- } else {
+ if (res != ACT_P_CREATED) {
/* make sure the program being replaced is no longer executing */
synchronize_rcu();
tcf_bpf_cfg_cleanup(&old);
ci->net = net;
ci->zone = parm->zone;
- tcf_idr_insert(tn, *a);
ret = ACT_P_CREATED;
} else if (ret > 0) {
ci = to_connmark(*a);
if (params_new)
kfree_rcu(params_new, rcu);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
-
return ret;
put_chain:
if (goto_ch)
tcf_chain_put_by_act(goto_ch);
if (params)
call_rcu(¶ms->rcu, tcf_ct_params_free);
- if (res == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
return res;
if (cp_new)
kfree_rcu(cp_new, rcu);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
-
return ret;
put_chain:
if (goto_ch)
tcf_chain_put_by_act(goto_ch);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
return ret;
release_idr:
tcf_idr_release(*a, bind);
if (goto_ch)
tcf_chain_put_by_act(goto_ch);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
-
return ret;
chain_put:
if (p)
kfree_rcu(p, rcu);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
-
return ret;
metadata_parse_err:
if (goto_ch)
ipt->tcfi_t = t;
ipt->tcfi_hook = hook;
spin_unlock_bh(&ipt->tcf_lock);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
return ret;
err3:
spin_lock(&mirred_list_lock);
list_add(&m->tcfm_list, &mirred_list);
spin_unlock(&mirred_list_lock);
-
- tcf_idr_insert(tn, *a);
}
return ret;
if (p)
kfree_rcu(p, rcu);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
return ret;
put_chain:
if (goto_ch)
if (goto_ch)
tcf_chain_put_by_act(goto_ch);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
-
return ret;
release_idr:
tcf_idr_release(*a, bind);
spin_unlock_bh(&p->tcf_lock);
if (goto_ch)
tcf_chain_put_by_act(goto_ch);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
return ret;
put_chain:
if (new)
kfree_rcu(new, rcu);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
return ret;
failure:
if (goto_ch)
tcf_chain_put_by_act(goto_ch);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
return ret;
put_chain:
if (goto_ch)
goto release_idr;
}
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
return ret;
put_chain:
if (goto_ch)
if (goto_ch)
tcf_chain_put_by_act(goto_ch);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
return ret;
put_chain:
if (goto_ch)
if (goto_ch)
tcf_chain_put_by_act(goto_ch);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
return ret;
put_chain:
if (goto_ch)
if (goto_ch)
tcf_chain_put_by_act(goto_ch);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
-
return ret;
put_chain:
if (p)
kfree_rcu(p, rcu);
- if (ret == ACT_P_CREATED)
- tcf_idr_insert(tn, *a);
return ret;
put_chain:
if (goto_ch)
out_err:
/* Clean up any successful allocations */
sctp_auth_destroy_hmacs(ep->auth_hmacs);
+ ep->auth_hmacs = NULL;
return -ENOMEM;
}
int kernel_sendpage(struct socket *sock, struct page *page, int offset,
size_t size, int flags)
{
- if (sock->ops->sendpage)
+ if (sock->ops->sendpage) {
+ /* Warn in case the improper page to zero-copy send */
+ WARN_ONCE(!sendpage_ok(page), "improper page for zero-copy send");
return sock->ops->sendpage(sock, page, offset, size, flags);
-
+ }
return sock_no_sendpage(sock, page, offset, size, flags);
}
EXPORT_SYMBOL(kernel_sendpage);
depends on SUNRPC && CRYPTO
depends on CRYPTO_MD5 && CRYPTO_DES && CRYPTO_CBC && CRYPTO_CTS
depends on CRYPTO_ECB && CRYPTO_HMAC && CRYPTO_SHA1 && CRYPTO_AES
- depends on CRYPTO_ARC4
default y
select SUNRPC_GSS
help
return crypto_ahash_update(req);
}
-static int
-arcfour_hmac_md5_usage_to_salt(unsigned int usage, u8 salt[4])
-{
- unsigned int ms_usage;
-
- switch (usage) {
- case KG_USAGE_SIGN:
- ms_usage = 15;
- break;
- case KG_USAGE_SEAL:
- ms_usage = 13;
- break;
- default:
- return -EINVAL;
- }
- salt[0] = (ms_usage >> 0) & 0xff;
- salt[1] = (ms_usage >> 8) & 0xff;
- salt[2] = (ms_usage >> 16) & 0xff;
- salt[3] = (ms_usage >> 24) & 0xff;
-
- return 0;
-}
-
-static u32
-make_checksum_hmac_md5(struct krb5_ctx *kctx, char *header, int hdrlen,
- struct xdr_buf *body, int body_offset, u8 *cksumkey,
- unsigned int usage, struct xdr_netobj *cksumout)
-{
- struct scatterlist sg[1];
- int err = -1;
- u8 *checksumdata;
- u8 *rc4salt;
- struct crypto_ahash *md5;
- struct crypto_ahash *hmac_md5;
- struct ahash_request *req;
-
- if (cksumkey == NULL)
- return GSS_S_FAILURE;
-
- if (cksumout->len < kctx->gk5e->cksumlength) {
- dprintk("%s: checksum buffer length, %u, too small for %s\n",
- __func__, cksumout->len, kctx->gk5e->name);
- return GSS_S_FAILURE;
- }
-
- rc4salt = kmalloc_array(4, sizeof(*rc4salt), GFP_NOFS);
- if (!rc4salt)
- return GSS_S_FAILURE;
-
- if (arcfour_hmac_md5_usage_to_salt(usage, rc4salt)) {
- dprintk("%s: invalid usage value %u\n", __func__, usage);
- goto out_free_rc4salt;
- }
-
- checksumdata = kmalloc(GSS_KRB5_MAX_CKSUM_LEN, GFP_NOFS);
- if (!checksumdata)
- goto out_free_rc4salt;
-
- md5 = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC);
- if (IS_ERR(md5))
- goto out_free_cksum;
-
- hmac_md5 = crypto_alloc_ahash(kctx->gk5e->cksum_name, 0,
- CRYPTO_ALG_ASYNC);
- if (IS_ERR(hmac_md5))
- goto out_free_md5;
-
- req = ahash_request_alloc(md5, GFP_NOFS);
- if (!req)
- goto out_free_hmac_md5;
-
- ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
-
- err = crypto_ahash_init(req);
- if (err)
- goto out;
- sg_init_one(sg, rc4salt, 4);
- ahash_request_set_crypt(req, sg, NULL, 4);
- err = crypto_ahash_update(req);
- if (err)
- goto out;
-
- sg_init_one(sg, header, hdrlen);
- ahash_request_set_crypt(req, sg, NULL, hdrlen);
- err = crypto_ahash_update(req);
- if (err)
- goto out;
- err = xdr_process_buf(body, body_offset, body->len - body_offset,
- checksummer, req);
- if (err)
- goto out;
- ahash_request_set_crypt(req, NULL, checksumdata, 0);
- err = crypto_ahash_final(req);
- if (err)
- goto out;
-
- ahash_request_free(req);
- req = ahash_request_alloc(hmac_md5, GFP_NOFS);
- if (!req)
- goto out_free_hmac_md5;
-
- ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
-
- err = crypto_ahash_setkey(hmac_md5, cksumkey, kctx->gk5e->keylength);
- if (err)
- goto out;
-
- sg_init_one(sg, checksumdata, crypto_ahash_digestsize(md5));
- ahash_request_set_crypt(req, sg, checksumdata,
- crypto_ahash_digestsize(md5));
- err = crypto_ahash_digest(req);
- if (err)
- goto out;
-
- memcpy(cksumout->data, checksumdata, kctx->gk5e->cksumlength);
- cksumout->len = kctx->gk5e->cksumlength;
-out:
- ahash_request_free(req);
-out_free_hmac_md5:
- crypto_free_ahash(hmac_md5);
-out_free_md5:
- crypto_free_ahash(md5);
-out_free_cksum:
- kfree(checksumdata);
-out_free_rc4salt:
- kfree(rc4salt);
- return err ? GSS_S_FAILURE : 0;
-}
-
/*
* checksum the plaintext data and hdrlen bytes of the token header
* The checksum is performed over the first 8 bytes of the
u8 *checksumdata;
unsigned int checksumlen;
- if (kctx->gk5e->ctype == CKSUMTYPE_HMAC_MD5_ARCFOUR)
- return make_checksum_hmac_md5(kctx, header, hdrlen,
- body, body_offset,
- cksumkey, usage, cksumout);
-
if (cksumout->len < kctx->gk5e->cksumlength) {
dprintk("%s: checksum buffer length, %u, too small for %s\n",
__func__, cksumout->len, kctx->gk5e->name);
ret = GSS_S_FAILURE;
return ret;
}
-
-/*
- * Compute Kseq given the initial session key and the checksum.
- * Set the key of the given cipher.
- */
-int
-krb5_rc4_setup_seq_key(struct krb5_ctx *kctx,
- struct crypto_sync_skcipher *cipher,
- unsigned char *cksum)
-{
- struct crypto_shash *hmac;
- struct shash_desc *desc;
- u8 Kseq[GSS_KRB5_MAX_KEYLEN];
- u32 zeroconstant = 0;
- int err;
-
- dprintk("%s: entered\n", __func__);
-
- hmac = crypto_alloc_shash(kctx->gk5e->cksum_name, 0, 0);
- if (IS_ERR(hmac)) {
- dprintk("%s: error %ld, allocating hash '%s'\n",
- __func__, PTR_ERR(hmac), kctx->gk5e->cksum_name);
- return PTR_ERR(hmac);
- }
-
- desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(hmac),
- GFP_NOFS);
- if (!desc) {
- dprintk("%s: failed to allocate shash descriptor for '%s'\n",
- __func__, kctx->gk5e->cksum_name);
- crypto_free_shash(hmac);
- return -ENOMEM;
- }
-
- desc->tfm = hmac;
-
- /* Compute intermediate Kseq from session key */
- err = crypto_shash_setkey(hmac, kctx->Ksess, kctx->gk5e->keylength);
- if (err)
- goto out_err;
-
- err = crypto_shash_digest(desc, (u8 *)&zeroconstant, 4, Kseq);
- if (err)
- goto out_err;
-
- /* Compute final Kseq from the checksum and intermediate Kseq */
- err = crypto_shash_setkey(hmac, Kseq, kctx->gk5e->keylength);
- if (err)
- goto out_err;
-
- err = crypto_shash_digest(desc, cksum, 8, Kseq);
- if (err)
- goto out_err;
-
- err = crypto_sync_skcipher_setkey(cipher, Kseq, kctx->gk5e->keylength);
- if (err)
- goto out_err;
-
- err = 0;
-
-out_err:
- kfree_sensitive(desc);
- crypto_free_shash(hmac);
- dprintk("%s: returning %d\n", __func__, err);
- return err;
-}
-
-/*
- * Compute Kcrypt given the initial session key and the plaintext seqnum.
- * Set the key of cipher kctx->enc.
- */
-int
-krb5_rc4_setup_enc_key(struct krb5_ctx *kctx,
- struct crypto_sync_skcipher *cipher,
- s32 seqnum)
-{
- struct crypto_shash *hmac;
- struct shash_desc *desc;
- u8 Kcrypt[GSS_KRB5_MAX_KEYLEN];
- u8 zeroconstant[4] = {0};
- u8 seqnumarray[4];
- int err, i;
-
- dprintk("%s: entered, seqnum %u\n", __func__, seqnum);
-
- hmac = crypto_alloc_shash(kctx->gk5e->cksum_name, 0, 0);
- if (IS_ERR(hmac)) {
- dprintk("%s: error %ld, allocating hash '%s'\n",
- __func__, PTR_ERR(hmac), kctx->gk5e->cksum_name);
- return PTR_ERR(hmac);
- }
-
- desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(hmac),
- GFP_NOFS);
- if (!desc) {
- dprintk("%s: failed to allocate shash descriptor for '%s'\n",
- __func__, kctx->gk5e->cksum_name);
- crypto_free_shash(hmac);
- return -ENOMEM;
- }
-
- desc->tfm = hmac;
-
- /* Compute intermediate Kcrypt from session key */
- for (i = 0; i < kctx->gk5e->keylength; i++)
- Kcrypt[i] = kctx->Ksess[i] ^ 0xf0;
-
- err = crypto_shash_setkey(hmac, Kcrypt, kctx->gk5e->keylength);
- if (err)
- goto out_err;
-
- err = crypto_shash_digest(desc, zeroconstant, 4, Kcrypt);
- if (err)
- goto out_err;
-
- /* Compute final Kcrypt from the seqnum and intermediate Kcrypt */
- err = crypto_shash_setkey(hmac, Kcrypt, kctx->gk5e->keylength);
- if (err)
- goto out_err;
-
- seqnumarray[0] = (unsigned char) ((seqnum >> 24) & 0xff);
- seqnumarray[1] = (unsigned char) ((seqnum >> 16) & 0xff);
- seqnumarray[2] = (unsigned char) ((seqnum >> 8) & 0xff);
- seqnumarray[3] = (unsigned char) ((seqnum >> 0) & 0xff);
-
- err = crypto_shash_digest(desc, seqnumarray, 4, Kcrypt);
- if (err)
- goto out_err;
-
- err = crypto_sync_skcipher_setkey(cipher, Kcrypt,
- kctx->gk5e->keylength);
- if (err)
- goto out_err;
-
- err = 0;
-
-out_err:
- kfree_sensitive(desc);
- crypto_free_shash(hmac);
- dprintk("%s: returning %d\n", __func__, err);
- return err;
-}
.keyed_cksum = 0,
},
#endif /* CONFIG_SUNRPC_DISABLE_INSECURE_ENCTYPES */
- /*
- * RC4-HMAC
- */
- {
- .etype = ENCTYPE_ARCFOUR_HMAC,
- .ctype = CKSUMTYPE_HMAC_MD5_ARCFOUR,
- .name = "rc4-hmac",
- .encrypt_name = "ecb(arc4)",
- .cksum_name = "hmac(md5)",
- .encrypt = krb5_encrypt,
- .decrypt = krb5_decrypt,
- .mk_key = NULL,
- .signalg = SGN_ALG_HMAC_MD5,
- .sealalg = SEAL_ALG_MICROSOFT_RC4,
- .keybytes = 16,
- .keylength = 16,
- .blocksize = 1,
- .conflen = 8,
- .cksumlength = 8,
- .keyed_cksum = 1,
- },
/*
* 3DES
*/
return -EINVAL;
}
-/*
- * Note that RC4 depends on deriving keys using the sequence
- * number or the checksum of a token. Therefore, the final keys
- * cannot be calculated until the token is being constructed!
- */
-static int
-context_derive_keys_rc4(struct krb5_ctx *ctx)
-{
- struct crypto_shash *hmac;
- char sigkeyconstant[] = "signaturekey";
- int slen = strlen(sigkeyconstant) + 1; /* include null terminator */
- struct shash_desc *desc;
- int err;
-
- dprintk("RPC: %s: entered\n", __func__);
- /*
- * derive cksum (aka Ksign) key
- */
- hmac = crypto_alloc_shash(ctx->gk5e->cksum_name, 0, 0);
- if (IS_ERR(hmac)) {
- dprintk("%s: error %ld allocating hash '%s'\n",
- __func__, PTR_ERR(hmac), ctx->gk5e->cksum_name);
- err = PTR_ERR(hmac);
- goto out_err;
- }
-
- err = crypto_shash_setkey(hmac, ctx->Ksess, ctx->gk5e->keylength);
- if (err)
- goto out_err_free_hmac;
-
-
- desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(hmac), GFP_NOFS);
- if (!desc) {
- dprintk("%s: failed to allocate hash descriptor for '%s'\n",
- __func__, ctx->gk5e->cksum_name);
- err = -ENOMEM;
- goto out_err_free_hmac;
- }
-
- desc->tfm = hmac;
-
- err = crypto_shash_digest(desc, sigkeyconstant, slen, ctx->cksum);
- kfree_sensitive(desc);
- if (err)
- goto out_err_free_hmac;
- /*
- * allocate hash, and skciphers for data and seqnum encryption
- */
- ctx->enc = crypto_alloc_sync_skcipher(ctx->gk5e->encrypt_name, 0, 0);
- if (IS_ERR(ctx->enc)) {
- err = PTR_ERR(ctx->enc);
- goto out_err_free_hmac;
- }
-
- ctx->seq = crypto_alloc_sync_skcipher(ctx->gk5e->encrypt_name, 0, 0);
- if (IS_ERR(ctx->seq)) {
- crypto_free_sync_skcipher(ctx->enc);
- err = PTR_ERR(ctx->seq);
- goto out_err_free_hmac;
- }
-
- dprintk("RPC: %s: returning success\n", __func__);
-
- err = 0;
-
-out_err_free_hmac:
- crypto_free_shash(hmac);
-out_err:
- dprintk("RPC: %s: returning %d\n", __func__, err);
- return err;
-}
-
static int
context_derive_keys_new(struct krb5_ctx *ctx, gfp_t gfp_mask)
{
switch (ctx->enctype) {
case ENCTYPE_DES3_CBC_RAW:
return context_derive_keys_des3(ctx, gfp_mask);
- case ENCTYPE_ARCFOUR_HMAC:
- return context_derive_keys_rc4(ctx);
case ENCTYPE_AES128_CTS_HMAC_SHA1_96:
case ENCTYPE_AES256_CTS_HMAC_SHA1_96:
return context_derive_keys_new(ctx, gfp_mask);
BUG();
case ENCTYPE_DES_CBC_RAW:
case ENCTYPE_DES3_CBC_RAW:
- case ENCTYPE_ARCFOUR_HMAC:
return gss_get_mic_v1(ctx, text, token);
case ENCTYPE_AES128_CTS_HMAC_SHA1_96:
case ENCTYPE_AES256_CTS_HMAC_SHA1_96:
# define RPCDBG_FACILITY RPCDBG_AUTH
#endif
-static s32
-krb5_make_rc4_seq_num(struct krb5_ctx *kctx, int direction, s32 seqnum,
- unsigned char *cksum, unsigned char *buf)
-{
- struct crypto_sync_skcipher *cipher;
- unsigned char *plain;
- s32 code;
-
- dprintk("RPC: %s:\n", __func__);
- cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name, 0, 0);
- if (IS_ERR(cipher))
- return PTR_ERR(cipher);
-
- plain = kmalloc(8, GFP_NOFS);
- if (!plain)
- return -ENOMEM;
-
- plain[0] = (unsigned char) ((seqnum >> 24) & 0xff);
- plain[1] = (unsigned char) ((seqnum >> 16) & 0xff);
- plain[2] = (unsigned char) ((seqnum >> 8) & 0xff);
- plain[3] = (unsigned char) ((seqnum >> 0) & 0xff);
- plain[4] = direction;
- plain[5] = direction;
- plain[6] = direction;
- plain[7] = direction;
-
- code = krb5_rc4_setup_seq_key(kctx, cipher, cksum);
- if (code)
- goto out;
-
- code = krb5_encrypt(cipher, cksum, plain, buf, 8);
-out:
- kfree(plain);
- crypto_free_sync_skcipher(cipher);
- return code;
-}
s32
krb5_make_seq_num(struct krb5_ctx *kctx,
struct crypto_sync_skcipher *key,
unsigned char *plain;
s32 code;
- if (kctx->enctype == ENCTYPE_ARCFOUR_HMAC)
- return krb5_make_rc4_seq_num(kctx, direction, seqnum,
- cksum, buf);
-
plain = kmalloc(8, GFP_NOFS);
if (!plain)
return -ENOMEM;
return code;
}
-static s32
-krb5_get_rc4_seq_num(struct krb5_ctx *kctx, unsigned char *cksum,
- unsigned char *buf, int *direction, s32 *seqnum)
-{
- struct crypto_sync_skcipher *cipher;
- unsigned char *plain;
- s32 code;
-
- dprintk("RPC: %s:\n", __func__);
- cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name, 0, 0);
- if (IS_ERR(cipher))
- return PTR_ERR(cipher);
-
- code = krb5_rc4_setup_seq_key(kctx, cipher, cksum);
- if (code)
- goto out;
-
- plain = kmalloc(8, GFP_NOFS);
- if (!plain) {
- code = -ENOMEM;
- goto out;
- }
-
- code = krb5_decrypt(cipher, cksum, buf, plain, 8);
- if (code)
- goto out_plain;
-
- if ((plain[4] != plain[5]) || (plain[4] != plain[6])
- || (plain[4] != plain[7])) {
- code = (s32)KG_BAD_SEQ;
- goto out_plain;
- }
-
- *direction = plain[4];
-
- *seqnum = ((plain[0] << 24) | (plain[1] << 16) |
- (plain[2] << 8) | (plain[3]));
-out_plain:
- kfree(plain);
-out:
- crypto_free_sync_skcipher(cipher);
- return code;
-}
-
s32
krb5_get_seq_num(struct krb5_ctx *kctx,
unsigned char *cksum,
dprintk("RPC: krb5_get_seq_num:\n");
- if (kctx->enctype == ENCTYPE_ARCFOUR_HMAC)
- return krb5_get_rc4_seq_num(kctx, cksum, buf,
- direction, seqnum);
plain = kmalloc(8, GFP_NOFS);
if (!plain)
return -ENOMEM;
BUG();
case ENCTYPE_DES_CBC_RAW:
case ENCTYPE_DES3_CBC_RAW:
- case ENCTYPE_ARCFOUR_HMAC:
return gss_verify_mic_v1(ctx, message_buffer, read_token);
case ENCTYPE_AES128_CTS_HMAC_SHA1_96:
case ENCTYPE_AES256_CTS_HMAC_SHA1_96:
seq_send, ptr + GSS_KRB5_TOK_HDR_LEN, ptr + 8)))
return GSS_S_FAILURE;
- if (kctx->enctype == ENCTYPE_ARCFOUR_HMAC) {
- struct crypto_sync_skcipher *cipher;
- int err;
- cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name,
- 0, 0);
- if (IS_ERR(cipher))
- return GSS_S_FAILURE;
-
- krb5_rc4_setup_enc_key(kctx, cipher, seq_send);
-
- err = gss_encrypt_xdr_buf(cipher, buf,
- offset + headlen - conflen, pages);
- crypto_free_sync_skcipher(cipher);
- if (err)
- return GSS_S_FAILURE;
- } else {
- if (gss_encrypt_xdr_buf(kctx->enc, buf,
- offset + headlen - conflen, pages))
- return GSS_S_FAILURE;
- }
+ if (gss_encrypt_xdr_buf(kctx->enc, buf,
+ offset + headlen - conflen, pages))
+ return GSS_S_FAILURE;
return (kctx->endtime < now) ? GSS_S_CONTEXT_EXPIRED : GSS_S_COMPLETE;
}
crypt_offset = ptr + (GSS_KRB5_TOK_HDR_LEN + kctx->gk5e->cksumlength) -
(unsigned char *)buf->head[0].iov_base;
- /*
- * Need plaintext seqnum to derive encryption key for arcfour-hmac
- */
- if (krb5_get_seq_num(kctx, ptr + GSS_KRB5_TOK_HDR_LEN,
- ptr + 8, &direction, &seqnum))
- return GSS_S_BAD_SIG;
-
- if ((kctx->initiate && direction != 0xff) ||
- (!kctx->initiate && direction != 0))
- return GSS_S_BAD_SIG;
-
buf->len = len;
- if (kctx->enctype == ENCTYPE_ARCFOUR_HMAC) {
- struct crypto_sync_skcipher *cipher;
- int err;
-
- cipher = crypto_alloc_sync_skcipher(kctx->gk5e->encrypt_name,
- 0, 0);
- if (IS_ERR(cipher))
- return GSS_S_FAILURE;
-
- krb5_rc4_setup_enc_key(kctx, cipher, seqnum);
-
- err = gss_decrypt_xdr_buf(cipher, buf, crypt_offset);
- crypto_free_sync_skcipher(cipher);
- if (err)
- return GSS_S_DEFECTIVE_TOKEN;
- } else {
- if (gss_decrypt_xdr_buf(kctx->enc, buf, crypt_offset))
- return GSS_S_DEFECTIVE_TOKEN;
- }
+ if (gss_decrypt_xdr_buf(kctx->enc, buf, crypt_offset))
+ return GSS_S_DEFECTIVE_TOKEN;
if (kctx->gk5e->keyed_cksum)
cksumkey = kctx->cksum;
/* do sequencing checks */
+ if (krb5_get_seq_num(kctx, ptr + GSS_KRB5_TOK_HDR_LEN,
+ ptr + 8, &direction, &seqnum))
+ return GSS_S_BAD_SIG;
+
+ if ((kctx->initiate && direction != 0xff) ||
+ (!kctx->initiate && direction != 0))
+ return GSS_S_BAD_SIG;
+
/* Copy the data back to the right position. XXX: Would probably be
* better to copy and encrypt at the same time. */
BUG();
case ENCTYPE_DES_CBC_RAW:
case ENCTYPE_DES3_CBC_RAW:
- case ENCTYPE_ARCFOUR_HMAC:
return gss_wrap_kerberos_v1(kctx, offset, buf, pages);
case ENCTYPE_AES128_CTS_HMAC_SHA1_96:
case ENCTYPE_AES256_CTS_HMAC_SHA1_96:
BUG();
case ENCTYPE_DES_CBC_RAW:
case ENCTYPE_DES3_CBC_RAW:
- case ENCTYPE_ARCFOUR_HMAC:
return gss_unwrap_kerberos_v1(kctx, offset, len, buf,
&gctx->slack, &gctx->align);
case ENCTYPE_AES128_CTS_HMAC_SHA1_96:
if (len > desc->count)
len = desc->count;
pos = desc->offset;
- csum2 = skb_copy_and_csum_bits(desc->skb, pos, to, len, 0);
+ csum2 = skb_copy_and_csum_bits(desc->skb, pos, to, len);
desc->csum = csum_block_add(desc->csum, csum2, pos);
desc->count -= len;
desc->offset += len;
static void svc_flush_bvec(const struct bio_vec *bvec, size_t size, size_t seek)
{
struct bvec_iter bi = {
- .bi_size = size,
+ .bi_size = size + seek,
};
struct bio_vec bv;
* @val: value passed unmodified to notifier function
* @dev: port device
* @info: notifier information data
- *
+ * @extack: netlink extended ack
* Call all network notifier blocks.
*/
int call_switchdev_notifiers(unsigned long val, struct net_device *dev,
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
struct tls_rec *rec, *tmp;
+ int pending;
/* Wait for any pending async encryptions to complete */
- smp_store_mb(ctx->async_notify, true);
- if (atomic_read(&ctx->encrypt_pending))
+ spin_lock_bh(&ctx->encrypt_compl_lock);
+ ctx->async_notify = true;
+ pending = atomic_read(&ctx->encrypt_pending);
+ spin_unlock_bh(&ctx->encrypt_compl_lock);
+
+ if (pending)
crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
tls_tx_records(sk, -1);
#include <net/af_unix.h>
#include <net/scm.h>
#include <linux/init.h>
+#include <linux/io_uring.h>
#include "scm.h"
if (err)
return err;
+ if (key.idx < 0)
+ return -EINVAL;
+
if (info->attrs[NL80211_ATTR_MAC])
mac_addr = nla_data(info->attrs[NL80211_ATTR_MAC]);
skb_shinfo(skb)->destructor_arg = (void *)(long)desc.addr;
skb->destructor = xsk_destruct_skb;
+ /* Hinder dev_direct_xmit from freeing the packet and
+ * therefore completing it in the destructor
+ */
+ refcount_inc(&skb->users);
err = dev_direct_xmit(skb, xs->queue_id);
+ if (err == NETDEV_TX_BUSY) {
+ /* Tell user-space to retry the send */
+ skb->destructor = sock_wfree;
+ /* Free skb without triggering the perf drop trace */
+ consume_skb(skb);
+ err = -EAGAIN;
+ goto out;
+ }
+
xskq_cons_release(xs->tx);
/* Ignore NET_XMIT_CN as packet might have been sent */
- if (err == NET_XMIT_DROP || err == NETDEV_TX_BUSY) {
+ if (err == NET_XMIT_DROP) {
/* SKB completed but not sent */
+ kfree_skb(skb);
err = -EBUSY;
goto out;
}
+ consume_skb(skb);
sent_frame = true;
}
static void handle_esp(struct sk_buff *skb, struct sock *sk)
{
+ struct tcp_skb_cb *tcp_cb = (struct tcp_skb_cb *)skb->cb;
+
skb_reset_transport_header(skb);
- memset(skb->cb, 0, sizeof(skb->cb));
+
+ /* restore IP CB, we need at least IP6CB->nhoff */
+ memmove(skb->cb, &tcp_cb->header, sizeof(tcp_cb->header));
rcu_read_lock();
skb->dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif);
}
mtu = dst_mtu(dst);
- if (!skb->ignore_df && skb->len > mtu) {
+ if (skb->len > mtu) {
skb_dst_update_pmtu_no_confirm(skb, mtu);
if (skb->protocol == htons(ETH_P_IPV6)) {
*/
if (x->km.state == XFRM_STATE_VALID) {
if ((x->sel.family &&
- !xfrm_selector_match(&x->sel, fl, x->sel.family)) ||
+ (x->sel.family != family ||
+ !xfrm_selector_match(&x->sel, fl, family))) ||
!security_xfrm_state_pol_flow_match(x, pol, fl))
return;
*acq_in_progress = 1;
} else if (x->km.state == XFRM_STATE_ERROR ||
x->km.state == XFRM_STATE_EXPIRED) {
- if (xfrm_selector_match(&x->sel, fl, x->sel.family) &&
+ if ((!x->sel.family ||
+ (x->sel.family == family &&
+ xfrm_selector_match(&x->sel, fl, family))) &&
security_xfrm_state_pol_flow_match(x, pol, fl))
*error = -ESRCH;
}
tmpl->mode == x->props.mode &&
tmpl->id.proto == x->id.proto &&
(tmpl->id.spi == x->id.spi || !tmpl->id.spi))
- xfrm_state_look_at(pol, x, fl, encap_family,
+ xfrm_state_look_at(pol, x, fl, family,
&best, &acquire_in_progress, &error);
}
if (best || acquire_in_progress)
tmpl->mode == x->props.mode &&
tmpl->id.proto == x->id.proto &&
(tmpl->id.spi == x->id.spi || !tmpl->id.spi))
- xfrm_state_look_at(pol, x, fl, encap_family,
+ xfrm_state_look_at(pol, x, fl, family,
&best, &acquire_in_progress, &error);
}
EXPORT_SYMBOL(xfrm_state_add);
#ifdef CONFIG_XFRM_MIGRATE
+static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security)
+{
+ struct xfrm_user_sec_ctx *uctx;
+ int size = sizeof(*uctx) + security->ctx_len;
+ int err;
+
+ uctx = kmalloc(size, GFP_KERNEL);
+ if (!uctx)
+ return -ENOMEM;
+
+ uctx->exttype = XFRMA_SEC_CTX;
+ uctx->len = size;
+ uctx->ctx_doi = security->ctx_doi;
+ uctx->ctx_alg = security->ctx_alg;
+ uctx->ctx_len = security->ctx_len;
+ memcpy(uctx + 1, security->ctx_str, security->ctx_len);
+ err = security_xfrm_state_alloc(x, uctx);
+ kfree(uctx);
+ if (err)
+ return err;
+
+ return 0;
+}
+
static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
struct xfrm_encap_tmpl *encap)
{
goto error;
}
+ if (orig->security)
+ if (clone_security(x, orig->security))
+ goto error;
+
if (orig->coaddr) {
x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr),
GFP_KERNEL);
}
memcpy(&x->mark, &orig->mark, sizeof(x->mark));
+ memcpy(&x->props.smark, &orig->props.smark, sizeof(x->props.smark));
if (xfrm_init_state(x) < 0)
goto error;
x->tfcpad = orig->tfcpad;
x->replay_maxdiff = orig->replay_maxdiff;
x->replay_maxage = orig->replay_maxage;
- x->curlft.add_time = orig->curlft.add_time;
+ memcpy(&x->curlft, &orig->curlft, sizeof(x->curlft));
x->km.state = orig->km.state;
x->km.seq = orig->km.seq;
x->replay = orig->replay;
* stack trace and selected registers when _do_fork() is called.
*
* For more information on theory of operation of kprobes, see
- * Documentation/staging/kprobes.rst
+ * Documentation/trace/kprobes.rst
*
* You will see the trace data in /var/log/messages and on the console
* whenever _do_fork() is invoked to create a new process.
* If no func_name is specified, _do_fork is instrumented
*
* For more information on theory of operation of kretprobes, see
- * Documentation/staging/kprobes.rst
+ * Documentation/trace/kprobes.rst
*
* Build and insert the kernel module as done in the kprobe example.
* You will see the trace data in /var/log/messages and on the console
# of some options does not break KCSAN nor causes false positive reports.
CFLAGS_KCSAN := -fsanitize=thread \
$(call cc-option,$(call cc-param,tsan-instrument-func-entry-exit=0) -fno-optimize-sibling-calls) \
- $(call cc-option,$(call cc-param,tsan-instrument-read-before-write=1)) \
+ $(call cc-option,$(call cc-param,tsan-compound-read-before-write=1),$(call cc-option,$(call cc-param,tsan-instrument-read-before-write=1))) \
$(call cc-param,tsan-distinguish-volatile=1)
cat <<EOF |
asm-generic/atomic-instrumented.h
asm-generic/atomic-long.h
+linux/atomic-arch-fallback.h
linux/atomic-fallback.h
EOF
while read header; do
. ${ATOMICDIR}/atomic-tbl.sh
-#gen_param_check(arg)
+#gen_param_check(meta, arg)
gen_param_check()
{
+ local meta="$1"; shift
local arg="$1"; shift
local type="${arg%%:*}"
local name="$(gen_param_name "${arg}")"
i) return;;
esac
- # We don't write to constant parameters
- [ ${type#c} != ${type} ] && rw="read"
+ if [ ${type#c} != ${type} ]; then
+ # We don't write to constant parameters.
+ rw="read"
+ elif [ "${meta}" != "s" ]; then
+ # An atomic RMW: if this parameter is not a constant, and this atomic is
+ # not just a 's'tore, this parameter is both read from and written to.
+ rw="read_write"
+ fi
printf "\tinstrument_atomic_${rw}(${name}, sizeof(*${name}));\n"
}
-#gen_param_check(arg...)
+#gen_params_checks(meta, arg...)
gen_params_checks()
{
+ local meta="$1"; shift
+
while [ "$#" -gt 0 ]; do
- gen_param_check "$1"
+ gen_param_check "$meta" "$1"
shift;
done
}
local ret="$(gen_ret_type "${meta}" "${int}")"
local params="$(gen_params "${int}" "${atomic}" "$@")"
- local checks="$(gen_params_checks "$@")"
+ local checks="$(gen_params_checks "${meta}" "$@")"
local args="$(gen_args "$@")"
local retstmt="$(gen_ret_stmt "${meta}")"
// SPDX-License-Identifier: GPL-2.0-only
///
-/// From Documentation/filesystems/sysfs.txt:
+/// From Documentation/filesystems/sysfs.rst:
/// show() must not use snprintf() when formatting the value to be
/// returned to user space. If you can guarantee that an overflow
/// will never happen you can use sprintf() otherwise you must use
dtc-objs += dtc-lexer.lex.o dtc-parser.tab.o
# Source files need to get at the userspace version of libfdt_env.h to compile
-HOST_EXTRACFLAGS := -I $(srctree)/$(src)/libfdt
+HOST_EXTRACFLAGS += -I $(srctree)/$(src)/libfdt
ifeq ($(shell pkg-config --exists yaml-0.1 2>/dev/null && echo yes),)
ifneq ($(CHECK_DT_BINDING)$(CHECK_DTBS),)
static bool is_ignored_symbol(const char *name, char type)
{
+ /* Symbol names that exactly match to the following are ignored.*/
static const char * const ignored_symbols[] = {
/*
* Symbols which vary between passes. Passes 1 and 2 must have
NULL
};
+ /* Symbol names that begin with the following are ignored.*/
static const char * const ignored_prefixes[] = {
"$", /* local symbols for ARM, MIPS, etc. */
".LASANPC", /* s390 kasan local symbols */
NULL
};
+ /* Symbol names that end with the following are ignored.*/
static const char * const ignored_suffixes[] = {
"_from_arm", /* arm */
"_from_thumb", /* arm */
NULL
};
+ /* Symbol names that contain the following are ignored.*/
+ static const char * const ignored_matches[] = {
+ ".long_branch.", /* ppc stub */
+ ".plt_branch.", /* ppc stub */
+ NULL
+ };
+
const char * const *p;
- /* Exclude symbols which vary between passes. */
for (p = ignored_symbols; *p; p++)
if (!strcmp(name, *p))
return true;
return true;
}
+ for (p = ignored_matches; *p; p++) {
+ if (strstr(name, *p))
+ return true;
+ }
+
if (type == 'U' || type == 'u')
return true;
/* exclude debugging symbols */
my $x = shift;
my $file = shift;
- if ($x =~ /(struct|union)\s+(\w+)\s*\{(.*)\}(\s*(__packed|__aligned|____cacheline_aligned_in_smp|__attribute__\s*\(\([a-z0-9,_\s\(\)]*\)\)))*/) {
+ if ($x =~ /(struct|union)\s+(\w+)\s*\{(.*)\}(\s*(__packed|__aligned|____cacheline_aligned_in_smp|____cacheline_aligned|__attribute__\s*\(\([a-z0-9,_\s\(\)]*\)\)))*/) {
my $decl_type = $1;
$declaration_name = $2;
my $members = $3;
$members =~ s/\s*__packed\s*/ /gos;
$members =~ s/\s*CRYPTO_MINALIGN_ATTR/ /gos;
$members =~ s/\s*____cacheline_aligned_in_smp/ /gos;
+ $members =~ s/\s*____cacheline_aligned/ /gos;
# replace DECLARE_BITMAP
$members =~ s/__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)/DECLARE_BITMAP($1, __ETHTOOL_LINK_MODE_MASK_NBITS)/gos;
my $file = shift;
my $noret = 0;
+ print_lineno($.);
+
$prototype =~ s/^static +//;
$prototype =~ s/^extern +//;
$prototype =~ s/^asmlinkage +//;
expresion||expression
exprimental||experimental
extened||extended
-exteneded||extended||extended
+exteneded||extended
extensability||extensibility
extention||extension
extenstion||extension
'/\<DEVICE_ATTR_\(RW\|RO\|WO\)(\([[:alnum:]_]\+\)/dev_attr_\2/'
'/\<DRIVER_ATTR_\(RW\|RO\|WO\)(\([[:alnum:]_]\+\)/driver_attr_\2/'
'/\<\(DEFINE\|DECLARE\)_STATIC_KEY_\(TRUE\|FALSE\)\(\|_RO\)(\([[:alnum:]_]\+\)/\4/'
+ '/^SEQCOUNT_LOCKTYPE(\([^,]*\),[[:space:]]*\([^,]*\),[^)]*)/seqcount_\2_t/'
+ '/^SEQCOUNT_LOCKTYPE(\([^,]*\),[[:space:]]*\([^,]*\),[^)]*)/seqcount_\2_init/'
)
regex_kconfig=(
'/^[[:blank:]]*\(menu\|\)config[[:blank:]]\+\([[:alnum:]_]\+\)/\2/'
memset(&pks, 0, sizeof(pks));
pks.hash_algo = hash_algo_name[hdr->hash_algo];
- if (hdr->hash_algo == HASH_ALGO_STREEBOG_256 ||
- hdr->hash_algo == HASH_ALGO_STREEBOG_512) {
+ switch (hdr->hash_algo) {
+ case HASH_ALGO_STREEBOG_256:
+ case HASH_ALGO_STREEBOG_512:
/* EC-RDSA and Streebog should go together. */
pks.pkey_algo = "ecrdsa";
pks.encoding = "raw";
- } else {
+ break;
+ case HASH_ALGO_SM3_256:
+ /* SM2 and SM3 should go together. */
+ pks.pkey_algo = "sm2";
+ pks.encoding = "raw";
+ break;
+ default:
pks.pkey_algo = "rsa";
pks.encoding = "pkcs1";
+ break;
}
pks.digest = (u8 *)data;
pks.digest_size = datalen;
}
/*
+ * load_moklist_certs() - Load MokList certs
+ *
+ * Load the certs contained in the UEFI MokListRT database into the
+ * platform trusted keyring.
+ *
+ * This routine checks the EFI MOK config table first. If and only if
+ * that fails, this routine uses the MokListRT ordinary UEFI variable.
+ *
+ * Return: Status
+ */
+static int __init load_moklist_certs(void)
+{
+ struct efi_mokvar_table_entry *mokvar_entry;
+ efi_guid_t mok_var = EFI_SHIM_LOCK_GUID;
+ void *mok;
+ unsigned long moksize;
+ efi_status_t status;
+ int rc;
+
+ /* First try to load certs from the EFI MOKvar config table.
+ * It's not an error if the MOKvar config table doesn't exist
+ * or the MokListRT entry is not found in it.
+ */
+ mokvar_entry = efi_mokvar_entry_find("MokListRT");
+ if (mokvar_entry) {
+ rc = parse_efi_signature_list("UEFI:MokListRT (MOKvar table)",
+ mokvar_entry->data,
+ mokvar_entry->data_size,
+ get_handler_for_db);
+ /* All done if that worked. */
+ if (!rc)
+ return rc;
+
+ pr_err("Couldn't parse MokListRT signatures from EFI MOKvar config table: %d\n",
+ rc);
+ }
+
+ /* Get MokListRT. It might not exist, so it isn't an error
+ * if we can't get it.
+ */
+ mok = get_cert_list(L"MokListRT", &mok_var, &moksize, &status);
+ if (mok) {
+ rc = parse_efi_signature_list("UEFI:MokListRT",
+ mok, moksize, get_handler_for_db);
+ kfree(mok);
+ if (rc)
+ pr_err("Couldn't parse MokListRT signatures: %d\n", rc);
+ return rc;
+ }
+ if (status == EFI_NOT_FOUND)
+ pr_debug("MokListRT variable wasn't found\n");
+ else
+ pr_info("Couldn't get UEFI MokListRT\n");
+ return 0;
+}
+
+/*
+ * load_uefi_certs() - Load certs from UEFI sources
+ *
* Load the certs contained in the UEFI databases into the platform trusted
* keyring and the UEFI blacklisted X.509 cert SHA256 hashes into the blacklist
* keyring.
static int __init load_uefi_certs(void)
{
efi_guid_t secure_var = EFI_IMAGE_SECURITY_DATABASE_GUID;
- efi_guid_t mok_var = EFI_SHIM_LOCK_GUID;
- void *db = NULL, *dbx = NULL, *mok = NULL;
- unsigned long dbsize = 0, dbxsize = 0, moksize = 0;
+ void *db = NULL, *dbx = NULL;
+ unsigned long dbsize = 0, dbxsize = 0;
efi_status_t status;
int rc = 0;
if (!efi_rt_services_supported(EFI_RT_SUPPORTED_GET_VARIABLE))
return false;
- /* Get db, MokListRT, and dbx. They might not exist, so it isn't
- * an error if we can't get them.
+ /* Get db and dbx. They might not exist, so it isn't an error
+ * if we can't get them.
*/
if (!uefi_check_ignore_db()) {
db = get_cert_list(L"db", &secure_var, &dbsize, &status);
}
}
- mok = get_cert_list(L"MokListRT", &mok_var, &moksize, &status);
- if (!mok) {
- if (status == EFI_NOT_FOUND)
- pr_debug("MokListRT variable wasn't found\n");
- else
- pr_info("Couldn't get UEFI MokListRT\n");
- } else {
- rc = parse_efi_signature_list("UEFI:MokListRT",
- mok, moksize, get_handler_for_db);
- if (rc)
- pr_err("Couldn't parse MokListRT signatures: %d\n", rc);
- kfree(mok);
- }
-
dbx = get_cert_list(L"dbx", &secure_var, &dbxsize, &status);
if (!dbx) {
if (status == EFI_NOT_FOUND)
kfree(dbx);
}
+ /* Load the MokListRT certs */
+ rc = load_moklist_certs();
+
return rc;
}
late_initcall(load_uefi_certs);
#include <linux/slab.h>
#include "internal.h"
-/*
- * Instantiate a key with the specified compatibility multipart payload and
- * link the key into the destination keyring if one is given.
- *
- * The caller must have the appropriate instantiation permit set for this to
- * work (see keyctl_assume_authority). No other permissions are required.
- *
- * If successful, 0 will be returned.
- */
-static long compat_keyctl_instantiate_key_iov(
- key_serial_t id,
- const struct compat_iovec __user *_payload_iov,
- unsigned ioc,
- key_serial_t ringid)
-{
- struct iovec iovstack[UIO_FASTIOV], *iov = iovstack;
- struct iov_iter from;
- long ret;
-
- if (!_payload_iov)
- ioc = 0;
-
- ret = compat_import_iovec(WRITE, _payload_iov, ioc,
- ARRAY_SIZE(iovstack), &iov,
- &from);
- if (ret < 0)
- return ret;
-
- ret = keyctl_instantiate_key_common(id, &from, ringid);
- kfree(iov);
- return ret;
-}
-
/*
* The key control system call, 32-bit compatibility version for 64-bit archs
*/
return keyctl_reject_key(arg2, arg3, arg4, arg5);
case KEYCTL_INSTANTIATE_IOV:
- return compat_keyctl_instantiate_key_iov(
- arg2, compat_ptr(arg3), arg4, arg5);
+ return keyctl_instantiate_key_iov(arg2, compat_ptr(arg3), arg4,
+ arg5);
case KEYCTL_INVALIDATE:
return keyctl_invalidate_key(arg2);
const struct iovec __user *,
unsigned, key_serial_t);
extern long keyctl_invalidate_key(key_serial_t);
-
-struct iov_iter;
-extern long keyctl_instantiate_key_common(key_serial_t,
- struct iov_iter *,
- key_serial_t);
extern long keyctl_restrict_keyring(key_serial_t id,
const char __user *_type,
const char __user *_restriction);
*
* If successful, 0 will be returned.
*/
-long keyctl_instantiate_key_common(key_serial_t id,
+static long keyctl_instantiate_key_common(key_serial_t id,
struct iov_iter *from,
key_serial_t ringid)
{
struct hpi_message hm;
struct hpi_response hr;
struct hpi_adapter adapter;
- struct hpi_pci pci;
+ struct hpi_pci pci = { 0 };
memset(&adapter, 0, sizeof(adapter));
return 0;
err:
- for (idx = 0; idx < HPI_MAX_ADAPTER_MEM_SPACES; idx++) {
+ while (--idx >= 0) {
if (pci.ap_mem_base[idx]) {
iounmap(pci.ap_mem_base[idx]);
pci.ap_mem_base[idx] = NULL;
SND_PCI_QUIRK(0x1462, 0x1276, "MSI-GL73", ALC1220_FIXUP_CLEVO_P950),
SND_PCI_QUIRK(0x1462, 0x1293, "MSI-GP65", ALC1220_FIXUP_CLEVO_P950),
SND_PCI_QUIRK(0x1462, 0x7350, "MSI-7350", ALC889_FIXUP_CD),
- SND_PCI_QUIRK(0x1462, 0x9c37, "MSI X570-A PRO", ALC1220_FIXUP_CLEVO_P950),
SND_PCI_QUIRK(0x1462, 0xda57, "MSI Z270-Gaming", ALC1220_FIXUP_GB_DUAL_CODECS),
SND_PCI_QUIRK_VENDOR(0x1462, "MSI", ALC882_FIXUP_GPIO3),
SND_PCI_QUIRK(0x147b, 0x107a, "Abit AW9D-MAX", ALC882_FIXUP_ABIT_AW9D_MAX),
/* 3k pull low control for Headset jack. */
/* NOTE: call this before clearing the pin, otherwise codec stalls */
- alc_update_coef_idx(codec, 0x46, 0, 3 << 12);
+ /* If disable 3k pulldown control for alc257, the Mic detection will not work correctly
+ * when booting with headset plugged. So skip setting it for the codec alc257
+ */
+ if (codec->core.vendor_id != 0x10ec0257)
+ alc_update_coef_idx(codec, 0x46, 0, 3 << 12);
if (!spec->no_shutup_pins)
snd_hda_codec_write(codec, hp_pin, 0,
#include "hp_x360_helper.c"
enum {
+ ALC269_FIXUP_GPIO2,
ALC269_FIXUP_SONY_VAIO,
ALC275_FIXUP_SONY_VAIO_GPIO2,
ALC269_FIXUP_DELL_M101Z,
};
static const struct hda_fixup alc269_fixups[] = {
+ [ALC269_FIXUP_GPIO2] = {
+ .type = HDA_FIXUP_FUNC,
+ .v.func = alc_fixup_gpio2,
+ },
[ALC269_FIXUP_SONY_VAIO] = {
.type = HDA_FIXUP_PINCTLS,
.v.pins = (const struct hda_pintbl[]) {
[ALC233_FIXUP_LENOVO_MULTI_CODECS] = {
.type = HDA_FIXUP_FUNC,
.v.func = alc233_alc662_fixup_lenovo_dual_codecs,
+ .chained = true,
+ .chain_id = ALC269_FIXUP_GPIO2
},
[ALC233_FIXUP_ACER_HEADSET_MIC] = {
.type = HDA_FIXUP_VERBS,
};
static const struct usbmix_name_map lenovo_p620_rear_map[] = {
- { 19, NULL, 2 }, /* FU, Volume */
{ 19, NULL, 12 }, /* FU, Input Gain Pad */
{}
};
&& (requesttype & USB_TYPE_MASK) == USB_TYPE_CLASS)
msleep(20);
- /* Zoom R16/24, Logitech H650e, Jabra 550a, Kingston HyperX needs a tiny
- * delay here, otherwise requests like get/set frequency return as
- * failed despite actually succeeding.
+ /* Zoom R16/24, Logitech H650e/H570e, Jabra 550a, Kingston HyperX
+ * needs a tiny delay here, otherwise requests like get/set
+ * frequency return as failed despite actually succeeding.
*/
if ((chip->usb_id == USB_ID(0x1686, 0x00dd) ||
chip->usb_id == USB_ID(0x046d, 0x0a46) ||
+ chip->usb_id == USB_ID(0x046d, 0x0a56) ||
chip->usb_id == USB_ID(0x0b0e, 0x0349) ||
chip->usb_id == USB_ID(0x0951, 0x16ad)) &&
(requesttype & USB_TYPE_MASK) == USB_TYPE_CLASS)
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _MCSAFE_TEST_H_
-#define _MCSAFE_TEST_H_
-
-.macro MCSAFE_TEST_CTL
-.endm
-
-.macro MCSAFE_TEST_SRC reg count target
-.endm
-
-.macro MCSAFE_TEST_DST reg count target
-.endm
-#endif /* _MCSAFE_TEST_H_ */
#include <linux/linkage.h>
#include <asm/errno.h>
#include <asm/cpufeatures.h>
-#include <asm/mcsafe_test.h>
#include <asm/alternative-asm.h>
#include <asm/export.h>
SYM_FUNC_END(memcpy_orig)
.popsection
-
-#ifndef CONFIG_UML
-
-MCSAFE_TEST_CTL
-
-/*
- * __memcpy_mcsafe - memory copy with machine check exception handling
- * Note that we only catch machine checks when reading the source addresses.
- * Writes to target are posted and don't generate machine checks.
- */
-SYM_FUNC_START(__memcpy_mcsafe)
- cmpl $8, %edx
- /* Less than 8 bytes? Go to byte copy loop */
- jb .L_no_whole_words
-
- /* Check for bad alignment of source */
- testl $7, %esi
- /* Already aligned */
- jz .L_8byte_aligned
-
- /* Copy one byte at a time until source is 8-byte aligned */
- movl %esi, %ecx
- andl $7, %ecx
- subl $8, %ecx
- negl %ecx
- subl %ecx, %edx
-.L_read_leading_bytes:
- movb (%rsi), %al
- MCSAFE_TEST_SRC %rsi 1 .E_leading_bytes
- MCSAFE_TEST_DST %rdi 1 .E_leading_bytes
-.L_write_leading_bytes:
- movb %al, (%rdi)
- incq %rsi
- incq %rdi
- decl %ecx
- jnz .L_read_leading_bytes
-
-.L_8byte_aligned:
- movl %edx, %ecx
- andl $7, %edx
- shrl $3, %ecx
- jz .L_no_whole_words
-
-.L_read_words:
- movq (%rsi), %r8
- MCSAFE_TEST_SRC %rsi 8 .E_read_words
- MCSAFE_TEST_DST %rdi 8 .E_write_words
-.L_write_words:
- movq %r8, (%rdi)
- addq $8, %rsi
- addq $8, %rdi
- decl %ecx
- jnz .L_read_words
-
- /* Any trailing bytes? */
-.L_no_whole_words:
- andl %edx, %edx
- jz .L_done_memcpy_trap
-
- /* Copy trailing bytes */
- movl %edx, %ecx
-.L_read_trailing_bytes:
- movb (%rsi), %al
- MCSAFE_TEST_SRC %rsi 1 .E_trailing_bytes
- MCSAFE_TEST_DST %rdi 1 .E_trailing_bytes
-.L_write_trailing_bytes:
- movb %al, (%rdi)
- incq %rsi
- incq %rdi
- decl %ecx
- jnz .L_read_trailing_bytes
-
- /* Copy successful. Return zero */
-.L_done_memcpy_trap:
- xorl %eax, %eax
-.L_done:
- ret
-SYM_FUNC_END(__memcpy_mcsafe)
-EXPORT_SYMBOL_GPL(__memcpy_mcsafe)
-
- .section .fixup, "ax"
- /*
- * Return number of bytes not copied for any failure. Note that
- * there is no "tail" handling since the source buffer is 8-byte
- * aligned and poison is cacheline aligned.
- */
-.E_read_words:
- shll $3, %ecx
-.E_leading_bytes:
- addl %edx, %ecx
-.E_trailing_bytes:
- mov %ecx, %eax
- jmp .L_done
-
- /*
- * For write fault handling, given the destination is unaligned,
- * we handle faults on multi-byte writes with a byte-by-byte
- * copy up to the write-protected page.
- */
-.E_write_words:
- shll $3, %ecx
- addl %edx, %ecx
- movl %ecx, %edx
- jmp mcsafe_handle_tail
-
- .previous
-
- _ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes)
- _ASM_EXTABLE_FAULT(.L_read_words, .E_read_words)
- _ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes)
- _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)
- _ASM_EXTABLE(.L_write_words, .E_write_words)
- _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)
-#endif
cat $TEMPCONF
xpass grep \'\"string\"\' $TEMPCONF
+echo "Repeat same-key tree"
+cat > $TEMPCONF << EOF
+foo
+bar
+foo { buz }
+EOF
+echo > $INITRD
+
+xpass $BOOTCONF -a $TEMPCONF $INITRD
+$BOOTCONF $INITRD > $OUTFILE
+xpass grep -q "bar" $OUTFILE
+
+
+echo "Remove/keep tailing spaces"
+cat > $TEMPCONF << EOF
+foo = val # comment
+bar = "val2 " # comment
+EOF
+echo > $INITRD
+
+xpass $BOOTCONF -a $TEMPCONF $INITRD
+$BOOTCONF $INITRD > $OUTFILE
+xfail grep -q val[[:space:]] $OUTFILE
+xpass grep -q val2[[:space:]] $OUTFILE
+
echo "=== expected failure cases ==="
for i in samples/bad-* ; do
xfail $BOOTCONF -a $i $INITRD
LIBBPF = $(LIBBPF_PATH)libbpf.a
-BPFTOOL_VERSION := $(shell make -rR --no-print-directory -sC ../../.. kernelversion)
+BPFTOOL_VERSION ?= $(shell make -rR --no-print-directory -sC ../../.. kernelversion)
$(LIBBPF): FORCE
$(if $(LIBBPF_OUTPUT),@mkdir -p $(LIBBPF_OUTPUT))
#include <sys/ioctl.h>
#include <sys/types.h>
#include <linux/gpio.h>
+#include "gpio-utils.h"
int monitor_device(const char *device_name,
- unsigned int line,
- uint32_t handleflags,
- uint32_t eventflags,
+ unsigned int *lines,
+ unsigned int num_lines,
+ struct gpio_v2_line_config *config,
unsigned int loops)
{
- struct gpioevent_request req;
- struct gpiohandle_data data;
+ struct gpio_v2_line_values values;
char *chrdev_name;
- int fd;
+ int cfd, lfd;
int ret;
int i = 0;
if (ret < 0)
return -ENOMEM;
- fd = open(chrdev_name, 0);
- if (fd == -1) {
+ cfd = open(chrdev_name, 0);
+ if (cfd == -1) {
ret = -errno;
fprintf(stderr, "Failed to open %s\n", chrdev_name);
goto exit_free_name;
}
- req.lineoffset = line;
- req.handleflags = handleflags;
- req.eventflags = eventflags;
- strcpy(req.consumer_label, "gpio-event-mon");
-
- ret = ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, &req);
- if (ret == -1) {
- ret = -errno;
- fprintf(stderr, "Failed to issue GET EVENT "
- "IOCTL (%d)\n",
- ret);
- goto exit_close_error;
- }
+ ret = gpiotools_request_line(device_name, lines, num_lines, config,
+ "gpio-event-mon");
+ if (ret < 0)
+ goto exit_device_close;
+ else
+ lfd = ret;
/* Read initial states */
- ret = ioctl(req.fd, GPIOHANDLE_GET_LINE_VALUES_IOCTL, &data);
- if (ret == -1) {
- ret = -errno;
- fprintf(stderr, "Failed to issue GPIOHANDLE GET LINE "
- "VALUES IOCTL (%d)\n",
+ values.mask = 0;
+ values.bits = 0;
+ for (i = 0; i < num_lines; i++)
+ gpiotools_set_bit(&values.mask, i);
+ ret = gpiotools_get_values(lfd, &values);
+ if (ret < 0) {
+ fprintf(stderr,
+ "Failed to issue GPIO LINE GET VALUES IOCTL (%d)\n",
ret);
- goto exit_close_error;
+ goto exit_line_close;
}
- fprintf(stdout, "Monitoring line %d on %s\n", line, device_name);
- fprintf(stdout, "Initial line value: %d\n", data.values[0]);
+ if (num_lines == 1) {
+ fprintf(stdout, "Monitoring line %d on %s\n", lines[0], device_name);
+ fprintf(stdout, "Initial line value: %d\n",
+ gpiotools_test_bit(values.bits, 0));
+ } else {
+ fprintf(stdout, "Monitoring lines %d", lines[0]);
+ for (i = 1; i < num_lines - 1; i++)
+ fprintf(stdout, ", %d", lines[i]);
+ fprintf(stdout, " and %d on %s\n", lines[i], device_name);
+ fprintf(stdout, "Initial line values: %d",
+ gpiotools_test_bit(values.bits, 0));
+ for (i = 1; i < num_lines - 1; i++)
+ fprintf(stdout, ", %d",
+ gpiotools_test_bit(values.bits, i));
+ fprintf(stdout, " and %d\n",
+ gpiotools_test_bit(values.bits, i));
+ }
while (1) {
- struct gpioevent_data event;
+ struct gpio_v2_line_event event;
- ret = read(req.fd, &event, sizeof(event));
+ ret = read(lfd, &event, sizeof(event));
if (ret == -1) {
if (errno == -EAGAIN) {
fprintf(stderr, "nothing available\n");
ret = -EIO;
break;
}
- fprintf(stdout, "GPIO EVENT %llu: ", event.timestamp);
+ fprintf(stdout, "GPIO EVENT at %llu on line %d (%d|%d) ",
+ event.timestamp_ns, event.offset, event.line_seqno,
+ event.seqno);
switch (event.id) {
- case GPIOEVENT_EVENT_RISING_EDGE:
+ case GPIO_V2_LINE_EVENT_RISING_EDGE:
fprintf(stdout, "rising edge");
break;
- case GPIOEVENT_EVENT_FALLING_EDGE:
+ case GPIO_V2_LINE_EVENT_FALLING_EDGE:
fprintf(stdout, "falling edge");
break;
default:
break;
}
-exit_close_error:
- if (close(fd) == -1)
+exit_line_close:
+ if (close(lfd) == -1)
+ perror("Failed to close line file");
+exit_device_close:
+ if (close(cfd) == -1)
perror("Failed to close GPIO character device file");
exit_free_name:
free(chrdev_name);
fprintf(stderr, "Usage: gpio-event-mon [options]...\n"
"Listen to events on GPIO lines, 0->1 1->0\n"
" -n <name> Listen on GPIOs on a named device (must be stated)\n"
- " -o <n> Offset to monitor\n"
+ " -o <n> Offset of line to monitor (may be repeated)\n"
" -d Set line as open drain\n"
" -s Set line as open source\n"
" -r Listen for rising edges\n"
" -f Listen for falling edges\n"
+ " -b <n> Debounce the line with period n microseconds\n"
" [-c <n>] Do <n> loops (optional, infinite loop if not stated)\n"
" -? This helptext\n"
"\n"
"Example:\n"
- "gpio-event-mon -n gpiochip0 -o 4 -r -f\n"
+ "gpio-event-mon -n gpiochip0 -o 4 -r -f -b 10000\n"
);
}
+#define EDGE_FLAGS \
+ (GPIO_V2_LINE_FLAG_EDGE_RISING | \
+ GPIO_V2_LINE_FLAG_EDGE_FALLING)
+
int main(int argc, char **argv)
{
const char *device_name = NULL;
- unsigned int line = -1;
+ unsigned int lines[GPIO_V2_LINES_MAX];
+ unsigned int num_lines = 0;
unsigned int loops = 0;
- uint32_t handleflags = GPIOHANDLE_REQUEST_INPUT;
- uint32_t eventflags = 0;
- int c;
+ struct gpio_v2_line_config config;
+ int c, attr, i;
+ unsigned long debounce_period_us = 0;
- while ((c = getopt(argc, argv, "c:n:o:dsrf?")) != -1) {
+ memset(&config, 0, sizeof(config));
+ config.flags = GPIO_V2_LINE_FLAG_INPUT;
+ while ((c = getopt(argc, argv, "c:n:o:b:dsrf?")) != -1) {
switch (c) {
case 'c':
loops = strtoul(optarg, NULL, 10);
device_name = optarg;
break;
case 'o':
- line = strtoul(optarg, NULL, 10);
+ if (num_lines >= GPIO_V2_LINES_MAX) {
+ print_usage();
+ return -1;
+ }
+ lines[num_lines] = strtoul(optarg, NULL, 10);
+ num_lines++;
+ break;
+ case 'b':
+ debounce_period_us = strtoul(optarg, NULL, 10);
break;
case 'd':
- handleflags |= GPIOHANDLE_REQUEST_OPEN_DRAIN;
+ config.flags |= GPIO_V2_LINE_FLAG_OPEN_DRAIN;
break;
case 's':
- handleflags |= GPIOHANDLE_REQUEST_OPEN_SOURCE;
+ config.flags |= GPIO_V2_LINE_FLAG_OPEN_SOURCE;
break;
case 'r':
- eventflags |= GPIOEVENT_REQUEST_RISING_EDGE;
+ config.flags |= GPIO_V2_LINE_FLAG_EDGE_RISING;
break;
case 'f':
- eventflags |= GPIOEVENT_REQUEST_FALLING_EDGE;
+ config.flags |= GPIO_V2_LINE_FLAG_EDGE_FALLING;
break;
case '?':
print_usage();
}
}
- if (!device_name || line == -1) {
+ if (debounce_period_us) {
+ attr = config.num_attrs;
+ config.num_attrs++;
+ for (i = 0; i < num_lines; i++)
+ gpiotools_set_bit(&config.attrs[attr].mask, i);
+ config.attrs[attr].attr.id = GPIO_V2_LINE_ATTR_ID_DEBOUNCE;
+ config.attrs[attr].attr.debounce_period_us = debounce_period_us;
+ }
+
+ if (!device_name || num_lines == 0) {
print_usage();
return -1;
}
- if (!eventflags) {
+ if (!(config.flags & EDGE_FLAGS)) {
printf("No flags specified, listening on both rising and "
"falling edges\n");
- eventflags = GPIOEVENT_REQUEST_BOTH_EDGES;
+ config.flags |= EDGE_FLAGS;
}
- return monitor_device(device_name, line, handleflags,
- eventflags, loops);
+ return monitor_device(device_name, lines, num_lines, &config, loops);
}
#include <linux/gpio.h>
#include "gpio-utils.h"
-int hammer_device(const char *device_name, unsigned int *lines, int nlines,
+int hammer_device(const char *device_name, unsigned int *lines, int num_lines,
unsigned int loops)
{
- struct gpiohandle_data data;
+ struct gpio_v2_line_values values;
+ struct gpio_v2_line_config config;
char swirr[] = "-\\|/";
int fd;
int ret;
int i, j;
unsigned int iteration = 0;
- memset(&data.values, 0, sizeof(data.values));
- ret = gpiotools_request_linehandle(device_name, lines, nlines,
- GPIOHANDLE_REQUEST_OUTPUT, &data,
- "gpio-hammer");
+ memset(&config, 0, sizeof(config));
+ config.flags = GPIO_V2_LINE_FLAG_OUTPUT;
+
+ ret = gpiotools_request_line(device_name, lines, num_lines,
+ &config, "gpio-hammer");
if (ret < 0)
goto exit_error;
else
fd = ret;
- ret = gpiotools_get_values(fd, &data);
+ values.mask = 0;
+ values.bits = 0;
+ for (i = 0; i < num_lines; i++)
+ gpiotools_set_bit(&values.mask, i);
+
+ ret = gpiotools_get_values(fd, &values);
if (ret < 0)
goto exit_close_error;
fprintf(stdout, "Hammer lines [");
- for (i = 0; i < nlines; i++) {
+ for (i = 0; i < num_lines; i++) {
fprintf(stdout, "%d", lines[i]);
- if (i != (nlines - 1))
+ if (i != (num_lines - 1))
fprintf(stdout, ", ");
}
fprintf(stdout, "] on %s, initial states: [", device_name);
- for (i = 0; i < nlines; i++) {
- fprintf(stdout, "%d", data.values[i]);
- if (i != (nlines - 1))
+ for (i = 0; i < num_lines; i++) {
+ fprintf(stdout, "%d", gpiotools_test_bit(values.bits, i));
+ if (i != (num_lines - 1))
fprintf(stdout, ", ");
}
fprintf(stdout, "]\n");
j = 0;
while (1) {
/* Invert all lines so we blink */
- for (i = 0; i < nlines; i++)
- data.values[i] = !data.values[i];
+ for (i = 0; i < num_lines; i++)
+ gpiotools_change_bit(&values.bits, i);
- ret = gpiotools_set_values(fd, &data);
+ ret = gpiotools_set_values(fd, &values);
if (ret < 0)
goto exit_close_error;
/* Re-read values to get status */
- ret = gpiotools_get_values(fd, &data);
+ ret = gpiotools_get_values(fd, &values);
if (ret < 0)
goto exit_close_error;
j = 0;
fprintf(stdout, "[");
- for (i = 0; i < nlines; i++) {
- fprintf(stdout, "%d: %d", lines[i], data.values[i]);
- if (i != (nlines - 1))
+ for (i = 0; i < num_lines; i++) {
+ fprintf(stdout, "%d: %d", lines[i],
+ gpiotools_test_bit(values.bits, i));
+ if (i != (num_lines - 1))
fprintf(stdout, ", ");
}
fprintf(stdout, "]\r");
ret = 0;
exit_close_error:
- gpiotools_release_linehandle(fd);
+ gpiotools_release_line(fd);
exit_error:
return ret;
}
const char *device_name = NULL;
unsigned int lines[GPIOHANDLES_MAX];
unsigned int loops = 0;
- int nlines;
+ int num_lines;
int c;
int i;
return -1;
}
- nlines = i;
+ num_lines = i;
- if (!device_name || !nlines) {
+ if (!device_name || !num_lines) {
print_usage();
return -1;
}
- return hammer_device(device_name, lines, nlines, loops);
+ return hammer_device(device_name, lines, num_lines, loops);
}
* such as "gpiochip0"
* @lines: An array desired lines, specified by offset
* index for the associated GPIO device.
- * @nline: The number of lines to request.
+ * @num_lines: The number of lines to request.
* @flag: The new flag for requsted gpio. Reference
* "linux/gpio.h" for the meaning of flag.
* @data: Default value will be set to gpio when flag is
* On failure return the errno.
*/
int gpiotools_request_linehandle(const char *device_name, unsigned int *lines,
- unsigned int nlines, unsigned int flag,
+ unsigned int num_lines, unsigned int flag,
struct gpiohandle_data *data,
const char *consumer_label)
{
goto exit_free_name;
}
- for (i = 0; i < nlines; i++)
+ for (i = 0; i < num_lines; i++)
req.lineoffsets[i] = lines[i];
req.flags = flag;
strcpy(req.consumer_label, consumer_label);
- req.lines = nlines;
+ req.lines = num_lines;
if (flag & GPIOHANDLE_REQUEST_OUTPUT)
memcpy(req.default_values, data, sizeof(req.default_values));
free(chrdev_name);
return ret < 0 ? ret : req.fd;
}
+
+/**
+ * gpiotools_request_line() - request gpio lines in a gpiochip
+ * @device_name: The name of gpiochip without prefix "/dev/",
+ * such as "gpiochip0"
+ * @lines: An array desired lines, specified by offset
+ * index for the associated GPIO device.
+ * @num_lines: The number of lines to request.
+ * @config: The new config for requested gpio. Reference
+ * "linux/gpio.h" for config details.
+ * @consumer: The name of consumer, such as "sysfs",
+ * "powerkey". This is useful for other users to
+ * know who is using.
+ *
+ * Request gpio lines through the ioctl provided by chardev. User
+ * could call gpiotools_set_values() and gpiotools_get_values() to
+ * read and write respectively through the returned fd. Call
+ * gpiotools_release_line() to release these lines after that.
+ *
+ * Return: On success return the fd;
+ * On failure return the errno.
+ */
+int gpiotools_request_line(const char *device_name, unsigned int *lines,
+ unsigned int num_lines,
+ struct gpio_v2_line_config *config,
+ const char *consumer)
+{
+ struct gpio_v2_line_request req;
+ char *chrdev_name;
+ int fd;
+ int i;
+ int ret;
+
+ ret = asprintf(&chrdev_name, "/dev/%s", device_name);
+ if (ret < 0)
+ return -ENOMEM;
+
+ fd = open(chrdev_name, 0);
+ if (fd == -1) {
+ ret = -errno;
+ fprintf(stderr, "Failed to open %s, %s\n",
+ chrdev_name, strerror(errno));
+ goto exit_free_name;
+ }
+
+ memset(&req, 0, sizeof(req));
+ for (i = 0; i < num_lines; i++)
+ req.offsets[i] = lines[i];
+
+ req.config = *config;
+ strcpy(req.consumer, consumer);
+ req.num_lines = num_lines;
+
+ ret = ioctl(fd, GPIO_V2_GET_LINE_IOCTL, &req);
+ if (ret == -1) {
+ ret = -errno;
+ fprintf(stderr, "Failed to issue %s (%d), %s\n",
+ "GPIO_GET_LINE_IOCTL", ret, strerror(errno));
+ }
+
+ if (close(fd) == -1)
+ perror("Failed to close GPIO character device file");
+exit_free_name:
+ free(chrdev_name);
+ return ret < 0 ? ret : req.fd;
+}
+
/**
* gpiotools_set_values(): Set the value of gpio(s)
* @fd: The fd returned by
- * gpiotools_request_linehandle().
- * @data: The array of values want to set.
+ * gpiotools_request_line().
+ * @values: The array of values want to set.
*
* Return: On success return 0;
* On failure return the errno.
*/
-int gpiotools_set_values(const int fd, struct gpiohandle_data *data)
+int gpiotools_set_values(const int fd, struct gpio_v2_line_values *values)
{
int ret;
- ret = ioctl(fd, GPIOHANDLE_SET_LINE_VALUES_IOCTL, data);
+ ret = ioctl(fd, GPIO_V2_LINE_SET_VALUES_IOCTL, values);
if (ret == -1) {
ret = -errno;
fprintf(stderr, "Failed to issue %s (%d), %s\n",
/**
* gpiotools_get_values(): Get the value of gpio(s)
* @fd: The fd returned by
- * gpiotools_request_linehandle().
- * @data: The array of values get from hardware.
+ * gpiotools_request_line().
+ * @values: The array of values get from hardware.
*
* Return: On success return 0;
* On failure return the errno.
*/
-int gpiotools_get_values(const int fd, struct gpiohandle_data *data)
+int gpiotools_get_values(const int fd, struct gpio_v2_line_values *values)
{
int ret;
- ret = ioctl(fd, GPIOHANDLE_GET_LINE_VALUES_IOCTL, data);
+ ret = ioctl(fd, GPIO_V2_LINE_GET_VALUES_IOCTL, values);
if (ret == -1) {
ret = -errno;
fprintf(stderr, "Failed to issue %s (%d), %s\n",
return ret;
}
+/**
+ * gpiotools_release_line(): Release the line(s) of gpiochip
+ * @fd: The fd returned by
+ * gpiotools_request_line().
+ *
+ * Return: On success return 0;
+ * On failure return the errno.
+ */
+int gpiotools_release_line(const int fd)
+{
+ int ret;
+
+ ret = close(fd);
+ if (ret == -1) {
+ perror("Failed to close GPIO LINE device file");
+ ret = -errno;
+ }
+
+ return ret;
+}
+
/**
* gpiotools_get(): Get value from specific line
* @device_name: The name of gpiochip without prefix "/dev/",
*/
int gpiotools_get(const char *device_name, unsigned int line)
{
- struct gpiohandle_data data;
+ int ret;
+ unsigned int value;
unsigned int lines[] = {line};
- gpiotools_gets(device_name, lines, 1, &data);
- return data.values[0];
+ ret = gpiotools_gets(device_name, lines, 1, &value);
+ if (ret)
+ return ret;
+ return value;
}
* such as "gpiochip0".
* @lines: An array desired lines, specified by offset
* index for the associated GPIO device.
- * @nline: The number of lines to request.
- * @data: The array of values get from gpiochip.
+ * @num_lines: The number of lines to request.
+ * @values: The array of values get from gpiochip.
*
* Return: On success return 0;
* On failure return the errno.
*/
int gpiotools_gets(const char *device_name, unsigned int *lines,
- unsigned int nlines, struct gpiohandle_data *data)
+ unsigned int num_lines, unsigned int *values)
{
- int fd;
+ int fd, i;
int ret;
int ret_close;
+ struct gpio_v2_line_config config;
+ struct gpio_v2_line_values lv;
- ret = gpiotools_request_linehandle(device_name, lines, nlines,
- GPIOHANDLE_REQUEST_INPUT, data,
- CONSUMER);
+ memset(&config, 0, sizeof(config));
+ config.flags = GPIO_V2_LINE_FLAG_INPUT;
+ ret = gpiotools_request_line(device_name, lines, num_lines,
+ &config, CONSUMER);
if (ret < 0)
return ret;
fd = ret;
- ret = gpiotools_get_values(fd, data);
- ret_close = gpiotools_release_linehandle(fd);
+ for (i = 0; i < num_lines; i++)
+ gpiotools_set_bit(&lv.mask, i);
+ ret = gpiotools_get_values(fd, &lv);
+ if (!ret)
+ for (i = 0; i < num_lines; i++)
+ values[i] = gpiotools_test_bit(lv.bits, i);
+ ret_close = gpiotools_release_line(fd);
return ret < 0 ? ret : ret_close;
}
int gpiotools_set(const char *device_name, unsigned int line,
unsigned int value)
{
- struct gpiohandle_data data;
unsigned int lines[] = {line};
- data.values[0] = value;
- return gpiotools_sets(device_name, lines, 1, &data);
+ return gpiotools_sets(device_name, lines, 1, &value);
}
/**
* such as "gpiochip0".
* @lines: An array desired lines, specified by offset
* index for the associated GPIO device.
- * @nline: The number of lines to request.
- * @data: The array of values set to gpiochip, must be
+ * @num_lines: The number of lines to request.
+ * @value: The array of values set to gpiochip, must be
* 0(low) or 1(high).
*
* Return: On success return 0;
* On failure return the errno.
*/
int gpiotools_sets(const char *device_name, unsigned int *lines,
- unsigned int nlines, struct gpiohandle_data *data)
+ unsigned int num_lines, unsigned int *values)
{
- int ret;
+ int ret, i;
+ struct gpio_v2_line_config config;
- ret = gpiotools_request_linehandle(device_name, lines, nlines,
- GPIOHANDLE_REQUEST_OUTPUT, data,
- CONSUMER);
+ memset(&config, 0, sizeof(config));
+ config.flags = GPIO_V2_LINE_FLAG_OUTPUT;
+ config.num_attrs = 1;
+ config.attrs[0].attr.id = GPIO_V2_LINE_ATTR_ID_OUTPUT_VALUES;
+ for (i = 0; i < num_lines; i++) {
+ gpiotools_set_bit(&config.attrs[0].mask, i);
+ gpiotools_assign_bit(&config.attrs[0].attr.values,
+ i, values[i]);
+ }
+ ret = gpiotools_request_line(device_name, lines, num_lines,
+ &config, CONSUMER);
if (ret < 0)
return ret;
- return gpiotools_release_linehandle(ret);
+ return gpiotools_release_line(ret);
}
#ifndef _GPIO_UTILS_H_
#define _GPIO_UTILS_H_
+#include <stdbool.h>
#include <string.h>
+#include <linux/types.h>
#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
}
int gpiotools_request_linehandle(const char *device_name, unsigned int *lines,
- unsigned int nlines, unsigned int flag,
+ unsigned int num_lines, unsigned int flag,
struct gpiohandle_data *data,
const char *consumer_label);
-int gpiotools_set_values(const int fd, struct gpiohandle_data *data);
-int gpiotools_get_values(const int fd, struct gpiohandle_data *data);
int gpiotools_release_linehandle(const int fd);
+int gpiotools_request_line(const char *device_name,
+ unsigned int *lines,
+ unsigned int num_lines,
+ struct gpio_v2_line_config *config,
+ const char *consumer);
+int gpiotools_set_values(const int fd, struct gpio_v2_line_values *values);
+int gpiotools_get_values(const int fd, struct gpio_v2_line_values *values);
+int gpiotools_release_line(const int fd);
+
int gpiotools_get(const char *device_name, unsigned int line);
int gpiotools_gets(const char *device_name, unsigned int *lines,
- unsigned int nlines, struct gpiohandle_data *data);
+ unsigned int num_lines, unsigned int *values);
int gpiotools_set(const char *device_name, unsigned int line,
unsigned int value);
int gpiotools_sets(const char *device_name, unsigned int *lines,
- unsigned int nlines, struct gpiohandle_data *data);
+ unsigned int num_lines, unsigned int *values);
+
+/* helper functions for gpio_v2_line_values bits */
+static inline void gpiotools_set_bit(__u64 *b, int n)
+{
+ *b |= _BITULL(n);
+}
+
+static inline void gpiotools_change_bit(__u64 *b, int n)
+{
+ *b ^= _BITULL(n);
+}
+
+static inline void gpiotools_clear_bit(__u64 *b, int n)
+{
+ *b &= ~_BITULL(n);
+}
+
+static inline int gpiotools_test_bit(__u64 b, int n)
+{
+ return !!(b & _BITULL(n));
+}
+
+static inline void gpiotools_assign_bit(__u64 *b, int n, bool value)
+{
+ if (value)
+ gpiotools_set_bit(b, n);
+ else
+ gpiotools_clear_bit(b, n);
+}
#endif /* _GPIO_UTILS_H_ */
int main(int argc, char **argv)
{
- struct gpioline_info_changed chg;
- struct gpioline_info req;
+ struct gpio_v2_line_info_changed chg;
+ struct gpio_v2_line_info req;
struct pollfd pfd;
int fd, i, j, ret;
char *event, *end;
for (i = 0, j = 2; i < argc - 2; i++, j++) {
memset(&req, 0, sizeof(req));
- req.line_offset = strtoul(argv[j], &end, 0);
+ req.offset = strtoul(argv[j], &end, 0);
if (*end != '\0')
goto err_usage;
- ret = ioctl(fd, GPIO_GET_LINEINFO_WATCH_IOCTL, &req);
+ ret = ioctl(fd, GPIO_V2_GET_LINEINFO_WATCH_IOCTL, &req);
if (ret) {
perror("unable to set up line watch");
return EXIT_FAILURE;
}
switch (chg.event_type) {
- case GPIOLINE_CHANGED_REQUESTED:
+ case GPIO_V2_LINE_CHANGED_REQUESTED:
event = "requested";
break;
- case GPIOLINE_CHANGED_RELEASED:
+ case GPIO_V2_LINE_CHANGED_RELEASED:
event = "released";
break;
- case GPIOLINE_CHANGED_CONFIG:
+ case GPIO_V2_LINE_CHANGED_CONFIG:
event = "config changed";
break;
default:
}
printf("line %u: %s at %llu\n",
- chg.info.line_offset, event, chg.timestamp);
+ chg.info.offset, event, chg.timestamp_ns);
}
}
struct gpio_flag {
char *name;
- unsigned long mask;
+ unsigned long long mask;
};
struct gpio_flag flagnames[] = {
{
- .name = "kernel",
- .mask = GPIOLINE_FLAG_KERNEL,
+ .name = "used",
+ .mask = GPIO_V2_LINE_FLAG_USED,
+ },
+ {
+ .name = "input",
+ .mask = GPIO_V2_LINE_FLAG_INPUT,
},
{
.name = "output",
- .mask = GPIOLINE_FLAG_IS_OUT,
+ .mask = GPIO_V2_LINE_FLAG_OUTPUT,
},
{
.name = "active-low",
- .mask = GPIOLINE_FLAG_ACTIVE_LOW,
+ .mask = GPIO_V2_LINE_FLAG_ACTIVE_LOW,
},
{
.name = "open-drain",
- .mask = GPIOLINE_FLAG_OPEN_DRAIN,
+ .mask = GPIO_V2_LINE_FLAG_OPEN_DRAIN,
},
{
.name = "open-source",
- .mask = GPIOLINE_FLAG_OPEN_SOURCE,
+ .mask = GPIO_V2_LINE_FLAG_OPEN_SOURCE,
},
{
.name = "pull-up",
- .mask = GPIOLINE_FLAG_BIAS_PULL_UP,
+ .mask = GPIO_V2_LINE_FLAG_BIAS_PULL_UP,
},
{
.name = "pull-down",
- .mask = GPIOLINE_FLAG_BIAS_PULL_DOWN,
+ .mask = GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN,
},
{
.name = "bias-disabled",
- .mask = GPIOLINE_FLAG_BIAS_DISABLE,
+ .mask = GPIO_V2_LINE_FLAG_BIAS_DISABLED,
},
};
-void print_flags(unsigned long flags)
+static void print_attributes(struct gpio_v2_line_info *info)
{
int i;
- int printed = 0;
+ const char *field_format = "%s";
for (i = 0; i < ARRAY_SIZE(flagnames); i++) {
- if (flags & flagnames[i].mask) {
- if (printed)
- fprintf(stdout, " ");
- fprintf(stdout, "%s", flagnames[i].name);
- printed++;
+ if (info->flags & flagnames[i].mask) {
+ fprintf(stdout, field_format, flagnames[i].name);
+ field_format = ", %s";
}
}
+
+ if ((info->flags & GPIO_V2_LINE_FLAG_EDGE_RISING) &&
+ (info->flags & GPIO_V2_LINE_FLAG_EDGE_FALLING))
+ fprintf(stdout, field_format, "both-edges");
+ else if (info->flags & GPIO_V2_LINE_FLAG_EDGE_RISING)
+ fprintf(stdout, field_format, "rising-edge");
+ else if (info->flags & GPIO_V2_LINE_FLAG_EDGE_FALLING)
+ fprintf(stdout, field_format, "falling-edge");
+
+ for (i = 0; i < info->num_attrs; i++) {
+ if (info->attrs[i].id == GPIO_V2_LINE_ATTR_ID_DEBOUNCE)
+ fprintf(stdout, ", debounce_period=%dusec",
+ info->attrs[0].debounce_period_us);
+ }
}
int list_device(const char *device_name)
/* Loop over the lines and print info */
for (i = 0; i < cinfo.lines; i++) {
- struct gpioline_info linfo;
+ struct gpio_v2_line_info linfo;
memset(&linfo, 0, sizeof(linfo));
- linfo.line_offset = i;
+ linfo.offset = i;
- ret = ioctl(fd, GPIO_GET_LINEINFO_IOCTL, &linfo);
+ ret = ioctl(fd, GPIO_V2_GET_LINEINFO_IOCTL, &linfo);
if (ret == -1) {
ret = -errno;
perror("Failed to issue LINEINFO IOCTL\n");
goto exit_close_error;
}
- fprintf(stdout, "\tline %2d:", linfo.line_offset);
+ fprintf(stdout, "\tline %2d:", linfo.offset);
if (linfo.name[0])
fprintf(stdout, " \"%s\"", linfo.name);
else
fprintf(stdout, " unused");
if (linfo.flags) {
fprintf(stdout, " [");
- print_flags(linfo.flags);
+ print_attributes(&linfo);
fprintf(stdout, "]");
}
fprintf(stdout, "\n");
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _STATIC_CALL_TYPES_H
+#define _STATIC_CALL_TYPES_H
+
+#include <linux/types.h>
+#include <linux/stringify.h>
+
+#define STATIC_CALL_KEY_PREFIX __SCK__
+#define STATIC_CALL_KEY_PREFIX_STR __stringify(STATIC_CALL_KEY_PREFIX)
+#define STATIC_CALL_KEY_PREFIX_LEN (sizeof(STATIC_CALL_KEY_PREFIX_STR) - 1)
+#define STATIC_CALL_KEY(name) __PASTE(STATIC_CALL_KEY_PREFIX, name)
+
+#define STATIC_CALL_TRAMP_PREFIX __SCT__
+#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX)
+#define STATIC_CALL_TRAMP_PREFIX_LEN (sizeof(STATIC_CALL_TRAMP_PREFIX_STR) - 1)
+#define STATIC_CALL_TRAMP(name) __PASTE(STATIC_CALL_TRAMP_PREFIX, name)
+#define STATIC_CALL_TRAMP_STR(name) __stringify(STATIC_CALL_TRAMP(name))
+
+/*
+ * Flags in the low bits of static_call_site::key.
+ */
+#define STATIC_CALL_SITE_TAIL 1UL /* tail call */
+#define STATIC_CALL_SITE_INIT 2UL /* init section */
+#define STATIC_CALL_SITE_FLAGS 3UL
+
+/*
+ * The static call site table needs to be created by external tooling (objtool
+ * or a compiler plugin).
+ */
+struct static_call_site {
+ s32 addr;
+ s32 key;
+};
+
+#endif /* _STATIC_CALL_TYPES_H */
#define __NR_umount2 39
__SYSCALL(__NR_umount2, sys_umount)
#define __NR_mount 40
-__SC_COMP(__NR_mount, sys_mount, compat_sys_mount)
+__SYSCALL(__NR_mount, sys_mount)
#define __NR_pivot_root 41
__SYSCALL(__NR_pivot_root, sys_pivot_root)
#define __NR_write 64
__SYSCALL(__NR_write, sys_write)
#define __NR_readv 65
-__SC_COMP(__NR_readv, sys_readv, compat_sys_readv)
+__SC_COMP(__NR_readv, sys_readv, sys_readv)
#define __NR_writev 66
-__SC_COMP(__NR_writev, sys_writev, compat_sys_writev)
+__SC_COMP(__NR_writev, sys_writev, sys_writev)
#define __NR_pread64 67
__SC_COMP(__NR_pread64, sys_pread64, compat_sys_pread64)
#define __NR_pwrite64 68
/* fs/splice.c */
#define __NR_vmsplice 75
-__SC_COMP(__NR_vmsplice, sys_vmsplice, compat_sys_vmsplice)
+__SYSCALL(__NR_vmsplice, sys_vmsplice)
#define __NR_splice 76
__SYSCALL(__NR_splice, sys_splice)
#define __NR_tee 77
#define __NR_sendmmsg 269
__SC_COMP(__NR_sendmmsg, sys_sendmmsg, compat_sys_sendmmsg)
#define __NR_process_vm_readv 270
-__SC_COMP(__NR_process_vm_readv, sys_process_vm_readv, \
- compat_sys_process_vm_readv)
+__SYSCALL(__NR_process_vm_readv, sys_process_vm_readv)
#define __NR_process_vm_writev 271
-__SC_COMP(__NR_process_vm_writev, sys_process_vm_writev, \
- compat_sys_process_vm_writev)
+__SYSCALL(__NR_process_vm_writev, sys_process_vm_writev)
#define __NR_kcmp 272
__SYSCALL(__NR_kcmp, sys_kcmp)
#define __NR_finit_module 273
err = -EIO;
goto err_out;
}
+ if (magic == __bswap_16(BTF_MAGIC)) {
+ /* non-native endian raw BTF */
+ pr_warn("non-native BTF endianness is not supported\n");
+ err = -LIBBPF_ERRNO__ENDIAN;
+ goto err_out;
+ }
if (magic != BTF_MAGIC) {
/* definitely not a raw BTF */
err = -EPROTO;
BPF_XDP_DEVMAP),
BPF_EAPROG_SEC("xdp_cpumap/", BPF_PROG_TYPE_XDP,
BPF_XDP_CPUMAP),
- BPF_EAPROG_SEC("xdp", BPF_PROG_TYPE_XDP,
+ BPF_APROG_SEC("xdp", BPF_PROG_TYPE_XDP,
BPF_XDP),
BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),
C Self R W RMW Self R W DR DW RMW SV
-- ---- - - --- ---- - - -- -- --- --
-Store, e.g., WRITE_ONCE() Y Y
-Load, e.g., READ_ONCE() Y Y Y Y
-Unsuccessful RMW operation Y Y Y Y
+Relaxed store Y Y
+Relaxed load Y Y Y Y
+Relaxed RMW operation Y Y Y Y
rcu_dereference() Y Y Y Y
Successful *_acquire() R Y Y Y Y Y Y
Successful *_release() C Y Y Y W Y
smp_mb__after_atomic() CP a a Y Y Y Y Y Y
-Key: C: Ordering is cumulative
- P: Ordering propagates
- R: Read, for example, READ_ONCE(), or read portion of RMW
- W: Write, for example, WRITE_ONCE(), or write portion of RMW
- Y: Provides ordering
- a: Provides ordering given intervening RMW atomic operation
- DR: Dependent read (address dependency)
- DW: Dependent write (address, data, or control dependency)
- RMW: Atomic read-modify-write operation
- SELF: Orders self, as opposed to accesses before and/or after
- SV: Orders later accesses to the same variable
+Key: Relaxed: A relaxed operation is either READ_ONCE(), WRITE_ONCE(),
+ a *_relaxed() RMW operation, an unsuccessful RMW
+ operation, a non-value-returning RMW operation such
+ as atomic_inc(), or one of the atomic*_read() and
+ atomic*_set() family of operations.
+ C: Ordering is cumulative
+ P: Ordering propagates
+ R: Read, for example, READ_ONCE(), or read portion of RMW
+ W: Write, for example, WRITE_ONCE(), or write portion of RMW
+ Y: Provides ordering
+ a: Provides ordering given intervening RMW atomic operation
+ DR: Dependent read (address dependency)
+ DW: Dependent write (address, data, or control dependency)
+ RMW: Atomic read-modify-write operation
+ SELF: Orders self, as opposed to accesses before and/or after
+ SV: Orders later accesses to the same variable
--- /dev/null
+Linux-Kernel Memory Model Litmus Tests
+======================================
+
+This file describes the LKMM litmus-test format by example, describes
+some tricks and traps, and finally outlines LKMM's limitations. Earlier
+versions of this material appeared in a number of LWN articles, including:
+
+https://lwn.net/Articles/720550/
+ A formal kernel memory-ordering model (part 2)
+https://lwn.net/Articles/608550/
+ Axiomatic validation of memory barriers and atomic instructions
+https://lwn.net/Articles/470681/
+ Validating Memory Barriers and Atomic Instructions
+
+This document presents information in decreasing order of applicability,
+so that, where possible, the information that has proven more commonly
+useful is shown near the beginning.
+
+For information on installing LKMM, including the underlying "herd7"
+tool, please see tools/memory-model/README.
+
+
+Copy-Pasta
+==========
+
+As with other software, it is often better (if less macho) to adapt an
+existing litmus test than it is to create one from scratch. A number
+of litmus tests may be found in the kernel source tree:
+
+ tools/memory-model/litmus-tests/
+ Documentation/litmus-tests/
+
+Several thousand more example litmus tests are available on github
+and kernel.org:
+
+ https://github.com/paulmckrcu/litmus
+ https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/perfbook.git/tree/CodeSamples/formal/herd
+ https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/perfbook.git/tree/CodeSamples/formal/litmus
+
+The -l and -L arguments to "git grep" can be quite helpful in identifying
+existing litmus tests that are similar to the one you need. But even if
+you start with an existing litmus test, it is still helpful to have a
+good understanding of the litmus-test format.
+
+
+Examples and Format
+===================
+
+This section describes the overall format of litmus tests, starting
+with a small example of the message-passing pattern and moving on to
+more complex examples that illustrate explicit initialization and LKMM's
+minimalistic set of flow-control statements.
+
+
+Message-Passing Example
+-----------------------
+
+This section gives an overview of the format of a litmus test using an
+example based on the common message-passing use case. This use case
+appears often in the Linux kernel. For example, a flag (modeled by "y"
+below) indicates that a buffer (modeled by "x" below) is now completely
+filled in and ready for use. It would be very bad if the consumer saw the
+flag set, but, due to memory misordering, saw old values in the buffer.
+
+This example asks whether smp_store_release() and smp_load_acquire()
+suffices to avoid this bad outcome:
+
+ 1 C MP+pooncerelease+poacquireonce
+ 2
+ 3 {}
+ 4
+ 5 P0(int *x, int *y)
+ 6 {
+ 7 WRITE_ONCE(*x, 1);
+ 8 smp_store_release(y, 1);
+ 9 }
+10
+11 P1(int *x, int *y)
+12 {
+13 int r0;
+14 int r1;
+15
+16 r0 = smp_load_acquire(y);
+17 r1 = READ_ONCE(*x);
+18 }
+19
+20 exists (1:r0=1 /\ 1:r1=0)
+
+Line 1 starts with "C", which identifies this file as being in the
+LKMM C-language format (which, as we will see, is a small fragment
+of the full C language). The remainder of line 1 is the name of
+the test, which by convention is the filename with the ".litmus"
+suffix stripped. In this case, the actual test may be found in
+tools/memory-model/litmus-tests/MP+pooncerelease+poacquireonce.litmus
+in the Linux-kernel source tree.
+
+Mechanically generated litmus tests will often have an optional
+double-quoted comment string on the second line. Such strings are ignored
+when running the test. Yes, you can add your own comments to litmus
+tests, but this is a bit involved due to the use of multiple parsers.
+For now, you can use C-language comments in the C code, and these comments
+may be in either the "/* */" or the "//" style. A later section will
+cover the full litmus-test commenting story.
+
+Line 3 is the initialization section. Because the default initialization
+to zero suffices for this test, the "{}" syntax is used, which mean the
+initialization section is empty. Litmus tests requiring non-default
+initialization must have non-empty initialization sections, as in the
+example that will be presented later in this document.
+
+Lines 5-9 show the first process and lines 11-18 the second process. Each
+process corresponds to a Linux-kernel task (or kthread, workqueue, thread,
+and so on; LKMM discussions often use these terms interchangeably).
+The name of the first process is "P0" and that of the second "P1".
+You can name your processes anything you like as long as the names consist
+of a single "P" followed by a number, and as long as the numbers are
+consecutive starting with zero. This can actually be quite helpful,
+for example, a .litmus file matching "^P1(" but not matching "^P2("
+must contain a two-process litmus test.
+
+The argument list for each function are pointers to the global variables
+used by that function. Unlike normal C-language function parameters, the
+names are significant. The fact that both P0() and P1() have a formal
+parameter named "x" means that these two processes are working with the
+same global variable, also named "x". So the "int *x, int *y" on P0()
+and P1() mean that both processes are working with two shared global
+variables, "x" and "y". Global variables are always passed to processes
+by reference, hence "P0(int *x, int *y)", but *never* "P0(int x, int y)".
+
+P0() has no local variables, but P1() has two of them named "r0" and "r1".
+These names may be freely chosen, but for historical reasons stemming from
+other litmus-test formats, it is conventional to use names consisting of
+"r" followed by a number as shown here. A common bug in litmus tests
+is forgetting to add a global variable to a process's parameter list.
+This will sometimes result in an error message, but can also cause the
+intended global to instead be silently treated as an undeclared local
+variable.
+
+Each process's code is similar to Linux-kernel C, as can be seen on lines
+7-8 and 13-17. This code may use many of the Linux kernel's atomic
+operations, some of its exclusive-lock functions, and some of its RCU
+and SRCU functions. An approximate list of the currently supported
+functions may be found in the linux-kernel.def file.
+
+The P0() process does "WRITE_ONCE(*x, 1)" on line 7. Because "x" is a
+pointer in P0()'s parameter list, this does an unordered store to global
+variable "x". Line 8 does "smp_store_release(y, 1)", and because "y"
+is also in P0()'s parameter list, this does a release store to global
+variable "y".
+
+The P1() process declares two local variables on lines 13 and 14.
+Line 16 does "r0 = smp_load_acquire(y)" which does an acquire load
+from global variable "y" into local variable "r0". Line 17 does a
+"r1 = READ_ONCE(*x)", which does an unordered load from "*x" into local
+variable "r1". Both "x" and "y" are in P1()'s parameter list, so both
+reference the same global variables that are used by P0().
+
+Line 20 is the "exists" assertion expression to evaluate the final state.
+This final state is evaluated after the dust has settled: both processes
+have completed and all of their memory references and memory barriers
+have propagated to all parts of the system. The references to the local
+variables "r0" and "r1" in line 24 must be prefixed with "1:" to specify
+which process they are local to.
+
+Note that the assertion expression is written in the litmus-test
+language rather than in C. For example, single "=" is an equality
+operator rather than an assignment. The "/\" character combination means
+"and". Similarly, "\/" stands for "or". Both of these are ASCII-art
+representations of the corresponding mathematical symbols. Finally,
+"~" stands for "logical not", which is "!" in C, and not to be confused
+with the C-language "~" operator which instead stands for "bitwise not".
+Parentheses may be used to override precedence.
+
+The "exists" assertion on line 20 is satisfied if the consumer sees the
+flag ("y") set but the buffer ("x") as not yet filled in, that is, if P1()
+loaded a value from "x" that was equal to 1 but loaded a value from "y"
+that was still equal to zero.
+
+This example can be checked by running the following command, which
+absolutely must be run from the tools/memory-model directory and from
+this directory only:
+
+herd7 -conf linux-kernel.cfg litmus-tests/MP+pooncerelease+poacquireonce.litmus
+
+The output is the result of something similar to a full state-space
+search, and is as follows:
+
+ 1 Test MP+pooncerelease+poacquireonce Allowed
+ 2 States 3
+ 3 1:r0=0; 1:r1=0;
+ 4 1:r0=0; 1:r1=1;
+ 5 1:r0=1; 1:r1=1;
+ 6 No
+ 7 Witnesses
+ 8 Positive: 0 Negative: 3
+ 9 Condition exists (1:r0=1 /\ 1:r1=0)
+10 Observation MP+pooncerelease+poacquireonce Never 0 3
+11 Time MP+pooncerelease+poacquireonce 0.00
+12 Hash=579aaa14d8c35a39429b02e698241d09
+
+The most pertinent line is line 10, which contains "Never 0 3", which
+indicates that the bad result flagged by the "exists" clause never
+happens. This line might instead say "Sometimes" to indicate that the
+bad result happened in some but not all executions, or it might say
+"Always" to indicate that the bad result happened in all executions.
+(The herd7 tool doesn't judge, so it is only an LKMM convention that the
+"exists" clause indicates a bad result. To see this, invert the "exists"
+clause's condition and run the test.) The numbers ("0 3") at the end
+of this line indicate the number of end states satisfying the "exists"
+clause (0) and the number not not satisfying that clause (3).
+
+Another important part of this output is shown in lines 2-5, repeated here:
+
+ 2 States 3
+ 3 1:r0=0; 1:r1=0;
+ 4 1:r0=0; 1:r1=1;
+ 5 1:r0=1; 1:r1=1;
+
+Line 2 gives the total number of end states, and each of lines 3-5 list
+one of these states, with the first ("1:r0=0; 1:r1=0;") indicating that
+both of P1()'s loads returned the value "0". As expected, given the
+"Never" on line 10, the state flagged by the "exists" clause is not
+listed. This full list of states can be helpful when debugging a new
+litmus test.
+
+The rest of the output is not normally needed, either due to irrelevance
+or due to being redundant with the lines discussed above. However, the
+following paragraph lists them for the benefit of readers possessed of
+an insatiable curiosity. Other readers should feel free to skip ahead.
+
+Line 1 echos the test name, along with the "Test" and "Allowed". Line 6's
+"No" says that the "exists" clause was not satisfied by any execution,
+and as such it has the same meaning as line 10's "Never". Line 7 is a
+lead-in to line 8's "Positive: 0 Negative: 3", which lists the number
+of end states satisfying and not satisfying the "exists" clause, just
+like the two numbers at the end of line 10. Line 9 repeats the "exists"
+clause so that you don't have to look it up in the litmus-test file.
+The number at the end of line 11 (which begins with "Time") gives the
+time in seconds required to analyze the litmus test. Small tests such
+as this one complete in a few milliseconds, so "0.00" is quite common.
+Line 12 gives a hash of the contents for the litmus-test file, and is used
+by tooling that manages litmus tests and their output. This tooling is
+used by people modifying LKMM itself, and among other things lets such
+people know which of the several thousand relevant litmus tests were
+affected by a given change to LKMM.
+
+
+Initialization
+--------------
+
+The previous example relied on the default zero initialization for
+"x" and "y", but a similar litmus test could instead initialize them
+to some other value:
+
+ 1 C MP+pooncerelease+poacquireonce
+ 2
+ 3 {
+ 4 x=42;
+ 5 y=42;
+ 6 }
+ 7
+ 8 P0(int *x, int *y)
+ 9 {
+10 WRITE_ONCE(*x, 1);
+11 smp_store_release(y, 1);
+12 }
+13
+14 P1(int *x, int *y)
+15 {
+16 int r0;
+17 int r1;
+18
+19 r0 = smp_load_acquire(y);
+20 r1 = READ_ONCE(*x);
+21 }
+22
+23 exists (1:r0=1 /\ 1:r1=42)
+
+Lines 3-6 now initialize both "x" and "y" to the value 42. This also
+means that the "exists" clause on line 23 must change "1:r1=0" to
+"1:r1=42".
+
+Running the test gives the same overall result as before, but with the
+value 42 appearing in place of the value zero:
+
+ 1 Test MP+pooncerelease+poacquireonce Allowed
+ 2 States 3
+ 3 1:r0=1; 1:r1=1;
+ 4 1:r0=42; 1:r1=1;
+ 5 1:r0=42; 1:r1=42;
+ 6 No
+ 7 Witnesses
+ 8 Positive: 0 Negative: 3
+ 9 Condition exists (1:r0=1 /\ 1:r1=42)
+10 Observation MP+pooncerelease+poacquireonce Never 0 3
+11 Time MP+pooncerelease+poacquireonce 0.02
+12 Hash=ab9a9b7940a75a792266be279a980156
+
+It is tempting to avoid the open-coded repetitions of the value "42"
+by defining another global variable "initval=42" and replacing all
+occurrences of "42" with "initval". This will not, repeat *not*,
+initialize "x" and "y" to 42, but instead to the address of "initval"
+(try it!). See the section below on linked lists to learn more about
+why this approach to initialization can be useful.
+
+
+Control Structures
+------------------
+
+LKMM supports the C-language "if" statement, which allows modeling of
+conditional branches. In LKMM, conditional branches can affect ordering,
+but only if you are *very* careful (compilers are surprisingly able
+to optimize away conditional branches). The following example shows
+the "load buffering" (LB) use case that is used in the Linux kernel to
+synchronize between ring-buffer producers and consumers. In the example
+below, P0() is one side checking to see if an operation may proceed and
+P1() is the other side completing its update.
+
+ 1 C LB+fencembonceonce+ctrlonceonce
+ 2
+ 3 {}
+ 4
+ 5 P0(int *x, int *y)
+ 6 {
+ 7 int r0;
+ 8
+ 9 r0 = READ_ONCE(*x);
+10 if (r0)
+11 WRITE_ONCE(*y, 1);
+12 }
+13
+14 P1(int *x, int *y)
+15 {
+16 int r0;
+17
+18 r0 = READ_ONCE(*y);
+19 smp_mb();
+20 WRITE_ONCE(*x, 1);
+21 }
+22
+23 exists (0:r0=1 /\ 1:r0=1)
+
+P1()'s "if" statement on line 10 works as expected, so that line 11 is
+executed only if line 9 loads a non-zero value from "x". Because P1()'s
+write of "1" to "x" happens only after P1()'s read from "y", one would
+hope that the "exists" clause cannot be satisfied. LKMM agrees:
+
+ 1 Test LB+fencembonceonce+ctrlonceonce Allowed
+ 2 States 2
+ 3 0:r0=0; 1:r0=0;
+ 4 0:r0=1; 1:r0=0;
+ 5 No
+ 6 Witnesses
+ 7 Positive: 0 Negative: 2
+ 8 Condition exists (0:r0=1 /\ 1:r0=1)
+ 9 Observation LB+fencembonceonce+ctrlonceonce Never 0 2
+10 Time LB+fencembonceonce+ctrlonceonce 0.00
+11 Hash=e5260556f6de495fd39b556d1b831c3b
+
+However, there is no "while" statement due to the fact that full
+state-space search has some difficulty with iteration. However, there
+are tricks that may be used to handle some special cases, which are
+discussed below. In addition, loop-unrolling tricks may be applied,
+albeit sparingly.
+
+
+Tricks and Traps
+================
+
+This section covers extracting debug output from herd7, emulating
+spin loops, handling trivial linked lists, adding comments to litmus tests,
+emulating call_rcu(), and finally tricks to improve herd7 performance
+in order to better handle large litmus tests.
+
+
+Debug Output
+------------
+
+By default, the herd7 state output includes all variables mentioned
+in the "exists" clause. But sometimes debugging efforts are greatly
+aided by the values of other variables. Consider this litmus test
+(tools/memory-order/litmus-tests/SB+rfionceonce-poonceonces.litmus but
+slightly modified), which probes an obscure corner of hardware memory
+ordering:
+
+ 1 C SB+rfionceonce-poonceonces
+ 2
+ 3 {}
+ 4
+ 5 P0(int *x, int *y)
+ 6 {
+ 7 int r1;
+ 8 int r2;
+ 9
+10 WRITE_ONCE(*x, 1);
+11 r1 = READ_ONCE(*x);
+12 r2 = READ_ONCE(*y);
+13 }
+14
+15 P1(int *x, int *y)
+16 {
+17 int r3;
+18 int r4;
+19
+20 WRITE_ONCE(*y, 1);
+21 r3 = READ_ONCE(*y);
+22 r4 = READ_ONCE(*x);
+23 }
+24
+25 exists (0:r2=0 /\ 1:r4=0)
+
+The herd7 output is as follows:
+
+ 1 Test SB+rfionceonce-poonceonces Allowed
+ 2 States 4
+ 3 0:r2=0; 1:r4=0;
+ 4 0:r2=0; 1:r4=1;
+ 5 0:r2=1; 1:r4=0;
+ 6 0:r2=1; 1:r4=1;
+ 7 Ok
+ 8 Witnesses
+ 9 Positive: 1 Negative: 3
+10 Condition exists (0:r2=0 /\ 1:r4=0)
+11 Observation SB+rfionceonce-poonceonces Sometimes 1 3
+12 Time SB+rfionceonce-poonceonces 0.01
+13 Hash=c7f30fe0faebb7d565405d55b7318ada
+
+(This output indicates that CPUs are permitted to "snoop their own
+store buffers", which all of Linux's CPU families other than s390 will
+happily do. Such snooping results in disagreement among CPUs on the
+order of stores from different CPUs, which is rarely an issue.)
+
+But the herd7 output shows only the two variables mentioned in the
+"exists" clause. Someone modifying this test might wish to know the
+values of "x", "y", "0:r1", and "0:r3" as well. The "locations"
+statement on line 25 shows how to cause herd7 to display additional
+variables:
+
+ 1 C SB+rfionceonce-poonceonces
+ 2
+ 3 {}
+ 4
+ 5 P0(int *x, int *y)
+ 6 {
+ 7 int r1;
+ 8 int r2;
+ 9
+10 WRITE_ONCE(*x, 1);
+11 r1 = READ_ONCE(*x);
+12 r2 = READ_ONCE(*y);
+13 }
+14
+15 P1(int *x, int *y)
+16 {
+17 int r3;
+18 int r4;
+19
+20 WRITE_ONCE(*y, 1);
+21 r3 = READ_ONCE(*y);
+22 r4 = READ_ONCE(*x);
+23 }
+24
+25 locations [0:r1; 1:r3; x; y]
+26 exists (0:r2=0 /\ 1:r4=0)
+
+The herd7 output then displays the values of all the variables:
+
+ 1 Test SB+rfionceonce-poonceonces Allowed
+ 2 States 4
+ 3 0:r1=1; 0:r2=0; 1:r3=1; 1:r4=0; x=1; y=1;
+ 4 0:r1=1; 0:r2=0; 1:r3=1; 1:r4=1; x=1; y=1;
+ 5 0:r1=1; 0:r2=1; 1:r3=1; 1:r4=0; x=1; y=1;
+ 6 0:r1=1; 0:r2=1; 1:r3=1; 1:r4=1; x=1; y=1;
+ 7 Ok
+ 8 Witnesses
+ 9 Positive: 1 Negative: 3
+10 Condition exists (0:r2=0 /\ 1:r4=0)
+11 Observation SB+rfionceonce-poonceonces Sometimes 1 3
+12 Time SB+rfionceonce-poonceonces 0.01
+13 Hash=40de8418c4b395388f6501cafd1ed38d
+
+What if you would like to know the value of a particular global variable
+at some particular point in a given process's execution? One approach
+is to use a READ_ONCE() to load that global variable into a new local
+variable, then add that local variable to the "locations" clause.
+But be careful: In some litmus tests, adding a READ_ONCE() will change
+the outcome! For one example, please see the C-READ_ONCE.litmus and
+C-READ_ONCE-omitted.litmus tests located here:
+
+ https://github.com/paulmckrcu/litmus/blob/master/manual/kernel/
+
+
+Spin Loops
+----------
+
+The analysis carried out by herd7 explores full state space, which is
+at best of exponential time complexity. Adding processes and increasing
+the amount of code in a give process can greatly increase execution time.
+Potentially infinite loops, such as those used to wait for locks to
+become available, are clearly problematic.
+
+Fortunately, it is possible to avoid state-space explosion by specially
+modeling such loops. For example, the following litmus tests emulates
+locking using xchg_acquire(), but instead of enclosing xchg_acquire()
+in a spin loop, it instead excludes executions that fail to acquire the
+lock using a herd7 "filter" clause. Note that for exclusive locking, you
+are better off using the spin_lock() and spin_unlock() that LKMM directly
+models, if for no other reason that these are much faster. However, the
+techniques illustrated in this section can be used for other purposes,
+such as emulating reader-writer locking, which LKMM does not yet model.
+
+ 1 C C-SB+l-o-o-u+l-o-o-u-X
+ 2
+ 3 {
+ 4 }
+ 5
+ 6 P0(int *sl, int *x0, int *x1)
+ 7 {
+ 8 int r2;
+ 9 int r1;
+10
+11 r2 = xchg_acquire(sl, 1);
+12 WRITE_ONCE(*x0, 1);
+13 r1 = READ_ONCE(*x1);
+14 smp_store_release(sl, 0);
+15 }
+16
+17 P1(int *sl, int *x0, int *x1)
+18 {
+19 int r2;
+20 int r1;
+21
+22 r2 = xchg_acquire(sl, 1);
+23 WRITE_ONCE(*x1, 1);
+24 r1 = READ_ONCE(*x0);
+25 smp_store_release(sl, 0);
+26 }
+27
+28 filter (0:r2=0 /\ 1:r2=0)
+29 exists (0:r1=0 /\ 1:r1=0)
+
+This litmus test may be found here:
+
+https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/perfbook.git/tree/CodeSamples/formal/herd/C-SB+l-o-o-u+l-o-o-u-X.litmus
+
+This test uses two global variables, "x1" and "x2", and also emulates a
+single global spinlock named "sl". This spinlock is held by whichever
+process changes the value of "sl" from "0" to "1", and is released when
+that process sets "sl" back to "0". P0()'s lock acquisition is emulated
+on line 11 using xchg_acquire(), which unconditionally stores the value
+"1" to "sl" and stores either "0" or "1" to "r2", depending on whether
+the lock acquisition was successful or unsuccessful (due to "sl" already
+having the value "1"), respectively. P1() operates in a similar manner.
+
+Rather unconventionally, execution appears to proceed to the critical
+section on lines 12 and 13 in either case. Line 14 then uses an
+smp_store_release() to store zero to "sl", thus emulating lock release.
+
+The case where xchg_acquire() fails to acquire the lock is handled by
+the "filter" clause on line 28, which tells herd7 to keep only those
+executions in which both "0:r2" and "1:r2" are zero, that is to pay
+attention only to those executions in which both locks are actually
+acquired. Thus, the bogus executions that would execute the critical
+sections are discarded and any effects that they might have had are
+ignored. Note well that the "filter" clause keeps those executions
+for which its expression is satisfied, that is, for which the expression
+evaluates to true. In other words, the "filter" clause says what to
+keep, not what to discard.
+
+The result of running this test is as follows:
+
+ 1 Test C-SB+l-o-o-u+l-o-o-u-X Allowed
+ 2 States 2
+ 3 0:r1=0; 1:r1=1;
+ 4 0:r1=1; 1:r1=0;
+ 5 No
+ 6 Witnesses
+ 7 Positive: 0 Negative: 2
+ 8 Condition exists (0:r1=0 /\ 1:r1=0)
+ 9 Observation C-SB+l-o-o-u+l-o-o-u-X Never 0 2
+10 Time C-SB+l-o-o-u+l-o-o-u-X 0.03
+
+The "Never" on line 9 indicates that this use of xchg_acquire() and
+smp_store_release() really does correctly emulate locking.
+
+Why doesn't the litmus test take the simpler approach of using a spin loop
+to handle failed spinlock acquisitions, like the kernel does? The key
+insight behind this litmus test is that spin loops have no effect on the
+possible "exists"-clause outcomes of program execution in the absence
+of deadlock. In other words, given a high-quality lock-acquisition
+primitive in a deadlock-free program running on high-quality hardware,
+each lock acquisition will eventually succeed. Because herd7 already
+explores the full state space, the length of time required to actually
+acquire the lock does not matter. After all, herd7 already models all
+possible durations of the xchg_acquire() statements.
+
+Why not just add the "filter" clause to the "exists" clause, thus
+avoiding the "filter" clause entirely? This does work, but is slower.
+The reason that the "filter" clause is faster is that (in the common case)
+herd7 knows to abandon an execution as soon as the "filter" expression
+fails to be satisfied. In contrast, the "exists" clause is evaluated
+only at the end of time, thus requiring herd7 to waste time on bogus
+executions in which both critical sections proceed concurrently. In
+addition, some LKMM users like the separation of concerns provided by
+using the both the "filter" and "exists" clauses.
+
+Readers lacking a pathological interest in odd corner cases should feel
+free to skip the remainder of this section.
+
+But what if the litmus test were to temporarily set "0:r2" to a non-zero
+value? Wouldn't that cause herd7 to abandon the execution prematurely
+due to an early mismatch of the "filter" clause?
+
+Why not just try it? Line 4 of the following modified litmus test
+introduces a new global variable "x2" that is initialized to "1". Line 23
+of P1() reads that variable into "1:r2" to force an early mismatch with
+the "filter" clause. Line 24 does a known-true "if" condition to avoid
+and static analysis that herd7 might do. Finally the "exists" clause
+on line 32 is updated to a condition that is alway satisfied at the end
+of the test.
+
+ 1 C C-SB+l-o-o-u+l-o-o-u-X
+ 2
+ 3 {
+ 4 x2=1;
+ 5 }
+ 6
+ 7 P0(int *sl, int *x0, int *x1)
+ 8 {
+ 9 int r2;
+10 int r1;
+11
+12 r2 = xchg_acquire(sl, 1);
+13 WRITE_ONCE(*x0, 1);
+14 r1 = READ_ONCE(*x1);
+15 smp_store_release(sl, 0);
+16 }
+17
+18 P1(int *sl, int *x0, int *x1, int *x2)
+19 {
+20 int r2;
+21 int r1;
+22
+23 r2 = READ_ONCE(*x2);
+24 if (r2)
+25 r2 = xchg_acquire(sl, 1);
+26 WRITE_ONCE(*x1, 1);
+27 r1 = READ_ONCE(*x0);
+28 smp_store_release(sl, 0);
+29 }
+30
+31 filter (0:r2=0 /\ 1:r2=0)
+32 exists (x1=1)
+
+If the "filter" clause were to check each variable at each point in the
+execution, running this litmus test would display no executions because
+all executions would be filtered out at line 23. However, the output
+is instead as follows:
+
+ 1 Test C-SB+l-o-o-u+l-o-o-u-X Allowed
+ 2 States 1
+ 3 x1=1;
+ 4 Ok
+ 5 Witnesses
+ 6 Positive: 2 Negative: 0
+ 7 Condition exists (x1=1)
+ 8 Observation C-SB+l-o-o-u+l-o-o-u-X Always 2 0
+ 9 Time C-SB+l-o-o-u+l-o-o-u-X 0.04
+10 Hash=080bc508da7f291e122c6de76c0088e3
+
+Line 3 shows that there is one execution that did not get filtered out,
+so the "filter" clause is evaluated only on the last assignment to
+the variables that it checks. In this case, the "filter" clause is a
+disjunction, so it might be evaluated twice, once at the final (and only)
+assignment to "0:r2" and once at the final assignment to "1:r2".
+
+
+Linked Lists
+------------
+
+LKMM can handle linked lists, but only linked lists in which each node
+contains nothing except a pointer to the next node in the list. This is
+of course quite restrictive, but there is nevertheless quite a bit that
+can be done within these confines, as can be seen in the litmus test
+at tools/memory-model/litmus-tests/MP+onceassign+derefonce.litmus:
+
+ 1 C MP+onceassign+derefonce
+ 2
+ 3 {
+ 4 y=z;
+ 5 z=0;
+ 6 }
+ 7
+ 8 P0(int *x, int **y)
+ 9 {
+10 WRITE_ONCE(*x, 1);
+11 rcu_assign_pointer(*y, x);
+12 }
+13
+14 P1(int *x, int **y)
+15 {
+16 int *r0;
+17 int r1;
+18
+19 rcu_read_lock();
+20 r0 = rcu_dereference(*y);
+21 r1 = READ_ONCE(*r0);
+22 rcu_read_unlock();
+23 }
+24
+25 exists (1:r0=x /\ 1:r1=0)
+
+Line 4's "y=z" may seem odd, given that "z" has not yet been initialized.
+But "y=z" does not set the value of "y" to that of "z", but instead
+sets the value of "y" to the *address* of "z". Lines 4 and 5 therefore
+create a simple linked list, with "y" pointing to "z" and "z" having a
+NULL pointer. A much longer linked list could be created if desired,
+and circular singly linked lists can also be created and manipulated.
+
+The "exists" clause works the same way, with the "1:r0=x" comparing P1()'s
+"r0" not to the value of "x", but again to its address. This term of the
+"exists" clause therefore tests whether line 20's load from "y" saw the
+value stored by line 11, which is in fact what is required in this case.
+
+P0()'s line 10 initializes "x" to the value 1 then line 11 links to "x"
+from "y", replacing "z".
+
+P1()'s line 20 loads a pointer from "y", and line 21 dereferences that
+pointer. The RCU read-side critical section spanning lines 19-22 is just
+for show in this example. Note that the address used for line 21's load
+depends on (in this case, "is exactly the same as") the value loaded by
+line 20. This is an example of what is called an "address dependency".
+This particular address dependency extends from the load on line 20 to the
+load on line 21. Address dependencies provide a weak form of ordering.
+
+Running this test results in the following:
+
+ 1 Test MP+onceassign+derefonce Allowed
+ 2 States 2
+ 3 1:r0=x; 1:r1=1;
+ 4 1:r0=z; 1:r1=0;
+ 5 No
+ 6 Witnesses
+ 7 Positive: 0 Negative: 2
+ 8 Condition exists (1:r0=x /\ 1:r1=0)
+ 9 Observation MP+onceassign+derefonce Never 0 2
+10 Time MP+onceassign+derefonce 0.00
+11 Hash=49ef7a741563570102448a256a0c8568
+
+The only possible outcomes feature P1() loading a pointer to "z"
+(which contains zero) on the one hand and P1() loading a pointer to "x"
+(which contains the value one) on the other. This should be reassuring
+because it says that RCU readers cannot see the old preinitialization
+values when accessing a newly inserted list node. This undesirable
+scenario is flagged by the "exists" clause, and would occur if P1()
+loaded a pointer to "x", but obtained the pre-initialization value of
+zero after dereferencing that pointer.
+
+
+Comments
+--------
+
+Different portions of a litmus test are processed by different parsers,
+which has the charming effect of requiring different comment syntax in
+different portions of the litmus test. The C-syntax portions use
+C-language comments (either "/* */" or "//"), while the other portions
+use Ocaml comments "(* *)".
+
+The following litmus test illustrates the comment style corresponding
+to each syntactic unit of the test:
+
+ 1 C MP+onceassign+derefonce (* A *)
+ 2
+ 3 (* B *)
+ 4
+ 5 {
+ 6 y=z; (* C *)
+ 7 z=0;
+ 8 } // D
+ 9
+10 // E
+11
+12 P0(int *x, int **y) // F
+13 {
+14 WRITE_ONCE(*x, 1); // G
+15 rcu_assign_pointer(*y, x);
+16 }
+17
+18 // H
+19
+20 P1(int *x, int **y)
+21 {
+22 int *r0;
+23 int r1;
+24
+25 rcu_read_lock();
+26 r0 = rcu_dereference(*y);
+27 r1 = READ_ONCE(*r0);
+28 rcu_read_unlock();
+29 }
+30
+31 // I
+32
+33 exists (* J *) (1:r0=x /\ (* K *) 1:r1=0) (* L *)
+
+In short, use C-language comments in the C code and Ocaml comments in
+the rest of the litmus test.
+
+On the other hand, if you prefer C-style comments everywhere, the
+C preprocessor is your friend.
+
+
+Asynchronous RCU Grace Periods
+------------------------------
+
+The following litmus test is derived from the example show in
+Documentation/litmus-tests/rcu/RCU+sync+free.litmus, but converted to
+emulate call_rcu():
+
+ 1 C RCU+sync+free
+ 2
+ 3 {
+ 4 int x = 1;
+ 5 int *y = &x;
+ 6 int z = 1;
+ 7 }
+ 8
+ 9 P0(int *x, int *z, int **y)
+10 {
+11 int *r0;
+12 int r1;
+13
+14 rcu_read_lock();
+15 r0 = rcu_dereference(*y);
+16 r1 = READ_ONCE(*r0);
+17 rcu_read_unlock();
+18 }
+19
+20 P1(int *z, int **y, int *c)
+21 {
+22 rcu_assign_pointer(*y, z);
+23 smp_store_release(*c, 1); // Emulate call_rcu().
+24 }
+25
+26 P2(int *x, int *z, int **y, int *c)
+27 {
+28 int r0;
+29
+30 r0 = smp_load_acquire(*c); // Note call_rcu() request.
+31 synchronize_rcu(); // Wait one grace period.
+32 WRITE_ONCE(*x, 0); // Emulate the RCU callback.
+33 }
+34
+35 filter (2:r0=1) (* Reject too-early starts. *)
+36 exists (0:r0=x /\ 0:r1=0)
+
+Lines 4-6 initialize a linked list headed by "y" that initially contains
+"x". In addition, "z" is pre-initialized to prepare for P1(), which
+will replace "x" with "z" in this list.
+
+P0() on lines 9-18 enters an RCU read-side critical section, loads the
+list header "y" and dereferences it, leaving the node in "0:r0" and
+the node's value in "0:r1".
+
+P1() on lines 20-24 updates the list header to instead reference "z",
+then emulates call_rcu() by doing a release store into "c".
+
+P2() on lines 27-33 emulates the behind-the-scenes effect of doing a
+call_rcu(). Line 30 first does an acquire load from "c", then line 31
+waits for an RCU grace period to elapse, and finally line 32 emulates
+the RCU callback, which in turn emulates a call to kfree().
+
+Of course, it is possible for P2() to start too soon, so that the
+value of "2:r0" is zero rather than the required value of "1".
+The "filter" clause on line 35 handles this possibility, rejecting
+all executions in which "2:r0" is not equal to the value "1".
+
+
+Performance
+-----------
+
+LKMM's exploration of the full state-space can be extremely helpful,
+but it does not come for free. The price is exponential computational
+complexity in terms of the number of processes, the average number
+of statements in each process, and the total number of stores in the
+litmus test.
+
+So it is best to start small and then work up. Where possible, break
+your code down into small pieces each representing a core concurrency
+requirement.
+
+That said, herd7 is quite fast. On an unprepossessing x86 laptop, it
+was able to analyze the following 10-process RCU litmus test in about
+six seconds.
+
+https://github.com/paulmckrcu/litmus/blob/master/auto/C-RW-R+RW-R+RW-G+RW-G+RW-G+RW-G+RW-R+RW-R+RW-R+RW-R.litmus
+
+One way to make herd7 run faster is to use the "-speedcheck true" option.
+This option prevents herd7 from generating all possible end states,
+instead causing it to focus solely on whether or not the "exists"
+clause can be satisfied. With this option, herd7 evaluates the above
+litmus test in about 300 milliseconds, for more than an order of magnitude
+improvement in performance.
+
+Larger 16-process litmus tests that would normally consume 15 minutes
+of time complete in about 40 seconds with this option. To be fair,
+you do get an extra 65,535 states when you leave off the "-speedcheck
+true" option.
+
+https://github.com/paulmckrcu/litmus/blob/master/auto/C-RW-R+RW-R+RW-G+RW-G+RW-G+RW-G+RW-R+RW-R+RW-R+RW-R+RW-G+RW-G+RW-G+RW-G+RW-R+RW-R.litmus
+
+Nevertheless, litmus-test analysis really is of exponential complexity,
+whether with or without "-speedcheck true". Increasing by just three
+processes to a 19-process litmus test requires 2 hours and 40 minutes
+without, and about 8 minutes with "-speedcheck true". Each of these
+results represent roughly an order of magnitude slowdown compared to the
+16-process litmus test. Again, to be fair, the multi-hour run explores
+no fewer than 524,287 additional states compared to the shorter one.
+
+https://github.com/paulmckrcu/litmus/blob/master/auto/C-RW-R+RW-R+RW-G+RW-G+RW-G+RW-G+RW-R+RW-R+RW-R+RW-R+RW-R+RW-R+RW-G+RW-G+RW-G+RW-G+RW-R+RW-R+RW-R.litmus
+
+If you don't like command-line arguments, you can obtain a similar speedup
+by adding a "filter" clause with exactly the same expression as your
+"exists" clause.
+
+However, please note that seeing the full set of states can be extremely
+helpful when developing and debugging litmus tests.
+
+
+LIMITATIONS
+===========
+
+Limitations of the Linux-kernel memory model (LKMM) include:
+
+1. Compiler optimizations are not accurately modeled. Of course,
+ the use of READ_ONCE() and WRITE_ONCE() limits the compiler's
+ ability to optimize, but under some circumstances it is possible
+ for the compiler to undermine the memory model. For more
+ information, see Documentation/explanation.txt (in particular,
+ the "THE PROGRAM ORDER RELATION: po AND po-loc" and "A WARNING"
+ sections).
+
+ Note that this limitation in turn limits LKMM's ability to
+ accurately model address, control, and data dependencies.
+ For example, if the compiler can deduce the value of some variable
+ carrying a dependency, then the compiler can break that dependency
+ by substituting a constant of that value.
+
+2. Multiple access sizes for a single variable are not supported,
+ and neither are misaligned or partially overlapping accesses.
+
+3. Exceptions and interrupts are not modeled. In some cases,
+ this limitation can be overcome by modeling the interrupt or
+ exception with an additional process.
+
+4. I/O such as MMIO or DMA is not supported.
+
+5. Self-modifying code (such as that found in the kernel's
+ alternatives mechanism, function tracer, Berkeley Packet Filter
+ JIT compiler, and module loader) is not supported.
+
+6. Complete modeling of all variants of atomic read-modify-write
+ operations, locking primitives, and RCU is not provided.
+ For example, call_rcu() and rcu_barrier() are not supported.
+ However, a substantial amount of support is provided for these
+ operations, as shown in the linux-kernel.def file.
+
+ Here are specific limitations:
+
+ a. When rcu_assign_pointer() is passed NULL, the Linux
+ kernel provides no ordering, but LKMM models this
+ case as a store release.
+
+ b. The "unless" RMW operations are not currently modeled:
+ atomic_long_add_unless(), atomic_inc_unless_negative(),
+ and atomic_dec_unless_positive(). These can be emulated
+ in litmus tests, for example, by using atomic_cmpxchg().
+
+ One exception of this limitation is atomic_add_unless(),
+ which is provided directly by herd7 (so no corresponding
+ definition in linux-kernel.def). atomic_add_unless() is
+ modeled by herd7 therefore it can be used in litmus tests.
+
+ c. The call_rcu() function is not modeled. As was shown above,
+ it can be emulated in litmus tests by adding another
+ process that invokes synchronize_rcu() and the body of the
+ callback function, with (for example) a release-acquire
+ from the site of the emulated call_rcu() to the beginning
+ of the additional process.
+
+ d. The rcu_barrier() function is not modeled. It can be
+ emulated in litmus tests emulating call_rcu() via
+ (for example) a release-acquire from the end of each
+ additional call_rcu() process to the site of the
+ emulated rcu-barrier().
+
+ e. Although sleepable RCU (SRCU) is now modeled, there
+ are some subtle differences between its semantics and
+ those in the Linux kernel. For example, the kernel
+ might interpret the following sequence as two partially
+ overlapping SRCU read-side critical sections:
+
+ 1 r1 = srcu_read_lock(&my_srcu);
+ 2 do_something_1();
+ 3 r2 = srcu_read_lock(&my_srcu);
+ 4 do_something_2();
+ 5 srcu_read_unlock(&my_srcu, r1);
+ 6 do_something_3();
+ 7 srcu_read_unlock(&my_srcu, r2);
+
+ In contrast, LKMM will interpret this as a nested pair of
+ SRCU read-side critical sections, with the outer critical
+ section spanning lines 1-7 and the inner critical section
+ spanning lines 3-5.
+
+ This difference would be more of a concern had anyone
+ identified a reasonable use case for partially overlapping
+ SRCU read-side critical sections. For more information
+ on the trickiness of such overlapping, please see:
+ https://paulmck.livejournal.com/40593.html
+
+ f. Reader-writer locking is not modeled. It can be
+ emulated in litmus tests using atomic read-modify-write
+ operations.
+
+The fragment of the C language supported by these litmus tests is quite
+limited and in some ways non-standard:
+
+1. There is no automatic C-preprocessor pass. You can of course
+ run it manually, if you choose.
+
+2. There is no way to create functions other than the Pn() functions
+ that model the concurrent processes.
+
+3. The Pn() functions' formal parameters must be pointers to the
+ global shared variables. Nothing can be passed by value into
+ these functions.
+
+4. The only functions that can be invoked are those built directly
+ into herd7 or that are defined in the linux-kernel.def file.
+
+5. The "switch", "do", "for", "while", and "goto" C statements are
+ not supported. The "switch" statement can be emulated by the
+ "if" statement. The "do", "for", and "while" statements can
+ often be emulated by manually unrolling the loop, or perhaps by
+ enlisting the aid of the C preprocessor to minimize the resulting
+ code duplication. Some uses of "goto" can be emulated by "if",
+ and some others by unrolling.
+
+6. Although you can use a wide variety of types in litmus-test
+ variable declarations, and especially in global-variable
+ declarations, the "herd7" tool understands only int and
+ pointer types. There is no support for floating-point types,
+ enumerations, characters, strings, arrays, or structures.
+
+7. Parsing of variable declarations is very loose, with almost no
+ type checking.
+
+8. Initializers differ from their C-language counterparts.
+ For example, when an initializer contains the name of a shared
+ variable, that name denotes a pointer to that variable, not
+ the current value of that variable. For example, "int x = y"
+ is interpreted the way "int x = &y" would be in C.
+
+9. Dynamic memory allocation is not supported, although this can
+ be worked around in some cases by supplying multiple statically
+ allocated variables.
+
+Some of these limitations may be overcome in the future, but others are
+more likely to be addressed by incorporating the Linux-kernel memory model
+into other tools.
+
+Finally, please note that LKMM is subject to change as hardware, use cases,
+and compilers evolve.
This document provides "recipes", that is, litmus tests for commonly
occurring situations, as well as a few that illustrate subtly broken but
attractive nuisances. Many of these recipes include example code from
-v4.13 of the Linux kernel.
+v5.7 of the Linux kernel.
The first section covers simple special cases, the second section
takes off the training wheels to cover more involved examples,
first place (control dependency). Note that the term "data dependency"
is sometimes casually used to cover both address and data dependencies.
-In lib/prime_numbers.c, the expand_to_next_prime() function invokes
+In lib/math/prime_numbers.c, the expand_to_next_prime() function invokes
rcu_assign_pointer(), and the next_prime_number() function invokes
rcu_dereference(). This combination mediates access to a bit vector
that is expanded as additional primes are needed.
o Jade Alglave, Patrick Cousot, and Luc Maranget. 2016. "Syntax and
semantics of the weak consistency model specification language
- cat". CoRR abs/1608.07531 (2016). http://arxiv.org/abs/1608.07531
+ cat". CoRR abs/1608.07531 (2016). https://arxiv.org/abs/1608.07531
Memory-model comparisons
--- /dev/null
+This document provides options for those wishing to keep their
+memory-ordering lives simple, as is necessary for those whose domain
+is complex. After all, there are bugs other than memory-ordering bugs,
+and the time spent gaining memory-ordering knowledge is not available
+for gaining domain knowledge. Furthermore Linux-kernel memory model
+(LKMM) is quite complex, with subtle differences in code often having
+dramatic effects on correctness.
+
+The options near the beginning of this list are quite simple. The idea
+is not that kernel hackers don't already know about them, but rather
+that they might need the occasional reminder.
+
+Please note that this is a generic guide, and that specific subsystems
+will often have special requirements or idioms. For example, developers
+of MMIO-based device drivers will often need to use mb(), rmb(), and
+wmb(), and therefore might find smp_mb(), smp_rmb(), and smp_wmb()
+to be more natural than smp_load_acquire() and smp_store_release().
+On the other hand, those coming in from other environments will likely
+be more familiar with these last two.
+
+
+Single-threaded code
+====================
+
+In single-threaded code, there is no reordering, at least assuming
+that your toolchain and hardware are working correctly. In addition,
+it is generally a mistake to assume your code will only run in a single
+threaded context as the kernel can enter the same code path on multiple
+CPUs at the same time. One important exception is a function that makes
+no external data references.
+
+In the general case, you will need to take explicit steps to ensure that
+your code really is executed within a single thread that does not access
+shared variables. A simple way to achieve this is to define a global lock
+that you acquire at the beginning of your code and release at the end,
+taking care to ensure that all references to your code's shared data are
+also carried out under that same lock. Because only one thread can hold
+this lock at a given time, your code will be executed single-threaded.
+This approach is called "code locking".
+
+Code locking can severely limit both performance and scalability, so it
+should be used with caution, and only on code paths that execute rarely.
+After all, a huge amount of effort was required to remove the Linux
+kernel's old "Big Kernel Lock", so let's please be very careful about
+adding new "little kernel locks".
+
+One of the advantages of locking is that, in happy contrast with the
+year 1981, almost all kernel developers are very familiar with locking.
+The Linux kernel's lockdep (CONFIG_PROVE_LOCKING=y) is very helpful with
+the formerly feared deadlock scenarios.
+
+Please use the standard locking primitives provided by the kernel rather
+than rolling your own. For one thing, the standard primitives interact
+properly with lockdep. For another thing, these primitives have been
+tuned to deal better with high contention. And for one final thing, it is
+surprisingly hard to correctly code production-quality lock acquisition
+and release functions. After all, even simple non-production-quality
+locking functions must carefully prevent both the CPU and the compiler
+from moving code in either direction across the locking function.
+
+Despite the scalability limitations of single-threaded code, RCU
+takes this approach for much of its grace-period processing and also
+for early-boot operation. The reason RCU is able to scale despite
+single-threaded grace-period processing is use of batching, where all
+updates that accumulated during one grace period are handled by the
+next one. In other words, slowing down grace-period processing makes
+it more efficient. Nor is RCU unique: Similar batching optimizations
+are used in many I/O operations.
+
+
+Packaged code
+=============
+
+Even if performance and scalability concerns prevent your code from
+being completely single-threaded, it is often possible to use library
+functions that handle the concurrency nearly or entirely on their own.
+This approach delegates any LKMM worries to the library maintainer.
+
+In the kernel, what is the "library"? Quite a bit. It includes the
+contents of the lib/ directory, much of the include/linux/ directory along
+with a lot of other heavily used APIs. But heavily used examples include
+the list macros (for example, include/linux/{,rcu}list.h), workqueues,
+smp_call_function(), and the various hash tables and search trees.
+
+
+Data locking
+============
+
+With code locking, we use single-threaded code execution to guarantee
+serialized access to the data that the code is accessing. However,
+we can also achieve this by instead associating the lock with specific
+instances of the data structures. This creates a "critical section"
+in the code execution that will execute as though it is single threaded.
+By placing all the accesses and modifications to a shared data structure
+inside a critical section, we ensure that the execution context that
+holds the lock has exclusive access to the shared data.
+
+The poster boy for this approach is the hash table, where placing a lock
+in each hash bucket allows operations on different buckets to proceed
+concurrently. This works because the buckets do not overlap with each
+other, so that an operation on one bucket does not interfere with any
+other bucket.
+
+As the number of buckets increases, data locking scales naturally.
+In particular, if the amount of data increases with the number of CPUs,
+increasing the number of buckets as the number of CPUs increase results
+in a naturally scalable data structure.
+
+
+Per-CPU processing
+==================
+
+Partitioning processing and data over CPUs allows each CPU to take
+a single-threaded approach while providing excellent performance and
+scalability. Of course, there is no free lunch: The dark side of this
+excellence is substantially increased memory footprint.
+
+In addition, it is sometimes necessary to occasionally update some global
+view of this processing and data, in which case something like locking
+must be used to protect this global view. This is the approach taken
+by the percpu_counter infrastructure. In many cases, there are already
+generic/library variants of commonly used per-cpu constructs available.
+Please use them rather than rolling your own.
+
+RCU uses DEFINE_PER_CPU*() declaration to create a number of per-CPU
+data sets. For example, each CPU does private quiescent-state processing
+within its instance of the per-CPU rcu_data structure, and then uses data
+locking to report quiescent states up the grace-period combining tree.
+
+
+Packaged primitives: Sequence locking
+=====================================
+
+Lockless programming is considered by many to be more difficult than
+lock-based programming, but there are a few lockless design patterns that
+have been built out into an API. One of these APIs is sequence locking.
+Although this APIs can be used in extremely complex ways, there are simple
+and effective ways of using it that avoid the need to pay attention to
+memory ordering.
+
+The basic keep-things-simple rule for sequence locking is "do not write
+in read-side code". Yes, you can do writes from within sequence-locking
+readers, but it won't be so simple. For example, such writes will be
+lockless and should be idempotent.
+
+For more sophisticated use cases, LKMM can guide you, including use
+cases involving combining sequence locking with other synchronization
+primitives. (LKMM does not yet know about sequence locking, so it is
+currently necessary to open-code it in your litmus tests.)
+
+Additional information may be found in include/linux/seqlock.h.
+
+Packaged primitives: RCU
+========================
+
+Another lockless design pattern that has been baked into an API
+is RCU. The Linux kernel makes sophisticated use of RCU, but the
+keep-things-simple rules for RCU are "do not write in read-side code"
+and "do not update anything that is visible to and accessed by readers",
+and "protect updates with locking".
+
+These rules are illustrated by the functions foo_update_a() and
+foo_get_a() shown in Documentation/RCU/whatisRCU.rst. Additional
+RCU usage patterns maybe found in Documentation/RCU and in the
+source code.
+
+
+Packaged primitives: Atomic operations
+======================================
+
+Back in the day, the Linux kernel had three types of atomic operations:
+
+1. Initialization and read-out, such as atomic_set() and atomic_read().
+
+2. Operations that did not return a value and provided no ordering,
+ such as atomic_inc() and atomic_dec().
+
+3. Operations that returned a value and provided full ordering, such as
+ atomic_add_return() and atomic_dec_and_test(). Note that some
+ value-returning operations provide full ordering only conditionally.
+ For example, cmpxchg() provides ordering only upon success.
+
+More recent kernels have operations that return a value but do not
+provide full ordering. These are flagged with either a _relaxed()
+suffix (providing no ordering), or an _acquire() or _release() suffix
+(providing limited ordering).
+
+Additional information may be found in these files:
+
+Documentation/atomic_t.txt
+Documentation/atomic_bitops.txt
+Documentation/core-api/atomic_ops.rst
+Documentation/core-api/refcount-vs-atomic.rst
+
+Reading code using these primitives is often also quite helpful.
+
+
+Lockless, fully ordered
+=======================
+
+When using locking, there often comes a time when it is necessary
+to access some variable or another without holding the data lock
+that serializes access to that variable.
+
+If you want to keep things simple, use the initialization and read-out
+operations from the previous section only when there are no racing
+accesses. Otherwise, use only fully ordered operations when accessing
+or modifying the variable. This approach guarantees that code prior
+to a given access to that variable will be seen by all CPUs has having
+happened before any code following any later access to that same variable.
+
+Please note that per-CPU functions are not atomic operations and
+hence they do not provide any ordering guarantees at all.
+
+If the lockless accesses are frequently executed reads that are used
+only for heuristics, or if they are frequently executed writes that
+are used only for statistics, please see the next section.
+
+
+Lockless statistics and heuristics
+==================================
+
+Unordered primitives such as atomic_read(), atomic_set(), READ_ONCE(), and
+WRITE_ONCE() can safely be used in some cases. These primitives provide
+no ordering, but they do prevent the compiler from carrying out a number
+of destructive optimizations (for which please see the next section).
+One example use for these primitives is statistics, such as per-CPU
+counters exemplified by the rt_cache_stat structure's routing-cache
+statistics counters. Another example use case is heuristics, such as
+the jiffies_till_first_fqs and jiffies_till_next_fqs kernel parameters
+controlling how often RCU scans for idle CPUs.
+
+But be careful. "Unordered" really does mean "unordered". It is all
+too easy to assume ordering, and this assumption must be avoided when
+using these primitives.
+
+
+Don't let the compiler trip you up
+==================================
+
+It can be quite tempting to use plain C-language accesses for lockless
+loads from and stores to shared variables. Although this is both
+possible and quite common in the Linux kernel, it does require a
+surprising amount of analysis, care, and knowledge about the compiler.
+Yes, some decades ago it was not unfair to consider a C compiler to be
+an assembler with added syntax and better portability, but the advent of
+sophisticated optimizing compilers mean that those days are long gone.
+Today's optimizing compilers can profoundly rewrite your code during the
+translation process, and have long been ready, willing, and able to do so.
+
+Therefore, if you really need to use C-language assignments instead of
+READ_ONCE(), WRITE_ONCE(), and so on, you will need to have a very good
+understanding of both the C standard and your compiler. Here are some
+introductory references and some tooling to start you on this noble quest:
+
+Who's afraid of a big bad optimizing compiler?
+ https://lwn.net/Articles/793253/
+Calibrating your fear of big bad optimizing compilers
+ https://lwn.net/Articles/799218/
+Concurrency bugs should fear the big bad data-race detector (part 1)
+ https://lwn.net/Articles/816850/
+Concurrency bugs should fear the big bad data-race detector (part 2)
+ https://lwn.net/Articles/816854/
+
+
+More complex use cases
+======================
+
+If the alternatives above do not do what you need, please look at the
+recipes-pairs.txt file to peel off the next layer of the memory-ordering
+onion.
==================
The memory model is used, in conjunction with "herd7", to exhaustively
-explore the state space of small litmus tests.
+explore the state space of small litmus tests. Documentation describing
+the format, features, capabilities and limitations of these litmus
+tests is available in tools/memory-model/Documentation/litmus-tests.txt.
-For example, to run SB+fencembonceonces.litmus against the memory model:
+Example litmus tests may be found in the Linux-kernel source tree:
+ tools/memory-model/litmus-tests/
+ Documentation/litmus-tests/
+
+Several thousand more example litmus tests are available here:
+
+ https://github.com/paulmckrcu/litmus
+ https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/perfbook.git/tree/CodeSamples/formal/herd
+ https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/perfbook.git/tree/CodeSamples/formal/litmus
+
+Documentation describing litmus tests and now to use them may be found
+here:
+
+ tools/memory-model/Documentation/litmus-tests.txt
+
+The remainder of this section uses the SB+fencembonceonces.litmus test
+located in the tools/memory-model directory.
+
+To run SB+fencembonceonces.litmus against the memory model:
+
+ $ cd $LINUX_SOURCE_TREE/tools/memory-model
$ herd7 -conf linux-kernel.cfg litmus-tests/SB+fencembonceonces.litmus
Here is the corresponding output:
The "Positive: 0 Negative: 3" and the "Never 0 3" each indicate that
this litmus test's "exists" clause can not be satisfied.
-See "herd7 -help" or "herdtools7/doc/" for more information.
+See "herd7 -help" or "herdtools7/doc/" for more information on running the
+tool itself, but please be aware that this documentation is intended for
+people who work on the memory model itself, that is, people making changes
+to the tools/memory-model/linux-kernel.* files. It is not intended for
+people focusing on writing, understanding, and running LKMM litmus tests.
=====================
test's "exists" clause was not reached.
And, as with "herd7", please see "klitmus7 -help" or "herdtools7/doc/"
-for more information.
+for more information. And again, please be aware that this documentation
+is intended for people who work on the memory model itself, that is,
+people making changes to the tools/memory-model/linux-kernel.* files.
+It is not intended for people focusing on writing, understanding, and
+running LKMM litmus tests.
====================
Documentation/explanation.txt
Describes the memory model in detail.
+Documentation/litmus-tests.txt
+ Describes the format, features, capabilities, and limitations
+ of the litmus tests that LKMM can evaluate.
+
Documentation/recipes.txt
Lists common memory-ordering patterns.
Documentation/references.txt
Provides background reading.
+Documentation/simple.txt
+ Starting point for someone new to Linux-kernel concurrency.
+ And also for those needing a reminder of the simpler approaches
+ to concurrency!
+
linux-kernel.bell
Categorizes the relevant instructions, including memory
references, memory barriers, atomic read-modify-write operations,
This file.
scripts Various scripts, see scripts/README.
-
-
-===========
-LIMITATIONS
-===========
-
-The Linux-kernel memory model (LKMM) has the following limitations:
-
-1. Compiler optimizations are not accurately modeled. Of course,
- the use of READ_ONCE() and WRITE_ONCE() limits the compiler's
- ability to optimize, but under some circumstances it is possible
- for the compiler to undermine the memory model. For more
- information, see Documentation/explanation.txt (in particular,
- the "THE PROGRAM ORDER RELATION: po AND po-loc" and "A WARNING"
- sections).
-
- Note that this limitation in turn limits LKMM's ability to
- accurately model address, control, and data dependencies.
- For example, if the compiler can deduce the value of some variable
- carrying a dependency, then the compiler can break that dependency
- by substituting a constant of that value.
-
-2. Multiple access sizes for a single variable are not supported,
- and neither are misaligned or partially overlapping accesses.
-
-3. Exceptions and interrupts are not modeled. In some cases,
- this limitation can be overcome by modeling the interrupt or
- exception with an additional process.
-
-4. I/O such as MMIO or DMA is not supported.
-
-5. Self-modifying code (such as that found in the kernel's
- alternatives mechanism, function tracer, Berkeley Packet Filter
- JIT compiler, and module loader) is not supported.
-
-6. Complete modeling of all variants of atomic read-modify-write
- operations, locking primitives, and RCU is not provided.
- For example, call_rcu() and rcu_barrier() are not supported.
- However, a substantial amount of support is provided for these
- operations, as shown in the linux-kernel.def file.
-
- a. When rcu_assign_pointer() is passed NULL, the Linux
- kernel provides no ordering, but LKMM models this
- case as a store release.
-
- b. The "unless" RMW operations are not currently modeled:
- atomic_long_add_unless(), atomic_inc_unless_negative(),
- and atomic_dec_unless_positive(). These can be emulated
- in litmus tests, for example, by using atomic_cmpxchg().
-
- One exception of this limitation is atomic_add_unless(),
- which is provided directly by herd7 (so no corresponding
- definition in linux-kernel.def). atomic_add_unless() is
- modeled by herd7 therefore it can be used in litmus tests.
-
- c. The call_rcu() function is not modeled. It can be
- emulated in litmus tests by adding another process that
- invokes synchronize_rcu() and the body of the callback
- function, with (for example) a release-acquire from
- the site of the emulated call_rcu() to the beginning
- of the additional process.
-
- d. The rcu_barrier() function is not modeled. It can be
- emulated in litmus tests emulating call_rcu() via
- (for example) a release-acquire from the end of each
- additional call_rcu() process to the site of the
- emulated rcu-barrier().
-
- e. Although sleepable RCU (SRCU) is now modeled, there
- are some subtle differences between its semantics and
- those in the Linux kernel. For example, the kernel
- might interpret the following sequence as two partially
- overlapping SRCU read-side critical sections:
-
- 1 r1 = srcu_read_lock(&my_srcu);
- 2 do_something_1();
- 3 r2 = srcu_read_lock(&my_srcu);
- 4 do_something_2();
- 5 srcu_read_unlock(&my_srcu, r1);
- 6 do_something_3();
- 7 srcu_read_unlock(&my_srcu, r2);
-
- In contrast, LKMM will interpret this as a nested pair of
- SRCU read-side critical sections, with the outer critical
- section spanning lines 1-7 and the inner critical section
- spanning lines 3-5.
-
- This difference would be more of a concern had anyone
- identified a reasonable use case for partially overlapping
- SRCU read-side critical sections. For more information,
- please see: https://paulmck.livejournal.com/40593.html
-
- f. Reader-writer locking is not modeled. It can be
- emulated in litmus tests using atomic read-modify-write
- operations.
-
-The "herd7" tool has some additional limitations of its own, apart from
-the memory model:
-
-1. Non-trivial data structures such as arrays or structures are
- not supported. However, pointers are supported, allowing trivial
- linked lists to be constructed.
-
-2. Dynamic memory allocation is not supported, although this can
- be worked around in some cases by supplying multiple statically
- allocated variables.
-
-Some of these limitations may be overcome in the future, but others are
-more likely to be addressed by incorporating the Linux-kernel memory model
-into other tools.
-
-Finally, please note that LKMM is subject to change as hardware, use cases,
-and compilers evolve.
#include <linux/hashtable.h>
#include <linux/kernel.h>
+#include <linux/static_call_types.h>
#define FAKE_JUMP_OFFSET -1
return 0;
}
+static int create_static_call_sections(struct objtool_file *file)
+{
+ struct section *sec, *reloc_sec;
+ struct reloc *reloc;
+ struct static_call_site *site;
+ struct instruction *insn;
+ struct symbol *key_sym;
+ char *key_name, *tmp;
+ int idx;
+
+ sec = find_section_by_name(file->elf, ".static_call_sites");
+ if (sec) {
+ INIT_LIST_HEAD(&file->static_call_list);
+ WARN("file already has .static_call_sites section, skipping");
+ return 0;
+ }
+
+ if (list_empty(&file->static_call_list))
+ return 0;
+
+ idx = 0;
+ list_for_each_entry(insn, &file->static_call_list, static_call_node)
+ idx++;
+
+ sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
+ sizeof(struct static_call_site), idx);
+ if (!sec)
+ return -1;
+
+ reloc_sec = elf_create_reloc_section(file->elf, sec, SHT_RELA);
+ if (!reloc_sec)
+ return -1;
+
+ idx = 0;
+ list_for_each_entry(insn, &file->static_call_list, static_call_node) {
+
+ site = (struct static_call_site *)sec->data->d_buf + idx;
+ memset(site, 0, sizeof(struct static_call_site));
+
+ /* populate reloc for 'addr' */
+ reloc = malloc(sizeof(*reloc));
+ if (!reloc) {
+ perror("malloc");
+ return -1;
+ }
+ memset(reloc, 0, sizeof(*reloc));
+ reloc->sym = insn->sec->sym;
+ reloc->addend = insn->offset;
+ reloc->type = R_X86_64_PC32;
+ reloc->offset = idx * sizeof(struct static_call_site);
+ reloc->sec = reloc_sec;
+ elf_add_reloc(file->elf, reloc);
+
+ /* find key symbol */
+ key_name = strdup(insn->call_dest->name);
+ if (!key_name) {
+ perror("strdup");
+ return -1;
+ }
+ if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
+ STATIC_CALL_TRAMP_PREFIX_LEN)) {
+ WARN("static_call: trampoline name malformed: %s", key_name);
+ return -1;
+ }
+ tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
+ memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
+
+ key_sym = find_symbol_by_name(file->elf, tmp);
+ if (!key_sym) {
+ WARN("static_call: can't find static_call_key symbol: %s", tmp);
+ return -1;
+ }
+ free(key_name);
+
+ /* populate reloc for 'key' */
+ reloc = malloc(sizeof(*reloc));
+ if (!reloc) {
+ perror("malloc");
+ return -1;
+ }
+ memset(reloc, 0, sizeof(*reloc));
+ reloc->sym = key_sym;
+ reloc->addend = is_sibling_call(insn) ? STATIC_CALL_SITE_TAIL : 0;
+ reloc->type = R_X86_64_PC32;
+ reloc->offset = idx * sizeof(struct static_call_site) + 4;
+ reloc->sec = reloc_sec;
+ elf_add_reloc(file->elf, reloc);
+
+ idx++;
+ }
+
+ if (elf_rebuild_reloc_section(file->elf, reloc_sec))
+ return -1;
+
+ return 0;
+}
+
/*
* Warnings shouldn't be reported for ignored functions.
*/
"__tsan_write4",
"__tsan_write8",
"__tsan_write16",
+ "__tsan_read_write1",
+ "__tsan_read_write2",
+ "__tsan_read_write4",
+ "__tsan_read_write8",
+ "__tsan_read_write16",
+ "__tsan_atomic8_load",
+ "__tsan_atomic16_load",
+ "__tsan_atomic32_load",
+ "__tsan_atomic64_load",
+ "__tsan_atomic8_store",
+ "__tsan_atomic16_store",
+ "__tsan_atomic32_store",
+ "__tsan_atomic64_store",
+ "__tsan_atomic8_exchange",
+ "__tsan_atomic16_exchange",
+ "__tsan_atomic32_exchange",
+ "__tsan_atomic64_exchange",
+ "__tsan_atomic8_fetch_add",
+ "__tsan_atomic16_fetch_add",
+ "__tsan_atomic32_fetch_add",
+ "__tsan_atomic64_fetch_add",
+ "__tsan_atomic8_fetch_sub",
+ "__tsan_atomic16_fetch_sub",
+ "__tsan_atomic32_fetch_sub",
+ "__tsan_atomic64_fetch_sub",
+ "__tsan_atomic8_fetch_and",
+ "__tsan_atomic16_fetch_and",
+ "__tsan_atomic32_fetch_and",
+ "__tsan_atomic64_fetch_and",
+ "__tsan_atomic8_fetch_or",
+ "__tsan_atomic16_fetch_or",
+ "__tsan_atomic32_fetch_or",
+ "__tsan_atomic64_fetch_or",
+ "__tsan_atomic8_fetch_xor",
+ "__tsan_atomic16_fetch_xor",
+ "__tsan_atomic32_fetch_xor",
+ "__tsan_atomic64_fetch_xor",
+ "__tsan_atomic8_fetch_nand",
+ "__tsan_atomic16_fetch_nand",
+ "__tsan_atomic32_fetch_nand",
+ "__tsan_atomic64_fetch_nand",
+ "__tsan_atomic8_compare_exchange_strong",
+ "__tsan_atomic16_compare_exchange_strong",
+ "__tsan_atomic32_compare_exchange_strong",
+ "__tsan_atomic64_compare_exchange_strong",
+ "__tsan_atomic8_compare_exchange_weak",
+ "__tsan_atomic16_compare_exchange_weak",
+ "__tsan_atomic32_compare_exchange_weak",
+ "__tsan_atomic64_compare_exchange_weak",
+ "__tsan_atomic8_compare_exchange_val",
+ "__tsan_atomic16_compare_exchange_val",
+ "__tsan_atomic32_compare_exchange_val",
+ "__tsan_atomic64_compare_exchange_val",
+ "__tsan_atomic_thread_fence",
+ "__tsan_atomic_signal_fence",
/* KCOV */
"write_comp_data",
"check_kcov_mode",
"__ubsan_handle_shift_out_of_bounds",
/* misc */
"csum_partial_copy_generic",
- "__memcpy_mcsafe",
- "mcsafe_handle_tail",
+ "copy_mc_fragile",
+ "copy_mc_fragile_handle_tail",
+ "copy_mc_enhanced_fast_string",
"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
NULL
};
} else {
/* external sibling call */
insn->call_dest = reloc->sym;
+ if (insn->call_dest->static_call_tramp) {
+ list_add_tail(&insn->static_call_node,
+ &file->static_call_list);
+ }
continue;
}
/* internal sibling call */
insn->call_dest = insn->jump_dest->func;
+ if (insn->call_dest->static_call_tramp) {
+ list_add_tail(&insn->static_call_node,
+ &file->static_call_list);
+ }
}
}
}
return 0;
}
+static int read_static_call_tramps(struct objtool_file *file)
+{
+ struct section *sec;
+ struct symbol *func;
+
+ for_each_sec(file, sec) {
+ list_for_each_entry(func, &sec->symbol_list, list) {
+ if (func->bind == STB_GLOBAL &&
+ !strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
+ strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
+ func->static_call_tramp = true;
+ }
+ }
+
+ return 0;
+}
+
static void mark_rodata(struct objtool_file *file)
{
struct section *sec;
if (ret)
return ret;
+ ret = read_static_call_tramps(file);
+ if (ret)
+ return ret;
+
ret = add_jump_destinations(file);
if (ret)
return ret;
if (dead_end_function(file, insn->call_dest))
return 0;
+ if (insn->type == INSN_CALL && insn->call_dest->static_call_tramp) {
+ list_add_tail(&insn->static_call_node,
+ &file->static_call_list);
+ }
+
break;
case INSN_JUMP_CONDITIONAL:
INIT_LIST_HEAD(&file.insn_list);
hash_init(file.insn_hash);
+ INIT_LIST_HEAD(&file.static_call_list);
file.c_file = !vmlinux && find_section_by_name(file.elf, ".comment");
file.ignore_unreachables = no_unreachable;
file.hints = false;
warnings += ret;
}
+ ret = create_static_call_sections(&file);
+ if (ret < 0)
+ goto out;
+ warnings += ret;
+
if (orc) {
ret = create_orc(&file);
if (ret < 0)
struct instruction {
struct list_head list;
struct hlist_node hash;
+ struct list_head static_call_node;
struct section *sec;
unsigned long offset;
unsigned int len;
}
struct section *elf_create_section(struct elf *elf, const char *name,
- size_t entsize, int nr)
+ unsigned int sh_flags, size_t entsize, int nr)
{
struct section *sec, *shstrtab;
size_t size = entsize * nr;
sec->sh.sh_entsize = entsize;
sec->sh.sh_type = SHT_PROGBITS;
sec->sh.sh_addralign = 1;
- sec->sh.sh_flags = SHF_ALLOC;
+ sec->sh.sh_flags = SHF_ALLOC | sh_flags;
/* Add section name to .shstrtab (or .strtab for Clang) */
strcpy(relocname, ".rel");
strcat(relocname, base->name);
- sec = elf_create_section(elf, relocname, sizeof(GElf_Rel), 0);
+ sec = elf_create_section(elf, relocname, 0, sizeof(GElf_Rel), 0);
free(relocname);
if (!sec)
return NULL;
strcpy(relocname, ".rela");
strcat(relocname, base->name);
- sec = elf_create_section(elf, relocname, sizeof(GElf_Rela), 0);
+ sec = elf_create_section(elf, relocname, 0, sizeof(GElf_Rela), 0);
free(relocname);
if (!sec)
return NULL;
unsigned int len;
struct symbol *pfunc, *cfunc, *alias;
bool uaccess_safe;
+ bool static_call_tramp;
};
struct reloc {
}
struct elf *elf_open_read(const char *name, int flags);
-struct section *elf_create_section(struct elf *elf, const char *name, size_t entsize, int nr);
+struct section *elf_create_section(struct elf *elf, const char *name, unsigned int sh_flags, size_t entsize, int nr);
struct section *elf_create_reloc_section(struct elf *elf, struct section *base, int reltype);
void elf_add_reloc(struct elf *elf, struct reloc *reloc);
int elf_write_insn(struct elf *elf, struct section *sec,
struct elf *elf;
struct list_head insn_list;
DECLARE_HASHTABLE(insn_hash, 20);
+ struct list_head static_call_list;
bool ignore_unreachables, c_file, hints, rodata;
};
/* create .orc_unwind_ip and .rela.orc_unwind_ip sections */
- sec = elf_create_section(file->elf, ".orc_unwind_ip", sizeof(int), idx);
+ sec = elf_create_section(file->elf, ".orc_unwind_ip", 0, sizeof(int), idx);
if (!sec)
return -1;
return -1;
/* create .orc_unwind section */
- u_sec = elf_create_section(file->elf, ".orc_unwind",
+ u_sec = elf_create_section(file->elf, ".orc_unwind", 0,
sizeof(struct orc_entry), idx);
/* populate sections */
arch/x86/include/asm/emulate_prefix.h
arch/x86/lib/x86-opcode-map.txt
arch/x86/tools/gen-insn-attr-x86.awk
+include/linux/static_call_types.h
'
check_2 () {
18 spu oldstat sys_ni_syscall
19 common lseek sys_lseek compat_sys_lseek
20 common getpid sys_getpid
-21 nospu mount sys_mount compat_sys_mount
+21 nospu mount sys_mount
22 32 umount sys_oldumount
22 64 umount sys_ni_syscall
22 spu umount sys_ni_syscall
142 common _newselect sys_select compat_sys_select
143 common flock sys_flock
144 common msync sys_msync
-145 common readv sys_readv compat_sys_readv
-146 common writev sys_writev compat_sys_writev
+145 common readv sys_readv
+146 common writev sys_writev
147 common getsid sys_getsid
148 common fdatasync sys_fdatasync
149 nospu _sysctl sys_ni_syscall
282 common unshare sys_unshare
283 common splice sys_splice
284 common tee sys_tee
-285 common vmsplice sys_vmsplice compat_sys_vmsplice
+285 common vmsplice sys_vmsplice
286 common openat sys_openat compat_sys_openat
287 common mkdirat sys_mkdirat
288 common mknodat sys_mknodat
348 common syncfs sys_syncfs
349 common sendmmsg sys_sendmmsg compat_sys_sendmmsg
350 common setns sys_setns
-351 nospu process_vm_readv sys_process_vm_readv compat_sys_process_vm_readv
-352 nospu process_vm_writev sys_process_vm_writev compat_sys_process_vm_writev
+351 nospu process_vm_readv sys_process_vm_readv
+352 nospu process_vm_writev sys_process_vm_writev
353 nospu finit_module sys_finit_module
354 nospu kcmp sys_kcmp
355 common sched_setattr sys_sched_setattr
16 32 lchown - compat_sys_s390_lchown16
19 common lseek sys_lseek compat_sys_lseek
20 common getpid sys_getpid sys_getpid
-21 common mount sys_mount compat_sys_mount
+21 common mount sys_mount
22 common umount sys_oldumount compat_sys_oldumount
23 32 setuid - compat_sys_s390_setuid16
24 32 getuid - compat_sys_s390_getuid16
142 64 select sys_select -
143 common flock sys_flock sys_flock
144 common msync sys_msync compat_sys_msync
-145 common readv sys_readv compat_sys_readv
-146 common writev sys_writev compat_sys_writev
+145 common readv sys_readv
+146 common writev sys_writev
147 common getsid sys_getsid sys_getsid
148 common fdatasync sys_fdatasync sys_fdatasync
149 common _sysctl - -
306 common splice sys_splice compat_sys_splice
307 common sync_file_range sys_sync_file_range compat_sys_s390_sync_file_range
308 common tee sys_tee compat_sys_tee
-309 common vmsplice sys_vmsplice compat_sys_vmsplice
+309 common vmsplice sys_vmsplice sys_vmsplice
310 common move_pages sys_move_pages compat_sys_move_pages
311 common getcpu sys_getcpu compat_sys_getcpu
312 common epoll_pwait sys_epoll_pwait compat_sys_epoll_pwait
337 common clock_adjtime sys_clock_adjtime compat_sys_clock_adjtime
338 common syncfs sys_syncfs sys_syncfs
339 common setns sys_setns sys_setns
-340 common process_vm_readv sys_process_vm_readv compat_sys_process_vm_readv
-341 common process_vm_writev sys_process_vm_writev compat_sys_process_vm_writev
+340 common process_vm_readv sys_process_vm_readv sys_process_vm_readv
+341 common process_vm_writev sys_process_vm_writev sys_process_vm_writev
342 common s390_runtime_instr sys_s390_runtime_instr sys_s390_runtime_instr
343 common kcmp sys_kcmp compat_sys_kcmp
344 common finit_module sys_finit_module compat_sys_finit_module
512 x32 rt_sigaction compat_sys_rt_sigaction
513 x32 rt_sigreturn compat_sys_x32_rt_sigreturn
514 x32 ioctl compat_sys_ioctl
-515 x32 readv compat_sys_readv
-516 x32 writev compat_sys_writev
+515 x32 readv sys_readv
+516 x32 writev sys_writev
517 x32 recvfrom compat_sys_recvfrom
518 x32 sendmsg compat_sys_sendmsg
519 x32 recvmsg compat_sys_recvmsg
529 x32 waitid compat_sys_waitid
530 x32 set_robust_list compat_sys_set_robust_list
531 x32 get_robust_list compat_sys_get_robust_list
-532 x32 vmsplice compat_sys_vmsplice
+532 x32 vmsplice sys_vmsplice
533 x32 move_pages compat_sys_move_pages
534 x32 preadv compat_sys_preadv64
535 x32 pwritev compat_sys_pwritev64
536 x32 rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo
537 x32 recvmmsg compat_sys_recvmmsg_time64
538 x32 sendmmsg compat_sys_sendmmsg
-539 x32 process_vm_readv compat_sys_process_vm_readv
-540 x32 process_vm_writev compat_sys_process_vm_writev
+539 x32 process_vm_readv sys_process_vm_readv
+540 x32 process_vm_writev sys_process_vm_writev
541 x32 setsockopt sys_setsockopt
542 x32 getsockopt sys_getsockopt
543 x32 io_setup compat_sys_io_setup
perf-y += kallsyms-parse.o
perf-y += find-bit-bench.o
-perf-$(CONFIG_X86_64) += mem-memcpy-x86-64-lib.o
perf-$(CONFIG_X86_64) += mem-memcpy-x86-64-asm.o
perf-$(CONFIG_X86_64) += mem-memset-x86-64-asm.o
+++ /dev/null
-/*
- * From code in arch/x86/lib/usercopy_64.c, copied to keep tools/ copy
- * of the kernel's arch/x86/lib/memcpy_64.s used in 'perf bench mem memcpy'
- * happy.
- */
-#include <linux/types.h>
-
-unsigned long __memcpy_mcsafe(void *dst, const void *src, size_t cnt);
-unsigned long mcsafe_handle_tail(char *to, char *from, unsigned len);
-
-unsigned long mcsafe_handle_tail(char *to, char *from, unsigned len)
-{
- for (; len; --len, to++, from++) {
- /*
- * Call the assembly routine back directly since
- * memcpy_mcsafe() may silently fallback to memcpy.
- */
- unsigned long rem = __memcpy_mcsafe(to, from, 1);
-
- if (rem)
- break;
- }
- return len;
-}
tracefuncs = {
'sys_sync': {},
'ksys_sync': {},
- '__pm_notifier_call_chain': {},
+ 'pm_notifier_call_chain_robust': {},
'pm_prepare_console': {},
'pm_notifier_call_chain': {},
'freeze_processes': {},
#include "nfit_test.h"
#include "../watermark.h"
-#include <asm/mcsafe_test.h>
+#include <asm/copy_mc_test.h>
+#include <asm/mce.h>
/*
* Generate an NFIT table to describe the following topology:
.id_table = nfit_test_id,
};
-static char mcsafe_buf[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE)));
+static char copy_mc_buf[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE)));
enum INJECT {
INJECT_NONE,
INJECT_DST,
};
-static void mcsafe_test_init(char *dst, char *src, size_t size)
+static void copy_mc_test_init(char *dst, char *src, size_t size)
{
size_t i;
src[i] = (char) i;
}
-static bool mcsafe_test_validate(unsigned char *dst, unsigned char *src,
+static bool copy_mc_test_validate(unsigned char *dst, unsigned char *src,
size_t size, unsigned long rem)
{
size_t i;
return true;
}
-void mcsafe_test(void)
+void copy_mc_test(void)
{
char *inject_desc[] = { "none", "source", "destination" };
enum INJECT inj;
- if (IS_ENABLED(CONFIG_MCSAFE_TEST)) {
+ if (IS_ENABLED(CONFIG_COPY_MC_TEST)) {
pr_info("%s: run...\n", __func__);
} else {
pr_info("%s: disabled, skip.\n", __func__);
switch (inj) {
case INJECT_NONE:
- mcsafe_inject_src(NULL);
- mcsafe_inject_dst(NULL);
- dst = &mcsafe_buf[2048];
- src = &mcsafe_buf[1024 - i];
+ copy_mc_inject_src(NULL);
+ copy_mc_inject_dst(NULL);
+ dst = ©_mc_buf[2048];
+ src = ©_mc_buf[1024 - i];
expect = 0;
break;
case INJECT_SRC:
- mcsafe_inject_src(&mcsafe_buf[1024]);
- mcsafe_inject_dst(NULL);
- dst = &mcsafe_buf[2048];
- src = &mcsafe_buf[1024 - i];
+ copy_mc_inject_src(©_mc_buf[1024]);
+ copy_mc_inject_dst(NULL);
+ dst = ©_mc_buf[2048];
+ src = ©_mc_buf[1024 - i];
expect = 512 - i;
break;
case INJECT_DST:
- mcsafe_inject_src(NULL);
- mcsafe_inject_dst(&mcsafe_buf[2048]);
- dst = &mcsafe_buf[2048 - i];
- src = &mcsafe_buf[1024];
+ copy_mc_inject_src(NULL);
+ copy_mc_inject_dst(©_mc_buf[2048]);
+ dst = ©_mc_buf[2048 - i];
+ src = ©_mc_buf[1024];
expect = 512 - i;
break;
}
- mcsafe_test_init(dst, src, 512);
- rem = __memcpy_mcsafe(dst, src, 512);
- valid = mcsafe_test_validate(dst, src, 512, expect);
+ copy_mc_test_init(dst, src, 512);
+ rem = copy_mc_fragile(dst, src, 512);
+ valid = copy_mc_test_validate(dst, src, 512, expect);
if (rem == expect && valid)
continue;
pr_info("%s: copy(%#lx, %#lx, %d) off: %d rem: %ld %s expect: %ld\n",
}
}
- mcsafe_inject_src(NULL);
- mcsafe_inject_dst(NULL);
+ copy_mc_inject_src(NULL);
+ copy_mc_inject_dst(NULL);
}
static __init int nfit_test_init(void)
libnvdimm_test();
acpi_nfit_test();
device_dax_test();
- mcsafe_test();
+ copy_mc_test();
dax_pmem_test();
dax_pmem_core_test();
#ifdef CONFIG_DEV_DAX_PMEM_COMPAT
ARCH ?= $(shell uname -m 2>/dev/null || echo not)
ifneq (,$(filter $(ARCH),aarch64 arm64))
-ARM64_SUBTARGETS ?= tags signal
+ARM64_SUBTARGETS ?= tags signal pauth fp mte
else
ARM64_SUBTARGETS :=
endif
--- /dev/null
+fpsimd-test
+sve-probe-vls
+sve-ptrace
+sve-test
+vlset
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+
+CFLAGS += -I../../../../../usr/include/
+TEST_GEN_PROGS := sve-ptrace sve-probe-vls
+TEST_PROGS_EXTENDED := fpsimd-test fpsimd-stress sve-test sve-stress vlset
+
+all: $(TEST_GEN_PROGS) $(TEST_PROGS_EXTENDED)
+
+fpsimd-test: fpsimd-test.o
+ $(CC) -nostdlib $^ -o $@
+sve-ptrace: sve-ptrace.o sve-ptrace-asm.o
+sve-probe-vls: sve-probe-vls.o
+sve-test: sve-test.o
+ $(CC) -nostdlib $^ -o $@
+vlset: vlset.o
+
+include ../../lib.mk
--- /dev/null
+This directory contains a mix of tests integrated with kselftest and
+standalone stress tests.
+
+kselftest tests
+===============
+
+sve-probe-vls - Checks the SVE vector length enumeration interface
+sve-ptrace - Checks the SVE ptrace interface
+
+Running the non-kselftest tests
+===============================
+
+sve-stress performs an SVE context switch stress test, as described
+below.
+
+(The fpsimd-stress test works the same way; just substitute "fpsimd" for
+"sve" in the following commands.)
+
+
+The test runs until killed by the user.
+
+If no context switch error was detected, you will see output such as
+the following:
+
+$ ./sve-stress
+(wait for some time)
+^C
+Vector length: 512 bits
+PID: 1573
+Terminated by signal 15, no error, iterations=9467, signals=1014
+Vector length: 512 bits
+PID: 1575
+Terminated by signal 15, no error, iterations=9448, signals=1028
+Vector length: 512 bits
+PID: 1577
+Terminated by signal 15, no error, iterations=9436, signals=1039
+Vector length: 512 bits
+PID: 1579
+Terminated by signal 15, no error, iterations=9421, signals=1039
+Vector length: 512 bits
+PID: 1581
+Terminated by signal 15, no error, iterations=9403, signals=1039
+Vector length: 512 bits
+PID: 1583
+Terminated by signal 15, no error, iterations=9385, signals=1036
+Vector length: 512 bits
+PID: 1585
+Terminated by signal 15, no error, iterations=9376, signals=1039
+Vector length: 512 bits
+PID: 1587
+Terminated by signal 15, no error, iterations=9361, signals=1039
+Vector length: 512 bits
+PID: 1589
+Terminated by signal 15, no error, iterations=9350, signals=1039
+
+
+If an error was detected, details of the mismatch will be printed
+instead of "no error".
+
+Ideally, the test should be allowed to run for many minutes or hours
+to maximise test coverage.
+
+
+KVM stress testing
+==================
+
+To try to reproduce the bugs that we have been observing, sve-stress
+should be run in parallel in two KVM guests, while simultaneously
+running on the host.
+
+1) Start 2 guests, using the following command for each:
+
+$ lkvm run --console=virtio -pconsole=hvc0 --sve Image
+
+(Depending on the hardware GIC implementation, you may also need
+--irqchip=gicv3. New kvmtool defaults to that if appropriate, but I
+can't remember whether my branch is new enough for that. Try without
+the option first.)
+
+Kvmtool occupies the terminal until you kill it (Ctrl+A x),
+or until the guest terminates. It is therefore recommended to run
+each instance in separate terminal (use screen or ssh etc.) This
+allows multiple guests to be run in parallel while running other
+commands on the host.
+
+Within the guest, the host filesystem is accessible, mounted on /host.
+
+2) Run the sve-stress on *each* guest with the Vector-Length set to 32:
+guest$ ./vlset --inherit 32 ./sve-stress
+
+3) Run the sve-stress on the host with the maximum Vector-Length:
+host$ ./vlset --inherit --max ./sve-stress
+
+
+Again, the test should be allowed to run for many minutes or hours to
+maximise test coverage.
+
+If no error is detected, you will see output from each sve-stress
+instance similar to that illustrated above; otherwise details of the
+observed mismatches will be printed.
--- /dev/null
+#define sa_sz 32
+#define sa_flags 8
+#define sa_handler 0
+#define sa_mask_sz 8
+#define SIGUSR1 10
+#define SIGTERM 15
+#define SIGINT 2
+#define SIGABRT 6
+#define SA_NODEFER 1073741824
+#define SA_SIGINFO 4
+#define ucontext_regs 184
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+// Copyright (C) 2015-2019 ARM Limited.
+// Original author: Dave Martin <Dave.Martin@arm.com>
+
+#ifndef ASSEMBLER_H
+#define ASSEMBLER_H
+
+.macro __for from:req, to:req
+ .if (\from) == (\to)
+ _for__body %\from
+ .else
+ __for \from, %(\from) + ((\to) - (\from)) / 2
+ __for %(\from) + ((\to) - (\from)) / 2 + 1, \to
+ .endif
+.endm
+
+.macro _for var:req, from:req, to:req, insn:vararg
+ .macro _for__body \var:req
+ .noaltmacro
+ \insn
+ .altmacro
+ .endm
+
+ .altmacro
+ __for \from, \to
+ .noaltmacro
+
+ .purgem _for__body
+.endm
+
+.macro function name
+ .macro endfunction
+ .type \name, @function
+ .purgem endfunction
+ .endm
+\name:
+.endm
+
+.macro define_accessor name, num, insn
+ .macro \name\()_entry n
+ \insn \n, 1
+ ret
+ .endm
+
+function \name
+ adr x2, .L__accessor_tbl\@
+ add x2, x2, x0, lsl #3
+ br x2
+
+.L__accessor_tbl\@:
+ _for x, 0, (\num) - 1, \name\()_entry \x
+endfunction
+
+ .purgem \name\()_entry
+.endm
+
+#endif /* ! ASSEMBLER_H */
--- /dev/null
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright (C) 2015-2019 ARM Limited.
+# Original author: Dave Martin <Dave.Martin@arm.com>
+
+set -ue
+
+NR_CPUS=`nproc`
+
+pids=
+logs=
+
+cleanup () {
+ trap - INT TERM CHLD
+ set +e
+
+ if [ -n "$pids" ]; then
+ kill $pids
+ wait $pids
+ pids=
+ fi
+
+ if [ -n "$logs" ]; then
+ cat $logs
+ rm $logs
+ logs=
+ fi
+}
+
+interrupt () {
+ cleanup
+ exit 0
+}
+
+child_died () {
+ cleanup
+ exit 1
+}
+
+trap interrupt INT TERM EXIT
+trap child_died CHLD
+
+for x in `seq 0 $((NR_CPUS * 4))`; do
+ log=`mktemp`
+ logs=$logs\ $log
+ ./fpsimd-test >$log &
+ pids=$pids\ $!
+done
+
+# Wait for all child processes to be created:
+sleep 10
+
+while :; do
+ kill -USR1 $pids
+done &
+pids=$pids\ $!
+
+wait
+
+exit 1
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+// Copyright (C) 2015-2019 ARM Limited.
+// Original author: Dave Martin <Dave.Martin@arm.com>
+//
+// Simple FPSIMD context switch test
+// Repeatedly writes unique test patterns into each FPSIMD register
+// and reads them back to verify integrity.
+//
+// for x in `seq 1 NR_CPUS`; do fpsimd-test & pids=$pids\ $! ; done
+// (leave it running for as long as you want...)
+// kill $pids
+
+#include <asm/unistd.h>
+#include "assembler.h"
+#include "asm-offsets.h"
+
+#define NVR 32
+#define MAXVL_B (128 / 8)
+
+.macro _vldr Vn:req, Xt:req
+ ld1 {v\Vn\().2d}, [x\Xt]
+.endm
+
+.macro _vstr Vn:req, Xt:req
+ st1 {v\Vn\().2d}, [x\Xt]
+.endm
+
+// Generate accessor functions to read/write programmatically selected
+// FPSIMD registers.
+// x0 is the register index to access
+// x1 is the memory address to read from (getv,setp) or store to (setv,setp)
+// All clobber x0-x2
+define_accessor setv, NVR, _vldr
+define_accessor getv, NVR, _vstr
+
+// Print a single character x0 to stdout
+// Clobbers x0-x2,x8
+function putc
+ str x0, [sp, #-16]!
+
+ mov x0, #1 // STDOUT_FILENO
+ mov x1, sp
+ mov x2, #1
+ mov x8, #__NR_write
+ svc #0
+
+ add sp, sp, #16
+ ret
+endfunction
+
+// Print a NUL-terminated string starting at address x0 to stdout
+// Clobbers x0-x3,x8
+function puts
+ mov x1, x0
+
+ mov x2, #0
+0: ldrb w3, [x0], #1
+ cbz w3, 1f
+ add x2, x2, #1
+ b 0b
+
+1: mov w0, #1 // STDOUT_FILENO
+ mov x8, #__NR_write
+ svc #0
+
+ ret
+endfunction
+
+// Utility macro to print a literal string
+// Clobbers x0-x4,x8
+.macro puts string
+ .pushsection .rodata.str1.1, "aMS", 1
+.L__puts_literal\@: .string "\string"
+ .popsection
+
+ ldr x0, =.L__puts_literal\@
+ bl puts
+.endm
+
+// Print an unsigned decimal number x0 to stdout
+// Clobbers x0-x4,x8
+function putdec
+ mov x1, sp
+ str x30, [sp, #-32]! // Result can't be > 20 digits
+
+ mov x2, #0
+ strb w2, [x1, #-1]! // Write the NUL terminator
+
+ mov x2, #10
+0: udiv x3, x0, x2 // div-mod loop to generate the digits
+ msub x0, x3, x2, x0
+ add w0, w0, #'0'
+ strb w0, [x1, #-1]!
+ mov x0, x3
+ cbnz x3, 0b
+
+ ldrb w0, [x1]
+ cbnz w0, 1f
+ mov w0, #'0' // Print "0" for 0, not ""
+ strb w0, [x1, #-1]!
+
+1: mov x0, x1
+ bl puts
+
+ ldr x30, [sp], #32
+ ret
+endfunction
+
+// Print an unsigned decimal number x0 to stdout, followed by a newline
+// Clobbers x0-x5,x8
+function putdecn
+ mov x5, x30
+
+ bl putdec
+ mov x0, #'\n'
+ bl putc
+
+ ret x5
+endfunction
+
+
+// Clobbers x0-x3,x8
+function puthexb
+ str x30, [sp, #-0x10]!
+
+ mov w3, w0
+ lsr w0, w0, #4
+ bl puthexnibble
+ mov w0, w3
+
+ ldr x30, [sp], #0x10
+ // fall through to puthexnibble
+endfunction
+// Clobbers x0-x2,x8
+function puthexnibble
+ and w0, w0, #0xf
+ cmp w0, #10
+ blo 1f
+ add w0, w0, #'a' - ('9' + 1)
+1: add w0, w0, #'0'
+ b putc
+endfunction
+
+// x0=data in, x1=size in, clobbers x0-x5,x8
+function dumphex
+ str x30, [sp, #-0x10]!
+
+ mov x4, x0
+ mov x5, x1
+
+0: subs x5, x5, #1
+ b.lo 1f
+ ldrb w0, [x4], #1
+ bl puthexb
+ b 0b
+
+1: ldr x30, [sp], #0x10
+ ret
+endfunction
+
+// Declare some storate space to shadow the SVE register contents:
+.pushsection .text
+.data
+.align 4
+vref:
+ .space MAXVL_B * NVR
+scratch:
+ .space MAXVL_B
+.popsection
+
+// Trivial memory copy: copy x2 bytes, starting at address x1, to address x0.
+// Clobbers x0-x3
+function memcpy
+ cmp x2, #0
+ b.eq 1f
+0: ldrb w3, [x1], #1
+ strb w3, [x0], #1
+ subs x2, x2, #1
+ b.ne 0b
+1: ret
+endfunction
+
+// Generate a test pattern for storage in SVE registers
+// x0: pid (16 bits)
+// x1: register number (6 bits)
+// x2: generation (4 bits)
+function pattern
+ orr w1, w0, w1, lsl #16
+ orr w2, w1, w2, lsl #28
+
+ ldr x0, =scratch
+ mov w1, #MAXVL_B / 4
+
+0: str w2, [x0], #4
+ add w2, w2, #(1 << 22)
+ subs w1, w1, #1
+ bne 0b
+
+ ret
+endfunction
+
+// Get the address of shadow data for FPSIMD V-register V<xn>
+.macro _adrv xd, xn, nrtmp
+ ldr \xd, =vref
+ mov x\nrtmp, #16
+ madd \xd, x\nrtmp, \xn, \xd
+.endm
+
+// Set up test pattern in a FPSIMD V-register
+// x0: pid
+// x1: register number
+// x2: generation
+function setup_vreg
+ mov x4, x30
+
+ mov x6, x1
+ bl pattern
+ _adrv x0, x6, 2
+ mov x5, x0
+ ldr x1, =scratch
+ bl memcpy
+
+ mov x0, x6
+ mov x1, x5
+ bl setv
+
+ ret x4
+endfunction
+
+// Fill x1 bytes starting at x0 with 0xae (for canary purposes)
+// Clobbers x1, x2.
+function memfill_ae
+ mov w2, #0xae
+ b memfill
+endfunction
+
+// Fill x1 bytes starting at x0 with 0.
+// Clobbers x1, x2.
+function memclr
+ mov w2, #0
+endfunction
+ // fall through to memfill
+
+// Trivial memory fill: fill x1 bytes starting at address x0 with byte w2
+// Clobbers x1
+function memfill
+ cmp x1, #0
+ b.eq 1f
+
+0: strb w2, [x0], #1
+ subs x1, x1, #1
+ b.ne 0b
+
+1: ret
+endfunction
+
+// Trivial memory compare: compare x2 bytes starting at address x0 with
+// bytes starting at address x1.
+// Returns only if all bytes match; otherwise, the program is aborted.
+// Clobbers x0-x5.
+function memcmp
+ cbz x2, 1f
+
+ mov x5, #0
+0: ldrb w3, [x0, x5]
+ ldrb w4, [x1, x5]
+ add x5, x5, #1
+ cmp w3, w4
+ b.ne barf
+ subs x2, x2, #1
+ b.ne 0b
+
+1: ret
+endfunction
+
+// Verify that a FPSIMD V-register matches its shadow in memory, else abort
+// x0: reg number
+// Clobbers x0-x5.
+function check_vreg
+ mov x3, x30
+
+ _adrv x5, x0, 6
+ mov x4, x0
+ ldr x7, =scratch
+
+ mov x0, x7
+ mov x1, x6
+ bl memfill_ae
+
+ mov x0, x4
+ mov x1, x7
+ bl getv
+
+ mov x0, x5
+ mov x1, x7
+ mov x2, x6
+ mov x30, x3
+ b memcmp
+endfunction
+
+// Any SVE register modified here can cause corruption in the main
+// thread -- but *only* the registers modified here.
+function irritator_handler
+ // Increment the irritation signal count (x23):
+ ldr x0, [x2, #ucontext_regs + 8 * 23]
+ add x0, x0, #1
+ str x0, [x2, #ucontext_regs + 8 * 23]
+
+ // Corrupt some random V-regs
+ adr x0, .text + (irritator_handler - .text) / 16 * 16
+ movi v0.8b, #7
+ movi v9.16b, #9
+ movi v31.8b, #31
+
+ ret
+endfunction
+
+function terminate_handler
+ mov w21, w0
+ mov x20, x2
+
+ puts "Terminated by signal "
+ mov w0, w21
+ bl putdec
+ puts ", no error, iterations="
+ ldr x0, [x20, #ucontext_regs + 8 * 22]
+ bl putdec
+ puts ", signals="
+ ldr x0, [x20, #ucontext_regs + 8 * 23]
+ bl putdecn
+
+ mov x0, #0
+ mov x8, #__NR_exit
+ svc #0
+endfunction
+
+// w0: signal number
+// x1: sa_action
+// w2: sa_flags
+// Clobbers x0-x6,x8
+function setsignal
+ str x30, [sp, #-((sa_sz + 15) / 16 * 16 + 16)]!
+
+ mov w4, w0
+ mov x5, x1
+ mov w6, w2
+
+ add x0, sp, #16
+ mov x1, #sa_sz
+ bl memclr
+
+ mov w0, w4
+ add x1, sp, #16
+ str w6, [x1, #sa_flags]
+ str x5, [x1, #sa_handler]
+ mov x2, #0
+ mov x3, #sa_mask_sz
+ mov x8, #__NR_rt_sigaction
+ svc #0
+
+ cbz w0, 1f
+
+ puts "sigaction failure\n"
+ b .Labort
+
+1: ldr x30, [sp], #((sa_sz + 15) / 16 * 16 + 16)
+ ret
+endfunction
+
+// Main program entry point
+.globl _start
+function _start
+_start:
+ // Sanity-check and report the vector length
+
+ mov x19, #128
+ cmp x19, #128
+ b.lo 1f
+ cmp x19, #2048
+ b.hi 1f
+ tst x19, #(8 - 1)
+ b.eq 2f
+
+1: puts "Bad vector length: "
+ mov x0, x19
+ bl putdecn
+ b .Labort
+
+2: puts "Vector length:\t"
+ mov x0, x19
+ bl putdec
+ puts " bits\n"
+
+ // Obtain our PID, to ensure test pattern uniqueness between processes
+
+ mov x8, #__NR_getpid
+ svc #0
+ mov x20, x0
+
+ puts "PID:\t"
+ mov x0, x20
+ bl putdecn
+
+ mov x23, #0 // Irritation signal count
+
+ mov w0, #SIGINT
+ adr x1, terminate_handler
+ mov w2, #SA_SIGINFO
+ bl setsignal
+
+ mov w0, #SIGTERM
+ adr x1, terminate_handler
+ mov w2, #SA_SIGINFO
+ bl setsignal
+
+ mov w0, #SIGUSR1
+ adr x1, irritator_handler
+ mov w2, #SA_SIGINFO
+ orr w2, w2, #SA_NODEFER
+ bl setsignal
+
+ mov x22, #0 // generation number, increments per iteration
+.Ltest_loop:
+
+ mov x21, #0 // Set up V-regs & shadow with test pattern
+0: mov x0, x20
+ mov x1, x21
+ and x2, x22, #0xf
+ bl setup_vreg
+ add x21, x21, #1
+ cmp x21, #NVR
+ b.lo 0b
+
+// Can't do this when SVE state is volatile across SVC:
+ mov x8, #__NR_sched_yield // Encourage preemption
+ svc #0
+
+ mov x21, #0
+0: mov x0, x21
+ bl check_vreg
+ add x21, x21, #1
+ cmp x21, #NVR
+ b.lo 0b
+
+ add x22, x22, #1
+ b .Ltest_loop
+
+.Labort:
+ mov x0, #0
+ mov x1, #SIGABRT
+ mov x8, #__NR_kill
+ svc #0
+endfunction
+
+function barf
+ mov x10, x0 // expected data
+ mov x11, x1 // actual data
+ mov x12, x2 // data size
+
+ puts "Mistatch: PID="
+ mov x0, x20
+ bl putdec
+ puts ", iteration="
+ mov x0, x22
+ bl putdec
+ puts ", reg="
+ mov x0, x21
+ bl putdecn
+ puts "\tExpected ["
+ mov x0, x10
+ mov x1, x12
+ bl dumphex
+ puts "]\n\tGot ["
+ mov x0, x11
+ mov x1, x12
+ bl dumphex
+ puts "]\n"
+
+ mov x8, #__NR_exit
+ mov x1, #1
+ svc #0
+endfunction
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2015-2020 ARM Limited.
+ * Original author: Dave Martin <Dave.Martin@arm.com>
+ */
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/auxv.h>
+#include <sys/prctl.h>
+#include <asm/sigcontext.h>
+
+#include "../../kselftest.h"
+
+int main(int argc, char **argv)
+{
+ unsigned int vq;
+ int vl;
+ static unsigned int vqs[SVE_VQ_MAX];
+ unsigned int nvqs = 0;
+
+ ksft_print_header();
+ ksft_set_plan(2);
+
+ if (!(getauxval(AT_HWCAP) & HWCAP_SVE))
+ ksft_exit_skip("SVE not available");
+
+ /*
+ * Enumerate up to SVE_VQ_MAX vector lengths
+ */
+ for (vq = SVE_VQ_MAX; vq > 0; --vq) {
+ vl = prctl(PR_SVE_SET_VL, vq * 16);
+ if (vl == -1)
+ ksft_exit_fail_msg("PR_SVE_SET_VL failed: %s (%d)\n",
+ strerror(errno), errno);
+
+ vl &= PR_SVE_VL_LEN_MASK;
+
+ if (!sve_vl_valid(vl))
+ ksft_exit_fail_msg("VL %d invalid\n", vl);
+ vq = sve_vq_from_vl(vl);
+
+ if (!(nvqs < SVE_VQ_MAX))
+ ksft_exit_fail_msg("Too many VLs %u >= SVE_VQ_MAX\n",
+ nvqs);
+ vqs[nvqs++] = vq;
+ }
+ ksft_test_result_pass("Enumerated %d vector lengths\n", nvqs);
+ ksft_test_result_pass("All vector lengths valid\n");
+
+ /* Print out the vector lengths in ascending order: */
+ while (nvqs--)
+ ksft_print_msg("%u\n", 16 * vqs[nvqs]);
+
+ ksft_exit_pass();
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+// Copyright (C) 2015-2019 ARM Limited.
+// Original author: Dave Martin <Dave.Martin@arm.com>
+#include <asm/unistd.h>
+
+.arch_extension sve
+
+.globl sve_store_patterns
+
+sve_store_patterns:
+ mov x1, x0
+
+ index z0.b, #0, #1
+ str q0, [x1]
+
+ mov w8, #__NR_getpid
+ svc #0
+ str q0, [x1, #0x10]
+
+ mov z1.d, z0.d
+ str q0, [x1, #0x20]
+
+ mov w8, #__NR_getpid
+ svc #0
+ str q0, [x1, #0x30]
+
+ mov z1.d, z0.d
+ str q0, [x1, #0x40]
+
+ ret
+
+.size sve_store_patterns, . - sve_store_patterns
+.type sve_store_patterns, @function
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2015-2020 ARM Limited.
+ * Original author: Dave Martin <Dave.Martin@arm.com>
+ */
+#include <errno.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/auxv.h>
+#include <sys/ptrace.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/wait.h>
+#include <asm/sigcontext.h>
+#include <asm/ptrace.h>
+
+#include "../../kselftest.h"
+
+/* <linux/elf.h> and <sys/auxv.h> don't like each other, so: */
+#ifndef NT_ARM_SVE
+#define NT_ARM_SVE 0x405
+#endif
+
+/* Number of registers filled in by sve_store_patterns */
+#define NR_VREGS 5
+
+void sve_store_patterns(__uint128_t v[NR_VREGS]);
+
+static void dump(const void *buf, size_t size)
+{
+ size_t i;
+ const unsigned char *p = buf;
+
+ for (i = 0; i < size; ++i)
+ printf(" %.2x", *p++);
+}
+
+static int check_vregs(const __uint128_t vregs[NR_VREGS])
+{
+ int i;
+ int ok = 1;
+
+ for (i = 0; i < NR_VREGS; ++i) {
+ printf("# v[%d]:", i);
+ dump(&vregs[i], sizeof vregs[i]);
+ putchar('\n');
+
+ if (vregs[i] != vregs[0])
+ ok = 0;
+ }
+
+ return ok;
+}
+
+static int do_child(void)
+{
+ if (ptrace(PTRACE_TRACEME, -1, NULL, NULL))
+ ksft_exit_fail_msg("PTRACE_TRACEME", strerror(errno));
+
+ if (raise(SIGSTOP))
+ ksft_exit_fail_msg("raise(SIGSTOP)", strerror(errno));
+
+ return EXIT_SUCCESS;
+}
+
+static struct user_sve_header *get_sve(pid_t pid, void **buf, size_t *size)
+{
+ struct user_sve_header *sve;
+ void *p;
+ size_t sz = sizeof *sve;
+ struct iovec iov;
+
+ while (1) {
+ if (*size < sz) {
+ p = realloc(*buf, sz);
+ if (!p) {
+ errno = ENOMEM;
+ goto error;
+ }
+
+ *buf = p;
+ *size = sz;
+ }
+
+ iov.iov_base = *buf;
+ iov.iov_len = sz;
+ if (ptrace(PTRACE_GETREGSET, pid, NT_ARM_SVE, &iov))
+ goto error;
+
+ sve = *buf;
+ if (sve->size <= sz)
+ break;
+
+ sz = sve->size;
+ }
+
+ return sve;
+
+error:
+ return NULL;
+}
+
+static int set_sve(pid_t pid, const struct user_sve_header *sve)
+{
+ struct iovec iov;
+
+ iov.iov_base = (void *)sve;
+ iov.iov_len = sve->size;
+ return ptrace(PTRACE_SETREGSET, pid, NT_ARM_SVE, &iov);
+}
+
+static void dump_sve_regs(const struct user_sve_header *sve, unsigned int num,
+ unsigned int vlmax)
+{
+ unsigned int vq;
+ unsigned int i;
+
+ if ((sve->flags & SVE_PT_REGS_MASK) != SVE_PT_REGS_SVE)
+ ksft_exit_fail_msg("Dumping non-SVE register\n");
+
+ if (vlmax > sve->vl)
+ vlmax = sve->vl;
+
+ vq = sve_vq_from_vl(sve->vl);
+ for (i = 0; i < num; ++i) {
+ printf("# z%u:", i);
+ dump((const char *)sve + SVE_PT_SVE_ZREG_OFFSET(vq, i),
+ vlmax);
+ printf("%s\n", vlmax == sve->vl ? "" : " ...");
+ }
+}
+
+static int do_parent(pid_t child)
+{
+ int ret = EXIT_FAILURE;
+ pid_t pid;
+ int status;
+ siginfo_t si;
+ void *svebuf = NULL, *newsvebuf;
+ size_t svebufsz = 0, newsvebufsz;
+ struct user_sve_header *sve, *new_sve;
+ struct user_fpsimd_state *fpsimd;
+ unsigned int i, j;
+ unsigned char *p;
+ unsigned int vq;
+
+ /* Attach to the child */
+ while (1) {
+ int sig;
+
+ pid = wait(&status);
+ if (pid == -1) {
+ perror("wait");
+ goto error;
+ }
+
+ /*
+ * This should never happen but it's hard to flag in
+ * the framework.
+ */
+ if (pid != child)
+ continue;
+
+ if (WIFEXITED(status) || WIFSIGNALED(status))
+ ksft_exit_fail_msg("Child died unexpectedly\n");
+
+ ksft_test_result(WIFSTOPPED(status), "WIFSTOPPED(%d)\n",
+ status);
+ if (!WIFSTOPPED(status))
+ goto error;
+
+ sig = WSTOPSIG(status);
+
+ if (ptrace(PTRACE_GETSIGINFO, pid, NULL, &si)) {
+ if (errno == ESRCH)
+ goto disappeared;
+
+ if (errno == EINVAL) {
+ sig = 0; /* bust group-stop */
+ goto cont;
+ }
+
+ ksft_test_result_fail("PTRACE_GETSIGINFO: %s\n",
+ strerror(errno));
+ goto error;
+ }
+
+ if (sig == SIGSTOP && si.si_code == SI_TKILL &&
+ si.si_pid == pid)
+ break;
+
+ cont:
+ if (ptrace(PTRACE_CONT, pid, NULL, sig)) {
+ if (errno == ESRCH)
+ goto disappeared;
+
+ ksft_test_result_fail("PTRACE_CONT: %s\n",
+ strerror(errno));
+ goto error;
+ }
+ }
+
+ sve = get_sve(pid, &svebuf, &svebufsz);
+ if (!sve) {
+ int e = errno;
+
+ ksft_test_result_fail("get_sve: %s\n", strerror(errno));
+ if (e == ESRCH)
+ goto disappeared;
+
+ goto error;
+ } else {
+ ksft_test_result_pass("get_sve\n");
+ }
+
+ ksft_test_result((sve->flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD,
+ "FPSIMD registers\n");
+ if ((sve->flags & SVE_PT_REGS_MASK) != SVE_PT_REGS_FPSIMD)
+ goto error;
+
+ fpsimd = (struct user_fpsimd_state *)((char *)sve +
+ SVE_PT_FPSIMD_OFFSET);
+ for (i = 0; i < 32; ++i) {
+ p = (unsigned char *)&fpsimd->vregs[i];
+
+ for (j = 0; j < sizeof fpsimd->vregs[i]; ++j)
+ p[j] = j;
+ }
+
+ if (set_sve(pid, sve)) {
+ int e = errno;
+
+ ksft_test_result_fail("set_sve(FPSIMD): %s\n",
+ strerror(errno));
+ if (e == ESRCH)
+ goto disappeared;
+
+ goto error;
+ }
+
+ vq = sve_vq_from_vl(sve->vl);
+
+ newsvebufsz = SVE_PT_SVE_ZREG_OFFSET(vq, 1);
+ new_sve = newsvebuf = malloc(newsvebufsz);
+ if (!new_sve) {
+ errno = ENOMEM;
+ perror(NULL);
+ goto error;
+ }
+
+ *new_sve = *sve;
+ new_sve->flags &= ~SVE_PT_REGS_MASK;
+ new_sve->flags |= SVE_PT_REGS_SVE;
+ memset((char *)new_sve + SVE_PT_SVE_ZREG_OFFSET(vq, 0),
+ 0, SVE_PT_SVE_ZREG_SIZE(vq));
+ new_sve->size = SVE_PT_SVE_ZREG_OFFSET(vq, 1);
+ if (set_sve(pid, new_sve)) {
+ int e = errno;
+
+ ksft_test_result_fail("set_sve(ZREG): %s\n", strerror(errno));
+ if (e == ESRCH)
+ goto disappeared;
+
+ goto error;
+ }
+
+ new_sve = get_sve(pid, &newsvebuf, &newsvebufsz);
+ if (!new_sve) {
+ int e = errno;
+
+ ksft_test_result_fail("get_sve(ZREG): %s\n", strerror(errno));
+ if (e == ESRCH)
+ goto disappeared;
+
+ goto error;
+ }
+
+ ksft_test_result((new_sve->flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_SVE,
+ "SVE registers\n");
+ if ((new_sve->flags & SVE_PT_REGS_MASK) != SVE_PT_REGS_SVE)
+ goto error;
+
+ dump_sve_regs(new_sve, 3, sizeof fpsimd->vregs[0]);
+
+ p = (unsigned char *)new_sve + SVE_PT_SVE_ZREG_OFFSET(vq, 1);
+ for (i = 0; i < sizeof fpsimd->vregs[0]; ++i) {
+ unsigned char expected = i;
+
+ if (__BYTE_ORDER == __BIG_ENDIAN)
+ expected = sizeof fpsimd->vregs[0] - 1 - expected;
+
+ ksft_test_result(p[i] == expected, "p[%d] == expected\n", i);
+ if (p[i] != expected)
+ goto error;
+ }
+
+ ret = EXIT_SUCCESS;
+
+error:
+ kill(child, SIGKILL);
+
+disappeared:
+ return ret;
+}
+
+int main(void)
+{
+ int ret = EXIT_SUCCESS;
+ __uint128_t v[NR_VREGS];
+ pid_t child;
+
+ ksft_print_header();
+ ksft_set_plan(20);
+
+ if (!(getauxval(AT_HWCAP) & HWCAP_SVE))
+ ksft_exit_skip("SVE not available\n");
+
+ sve_store_patterns(v);
+
+ if (!check_vregs(v))
+ ksft_exit_fail_msg("Initial check_vregs() failed\n");
+
+ child = fork();
+ if (!child)
+ return do_child();
+
+ if (do_parent(child))
+ ret = EXIT_FAILURE;
+
+ ksft_print_cnts();
+
+ return 0;
+}
--- /dev/null
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0-only
+# Copyright (C) 2015-2019 ARM Limited.
+# Original author: Dave Martin <Dave.Martin@arm.com>
+
+set -ue
+
+NR_CPUS=`nproc`
+
+pids=
+logs=
+
+cleanup () {
+ trap - INT TERM CHLD
+ set +e
+
+ if [ -n "$pids" ]; then
+ kill $pids
+ wait $pids
+ pids=
+ fi
+
+ if [ -n "$logs" ]; then
+ cat $logs
+ rm $logs
+ logs=
+ fi
+}
+
+interrupt () {
+ cleanup
+ exit 0
+}
+
+child_died () {
+ cleanup
+ exit 1
+}
+
+trap interrupt INT TERM EXIT
+
+for x in `seq 0 $((NR_CPUS * 4))`; do
+ log=`mktemp`
+ logs=$logs\ $log
+ ./sve-test >$log &
+ pids=$pids\ $!
+done
+
+# Wait for all child processes to be created:
+sleep 10
+
+while :; do
+ kill -USR1 $pids
+done &
+pids=$pids\ $!
+
+wait
+
+exit 1
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+// Copyright (C) 2015-2019 ARM Limited.
+// Original author: Dave Martin <Dave.Martin@arm.com>
+//
+// Simple Scalable Vector Extension context switch test
+// Repeatedly writes unique test patterns into each SVE register
+// and reads them back to verify integrity.
+//
+// for x in `seq 1 NR_CPUS`; do sve-test & pids=$pids\ $! ; done
+// (leave it running for as long as you want...)
+// kill $pids
+
+#include <asm/unistd.h>
+#include "assembler.h"
+#include "asm-offsets.h"
+
+#define NZR 32
+#define NPR 16
+#define MAXVL_B (2048 / 8)
+
+.arch_extension sve
+
+.macro _sve_ldr_v zt, xn
+ ldr z\zt, [x\xn]
+.endm
+
+.macro _sve_str_v zt, xn
+ str z\zt, [x\xn]
+.endm
+
+.macro _sve_ldr_p pt, xn
+ ldr p\pt, [x\xn]
+.endm
+
+.macro _sve_str_p pt, xn
+ str p\pt, [x\xn]
+.endm
+
+// Generate accessor functions to read/write programmatically selected
+// SVE registers.
+// x0 is the register index to access
+// x1 is the memory address to read from (getz,setp) or store to (setz,setp)
+// All clobber x0-x2
+define_accessor setz, NZR, _sve_ldr_v
+define_accessor getz, NZR, _sve_str_v
+define_accessor setp, NPR, _sve_ldr_p
+define_accessor getp, NPR, _sve_str_p
+
+// Print a single character x0 to stdout
+// Clobbers x0-x2,x8
+function putc
+ str x0, [sp, #-16]!
+
+ mov x0, #1 // STDOUT_FILENO
+ mov x1, sp
+ mov x2, #1
+ mov x8, #__NR_write
+ svc #0
+
+ add sp, sp, #16
+ ret
+endfunction
+
+// Print a NUL-terminated string starting at address x0 to stdout
+// Clobbers x0-x3,x8
+function puts
+ mov x1, x0
+
+ mov x2, #0
+0: ldrb w3, [x0], #1
+ cbz w3, 1f
+ add x2, x2, #1
+ b 0b
+
+1: mov w0, #1 // STDOUT_FILENO
+ mov x8, #__NR_write
+ svc #0
+
+ ret
+endfunction
+
+// Utility macro to print a literal string
+// Clobbers x0-x4,x8
+.macro puts string
+ .pushsection .rodata.str1.1, "aMS", 1
+.L__puts_literal\@: .string "\string"
+ .popsection
+
+ ldr x0, =.L__puts_literal\@
+ bl puts
+.endm
+
+// Print an unsigned decimal number x0 to stdout
+// Clobbers x0-x4,x8
+function putdec
+ mov x1, sp
+ str x30, [sp, #-32]! // Result can't be > 20 digits
+
+ mov x2, #0
+ strb w2, [x1, #-1]! // Write the NUL terminator
+
+ mov x2, #10
+0: udiv x3, x0, x2 // div-mod loop to generate the digits
+ msub x0, x3, x2, x0
+ add w0, w0, #'0'
+ strb w0, [x1, #-1]!
+ mov x0, x3
+ cbnz x3, 0b
+
+ ldrb w0, [x1]
+ cbnz w0, 1f
+ mov w0, #'0' // Print "0" for 0, not ""
+ strb w0, [x1, #-1]!
+
+1: mov x0, x1
+ bl puts
+
+ ldr x30, [sp], #32
+ ret
+endfunction
+
+// Print an unsigned decimal number x0 to stdout, followed by a newline
+// Clobbers x0-x5,x8
+function putdecn
+ mov x5, x30
+
+ bl putdec
+ mov x0, #'\n'
+ bl putc
+
+ ret x5
+endfunction
+
+// Clobbers x0-x3,x8
+function puthexb
+ str x30, [sp, #-0x10]!
+
+ mov w3, w0
+ lsr w0, w0, #4
+ bl puthexnibble
+ mov w0, w3
+
+ ldr x30, [sp], #0x10
+ // fall through to puthexnibble
+endfunction
+// Clobbers x0-x2,x8
+function puthexnibble
+ and w0, w0, #0xf
+ cmp w0, #10
+ blo 1f
+ add w0, w0, #'a' - ('9' + 1)
+1: add w0, w0, #'0'
+ b putc
+endfunction
+
+// x0=data in, x1=size in, clobbers x0-x5,x8
+function dumphex
+ str x30, [sp, #-0x10]!
+
+ mov x4, x0
+ mov x5, x1
+
+0: subs x5, x5, #1
+ b.lo 1f
+ ldrb w0, [x4], #1
+ bl puthexb
+ b 0b
+
+1: ldr x30, [sp], #0x10
+ ret
+endfunction
+
+// Declare some storate space to shadow the SVE register contents:
+.pushsection .text
+.data
+.align 4
+zref:
+ .space MAXVL_B * NZR
+pref:
+ .space MAXVL_B / 8 * NPR
+ffrref:
+ .space MAXVL_B / 8
+scratch:
+ .space MAXVL_B
+.popsection
+
+// Trivial memory copy: copy x2 bytes, starting at address x1, to address x0.
+// Clobbers x0-x3
+function memcpy
+ cmp x2, #0
+ b.eq 1f
+0: ldrb w3, [x1], #1
+ strb w3, [x0], #1
+ subs x2, x2, #1
+ b.ne 0b
+1: ret
+endfunction
+
+// Generate a test pattern for storage in SVE registers
+// x0: pid (16 bits)
+// x1: register number (6 bits)
+// x2: generation (4 bits)
+
+// These values are used to constuct a 32-bit pattern that is repeated in the
+// scratch buffer as many times as will fit:
+// bits 31:28 generation number (increments once per test_loop)
+// bits 27:22 32-bit lane index
+// bits 21:16 register number
+// bits 15: 0 pid
+
+function pattern
+ orr w1, w0, w1, lsl #16
+ orr w2, w1, w2, lsl #28
+
+ ldr x0, =scratch
+ mov w1, #MAXVL_B / 4
+
+0: str w2, [x0], #4
+ add w2, w2, #(1 << 22)
+ subs w1, w1, #1
+ bne 0b
+
+ ret
+endfunction
+
+// Get the address of shadow data for SVE Z-register Z<xn>
+.macro _adrz xd, xn, nrtmp
+ ldr \xd, =zref
+ rdvl x\nrtmp, #1
+ madd \xd, x\nrtmp, \xn, \xd
+.endm
+
+// Get the address of shadow data for SVE P-register P<xn - NZR>
+.macro _adrp xd, xn, nrtmp
+ ldr \xd, =pref
+ rdvl x\nrtmp, #1
+ lsr x\nrtmp, x\nrtmp, #3
+ sub \xn, \xn, #NZR
+ madd \xd, x\nrtmp, \xn, \xd
+.endm
+
+// Set up test pattern in a SVE Z-register
+// x0: pid
+// x1: register number
+// x2: generation
+function setup_zreg
+ mov x4, x30
+
+ mov x6, x1
+ bl pattern
+ _adrz x0, x6, 2
+ mov x5, x0
+ ldr x1, =scratch
+ bl memcpy
+
+ mov x0, x6
+ mov x1, x5
+ bl setz
+
+ ret x4
+endfunction
+
+// Set up test pattern in a SVE P-register
+// x0: pid
+// x1: register number
+// x2: generation
+function setup_preg
+ mov x4, x30
+
+ mov x6, x1
+ bl pattern
+ _adrp x0, x6, 2
+ mov x5, x0
+ ldr x1, =scratch
+ bl memcpy
+
+ mov x0, x6
+ mov x1, x5
+ bl setp
+
+ ret x4
+endfunction
+
+// Set up test pattern in the FFR
+// x0: pid
+// x2: generation
+// Beware: corrupts P0.
+function setup_ffr
+ mov x4, x30
+
+ bl pattern
+ ldr x0, =ffrref
+ ldr x1, =scratch
+ rdvl x2, #1
+ lsr x2, x2, #3
+ bl memcpy
+
+ mov x0, #0
+ ldr x1, =ffrref
+ bl setp
+
+ wrffr p0.b
+
+ ret x4
+endfunction
+
+// Fill x1 bytes starting at x0 with 0xae (for canary purposes)
+// Clobbers x1, x2.
+function memfill_ae
+ mov w2, #0xae
+ b memfill
+endfunction
+
+// Fill x1 bytes starting at x0 with 0.
+// Clobbers x1, x2.
+function memclr
+ mov w2, #0
+endfunction
+ // fall through to memfill
+
+// Trivial memory fill: fill x1 bytes starting at address x0 with byte w2
+// Clobbers x1
+function memfill
+ cmp x1, #0
+ b.eq 1f
+
+0: strb w2, [x0], #1
+ subs x1, x1, #1
+ b.ne 0b
+
+1: ret
+endfunction
+
+// Trivial memory compare: compare x2 bytes starting at address x0 with
+// bytes starting at address x1.
+// Returns only if all bytes match; otherwise, the program is aborted.
+// Clobbers x0-x5.
+function memcmp
+ cbz x2, 2f
+
+ stp x0, x1, [sp, #-0x20]!
+ str x2, [sp, #0x10]
+
+ mov x5, #0
+0: ldrb w3, [x0, x5]
+ ldrb w4, [x1, x5]
+ add x5, x5, #1
+ cmp w3, w4
+ b.ne 1f
+ subs x2, x2, #1
+ b.ne 0b
+
+1: ldr x2, [sp, #0x10]
+ ldp x0, x1, [sp], #0x20
+ b.ne barf
+
+2: ret
+endfunction
+
+// Verify that a SVE Z-register matches its shadow in memory, else abort
+// x0: reg number
+// Clobbers x0-x7.
+function check_zreg
+ mov x3, x30
+
+ _adrz x5, x0, 6
+ mov x4, x0
+ ldr x7, =scratch
+
+ mov x0, x7
+ mov x1, x6
+ bl memfill_ae
+
+ mov x0, x4
+ mov x1, x7
+ bl getz
+
+ mov x0, x5
+ mov x1, x7
+ mov x2, x6
+ mov x30, x3
+ b memcmp
+endfunction
+
+// Verify that a SVE P-register matches its shadow in memory, else abort
+// x0: reg number
+// Clobbers x0-x7.
+function check_preg
+ mov x3, x30
+
+ _adrp x5, x0, 6
+ mov x4, x0
+ ldr x7, =scratch
+
+ mov x0, x7
+ mov x1, x6
+ bl memfill_ae
+
+ mov x0, x4
+ mov x1, x7
+ bl getp
+
+ mov x0, x5
+ mov x1, x7
+ mov x2, x6
+ mov x30, x3
+ b memcmp
+endfunction
+
+// Verify that the FFR matches its shadow in memory, else abort
+// Beware -- corrupts P0.
+// Clobbers x0-x5.
+function check_ffr
+ mov x3, x30
+
+ ldr x4, =scratch
+ rdvl x5, #1
+ lsr x5, x5, #3
+
+ mov x0, x4
+ mov x1, x5
+ bl memfill_ae
+
+ rdffr p0.b
+ mov x0, #0
+ mov x1, x4
+ bl getp
+
+ ldr x0, =ffrref
+ mov x1, x4
+ mov x2, x5
+ mov x30, x3
+ b memcmp
+endfunction
+
+// Any SVE register modified here can cause corruption in the main
+// thread -- but *only* the registers modified here.
+function irritator_handler
+ // Increment the irritation signal count (x23):
+ ldr x0, [x2, #ucontext_regs + 8 * 23]
+ add x0, x0, #1
+ str x0, [x2, #ucontext_regs + 8 * 23]
+
+ // Corrupt some random Z-regs
+ adr x0, .text + (irritator_handler - .text) / 16 * 16
+ movi v0.8b, #1
+ movi v9.16b, #2
+ movi v31.8b, #3
+ // And P0
+ rdffr p0.b
+ // And FFR
+ wrffr p15.b
+
+ ret
+endfunction
+
+function terminate_handler
+ mov w21, w0
+ mov x20, x2
+
+ puts "Terminated by signal "
+ mov w0, w21
+ bl putdec
+ puts ", no error, iterations="
+ ldr x0, [x20, #ucontext_regs + 8 * 22]
+ bl putdec
+ puts ", signals="
+ ldr x0, [x20, #ucontext_regs + 8 * 23]
+ bl putdecn
+
+ mov x0, #0
+ mov x8, #__NR_exit
+ svc #0
+endfunction
+
+// w0: signal number
+// x1: sa_action
+// w2: sa_flags
+// Clobbers x0-x6,x8
+function setsignal
+ str x30, [sp, #-((sa_sz + 15) / 16 * 16 + 16)]!
+
+ mov w4, w0
+ mov x5, x1
+ mov w6, w2
+
+ add x0, sp, #16
+ mov x1, #sa_sz
+ bl memclr
+
+ mov w0, w4
+ add x1, sp, #16
+ str w6, [x1, #sa_flags]
+ str x5, [x1, #sa_handler]
+ mov x2, #0
+ mov x3, #sa_mask_sz
+ mov x8, #__NR_rt_sigaction
+ svc #0
+
+ cbz w0, 1f
+
+ puts "sigaction failure\n"
+ b .Labort
+
+1: ldr x30, [sp], #((sa_sz + 15) / 16 * 16 + 16)
+ ret
+endfunction
+
+// Main program entry point
+.globl _start
+function _start
+_start:
+ // Sanity-check and report the vector length
+
+ rdvl x19, #8
+ cmp x19, #128
+ b.lo 1f
+ cmp x19, #2048
+ b.hi 1f
+ tst x19, #(8 - 1)
+ b.eq 2f
+
+1: puts "Bad vector length: "
+ mov x0, x19
+ bl putdecn
+ b .Labort
+
+2: puts "Vector length:\t"
+ mov x0, x19
+ bl putdec
+ puts " bits\n"
+
+ // Obtain our PID, to ensure test pattern uniqueness between processes
+
+ mov x8, #__NR_getpid
+ svc #0
+ mov x20, x0
+
+ puts "PID:\t"
+ mov x0, x20
+ bl putdecn
+
+ mov x23, #0 // Irritation signal count
+
+ mov w0, #SIGINT
+ adr x1, terminate_handler
+ mov w2, #SA_SIGINFO
+ bl setsignal
+
+ mov w0, #SIGTERM
+ adr x1, terminate_handler
+ mov w2, #SA_SIGINFO
+ bl setsignal
+
+ mov w0, #SIGUSR1
+ adr x1, irritator_handler
+ mov w2, #SA_SIGINFO
+ orr w2, w2, #SA_NODEFER
+ bl setsignal
+
+ mov x22, #0 // generation number, increments per iteration
+.Ltest_loop:
+ rdvl x0, #8
+ cmp x0, x19
+ b.ne vl_barf
+
+ mov x21, #0 // Set up Z-regs & shadow with test pattern
+0: mov x0, x20
+ mov x1, x21
+ and x2, x22, #0xf
+ bl setup_zreg
+ add x21, x21, #1
+ cmp x21, #NZR
+ b.lo 0b
+
+ mov x0, x20 // Set up FFR & shadow with test pattern
+ mov x1, #NZR + NPR
+ and x2, x22, #0xf
+ bl setup_ffr
+
+0: mov x0, x20 // Set up P-regs & shadow with test pattern
+ mov x1, x21
+ and x2, x22, #0xf
+ bl setup_preg
+ add x21, x21, #1
+ cmp x21, #NZR + NPR
+ b.lo 0b
+
+// Can't do this when SVE state is volatile across SVC:
+// mov x8, #__NR_sched_yield // Encourage preemption
+// svc #0
+
+ mov x21, #0
+0: mov x0, x21
+ bl check_zreg
+ add x21, x21, #1
+ cmp x21, #NZR
+ b.lo 0b
+
+0: mov x0, x21
+ bl check_preg
+ add x21, x21, #1
+ cmp x21, #NZR + NPR
+ b.lo 0b
+
+ bl check_ffr
+
+ add x22, x22, #1
+ b .Ltest_loop
+
+.Labort:
+ mov x0, #0
+ mov x1, #SIGABRT
+ mov x8, #__NR_kill
+ svc #0
+endfunction
+
+function barf
+// fpsimd.c acitivty log dump hack
+// ldr w0, =0xdeadc0de
+// mov w8, #__NR_exit
+// svc #0
+// end hack
+ mov x10, x0 // expected data
+ mov x11, x1 // actual data
+ mov x12, x2 // data size
+
+ puts "Mistatch: PID="
+ mov x0, x20
+ bl putdec
+ puts ", iteration="
+ mov x0, x22
+ bl putdec
+ puts ", reg="
+ mov x0, x21
+ bl putdecn
+ puts "\tExpected ["
+ mov x0, x10
+ mov x1, x12
+ bl dumphex
+ puts "]\n\tGot ["
+ mov x0, x11
+ mov x1, x12
+ bl dumphex
+ puts "]\n"
+
+ mov x8, #__NR_getpid
+ svc #0
+// fpsimd.c acitivty log dump hack
+// ldr w0, =0xdeadc0de
+// mov w8, #__NR_exit
+// svc #0
+// ^ end of hack
+ mov x1, #SIGABRT
+ mov x8, #__NR_kill
+ svc #0
+// mov x8, #__NR_exit
+// mov x1, #1
+// svc #0
+endfunction
+
+function vl_barf
+ mov x10, x0
+
+ puts "Bad active VL: "
+ mov x0, x10
+ bl putdecn
+
+ mov x8, #__NR_exit
+ mov x1, #1
+ svc #0
+endfunction
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2015-2019 ARM Limited.
+ * Original author: Dave Martin <Dave.Martin@arm.com>
+ */
+#define _GNU_SOURCE
+#include <assert.h>
+#include <errno.h>
+#include <limits.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <getopt.h>
+#include <unistd.h>
+#include <sys/auxv.h>
+#include <sys/prctl.h>
+#include <asm/hwcap.h>
+#include <asm/sigcontext.h>
+
+static int inherit = 0;
+static int no_inherit = 0;
+static int force = 0;
+static unsigned long vl;
+
+static const struct option options[] = {
+ { "force", no_argument, NULL, 'f' },
+ { "inherit", no_argument, NULL, 'i' },
+ { "max", no_argument, NULL, 'M' },
+ { "no-inherit", no_argument, &no_inherit, 1 },
+ { "help", no_argument, NULL, '?' },
+ {}
+};
+
+static char const *program_name;
+
+static int parse_options(int argc, char **argv)
+{
+ int c;
+ char *rest;
+
+ program_name = strrchr(argv[0], '/');
+ if (program_name)
+ ++program_name;
+ else
+ program_name = argv[0];
+
+ while ((c = getopt_long(argc, argv, "Mfhi", options, NULL)) != -1)
+ switch (c) {
+ case 'M': vl = SVE_VL_MAX; break;
+ case 'f': force = 1; break;
+ case 'i': inherit = 1; break;
+ case 0: break;
+ default: goto error;
+ }
+
+ if (inherit && no_inherit)
+ goto error;
+
+ if (!vl) {
+ /* vector length */
+ if (optind >= argc)
+ goto error;
+
+ errno = 0;
+ vl = strtoul(argv[optind], &rest, 0);
+ if (*rest) {
+ vl = ULONG_MAX;
+ errno = EINVAL;
+ }
+ if (vl == ULONG_MAX && errno) {
+ fprintf(stderr, "%s: %s: %s\n",
+ program_name, argv[optind], strerror(errno));
+ goto error;
+ }
+
+ ++optind;
+ }
+
+ /* command */
+ if (optind >= argc)
+ goto error;
+
+ return 0;
+
+error:
+ fprintf(stderr,
+ "Usage: %s [-f | --force] "
+ "[-i | --inherit | --no-inherit] "
+ "{-M | --max | <vector length>} "
+ "<command> [<arguments> ...]\n",
+ program_name);
+ return -1;
+}
+
+int main(int argc, char **argv)
+{
+ int ret = 126; /* same as sh(1) command-not-executable error */
+ long flags;
+ char *path;
+ int t, e;
+
+ if (parse_options(argc, argv))
+ return 2; /* same as sh(1) builtin incorrect-usage */
+
+ if (vl & ~(vl & PR_SVE_VL_LEN_MASK)) {
+ fprintf(stderr, "%s: Invalid vector length %lu\n",
+ program_name, vl);
+ return 2; /* same as sh(1) builtin incorrect-usage */
+ }
+
+ if (!(getauxval(AT_HWCAP) & HWCAP_SVE)) {
+ fprintf(stderr, "%s: Scalable Vector Extension not present\n",
+ program_name);
+
+ if (!force)
+ goto error;
+
+ fputs("Going ahead anyway (--force): "
+ "This is a debug option. Don't rely on it.\n",
+ stderr);
+ }
+
+ flags = PR_SVE_SET_VL_ONEXEC;
+ if (inherit)
+ flags |= PR_SVE_VL_INHERIT;
+
+ t = prctl(PR_SVE_SET_VL, vl | flags);
+ if (t < 0) {
+ fprintf(stderr, "%s: PR_SVE_SET_VL: %s\n",
+ program_name, strerror(errno));
+ goto error;
+ }
+
+ t = prctl(PR_SVE_GET_VL);
+ if (t == -1) {
+ fprintf(stderr, "%s: PR_SVE_GET_VL: %s\n",
+ program_name, strerror(errno));
+ goto error;
+ }
+ flags = PR_SVE_VL_LEN_MASK;
+ flags = t & ~flags;
+
+ assert(optind < argc);
+ path = argv[optind];
+
+ execvp(path, &argv[optind]);
+ e = errno;
+ if (errno == ENOENT)
+ ret = 127; /* same as sh(1) not-found error */
+ fprintf(stderr, "%s: %s: %s\n", program_name, path, strerror(e));
+
+error:
+ return ret; /* same as sh(1) not-executable error */
+}
--- /dev/null
+check_buffer_fill
+check_tags_inclusion
+check_child_memory
+check_mmap_options
+check_ksm_options
+check_user_mem
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2020 ARM Limited
+
+CFLAGS += -std=gnu99 -I.
+SRCS := $(filter-out mte_common_util.c,$(wildcard *.c))
+PROGS := $(patsubst %.c,%,$(SRCS))
+
+#Add mte compiler option
+ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep gcc),)
+CFLAGS += -march=armv8.5-a+memtag
+endif
+
+#check if the compiler works well
+mte_cc_support := $(shell if ($(CC) $(CFLAGS) -E -x c /dev/null -o /dev/null 2>&1) then echo "1"; fi)
+
+ifeq ($(mte_cc_support),1)
+# Generated binaries to be installed by top KSFT script
+TEST_GEN_PROGS := $(PROGS)
+
+# Get Kernel headers installed and use them.
+KSFT_KHDR_INSTALL := 1
+endif
+
+# Include KSFT lib.mk.
+include ../../lib.mk
+
+ifeq ($(mte_cc_support),1)
+$(TEST_GEN_PROGS): mte_common_util.c mte_common_util.h mte_helper.S
+endif
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 ARM Limited
+
+#define _GNU_SOURCE
+
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "kselftest.h"
+#include "mte_common_util.h"
+#include "mte_def.h"
+
+#define OVERFLOW_RANGE MT_GRANULE_SIZE
+
+static int sizes[] = {
+ 1, 555, 1033, MT_GRANULE_SIZE - 1, MT_GRANULE_SIZE,
+ /* page size - 1*/ 0, /* page_size */ 0, /* page size + 1 */ 0
+};
+
+enum mte_block_test_alloc {
+ UNTAGGED_TAGGED,
+ TAGGED_UNTAGGED,
+ TAGGED_TAGGED,
+ BLOCK_ALLOC_MAX,
+};
+
+static int check_buffer_by_byte(int mem_type, int mode)
+{
+ char *ptr;
+ int i, j, item;
+ bool err;
+
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ item = sizeof(sizes)/sizeof(int);
+
+ for (i = 0; i < item; i++) {
+ ptr = (char *)mte_allocate_memory(sizes[i], mem_type, 0, true);
+ if (check_allocated_memory(ptr, sizes[i], mem_type, true) != KSFT_PASS)
+ return KSFT_FAIL;
+ mte_initialize_current_context(mode, (uintptr_t)ptr, sizes[i]);
+ /* Set some value in tagged memory */
+ for (j = 0; j < sizes[i]; j++)
+ ptr[j] = '1';
+ mte_wait_after_trig();
+ err = cur_mte_cxt.fault_valid;
+ /* Check the buffer whether it is filled. */
+ for (j = 0; j < sizes[i] && !err; j++) {
+ if (ptr[j] != '1')
+ err = true;
+ }
+ mte_free_memory((void *)ptr, sizes[i], mem_type, true);
+
+ if (err)
+ break;
+ }
+ if (!err)
+ return KSFT_PASS;
+ else
+ return KSFT_FAIL;
+}
+
+static int check_buffer_underflow_by_byte(int mem_type, int mode,
+ int underflow_range)
+{
+ char *ptr;
+ int i, j, item, last_index;
+ bool err;
+ char *und_ptr = NULL;
+
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ item = sizeof(sizes)/sizeof(int);
+ for (i = 0; i < item; i++) {
+ ptr = (char *)mte_allocate_memory_tag_range(sizes[i], mem_type, 0,
+ underflow_range, 0);
+ if (check_allocated_memory_range(ptr, sizes[i], mem_type,
+ underflow_range, 0) != KSFT_PASS)
+ return KSFT_FAIL;
+
+ mte_initialize_current_context(mode, (uintptr_t)ptr, -underflow_range);
+ last_index = 0;
+ /* Set some value in tagged memory and make the buffer underflow */
+ for (j = sizes[i] - 1; (j >= -underflow_range) &&
+ (cur_mte_cxt.fault_valid == false); j--) {
+ ptr[j] = '1';
+ last_index = j;
+ }
+ mte_wait_after_trig();
+ err = false;
+ /* Check whether the buffer is filled */
+ for (j = 0; j < sizes[i]; j++) {
+ if (ptr[j] != '1') {
+ err = true;
+ ksft_print_msg("Buffer is not filled at index:%d of ptr:0x%lx\n",
+ j, ptr);
+ break;
+ }
+ }
+ if (err)
+ goto check_buffer_underflow_by_byte_err;
+
+ switch (mode) {
+ case MTE_NONE_ERR:
+ if (cur_mte_cxt.fault_valid == true || last_index != -underflow_range) {
+ err = true;
+ break;
+ }
+ /* There were no fault so the underflow area should be filled */
+ und_ptr = (char *) MT_CLEAR_TAG((size_t) ptr - underflow_range);
+ for (j = 0 ; j < underflow_range; j++) {
+ if (und_ptr[j] != '1') {
+ err = true;
+ break;
+ }
+ }
+ break;
+ case MTE_ASYNC_ERR:
+ /* Imprecise fault should occur otherwise return error */
+ if (cur_mte_cxt.fault_valid == false) {
+ err = true;
+ break;
+ }
+ /*
+ * The imprecise fault is checked after the write to the buffer,
+ * so the underflow area before the fault should be filled.
+ */
+ und_ptr = (char *) MT_CLEAR_TAG((size_t) ptr);
+ for (j = last_index ; j < 0 ; j++) {
+ if (und_ptr[j] != '1') {
+ err = true;
+ break;
+ }
+ }
+ break;
+ case MTE_SYNC_ERR:
+ /* Precise fault should occur otherwise return error */
+ if (!cur_mte_cxt.fault_valid || (last_index != (-1))) {
+ err = true;
+ break;
+ }
+ /* Underflow area should not be filled */
+ und_ptr = (char *) MT_CLEAR_TAG((size_t) ptr);
+ if (und_ptr[-1] == '1')
+ err = true;
+ break;
+ default:
+ err = true;
+ break;
+ }
+check_buffer_underflow_by_byte_err:
+ mte_free_memory_tag_range((void *)ptr, sizes[i], mem_type, underflow_range, 0);
+ if (err)
+ break;
+ }
+ return (err ? KSFT_FAIL : KSFT_PASS);
+}
+
+static int check_buffer_overflow_by_byte(int mem_type, int mode,
+ int overflow_range)
+{
+ char *ptr;
+ int i, j, item, last_index;
+ bool err;
+ size_t tagged_size, overflow_size;
+ char *over_ptr = NULL;
+
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ item = sizeof(sizes)/sizeof(int);
+ for (i = 0; i < item; i++) {
+ ptr = (char *)mte_allocate_memory_tag_range(sizes[i], mem_type, 0,
+ 0, overflow_range);
+ if (check_allocated_memory_range(ptr, sizes[i], mem_type,
+ 0, overflow_range) != KSFT_PASS)
+ return KSFT_FAIL;
+
+ tagged_size = MT_ALIGN_UP(sizes[i]);
+
+ mte_initialize_current_context(mode, (uintptr_t)ptr, sizes[i] + overflow_range);
+
+ /* Set some value in tagged memory and make the buffer underflow */
+ for (j = 0, last_index = 0 ; (j < (sizes[i] + overflow_range)) &&
+ (cur_mte_cxt.fault_valid == false); j++) {
+ ptr[j] = '1';
+ last_index = j;
+ }
+ mte_wait_after_trig();
+ err = false;
+ /* Check whether the buffer is filled */
+ for (j = 0; j < sizes[i]; j++) {
+ if (ptr[j] != '1') {
+ err = true;
+ ksft_print_msg("Buffer is not filled at index:%d of ptr:0x%lx\n",
+ j, ptr);
+ break;
+ }
+ }
+ if (err)
+ goto check_buffer_overflow_by_byte_err;
+
+ overflow_size = overflow_range - (tagged_size - sizes[i]);
+
+ switch (mode) {
+ case MTE_NONE_ERR:
+ if ((cur_mte_cxt.fault_valid == true) ||
+ (last_index != (sizes[i] + overflow_range - 1))) {
+ err = true;
+ break;
+ }
+ /* There were no fault so the overflow area should be filled */
+ over_ptr = (char *) MT_CLEAR_TAG((size_t) ptr + tagged_size);
+ for (j = 0 ; j < overflow_size; j++) {
+ if (over_ptr[j] != '1') {
+ err = true;
+ break;
+ }
+ }
+ break;
+ case MTE_ASYNC_ERR:
+ /* Imprecise fault should occur otherwise return error */
+ if (cur_mte_cxt.fault_valid == false) {
+ err = true;
+ break;
+ }
+ /*
+ * The imprecise fault is checked after the write to the buffer,
+ * so the overflow area should be filled before the fault.
+ */
+ over_ptr = (char *) MT_CLEAR_TAG((size_t) ptr);
+ for (j = tagged_size ; j < last_index; j++) {
+ if (over_ptr[j] != '1') {
+ err = true;
+ break;
+ }
+ }
+ break;
+ case MTE_SYNC_ERR:
+ /* Precise fault should occur otherwise return error */
+ if (!cur_mte_cxt.fault_valid || (last_index != tagged_size)) {
+ err = true;
+ break;
+ }
+ /* Underflow area should not be filled */
+ over_ptr = (char *) MT_CLEAR_TAG((size_t) ptr + tagged_size);
+ for (j = 0 ; j < overflow_size; j++) {
+ if (over_ptr[j] == '1')
+ err = true;
+ }
+ break;
+ default:
+ err = true;
+ break;
+ }
+check_buffer_overflow_by_byte_err:
+ mte_free_memory_tag_range((void *)ptr, sizes[i], mem_type, 0, overflow_range);
+ if (err)
+ break;
+ }
+ return (err ? KSFT_FAIL : KSFT_PASS);
+}
+
+static int check_buffer_by_block_iterate(int mem_type, int mode, size_t size)
+{
+ char *src, *dst;
+ int j, result = KSFT_PASS;
+ enum mte_block_test_alloc alloc_type = UNTAGGED_TAGGED;
+
+ for (alloc_type = UNTAGGED_TAGGED; alloc_type < (int) BLOCK_ALLOC_MAX; alloc_type++) {
+ switch (alloc_type) {
+ case UNTAGGED_TAGGED:
+ src = (char *)mte_allocate_memory(size, mem_type, 0, false);
+ if (check_allocated_memory(src, size, mem_type, false) != KSFT_PASS)
+ return KSFT_FAIL;
+
+ dst = (char *)mte_allocate_memory(size, mem_type, 0, true);
+ if (check_allocated_memory(dst, size, mem_type, true) != KSFT_PASS) {
+ mte_free_memory((void *)src, size, mem_type, false);
+ return KSFT_FAIL;
+ }
+
+ break;
+ case TAGGED_UNTAGGED:
+ dst = (char *)mte_allocate_memory(size, mem_type, 0, false);
+ if (check_allocated_memory(dst, size, mem_type, false) != KSFT_PASS)
+ return KSFT_FAIL;
+
+ src = (char *)mte_allocate_memory(size, mem_type, 0, true);
+ if (check_allocated_memory(src, size, mem_type, true) != KSFT_PASS) {
+ mte_free_memory((void *)dst, size, mem_type, false);
+ return KSFT_FAIL;
+ }
+ break;
+ case TAGGED_TAGGED:
+ src = (char *)mte_allocate_memory(size, mem_type, 0, true);
+ if (check_allocated_memory(src, size, mem_type, true) != KSFT_PASS)
+ return KSFT_FAIL;
+
+ dst = (char *)mte_allocate_memory(size, mem_type, 0, true);
+ if (check_allocated_memory(dst, size, mem_type, true) != KSFT_PASS) {
+ mte_free_memory((void *)src, size, mem_type, true);
+ return KSFT_FAIL;
+ }
+ break;
+ default:
+ return KSFT_FAIL;
+ }
+
+ cur_mte_cxt.fault_valid = false;
+ result = KSFT_PASS;
+ mte_initialize_current_context(mode, (uintptr_t)dst, size);
+ /* Set some value in memory and copy*/
+ memset((void *)src, (int)'1', size);
+ memcpy((void *)dst, (void *)src, size);
+ mte_wait_after_trig();
+ if (cur_mte_cxt.fault_valid) {
+ result = KSFT_FAIL;
+ goto check_buffer_by_block_err;
+ }
+ /* Check the buffer whether it is filled. */
+ for (j = 0; j < size; j++) {
+ if (src[j] != dst[j] || src[j] != '1') {
+ result = KSFT_FAIL;
+ break;
+ }
+ }
+check_buffer_by_block_err:
+ mte_free_memory((void *)src, size, mem_type,
+ MT_FETCH_TAG((uintptr_t)src) ? true : false);
+ mte_free_memory((void *)dst, size, mem_type,
+ MT_FETCH_TAG((uintptr_t)dst) ? true : false);
+ if (result != KSFT_PASS)
+ return result;
+ }
+ return result;
+}
+
+static int check_buffer_by_block(int mem_type, int mode)
+{
+ int i, item, result = KSFT_PASS;
+
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ item = sizeof(sizes)/sizeof(int);
+ cur_mte_cxt.fault_valid = false;
+ for (i = 0; i < item; i++) {
+ result = check_buffer_by_block_iterate(mem_type, mode, sizes[i]);
+ if (result != KSFT_PASS)
+ break;
+ }
+ return result;
+}
+
+static int compare_memory_tags(char *ptr, size_t size, int tag)
+{
+ int i, new_tag;
+
+ for (i = 0 ; i < size ; i += MT_GRANULE_SIZE) {
+ new_tag = MT_FETCH_TAG((uintptr_t)(mte_get_tag_address(ptr + i)));
+ if (tag != new_tag) {
+ ksft_print_msg("FAIL: child mte tag mismatch\n");
+ return KSFT_FAIL;
+ }
+ }
+ return KSFT_PASS;
+}
+
+static int check_memory_initial_tags(int mem_type, int mode, int mapping)
+{
+ char *ptr;
+ int run, fd;
+ int total = sizeof(sizes)/sizeof(int);
+
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ for (run = 0; run < total; run++) {
+ /* check initial tags for anonymous mmap */
+ ptr = (char *)mte_allocate_memory(sizes[run], mem_type, mapping, false);
+ if (check_allocated_memory(ptr, sizes[run], mem_type, false) != KSFT_PASS)
+ return KSFT_FAIL;
+ if (compare_memory_tags(ptr, sizes[run], 0) != KSFT_PASS) {
+ mte_free_memory((void *)ptr, sizes[run], mem_type, false);
+ return KSFT_FAIL;
+ }
+ mte_free_memory((void *)ptr, sizes[run], mem_type, false);
+
+ /* check initial tags for file mmap */
+ fd = create_temp_file();
+ if (fd == -1)
+ return KSFT_FAIL;
+ ptr = (char *)mte_allocate_file_memory(sizes[run], mem_type, mapping, false, fd);
+ if (check_allocated_memory(ptr, sizes[run], mem_type, false) != KSFT_PASS) {
+ close(fd);
+ return KSFT_FAIL;
+ }
+ if (compare_memory_tags(ptr, sizes[run], 0) != KSFT_PASS) {
+ mte_free_memory((void *)ptr, sizes[run], mem_type, false);
+ close(fd);
+ return KSFT_FAIL;
+ }
+ mte_free_memory((void *)ptr, sizes[run], mem_type, false);
+ close(fd);
+ }
+ return KSFT_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+ int err;
+ size_t page_size = getpagesize();
+ int item = sizeof(sizes)/sizeof(int);
+
+ sizes[item - 3] = page_size - 1;
+ sizes[item - 2] = page_size;
+ sizes[item - 1] = page_size + 1;
+
+ err = mte_default_setup();
+ if (err)
+ return err;
+
+ /* Register SIGSEGV handler */
+ mte_register_signal(SIGSEGV, mte_default_handler);
+
+ /* Buffer by byte tests */
+ evaluate_test(check_buffer_by_byte(USE_MMAP, MTE_SYNC_ERR),
+ "Check buffer correctness by byte with sync err mode and mmap memory\n");
+ evaluate_test(check_buffer_by_byte(USE_MMAP, MTE_ASYNC_ERR),
+ "Check buffer correctness by byte with async err mode and mmap memory\n");
+ evaluate_test(check_buffer_by_byte(USE_MPROTECT, MTE_SYNC_ERR),
+ "Check buffer correctness by byte with sync err mode and mmap/mprotect memory\n");
+ evaluate_test(check_buffer_by_byte(USE_MPROTECT, MTE_ASYNC_ERR),
+ "Check buffer correctness by byte with async err mode and mmap/mprotect memory\n");
+
+ /* Check buffer underflow with underflow size as 16 */
+ evaluate_test(check_buffer_underflow_by_byte(USE_MMAP, MTE_SYNC_ERR, MT_GRANULE_SIZE),
+ "Check buffer write underflow by byte with sync mode and mmap memory\n");
+ evaluate_test(check_buffer_underflow_by_byte(USE_MMAP, MTE_ASYNC_ERR, MT_GRANULE_SIZE),
+ "Check buffer write underflow by byte with async mode and mmap memory\n");
+ evaluate_test(check_buffer_underflow_by_byte(USE_MMAP, MTE_NONE_ERR, MT_GRANULE_SIZE),
+ "Check buffer write underflow by byte with tag check fault ignore and mmap memory\n");
+
+ /* Check buffer underflow with underflow size as page size */
+ evaluate_test(check_buffer_underflow_by_byte(USE_MMAP, MTE_SYNC_ERR, page_size),
+ "Check buffer write underflow by byte with sync mode and mmap memory\n");
+ evaluate_test(check_buffer_underflow_by_byte(USE_MMAP, MTE_ASYNC_ERR, page_size),
+ "Check buffer write underflow by byte with async mode and mmap memory\n");
+ evaluate_test(check_buffer_underflow_by_byte(USE_MMAP, MTE_NONE_ERR, page_size),
+ "Check buffer write underflow by byte with tag check fault ignore and mmap memory\n");
+
+ /* Check buffer overflow with overflow size as 16 */
+ evaluate_test(check_buffer_overflow_by_byte(USE_MMAP, MTE_SYNC_ERR, MT_GRANULE_SIZE),
+ "Check buffer write overflow by byte with sync mode and mmap memory\n");
+ evaluate_test(check_buffer_overflow_by_byte(USE_MMAP, MTE_ASYNC_ERR, MT_GRANULE_SIZE),
+ "Check buffer write overflow by byte with async mode and mmap memory\n");
+ evaluate_test(check_buffer_overflow_by_byte(USE_MMAP, MTE_NONE_ERR, MT_GRANULE_SIZE),
+ "Check buffer write overflow by byte with tag fault ignore mode and mmap memory\n");
+
+ /* Buffer by block tests */
+ evaluate_test(check_buffer_by_block(USE_MMAP, MTE_SYNC_ERR),
+ "Check buffer write correctness by block with sync mode and mmap memory\n");
+ evaluate_test(check_buffer_by_block(USE_MMAP, MTE_ASYNC_ERR),
+ "Check buffer write correctness by block with async mode and mmap memory\n");
+ evaluate_test(check_buffer_by_block(USE_MMAP, MTE_NONE_ERR),
+ "Check buffer write correctness by block with tag fault ignore and mmap memory\n");
+
+ /* Initial tags are supposed to be 0 */
+ evaluate_test(check_memory_initial_tags(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE),
+ "Check initial tags with private mapping, sync error mode and mmap memory\n");
+ evaluate_test(check_memory_initial_tags(USE_MPROTECT, MTE_SYNC_ERR, MAP_PRIVATE),
+ "Check initial tags with private mapping, sync error mode and mmap/mprotect memory\n");
+ evaluate_test(check_memory_initial_tags(USE_MMAP, MTE_SYNC_ERR, MAP_SHARED),
+ "Check initial tags with shared mapping, sync error mode and mmap memory\n");
+ evaluate_test(check_memory_initial_tags(USE_MPROTECT, MTE_SYNC_ERR, MAP_SHARED),
+ "Check initial tags with shared mapping, sync error mode and mmap/mprotect memory\n");
+
+ mte_restore_setup();
+ ksft_print_cnts();
+ return ksft_get_fail_cnt() == 0 ? KSFT_PASS : KSFT_FAIL;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 ARM Limited
+
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ucontext.h>
+#include <sys/wait.h>
+
+#include "kselftest.h"
+#include "mte_common_util.h"
+#include "mte_def.h"
+
+#define BUFFER_SIZE (5 * MT_GRANULE_SIZE)
+#define RUNS (MT_TAG_COUNT)
+#define UNDERFLOW MT_GRANULE_SIZE
+#define OVERFLOW MT_GRANULE_SIZE
+
+static size_t page_size;
+static int sizes[] = {
+ 1, 537, 989, 1269, MT_GRANULE_SIZE - 1, MT_GRANULE_SIZE,
+ /* page size - 1*/ 0, /* page_size */ 0, /* page size + 1 */ 0
+};
+
+static int check_child_tag_inheritance(char *ptr, int size, int mode)
+{
+ int i, parent_tag, child_tag, fault, child_status;
+ pid_t child;
+
+ parent_tag = MT_FETCH_TAG((uintptr_t)ptr);
+ fault = 0;
+
+ child = fork();
+ if (child == -1) {
+ ksft_print_msg("FAIL: child process creation\n");
+ return KSFT_FAIL;
+ } else if (child == 0) {
+ mte_initialize_current_context(mode, (uintptr_t)ptr, size);
+ /* Do copy on write */
+ memset(ptr, '1', size);
+ mte_wait_after_trig();
+ if (cur_mte_cxt.fault_valid == true) {
+ fault = 1;
+ goto check_child_tag_inheritance_err;
+ }
+ for (i = 0 ; i < size ; i += MT_GRANULE_SIZE) {
+ child_tag = MT_FETCH_TAG((uintptr_t)(mte_get_tag_address(ptr + i)));
+ if (parent_tag != child_tag) {
+ ksft_print_msg("FAIL: child mte tag mismatch\n");
+ fault = 1;
+ goto check_child_tag_inheritance_err;
+ }
+ }
+ mte_initialize_current_context(mode, (uintptr_t)ptr, -UNDERFLOW);
+ memset(ptr - UNDERFLOW, '2', UNDERFLOW);
+ mte_wait_after_trig();
+ if (cur_mte_cxt.fault_valid == false) {
+ fault = 1;
+ goto check_child_tag_inheritance_err;
+ }
+ mte_initialize_current_context(mode, (uintptr_t)ptr, size + OVERFLOW);
+ memset(ptr + size, '3', OVERFLOW);
+ mte_wait_after_trig();
+ if (cur_mte_cxt.fault_valid == false) {
+ fault = 1;
+ goto check_child_tag_inheritance_err;
+ }
+check_child_tag_inheritance_err:
+ _exit(fault);
+ }
+ /* Wait for child process to terminate */
+ wait(&child_status);
+ if (WIFEXITED(child_status))
+ fault = WEXITSTATUS(child_status);
+ else
+ fault = 1;
+ return (fault) ? KSFT_FAIL : KSFT_PASS;
+}
+
+static int check_child_memory_mapping(int mem_type, int mode, int mapping)
+{
+ char *ptr;
+ int run, result;
+ int item = sizeof(sizes)/sizeof(int);
+
+ item = sizeof(sizes)/sizeof(int);
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ for (run = 0; run < item; run++) {
+ ptr = (char *)mte_allocate_memory_tag_range(sizes[run], mem_type, mapping,
+ UNDERFLOW, OVERFLOW);
+ if (check_allocated_memory_range(ptr, sizes[run], mem_type,
+ UNDERFLOW, OVERFLOW) != KSFT_PASS)
+ return KSFT_FAIL;
+ result = check_child_tag_inheritance(ptr, sizes[run], mode);
+ mte_free_memory_tag_range((void *)ptr, sizes[run], mem_type, UNDERFLOW, OVERFLOW);
+ if (result == KSFT_FAIL)
+ return result;
+ }
+ return KSFT_PASS;
+}
+
+static int check_child_file_mapping(int mem_type, int mode, int mapping)
+{
+ char *ptr, *map_ptr;
+ int run, fd, map_size, result = KSFT_PASS;
+ int total = sizeof(sizes)/sizeof(int);
+
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ for (run = 0; run < total; run++) {
+ fd = create_temp_file();
+ if (fd == -1)
+ return KSFT_FAIL;
+
+ map_size = sizes[run] + OVERFLOW + UNDERFLOW;
+ map_ptr = (char *)mte_allocate_file_memory(map_size, mem_type, mapping, false, fd);
+ if (check_allocated_memory(map_ptr, map_size, mem_type, false) != KSFT_PASS) {
+ close(fd);
+ return KSFT_FAIL;
+ }
+ ptr = map_ptr + UNDERFLOW;
+ mte_initialize_current_context(mode, (uintptr_t)ptr, sizes[run]);
+ /* Only mte enabled memory will allow tag insertion */
+ ptr = mte_insert_tags((void *)ptr, sizes[run]);
+ if (!ptr || cur_mte_cxt.fault_valid == true) {
+ ksft_print_msg("FAIL: Insert tags on file based memory\n");
+ munmap((void *)map_ptr, map_size);
+ close(fd);
+ return KSFT_FAIL;
+ }
+ result = check_child_tag_inheritance(ptr, sizes[run], mode);
+ mte_clear_tags((void *)ptr, sizes[run]);
+ munmap((void *)map_ptr, map_size);
+ close(fd);
+ if (result != KSFT_PASS)
+ return KSFT_FAIL;
+ }
+ return KSFT_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+ int err;
+ int item = sizeof(sizes)/sizeof(int);
+
+ page_size = getpagesize();
+ if (!page_size) {
+ ksft_print_msg("ERR: Unable to get page size\n");
+ return KSFT_FAIL;
+ }
+ sizes[item - 3] = page_size - 1;
+ sizes[item - 2] = page_size;
+ sizes[item - 1] = page_size + 1;
+
+ err = mte_default_setup();
+ if (err)
+ return err;
+
+ /* Register SIGSEGV handler */
+ mte_register_signal(SIGSEGV, mte_default_handler);
+ mte_register_signal(SIGBUS, mte_default_handler);
+
+ evaluate_test(check_child_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE),
+ "Check child anonymous memory with private mapping, precise mode and mmap memory\n");
+ evaluate_test(check_child_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_SHARED),
+ "Check child anonymous memory with shared mapping, precise mode and mmap memory\n");
+ evaluate_test(check_child_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_PRIVATE),
+ "Check child anonymous memory with private mapping, imprecise mode and mmap memory\n");
+ evaluate_test(check_child_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_SHARED),
+ "Check child anonymous memory with shared mapping, imprecise mode and mmap memory\n");
+ evaluate_test(check_child_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_PRIVATE),
+ "Check child anonymous memory with private mapping, precise mode and mmap/mprotect memory\n");
+ evaluate_test(check_child_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_SHARED),
+ "Check child anonymous memory with shared mapping, precise mode and mmap/mprotect memory\n");
+
+ evaluate_test(check_child_file_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE),
+ "Check child file memory with private mapping, precise mode and mmap memory\n");
+ evaluate_test(check_child_file_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_SHARED),
+ "Check child file memory with shared mapping, precise mode and mmap memory\n");
+ evaluate_test(check_child_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_PRIVATE),
+ "Check child file memory with private mapping, imprecise mode and mmap memory\n");
+ evaluate_test(check_child_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_SHARED),
+ "Check child file memory with shared mapping, imprecise mode and mmap memory\n");
+ evaluate_test(check_child_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_PRIVATE),
+ "Check child file memory with private mapping, precise mode and mmap/mprotect memory\n");
+ evaluate_test(check_child_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_SHARED),
+ "Check child file memory with shared mapping, precise mode and mmap/mprotect memory\n");
+
+ mte_restore_setup();
+ ksft_print_cnts();
+ return ksft_get_fail_cnt() == 0 ? KSFT_PASS : KSFT_FAIL;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 ARM Limited
+
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ucontext.h>
+#include <sys/mman.h>
+
+#include "kselftest.h"
+#include "mte_common_util.h"
+#include "mte_def.h"
+
+#define TEST_UNIT 10
+#define PATH_KSM "/sys/kernel/mm/ksm/"
+#define MAX_LOOP 4
+
+static size_t page_sz;
+static unsigned long ksm_sysfs[5];
+
+static unsigned long read_sysfs(char *str)
+{
+ FILE *f;
+ unsigned long val = 0;
+
+ f = fopen(str, "r");
+ if (!f) {
+ ksft_print_msg("ERR: missing %s\n", str);
+ return 0;
+ }
+ fscanf(f, "%lu", &val);
+ fclose(f);
+ return val;
+}
+
+static void write_sysfs(char *str, unsigned long val)
+{
+ FILE *f;
+
+ f = fopen(str, "w");
+ if (!f) {
+ ksft_print_msg("ERR: missing %s\n", str);
+ return;
+ }
+ fprintf(f, "%lu", val);
+ fclose(f);
+}
+
+static void mte_ksm_setup(void)
+{
+ ksm_sysfs[0] = read_sysfs(PATH_KSM "merge_across_nodes");
+ write_sysfs(PATH_KSM "merge_across_nodes", 1);
+ ksm_sysfs[1] = read_sysfs(PATH_KSM "sleep_millisecs");
+ write_sysfs(PATH_KSM "sleep_millisecs", 0);
+ ksm_sysfs[2] = read_sysfs(PATH_KSM "run");
+ write_sysfs(PATH_KSM "run", 1);
+ ksm_sysfs[3] = read_sysfs(PATH_KSM "max_page_sharing");
+ write_sysfs(PATH_KSM "max_page_sharing", ksm_sysfs[3] + TEST_UNIT);
+ ksm_sysfs[4] = read_sysfs(PATH_KSM "pages_to_scan");
+ write_sysfs(PATH_KSM "pages_to_scan", ksm_sysfs[4] + TEST_UNIT);
+}
+
+static void mte_ksm_restore(void)
+{
+ write_sysfs(PATH_KSM "merge_across_nodes", ksm_sysfs[0]);
+ write_sysfs(PATH_KSM "sleep_millisecs", ksm_sysfs[1]);
+ write_sysfs(PATH_KSM "run", ksm_sysfs[2]);
+ write_sysfs(PATH_KSM "max_page_sharing", ksm_sysfs[3]);
+ write_sysfs(PATH_KSM "pages_to_scan", ksm_sysfs[4]);
+}
+
+static void mte_ksm_scan(void)
+{
+ int cur_count = read_sysfs(PATH_KSM "full_scans");
+ int scan_count = cur_count + 1;
+ int max_loop_count = MAX_LOOP;
+
+ while ((cur_count < scan_count) && max_loop_count) {
+ sleep(1);
+ cur_count = read_sysfs(PATH_KSM "full_scans");
+ max_loop_count--;
+ }
+#ifdef DEBUG
+ ksft_print_msg("INFO: pages_shared=%lu pages_sharing=%lu\n",
+ read_sysfs(PATH_KSM "pages_shared"),
+ read_sysfs(PATH_KSM "pages_sharing"));
+#endif
+}
+
+static int check_madvise_options(int mem_type, int mode, int mapping)
+{
+ char *ptr;
+ int err, ret;
+
+ err = KSFT_FAIL;
+ if (access(PATH_KSM, F_OK) == -1) {
+ ksft_print_msg("ERR: Kernel KSM config not enabled\n");
+ return err;
+ }
+
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ ptr = mte_allocate_memory(TEST_UNIT * page_sz, mem_type, mapping, true);
+ if (check_allocated_memory(ptr, TEST_UNIT * page_sz, mem_type, false) != KSFT_PASS)
+ return KSFT_FAIL;
+
+ /* Insert same data in all the pages */
+ memset(ptr, 'A', TEST_UNIT * page_sz);
+ ret = madvise(ptr, TEST_UNIT * page_sz, MADV_MERGEABLE);
+ if (ret) {
+ ksft_print_msg("ERR: madvise failed to set MADV_UNMERGEABLE\n");
+ goto madvise_err;
+ }
+ mte_ksm_scan();
+ /* Tagged pages should not merge */
+ if ((read_sysfs(PATH_KSM "pages_shared") < 1) ||
+ (read_sysfs(PATH_KSM "pages_sharing") < (TEST_UNIT - 1)))
+ err = KSFT_PASS;
+madvise_err:
+ mte_free_memory(ptr, TEST_UNIT * page_sz, mem_type, true);
+ return err;
+}
+
+int main(int argc, char *argv[])
+{
+ int err;
+
+ err = mte_default_setup();
+ if (err)
+ return err;
+ page_sz = getpagesize();
+ if (!page_sz) {
+ ksft_print_msg("ERR: Unable to get page size\n");
+ return KSFT_FAIL;
+ }
+ /* Register signal handlers */
+ mte_register_signal(SIGBUS, mte_default_handler);
+ mte_register_signal(SIGSEGV, mte_default_handler);
+ /* Enable KSM */
+ mte_ksm_setup();
+
+ evaluate_test(check_madvise_options(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE),
+ "Check KSM mte page merge for private mapping, sync mode and mmap memory\n");
+ evaluate_test(check_madvise_options(USE_MMAP, MTE_ASYNC_ERR, MAP_PRIVATE),
+ "Check KSM mte page merge for private mapping, async mode and mmap memory\n");
+ evaluate_test(check_madvise_options(USE_MMAP, MTE_SYNC_ERR, MAP_SHARED),
+ "Check KSM mte page merge for shared mapping, sync mode and mmap memory\n");
+ evaluate_test(check_madvise_options(USE_MMAP, MTE_ASYNC_ERR, MAP_SHARED),
+ "Check KSM mte page merge for shared mapping, async mode and mmap memory\n");
+
+ mte_ksm_restore();
+ mte_restore_setup();
+ ksft_print_cnts();
+ return ksft_get_fail_cnt() == 0 ? KSFT_PASS : KSFT_FAIL;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 ARM Limited
+
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ucontext.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "kselftest.h"
+#include "mte_common_util.h"
+#include "mte_def.h"
+
+#define RUNS (MT_TAG_COUNT)
+#define UNDERFLOW MT_GRANULE_SIZE
+#define OVERFLOW MT_GRANULE_SIZE
+#define TAG_CHECK_ON 0
+#define TAG_CHECK_OFF 1
+
+static size_t page_size;
+static int sizes[] = {
+ 1, 537, 989, 1269, MT_GRANULE_SIZE - 1, MT_GRANULE_SIZE,
+ /* page size - 1*/ 0, /* page_size */ 0, /* page size + 1 */ 0
+};
+
+static int check_mte_memory(char *ptr, int size, int mode, int tag_check)
+{
+ mte_initialize_current_context(mode, (uintptr_t)ptr, size);
+ memset(ptr, '1', size);
+ mte_wait_after_trig();
+ if (cur_mte_cxt.fault_valid == true)
+ return KSFT_FAIL;
+
+ mte_initialize_current_context(mode, (uintptr_t)ptr, -UNDERFLOW);
+ memset(ptr - UNDERFLOW, '2', UNDERFLOW);
+ mte_wait_after_trig();
+ if (cur_mte_cxt.fault_valid == false && tag_check == TAG_CHECK_ON)
+ return KSFT_FAIL;
+ if (cur_mte_cxt.fault_valid == true && tag_check == TAG_CHECK_OFF)
+ return KSFT_FAIL;
+
+ mte_initialize_current_context(mode, (uintptr_t)ptr, size + OVERFLOW);
+ memset(ptr + size, '3', OVERFLOW);
+ mte_wait_after_trig();
+ if (cur_mte_cxt.fault_valid == false && tag_check == TAG_CHECK_ON)
+ return KSFT_FAIL;
+ if (cur_mte_cxt.fault_valid == true && tag_check == TAG_CHECK_OFF)
+ return KSFT_FAIL;
+
+ return KSFT_PASS;
+}
+
+static int check_anonymous_memory_mapping(int mem_type, int mode, int mapping, int tag_check)
+{
+ char *ptr, *map_ptr;
+ int run, result, map_size;
+ int item = sizeof(sizes)/sizeof(int);
+
+ item = sizeof(sizes)/sizeof(int);
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ for (run = 0; run < item; run++) {
+ map_size = sizes[run] + OVERFLOW + UNDERFLOW;
+ map_ptr = (char *)mte_allocate_memory(map_size, mem_type, mapping, false);
+ if (check_allocated_memory(map_ptr, map_size, mem_type, false) != KSFT_PASS)
+ return KSFT_FAIL;
+
+ ptr = map_ptr + UNDERFLOW;
+ mte_initialize_current_context(mode, (uintptr_t)ptr, sizes[run]);
+ /* Only mte enabled memory will allow tag insertion */
+ ptr = mte_insert_tags((void *)ptr, sizes[run]);
+ if (!ptr || cur_mte_cxt.fault_valid == true) {
+ ksft_print_msg("FAIL: Insert tags on anonymous mmap memory\n");
+ munmap((void *)map_ptr, map_size);
+ return KSFT_FAIL;
+ }
+ result = check_mte_memory(ptr, sizes[run], mode, tag_check);
+ mte_clear_tags((void *)ptr, sizes[run]);
+ mte_free_memory((void *)map_ptr, map_size, mem_type, false);
+ if (result == KSFT_FAIL)
+ return KSFT_FAIL;
+ }
+ return KSFT_PASS;
+}
+
+static int check_file_memory_mapping(int mem_type, int mode, int mapping, int tag_check)
+{
+ char *ptr, *map_ptr;
+ int run, fd, map_size;
+ int total = sizeof(sizes)/sizeof(int);
+ int result = KSFT_PASS;
+
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ for (run = 0; run < total; run++) {
+ fd = create_temp_file();
+ if (fd == -1)
+ return KSFT_FAIL;
+
+ map_size = sizes[run] + UNDERFLOW + OVERFLOW;
+ map_ptr = (char *)mte_allocate_file_memory(map_size, mem_type, mapping, false, fd);
+ if (check_allocated_memory(map_ptr, map_size, mem_type, false) != KSFT_PASS) {
+ close(fd);
+ return KSFT_FAIL;
+ }
+ ptr = map_ptr + UNDERFLOW;
+ mte_initialize_current_context(mode, (uintptr_t)ptr, sizes[run]);
+ /* Only mte enabled memory will allow tag insertion */
+ ptr = mte_insert_tags((void *)ptr, sizes[run]);
+ if (!ptr || cur_mte_cxt.fault_valid == true) {
+ ksft_print_msg("FAIL: Insert tags on file based memory\n");
+ munmap((void *)map_ptr, map_size);
+ close(fd);
+ return KSFT_FAIL;
+ }
+ result = check_mte_memory(ptr, sizes[run], mode, tag_check);
+ mte_clear_tags((void *)ptr, sizes[run]);
+ munmap((void *)map_ptr, map_size);
+ close(fd);
+ if (result == KSFT_FAIL)
+ break;
+ }
+ return result;
+}
+
+static int check_clear_prot_mte_flag(int mem_type, int mode, int mapping)
+{
+ char *ptr, *map_ptr;
+ int run, prot_flag, result, fd, map_size;
+ int total = sizeof(sizes)/sizeof(int);
+
+ prot_flag = PROT_READ | PROT_WRITE;
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ for (run = 0; run < total; run++) {
+ map_size = sizes[run] + OVERFLOW + UNDERFLOW;
+ ptr = (char *)mte_allocate_memory_tag_range(sizes[run], mem_type, mapping,
+ UNDERFLOW, OVERFLOW);
+ if (check_allocated_memory_range(ptr, sizes[run], mem_type,
+ UNDERFLOW, OVERFLOW) != KSFT_PASS)
+ return KSFT_FAIL;
+ map_ptr = ptr - UNDERFLOW;
+ /* Try to clear PROT_MTE property and verify it by tag checking */
+ if (mprotect(map_ptr, map_size, prot_flag)) {
+ mte_free_memory_tag_range((void *)ptr, sizes[run], mem_type,
+ UNDERFLOW, OVERFLOW);
+ ksft_print_msg("FAIL: mprotect not ignoring clear PROT_MTE property\n");
+ return KSFT_FAIL;
+ }
+ result = check_mte_memory(ptr, sizes[run], mode, TAG_CHECK_ON);
+ mte_free_memory_tag_range((void *)ptr, sizes[run], mem_type, UNDERFLOW, OVERFLOW);
+ if (result != KSFT_PASS)
+ return KSFT_FAIL;
+
+ fd = create_temp_file();
+ if (fd == -1)
+ return KSFT_FAIL;
+ ptr = (char *)mte_allocate_file_memory_tag_range(sizes[run], mem_type, mapping,
+ UNDERFLOW, OVERFLOW, fd);
+ if (check_allocated_memory_range(ptr, sizes[run], mem_type,
+ UNDERFLOW, OVERFLOW) != KSFT_PASS) {
+ close(fd);
+ return KSFT_FAIL;
+ }
+ map_ptr = ptr - UNDERFLOW;
+ /* Try to clear PROT_MTE property and verify it by tag checking */
+ if (mprotect(map_ptr, map_size, prot_flag)) {
+ ksft_print_msg("FAIL: mprotect not ignoring clear PROT_MTE property\n");
+ mte_free_memory_tag_range((void *)ptr, sizes[run], mem_type,
+ UNDERFLOW, OVERFLOW);
+ close(fd);
+ return KSFT_FAIL;
+ }
+ result = check_mte_memory(ptr, sizes[run], mode, TAG_CHECK_ON);
+ mte_free_memory_tag_range((void *)ptr, sizes[run], mem_type, UNDERFLOW, OVERFLOW);
+ close(fd);
+ if (result != KSFT_PASS)
+ return KSFT_FAIL;
+ }
+ return KSFT_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+ int err;
+ int item = sizeof(sizes)/sizeof(int);
+
+ err = mte_default_setup();
+ if (err)
+ return err;
+ page_size = getpagesize();
+ if (!page_size) {
+ ksft_print_msg("ERR: Unable to get page size\n");
+ return KSFT_FAIL;
+ }
+ sizes[item - 3] = page_size - 1;
+ sizes[item - 2] = page_size;
+ sizes[item - 1] = page_size + 1;
+
+ /* Register signal handlers */
+ mte_register_signal(SIGBUS, mte_default_handler);
+ mte_register_signal(SIGSEGV, mte_default_handler);
+
+ mte_enable_pstate_tco();
+ evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_OFF),
+ "Check anonymous memory with private mapping, sync error mode, mmap memory and tag check off\n");
+ evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_OFF),
+ "Check file memory with private mapping, sync error mode, mmap/mprotect memory and tag check off\n");
+
+ mte_disable_pstate_tco();
+ evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_NONE_ERR, MAP_PRIVATE, TAG_CHECK_OFF),
+ "Check anonymous memory with private mapping, no error mode, mmap memory and tag check off\n");
+ evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_NONE_ERR, MAP_PRIVATE, TAG_CHECK_OFF),
+ "Check file memory with private mapping, no error mode, mmap/mprotect memory and tag check off\n");
+
+ evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON),
+ "Check anonymous memory with private mapping, sync error mode, mmap memory and tag check on\n");
+ evaluate_test(check_anonymous_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON),
+ "Check anonymous memory with private mapping, sync error mode, mmap/mprotect memory and tag check on\n");
+ evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_SHARED, TAG_CHECK_ON),
+ "Check anonymous memory with shared mapping, sync error mode, mmap memory and tag check on\n");
+ evaluate_test(check_anonymous_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_SHARED, TAG_CHECK_ON),
+ "Check anonymous memory with shared mapping, sync error mode, mmap/mprotect memory and tag check on\n");
+ evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON),
+ "Check anonymous memory with private mapping, async error mode, mmap memory and tag check on\n");
+ evaluate_test(check_anonymous_memory_mapping(USE_MPROTECT, MTE_ASYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON),
+ "Check anonymous memory with private mapping, async error mode, mmap/mprotect memory and tag check on\n");
+ evaluate_test(check_anonymous_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_SHARED, TAG_CHECK_ON),
+ "Check anonymous memory with shared mapping, async error mode, mmap memory and tag check on\n");
+ evaluate_test(check_anonymous_memory_mapping(USE_MPROTECT, MTE_ASYNC_ERR, MAP_SHARED, TAG_CHECK_ON),
+ "Check anonymous memory with shared mapping, async error mode, mmap/mprotect memory and tag check on\n");
+
+ evaluate_test(check_file_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON),
+ "Check file memory with private mapping, sync error mode, mmap memory and tag check on\n");
+ evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON),
+ "Check file memory with private mapping, sync error mode, mmap/mprotect memory and tag check on\n");
+ evaluate_test(check_file_memory_mapping(USE_MMAP, MTE_SYNC_ERR, MAP_SHARED, TAG_CHECK_ON),
+ "Check file memory with shared mapping, sync error mode, mmap memory and tag check on\n");
+ evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_SYNC_ERR, MAP_SHARED, TAG_CHECK_ON),
+ "Check file memory with shared mapping, sync error mode, mmap/mprotect memory and tag check on\n");
+ evaluate_test(check_file_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON),
+ "Check file memory with private mapping, async error mode, mmap memory and tag check on\n");
+ evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_ASYNC_ERR, MAP_PRIVATE, TAG_CHECK_ON),
+ "Check file memory with private mapping, async error mode, mmap/mprotect memory and tag check on\n");
+ evaluate_test(check_file_memory_mapping(USE_MMAP, MTE_ASYNC_ERR, MAP_SHARED, TAG_CHECK_ON),
+ "Check file memory with shared mapping, async error mode, mmap memory and tag check on\n");
+ evaluate_test(check_file_memory_mapping(USE_MPROTECT, MTE_ASYNC_ERR, MAP_SHARED, TAG_CHECK_ON),
+ "Check file memory with shared mapping, async error mode, mmap/mprotect memory and tag check on\n");
+
+ evaluate_test(check_clear_prot_mte_flag(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE),
+ "Check clear PROT_MTE flags with private mapping, sync error mode and mmap memory\n");
+ evaluate_test(check_clear_prot_mte_flag(USE_MPROTECT, MTE_SYNC_ERR, MAP_PRIVATE),
+ "Check clear PROT_MTE flags with private mapping and sync error mode and mmap/mprotect memory\n");
+
+ mte_restore_setup();
+ ksft_print_cnts();
+ return ksft_get_fail_cnt() == 0 ? KSFT_PASS : KSFT_FAIL;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 ARM Limited
+
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ucontext.h>
+#include <sys/wait.h>
+
+#include "kselftest.h"
+#include "mte_common_util.h"
+#include "mte_def.h"
+
+#define BUFFER_SIZE (5 * MT_GRANULE_SIZE)
+#define RUNS (MT_TAG_COUNT * 2)
+#define MTE_LAST_TAG_MASK (0x7FFF)
+
+static int verify_mte_pointer_validity(char *ptr, int mode)
+{
+ mte_initialize_current_context(mode, (uintptr_t)ptr, BUFFER_SIZE);
+ /* Check the validity of the tagged pointer */
+ memset((void *)ptr, '1', BUFFER_SIZE);
+ mte_wait_after_trig();
+ if (cur_mte_cxt.fault_valid)
+ return KSFT_FAIL;
+ /* Proceed further for nonzero tags */
+ if (!MT_FETCH_TAG((uintptr_t)ptr))
+ return KSFT_PASS;
+ mte_initialize_current_context(mode, (uintptr_t)ptr, BUFFER_SIZE + 1);
+ /* Check the validity outside the range */
+ ptr[BUFFER_SIZE] = '2';
+ mte_wait_after_trig();
+ if (!cur_mte_cxt.fault_valid)
+ return KSFT_FAIL;
+ else
+ return KSFT_PASS;
+}
+
+static int check_single_included_tags(int mem_type, int mode)
+{
+ char *ptr;
+ int tag, run, result = KSFT_PASS;
+
+ ptr = (char *)mte_allocate_memory(BUFFER_SIZE + MT_GRANULE_SIZE, mem_type, 0, false);
+ if (check_allocated_memory(ptr, BUFFER_SIZE + MT_GRANULE_SIZE,
+ mem_type, false) != KSFT_PASS)
+ return KSFT_FAIL;
+
+ for (tag = 0; (tag < MT_TAG_COUNT) && (result == KSFT_PASS); tag++) {
+ mte_switch_mode(mode, MT_INCLUDE_VALID_TAG(tag));
+ /* Try to catch a excluded tag by a number of tries. */
+ for (run = 0; (run < RUNS) && (result == KSFT_PASS); run++) {
+ ptr = (char *)mte_insert_tags(ptr, BUFFER_SIZE);
+ /* Check tag value */
+ if (MT_FETCH_TAG((uintptr_t)ptr) == tag) {
+ ksft_print_msg("FAIL: wrong tag = 0x%x with include mask=0x%x\n",
+ MT_FETCH_TAG((uintptr_t)ptr),
+ MT_INCLUDE_VALID_TAG(tag));
+ result = KSFT_FAIL;
+ break;
+ }
+ result = verify_mte_pointer_validity(ptr, mode);
+ }
+ }
+ mte_free_memory_tag_range((void *)ptr, BUFFER_SIZE, mem_type, 0, MT_GRANULE_SIZE);
+ return result;
+}
+
+static int check_multiple_included_tags(int mem_type, int mode)
+{
+ char *ptr;
+ int tag, run, result = KSFT_PASS;
+ unsigned long excl_mask = 0;
+
+ ptr = (char *)mte_allocate_memory(BUFFER_SIZE + MT_GRANULE_SIZE, mem_type, 0, false);
+ if (check_allocated_memory(ptr, BUFFER_SIZE + MT_GRANULE_SIZE,
+ mem_type, false) != KSFT_PASS)
+ return KSFT_FAIL;
+
+ for (tag = 0; (tag < MT_TAG_COUNT - 1) && (result == KSFT_PASS); tag++) {
+ excl_mask |= 1 << tag;
+ mte_switch_mode(mode, MT_INCLUDE_VALID_TAGS(excl_mask));
+ /* Try to catch a excluded tag by a number of tries. */
+ for (run = 0; (run < RUNS) && (result == KSFT_PASS); run++) {
+ ptr = (char *)mte_insert_tags(ptr, BUFFER_SIZE);
+ /* Check tag value */
+ if (MT_FETCH_TAG((uintptr_t)ptr) < tag) {
+ ksft_print_msg("FAIL: wrong tag = 0x%x with include mask=0x%x\n",
+ MT_FETCH_TAG((uintptr_t)ptr),
+ MT_INCLUDE_VALID_TAGS(excl_mask));
+ result = KSFT_FAIL;
+ break;
+ }
+ result = verify_mte_pointer_validity(ptr, mode);
+ }
+ }
+ mte_free_memory_tag_range((void *)ptr, BUFFER_SIZE, mem_type, 0, MT_GRANULE_SIZE);
+ return result;
+}
+
+static int check_all_included_tags(int mem_type, int mode)
+{
+ char *ptr;
+ int run, result = KSFT_PASS;
+
+ ptr = (char *)mte_allocate_memory(BUFFER_SIZE + MT_GRANULE_SIZE, mem_type, 0, false);
+ if (check_allocated_memory(ptr, BUFFER_SIZE + MT_GRANULE_SIZE,
+ mem_type, false) != KSFT_PASS)
+ return KSFT_FAIL;
+
+ mte_switch_mode(mode, MT_INCLUDE_TAG_MASK);
+ /* Try to catch a excluded tag by a number of tries. */
+ for (run = 0; (run < RUNS) && (result == KSFT_PASS); run++) {
+ ptr = (char *)mte_insert_tags(ptr, BUFFER_SIZE);
+ /*
+ * Here tag byte can be between 0x0 to 0xF (full allowed range)
+ * so no need to match so just verify if it is writable.
+ */
+ result = verify_mte_pointer_validity(ptr, mode);
+ }
+ mte_free_memory_tag_range((void *)ptr, BUFFER_SIZE, mem_type, 0, MT_GRANULE_SIZE);
+ return result;
+}
+
+static int check_none_included_tags(int mem_type, int mode)
+{
+ char *ptr;
+ int run;
+
+ ptr = (char *)mte_allocate_memory(BUFFER_SIZE, mem_type, 0, false);
+ if (check_allocated_memory(ptr, BUFFER_SIZE, mem_type, false) != KSFT_PASS)
+ return KSFT_FAIL;
+
+ mte_switch_mode(mode, MT_EXCLUDE_TAG_MASK);
+ /* Try to catch a excluded tag by a number of tries. */
+ for (run = 0; run < RUNS; run++) {
+ ptr = (char *)mte_insert_tags(ptr, BUFFER_SIZE);
+ /* Here all tags exluded so tag value generated should be 0 */
+ if (MT_FETCH_TAG((uintptr_t)ptr)) {
+ ksft_print_msg("FAIL: included tag value found\n");
+ mte_free_memory((void *)ptr, BUFFER_SIZE, mem_type, true);
+ return KSFT_FAIL;
+ }
+ mte_initialize_current_context(mode, (uintptr_t)ptr, BUFFER_SIZE);
+ /* Check the write validity of the untagged pointer */
+ memset((void *)ptr, '1', BUFFER_SIZE);
+ mte_wait_after_trig();
+ if (cur_mte_cxt.fault_valid)
+ break;
+ }
+ mte_free_memory((void *)ptr, BUFFER_SIZE, mem_type, false);
+ if (cur_mte_cxt.fault_valid)
+ return KSFT_FAIL;
+ else
+ return KSFT_PASS;
+}
+
+int main(int argc, char *argv[])
+{
+ int err;
+
+ err = mte_default_setup();
+ if (err)
+ return err;
+
+ /* Register SIGSEGV handler */
+ mte_register_signal(SIGSEGV, mte_default_handler);
+
+ evaluate_test(check_single_included_tags(USE_MMAP, MTE_SYNC_ERR),
+ "Check an included tag value with sync mode\n");
+ evaluate_test(check_multiple_included_tags(USE_MMAP, MTE_SYNC_ERR),
+ "Check different included tags value with sync mode\n");
+ evaluate_test(check_none_included_tags(USE_MMAP, MTE_SYNC_ERR),
+ "Check none included tags value with sync mode\n");
+ evaluate_test(check_all_included_tags(USE_MMAP, MTE_SYNC_ERR),
+ "Check all included tags value with sync mode\n");
+
+ mte_restore_setup();
+ ksft_print_cnts();
+ return ksft_get_fail_cnt() == 0 ? KSFT_PASS : KSFT_FAIL;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 ARM Limited
+
+#define _GNU_SOURCE
+
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ucontext.h>
+#include <unistd.h>
+#include <sys/mman.h>
+
+#include "kselftest.h"
+#include "mte_common_util.h"
+#include "mte_def.h"
+
+static size_t page_sz;
+
+static int check_usermem_access_fault(int mem_type, int mode, int mapping)
+{
+ int fd, i, err;
+ char val = 'A';
+ size_t len, read_len;
+ void *ptr, *ptr_next;
+
+ err = KSFT_FAIL;
+ len = 2 * page_sz;
+ mte_switch_mode(mode, MTE_ALLOW_NON_ZERO_TAG);
+ fd = create_temp_file();
+ if (fd == -1)
+ return KSFT_FAIL;
+ for (i = 0; i < len; i++)
+ write(fd, &val, sizeof(val));
+ lseek(fd, 0, 0);
+ ptr = mte_allocate_memory(len, mem_type, mapping, true);
+ if (check_allocated_memory(ptr, len, mem_type, true) != KSFT_PASS) {
+ close(fd);
+ return KSFT_FAIL;
+ }
+ mte_initialize_current_context(mode, (uintptr_t)ptr, len);
+ /* Copy from file into buffer with valid tag */
+ read_len = read(fd, ptr, len);
+ mte_wait_after_trig();
+ if (cur_mte_cxt.fault_valid || read_len < len)
+ goto usermem_acc_err;
+ /* Verify same pattern is read */
+ for (i = 0; i < len; i++)
+ if (*(char *)(ptr + i) != val)
+ break;
+ if (i < len)
+ goto usermem_acc_err;
+
+ /* Tag the next half of memory with different value */
+ ptr_next = (void *)((unsigned long)ptr + page_sz);
+ ptr_next = mte_insert_new_tag(ptr_next);
+ mte_set_tag_address_range(ptr_next, page_sz);
+
+ lseek(fd, 0, 0);
+ /* Copy from file into buffer with invalid tag */
+ read_len = read(fd, ptr, len);
+ mte_wait_after_trig();
+ /*
+ * Accessing user memory in kernel with invalid tag should fail in sync
+ * mode without fault but may not fail in async mode as per the
+ * implemented MTE userspace support in Arm64 kernel.
+ */
+ if (mode == MTE_SYNC_ERR &&
+ !cur_mte_cxt.fault_valid && read_len < len) {
+ err = KSFT_PASS;
+ } else if (mode == MTE_ASYNC_ERR &&
+ !cur_mte_cxt.fault_valid && read_len == len) {
+ err = KSFT_PASS;
+ }
+usermem_acc_err:
+ mte_free_memory((void *)ptr, len, mem_type, true);
+ close(fd);
+ return err;
+}
+
+int main(int argc, char *argv[])
+{
+ int err;
+
+ page_sz = getpagesize();
+ if (!page_sz) {
+ ksft_print_msg("ERR: Unable to get page size\n");
+ return KSFT_FAIL;
+ }
+ err = mte_default_setup();
+ if (err)
+ return err;
+ /* Register signal handlers */
+ mte_register_signal(SIGSEGV, mte_default_handler);
+
+ evaluate_test(check_usermem_access_fault(USE_MMAP, MTE_SYNC_ERR, MAP_PRIVATE),
+ "Check memory access from kernel in sync mode, private mapping and mmap memory\n");
+ evaluate_test(check_usermem_access_fault(USE_MMAP, MTE_SYNC_ERR, MAP_SHARED),
+ "Check memory access from kernel in sync mode, shared mapping and mmap memory\n");
+
+ evaluate_test(check_usermem_access_fault(USE_MMAP, MTE_ASYNC_ERR, MAP_PRIVATE),
+ "Check memory access from kernel in async mode, private mapping and mmap memory\n");
+ evaluate_test(check_usermem_access_fault(USE_MMAP, MTE_ASYNC_ERR, MAP_SHARED),
+ "Check memory access from kernel in async mode, shared mapping and mmap memory\n");
+
+ mte_restore_setup();
+ ksft_print_cnts();
+ return ksft_get_fail_cnt() == 0 ? KSFT_PASS : KSFT_FAIL;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 ARM Limited
+
+#include <fcntl.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <linux/auxvec.h>
+#include <sys/auxv.h>
+#include <sys/mman.h>
+#include <sys/prctl.h>
+
+#include <asm/hwcap.h>
+
+#include "kselftest.h"
+#include "mte_common_util.h"
+#include "mte_def.h"
+
+#define INIT_BUFFER_SIZE 256
+
+struct mte_fault_cxt cur_mte_cxt;
+static unsigned int mte_cur_mode;
+static unsigned int mte_cur_pstate_tco;
+
+void mte_default_handler(int signum, siginfo_t *si, void *uc)
+{
+ unsigned long addr = (unsigned long)si->si_addr;
+
+ if (signum == SIGSEGV) {
+#ifdef DEBUG
+ ksft_print_msg("INFO: SIGSEGV signal at pc=%lx, fault addr=%lx, si_code=%lx\n",
+ ((ucontext_t *)uc)->uc_mcontext.pc, addr, si->si_code);
+#endif
+ if (si->si_code == SEGV_MTEAERR) {
+ if (cur_mte_cxt.trig_si_code == si->si_code)
+ cur_mte_cxt.fault_valid = true;
+ return;
+ }
+ /* Compare the context for precise error */
+ else if (si->si_code == SEGV_MTESERR) {
+ if (cur_mte_cxt.trig_si_code == si->si_code &&
+ ((cur_mte_cxt.trig_range >= 0 &&
+ addr >= MT_CLEAR_TAG(cur_mte_cxt.trig_addr) &&
+ addr <= (MT_CLEAR_TAG(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range)) ||
+ (cur_mte_cxt.trig_range < 0 &&
+ addr <= MT_CLEAR_TAG(cur_mte_cxt.trig_addr) &&
+ addr >= (MT_CLEAR_TAG(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range)))) {
+ cur_mte_cxt.fault_valid = true;
+ /* Adjust the pc by 4 */
+ ((ucontext_t *)uc)->uc_mcontext.pc += 4;
+ } else {
+ ksft_print_msg("Invalid MTE synchronous exception caught!\n");
+ exit(1);
+ }
+ } else {
+ ksft_print_msg("Unknown SIGSEGV exception caught!\n");
+ exit(1);
+ }
+ } else if (signum == SIGBUS) {
+ ksft_print_msg("INFO: SIGBUS signal at pc=%lx, fault addr=%lx, si_code=%lx\n",
+ ((ucontext_t *)uc)->uc_mcontext.pc, addr, si->si_code);
+ if ((cur_mte_cxt.trig_range >= 0 &&
+ addr >= MT_CLEAR_TAG(cur_mte_cxt.trig_addr) &&
+ addr <= (MT_CLEAR_TAG(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range)) ||
+ (cur_mte_cxt.trig_range < 0 &&
+ addr <= MT_CLEAR_TAG(cur_mte_cxt.trig_addr) &&
+ addr >= (MT_CLEAR_TAG(cur_mte_cxt.trig_addr) + cur_mte_cxt.trig_range))) {
+ cur_mte_cxt.fault_valid = true;
+ /* Adjust the pc by 4 */
+ ((ucontext_t *)uc)->uc_mcontext.pc += 4;
+ }
+ }
+}
+
+void mte_register_signal(int signal, void (*handler)(int, siginfo_t *, void *))
+{
+ struct sigaction sa;
+
+ sa.sa_sigaction = handler;
+ sa.sa_flags = SA_SIGINFO;
+ sigemptyset(&sa.sa_mask);
+ sigaction(signal, &sa, NULL);
+}
+
+void mte_wait_after_trig(void)
+{
+ sched_yield();
+}
+
+void *mte_insert_tags(void *ptr, size_t size)
+{
+ void *tag_ptr;
+ int align_size;
+
+ if (!ptr || (unsigned long)(ptr) & MT_ALIGN_GRANULE) {
+ ksft_print_msg("FAIL: Addr=%lx: invalid\n", ptr);
+ return NULL;
+ }
+ align_size = MT_ALIGN_UP(size);
+ tag_ptr = mte_insert_random_tag(ptr);
+ mte_set_tag_address_range(tag_ptr, align_size);
+ return tag_ptr;
+}
+
+void mte_clear_tags(void *ptr, size_t size)
+{
+ if (!ptr || (unsigned long)(ptr) & MT_ALIGN_GRANULE) {
+ ksft_print_msg("FAIL: Addr=%lx: invalid\n", ptr);
+ return;
+ }
+ size = MT_ALIGN_UP(size);
+ ptr = (void *)MT_CLEAR_TAG((unsigned long)ptr);
+ mte_clear_tag_address_range(ptr, size);
+}
+
+static void *__mte_allocate_memory_range(size_t size, int mem_type, int mapping,
+ size_t range_before, size_t range_after,
+ bool tags, int fd)
+{
+ void *ptr;
+ int prot_flag, map_flag;
+ size_t entire_size = size + range_before + range_after;
+
+ if (mem_type != USE_MALLOC && mem_type != USE_MMAP &&
+ mem_type != USE_MPROTECT) {
+ ksft_print_msg("FAIL: Invalid allocate request\n");
+ return NULL;
+ }
+ if (mem_type == USE_MALLOC)
+ return malloc(entire_size) + range_before;
+
+ prot_flag = PROT_READ | PROT_WRITE;
+ if (mem_type == USE_MMAP)
+ prot_flag |= PROT_MTE;
+
+ map_flag = mapping;
+ if (fd == -1)
+ map_flag = MAP_ANONYMOUS | map_flag;
+ if (!(mapping & MAP_SHARED))
+ map_flag |= MAP_PRIVATE;
+ ptr = mmap(NULL, entire_size, prot_flag, map_flag, fd, 0);
+ if (ptr == MAP_FAILED) {
+ ksft_print_msg("FAIL: mmap allocation\n");
+ return NULL;
+ }
+ if (mem_type == USE_MPROTECT) {
+ if (mprotect(ptr, entire_size, prot_flag | PROT_MTE)) {
+ munmap(ptr, size);
+ ksft_print_msg("FAIL: mprotect PROT_MTE property\n");
+ return NULL;
+ }
+ }
+ if (tags)
+ ptr = mte_insert_tags(ptr + range_before, size);
+ return ptr;
+}
+
+void *mte_allocate_memory_tag_range(size_t size, int mem_type, int mapping,
+ size_t range_before, size_t range_after)
+{
+ return __mte_allocate_memory_range(size, mem_type, mapping, range_before,
+ range_after, true, -1);
+}
+
+void *mte_allocate_memory(size_t size, int mem_type, int mapping, bool tags)
+{
+ return __mte_allocate_memory_range(size, mem_type, mapping, 0, 0, tags, -1);
+}
+
+void *mte_allocate_file_memory(size_t size, int mem_type, int mapping, bool tags, int fd)
+{
+ int index;
+ char buffer[INIT_BUFFER_SIZE];
+
+ if (mem_type != USE_MPROTECT && mem_type != USE_MMAP) {
+ ksft_print_msg("FAIL: Invalid mmap file request\n");
+ return NULL;
+ }
+ /* Initialize the file for mappable size */
+ lseek(fd, 0, SEEK_SET);
+ for (index = INIT_BUFFER_SIZE; index < size; index += INIT_BUFFER_SIZE)
+ write(fd, buffer, INIT_BUFFER_SIZE);
+ index -= INIT_BUFFER_SIZE;
+ write(fd, buffer, size - index);
+ return __mte_allocate_memory_range(size, mem_type, mapping, 0, 0, tags, fd);
+}
+
+void *mte_allocate_file_memory_tag_range(size_t size, int mem_type, int mapping,
+ size_t range_before, size_t range_after, int fd)
+{
+ int index;
+ char buffer[INIT_BUFFER_SIZE];
+ int map_size = size + range_before + range_after;
+
+ if (mem_type != USE_MPROTECT && mem_type != USE_MMAP) {
+ ksft_print_msg("FAIL: Invalid mmap file request\n");
+ return NULL;
+ }
+ /* Initialize the file for mappable size */
+ lseek(fd, 0, SEEK_SET);
+ for (index = INIT_BUFFER_SIZE; index < map_size; index += INIT_BUFFER_SIZE)
+ write(fd, buffer, INIT_BUFFER_SIZE);
+ index -= INIT_BUFFER_SIZE;
+ write(fd, buffer, map_size - index);
+ return __mte_allocate_memory_range(size, mem_type, mapping, range_before,
+ range_after, true, fd);
+}
+
+static void __mte_free_memory_range(void *ptr, size_t size, int mem_type,
+ size_t range_before, size_t range_after, bool tags)
+{
+ switch (mem_type) {
+ case USE_MALLOC:
+ free(ptr - range_before);
+ break;
+ case USE_MMAP:
+ case USE_MPROTECT:
+ if (tags)
+ mte_clear_tags(ptr, size);
+ munmap(ptr - range_before, size + range_before + range_after);
+ break;
+ default:
+ ksft_print_msg("FAIL: Invalid free request\n");
+ break;
+ }
+}
+
+void mte_free_memory_tag_range(void *ptr, size_t size, int mem_type,
+ size_t range_before, size_t range_after)
+{
+ __mte_free_memory_range(ptr, size, mem_type, range_before, range_after, true);
+}
+
+void mte_free_memory(void *ptr, size_t size, int mem_type, bool tags)
+{
+ __mte_free_memory_range(ptr, size, mem_type, 0, 0, tags);
+}
+
+void mte_initialize_current_context(int mode, uintptr_t ptr, ssize_t range)
+{
+ cur_mte_cxt.fault_valid = false;
+ cur_mte_cxt.trig_addr = ptr;
+ cur_mte_cxt.trig_range = range;
+ if (mode == MTE_SYNC_ERR)
+ cur_mte_cxt.trig_si_code = SEGV_MTESERR;
+ else if (mode == MTE_ASYNC_ERR)
+ cur_mte_cxt.trig_si_code = SEGV_MTEAERR;
+ else
+ cur_mte_cxt.trig_si_code = 0;
+}
+
+int mte_switch_mode(int mte_option, unsigned long incl_mask)
+{
+ unsigned long en = 0;
+
+ if (!(mte_option == MTE_SYNC_ERR || mte_option == MTE_ASYNC_ERR ||
+ mte_option == MTE_NONE_ERR || incl_mask <= MTE_ALLOW_NON_ZERO_TAG)) {
+ ksft_print_msg("FAIL: Invalid mte config option\n");
+ return -EINVAL;
+ }
+ en = PR_TAGGED_ADDR_ENABLE;
+ if (mte_option == MTE_SYNC_ERR)
+ en |= PR_MTE_TCF_SYNC;
+ else if (mte_option == MTE_ASYNC_ERR)
+ en |= PR_MTE_TCF_ASYNC;
+ else if (mte_option == MTE_NONE_ERR)
+ en |= PR_MTE_TCF_NONE;
+
+ en |= (incl_mask << PR_MTE_TAG_SHIFT);
+ /* Enable address tagging ABI, mte error reporting mode and tag inclusion mask. */
+ if (!prctl(PR_SET_TAGGED_ADDR_CTRL, en, 0, 0, 0) == 0) {
+ ksft_print_msg("FAIL:prctl PR_SET_TAGGED_ADDR_CTRL for mte mode\n");
+ return -EINVAL;
+ }
+ return 0;
+}
+
+#define ID_AA64PFR1_MTE_SHIFT 8
+#define ID_AA64PFR1_MTE 2
+
+int mte_default_setup(void)
+{
+ unsigned long hwcaps = getauxval(AT_HWCAP);
+ unsigned long en = 0;
+ int ret;
+
+ if (!(hwcaps & HWCAP_CPUID)) {
+ ksft_print_msg("FAIL: CPUID registers unavailable\n");
+ return KSFT_FAIL;
+ }
+ /* Read ID_AA64PFR1_EL1 register */
+ asm volatile("mrs %0, id_aa64pfr1_el1" : "=r"(hwcaps) : : "memory");
+ if (((hwcaps >> ID_AA64PFR1_MTE_SHIFT) & MT_TAG_MASK) != ID_AA64PFR1_MTE) {
+ ksft_print_msg("FAIL: MTE features unavailable\n");
+ return KSFT_SKIP;
+ }
+ /* Get current mte mode */
+ ret = prctl(PR_GET_TAGGED_ADDR_CTRL, en, 0, 0, 0);
+ if (ret < 0) {
+ ksft_print_msg("FAIL:prctl PR_GET_TAGGED_ADDR_CTRL with error =%d\n", ret);
+ return KSFT_FAIL;
+ }
+ if (ret & PR_MTE_TCF_SYNC)
+ mte_cur_mode = MTE_SYNC_ERR;
+ else if (ret & PR_MTE_TCF_ASYNC)
+ mte_cur_mode = MTE_ASYNC_ERR;
+ else if (ret & PR_MTE_TCF_NONE)
+ mte_cur_mode = MTE_NONE_ERR;
+
+ mte_cur_pstate_tco = mte_get_pstate_tco();
+ /* Disable PSTATE.TCO */
+ mte_disable_pstate_tco();
+ return 0;
+}
+
+void mte_restore_setup(void)
+{
+ mte_switch_mode(mte_cur_mode, MTE_ALLOW_NON_ZERO_TAG);
+ if (mte_cur_pstate_tco == MT_PSTATE_TCO_EN)
+ mte_enable_pstate_tco();
+ else if (mte_cur_pstate_tco == MT_PSTATE_TCO_DIS)
+ mte_disable_pstate_tco();
+}
+
+int create_temp_file(void)
+{
+ int fd;
+ char filename[] = "/dev/shm/tmp_XXXXXX";
+
+ /* Create a file in the tmpfs filesystem */
+ fd = mkstemp(&filename[0]);
+ if (fd == -1) {
+ ksft_print_msg("FAIL: Unable to open temporary file\n");
+ return 0;
+ }
+ unlink(&filename[0]);
+ return fd;
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2020 ARM Limited */
+
+#ifndef _MTE_COMMON_UTIL_H
+#define _MTE_COMMON_UTIL_H
+
+#include <signal.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <sys/auxv.h>
+#include <sys/mman.h>
+#include <sys/prctl.h>
+#include "mte_def.h"
+#include "kselftest.h"
+
+enum mte_mem_type {
+ USE_MALLOC,
+ USE_MMAP,
+ USE_MPROTECT,
+};
+
+enum mte_mode {
+ MTE_NONE_ERR,
+ MTE_SYNC_ERR,
+ MTE_ASYNC_ERR,
+};
+
+struct mte_fault_cxt {
+ /* Address start which triggers mte tag fault */
+ unsigned long trig_addr;
+ /* Address range for mte tag fault and negative value means underflow */
+ ssize_t trig_range;
+ /* siginfo si code */
+ unsigned long trig_si_code;
+ /* Flag to denote if correct fault caught */
+ bool fault_valid;
+};
+
+extern struct mte_fault_cxt cur_mte_cxt;
+
+/* MTE utility functions */
+void mte_default_handler(int signum, siginfo_t *si, void *uc);
+void mte_register_signal(int signal, void (*handler)(int, siginfo_t *, void *));
+void mte_wait_after_trig(void);
+void *mte_allocate_memory(size_t size, int mem_type, int mapping, bool tags);
+void *mte_allocate_memory_tag_range(size_t size, int mem_type, int mapping,
+ size_t range_before, size_t range_after);
+void *mte_allocate_file_memory(size_t size, int mem_type, int mapping,
+ bool tags, int fd);
+void *mte_allocate_file_memory_tag_range(size_t size, int mem_type, int mapping,
+ size_t range_before, size_t range_after, int fd);
+void mte_free_memory(void *ptr, size_t size, int mem_type, bool tags);
+void mte_free_memory_tag_range(void *ptr, size_t size, int mem_type,
+ size_t range_before, size_t range_after);
+void *mte_insert_tags(void *ptr, size_t size);
+void mte_clear_tags(void *ptr, size_t size);
+int mte_default_setup(void);
+void mte_restore_setup(void);
+int mte_switch_mode(int mte_option, unsigned long incl_mask);
+void mte_initialize_current_context(int mode, uintptr_t ptr, ssize_t range);
+
+/* Common utility functions */
+int create_temp_file(void);
+
+/* Assembly MTE utility functions */
+void *mte_insert_random_tag(void *ptr);
+void *mte_insert_new_tag(void *ptr);
+void *mte_get_tag_address(void *ptr);
+void mte_set_tag_address_range(void *ptr, int range);
+void mte_clear_tag_address_range(void *ptr, int range);
+void mte_disable_pstate_tco(void);
+void mte_enable_pstate_tco(void);
+unsigned int mte_get_pstate_tco(void);
+
+/* Test framework static inline functions/macros */
+static inline void evaluate_test(int err, const char *msg)
+{
+ if (err == KSFT_PASS)
+ ksft_test_result_pass(msg);
+ else if (err == KSFT_FAIL)
+ ksft_test_result_fail(msg);
+}
+
+static inline int check_allocated_memory(void *ptr, size_t size,
+ int mem_type, bool tags)
+{
+ if (ptr == NULL) {
+ ksft_print_msg("FAIL: memory allocation\n");
+ return KSFT_FAIL;
+ }
+
+ if (tags && !MT_FETCH_TAG((uintptr_t)ptr)) {
+ ksft_print_msg("FAIL: tag not found at addr(%p)\n", ptr);
+ mte_free_memory((void *)ptr, size, mem_type, false);
+ return KSFT_FAIL;
+ }
+
+ return KSFT_PASS;
+}
+
+static inline int check_allocated_memory_range(void *ptr, size_t size, int mem_type,
+ size_t range_before, size_t range_after)
+{
+ if (ptr == NULL) {
+ ksft_print_msg("FAIL: memory allocation\n");
+ return KSFT_FAIL;
+ }
+
+ if (!MT_FETCH_TAG((uintptr_t)ptr)) {
+ ksft_print_msg("FAIL: tag not found at addr(%p)\n", ptr);
+ mte_free_memory_tag_range((void *)ptr, size, mem_type, range_before,
+ range_after);
+ return KSFT_FAIL;
+ }
+ return KSFT_PASS;
+}
+
+#endif /* _MTE_COMMON_UTIL_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2020 ARM Limited */
+
+/*
+ * Below definitions may be found in kernel headers, However, they are
+ * redefined here to decouple the MTE selftests compilations from them.
+ */
+#ifndef SEGV_MTEAERR
+#define SEGV_MTEAERR 8
+#endif
+#ifndef SEGV_MTESERR
+#define SEGV_MTESERR 9
+#endif
+#ifndef PROT_MTE
+#define PROT_MTE 0x20
+#endif
+#ifndef HWCAP2_MTE
+#define HWCAP2_MTE (1 << 18)
+#endif
+
+#ifndef PR_MTE_TCF_SHIFT
+#define PR_MTE_TCF_SHIFT 1
+#endif
+#ifndef PR_MTE_TCF_NONE
+#define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
+#endif
+#ifndef PR_MTE_TCF_SYNC
+#define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
+#endif
+#ifndef PR_MTE_TCF_ASYNC
+#define PR_MTE_TCF_ASYNC (2UL << PR_MTE_TCF_SHIFT)
+#endif
+#ifndef PR_MTE_TAG_SHIFT
+#define PR_MTE_TAG_SHIFT 3
+#endif
+
+/* MTE Hardware feature definitions below. */
+#define MT_TAG_SHIFT 56
+#define MT_TAG_MASK 0xFUL
+#define MT_FREE_TAG 0x0UL
+#define MT_GRANULE_SIZE 16
+#define MT_TAG_COUNT 16
+#define MT_INCLUDE_TAG_MASK 0xFFFF
+#define MT_EXCLUDE_TAG_MASK 0x0
+
+#define MT_ALIGN_GRANULE (MT_GRANULE_SIZE - 1)
+#define MT_CLEAR_TAG(x) ((x) & ~(MT_TAG_MASK << MT_TAG_SHIFT))
+#define MT_SET_TAG(x, y) ((x) | (y << MT_TAG_SHIFT))
+#define MT_FETCH_TAG(x) ((x >> MT_TAG_SHIFT) & (MT_TAG_MASK))
+#define MT_ALIGN_UP(x) ((x + MT_ALIGN_GRANULE) & ~(MT_ALIGN_GRANULE))
+
+#define MT_PSTATE_TCO_SHIFT 25
+#define MT_PSTATE_TCO_MASK ~(0x1 << MT_PSTATE_TCO_SHIFT)
+#define MT_PSTATE_TCO_EN 1
+#define MT_PSTATE_TCO_DIS 0
+
+#define MT_EXCLUDE_TAG(x) (1 << (x))
+#define MT_INCLUDE_VALID_TAG(x) (MT_INCLUDE_TAG_MASK ^ MT_EXCLUDE_TAG(x))
+#define MT_INCLUDE_VALID_TAGS(x) (MT_INCLUDE_TAG_MASK ^ (x))
+#define MTE_ALLOW_NON_ZERO_TAG MT_INCLUDE_VALID_TAG(0)
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2020 ARM Limited */
+
+#include "mte_def.h"
+
+#define ENTRY(name) \
+ .globl name ;\
+ .p2align 2;\
+ .type name, @function ;\
+name:
+
+#define ENDPROC(name) \
+ .size name, .-name ;
+
+ .text
+/*
+ * mte_insert_random_tag: Insert random tag and might be same as the source tag if
+ * the source pointer has it.
+ * Input:
+ * x0 - source pointer with a tag/no-tag
+ * Return:
+ * x0 - pointer with random tag
+ */
+ENTRY(mte_insert_random_tag)
+ irg x0, x0, xzr
+ ret
+ENDPROC(mte_insert_random_tag)
+
+/*
+ * mte_insert_new_tag: Insert new tag and different from the source tag if
+ * source pointer has it.
+ * Input:
+ * x0 - source pointer with a tag/no-tag
+ * Return:
+ * x0 - pointer with random tag
+ */
+ENTRY(mte_insert_new_tag)
+ gmi x1, x0, xzr
+ irg x0, x0, x1
+ ret
+ENDPROC(mte_insert_new_tag)
+
+/*
+ * mte_get_tag_address: Get the tag from given address.
+ * Input:
+ * x0 - source pointer
+ * Return:
+ * x0 - pointer with appended tag
+ */
+ENTRY(mte_get_tag_address)
+ ldg x0, [x0]
+ ret
+ENDPROC(mte_get_tag_address)
+
+/*
+ * mte_set_tag_address_range: Set the tag range from the given address
+ * Input:
+ * x0 - source pointer with tag data
+ * x1 - range
+ * Return:
+ * none
+ */
+ENTRY(mte_set_tag_address_range)
+ cbz x1, 2f
+1:
+ stg x0, [x0, #0x0]
+ add x0, x0, #MT_GRANULE_SIZE
+ sub x1, x1, #MT_GRANULE_SIZE
+ cbnz x1, 1b
+2:
+ ret
+ENDPROC(mte_set_tag_address_range)
+
+/*
+ * mt_clear_tag_address_range: Clear the tag range from the given address
+ * Input:
+ * x0 - source pointer with tag data
+ * x1 - range
+ * Return:
+ * none
+ */
+ENTRY(mte_clear_tag_address_range)
+ cbz x1, 2f
+1:
+ stzg x0, [x0, #0x0]
+ add x0, x0, #MT_GRANULE_SIZE
+ sub x1, x1, #MT_GRANULE_SIZE
+ cbnz x1, 1b
+2:
+ ret
+ENDPROC(mte_clear_tag_address_range)
+
+/*
+ * mte_enable_pstate_tco: Enable PSTATE.TCO (tag check override) field
+ * Input:
+ * none
+ * Return:
+ * none
+ */
+ENTRY(mte_enable_pstate_tco)
+ msr tco, #MT_PSTATE_TCO_EN
+ ret
+ENDPROC(mte_enable_pstate_tco)
+
+/*
+ * mte_disable_pstate_tco: Disable PSTATE.TCO (tag check override) field
+ * Input:
+ * none
+ * Return:
+ * none
+ */
+ENTRY(mte_disable_pstate_tco)
+ msr tco, #MT_PSTATE_TCO_DIS
+ ret
+ENDPROC(mte_disable_pstate_tco)
+
+/*
+ * mte_get_pstate_tco: Get PSTATE.TCO (tag check override) field
+ * Input:
+ * none
+ * Return:
+ * x0
+ */
+ENTRY(mte_get_pstate_tco)
+ mrs x0, tco
+ ubfx x0, x0, #MT_PSTATE_TCO_SHIFT, #1
+ ret
+ENDPROC(mte_get_pstate_tco)
--- /dev/null
+exec_target
+pac
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2020 ARM Limited
+
+# preserve CC value from top level Makefile
+ifeq ($(CC),cc)
+CC := $(CROSS_COMPILE)gcc
+endif
+
+CFLAGS += -mbranch-protection=pac-ret
+# check if the compiler supports ARMv8.3 and branch protection with PAuth
+pauth_cc_support := $(shell if ($(CC) $(CFLAGS) -march=armv8.3-a -E -x c /dev/null -o /dev/null 2>&1) then echo "1"; fi)
+
+ifeq ($(pauth_cc_support),1)
+TEST_GEN_PROGS := pac
+TEST_GEN_FILES := pac_corruptor.o helper.o
+TEST_GEN_PROGS_EXTENDED := exec_target
+endif
+
+include ../../lib.mk
+
+ifeq ($(pauth_cc_support),1)
+# pac* and aut* instructions are not available on architectures berfore
+# ARMv8.3. Therefore target ARMv8.3 wherever they are used directly
+$(OUTPUT)/pac_corruptor.o: pac_corruptor.S
+ $(CC) -c $^ -o $@ $(CFLAGS) -march=armv8.3-a
+
+$(OUTPUT)/helper.o: helper.c
+ $(CC) -c $^ -o $@ $(CFLAGS) -march=armv8.3-a
+
+# when -mbranch-protection is enabled and the target architecture is ARMv8.3 or
+# greater, gcc emits pac* instructions which are not in HINT NOP space,
+# preventing the tests from occurring at all. Compile for ARMv8.2 so tests can
+# run on earlier targets and print a meaningful error messages
+$(OUTPUT)/exec_target: exec_target.c $(OUTPUT)/helper.o
+ $(CC) $^ -o $@ $(CFLAGS) -march=armv8.2-a
+
+$(OUTPUT)/pac: pac.c $(OUTPUT)/pac_corruptor.o $(OUTPUT)/helper.o
+ $(CC) $^ -o $@ $(CFLAGS) -march=armv8.2-a
+endif
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 ARM Limited
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/auxv.h>
+
+#include "helper.h"
+
+int main(void)
+{
+ struct signatures signed_vals;
+ unsigned long hwcaps;
+ size_t val;
+
+ fread(&val, sizeof(size_t), 1, stdin);
+
+ /* don't try to execute illegal (unimplemented) instructions) caller
+ * should have checked this and keep worker simple
+ */
+ hwcaps = getauxval(AT_HWCAP);
+
+ if (hwcaps & HWCAP_PACA) {
+ signed_vals.keyia = keyia_sign(val);
+ signed_vals.keyib = keyib_sign(val);
+ signed_vals.keyda = keyda_sign(val);
+ signed_vals.keydb = keydb_sign(val);
+ }
+ signed_vals.keyg = (hwcaps & HWCAP_PACG) ? keyg_sign(val) : 0;
+
+ fwrite(&signed_vals, sizeof(struct signatures), 1, stdout);
+
+ return 0;
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 ARM Limited
+
+#include "helper.h"
+
+size_t keyia_sign(size_t ptr)
+{
+ asm volatile("paciza %0" : "+r" (ptr));
+ return ptr;
+}
+
+size_t keyib_sign(size_t ptr)
+{
+ asm volatile("pacizb %0" : "+r" (ptr));
+ return ptr;
+}
+
+size_t keyda_sign(size_t ptr)
+{
+ asm volatile("pacdza %0" : "+r" (ptr));
+ return ptr;
+}
+
+size_t keydb_sign(size_t ptr)
+{
+ asm volatile("pacdzb %0" : "+r" (ptr));
+ return ptr;
+}
+
+size_t keyg_sign(size_t ptr)
+{
+ /* output is encoded in the upper 32 bits */
+ size_t dest = 0;
+ size_t modifier = 0;
+
+ asm volatile("pacga %0, %1, %2" : "=r" (dest) : "r" (ptr), "r" (modifier));
+
+ return dest;
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2020 ARM Limited */
+
+#ifndef _HELPER_H_
+#define _HELPER_H_
+
+#include <stdlib.h>
+
+#define NKEYS 5
+
+struct signatures {
+ size_t keyia;
+ size_t keyib;
+ size_t keyda;
+ size_t keydb;
+ size_t keyg;
+};
+
+void pac_corruptor(void);
+
+/* PAuth sign a value with key ia and modifier value 0 */
+size_t keyia_sign(size_t val);
+size_t keyib_sign(size_t val);
+size_t keyda_sign(size_t val);
+size_t keydb_sign(size_t val);
+size_t keyg_sign(size_t val);
+
+#endif
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2020 ARM Limited
+
+#define _GNU_SOURCE
+
+#include <sys/auxv.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <signal.h>
+#include <setjmp.h>
+#include <sched.h>
+
+#include "../../kselftest_harness.h"
+#include "helper.h"
+
+#define PAC_COLLISION_ATTEMPTS 10
+/*
+ * The kernel sets TBID by default. So bits 55 and above should remain
+ * untouched no matter what.
+ * The VA space size is 48 bits. Bigger is opt-in.
+ */
+#define PAC_MASK (~0xff80ffffffffffff)
+#define ARBITRARY_VALUE (0x1234)
+#define ASSERT_PAUTH_ENABLED() \
+do { \
+ unsigned long hwcaps = getauxval(AT_HWCAP); \
+ /* data key instructions are not in NOP space. This prevents a SIGILL */ \
+ ASSERT_NE(0, hwcaps & HWCAP_PACA) TH_LOG("PAUTH not enabled"); \
+} while (0)
+#define ASSERT_GENERIC_PAUTH_ENABLED() \
+do { \
+ unsigned long hwcaps = getauxval(AT_HWCAP); \
+ /* generic key instructions are not in NOP space. This prevents a SIGILL */ \
+ ASSERT_NE(0, hwcaps & HWCAP_PACG) TH_LOG("Generic PAUTH not enabled"); \
+} while (0)
+
+void sign_specific(struct signatures *sign, size_t val)
+{
+ sign->keyia = keyia_sign(val);
+ sign->keyib = keyib_sign(val);
+ sign->keyda = keyda_sign(val);
+ sign->keydb = keydb_sign(val);
+}
+
+void sign_all(struct signatures *sign, size_t val)
+{
+ sign->keyia = keyia_sign(val);
+ sign->keyib = keyib_sign(val);
+ sign->keyda = keyda_sign(val);
+ sign->keydb = keydb_sign(val);
+ sign->keyg = keyg_sign(val);
+}
+
+int n_same(struct signatures *old, struct signatures *new, int nkeys)
+{
+ int res = 0;
+
+ res += old->keyia == new->keyia;
+ res += old->keyib == new->keyib;
+ res += old->keyda == new->keyda;
+ res += old->keydb == new->keydb;
+ if (nkeys == NKEYS)
+ res += old->keyg == new->keyg;
+
+ return res;
+}
+
+int n_same_single_set(struct signatures *sign, int nkeys)
+{
+ size_t vals[nkeys];
+ int same = 0;
+
+ vals[0] = sign->keyia & PAC_MASK;
+ vals[1] = sign->keyib & PAC_MASK;
+ vals[2] = sign->keyda & PAC_MASK;
+ vals[3] = sign->keydb & PAC_MASK;
+
+ if (nkeys >= 4)
+ vals[4] = sign->keyg & PAC_MASK;
+
+ for (int i = 0; i < nkeys - 1; i++) {
+ for (int j = i + 1; j < nkeys; j++) {
+ if (vals[i] == vals[j])
+ same += 1;
+ }
+ }
+ return same;
+}
+
+int exec_sign_all(struct signatures *signed_vals, size_t val)
+{
+ int new_stdin[2];
+ int new_stdout[2];
+ int status;
+ int i;
+ ssize_t ret;
+ pid_t pid;
+ cpu_set_t mask;
+
+ ret = pipe(new_stdin);
+ if (ret == -1) {
+ perror("pipe returned error");
+ return -1;
+ }
+
+ ret = pipe(new_stdout);
+ if (ret == -1) {
+ perror("pipe returned error");
+ return -1;
+ }
+
+ /*
+ * pin this process and all its children to a single CPU, so it can also
+ * guarantee a context switch with its child
+ */
+ sched_getaffinity(0, sizeof(mask), &mask);
+
+ for (i = 0; i < sizeof(cpu_set_t); i++)
+ if (CPU_ISSET(i, &mask))
+ break;
+
+ CPU_ZERO(&mask);
+ CPU_SET(i, &mask);
+ sched_setaffinity(0, sizeof(mask), &mask);
+
+ pid = fork();
+ // child
+ if (pid == 0) {
+ dup2(new_stdin[0], STDIN_FILENO);
+ if (ret == -1) {
+ perror("dup2 returned error");
+ exit(1);
+ }
+
+ dup2(new_stdout[1], STDOUT_FILENO);
+ if (ret == -1) {
+ perror("dup2 returned error");
+ exit(1);
+ }
+
+ close(new_stdin[0]);
+ close(new_stdin[1]);
+ close(new_stdout[0]);
+ close(new_stdout[1]);
+
+ ret = execl("exec_target", "exec_target", (char *)NULL);
+ if (ret == -1) {
+ perror("exec returned error");
+ exit(1);
+ }
+ }
+
+ close(new_stdin[0]);
+ close(new_stdout[1]);
+
+ ret = write(new_stdin[1], &val, sizeof(size_t));
+ if (ret == -1) {
+ perror("write returned error");
+ return -1;
+ }
+
+ /*
+ * wait for the worker to finish, so that read() reads all data
+ * will also context switch with worker so that this function can be used
+ * for context switch tests
+ */
+ waitpid(pid, &status, 0);
+ if (WIFEXITED(status) == 0) {
+ fprintf(stderr, "worker exited unexpectedly\n");
+ return -1;
+ }
+ if (WEXITSTATUS(status) != 0) {
+ fprintf(stderr, "worker exited with error\n");
+ return -1;
+ }
+
+ ret = read(new_stdout[0], signed_vals, sizeof(struct signatures));
+ if (ret == -1) {
+ perror("read returned error");
+ return -1;
+ }
+
+ return 0;
+}
+
+sigjmp_buf jmpbuf;
+void pac_signal_handler(int signum, siginfo_t *si, void *uc)
+{
+ if (signum == SIGSEGV || signum == SIGILL)
+ siglongjmp(jmpbuf, 1);
+}
+
+/* check that a corrupted PAC results in SIGSEGV or SIGILL */
+TEST(corrupt_pac)
+{
+ struct sigaction sa;
+
+ ASSERT_PAUTH_ENABLED();
+ if (sigsetjmp(jmpbuf, 1) == 0) {
+ sa.sa_sigaction = pac_signal_handler;
+ sa.sa_flags = SA_SIGINFO | SA_RESETHAND;
+ sigemptyset(&sa.sa_mask);
+
+ sigaction(SIGSEGV, &sa, NULL);
+ sigaction(SIGILL, &sa, NULL);
+
+ pac_corruptor();
+ ASSERT_TRUE(0) TH_LOG("SIGSEGV/SIGILL signal did not occur");
+ }
+}
+
+/*
+ * There are no separate pac* and aut* controls so checking only the pac*
+ * instructions is sufficient
+ */
+TEST(pac_instructions_not_nop)
+{
+ size_t keyia = 0;
+ size_t keyib = 0;
+ size_t keyda = 0;
+ size_t keydb = 0;
+
+ ASSERT_PAUTH_ENABLED();
+
+ for (int i = 0; i < PAC_COLLISION_ATTEMPTS; i++) {
+ keyia |= keyia_sign(i) & PAC_MASK;
+ keyib |= keyib_sign(i) & PAC_MASK;
+ keyda |= keyda_sign(i) & PAC_MASK;
+ keydb |= keydb_sign(i) & PAC_MASK;
+ }
+
+ ASSERT_NE(0, keyia) TH_LOG("keyia instructions did nothing");
+ ASSERT_NE(0, keyib) TH_LOG("keyib instructions did nothing");
+ ASSERT_NE(0, keyda) TH_LOG("keyda instructions did nothing");
+ ASSERT_NE(0, keydb) TH_LOG("keydb instructions did nothing");
+}
+
+TEST(pac_instructions_not_nop_generic)
+{
+ size_t keyg = 0;
+
+ ASSERT_GENERIC_PAUTH_ENABLED();
+
+ for (int i = 0; i < PAC_COLLISION_ATTEMPTS; i++)
+ keyg |= keyg_sign(i) & PAC_MASK;
+
+ ASSERT_NE(0, keyg) TH_LOG("keyg instructions did nothing");
+}
+
+TEST(single_thread_different_keys)
+{
+ int same = 10;
+ int nkeys = NKEYS;
+ int tmp;
+ struct signatures signed_vals;
+ unsigned long hwcaps = getauxval(AT_HWCAP);
+
+ /* generic and data key instructions are not in NOP space. This prevents a SIGILL */
+ ASSERT_NE(0, hwcaps & HWCAP_PACA) TH_LOG("PAUTH not enabled");
+ if (!(hwcaps & HWCAP_PACG)) {
+ TH_LOG("WARNING: Generic PAUTH not enabled. Skipping generic key checks");
+ nkeys = NKEYS - 1;
+ }
+
+ /*
+ * In Linux the PAC field can be up to 7 bits wide. Even if keys are
+ * different, there is about 5% chance for PACs to collide with
+ * different addresses. This chance rapidly increases with fewer bits
+ * allocated for the PAC (e.g. wider address). A comparison of the keys
+ * directly will be more reliable.
+ * All signed values need to be different at least once out of n
+ * attempts to be certain that the keys are different
+ */
+ for (int i = 0; i < PAC_COLLISION_ATTEMPTS; i++) {
+ if (nkeys == NKEYS)
+ sign_all(&signed_vals, i);
+ else
+ sign_specific(&signed_vals, i);
+
+ tmp = n_same_single_set(&signed_vals, nkeys);
+ if (tmp < same)
+ same = tmp;
+ }
+
+ ASSERT_EQ(0, same) TH_LOG("%d keys clashed every time", same);
+}
+
+/*
+ * fork() does not change keys. Only exec() does so call a worker program.
+ * Its only job is to sign a value and report back the resutls
+ */
+TEST(exec_changed_keys)
+{
+ struct signatures new_keys;
+ struct signatures old_keys;
+ int ret;
+ int same = 10;
+ int nkeys = NKEYS;
+ unsigned long hwcaps = getauxval(AT_HWCAP);
+
+ /* generic and data key instructions are not in NOP space. This prevents a SIGILL */
+ ASSERT_NE(0, hwcaps & HWCAP_PACA) TH_LOG("PAUTH not enabled");
+ if (!(hwcaps & HWCAP_PACG)) {
+ TH_LOG("WARNING: Generic PAUTH not enabled. Skipping generic key checks");
+ nkeys = NKEYS - 1;
+ }
+
+ for (int i = 0; i < PAC_COLLISION_ATTEMPTS; i++) {
+ ret = exec_sign_all(&new_keys, i);
+ ASSERT_EQ(0, ret) TH_LOG("failed to run worker");
+
+ if (nkeys == NKEYS)
+ sign_all(&old_keys, i);
+ else
+ sign_specific(&old_keys, i);
+
+ ret = n_same(&old_keys, &new_keys, nkeys);
+ if (ret < same)
+ same = ret;
+ }
+
+ ASSERT_EQ(0, same) TH_LOG("exec() did not change %d keys", same);
+}
+
+TEST(context_switch_keep_keys)
+{
+ int ret;
+ struct signatures trash;
+ struct signatures before;
+ struct signatures after;
+
+ ASSERT_PAUTH_ENABLED();
+
+ sign_specific(&before, ARBITRARY_VALUE);
+
+ /* will context switch with a process with different keys at least once */
+ ret = exec_sign_all(&trash, ARBITRARY_VALUE);
+ ASSERT_EQ(0, ret) TH_LOG("failed to run worker");
+
+ sign_specific(&after, ARBITRARY_VALUE);
+
+ ASSERT_EQ(before.keyia, after.keyia) TH_LOG("keyia changed after context switching");
+ ASSERT_EQ(before.keyib, after.keyib) TH_LOG("keyib changed after context switching");
+ ASSERT_EQ(before.keyda, after.keyda) TH_LOG("keyda changed after context switching");
+ ASSERT_EQ(before.keydb, after.keydb) TH_LOG("keydb changed after context switching");
+}
+
+TEST(context_switch_keep_keys_generic)
+{
+ int ret;
+ struct signatures trash;
+ size_t before;
+ size_t after;
+
+ ASSERT_GENERIC_PAUTH_ENABLED();
+
+ before = keyg_sign(ARBITRARY_VALUE);
+
+ /* will context switch with a process with different keys at least once */
+ ret = exec_sign_all(&trash, ARBITRARY_VALUE);
+ ASSERT_EQ(0, ret) TH_LOG("failed to run worker");
+
+ after = keyg_sign(ARBITRARY_VALUE);
+
+ ASSERT_EQ(before, after) TH_LOG("keyg changed after context switching");
+}
+
+TEST_HARNESS_MAIN
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2020 ARM Limited */
+
+.global pac_corruptor
+
+.text
+/*
+ * Corrupting a single bit of the PAC ensures the authentication will fail. It
+ * also guarantees no possible collision. TCR_EL1.TBI0 is set by default so no
+ * top byte PAC is tested
+ */
+ pac_corruptor:
+ paciasp
+
+ /* corrupt the top bit of the PAC */
+ eor lr, lr, #1 << 53
+
+ autiasp
+ ret
int i;
/* Instruction lengths starting at ss_start */
int ss_size[4] = {
- 3, /* xor */
+ 2, /* xor */
2, /* cpuid */
5, /* mov */
2, /* rdmsr */
copyuser_64_exc_t0
copyuser_64_exc_t1
copyuser_64_exc_t2
-memcpy_mcsafe_64
+copy_mc_64
TEST_GEN_PROGS := copyuser_64_t0 copyuser_64_t1 copyuser_64_t2 \
copyuser_p7_t0 copyuser_p7_t1 \
memcpy_64_t0 memcpy_64_t1 memcpy_64_t2 \
- memcpy_p7_t0 memcpy_p7_t1 memcpy_mcsafe_64 \
+ memcpy_p7_t0 memcpy_p7_t1 copy_mc_64 \
copyuser_64_exc_t0 copyuser_64_exc_t1 copyuser_64_exc_t2
EXTRA_SOURCES := validate.c ../harness.c stubs.S
-D SELFTEST_CASE=$(subst memcpy_p7_t,,$(notdir $@)) \
-o $@ $^
-$(OUTPUT)/memcpy_mcsafe_64: memcpy_mcsafe_64.S $(EXTRA_SOURCES)
+$(OUTPUT)/copy_mc_64: copy_mc_64.S $(EXTRA_SOURCES)
$(CC) $(CPPFLAGS) $(CFLAGS) \
- -D COPY_LOOP=test_memcpy_mcsafe \
+ -D COPY_LOOP=test_copy_mc_generic \
-o $@ $^
$(OUTPUT)/copyuser_64_exc_t%: copyuser_64.S exc_validate.c ../harness.c \
--- /dev/null
+../../../../../arch/powerpc/lib/copy_mc_64.S
\ No newline at end of file
+++ /dev/null
-../../../../../arch/powerpc/lib/memcpy_mcsafe_64.S
\ No newline at end of file
// SPDX-License-Identifier: LGPL-2.1
#define _GNU_SOURCE
#include <assert.h>
+#include <linux/membarrier.h>
#include <pthread.h>
#include <sched.h>
+#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
return ret;
}
+struct test_membarrier_thread_args {
+ int stop;
+ intptr_t percpu_list_ptr;
+};
+
+/* Worker threads modify data in their "active" percpu lists. */
+void *test_membarrier_worker_thread(void *arg)
+{
+ struct test_membarrier_thread_args *args =
+ (struct test_membarrier_thread_args *)arg;
+ const int iters = opt_reps;
+ int i;
+
+ if (rseq_register_current_thread()) {
+ fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
+ errno, strerror(errno));
+ abort();
+ }
+
+ /* Wait for initialization. */
+ while (!atomic_load(&args->percpu_list_ptr)) {}
+
+ for (i = 0; i < iters; ++i) {
+ int ret;
+
+ do {
+ int cpu = rseq_cpu_start();
+
+ ret = rseq_offset_deref_addv(&args->percpu_list_ptr,
+ sizeof(struct percpu_list_entry) * cpu, 1, cpu);
+ } while (rseq_unlikely(ret));
+ }
+
+ if (rseq_unregister_current_thread()) {
+ fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
+ errno, strerror(errno));
+ abort();
+ }
+ return NULL;
+}
+
+void test_membarrier_init_percpu_list(struct percpu_list *list)
+{
+ int i;
+
+ memset(list, 0, sizeof(*list));
+ for (i = 0; i < CPU_SETSIZE; i++) {
+ struct percpu_list_node *node;
+
+ node = malloc(sizeof(*node));
+ assert(node);
+ node->data = 0;
+ node->next = NULL;
+ list->c[i].head = node;
+ }
+}
+
+void test_membarrier_free_percpu_list(struct percpu_list *list)
+{
+ int i;
+
+ for (i = 0; i < CPU_SETSIZE; i++)
+ free(list->c[i].head);
+}
+
+static int sys_membarrier(int cmd, int flags, int cpu_id)
+{
+ return syscall(__NR_membarrier, cmd, flags, cpu_id);
+}
+
+/*
+ * The manager thread swaps per-cpu lists that worker threads see,
+ * and validates that there are no unexpected modifications.
+ */
+void *test_membarrier_manager_thread(void *arg)
+{
+ struct test_membarrier_thread_args *args =
+ (struct test_membarrier_thread_args *)arg;
+ struct percpu_list list_a, list_b;
+ intptr_t expect_a = 0, expect_b = 0;
+ int cpu_a = 0, cpu_b = 0;
+
+ if (rseq_register_current_thread()) {
+ fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n",
+ errno, strerror(errno));
+ abort();
+ }
+
+ /* Init lists. */
+ test_membarrier_init_percpu_list(&list_a);
+ test_membarrier_init_percpu_list(&list_b);
+
+ atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+
+ while (!atomic_load(&args->stop)) {
+ /* list_a is "active". */
+ cpu_a = rand() % CPU_SETSIZE;
+ /*
+ * As list_b is "inactive", we should never see changes
+ * to list_b.
+ */
+ if (expect_b != atomic_load(&list_b.c[cpu_b].head->data)) {
+ fprintf(stderr, "Membarrier test failed\n");
+ abort();
+ }
+
+ /* Make list_b "active". */
+ atomic_store(&args->percpu_list_ptr, (intptr_t)&list_b);
+ if (sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,
+ MEMBARRIER_CMD_FLAG_CPU, cpu_a) &&
+ errno != ENXIO /* missing CPU */) {
+ perror("sys_membarrier");
+ abort();
+ }
+ /*
+ * Cpu A should now only modify list_b, so the values
+ * in list_a should be stable.
+ */
+ expect_a = atomic_load(&list_a.c[cpu_a].head->data);
+
+ cpu_b = rand() % CPU_SETSIZE;
+ /*
+ * As list_a is "inactive", we should never see changes
+ * to list_a.
+ */
+ if (expect_a != atomic_load(&list_a.c[cpu_a].head->data)) {
+ fprintf(stderr, "Membarrier test failed\n");
+ abort();
+ }
+
+ /* Make list_a "active". */
+ atomic_store(&args->percpu_list_ptr, (intptr_t)&list_a);
+ if (sys_membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ,
+ MEMBARRIER_CMD_FLAG_CPU, cpu_b) &&
+ errno != ENXIO /* missing CPU*/) {
+ perror("sys_membarrier");
+ abort();
+ }
+ /* Remember a value from list_b. */
+ expect_b = atomic_load(&list_b.c[cpu_b].head->data);
+ }
+
+ test_membarrier_free_percpu_list(&list_a);
+ test_membarrier_free_percpu_list(&list_b);
+
+ if (rseq_unregister_current_thread()) {
+ fprintf(stderr, "Error: rseq_unregister_current_thread(...) failed(%d): %s\n",
+ errno, strerror(errno));
+ abort();
+ }
+ return NULL;
+}
+
+/* Test MEMBARRIER_CMD_PRIVATE_RESTART_RSEQ_ON_CPU membarrier command. */
+#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
+void test_membarrier(void)
+{
+ const int num_threads = opt_threads;
+ struct test_membarrier_thread_args thread_args;
+ pthread_t worker_threads[num_threads];
+ pthread_t manager_thread;
+ int i, ret;
+
+ if (sys_membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ, 0, 0)) {
+ perror("sys_membarrier");
+ abort();
+ }
+
+ thread_args.stop = 0;
+ thread_args.percpu_list_ptr = 0;
+ ret = pthread_create(&manager_thread, NULL,
+ test_membarrier_manager_thread, &thread_args);
+ if (ret) {
+ errno = ret;
+ perror("pthread_create");
+ abort();
+ }
+
+ for (i = 0; i < num_threads; i++) {
+ ret = pthread_create(&worker_threads[i], NULL,
+ test_membarrier_worker_thread, &thread_args);
+ if (ret) {
+ errno = ret;
+ perror("pthread_create");
+ abort();
+ }
+ }
+
+
+ for (i = 0; i < num_threads; i++) {
+ ret = pthread_join(worker_threads[i], NULL);
+ if (ret) {
+ errno = ret;
+ perror("pthread_join");
+ abort();
+ }
+ }
+
+ atomic_store(&thread_args.stop, 1);
+ ret = pthread_join(manager_thread, NULL);
+ if (ret) {
+ errno = ret;
+ perror("pthread_join");
+ abort();
+ }
+}
+#else /* RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV */
+void test_membarrier(void)
+{
+ fprintf(stderr, "rseq_offset_deref_addv is not implemented on this architecture. "
+ "Skipping membarrier test.\n");
+}
+#endif
+
static void show_usage(int argc, char **argv)
{
printf("Usage : %s <OPTIONS>\n",
printf(" [-r N] Number of repetitions per thread (default 5000)\n");
printf(" [-d] Disable rseq system call (no initialization)\n");
printf(" [-D M] Disable rseq for each M threads\n");
- printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement\n");
+ printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\n");
printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n");
printf(" [-v] Verbose output.\n");
printf(" [-h] Show this help.\n");
case 'i':
case 'b':
case 'm':
+ case 'r':
break;
default:
show_usage(argc, argv);
printf_verbose("counter increment\n");
test_percpu_inc();
break;
+ case 'r':
+ printf_verbose("membarrier\n");
+ test_membarrier();
+ break;
}
if (!opt_disable_rseq && rseq_unregister_current_thread())
abort();
#endif
}
+#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
+
+/*
+ * pval = *(ptr+off)
+ * *pval += inc;
+ */
+static inline __attribute__((always_inline))
+int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
+{
+ RSEQ_INJECT_C(9)
+
+ __asm__ __volatile__ goto (
+ RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
+#ifdef RSEQ_COMPARE_TWICE
+ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
+#endif
+ /* Start rseq by storing table entry pointer into rseq_cs. */
+ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
+ RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
+ RSEQ_INJECT_ASM(3)
+#ifdef RSEQ_COMPARE_TWICE
+ RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
+#endif
+ /* get p+v */
+ "movq %[ptr], %%rbx\n\t"
+ "addq %[off], %%rbx\n\t"
+ /* get pv */
+ "movq (%%rbx), %%rcx\n\t"
+ /* *pv += inc */
+ "addq %[inc], (%%rcx)\n\t"
+ "2:\n\t"
+ RSEQ_INJECT_ASM(4)
+ RSEQ_ASM_DEFINE_ABORT(4, "", abort)
+ : /* gcc asm goto does not allow outputs */
+ : [cpu_id] "r" (cpu),
+ [rseq_abi] "r" (&__rseq_abi),
+ /* final store input */
+ [ptr] "m" (*ptr),
+ [off] "er" (off),
+ [inc] "er" (inc)
+ : "memory", "cc", "rax", "rbx", "rcx"
+ RSEQ_INJECT_CLOBBER
+ : abort
+#ifdef RSEQ_COMPARE_TWICE
+ , error1
+#endif
+ );
+ return 0;
+abort:
+ RSEQ_INJECT_FAILED
+ return -1;
+#ifdef RSEQ_COMPARE_TWICE
+error1:
+ rseq_bug("cpu_id comparison failed");
+#endif
+}
+
static inline __attribute__((always_inline))
int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
intptr_t *v2, intptr_t newv2,
"-T m"
"-T m -M"
"-T i"
+ "-T r"
)
TEST_NAME=(
"memcpy"
"memcpy with barrier"
"increment"
+ "membarrier"
)
IFS="$OLDIFS"
#define USER_REGS_OFFSET(r) offsetof(struct user_regs_struct, r)
+static void test_ptrace_write_gs_read_base(void)
+{
+ int status;
+ pid_t child = fork();
+
+ if (child < 0)
+ err(1, "fork");
+
+ if (child == 0) {
+ printf("[RUN]\tPTRACE_POKE GS, read GSBASE back\n");
+
+ printf("[RUN]\tARCH_SET_GS to 1\n");
+ if (syscall(SYS_arch_prctl, ARCH_SET_GS, 1) != 0)
+ err(1, "ARCH_SET_GS");
+
+ if (ptrace(PTRACE_TRACEME, 0, NULL, NULL) != 0)
+ err(1, "PTRACE_TRACEME");
+
+ raise(SIGTRAP);
+ _exit(0);
+ }
+
+ wait(&status);
+
+ if (WSTOPSIG(status) == SIGTRAP) {
+ unsigned long base;
+ unsigned long gs_offset = USER_REGS_OFFSET(gs);
+ unsigned long base_offset = USER_REGS_OFFSET(gs_base);
+
+ /* Read the initial base. It should be 1. */
+ base = ptrace(PTRACE_PEEKUSER, child, base_offset, NULL);
+ if (base == 1) {
+ printf("[OK]\tGSBASE started at 1\n");
+ } else {
+ nerrs++;
+ printf("[FAIL]\tGSBASE started at 0x%lx\n", base);
+ }
+
+ printf("[RUN]\tSet GS = 0x7, read GSBASE\n");
+
+ /* Poke an LDT selector into GS. */
+ if (ptrace(PTRACE_POKEUSER, child, gs_offset, 0x7) != 0)
+ err(1, "PTRACE_POKEUSER");
+
+ /* And read the base. */
+ base = ptrace(PTRACE_PEEKUSER, child, base_offset, NULL);
+
+ if (base == 0 || base == 1) {
+ printf("[OK]\tGSBASE reads as 0x%lx with invalid GS\n", base);
+ } else {
+ nerrs++;
+ printf("[FAIL]\tGSBASE=0x%lx (should be 0 or 1)\n", base);
+ }
+ }
+
+ ptrace(PTRACE_CONT, child, NULL, NULL);
+
+ wait(&status);
+ if (!WIFEXITED(status))
+ printf("[WARN]\tChild didn't exit cleanly.\n");
+}
+
static void test_ptrace_write_gsbase(void)
{
int status;
END:
ptrace(PTRACE_CONT, child, NULL, NULL);
+ wait(&status);
+ if (!WIFEXITED(status))
+ printf("[WARN]\tChild didn't exit cleanly.\n");
}
int main()
shared_scratch = mmap(NULL, 4096, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
+ /* Do these tests before we have an LDT. */
+ test_ptrace_write_gs_read_base();
+
/* Probe FSGSBASE */
sethandler(SIGILL, sigill, 0);
if (sigsetjmp(jmpbuf, 1) == 0) {
#define KPF_ARCH 38
#define KPF_UNCACHED 39
#define KPF_SOFTDIRTY 40
+#define KPF_ARCH_2 41
/* [48-] take some arbitrary free slots for expanding overloaded flags
* not part of kernel API
[KPF_ARCH] = "h:arch",
[KPF_UNCACHED] = "c:uncached",
[KPF_SOFTDIRTY] = "f:softdirty",
+ [KPF_ARCH_2] = "H:arch_2",
[KPF_READAHEAD] = "I:readahead",
[KPF_SLOB_FREE] = "P:slob_free",
projects / linux-2.6-block.git / commitdiff
